Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

24.8 KB
Raw Blame
"""MinerU startup configuration."""

from __future__ import annotations
from docvortex.options import LatexDelimiterConfig, LatexDelimitersConfig

import copy
import logging
import os
import re
import secrets
import socket
import tempfile
import threading
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, Literal, get_args, get_origin
from urllib.parse import urlsplit, urlunsplit

import yaml
from filelock import FileLock
from pydantic import BaseModel, Field, SecretStr, field_validator, model_validator

from .utils.private_file import create_private_tempfile, protect_private_file

_logger = logging.getLogger(__name__)

MINERU_HOME_ENV = "MINERU_HOME"
MINERU_CONFIG_ENV = "MINERU_CONFIG"
MINERU_ENV_PREFIX = "MINERU_"

AutoBool = Literal["auto"] | bool
ConfigSource = Literal["default", "file", "env"]
ModelSource = Literal["auto", "huggingface", "modelscope", "local"]
OnnxDevice = Literal["auto", "cpu", "gpu"]
LogLevel = Literal["trace", "debug", "info", "warning", "error", "critical"]

_INTERPOLATION_RE = re.compile(r"\$\{(\w+)(?::-([^${}]*))?\}")


def _mineru_home() -> str:
    configured = os.getenv(MINERU_HOME_ENV)
    if configured not in (None, ""):
        return os.path.expanduser(configured)
    return os.path.join(os.path.expanduser("~"), ".mineru")


def _default_path(path1: str, /, *paths: str) -> str:
    return os.path.join(_mineru_home(), path1, *paths)


def _normalize_log_level(value: LogLevel | str | None) -> LogLevel | str | None:
    """归一化日志级别输入;None 表示未设置局部覆盖。"""
    if value is None or not isinstance(value, str):
        return value
    return value.strip().lower()


def _uds_available() -> bool:
    try:
        socket.socket(socket.AF_UNIX, socket.SOCK_STREAM).close()
        return True
    except Exception:
        return False


def _strip_quotes(value: str) -> str:
    value = value.strip()
    if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]:
        return value[1:-1]
    return value


def _interpolation_replace(match: re.Match[str]) -> str:
    var_name, raw_default = match.group(1), match.group(2)
    value = os.environ.get(var_name)
    if value is not None:
        return value
    if raw_default is not None:
        return _strip_quotes(raw_default)
    raise ValueError(
        f"Environment variable {var_name!r} is referenced in MinerU config but is not set and has no default value."
    )


def _substitute(value: str, max_depth: int = 20) -> str:
    for _ in range(max_depth):
        substituted = _INTERPOLATION_RE.sub(_interpolation_replace, value)
        if substituted == value:
            return value
        value = substituted
    raise ValueError(f"MinerU config interpolation did not converge after {max_depth} passes.")


def _interpolate_env(value: Any) -> Any:
    """Recursively substitute ${VAR} and ${VAR:-default} placeholders."""
    if isinstance(value, str):
        return _substitute(value)
    if isinstance(value, dict):
        return {key: _interpolate_env(item) for key, item in value.items()}
    if isinstance(value, list):
        return [_interpolate_env(item) for item in value]
    return value


def _load_config(config_file: str) -> dict[str, Any]:
    with open(config_file, encoding="utf-8") as file:
        raw = yaml.safe_load(file)
    if isinstance(raw, dict):
        return _interpolate_env(raw)
    _logger.warning("MinerU config file [%s] is empty or invalid.", config_file)
    return {}


def _resolve_config_file() -> tuple[str, bool]:
    config_file = os.getenv(MINERU_CONFIG_ENV)
    if config_file not in (None, ""):
        config_file = os.path.expanduser(config_file)
        if not os.path.isfile(config_file):
            raise FileNotFoundError(f"MinerU config file [{config_file}] does not exist.")
        return config_file, True

    default_config_file = _default_path("config.yaml")
    exists = os.path.isfile(default_config_file)
    if not exists:
        _logger.debug(
            "MinerU config file not found. Default path is %s. Use %s to specify a custom path.",
            default_config_file,
            MINERU_CONFIG_ENV,
        )
    return default_config_file, exists


def _collect_path(remaining: str, model_class: type[BaseModel]) -> list[str] | None:
    """Greedily match an UPPER_CASE env suffix to a pydantic field path."""
    remaining_lower = remaining.lower()
    fields = sorted(model_class.model_fields.keys(), key=len, reverse=True)
    for field_name in fields:
        if remaining_lower == field_name:
            return [field_name]
        if remaining_lower.startswith(field_name + "_"):
            annotation = model_class.model_fields[field_name].annotation
            if isinstance(annotation, type) and issubclass(annotation, BaseModel):
                sub_remaining = remaining[len(field_name) + 1 :]
                sub_path = _collect_path(sub_remaining, annotation)
                if sub_path is not None:
                    return [field_name, *sub_path]
    return None


def _deep_merge(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
    result = copy.deepcopy(base)
    for key, value in overrides.items():
        if key in result and isinstance(result[key], dict) and isinstance(value, dict):
            result[key] = _deep_merge(result[key], value)
        else:
            result[key] = value
    return result


def _collect_env_overrides(prefix: str = MINERU_ENV_PREFIX) -> tuple[dict[str, Any], set[tuple[str, ...]]]:
    prefix_upper = prefix.upper()
    overrides: dict[str, Any] = {}
    paths: set[tuple[str, ...]] = set()

    for key, value in os.environ.items():
        if not key.startswith(prefix_upper):
            continue
        remaining = key[len(prefix_upper) :]
        if not remaining or remaining == "CONFIG":
            continue
        path = _collect_path(remaining, Config)
        if path is None:
            continue
        node = overrides
        for part in path[:-1]:
            node = node.setdefault(part, {})
        node[path[-1]] = value
        paths.add(tuple(path))

    return overrides, paths


def _model_annotation(annotation: Any) -> type[BaseModel] | None:
    if isinstance(annotation, type) and issubclass(annotation, BaseModel):
        return annotation
    origin = get_origin(annotation)
    if origin is None:
        return None
    for arg in get_args(annotation):
        if isinstance(arg, type) and issubclass(arg, BaseModel):
            return arg
    return None


def _default_source_paths(model_class: type[BaseModel], prefix: tuple[str, ...] = ()) -> set[tuple[str, ...]]:
    paths: set[tuple[str, ...]] = set()
    for field_name, field in model_class.model_fields.items():
        sub_model = _model_annotation(field.annotation)
        current = (*prefix, field_name)
        if sub_model is not None:
            paths.update(_default_source_paths(sub_model, current))
        else:
            paths.add(current)
    return paths


def _configured_source_paths(
    data: dict[str, Any],
    model_class: type[BaseModel],
    prefix: tuple[str, ...] = (),
) -> set[tuple[str, ...]]:
    paths: set[tuple[str, ...]] = set()
    for key, value in data.items():
        if key not in model_class.model_fields:
            continue
        field = model_class.model_fields[key]
        sub_model = _model_annotation(field.annotation)
        current = (*prefix, key)
        if sub_model is not None and isinstance(value, dict):
            paths.update(_configured_source_paths(value, sub_model, current))
        else:
            paths.add(current)
    return paths


def _normalize_config_source_path(path: str | Sequence[str]) -> tuple[str, ...]:
    if isinstance(path, str):
        return tuple(part for part in path.split(".") if part)
    return tuple(path)


@dataclass(frozen=True)
class LoadedConfig:
    config: "Config"
    sources: dict[tuple[str, ...], ConfigSource]
    config_file: str
    config_file_exists: bool


def _load_effective_config() -> LoadedConfig:
    if "MINERU_MODEL_STACK" in os.environ:
        raise ValueError("MINERU_MODEL_STACK was removed; use MINERU_MODEL_SMALL_BACKEND and MINERU_MODEL_VLM_ENGINE")
    config_file, config_file_exists = _resolve_config_file()
    raw_config = _load_config(config_file) if config_file_exists else {}
    sources: dict[tuple[str, ...], ConfigSource] = dict.fromkeys(_default_source_paths(Config), "default")
    for path in _configured_source_paths(raw_config, Config):
        sources[path] = "file"

    base_config = Config(**raw_config)
    overrides, env_paths = _collect_env_overrides()
    if overrides:
        base_config = Config(**_deep_merge(base_config.model_dump(), overrides))
        for path in env_paths:
            sources[path] = "env"
    return LoadedConfig(
        config=base_config,
        sources=sources,
        config_file=config_file,
        config_file_exists=config_file_exists,
    )


def load_config() -> LoadedConfig:
    """按当前文件和环境独立装载配置及来源,不替换进程默认配置。"""
    return _load_effective_config()


def _get_loaded_config() -> LoadedConfig:
    """首次使用时在线程锁内装载进程默认配置,后续复用同一份来源记录。"""
    global _loaded_config
    if _loaded_config is None:
        with _config_load_lock:
            if _loaded_config is None:
                _loaded_config = load_config()
    return _loaded_config


def get_config() -> Config:
    """返回首次使用时装载并缓存的进程默认配置。"""
    return _get_loaded_config().config


def get_config_source(path: str | Sequence[str]) -> ConfigSource:
    """从进程默认配置的同一份装载结果查询字段来源。"""
    return _get_loaded_config().sources.get(_normalize_config_source_path(path), "default")


def get_config_file_path() -> str:
    """返回进程默认配置首次装载时确定的文件路径。"""
    return _get_loaded_config().config_file


def get_config_file_exists() -> bool:
    """返回进程默认配置首次装载时的文件存在状态。"""
    return _get_loaded_config().config_file_exists


def update_config_file(patch: dict[str, Any], *, only_if_onnx_auto: bool = False) -> bool:
    """Atomically merge a patch, optionally comparing the current ONNX choice under the lock."""
    config_file = get_config_file_path()
    config_dir = os.path.dirname(config_file)
    if config_dir:
        os.makedirs(config_dir, exist_ok=True)

    lock = FileLock(f"{config_file}.lock")
    with lock:
        if os.path.exists(config_file):
            with open(config_file, encoding="utf-8") as file:
                loaded = yaml.safe_load(file)
            current = loaded if isinstance(loaded, dict) else {}
        else:
            current = {}

        if only_if_onnx_auto:
            model = current.get("model", {})
            if not isinstance(model, dict) or model.get("onnx_device", "auto") != "auto":
                return False

        updated = _deep_merge(current, patch)
        _write_config_file(config_file, updated)
        return True


def _write_config_file(config_file: str, config: dict[str, Any]) -> None:
    """在写入配置正文之前限制临时文件权限,原子替换后保留同样的私密权限。"""
    directory = os.path.dirname(config_file) or "."
    doclib = config.get("doclib")
    auth = doclib.get("auth") if isinstance(doclib, dict) else None
    if isinstance(auth, dict) and auth.get("token") is not None:
        fd, tmp_path = create_private_tempfile(directory)
    else:
        fd, tmp_path = tempfile.mkstemp(prefix=".config.", suffix=".tmp", dir=directory)
    try:
        with os.fdopen(fd, "w", encoding="utf-8") as file:
            yaml.safe_dump(config, file, allow_unicode=True, sort_keys=False)
        os.replace(tmp_path, config_file)
    finally:
        if os.path.exists(tmp_path):
            os.unlink(tmp_path)


def load_doclib_server_config() -> LoadedConfig:
    """启动时在配置锁内生成或复用本机凭证;普通配置读取不生成文件或凭证。"""
    loaded = load_config()
    config_file = loaded.config_file
    os.makedirs(os.path.dirname(os.path.abspath(config_file)), mode=0o700, exist_ok=True)
    with FileLock(f"{config_file}.lock"):
        loaded = load_config()
        if os.path.isfile(config_file):
            protect_private_file(config_file)
        if loaded.config.doclib.auth.token is not None:
            validate_doclib_token(loaded.config.doclib.auth.token.get_secret_value())
            return loaded
        if os.path.isfile(config_file):
            with open(config_file, encoding="utf-8") as file:
                current = yaml.safe_load(file) or {}
        else:
            current = {}
        token = secrets.token_urlsafe(32)
        updated = _deep_merge(current, {"doclib": {"auth": {"token": token}}})
        _write_config_file(config_file, updated)
        return load_config()


def validate_doclib_token(token: str) -> str:
    """校验本机凭证的长度和 HTTP Bearer 可用字符,错误消息不包含凭证。"""
    if not re.fullmatch(r"[A-Za-z0-9._~+/=-]{32,}", token):
        raise ValueError("Doclib token must contain at least 32 ASCII Bearer-token characters.")
    return token


def read_onnx_device_from_file() -> OnnxDevice:
    """Re-read only the device choice after acquiring the cross-process selection lock."""
    path = get_config_file_path()
    raw = _load_config(path) if os.path.isfile(path) else {}
    return ModelConfig(**raw.get("model", {})).onnx_device


class UDSConfig(BaseModel):
    enabled: AutoBool = "auto"
    path: str = Field(default_factory=lambda: _default_path("doclib.sock"))
    permission: int = 0o600


class TCPConfig(BaseModel):
    enabled: AutoBool = "auto"
    host: str = "127.0.0.1"
    port: int = 15980
    strict_port: bool = False
    port_probe_count: int = Field(default=100, ge=1)
    backlog: int = 128
    timeout: int = 600


class GlobalLogConfig(BaseModel):
    """MinerU 全局日志配置,控制当前进程的 Loguru 默认输出。"""

    level: LogLevel = "info"

    @field_validator("level", mode="before")
    @classmethod
    def _normalize_level(cls, value: LogLevel | str | None) -> LogLevel | str | None:
        """允许配置文件和环境变量使用大小写混写,并忽略首尾空白。"""
        return _normalize_log_level(value)


class LogConfig(BaseModel):
    dir: str = Field(default_factory=lambda: _default_path("logs"))
    app_path: str | None = None
    access_path: str | None = None
    stdout_path: str | None = None
    stderr_path: str | None = None
    parse_server_stdout_path: str | None = None
    parse_server_stderr_path: str | None = None
    level: LogLevel | None = None

    @field_validator("level", mode="before")
    @classmethod
    def _normalize_level(cls, value: LogLevel | str | None) -> LogLevel | str | None:
        """归一化 Doclib 的局部日志级别覆盖。"""
        return _normalize_log_level(value)

    @property
    def resolved_app_path(self) -> str:
        return self.app_path or os.path.join(self.dir, "doclib.log")

    @property
    def resolved_access_path(self) -> str:
        return self.access_path or os.path.join(self.dir, "doclib.access.log")

    @property
    def resolved_stdout_path(self) -> str:
        return self.stdout_path or os.path.join(self.dir, "doclib.stdout.log")

    @property
    def resolved_stderr_path(self) -> str:
        return self.stderr_path or os.path.join(self.dir, "doclib.stderr.log")

    @property
    def resolved_parse_server_stdout_path(self) -> str:
        return self.parse_server_stdout_path or os.path.join(self.dir, "doclib.parse-server.stdout.log")

    @property
    def resolved_parse_server_stderr_path(self) -> str:
        return self.parse_server_stderr_path or os.path.join(self.dir, "doclib.parse-server.stderr.log")


class SQLiteConfig(BaseModel):
    path: str = Field(default_factory=lambda: _default_path("doclib.db"))
    busy_timeout_ms: int = Field(default=5000, ge=0)
    lock_retry_attempts: int = Field(default=3, ge=0)
    lock_retry_base_delay_ms: int = Field(default=50, ge=0)
    mmap_size: int = 268435456
    cache_size: int = -20000
    wal_autocheckpoint: int = 1000
    journal_size_limit: int = 33_554_432
    temp_store: str = "memory"
    synchronous: str = "NORMAL"


class ManagedParseServerConfig(BaseModel):
    host: str = "127.0.0.1"
    port: int = 16580
    strict_port: bool = False
    port_probe_count: int = Field(default=100, ge=1)


class RenderConfig(BaseModel):
    """MiddleJson 输出渲染配置。"""

    latex_delimiters: LatexDelimitersConfig = Field(default_factory=LatexDelimitersConfig)


class LLMAidedFeaturesConfig(BaseModel):
    """LLM 辅助后处理的独立功能开关。"""

    title_leveling: bool = False
    cross_page_table_cell_merge: bool = False


class LLMAidedConfig(BaseModel):
    """OpenAI-compatible LLM 辅助后处理配置。"""

    api_key: str = Field(default="", repr=False)
    base_url: str = "https://dashscope.aliyuncs.com/compatible-mode/v1"
    model: str = "qwen3.8-flash"
    enable_thinking: bool | None = None
    max_concurrency: int = Field(default=16, ge=1)
    allow_partial_title_leveling: bool = False
    features: LLMAidedFeaturesConfig = Field(default_factory=LLMAidedFeaturesConfig)

    @field_validator("max_concurrency", mode="before")
    @classmethod
    def _reject_boolean_max_concurrency(cls, value: Any) -> Any:
        """拒绝被 Python 视作整数的布尔并发数,同时保留环境变量字符串转换。"""
        if isinstance(value, bool):
            raise ValueError("llm_aided.max_concurrency must be a positive integer")
        return value

    @model_validator(mode="after")
    def _validate_enabled_credentials(self) -> "LLMAidedConfig":
        """任一 LLM 功能启用时要求连接参数均为非空字符串。"""
        enabled = self.features.title_leveling or self.features.cross_page_table_cell_merge
        if not enabled:
            return self

        for field_name in ("api_key", "base_url", "model"):
            value = getattr(self, field_name)
            if not isinstance(value, str) or not value.strip():
                raise ValueError(f"llm_aided.{field_name} must be a non-empty string when an LLM feature is enabled")
        return self


class VlmConfig(BaseModel):
    """所有解析入口共享的 VLM 引擎与连接配置;远程地址优先于本地引擎。"""

    model_config = {"hide_input_in_errors": True}

    engine: Literal["auto", "llama-cpp", "vllm", "lmdeploy", "mlx"] = "auto"
    server_url: str = ""
    api_key: str = Field(default="", repr=False)
    model: str = ""
    http_timeout: int = Field(default=600, ge=1)
    max_concurrency: int = Field(default=100, ge=1)

    @field_validator("api_key", "model")
    @classmethod
    def _strip_connection_value(cls, value: str) -> str:
        """清理连接字段首尾空白,保持显式空值可覆盖全局配置。"""
        return value.strip()

    @field_validator("server_url")
    @classmethod
    def _normalize_server_url(cls, value: str) -> str:
        """保留代理前缀并移除末尾 v1,以尾斜线阻止底层客户端丢弃路径。"""
        value = value.strip()
        if not value:
            return ""
        try:
            parts = urlsplit(value)
            valid = (
                parts.scheme in {"http", "https"}
                and bool(parts.hostname)
                and parts.username is None
                and parts.password is None
                and not parts.query
                and not parts.fragment
                and not any(char.isspace() for char in value)
            )
            _ = parts.port
        except ValueError:
            valid = False
        if not valid:
            raise ValueError("model.vlm.server_url must be an HTTP(S) service URL without credentials, query or fragment")
        path = parts.path.rstrip("/")
        if path.endswith("/v1"):
            path = path[:-3]
        return urlunsplit((parts.scheme, parts.netloc, path + "/", "", ""))

    @field_validator("http_timeout", "max_concurrency", mode="before")
    @classmethod
    def _reject_boolean_limit(cls, value: Any) -> Any:
        """拒绝布尔数值,同时允许配置环境变量提供整数文本。"""
        if isinstance(value, bool):
            raise ValueError("VLM timeout and concurrency must be positive integers")
        return value

    def validate_environment(self) -> None:
        """远程推理前拒绝会隐式覆盖显式配置的旧环境变量,错误不包含凭据。"""
        if not self.server_url:
            return
        for env_name, expected, field in (
            ("MINERU_VL_API_KEY", self.api_key, "api_key"),
            ("MINERU_VL_MODEL_NAME", self.model, "model"),
        ):
            # 底层只清理 API Key;模型名按原始环境变量值读取。
            actual = os.getenv(env_name, "")
            if field == "api_key":
                actual = actual.strip()
            if actual and actual != expected:
                raise ValueError(f"{env_name} conflicts with model.vlm.{field}; unset the legacy environment variable")


class ModelConfig(BaseModel):
    base_dir: str = Field(default_factory=lambda: _default_path("models"))
    source: str = "auto"
    small_backend: Literal["auto", "onnx", "torch"] = "auto"
    onnx_device: OnnxDevice = "auto"
    vlm: VlmConfig = Field(default_factory=VlmConfig)

    @model_validator(mode="before")
    @classmethod
    def _reject_legacy_stack(cls, value: Any) -> Any:
        """拒绝旧模型栈字段,避免配置被忽略后意外切换推理后端。"""
        if isinstance(value, dict) and "stack" in value:
            raise ValueError("model.stack was removed; use model.small_backend and model.vlm.engine")
        return value


class DoclibAuthConfig(BaseModel):
    """保存跨服务重启稳定的本机凭证,默认序列化和 repr 都隐藏正文。"""

    token: SecretStr | None = Field(default=None, repr=False)


class DoclibConfig(BaseModel):
    """Doclib startup configuration.

    Only configuration needed before the doclib server starts belongs here.
    Runtime configuration that can be read from SQLite stays in the config
    service.
    """

    auth: DoclibAuthConfig = Field(default_factory=DoclibAuthConfig)
    uds: UDSConfig = Field(default_factory=UDSConfig)
    tcp: TCPConfig = Field(default_factory=TCPConfig)
    log: LogConfig = Field(default_factory=LogConfig)
    sqlite: SQLiteConfig = Field(default_factory=SQLiteConfig)
    endpoint_path: str = Field(default_factory=lambda: _default_path("doclib.endpoint.json"))
    data_dir: str = Field(default_factory=lambda: _default_path("doclib"))
    managed_parse_server: ManagedParseServerConfig = Field(default_factory=ManagedParseServerConfig)
    ingest_workers: int = 2
    parse_workers: int = 2
    scan_interval_sec: int = 300
    device_check_interval_sec: int = 5
    ingest_lock_timeout_sec: int = 60
    parse_lock_timeout_sec: int = 1800
    scan_lock_timeout_sec: int = 1800
    compaction_interval_sec: int = 3600
    parse_server_health_check_interval_sec: int = 30
    parse_server_probe_timeout_sec: int = 10
    parse_server_startup_grace_sec: int = 30
    parse_server_startup_timeout_sec: int = 600
    parse_server_stop_timeout_sec: int = 10

    @property
    def resolved_uds_enabled(self) -> bool:
        if self.uds.enabled == "auto":
            return _uds_available()
        return self.uds.enabled

    @property
    def resolved_tcp_enabled(self) -> bool:
        if self.tcp.enabled == "auto":
            return not _uds_available()
        return self.tcp.enabled


class Config(BaseModel):
    """Top-level MinerU startup configuration."""

    log: GlobalLogConfig = Field(default_factory=GlobalLogConfig)
    doclib: DoclibConfig = Field(default_factory=DoclibConfig)
    render: RenderConfig = Field(default_factory=RenderConfig)
    llm_aided: LLMAidedConfig = Field(default_factory=LLMAidedConfig)
    model: ModelConfig = Field(default_factory=ModelConfig)


_loaded_config: LoadedConfig | None = None
_config_load_lock = threading.Lock()


def PatchedConfig(**kwargs: Any) -> Config:
    """在进程默认配置的独立副本上应用局部覆盖并重新校验。"""
    merged = _deep_merge(get_config().model_dump(), kwargs)
    return Config(**merged)


__all__ = [
    "DoclibAuthConfig",
    "load_doclib_server_config",
    "validate_doclib_token",
    "OnnxDevice",
    "read_onnx_device_from_file",
    "AutoBool",
    "ConfigSource",
    "LoadedConfig",
    "LogLevel",
    "GlobalLogConfig",
    "ModelConfig",
    "VlmConfig",
    "ModelSource",
    "get_config",
    "load_config",
    "Config",
    "DoclibConfig",
    "LLMAidedConfig",
    "LLMAidedFeaturesConfig",
    "LatexDelimiterConfig",
    "LatexDelimitersConfig",
    "RenderConfig",
    "TCPConfig",
    "LogConfig",
    "SQLiteConfig",
    "UDSConfig",
    "PatchedConfig",
    "get_config_source",
    "get_config_file_path",
    "get_config_file_exists",
    "update_config_file",
    "MINERU_HOME_ENV",
    "MINERU_CONFIG_ENV",
    "MINERU_ENV_PREFIX",
]