Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
"""MinerU startup configuration."""
from __future__ import annotations
from docvortex.options import LatexDelimiterConfig, LatexDelimitersConfig
import copy
import logging
import os
import re
import secrets
import socket
import tempfile
import threading
from collections.abc import Sequence
from dataclasses import dataclass
from typing import Any, Literal, get_args, get_origin
from urllib.parse import urlsplit, urlunsplit
import yaml
from filelock import FileLock
from pydantic import BaseModel, Field, SecretStr, field_validator, model_validator
from .utils.private_file import create_private_tempfile, protect_private_file
_logger = logging.getLogger(__name__)
MINERU_HOME_ENV = "MINERU_HOME"
MINERU_CONFIG_ENV = "MINERU_CONFIG"
MINERU_ENV_PREFIX = "MINERU_"
AutoBool = Literal["auto"] | bool
ConfigSource = Literal["default", "file", "env"]
ModelSource = Literal["auto", "huggingface", "modelscope", "local"]
OnnxDevice = Literal["auto", "cpu", "gpu"]
LogLevel = Literal["trace", "debug", "info", "warning", "error", "critical"]
_INTERPOLATION_RE = re.compile(r"\$\{(\w+)(?::-([^${}]*))?\}")
def _mineru_home() -> str:
configured = os.getenv(MINERU_HOME_ENV)
if configured not in (None, ""):
return os.path.expanduser(configured)
return os.path.join(os.path.expanduser("~"), ".mineru")
def _default_path(path1: str, /, *paths: str) -> str:
return os.path.join(_mineru_home(), path1, *paths)
def _normalize_log_level(value: LogLevel | str | None) -> LogLevel | str | None:
"""归一化日志级别输入;None 表示未设置局部覆盖。"""
if value is None or not isinstance(value, str):
return value
return value.strip().lower()
def _uds_available() -> bool:
try:
socket.socket(socket.AF_UNIX, socket.SOCK_STREAM).close()
return True
except Exception:
return False
def _strip_quotes(value: str) -> str:
value = value.strip()
if len(value) >= 2 and value[0] in ('"', "'") and value[-1] == value[0]:
return value[1:-1]
return value
def _interpolation_replace(match: re.Match[str]) -> str:
var_name, raw_default = match.group(1), match.group(2)
value = os.environ.get(var_name)
if value is not None:
return value
if raw_default is not None:
return _strip_quotes(raw_default)
raise ValueError(
f"Environment variable {var_name!r} is referenced in MinerU config but is not set and has no default value."
)
def _substitute(value: str, max_depth: int = 20) -> str:
for _ in range(max_depth):
substituted = _INTERPOLATION_RE.sub(_interpolation_replace, value)
if substituted == value:
return value
value = substituted
raise ValueError(f"MinerU config interpolation did not converge after {max_depth} passes.")
def _interpolate_env(value: Any) -> Any:
"""Recursively substitute ${VAR} and ${VAR:-default} placeholders."""
if isinstance(value, str):
return _substitute(value)
if isinstance(value, dict):
return {key: _interpolate_env(item) for key, item in value.items()}
if isinstance(value, list):
return [_interpolate_env(item) for item in value]
return value
def _load_config(config_file: str) -> dict[str, Any]:
with open(config_file, encoding="utf-8") as file:
raw = yaml.safe_load(file)
if isinstance(raw, dict):
return _interpolate_env(raw)
_logger.warning("MinerU config file [%s] is empty or invalid.", config_file)
return {}
def _resolve_config_file() -> tuple[str, bool]:
config_file = os.getenv(MINERU_CONFIG_ENV)
if config_file not in (None, ""):
config_file = os.path.expanduser(config_file)
if not os.path.isfile(config_file):
raise FileNotFoundError(f"MinerU config file [{config_file}] does not exist.")
return config_file, True
default_config_file = _default_path("config.yaml")
exists = os.path.isfile(default_config_file)
if not exists:
_logger.debug(
"MinerU config file not found. Default path is %s. Use %s to specify a custom path.",
default_config_file,
MINERU_CONFIG_ENV,
)
return default_config_file, exists
def _collect_path(remaining: str, model_class: type[BaseModel]) -> list[str] | None:
"""Greedily match an UPPER_CASE env suffix to a pydantic field path."""
remaining_lower = remaining.lower()
fields = sorted(model_class.model_fields.keys(), key=len, reverse=True)
for field_name in fields:
if remaining_lower == field_name:
return [field_name]
if remaining_lower.startswith(field_name + "_"):
annotation = model_class.model_fields[field_name].annotation
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
sub_remaining = remaining[len(field_name) + 1 :]
sub_path = _collect_path(sub_remaining, annotation)
if sub_path is not None:
return [field_name, *sub_path]
return None
def _deep_merge(base: dict[str, Any], overrides: dict[str, Any]) -> dict[str, Any]:
result = copy.deepcopy(base)
for key, value in overrides.items():
if key in result and isinstance(result[key], dict) and isinstance(value, dict):
result[key] = _deep_merge(result[key], value)
else:
result[key] = value
return result
def _collect_env_overrides(prefix: str = MINERU_ENV_PREFIX) -> tuple[dict[str, Any], set[tuple[str, ...]]]:
prefix_upper = prefix.upper()
overrides: dict[str, Any] = {}
paths: set[tuple[str, ...]] = set()
for key, value in os.environ.items():
if not key.startswith(prefix_upper):
continue
remaining = key[len(prefix_upper) :]
if not remaining or remaining == "CONFIG":
continue
path = _collect_path(remaining, Config)
if path is None:
continue
node = overrides
for part in path[:-1]:
node = node.setdefault(part, {})
node[path[-1]] = value
paths.add(tuple(path))
return overrides, paths
def _model_annotation(annotation: Any) -> type[BaseModel] | None:
if isinstance(annotation, type) and issubclass(annotation, BaseModel):
return annotation
origin = get_origin(annotation)
if origin is None:
return None
for arg in get_args(annotation):
if isinstance(arg, type) and issubclass(arg, BaseModel):
return arg
return None
def _default_source_paths(model_class: type[BaseModel], prefix: tuple[str, ...] = ()) -> set[tuple[str, ...]]:
paths: set[tuple[str, ...]] = set()
for field_name, field in model_class.model_fields.items():
sub_model = _model_annotation(field.annotation)
current = (*prefix, field_name)
if sub_model is not None:
paths.update(_default_source_paths(sub_model, current))
else:
paths.add(current)
return paths
def _configured_source_paths(
data: dict[str, Any],
model_class: type[BaseModel],
prefix: tuple[str, ...] = (),
) -> set[tuple[str, ...]]:
paths: set[tuple[str, ...]] = set()
for key, value in data.items():
if key not in model_class.model_fields:
continue
field = model_class.model_fields[key]
sub_model = _model_annotation(field.annotation)
current = (*prefix, key)
if sub_model is not None and isinstance(value, dict):
paths.update(_configured_source_paths(value, sub_model, current))
else:
paths.add(current)
return paths
def _normalize_config_source_path(path: str | Sequence[str]) -> tuple[str, ...]:
if isinstance(path, str):
return tuple(part for part in path.split(".") if part)
return tuple(path)
@dataclass(frozen=True)
class LoadedConfig:
config: "Config"
sources: dict[tuple[str, ...], ConfigSource]
config_file: str
config_file_exists: bool
def _load_effective_config() -> LoadedConfig:
if "MINERU_MODEL_STACK" in os.environ:
raise ValueError("MINERU_MODEL_STACK was removed; use MINERU_MODEL_SMALL_BACKEND and MINERU_MODEL_VLM_ENGINE")
config_file, config_file_exists = _resolve_config_file()
raw_config = _load_config(config_file) if config_file_exists else {}
sources: dict[tuple[str, ...], ConfigSource] = dict.fromkeys(_default_source_paths(Config), "default")
for path in _configured_source_paths(raw_config, Config):
sources[path] = "file"
base_config = Config(**raw_config)
overrides, env_paths = _collect_env_overrides()
if overrides:
base_config = Config(**_deep_merge(base_config.model_dump(), overrides))
for path in env_paths:
sources[path] = "env"
return LoadedConfig(
config=base_config,
sources=sources,
config_file=config_file,
config_file_exists=config_file_exists,
)
def load_config() -> LoadedConfig:
"""按当前文件和环境独立装载配置及来源,不替换进程默认配置。"""
return _load_effective_config()
def _get_loaded_config() -> LoadedConfig:
"""首次使用时在线程锁内装载进程默认配置,后续复用同一份来源记录。"""
global _loaded_config
if _loaded_config is None:
with _config_load_lock:
if _loaded_config is None:
_loaded_config = load_config()
return _loaded_config
def get_config() -> Config:
"""返回首次使用时装载并缓存的进程默认配置。"""
return _get_loaded_config().config
def get_config_source(path: str | Sequence[str]) -> ConfigSource:
"""从进程默认配置的同一份装载结果查询字段来源。"""
return _get_loaded_config().sources.get(_normalize_config_source_path(path), "default")
def get_config_file_path() -> str:
"""返回进程默认配置首次装载时确定的文件路径。"""
return _get_loaded_config().config_file
def get_config_file_exists() -> bool:
"""返回进程默认配置首次装载时的文件存在状态。"""
return _get_loaded_config().config_file_exists
def update_config_file(patch: dict[str, Any], *, only_if_onnx_auto: bool = False) -> bool:
"""Atomically merge a patch, optionally comparing the current ONNX choice under the lock."""
config_file = get_config_file_path()
config_dir = os.path.dirname(config_file)
if config_dir:
os.makedirs(config_dir, exist_ok=True)
lock = FileLock(f"{config_file}.lock")
with lock:
if os.path.exists(config_file):
with open(config_file, encoding="utf-8") as file:
loaded = yaml.safe_load(file)
current = loaded if isinstance(loaded, dict) else {}
else:
current = {}
if only_if_onnx_auto:
model = current.get("model", {})
if not isinstance(model, dict) or model.get("onnx_device", "auto") != "auto":
return False
updated = _deep_merge(current, patch)
_write_config_file(config_file, updated)
return True
def _write_config_file(config_file: str, config: dict[str, Any]) -> None:
"""在写入配置正文之前限制临时文件权限,原子替换后保留同样的私密权限。"""
directory = os.path.dirname(config_file) or "."
doclib = config.get("doclib")
auth = doclib.get("auth") if isinstance(doclib, dict) else None
if isinstance(auth, dict) and auth.get("token") is not None:
fd, tmp_path = create_private_tempfile(directory)
else:
fd, tmp_path = tempfile.mkstemp(prefix=".config.", suffix=".tmp", dir=directory)
try:
with os.fdopen(fd, "w", encoding="utf-8") as file:
yaml.safe_dump(config, file, allow_unicode=True, sort_keys=False)
os.replace(tmp_path, config_file)
finally:
if os.path.exists(tmp_path):
os.unlink(tmp_path)
def load_doclib_server_config() -> LoadedConfig:
"""启动时在配置锁内生成或复用本机凭证;普通配置读取不生成文件或凭证。"""
loaded = load_config()
config_file = loaded.config_file
os.makedirs(os.path.dirname(os.path.abspath(config_file)), mode=0o700, exist_ok=True)
with FileLock(f"{config_file}.lock"):
loaded = load_config()
if os.path.isfile(config_file):
protect_private_file(config_file)
if loaded.config.doclib.auth.token is not None:
validate_doclib_token(loaded.config.doclib.auth.token.get_secret_value())
return loaded
if os.path.isfile(config_file):
with open(config_file, encoding="utf-8") as file:
current = yaml.safe_load(file) or {}
else:
current = {}
token = secrets.token_urlsafe(32)
updated = _deep_merge(current, {"doclib": {"auth": {"token": token}}})
_write_config_file(config_file, updated)
return load_config()
def validate_doclib_token(token: str) -> str:
"""校验本机凭证的长度和 HTTP Bearer 可用字符,错误消息不包含凭证。"""
if not re.fullmatch(r"[A-Za-z0-9._~+/=-]{32,}", token):
raise ValueError("Doclib token must contain at least 32 ASCII Bearer-token characters.")
return token
def read_onnx_device_from_file() -> OnnxDevice:
"""Re-read only the device choice after acquiring the cross-process selection lock."""
path = get_config_file_path()
raw = _load_config(path) if os.path.isfile(path) else {}
return ModelConfig(**raw.get("model", {})).onnx_device
class UDSConfig(BaseModel):
enabled: AutoBool = "auto"
path: str = Field(default_factory=lambda: _default_path("doclib.sock"))
permission: int = 0o600
class TCPConfig(BaseModel):
enabled: AutoBool = "auto"
host: str = "127.0.0.1"
port: int = 15980
strict_port: bool = False
port_probe_count: int = Field(default=100, ge=1)
backlog: int = 128
timeout: int = 600
class GlobalLogConfig(BaseModel):
"""MinerU 全局日志配置,控制当前进程的 Loguru 默认输出。"""
level: LogLevel = "info"
@field_validator("level", mode="before")
@classmethod
def _normalize_level(cls, value: LogLevel | str | None) -> LogLevel | str | None:
"""允许配置文件和环境变量使用大小写混写,并忽略首尾空白。"""
return _normalize_log_level(value)
class LogConfig(BaseModel):
dir: str = Field(default_factory=lambda: _default_path("logs"))
app_path: str | None = None
access_path: str | None = None
stdout_path: str | None = None
stderr_path: str | None = None
parse_server_stdout_path: str | None = None
parse_server_stderr_path: str | None = None
level: LogLevel | None = None
@field_validator("level", mode="before")
@classmethod
def _normalize_level(cls, value: LogLevel | str | None) -> LogLevel | str | None:
"""归一化 Doclib 的局部日志级别覆盖。"""
return _normalize_log_level(value)
@property
def resolved_app_path(self) -> str:
return self.app_path or os.path.join(self.dir, "doclib.log")
@property
def resolved_access_path(self) -> str:
return self.access_path or os.path.join(self.dir, "doclib.access.log")
@property
def resolved_stdout_path(self) -> str:
return self.stdout_path or os.path.join(self.dir, "doclib.stdout.log")
@property
def resolved_stderr_path(self) -> str:
return self.stderr_path or os.path.join(self.dir, "doclib.stderr.log")
@property
def resolved_parse_server_stdout_path(self) -> str:
return self.parse_server_stdout_path or os.path.join(self.dir, "doclib.parse-server.stdout.log")
@property
def resolved_parse_server_stderr_path(self) -> str:
return self.parse_server_stderr_path or os.path.join(self.dir, "doclib.parse-server.stderr.log")
class SQLiteConfig(BaseModel):
path: str = Field(default_factory=lambda: _default_path("doclib.db"))
busy_timeout_ms: int = Field(default=5000, ge=0)
lock_retry_attempts: int = Field(default=3, ge=0)
lock_retry_base_delay_ms: int = Field(default=50, ge=0)
mmap_size: int = 268435456
cache_size: int = -20000
wal_autocheckpoint: int = 1000
journal_size_limit: int = 33_554_432
temp_store: str = "memory"
synchronous: str = "NORMAL"
class ManagedParseServerConfig(BaseModel):
host: str = "127.0.0.1"
port: int = 16580
strict_port: bool = False
port_probe_count: int = Field(default=100, ge=1)
class RenderConfig(BaseModel):
"""MiddleJson 输出渲染配置。"""
latex_delimiters: LatexDelimitersConfig = Field(default_factory=LatexDelimitersConfig)
class LLMAidedFeaturesConfig(BaseModel):
"""LLM 辅助后处理的独立功能开关。"""
title_leveling: bool = False
cross_page_table_cell_merge: bool = False
class LLMAidedConfig(BaseModel):
"""OpenAI-compatible LLM 辅助后处理配置。"""
api_key: str = Field(default="", repr=False)
base_url: str = "https://dashscope.aliyuncs.com/compatible-mode/v1"
model: str = "qwen3.8-flash"
enable_thinking: bool | None = None
max_concurrency: int = Field(default=16, ge=1)
allow_partial_title_leveling: bool = False
features: LLMAidedFeaturesConfig = Field(default_factory=LLMAidedFeaturesConfig)
@field_validator("max_concurrency", mode="before")
@classmethod
def _reject_boolean_max_concurrency(cls, value: Any) -> Any:
"""拒绝被 Python 视作整数的布尔并发数,同时保留环境变量字符串转换。"""
if isinstance(value, bool):
raise ValueError("llm_aided.max_concurrency must be a positive integer")
return value
@model_validator(mode="after")
def _validate_enabled_credentials(self) -> "LLMAidedConfig":
"""任一 LLM 功能启用时要求连接参数均为非空字符串。"""
enabled = self.features.title_leveling or self.features.cross_page_table_cell_merge
if not enabled:
return self
for field_name in ("api_key", "base_url", "model"):
value = getattr(self, field_name)
if not isinstance(value, str) or not value.strip():
raise ValueError(f"llm_aided.{field_name} must be a non-empty string when an LLM feature is enabled")
return self
class VlmConfig(BaseModel):
"""所有解析入口共享的 VLM 引擎与连接配置;远程地址优先于本地引擎。"""
model_config = {"hide_input_in_errors": True}
engine: Literal["auto", "llama-cpp", "vllm", "lmdeploy", "mlx"] = "auto"
server_url: str = ""
api_key: str = Field(default="", repr=False)
model: str = ""
http_timeout: int = Field(default=600, ge=1)
max_concurrency: int = Field(default=100, ge=1)
@field_validator("api_key", "model")
@classmethod
def _strip_connection_value(cls, value: str) -> str:
"""清理连接字段首尾空白,保持显式空值可覆盖全局配置。"""
return value.strip()
@field_validator("server_url")
@classmethod
def _normalize_server_url(cls, value: str) -> str:
"""保留代理前缀并移除末尾 v1,以尾斜线阻止底层客户端丢弃路径。"""
value = value.strip()
if not value:
return ""
try:
parts = urlsplit(value)
valid = (
parts.scheme in {"http", "https"}
and bool(parts.hostname)
and parts.username is None
and parts.password is None
and not parts.query
and not parts.fragment
and not any(char.isspace() for char in value)
)
_ = parts.port
except ValueError:
valid = False
if not valid:
raise ValueError("model.vlm.server_url must be an HTTP(S) service URL without credentials, query or fragment")
path = parts.path.rstrip("/")
if path.endswith("/v1"):
path = path[:-3]
return urlunsplit((parts.scheme, parts.netloc, path + "/", "", ""))
@field_validator("http_timeout", "max_concurrency", mode="before")
@classmethod
def _reject_boolean_limit(cls, value: Any) -> Any:
"""拒绝布尔数值,同时允许配置环境变量提供整数文本。"""
if isinstance(value, bool):
raise ValueError("VLM timeout and concurrency must be positive integers")
return value
def validate_environment(self) -> None:
"""远程推理前拒绝会隐式覆盖显式配置的旧环境变量,错误不包含凭据。"""
if not self.server_url:
return
for env_name, expected, field in (
("MINERU_VL_API_KEY", self.api_key, "api_key"),
("MINERU_VL_MODEL_NAME", self.model, "model"),
):
# 底层只清理 API Key;模型名按原始环境变量值读取。
actual = os.getenv(env_name, "")
if field == "api_key":
actual = actual.strip()
if actual and actual != expected:
raise ValueError(f"{env_name} conflicts with model.vlm.{field}; unset the legacy environment variable")
class ModelConfig(BaseModel):
base_dir: str = Field(default_factory=lambda: _default_path("models"))
source: str = "auto"
small_backend: Literal["auto", "onnx", "torch"] = "auto"
onnx_device: OnnxDevice = "auto"
vlm: VlmConfig = Field(default_factory=VlmConfig)
@model_validator(mode="before")
@classmethod
def _reject_legacy_stack(cls, value: Any) -> Any:
"""拒绝旧模型栈字段,避免配置被忽略后意外切换推理后端。"""
if isinstance(value, dict) and "stack" in value:
raise ValueError("model.stack was removed; use model.small_backend and model.vlm.engine")
return value
class DoclibAuthConfig(BaseModel):
"""保存跨服务重启稳定的本机凭证,默认序列化和 repr 都隐藏正文。"""
token: SecretStr | None = Field(default=None, repr=False)
class DoclibConfig(BaseModel):
"""Doclib startup configuration.
Only configuration needed before the doclib server starts belongs here.
Runtime configuration that can be read from SQLite stays in the config
service.
"""
auth: DoclibAuthConfig = Field(default_factory=DoclibAuthConfig)
uds: UDSConfig = Field(default_factory=UDSConfig)
tcp: TCPConfig = Field(default_factory=TCPConfig)
log: LogConfig = Field(default_factory=LogConfig)
sqlite: SQLiteConfig = Field(default_factory=SQLiteConfig)
endpoint_path: str = Field(default_factory=lambda: _default_path("doclib.endpoint.json"))
data_dir: str = Field(default_factory=lambda: _default_path("doclib"))
managed_parse_server: ManagedParseServerConfig = Field(default_factory=ManagedParseServerConfig)
ingest_workers: int = 2
parse_workers: int = 2
scan_interval_sec: int = 300
device_check_interval_sec: int = 5
ingest_lock_timeout_sec: int = 60
parse_lock_timeout_sec: int = 1800
scan_lock_timeout_sec: int = 1800
compaction_interval_sec: int = 3600
parse_server_health_check_interval_sec: int = 30
parse_server_probe_timeout_sec: int = 10
parse_server_startup_grace_sec: int = 30
parse_server_startup_timeout_sec: int = 600
parse_server_stop_timeout_sec: int = 10
@property
def resolved_uds_enabled(self) -> bool:
if self.uds.enabled == "auto":
return _uds_available()
return self.uds.enabled
@property
def resolved_tcp_enabled(self) -> bool:
if self.tcp.enabled == "auto":
return not _uds_available()
return self.tcp.enabled
class Config(BaseModel):
"""Top-level MinerU startup configuration."""
log: GlobalLogConfig = Field(default_factory=GlobalLogConfig)
doclib: DoclibConfig = Field(default_factory=DoclibConfig)
render: RenderConfig = Field(default_factory=RenderConfig)
llm_aided: LLMAidedConfig = Field(default_factory=LLMAidedConfig)
model: ModelConfig = Field(default_factory=ModelConfig)
_loaded_config: LoadedConfig | None = None
_config_load_lock = threading.Lock()
def PatchedConfig(**kwargs: Any) -> Config:
"""在进程默认配置的独立副本上应用局部覆盖并重新校验。"""
merged = _deep_merge(get_config().model_dump(), kwargs)
return Config(**merged)
__all__ = [
"DoclibAuthConfig",
"load_doclib_server_config",
"validate_doclib_token",
"OnnxDevice",
"read_onnx_device_from_file",
"AutoBool",
"ConfigSource",
"LoadedConfig",
"LogLevel",
"GlobalLogConfig",
"ModelConfig",
"VlmConfig",
"ModelSource",
"get_config",
"load_config",
"Config",
"DoclibConfig",
"LLMAidedConfig",
"LLMAidedFeaturesConfig",
"LatexDelimiterConfig",
"LatexDelimitersConfig",
"RenderConfig",
"TCPConfig",
"LogConfig",
"SQLiteConfig",
"UDSConfig",
"PatchedConfig",
"get_config_source",
"get_config_file_path",
"get_config_file_exists",
"update_config_file",
"MINERU_HOME_ENV",
"MINERU_CONFIG_ENV",
"MINERU_ENV_PREFIX",
]