Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
# Copyright (c) Opendatalab. All rights reserved.
from __future__ import annotations
import atexit
import gc
import hashlib
import json
import os
import threading
import time
from collections.abc import Callable
from contextvars import ContextVar, copy_context
from dataclasses import dataclass, field
from functools import partial
from pathlib import Path
from typing import Any, Literal
from loguru import logger
from mineru_vl_utils import MinerUClient
from ..registry import MINERU_2_5_PRO_2605_1_2B, vlm_model_repo
from ..runtime.closing import in_runtime_close_scope, runtime_close_scope
from ..runtime.device import get_device
from ..runtime.ownership import RuntimeOwner, current_runtime_owner, runtime_initialization_scope
from ..runtime.platform import is_mac_os_version_supported
from .async_runtime import AsyncVlmPredictor
from .engine_utils import (
enable_custom_logits_processors,
mod_kwargs_by_device_type,
set_default_batch_size,
set_default_gpu_memory_utilization,
set_lmdeploy_backend,
)
from .errors import EngineNotReadyError
from .handles import RuntimeHandle
@dataclass(eq=False)
class _RuntimeInitialization:
"""保存同键初始化屏障,直到构造发布或失败清理完整结束。"""
finished: threading.Event = field(default_factory=threading.Event)
_initializing_runtime: ContextVar[_RuntimeInitialization | None] = ContextVar("mineru_initializing_vlm", default=None)
class ModelSingleton:
_instance = None
_models: dict[tuple[object, ...], RuntimeHandle | AsyncVlmPredictor] = {}
_closing: dict[tuple[object, ...], threading.Event] = {}
_initializing: dict[tuple[object, ...], _RuntimeInitialization] = {}
_lock = threading.RLock()
_shutdown_registered = False
def __new__(cls, *args: object, **kwargs: object) -> ModelSingleton:
with cls._lock:
if cls._instance is None:
cls._instance = super().__new__(cls)
return cls._instance
def get_model(
self,
backend: Literal[
"http-client",
"llama-cpp-engine",
"transformers",
"mlx-engine",
"lmdeploy-engine",
"vllm-engine",
"vllm-async-engine",
],
model_path: str | None,
server_url: str | None,
*,
model_name: str | None = None,
**kwargs: Any,
) -> RuntimeHandle | AsyncVlmPredictor:
"""先接纳应用操作,再沿原缓存键复用运行时;路径解析和构造都不持有缓存锁。"""
with runtime_initialization_scope():
if backend == "vllm-engine":
backend = "vllm-async-engine"
native_async = backend in {"http-client", "vllm-async-engine"}
options = {"model_name": model_name, **kwargs}
if backend == "vllm-async-engine":
options["device"] = get_device()
model_path = (
str(Path(model_path).expanduser().resolve()) if model_path else str(MINERU_2_5_PRO_2605_1_2B.ensure())
)
fingerprint = hashlib.sha256(json.dumps(options, sort_keys=True, default=str).encode()).hexdigest()
cache_key = (backend, model_path, server_url, fingerprint)
factory = partial(self._create_model, backend, model_path, server_url, model_name=model_name, **kwargs)
return self._get_or_initialize(
cache_key,
factory,
native_async=native_async,
backend=backend,
max_concurrency=kwargs.get("max_concurrency", 100),
)
def _retain(self, predictor: RuntimeHandle | AsyncVlmPredictor) -> None:
"""发布或复用前重新校验应用,防止关闭后的预热结果进入缓存。"""
owner = current_runtime_owner()
predictor.lifecycle.retain(owner)
if owner is not None and predictor.supports_health_probe:
owner.has_engine_health_probe = True
def _get_or_initialize(
self,
key: tuple[object, ...],
factory: Callable[[], RuntimeHandle],
*,
native_async: bool,
backend: str,
max_concurrency: int,
) -> RuntimeHandle | AsyncVlmPredictor:
"""同键共享初始化或关闭屏障,外部构造和关闭动作始终在全局锁之外。"""
while True:
detached = []
reservation = None
with self._lock:
current_runtime_owner()
closing = self._closing.get(key)
initializing = self._initializing.get(key)
cached = self._models.get(key)
if in_runtime_close_scope() and (
closing is not None or initializing is not None or (cached is not None and cached.is_closed)
):
raise RuntimeError("Cannot acquire a closing VLM runtime from a close callback")
if initializing is not None and initializing is _initializing_runtime.get():
raise RuntimeError("Cannot recursively acquire a VLM runtime during its initialization")
if closing is not None:
pending = closing
elif initializing is not None:
pending = initializing.finished
elif cached is not None and cached.is_closed:
pending = None
detached.append(self._detach(key, cached))
elif cached is not None:
self._retain(cached)
cached.require_ready(allow_unavailable=True)
return cached
else:
pending = None
reservation = _RuntimeInitialization()
self._initializing[key] = reservation
if not self._shutdown_registered:
atexit.register(shutdown_cached_models)
type(self)._shutdown_registered = True
if pending is not None:
pending.wait()
elif detached:
self._close_detached(detached)
elif reservation is not None:
return self._initialize_model(
key, reservation, factory, native_async=native_async, backend=backend, max_concurrency=max_concurrency
)
def _initialize_model(
self,
key: tuple[object, ...],
reservation: _RuntimeInitialization,
factory: Callable[[], RuntimeHandle],
*,
native_async: bool,
backend: str,
max_concurrency: int,
) -> RuntimeHandle | AsyncVlmPredictor:
"""锁外构造并等待就绪,失效应用或失败构造完整清理后才解除同键屏障。"""
predictor: RuntimeHandle | AsyncVlmPredictor | None = None
token = _initializing_runtime.set(reservation)
try:
if native_async:
factory_context = copy_context()
predictor = AsyncVlmPredictor(
partial(factory_context.run, factory), backend=backend, max_concurrency=max_concurrency
)
predictor.wait_ready()
else:
predictor = factory()
with self._lock:
self._retain(predictor)
self._models[key] = predictor
return predictor
except BaseException as initialization_error:
if predictor is not None:
try:
with runtime_close_scope():
predictor.shutdown()
except BaseException as cleanup_error:
raise initialization_error from cleanup_error
raise
finally:
_initializing_runtime.reset(token)
with self._lock:
self._initializing.pop(key, None)
reservation.finished.set()
def _detach(
self, key: tuple[object, ...], predictor: RuntimeHandle | AsyncVlmPredictor
) -> tuple[tuple[object, ...], RuntimeHandle | AsyncVlmPredictor, threading.Event]:
"""在缓存锁内撤销条目并保留关闭屏障,防止相同权重在关闭完成前重新创建。"""
del self._models[key]
finished = threading.Event()
self._closing[key] = finished
return key, predictor, finished
def _close_detached(
self, entries: list[tuple[tuple[object, ...], RuntimeHandle | AsyncVlmPredictor, threading.Event]]
) -> None:
"""锁外逐个关闭已撤销条目,某个后端失败也释放其余条目和等待屏障。"""
first_error: BaseException | None = None
for key, predictor, finished in entries:
try:
with runtime_close_scope():
predictor.shutdown()
except BaseException as exc:
if first_error is None:
first_error = exc
finally:
with self._lock:
self._closing.pop(key, None)
finished.set()
if first_error is not None:
raise first_error
def _create_model(
self,
backend: str,
model_path: str | None,
server_url: str | None,
*,
model_name: str | None = None,
**kwargs: Any,
) -> RuntimeHandle:
"""只构造一个底层客户端,不操作缓存;原生异步引擎由所属运行时调用。"""
start_time = time.time()
model = None
processor = None
vllm_llm = None
lmdeploy_engine = None
vllm_async_llm = None
close_engine = None
llama_cpp_engine = None
batch_size = kwargs.get("batch_size", 0) # 本地引擎批次大小;MLX 的 0 由客户端解析为保守默认值。
max_concurrency = kwargs.get("max_concurrency", 100) # 原生异步运行时在所有文档间共用该额度。
http_timeout = kwargs.get("http_timeout", 600) # for http-client backend only
server_headers = kwargs.get("server_headers", None) # for http-client backend only
max_retries = kwargs.get("max_retries", 3) # for http-client backend only
retry_backoff_factor = kwargs.get("retry_backoff_factor", 0.5) # for http-client backend only
# 从kwargs中移除这些参数,避免传递给不相关的初始化函数
for param in [
"batch_size",
"max_concurrency",
"http_timeout",
"server_headers",
"max_retries",
"retry_backoff_factor",
]:
if param in kwargs:
del kwargs[param]
if backend not in ["http-client", "llama-cpp-engine"] and not model_path:
model_path = str(MINERU_2_5_PRO_2605_1_2B.ensure())
if backend == "llama-cpp-engine":
# The GGUF repo carries two files: the main model and its
# multi-modal projector (mmproj). ModelRepo.paths is
# {"main": "...", "mmproj": "..."} — resolve each to its
# absolute path under model_dir and hand both to Engine.
repo = vlm_model_repo("llama-cpp")
model_dir = Path(model_path).expanduser() if model_path else repo.ensure()
model_gguf = model_dir / repo.paths["main"]
mmproj_gguf = model_dir / repo.paths["mmproj"]
# Engine-specific kwargs are pulled out of **kwargs so they
# are not forwarded to MinerUClient (which doesn't accept
# them). CPU threads default to 4 on GPU, otherwise Engine's auto.
engine_kwargs = {}
for key in ("n_ctx_seq", "n_gpu_layers", "n_parallel", "verbosity", "n_threads"):
if key in kwargs:
engine_kwargs[key] = kwargs.pop(key)
from ..runtime.llama_device import default_llama_threads, llama_engine_scope, resolve_llama_device
selection = resolve_llama_device(n_gpu_layers=engine_kwargs.get("n_gpu_layers"))
if selection.device == "cpu":
engine_kwargs["n_gpu_layers"] = 0
default_threads = default_llama_threads(selection)
if default_threads is not None:
engine_kwargs.setdefault("n_threads", default_threads)
try:
from mineru_llama_cpp import Engine
except ImportError:
raise ImportError("Please install mineru-llama-cpp to use the llama-cpp-engine backend.")
with llama_engine_scope(selection):
llama_cpp_engine = Engine(model_gguf, mmproj_gguf, **engine_kwargs)
close_engine = llama_cpp_engine.close
elif backend == "transformers":
from ..runtime.cpu_threads import initialize_torch_threads
initialize_torch_threads()
try:
from mineru_vl_utils.transformers_loading import (
load_transformers_model,
load_transformers_processor,
)
model = load_transformers_model(model_path, device_map={"": get_device()})
processor = load_transformers_processor(model_path)
except ImportError as exc:
raise ImportError("Please install transformers to use the transformers backend.") from exc
if batch_size == 0:
batch_size = set_default_batch_size()
elif backend == "mlx-engine":
mlx_supported = is_mac_os_version_supported("14.0")
if not mlx_supported:
raise EnvironmentError("mlx-engine backend is only supported on macOS 14.0+ with Apple Silicon.")
from mineru_vl_utils.mlx_compat import load_mlx_model
model, processor = load_mlx_model(model_path)
elif backend != "http-client":
if os.getenv("OMP_NUM_THREADS") is None:
os.environ["OMP_NUM_THREADS"] = "1"
if backend == "vllm-async-engine":
try:
from vllm.config import CompilationConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.v1.engine.async_llm import AsyncLLM
except ImportError:
raise ImportError("Please install vllm to use the vllm-async-engine backend.")
kwargs = mod_kwargs_by_device_type(kwargs, vllm_mode="async_engine")
if "compilation_config" in kwargs:
if isinstance(kwargs["compilation_config"], dict):
# 如果是字典,转换为 CompilationConfig 对象
kwargs["compilation_config"] = CompilationConfig(**kwargs["compilation_config"])
elif isinstance(kwargs["compilation_config"], str):
# 如果是 JSON 字符串,先解析再转换
try:
config_dict = json.loads(kwargs["compilation_config"])
kwargs["compilation_config"] = CompilationConfig(**config_dict)
except (json.JSONDecodeError, TypeError) as e:
logger.warning(f"Failed to parse compilation_config: {kwargs['compilation_config']}, error: {e}")
del kwargs["compilation_config"]
if "gpu_memory_utilization" not in kwargs:
kwargs["gpu_memory_utilization"] = set_default_gpu_memory_utilization()
if "model" not in kwargs:
kwargs["model"] = model_path
if enable_custom_logits_processors() and ("logits_processors" not in kwargs):
from mineru_vl_utils import MinerULogitsProcessor
kwargs["logits_processors"] = [MinerULogitsProcessor]
# 使用kwargs为 vllm初始化参数
vllm_async_llm = AsyncLLM.from_engine_args(AsyncEngineArgs(**kwargs))
close_engine = vllm_async_llm.shutdown
elif backend == "lmdeploy-engine":
try:
from lmdeploy import PytorchEngineConfig, TurbomindEngineConfig, pipeline
except ImportError:
raise ImportError("Please install lmdeploy to use the lmdeploy-engine backend.")
if "cache_max_entry_count" not in kwargs:
kwargs["cache_max_entry_count"] = set_default_gpu_memory_utilization(backend="lmdeploy")
device_type = os.getenv("MINERU_LMDEPLOY_DEVICE", "")
if device_type == "":
if "lmdeploy_device" in kwargs:
device_type = kwargs.pop("lmdeploy_device")
if device_type not in ["cuda", "ascend", "maca", "camb"]:
raise ValueError(f"Unsupported lmdeploy device type: {device_type}")
else:
device_type = "cuda"
lm_backend = os.getenv("MINERU_LMDEPLOY_BACKEND", "")
if lm_backend == "":
if "lmdeploy_backend" in kwargs:
lm_backend = kwargs.pop("lmdeploy_backend")
if lm_backend not in ["pytorch", "turbomind"]:
raise ValueError(f"Unsupported lmdeploy backend: {lm_backend}")
else:
lm_backend = set_lmdeploy_backend(device_type)
logger.info(f"lmdeploy device is: {device_type}, lmdeploy backend is: {lm_backend}")
if lm_backend == "pytorch":
kwargs["device_type"] = device_type
backend_config = PytorchEngineConfig(**kwargs)
elif lm_backend == "turbomind":
backend_config = TurbomindEngineConfig(**kwargs)
else:
raise ValueError(f"Unsupported lmdeploy backend: {lm_backend}")
log_level = "ERROR"
from lmdeploy.utils import get_logger
lm_logger = get_logger("lmdeploy")
lm_logger.setLevel(log_level)
if os.getenv("TM_LOG_LEVEL") is None:
os.environ["TM_LOG_LEVEL"] = log_level
lmdeploy_engine = pipeline(
model_path,
backend_config=backend_config,
log_level=log_level,
)
close_engine = lmdeploy_engine.close
try:
predictor = MinerUClient(
backend=backend,
model_name=model_name,
model=model,
processor=processor,
lmdeploy_engine=lmdeploy_engine,
vllm_llm=vllm_llm,
vllm_async_llm=vllm_async_llm,
llama_cpp_engine=llama_cpp_engine,
server_url=server_url,
batch_size=batch_size,
max_concurrency=max_concurrency,
http_timeout=http_timeout,
server_headers=server_headers,
max_retries=max_retries,
retry_backoff_factor=retry_backoff_factor,
enable_table_formula_eq_wrap=True,
image_analysis=True,
enable_cross_page_table_merge=False,
)
except BaseException as initialization_error:
# 客户端构造失败后,宿主仍负责关闭已经成功创建的引擎。
if close_engine is not None:
try:
with runtime_close_scope():
close_engine()
except BaseException as cleanup_error:
raise initialization_error from cleanup_error
raise
elapsed = round(time.time() - start_time, 2)
logger.info(f"get {backend} predictor cost: {elapsed}s")
return RuntimeHandle(
predictor,
backend=backend,
close_engine=close_engine,
probe_engine=predictor.client.probe_health if predictor.client.health_probe_mode == "sync" else None,
aio_probe_engine=predictor.client.aio_probe_health if predictor.client.health_probe_mode == "async" else None,
# 托管进程的关闭期限由外部管理器控制;释放 Pipeline 前必须真正排空健康工作。
close_health_probe=(
partial(predictor.client.close_health_probe, timeout=None) if backend == "lmdeploy-engine" else None
),
)
def probe_owner_health(self, owner: RuntimeOwner, timeout: float = 5.0) -> None:
"""只检查该应用已经创建的运行时;快照之后释放缓存锁再等待后端。"""
with self._lock:
handles = tuple(
predictor
for predictor in self._models.values()
if owner in predictor.owners and predictor.supports_health_probe
)
unavailable: EngineNotReadyError | None = None
for predictor in handles:
try:
predictor.probe_health(timeout)
except EngineNotReadyError as exc:
unavailable = unavailable or exc
if unavailable is not None:
raise unavailable
def release_owner(self, owner: RuntimeOwner) -> None:
"""排空该应用文档后撤销全部后端租约,保留其它应用和独立解析持有的实例。"""
owner.close()
with self._lock:
closing = []
for key, predictor in tuple(self._models.items()):
predictor.lifecycle.release(owner)
if not predictor.owners and not predictor.pinned:
closing.append(self._detach(key, predictor))
self._close_detached(closing)
def shutdown(self) -> None:
"""等待已开始的初始化收尾,撤销缓存后在锁外排空运行时。"""
while True:
with self._lock:
pending = tuple(self._initializing.values())
if not pending:
closing = [self._detach(key, predictor) for key, predictor in tuple(self._models.items())]
break
if _initializing_runtime.get() in pending or in_runtime_close_scope():
raise RuntimeError("Cannot wait for VLM initialization from its cleanup callback")
for item in pending:
item.finished.wait()
try:
self._close_detached(closing)
finally:
gc.collect()
async def _get_model_async(
backend: Literal[
"http-client",
"llama-cpp-engine",
"transformers",
"mlx-engine",
"lmdeploy-engine",
"vllm-engine",
"vllm-async-engine",
],
model_path: str | None,
server_url: str | None,
**kwargs: Any,
) -> RuntimeHandle | AsyncVlmPredictor:
"""模型初始化被取消时等待后台加载结束,防止清理后又写入迟到的模型句柄。"""
from mineru_vl_utils.vlm_client.utils import run_in_thread_until_complete
return await run_in_thread_until_complete(
ModelSingleton().get_model,
backend,
model_path,
server_url,
**kwargs,
)
def shutdown_cached_models() -> None:
"""进程结束时关闭包含独立解析固定租约在内的全部 VLM 运行时。"""
ModelSingleton().shutdown()
__all__ = ["ModelSingleton", "shutdown_cached_models"]