Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

22.3 KB
Raw Blame
# Copyright (c) Opendatalab. All rights reserved.
"""Hybrid 与本地小模型共享的运行时初始化模块。"""

from __future__ import annotations

import os
import threading
from collections.abc import Callable
from functools import cached_property
from typing import TYPE_CHECKING, Any

from loguru import logger

from ...utils.timing import stage_timer

from ..ocr.language import normalize_ocr_model_lang
from ..ocr.resources import PPOCRV6_DICT_PATH
from ..registry import MINERU_4_MODELS_ONNX, small_model_repo
from .contracts import AtomicModelName
from .local_cache import (
    LOCAL_MODEL_CACHE_LOCK,
    LocalModelResource,
    current_local_resource,
    get_local_model,
    local_resource_scope,
    release_local_owner,
)
from .ownership import RuntimeOwner

if TYPE_CHECKING:
    from ..table.cls.mineru_table_ori_cls import MineruTableOrientationClsModel
    from ..table.cls.paddle_table_cls import PaddleTableClsModel
    from ..table.rec.slanet_plus.main import PaddleTableModel
    from ..table.rec.unet_table.main import UnetTableModel
    from ..layout.pp_doclayoutv2 import PPDocLayoutV2LayoutModel
    from ..mfr.pp_formulanet.predict_formula import FormulaRecognizer
    from ..ocr.pytorch_paddle import PytorchPaddleOCR

from .device_state import model_device_scope, resolve_model_backend, resolve_model_device, resolve_runtime_device

LOCAL_MODEL_INIT_LOCK = LOCAL_MODEL_CACHE_LOCK
# 这些锁保护 Hybrid medium/high/xhigh 共享的 atom model/native 模型推理调用,避免多线程同时进入同一个模型对象。
LOCAL_MODEL_LAYOUT_INFERENCE_LOCK = threading.RLock()
LOCAL_MODEL_MFR_INFERENCE_LOCK = threading.RLock()
LOCAL_MODEL_OCR_INFERENCE_LOCK = threading.RLock()


def _read_bool_env(primary_name: str, fallback_name: str | None = None, default: bool = False) -> bool:
    """读取布尔环境变量;新变量未配置时回退到旧变量,保持已有部署兼容。"""
    raw_value = os.getenv(primary_name)
    if raw_value is None and fallback_name is not None:
        raw_value = os.getenv(fallback_name)
    if raw_value is None:
        return default
    return raw_value.lower() in ["true", "1", "yes"]


# 临时关闭共享推理阶段锁;旧 PIPELINE 变量仅作兼容回退,新的 Hybrid 本地变量优先生效。
LOCAL_MODEL_INFERENCE_LOCKS_ENABLED = _read_bool_env(
    "MINERU_ENABLE_LOCAL_MODEL_INFERENCE_LOCKS",
    fallback_name="MINERU_ENABLE_PIPELINE_INFERENCE_LOCKS",
    default=False,
)


def _run_with_inference_lock(
    inference_lock: threading.RLock, inference_callable: Callable[..., Any], *args: Any, **kwargs: Any
) -> object:
    """按实验开关决定是否在指定推理锁内执行真实 native 模型调用。"""
    if not LOCAL_MODEL_INFERENCE_LOCKS_ENABLED:
        return inference_callable(*args, **kwargs)

    with inference_lock:
        return inference_callable(*args, **kwargs)


def run_layout_inference(inference_callable: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
    """按实验开关执行共享 Layout 模型调用。"""
    return _run_with_inference_lock(LOCAL_MODEL_LAYOUT_INFERENCE_LOCK, inference_callable, *args, **kwargs)


def run_mfr_inference(inference_callable: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
    """按实验开关执行共享 MFR 模型调用。"""
    return _run_with_inference_lock(LOCAL_MODEL_MFR_INFERENCE_LOCK, inference_callable, *args, **kwargs)


def run_ocr_inference(inference_callable: Callable[..., Any], *args: Any, **kwargs: Any) -> Any:
    """按实验开关执行共享 OCR native 模型调用。"""
    return _run_with_inference_lock(LOCAL_MODEL_OCR_INFERENCE_LOCK, inference_callable, *args, **kwargs)


def table_orientation_cls_model_init(
    small_backend: str | None = None, device: str | None = None
) -> MineruTableOrientationClsModel:
    """初始化表格方向分类包装器,并注入适配方向检测的 OCR 引擎。"""
    from ..table.cls.mineru_table_ori_cls import MineruTableOrientationClsModel

    atom_model_manager = AtomModelSingleton()
    ocr_engine = atom_model_manager.get_atom_model(
        atom_model_name=AtomicModelName.OCR,
        small_backend=small_backend,
        device=device,
        det_db_box_thresh=0.5,
        det_db_unclip_ratio=1.6,
        lang="ch",
        enable_merge_det_boxes=False,
    )
    cls_model = MineruTableOrientationClsModel(ocr_engine)
    return cls_model


def table_cls_model_init(small_backend: str | None = None) -> PaddleTableClsModel:
    """初始化有线与无线表格类型分类模型。"""
    from ..table.cls.paddle_table_cls import PaddleTableClsModel

    return PaddleTableClsModel(
        model_path=str(small_model_repo(small_backend).paddle_table_cls.ensure()), small_backend=small_backend
    )


def wired_table_model_init(
    lang: str | None = None, *, small_backend: str | None = None, device: str | None = None
) -> UnetTableModel:
    """初始化有线表格识别模型,并注入指定语言的 OCR 引擎。"""
    from ..table.rec.unet_table.main import UnetTableModel

    atom_model_manager = AtomModelSingleton()
    ocr_engine = atom_model_manager.get_atom_model(
        atom_model_name=AtomicModelName.OCR,
        small_backend=small_backend,
        device=device,
        det_db_box_thresh=0.5,
        det_db_unclip_ratio=1.6,
        lang=lang,
        enable_merge_det_boxes=False,
    )
    table_model = UnetTableModel(
        ocr_engine, model_path=str(small_model_repo(small_backend).unet_structure.ensure()), small_backend=small_backend
    )
    return table_model


def wireless_table_model_init(
    lang: str | None = None, *, small_backend: str | None = None, device: str | None = None
) -> PaddleTableModel:
    """初始化无线表格识别模型,并注入指定语言的 OCR 引擎。"""
    from ..table.rec.slanet_plus.main import PaddleTableModel

    atom_model_manager = AtomModelSingleton()
    ocr_engine = atom_model_manager.get_atom_model(
        atom_model_name=AtomicModelName.OCR,
        small_backend=small_backend,
        device=device,
        det_db_box_thresh=0.5,
        det_db_unclip_ratio=1.6,
        lang=lang,
        enable_merge_det_boxes=False,
    )
    table_model = PaddleTableModel(
        ocr_engine, model_path=str(small_model_repo(small_backend).slanet_plus.ensure()), small_backend=small_backend
    )
    return table_model


def mfr_model_init(weight_dir: str, device: str = "cpu") -> "FormulaRecognizer":
    """初始化默认 Torch 公式模型 plus-M,精度由设备策略决定。"""
    from ..mfr.pp_formulanet.predict_formula import FormulaRecognizer

    return FormulaRecognizer(weight_dir, device, model_name="PP-FormulaNet_plus-M")


def pp_doclayout_v2_model_init(weight: str, device: str = "cpu") -> "PPDocLayoutV2LayoutModel":
    """在指定设备上初始化 PP-DocLayoutV2 版面分析模型。"""

    from ..layout.pp_doclayoutv2 import PPDocLayoutV2LayoutModel

    if str(device).startswith("npu"):
        import torch

        device = torch.device(device)
    model = PPDocLayoutV2LayoutModel(weight, device)
    return model


def ocr_model_init(
    det_db_box_thresh: float = 0.5,
    lang: str | None = None,
    det_db_unclip_ratio: float = 1.5,
    enable_merge_det_boxes: bool = True,
    device: str | None = None,
) -> "PytorchPaddleOCR":
    """按语言和检测阈值初始化本地 Paddle OCR 模型。"""

    from ..ocr.pytorch_paddle import PytorchPaddleOCR

    ocr_kwargs = {
        "device": device,
        "lang": normalize_ocr_model_lang(lang),
        "det_db_box_thresh": det_db_box_thresh,
        "det_db_unclip_ratio": det_db_unclip_ratio,
        "enable_merge_det_boxes": enable_merge_det_boxes,
    }
    return PytorchPaddleOCR(**ocr_kwargs)


class AtomModelSingleton:
    """按模型配置缓存并复用本地原子模型实例。"""

    _instance: AtomModelSingleton | None = None
    _models: dict[object, LocalModelResource] = {}
    _lock: threading.RLock = LOCAL_MODEL_INIT_LOCK

    def __new__(cls, *args: Any, **kwargs: Any) -> AtomModelSingleton:
        """在线程锁保护下创建或返回唯一的原子模型管理器。"""
        with cls._lock:
            if cls._instance is None:
                cls._instance = super().__new__(cls)
        return cls._instance

    def get_atom_model(self, atom_model_name: str, **kwargs: Any) -> Any:
        """根据模型名称和关键配置生成缓存键,并获取对应原子模型。"""
        decision = resolve_model_device(kwargs.get("small_backend"), kwargs.get("device"))
        small_backend = decision.backend
        if small_backend == "torch":
            from .cpu_threads import initialize_torch_threads

            initialize_torch_threads()
        device = decision.model_device
        kwargs = {**kwargs, "small_backend": small_backend, "device": device}
        lang = kwargs.get("lang", None)
        ocr_singleton_lang = normalize_ocr_model_lang(lang)

        if atom_model_name in [AtomicModelName.WiredTable, AtomicModelName.WirelessTable]:
            key = (atom_model_name, ocr_singleton_lang)
        elif atom_model_name in [AtomicModelName.OCR]:
            key = (
                atom_model_name,
                kwargs.get("det_db_box_thresh", 0.5),
                ocr_singleton_lang,
                kwargs.get("det_db_unclip_ratio", 1.5),
                kwargs.get("enable_merge_det_boxes", True),
            )
        elif atom_model_name in [AtomicModelName.Layout, AtomicModelName.MFR]:
            key = (
                atom_model_name,
                kwargs.get("device"),
            )
        else:
            key = atom_model_name

        key = (decision, key)

        def initialize() -> Any:
            """在已确定的设备作用域中创建模型,使嵌套 ONNX 会话复用相同选型。"""
            with model_device_scope(decision):
                return atom_model_init(model_name=atom_model_name, **kwargs)

        return get_local_model(self._models, key, initialize)

    def release_owner(self, owner: RuntimeOwner) -> None:
        """释放应用直接持有的原子模型,仍被其它上下文引用的依赖继续保留。"""
        release_local_owner(self._models, owner)


@stage_timer("model.initialize")
def atom_model_init(model_name: str, **kwargs: Any) -> Any:
    """将原子模型名称分派到具体初始化函数,并校验初始化结果。"""
    decision = resolve_model_device(kwargs.get("small_backend"), kwargs.get("device"))
    with model_device_scope(decision):
        return _create_atom_model(model_name, **{**kwargs, "small_backend": decision.backend, "device": decision.model_device})


def _create_atom_model(model_name: str, **kwargs: Any) -> Any:
    """在调用者已固定的设备作用域内执行具体模型构造。"""
    small_backend = kwargs["small_backend"]
    if small_backend == "torch":
        from .cpu_threads import initialize_torch_threads

        initialize_torch_threads()
    atom_model = None
    if model_name == AtomicModelName.Layout:
        if small_backend == "onnx":
            from ..layout.pp_doclayout_v2_onnx import PPDocLayoutV2LayoutModelONNX

            atom_model = PPDocLayoutV2LayoutModelONNX(
                weight=str(MINERU_4_MODELS_ONNX.pp_doclayout_v2.ensure()),
                config_path=str(MINERU_4_MODELS_ONNX.pp_doclayout_v2_config.ensure()),
                device=kwargs.get("device"),
            )
        else:
            from ..registry import MINERU_4_MODELS_TORCH

            atom_model = pp_doclayout_v2_model_init(
                str(MINERU_4_MODELS_TORCH.pp_doclayout_v2.ensure()),
                kwargs.get("device"),
            )
    elif model_name == AtomicModelName.MFR:
        if small_backend == "onnx":
            from ..mfr.pp_formulanet_plus_m_onnx import PPFormulaNetPlusMONNX

            atom_model = PPFormulaNetPlusMONNX(
                model_path=str(MINERU_4_MODELS_ONNX.pp_formulanet_plus_m_weights.ensure()),
                config_path=str(MINERU_4_MODELS_ONNX.pp_formulanet_plus_m_config.ensure()),
                device=kwargs.get("device"),
            )
        else:
            from ..registry import MINERU_4_MODELS_TORCH

            # 按文件检查手工放置的资源,不要求模型目录存在下载完成标记。
            weights_path = MINERU_4_MODELS_TORCH.pp_formulanet_plus_m_weights.ensure()
            MINERU_4_MODELS_TORCH.pp_formulanet_plus_m_config.ensure()
            atom_model = mfr_model_init(
                str(weights_path.parent),
                kwargs.get("device"),
            )
    elif model_name == AtomicModelName.OCR:
        if small_backend == "onnx":
            from ..ocr.pp_ocr_v6_onnx import PPOCRv6ONNX

            lang = normalize_ocr_model_lang(kwargs.get("lang"))
            detector = MINERU_4_MODELS_ONNX.seal_det if lang == "seal" else MINERU_4_MODELS_ONNX.ocr_det

            atom_model = PPOCRv6ONNX(
                det_model_path=str(detector.ensure()),
                rec_model_path=str(MINERU_4_MODELS_ONNX.ocr_rec.ensure()),
                dict_path=str(PPOCRV6_DICT_PATH),
                lang=lang,
                device=kwargs.get("device"),
                det_db_box_thresh=kwargs.get("det_db_box_thresh", 0.5),
                det_db_unclip_ratio=kwargs.get("det_db_unclip_ratio", 1.5),
                enable_merge_det_boxes=kwargs.get("enable_merge_det_boxes", True),
            )
        else:
            atom_model = ocr_model_init(
                kwargs.get("det_db_box_thresh", 0.5),
                kwargs.get("lang"),
                kwargs.get("det_db_unclip_ratio", 1.5),
                kwargs.get("enable_merge_det_boxes", True),
                device=kwargs.get("device"),
            )
    elif model_name == AtomicModelName.WirelessTable:
        atom_model = wireless_table_model_init(
            kwargs.get("lang"),
            small_backend=small_backend,
            device=kwargs.get("device"),
        )
    elif model_name == AtomicModelName.WiredTable:
        atom_model = wired_table_model_init(
            kwargs.get("lang"),
            small_backend=small_backend,
            device=kwargs.get("device"),
        )
    elif model_name == AtomicModelName.TableCls:
        atom_model = table_cls_model_init(small_backend=small_backend)
    elif model_name == AtomicModelName.TableOrientationCls:
        atom_model = table_orientation_cls_model_init(small_backend=small_backend, device=kwargs.get("device"))
    else:
        logger.error("model name not allow")
        exit(1)

    if atom_model is None:
        logger.error("model init failed")
        exit(1)
    else:
        return atom_model


class HybridLocalModelContextSingleton:
    """全局缓存并复用 Hybrid 本地模型上下文。"""

    _instance: HybridLocalModelContextSingleton | None = None
    _models: dict[object, LocalModelResource] = {}
    _lock: threading.RLock = LOCAL_MODEL_INIT_LOCK

    def __new__(cls, *args: Any, **kwargs: Any) -> HybridLocalModelContextSingleton:
        """在线程锁保护下创建或返回唯一的上下文管理器。"""
        with cls._lock:
            if cls._instance is None:
                cls._instance = super().__new__(cls)
        return cls._instance

    def get_model(
        self,
    ) -> HybridLocalModelContext:
        """按进程稳定设备复用上下文,配置写回不创建另一份模型。"""
        decision = resolve_runtime_device()

        def initialize() -> HybridLocalModelContext:
            """在同一设备作用域内创建上下文及其嵌套原子模型。"""
            with model_device_scope(decision):
                return HybridLocalModelContext(device=decision.model_device, small_backend=decision.backend)

        return get_local_model(self._models, decision, initialize)

    def release_owner(self, owner: RuntimeOwner) -> None:
        """释放应用对上下文的租约,并按依赖关系回收无人持有的原子模型。"""
        release_local_owner(self._models, owner)


def ocr_det_batch_setting() -> bool:
    """根据运行设备和 PyTorch 版本确定是否启用 OCR 检测批处理。"""
    if resolve_model_backend() == "onnx":
        return True

    try:
        import torch as _torch
        from packaging import version
    except ImportError:
        return True

    device_type = os.getenv("MINERU_LMDEPLOY_DEVICE", "")
    if device_type.lower() in ["corex"]:
        enable_ocr_det_batch = False
    else:
        if version.parse(_torch.__version__) >= version.parse("2.8.0"):
            os.environ["TORCH_CUDNN_V8_API_DISABLED"] = "1"
        enable_ocr_det_batch = True

    return enable_ocr_det_batch


class HybridLocalModelContext:
    """集中管理 Hybrid 分析所需本地模型及其惰性加载生命周期。"""

    def __init__(
        self,
        device: str | None = None,
        *,
        small_backend: str | None = None,
    ) -> None:
        """初始化 Hybrid 基础运行时,其他模型在首次访问对应属性时加载。"""
        self._resource = current_local_resource()
        self._device_decision = resolve_model_device(small_backend, device)
        self.small_backend = self._device_decision.backend
        self.device = self._device_decision.model_device
        with model_device_scope(self._device_decision):
            self.enable_ocr_det_batch = True if self.small_backend == "onnx" else ocr_det_batch_setting()

        if str(self.device).startswith("npu"):
            try:
                import torch_npu

                if torch_npu.npu.is_available():
                    torch_npu.npu.set_compile_mode(jit_compile=False)
            except Exception as e:
                raise RuntimeError(
                    "NPU is selected as device, but torch_npu is not available. "
                    "Please ensure that the torch_npu package is installed correctly."
                ) from e

        self.atom_model_manager = AtomModelSingleton()

        # 初始化OCR模型
        self.ocr_model = self.get_ocr_model()

        # 初始化layout模型,用于提供行内公式检测框和Hybrid标题拆分
        self.layout_model = self.get_layout_model()

    def _get_atom_model(self, atom_model_name: str, **kwargs: Any) -> Any:
        """把惰性模型登记为上下文依赖,复用 cached_property 时无需重复登记应用。"""
        with local_resource_scope(self._resource), model_device_scope(self._device_decision):
            return self.atom_model_manager.get_atom_model(atom_model_name=atom_model_name, **kwargs)

    @cached_property
    def mfr_model(self) -> FormulaRecognizer:
        """首次访问时加载公式识别模型,并在当前 Context 内复用。"""
        return self.get_mfr_model()

    @cached_property
    def wireless_table_model(self) -> PaddleTableModel:
        """首次访问时加载无线表格识别模型,并在当前 Context 内复用。"""
        return self.get_wireless_table_model()

    @cached_property
    def wired_table_model(self) -> UnetTableModel:
        """首次访问时加载有线表格识别模型,并在当前 Context 内复用。"""
        return self.get_wired_table_model()

    @cached_property
    def table_cls_model(self) -> PaddleTableClsModel:
        """首次访问时加载表格类型分类模型,并在当前 Context 内复用。"""
        return self.get_table_cls_model()

    @cached_property
    def table_orientation_cls_model(self) -> MineruTableOrientationClsModel:
        """首次访问时加载基于 OCR 的表格方向包装器,并在当前 Context 内复用。"""
        return self.get_table_orientation_cls_model()

    @cached_property
    def seal_model(self) -> PytorchPaddleOCR:
        """首次访问时加载印章 OCR 模型,并在当前 Context 内复用。"""
        return self.get_seal_ocr_model()

    def get_ocr_model(
        self,
        det_db_box_thresh: float = 0.5,
        det_db_unclip_ratio: float = 1.5,
        enable_merge_det_boxes: bool = True,
    ) -> "PytorchPaddleOCR":
        """获取 OCR 原子模型,默认使用当前 Hybrid 本地上下文语言并复用 singleton 缓存。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.OCR,
            det_db_box_thresh=det_db_box_thresh,
            lang="ch",
            det_db_unclip_ratio=det_db_unclip_ratio,
            enable_merge_det_boxes=enable_merge_det_boxes,
        )

    def get_layout_model(self) -> "PPDocLayoutV2LayoutModel":
        """获取 Layout 原子模型,供 Hybrid 本地 layout、标题拆分和公式框检测复用。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.Layout,
        )

    def get_mfr_model(self) -> "FormulaRecognizer":
        """获取公式识别原子模型,统一复用当前公式模型配置和设备。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.MFR,
        )

    def get_wireless_table_model(self) -> PaddleTableModel:
        """获取无线表格识别原子模型。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.WirelessTable,
            lang="ch",
        )

    def get_wired_table_model(self) -> UnetTableModel:
        """获取有线表格识别原子模型。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.WiredTable,
            lang="ch",
        )

    def get_table_cls_model(self) -> PaddleTableClsModel:
        """获取表格分类原子模型。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.TableCls,
        )

    def get_table_orientation_cls_model(self) -> MineruTableOrientationClsModel:
        """获取表格方向分类原子模型。"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.TableOrientationCls,
        )

    def get_seal_ocr_model(self) -> PytorchPaddleOCR:
        """获取印章识别 OCR 原子模型"""
        return self._get_atom_model(
            small_backend=self.small_backend,
            device=self.device,
            atom_model_name=AtomicModelName.OCR,
            lang="seal",
        )