Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

feat(onnx): route UNet through shared device selection

myhloliee3ba4dc4005
diff --git a/docs/next/onnx-device.md b/docs/next/onnx-device.md
index d609b0ba..7c228cd8 100644
--- a/docs/next/onnx-device.md
+++ b/docs/next/onnx-device.md
@@ -33,15 +33,16 @@ Windows 使用 DXGI 高性能顺序、D3D12 UMA 属性和 LUID 匹配;适配
## 模型预设
-GPU 路线只将 layout、rec、MFR 交给 DML,检测、表格等其他 ONNX 模型保持 CPU。
+GPU 路线将 layout、rec、MFR 和 UNet 有线表格结构模型交给 DML,检测、表格分类和无线表格等其他 ONNX 模型保持 CPU。
+UNet 与其他模型共享 `model.onnx_device` 的进程内选择,不单独测速;ONNX 栈的 UNet 不受旧 `MINERU_TABLE_DEVICE` 覆盖。
Torch 栈中的表格仍使用其原有 CPU/CUDA 策略。旧实验引擎和实验开关已经移除。
-| 厂商 | layout | MFR | rec |
-|---|---|---|---|
-| Intel | 关 | 开 | 关 |
-| AMD | 开 | 开 | 开 |
-| NVIDIA | 开 | 开 | 关 |
-| 其他 | ORT 默认 | ORT 默认 | ORT 默认 |
+| 厂商 | layout | MFR | rec | UNet |
+|---|---|---|---|---|
+| Intel | 关 | 开 | 关 | 关 |
+| AMD | 开 | 开 | 开 | 开 |
+| NVIDIA | 开 | 开 | 关 | 关 |
+| 其他 | ORT 默认 | ORT 默认 | ORT 默认 | ORT 默认 |
表格中的开关指 metacommands,关闭对应 `disable_metacommands=true`。
Intel layout 优先首轮响应;NVIDIA layout 采用后续补测结论。
diff --git a/mineru/model/runtime/onnx.py b/mineru/model/runtime/onnx.py
index e0547c0f..b2e89188 100644
--- a/mineru/model/runtime/onnx.py
+++ b/mineru/model/runtime/onnx.py
@@ -83,9 +83,10 @@ def create_ort_session(
intra_op_num_threads: int = 0,
inter_op_num_threads: int = 0,
profile_prefix: str | None = None,
+ sess_options: ort.SessionOptions | None = None,
) -> ort.InferenceSession | DirectMLSession:
"""Create an explicit CPU/DML session without selection, downloads or config writes."""
- opts = ort.SessionOptions()
+ opts = sess_options if sess_options is not None else ort.SessionOptions()
opts.log_severity_level = 3
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
configure_ort_threads(opts, intra_op_num_threads=intra_op_num_threads, inter_op_num_threads=inter_op_num_threads)
@@ -119,21 +120,30 @@ def ort_session(
*,
model_kind: str = "",
selection: OnnxSelection | None = None,
+ sess_options: ort.SessionOptions | None = None,
) -> ort.InferenceSession | DirectMLSession:
- """Use the process ONNX choice only for layout, rec and MFR; other models stay on CPU."""
- eligible = model_kind in {"layout", "ocr_rec", "mfr"}
- route = (selection or resolve_onnx_device(intra_op_num_threads=intra_op_num_threads)) if eligible else OnnxSelection()
+ """Use the process ONNX choice for layout, rec, MFR and UNet; other models stay on CPU."""
+ eligible = model_kind in {"layout", "ocr_rec", "mfr", "unet"}
+ selection_threads = intra_op_num_threads
+ if selection_threads <= 0 and sess_options is not None:
+ selection_threads = sess_options.intra_op_num_threads
+ route = (selection or resolve_onnx_device(intra_op_num_threads=selection_threads)) if eligible else OnnxSelection()
adapter = route.adapter if route.device == "gpu" else None
started = time.perf_counter()
try:
session = create_ort_session(
- model_path, adapter=adapter, model_kind=model_kind, intra_op_num_threads=intra_op_num_threads
+ model_path,
+ adapter=adapter,
+ model_kind=model_kind,
+ intra_op_num_threads=intra_op_num_threads,
+ sess_options=sess_options,
)
except Exception as error:
if adapter is None:
raise
logger.warning("ONNX {} DML initialization failed; using CPU: {}", model_kind, error)
- session = create_ort_session(model_path, intra_op_num_threads=intra_op_num_threads)
+ session = create_ort_session(model_path, intra_op_num_threads=intra_op_num_threads, sess_options=sess_options)
+ adapter = None
else:
if adapter is not None:
confirm_gpu_session(route)
@@ -149,15 +159,23 @@ def ort_session(
def table_ort_session(
- model_path: str, *, sess_options: ort.SessionOptions | None = None, small_backend: str | None = None
-) -> ort.InferenceSession:
- """表格默认按可用 provider 选择设备;支持固定 CPU/CUDA,并明确记录 CUDA 失败回退。"""
- device = os.getenv("MINERU_TABLE_DEVICE", "auto").strip().lower()
- if device not in {"auto", "cpu", "cuda"}:
- raise ValueError("MINERU_TABLE_DEVICE must be 'auto', 'cpu' or 'cuda'")
+ model_path: str,
+ *,
+ sess_options: ort.SessionOptions | None = None,
+ small_backend: str | None = None,
+ model_kind: str = "",
+) -> ort.InferenceSession | DirectMLSession:
+ """ONNX 栈的 UNet 复用统一设备选择;其他表格保留原有 CPU/CUDA 策略。"""
from .device import resolve_small_model_backend
- if resolve_small_model_backend(small_backend) == "onnx":
+ onnx_backend = resolve_small_model_backend(small_backend) == "onnx"
+ unified_unet = onnx_backend and model_kind == "unet"
+ device = "cpu"
+ if not unified_unet:
+ device = os.getenv("MINERU_TABLE_DEVICE", "auto").strip().lower()
+ if device not in {"auto", "cpu", "cuda"}:
+ raise ValueError("MINERU_TABLE_DEVICE must be 'auto', 'cpu' or 'cuda'")
+ if onnx_backend:
device = "cpu"
opts = sess_options if sess_options is not None else ort.SessionOptions()
opts.graph_optimization_level = ort.GraphOptimizationLevel.ORT_ENABLE_ALL
@@ -168,6 +186,8 @@ def table_ort_session(
Path(profile_dir).mkdir(parents=True, exist_ok=True)
opts.enable_profiling = True
opts.profile_file_prefix = str(Path(profile_dir) / f"{Path(model_path).stem}-{os.getpid()}")
+ if unified_unet:
+ return ort_session(model_path, model_kind="unet", sess_options=opts)
providers = ort_providers()
use_cuda = False
if device != "cpu":
diff --git a/mineru/model/runtime/onnx_device.py b/mineru/model/runtime/onnx_device.py
index 82db4ecd..7771cfee 100644
--- a/mineru/model/runtime/onnx_device.py
+++ b/mineru/model/runtime/onnx_device.py
@@ -18,9 +18,9 @@ from .windows_gpu import GpuAdapter, discover_gpus, preferred_gpu
# Evidence: docs/next/onnx-selection-validation.md. None disables unvalidated auto-iGPU choices.
ONNX_GPU_TIME_RATIO_THRESHOLD: float | None = 0.8
METACOMMANDS: dict[str, dict[str, bool]] = {
- "intel": {"layout": False, "mfr": True, "ocr_rec": False},
- "amd": {"layout": True, "mfr": True, "ocr_rec": True},
- "nvidia": {"layout": True, "mfr": True, "ocr_rec": False},
+ "intel": {"layout": False, "mfr": True, "ocr_rec": False, "unet": False},
+ "amd": {"layout": True, "mfr": True, "ocr_rec": True, "unet": True},
+ "nvidia": {"layout": True, "mfr": True, "ocr_rec": False, "unet": False},
}
diff --git a/mineru/model/table/rec/unet_table/utils.py b/mineru/model/table/rec/unet_table/utils.py
index ec123134..d6d6f088 100644
--- a/mineru/model/table/rec/unet_table/utils.py
+++ b/mineru/model/table/rec/unet_table/utils.py
@@ -27,6 +27,7 @@ class OrtInferSession:
model_path,
sess_options=self._init_sess_opts(config),
small_backend=config.get("small_backend"),
+ model_kind="unet",
)
@staticmethod
diff --git a/tests/unittest/test_directml_runtime.py b/tests/unittest/test_directml_runtime.py
index 4c07ac8c..294a1f27 100644
--- a/tests/unittest/test_directml_runtime.py
+++ b/tests/unittest/test_directml_runtime.py
@@ -18,7 +18,12 @@ from mineru.model.runtime.windows_gpu import GpuAdapter
@pytest.mark.parametrize(
("vendor", "enabled"),
- [(0x8086, [False, True, False]), (0x1002, [True, True, True]), (0x10DE, [True, True, False]), (0xFFFF, [None, None, None])],
+ [
+ (0x8086, [False, True, False, False]),
+ (0x1002, [True, True, True, True]),
+ (0x10DE, [True, True, False, False]),
+ (0xFFFF, [None, None, None, None]),
+ ],
)
def test_vendor_presets_and_actual_index(monkeypatch: pytest.MonkeyPatch, vendor: int, enabled: list[bool | None]) -> None:
adapter = GpuAdapter(7, "luid", "gpu", vendor, 1, True, 0)
@@ -26,7 +31,7 @@ def test_vendor_presets_and_actual_index(monkeypatch: pytest.MonkeyPatch, vendor
native = SimpleNamespace(get_providers=lambda: ["DmlExecutionProvider", "CPUExecutionProvider"], disable_fallback=Mock())
factory = Mock(return_value=native)
monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
- for kind, preset in zip(("layout", "mfr", "ocr_rec"), enabled):
+ for kind, preset in zip(("layout", "mfr", "ocr_rec", "unet"), enabled):
session = onnx.create_ort_session("test.onnx", adapter=adapter, model_kind=kind)
assert isinstance(session, onnx.DirectMLSession)
kwargs = factory.call_args.kwargs
@@ -35,7 +40,7 @@ def test_vendor_presets_and_actual_index(monkeypatch: pytest.MonkeyPatch, vendor
assert options.get("disable_metacommands") == (None if preset is None else "false" if preset else "true")
assert not kwargs["sess_options"].enable_mem_pattern
assert kwargs["sess_options"].execution_mode == onnx.ort.ExecutionMode.ORT_SEQUENTIAL
- assert native.disable_fallback.call_count == 3
+ assert native.disable_fallback.call_count == 4
def test_cpu_models_do_not_select_gpu(monkeypatch: pytest.MonkeyPatch) -> None:
@@ -50,7 +55,8 @@ def test_cpu_models_do_not_select_gpu(monkeypatch: pytest.MonkeyPatch) -> None:
@pytest.mark.parametrize("failure", ["unavailable", "exception", "silent"])
-def test_production_fallback_does_not_save_gpu(monkeypatch: pytest.MonkeyPatch, failure: str) -> None:
[email protected]("kind", ["layout", "unet"])
+def test_production_fallback_does_not_save_gpu(monkeypatch: pytest.MonkeyPatch, failure: str, kind: str) -> None:
adapter = GpuAdapter(1, "id", "gpu", 0x8086, 1, False, 0)
cpu = SimpleNamespace(get_providers=lambda: ["CPUExecutionProvider"])
monkeypatch.setattr(
@@ -60,9 +66,25 @@ def test_production_fallback_does_not_save_gpu(monkeypatch: pytest.MonkeyPatch,
monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
save = Mock()
monkeypatch.setattr(onnx, "confirm_gpu_session", save)
- assert onnx.ort_session("test.onnx", model_kind="layout", selection=OnnxSelection("gpu", adapter, "discrete_gpu")) is cpu
+ opts = onnx.ort.SessionOptions()
+ opts.intra_op_num_threads = 2
+ opts.inter_op_num_threads = 1
+ opts.enable_cpu_mem_arena = False
+ opts.enable_profiling = True
+ opts.profile_file_prefix = "unet-profile"
+ assert (
+ onnx.ort_session(
+ "test.onnx", model_kind=kind, selection=OnnxSelection("gpu", adapter, "discrete_gpu"), sess_options=opts
+ )
+ is cpu
+ )
save.assert_not_called()
assert factory.call_args.kwargs["providers"] == onnx.ort_providers()
+ for call in factory.call_args_list:
+ assert call.kwargs["sess_options"] is opts
+ assert (opts.intra_op_num_threads, opts.inter_op_num_threads) == (2, 1)
+ assert not opts.enable_cpu_mem_arena
+ assert opts.enable_profiling and opts.profile_file_prefix == "unet-profile"
def test_calibration_factory_fails_instead_of_falling_back(monkeypatch: pytest.MonkeyPatch) -> None:
diff --git a/tests/unittest/test_onnx_device_selection.py b/tests/unittest/test_onnx_device_selection.py
index cc30971b..c72d3fc4 100644
--- a/tests/unittest/test_onnx_device_selection.py
+++ b/tests/unittest/test_onnx_device_selection.py
@@ -224,3 +224,52 @@ def test_manual_reset_to_auto_recalibrates(config_file: Path) -> None:
policy._selections.clear() # New process after the explicit reset.
policy.resolve_onnx_device()
assert onnx_calibration.run_calibration.call_count == 2
+
+
+ ("requested", "uma", "gpu_time", "expected"),
+ [
+ ("cpu", True, 1.0, "cpu"),
+ ("gpu", True, 1.0, "gpu"),
+ ("auto", True, 1.0, "gpu"),
+ ("auto", True, 1.6, "cpu"),
+ ("auto", False, 1.0, "gpu"),
+ ],
+)
+def test_unet_first_model_shares_selection_and_persistence(
+ config_file: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ requested: str,
+ uma: bool,
+ gpu_time: float,
+ expected: str,
+) -> None:
+ from mineru.model.runtime import onnx, onnx_calibration
+ from mineru.model.table.rec.unet_table.table_structure_unet import TSRUnet
+
+ monkeypatch.setattr(settings.config.model, "onnx_device", requested)
+ config_file.write_text(f"model:\n onnx_device: {requested}\n source: local\n", encoding="utf-8")
+ monkeypatch.setenv("MINERU_TABLE_DEVICE", "cpu" if expected == "gpu" else "cuda")
+ discover = Mock(return_value=(adapter(index=7, uma=uma),))
+ monkeypatch.setattr(policy, "discover_gpus", discover)
+ measure = Mock(return_value={"cpu_elapsed": 2.0, "gpu_elapsed": gpu_time})
+ monkeypatch.setattr(onnx_calibration, "run_calibration", measure)
+ factory = Mock(
+ side_effect=lambda *args, **kwargs: SimpleNamespace(
+ get_providers=lambda: [name for name, options in kwargs["providers"]], disable_fallback=Mock()
+ )
+ )
+ monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
+ model = TSRUnet({"model_path": "unet.onnx", "small_backend": "onnx", "intra_op_num_threads": 2})
+ provider = "DmlExecutionProvider" if expected == "gpu" else "CPUExecutionProvider"
+ assert model.session.session.get_providers()[0] == provider
+ onnx.ort_session("layout.onnx", model_kind="layout")
+ onnx.ort_session("rec.onnx", model_kind="ocr_rec")
+ assert all(call.kwargs["providers"][0][0] == provider for call in factory.call_args_list)
+ assert discover.call_count == (0 if requested == "cpu" else 1)
+ assert measure.call_count == (1 if requested == "auto" and uma else 0)
+ if measure.called:
+ assert measure.call_args.kwargs["intra_op_num_threads"] == 2
+ if expected == "gpu":
+ assert factory.call_args_list[0].kwargs["providers"][0][1] == {"device_id": 7, "disable_metacommands": "true"}
+ assert yaml.safe_load(config_file.read_text())["model"] == {"onnx_device": expected, "source": "local"}
diff --git a/tests/unittest/test_table_onnx_runtime.py b/tests/unittest/test_table_onnx_runtime.py
index 46b2cac9..6e238a37 100644
--- a/tests/unittest/test_table_onnx_runtime.py
+++ b/tests/unittest/test_table_onnx_runtime.py
@@ -102,7 +102,7 @@ def test_cuda_selection_and_other_onnx_models_stay_cpu(monkeypatch: pytest.Monke
monkeypatch.setattr(onnx.ort, "get_available_providers", lambda: ["CUDAExecutionProvider", "CPUExecutionProvider"])
factory = Mock(return_value=SimpleNamespace(get_providers=lambda: ["CUDAExecutionProvider", "CPUExecutionProvider"]))
monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
- onnx.table_ort_session("model.onnx")
+ onnx.table_ort_session("model.onnx", small_backend="torch")
call = factory.call_args.kwargs
assert call["providers"][0][0] == "CUDAExecutionProvider"
assert call["providers"][0][1]["cudnn_conv_algo_search"] == "HEURISTIC"
@@ -148,7 +148,7 @@ def test_cuda_failure_is_reported_and_cpu_remains_available(monkeypatch: pytest.
warning = Mock()
monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
monkeypatch.setattr(onnx.logger, "warning", warning)
- assert onnx.table_ort_session("table.onnx") is cpu
+ assert onnx.table_ort_session("table.onnx", small_backend="torch") is cpu
assert warning.called == (mode != "unavailable" or device == "cuda")
for call in factory.call_args_list:
opts = call.kwargs["sess_options"]
@@ -183,6 +183,7 @@ def test_all_table_wrappers_use_shared_factory(monkeypatch: pytest.MonkeyPatch,
utils.OrtInferSession({"model_path": str(path)})
assert factory.call_count == 3
assert all(call.args == (str(path),) for call in factory.call_args_list)
+ assert [call.kwargs.get("model_kind") for call in factory.call_args_list] == [None, None, "unet"]
@pytest.mark.parametrize("explicit", [False, True])
@@ -198,7 +199,7 @@ def test_table_structure_entrypoints_use_shared_threads(
monkeypatch.setenv("MINERU_INTER_OP_NUM_THREADS", "3")
model = tmp_path / "table.onnx"
model.touch()
- config = {"model_path": str(model)}
+ config = {"model_path": str(model), "small_backend": "onnx"}
if explicit:
config.update(intra_op_num_threads=2, inter_op_num_threads=1)
original = config.copy()
@@ -216,3 +217,45 @@ def test_table_structure_entrypoints_use_shared_threads(
opts = call.kwargs["sess_options"]
assert (opts.intra_op_num_threads, opts.inter_op_num_threads) == ((2, 1) if explicit else (8, 3))
assert opts.enable_cpu_mem_arena is False
+
+
+def test_onnx_unet_preserves_options_profile_and_dml_lock(monkeypatch: pytest.MonkeyPatch, tmp_path: Path) -> None:
+ from mineru.model.runtime.onnx_device import OnnxSelection
+ from mineru.model.runtime.windows_gpu import GpuAdapter
+ from mineru.model.table.rec.unet_table.table_structure_unet import TSRUnet
+
+ monkeypatch.setenv("MINERU_TABLE_DEVICE", "cpu")
+ monkeypatch.setenv("MINERU_ONNX_PROFILE_DIR", str(tmp_path / "profiles"))
+ monkeypatch.setenv("MINERU_INTRA_OP_NUM_THREADS", "8")
+ select = Mock(return_value=OnnxSelection("gpu", GpuAdapter(7, "id", "intel", 0x8086, 1, True, 0)))
+ monkeypatch.setattr(onnx, "resolve_onnx_device", select)
+ monkeypatch.setattr(onnx.ort, "get_available_providers", lambda: ["DmlExecutionProvider", "CPUExecutionProvider"])
+ native = SimpleNamespace(get_providers=lambda: ["DmlExecutionProvider", "CPUExecutionProvider"], disable_fallback=Mock())
+ factory = Mock(return_value=native)
+ monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
+ model = TSRUnet({"model_path": "unet.onnx", "small_backend": "onnx", "intra_op_num_threads": 2})
+ assert isinstance(model.session.session, onnx.DirectMLSession)
+ select.assert_called_once_with(intra_op_num_threads=2)
+ opts = factory.call_args.kwargs["sess_options"]
+ assert (opts.intra_op_num_threads, opts.inter_op_num_threads) == (2, 1)
+ assert not opts.enable_cpu_mem_arena and not opts.enable_mem_pattern
+ assert opts.execution_mode == onnx.ort.ExecutionMode.ORT_SEQUENTIAL
+ assert opts.enable_profiling and Path(opts.profile_file_prefix).parent == tmp_path / "profiles"
+ assert factory.call_args.kwargs["providers"][0] == (
+ "DmlExecutionProvider",
+ {"device_id": 7, "disable_metacommands": "true"},
+ )
+ native.disable_fallback.assert_called_once()
+
+
[email protected]("legacy_device", ["cpu", "auto", "cuda"])
+def test_torch_unet_retains_legacy_table_device(monkeypatch: pytest.MonkeyPatch, legacy_device: str) -> None:
+ monkeypatch.setenv("MINERU_TABLE_DEVICE", legacy_device)
+ monkeypatch.setattr(onnx, "resolve_onnx_device", Mock(side_effect=AssertionError("ONNX selection")))
+ monkeypatch.setattr(onnx.ort, "get_available_providers", lambda: ["CUDAExecutionProvider", "CPUExecutionProvider"])
+ factory = Mock(return_value=SimpleNamespace(get_providers=lambda: ["CPUExecutionProvider"]))
+ monkeypatch.setattr(onnx.ort, "InferenceSession", factory)
+ onnx.table_ort_session("unet.onnx", small_backend="torch", model_kind="unet")
+ assert factory.call_args.kwargs["providers"][0][0] == (
+ "CPUExecutionProvider" if legacy_device == "cpu" else "CUDAExecutionProvider"
+ )