Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

feat(pdf): recognize inline formulas with VLM text extraction

myhloli115693124d27
diff --git a/docs/next/tiers.md b/docs/next/tiers.md
index 7a9f450e..43dbfefb 100644
--- a/docs/next/tiers.md
+++ b/docs/next/tiers.md
@@ -13,6 +13,14 @@
基础包已经提供 ONNX 与 llama.cpp;Basic 不要求额外安装 Torch。
非 PDF 原生格式使用 DocVortex,与 PDF 推理档位不是同一种实现;不承诺 Flash 对原生格式的质量总是最低。
+Standard / Advanced 在 PDF 文本模式(`txt`,包括 `auto` 分类为文本的文档)中,
+使用 VLM 的 Text Recognition 识别最终表格外的行内公式切片。纯文本直接回填正文,
+带公式定界符的响应拆成文本与行内公式;不会强制把可用 Unicode 表达的内容转成 LaTeX。
+单个切片识别失败、返回空内容或残缺公式定界符时,仅该切片回退 MFR,并记录告警;
+回退仍无有效结果时报告错误。取消、引擎死亡和程序契约错误不触发此回退。
+Basic 继续使用 MFR;Standard / Advanced 的 OCR 模式继续由 VLM 识别正文,
+表内内容和行间公式沿用原有识别流程。跨行检测框及字体样式不由此切片识别策略修复。
+
## 各入口的默认选择
| 场景 | 当前规则 |
diff --git a/mineru/backend/analysis/pdf/inline_recognition.py b/mineru/backend/analysis/pdf/inline_recognition.py
new file mode 100644
index 00000000..a681a395
--- /dev/null
+++ b/mineru/backend/analysis/pdf/inline_recognition.py
@@ -0,0 +1,201 @@
+"""表外行内切片的 VLM 文本识别准备、定界符校验及局部 MFR 回退。"""
+
+from __future__ import annotations
+
+import math
+import re
+from dataclasses import dataclass, field
+from typing import TYPE_CHECKING, Any
+
+import numpy as np
+from loguru import logger
+
+from ....model.vlm.text_recognition import TextRecognitionResult, raise_unrecoverable_text_error
+from ....utils.timing import stage_timer
+from .constants import BATCH_RATIO, MFR_BASE_BATCH_SIZE
+from .formulas import _build_formula_inputs, _normalize_hybrid_formula_bbox
+from .tables import _build_table_external_mfr_inputs
+
+if TYPE_CHECKING:
+ from PIL.Image import Image
+
+ from ....model.runtime.hybrid import HybridLocalModelContext
+
+_DELIMITER = re.compile(r"\\[()\[\]]")
+_TEXT_DOLLAR_ESCAPE = re.compile(r"(?<!\\)(?P<pairs>(?:\\\\)*)\\\$")
+
+
+@dataclass
+class _InlineCrop:
+ """显式保存切片在窗口内的落点,不用识别文本反查公式。"""
+
+ page_index: int
+ formula_index: int
+ image: Image
+
+
+@dataclass
+class InlineRecognitionBatch:
+ """窗口拥有公式副本和切片资源;原始 MFD 继续独立服务 OCR 遮罩。"""
+
+ formulas: list[list[dict[str, Any]]]
+ crops: list[_InlineCrop] = field(default_factory=list)
+
+ @property
+ def images(self) -> list[Image]:
+ """按稳定任务顺序交付图片,不向模型传递页面资源或裸 PDF 句柄。"""
+ return [crop.image for crop in self.crops]
+
+ def close(self) -> None:
+ """在识别及回填退出后关闭当前窗口持有的全部切片。"""
+ for crop in self.crops:
+ crop.image.close()
+ self.crops.clear()
+
+
+def prepare_inline_recognition(
+ images_layout_res: list[list[dict[str, Any]]],
+ model_list: list[list[dict[str, Any]]],
+ images: list[Image],
+) -> InlineRecognitionBatch:
+ """依据最终表格归属选框,按现有 MFR 的取整和裁边规则创建独立切片。"""
+ if not (len(images_layout_res) == len(model_list) == len(images)):
+ raise ValueError("Inline recognition page count mismatch")
+ inputs = _build_table_external_mfr_inputs(
+ _build_formula_inputs(images_layout_res), model_list, [image.size for image in images]
+ )
+ batch = InlineRecognitionBatch([[] for _ in images])
+ try:
+ for page_index, (formulas, image) in enumerate(zip(inputs, images, strict=True)):
+ for formula in formulas:
+ bbox = _normalize_hybrid_formula_bbox(formula.get("bbox"))
+ if bbox is None:
+ continue
+ x0, y0, x1, y1 = bbox
+ box = (
+ max(0, min(image.width, math.floor(x0))),
+ max(0, min(image.height, math.floor(y0))),
+ max(0, min(image.width, math.ceil(x1))),
+ max(0, min(image.height, math.ceil(y1))),
+ )
+ if box[2] <= box[0] or box[3] <= box[1]:
+ continue
+ crop = image.crop(box)
+ batch.crops.append(_InlineCrop(page_index, len(batch.formulas[page_index]), crop))
+ batch.formulas[page_index].append(formula)
+ return batch
+ except BaseException:
+ batch.close()
+ raise
+
+
+def _unescape_text_dollars(text: str) -> str:
+ """仅解除 VLM 普通文本中美元前的单层转义,保留成对反斜杠和其他命令。"""
+ return _TEXT_DOLLAR_ESCAPE.sub(r"\g<pairs>$", text)
+
+
+def normalize_inline_text(text: str | None) -> str:
+ """规范公式定界符和公式外的美元转义,拒绝空或残缺响应。"""
+ if text is None or not text.strip() or text.strip() == "[Non-Text]":
+ raise ValueError("empty VLM inline response")
+ parts: list[str] = []
+ cursor = 0
+ opening: re.Match[str] | None = None
+ for match in _DELIMITER.finditer(text):
+ # 偶数个连续反斜杠表示转义文字,不作为公式边界。
+ preceding = match.start() - 1
+ while preceding >= 0 and text[preceding] == "\\":
+ preceding -= 1
+ if (match.start() - preceding - 1) % 2:
+ continue
+ token = match.group()
+ if token in {r"\(", r"\["}:
+ if opening is not None:
+ raise ValueError("nested VLM inline delimiter")
+ opening = match
+ else:
+ expected = {r"\)": r"\(", r"\]": r"\["}[token]
+ if opening is None or opening.group() != expected:
+ raise ValueError("unmatched VLM inline delimiter")
+ latex = text[opening.end() : match.start()].strip()
+ if not latex:
+ raise ValueError("empty VLM inline formula")
+ parts.extend((_unescape_text_dollars(text[cursor : opening.start()]), f"\\({latex}\\)"))
+ cursor = match.end()
+ opening = None
+ if opening is not None:
+ raise ValueError("unclosed VLM inline delimiter")
+ parts.append(_unescape_text_dollars(text[cursor:]))
+ return "".join(parts).strip()
+
+
+def finish_inline_recognition(
+ batch: InlineRecognitionBatch,
+ outputs: list[TextRecognitionResult],
+ context: HybridLocalModelContext,
+ np_images: list[np.ndarray],
+ *,
+ page_offset: int,
+) -> list[list[dict[str, Any]]]:
+ """在本地模型锁内仅对失败片段合批回退,任何二次失败均显式终止。"""
+ if len(outputs) != len(batch.crops):
+ raise ValueError(f"Inline recognition count mismatch: inputs={len(batch.crops)}, outputs={len(outputs)}")
+ fallback_inputs: list[list[dict[str, Any]]] = [[] for _ in batch.formulas]
+ failed_crops: list[list[_InlineCrop]] = [[] for _ in batch.formulas]
+ for crop, output in zip(batch.crops, outputs, strict=True):
+ formula = batch.formulas[crop.page_index][crop.formula_index]
+ try:
+ if isinstance(output, Exception):
+ raise_unrecoverable_text_error(output)
+ raise output
+ content = normalize_inline_text(output)
+ except Exception as error:
+ raise_unrecoverable_text_error(error)
+ logger.warning(
+ "Inline VLM fallback to MFR: page={}, formula_index={}, bbox={}, reason={}: {}",
+ page_offset + crop.page_index + 1,
+ crop.formula_index,
+ formula["bbox"],
+ type(error).__name__,
+ error,
+ )
+ fallback_inputs[crop.page_index].append(dict(formula))
+ failed_crops[crop.page_index].append(crop)
+ else:
+ formula["content"] = content
+ formula["content_format"] = "text_with_inline_math"
+ fallback_count = sum(len(page) for page in failed_crops)
+ if fallback_count:
+ with stage_timer("pdf.inline_mfr_fallback"):
+ fallback_results = context.mfr_model.batch_predict(
+ fallback_inputs,
+ np_images,
+ batch_size=BATCH_RATIO * MFR_BASE_BATCH_SIZE,
+ interline_enable=False,
+ )
+ if len(fallback_results) != len(failed_crops):
+ raise ValueError("Inline MFR fallback page count mismatch")
+ for crops, results in zip(failed_crops, fallback_results, strict=True):
+ if len(crops) != len(results):
+ raise ValueError("Inline MFR fallback result count mismatch")
+ for crop, result in zip(crops, results, strict=True):
+ formula = batch.formulas[crop.page_index][crop.formula_index]
+ if result.get("bbox") != formula["bbox"] or result.get("label") != "inline_formula":
+ raise ValueError("Inline MFR fallback result identity mismatch")
+ latex = result.get("latex")
+ if not isinstance(latex, str) or not latex.strip():
+ raise ValueError(
+ f"Empty inline MFR fallback: page={page_offset + crop.page_index + 1}, bbox={formula['bbox']}"
+ )
+ formula["latex"] = latex.strip()
+ if batch.crops:
+ logger.info(
+ "Inline recognition: total={}, vlm_success={}, mfr_fallback={}",
+ len(batch.crops),
+ len(batch.crops) - fallback_count,
+ fallback_count,
+ )
+ return batch.formulas
+
+
+__all__ = ["InlineRecognitionBatch", "prepare_inline_recognition", "finish_inline_recognition", "normalize_inline_text"]
diff --git a/mineru/backend/analysis/pdf/normalization.py b/mineru/backend/analysis/pdf/normalization.py
index 13b4a067..0e0d04f8 100644
--- a/mineru/backend/analysis/pdf/normalization.py
+++ b/mineru/backend/analysis/pdf/normalization.py
@@ -98,13 +98,22 @@ def _is_valid_pdf_text_block(block: dict[str, Any]) -> bool:
def _natural_language_spans(content: str) -> list[dict[str, Any]]:
"""把 PDF 自然语言字符串及圆括号公式定界符直接转换为 Span。"""
- pattern = re.compile(r"\\\((?P<round>.*?)\\\)", re.DOTALL)
+ # 偶数个前导反斜杠属于内容,后面的奇数位反斜杠才可作为模型公式定界符。
+ pattern = re.compile(r"(?<!\\)(?:\\\\)*(?P<delimiter>\\[()])")
spans: list[dict[str, Any]] = []
cursor = 0
+ opening: tuple[int, int] | None = None
for match in pattern.finditer(content):
- append_text_span(spans, content[cursor : match.start()])
- append_equation_span(spans, match.group("round"))
+ if match.group("delimiter") == r"\(":
+ if opening is None:
+ opening = (match.start("delimiter"), match.end())
+ continue
+ if opening is None:
+ continue
+ append_text_span(spans, content[cursor : opening[0]])
+ append_equation_span(spans, content[opening[1] : match.start("delimiter")])
cursor = match.end()
+ opening = None
append_text_span(spans, content[cursor:])
return strip_span_dicts(spans)
diff --git a/mineru/backend/analysis/pdf/ocr.py b/mineru/backend/analysis/pdf/ocr.py
index f3257cc1..7be5b6d8 100644
--- a/mineru/backend/analysis/pdf/ocr.py
+++ b/mineru/backend/analysis/pdf/ocr.py
@@ -54,7 +54,7 @@ def _build_ocr_det_type_and_mfr_enable(
if effort == "medium":
return PIPELINE_DET_TYPE, True
if parse_mode == "txt":
- return VLM_TXT_DET_TYPE, True
+ return VLM_TXT_DET_TYPE, False
return VLM_OCR_DET_TYPE, False
diff --git a/mineru/backend/analysis/pdf/text/content.py b/mineru/backend/analysis/pdf/text/content.py
index 7bf2d639..b6469d3a 100644
--- a/mineru/backend/analysis/pdf/text/content.py
+++ b/mineru/backend/analysis/pdf/text/content.py
@@ -121,8 +121,15 @@ def _build_page_text_formula_spans(
_AnalyzeSpan(
type=ContentType.INLINE_EQUATION,
bbox=bbox,
- content=str(formula.get("latex", "") or "").strip(),
+ content=str(
+ formula.get("content", "")
+ if formula.get("content_format") == "text_with_inline_math"
+ else formula.get("latex", "") or ""
+ ).strip(),
score=float(formula.get("score", 0.0) or 0.0),
+ content_format=(
+ "text_with_inline_math" if formula.get("content_format") == "text_with_inline_math" else "default"
+ ),
)
)
@@ -400,7 +407,7 @@ def _line_content_parts(line: _AnalyzeLine) -> list[tuple[str, str]]:
content = str(span.content or "").strip()
elif span.type == ContentType.INLINE_EQUATION:
latex = str(span.content or "").strip()
- content = f"\\({latex}\\)" if latex else ""
+ content = latex if span.content_format == "text_with_inline_math" else (f"\\({latex}\\)" if latex else "")
else:
continue
if content:
diff --git a/mineru/backend/analysis/pdf/text/models.py b/mineru/backend/analysis/pdf/text/models.py
index 7ae00765..f4c6e7ce 100644
--- a/mineru/backend/analysis/pdf/text/models.py
+++ b/mineru/backend/analysis/pdf/text/models.py
@@ -4,7 +4,7 @@
from __future__ import annotations
from dataclasses import dataclass, field
-from typing import Any
+from typing import Any, Literal
from .....types import BBox
@@ -19,6 +19,7 @@ class _AnalyzeSpan:
score: float = 0.0
image: Any = None
metadata: dict[str, Any] = field(default_factory=dict)
+ content_format: Literal["default", "text_with_inline_math"] = "default"
@dataclass(slots=True)
diff --git a/mineru/backend/analysis/pdf/window.py b/mineru/backend/analysis/pdf/window.py
index 21c8e845..79417b13 100644
--- a/mineru/backend/analysis/pdf/window.py
+++ b/mineru/backend/analysis/pdf/window.py
@@ -12,6 +12,8 @@ if TYPE_CHECKING:
from docvortex.document.pdf import PDFRenderSession
from PIL.Image import Image
from ....model.vlm.contracts import VlmPredictor
+ from ....model.vlm.text_recognition import TextRecognitionResult
+ from .inline_recognition import InlineRecognitionBatch
import numpy as np
from docvortex.assets import image_size as _normalize_page_size
@@ -196,6 +198,7 @@ def _process_text_and_formulas(
page_vector_geometries: list[PDFPageVectorGeometry | None] | None = None,
page_snapshots: PageSnapshotCache | None = None,
np_images: list[np.ndarray] | None = None,
+ recognized_inline_formulas: list[list[dict[str, Any]]] | None = None,
) -> list[list[dict[str, Any]]]:
"""在当前窗口内完成 OCR、公式、原生文本及 block 行信息回填。"""
from .formulas import (
@@ -206,7 +209,7 @@ def _process_text_and_formulas(
optimize_hybrid_formula_number_blocks,
)
from .ocr import _apply_ocr_rec_results, _build_ocr_det_type_and_mfr_enable, _ocr_det
- from .tables import _apply_medium_table_recognition, _build_table_external_mfr_inputs
+ from .tables import _apply_medium_table_recognition
from .text.content import _fill_window_block_content_and_lines, _validate_text_formula_window_inputs
@@ -233,13 +236,15 @@ def _process_text_and_formulas(
images_formula_list = mfd_res
interline_enable = effort == "medium"
- # medium 保留未被原生表格消费的公式;high/xhigh 只识别最终表格外的行内公式。
+ # high/xhigh TXT 的 VLM 与局部 MFR 回退已在窗口阶段完成,不能重新识别成功切片。
+ if effort in {"high", "xhigh"} and parse_mode == "txt":
+ if recognized_inline_formulas is None:
+ raise ValueError("High/Xhigh TXT requires window inline recognition results")
+ images_formula_list = recognized_inline_formulas
+
+ # 只有 medium 在此阶段运行完整 MFR;OCR 遮罩始终使用原始 mfd_res。
if mfr_enable:
- mfr_inputs = (
- mfd_res
- if effort == "medium"
- else _build_table_external_mfr_inputs(mfd_res, model_list, [image.size for image in images_pil_list])
- )
+ mfr_inputs = mfd_res
# 全部被过滤时保持逐页空结果,不能让原始表内公式重新进入正文 sidecar。
images_formula_list = [[] for _ in mfd_res]
if any(mfr_inputs):
@@ -329,12 +334,20 @@ class _WindowInputs:
page_text_geometries: list[PDFPageTextGeometry | None] | None
page_vector_geometries: list[PDFPageVectorGeometry | None] | None = None
page_snapshots: PageSnapshotCache | None = None
+ inline_recognition: InlineRecognitionBatch | None = None
+ inline_outputs: list[TextRecognitionResult] | None = None
def close(self) -> None:
"""推理及回填真正退出后关闭图片,仅清除本层拥有的容器。"""
try:
- _close_images(self.images_list)
+ try:
+ if self.inline_recognition is not None:
+ self.inline_recognition.close()
+ finally:
+ _close_images(self.images_list)
finally:
+ self.inline_outputs = None
+ self.inline_recognition = None
self.images_list.clear()
self.images_pil_list.clear()
self.np_images.clear()
@@ -483,38 +496,63 @@ def _prepare_pdf_window(
table_items.clear()
-def _finish_pdf_window(
+def _prepare_window_result(
state: _WindowInputs,
window_model_list: list[Any],
*,
effort: AnalyzeEffort,
parse_mode: Literal["txt", "ocr"],
+) -> list[list[dict[str, Any]]]:
+ """先确定最终版面与原生表格归属,再准备表外行内切片。"""
+ from .inline_recognition import prepare_inline_recognition
+ from .tables import _restore_native_high_table_blocks
+
+ if parse_mode == "txt" and effort == "high":
+ window_model_list = _restore_native_high_table_blocks(window_model_list, state.accepted_native_tables)
+ if effort in {"high", "xhigh"}:
+ window_model_list = _convert_vlm_results_to_model_list(window_model_list)
+ if effort == "xhigh":
+ _normalize_xhigh_vlm_blocks(window_model_list)
+ _apply_layout_title_split(
+ window_model_list,
+ state.images_layout_res,
+ [_normalize_page_size(image) for image in state.images_pil_list],
+ )
+ if parse_mode == "txt" and effort in {"high", "xhigh"}:
+ state.inline_recognition = prepare_inline_recognition(
+ state.images_layout_res, window_model_list, state.images_pil_list
+ )
+ state.inline_outputs = []
+ return window_model_list
+
+
+def _finish_pdf_window(
+ state: _WindowInputs,
+ window_model_list: list[list[dict[str, Any]]],
+ *,
+ effort: AnalyzeEffort,
+ parse_mode: Literal["txt", "ocr"],
hybrid_model: HybridLocalModelContext,
) -> list[list[dict[str, Any]]]:
- """按原有顺序回填推理结果、文本公式与视觉素材。"""
+ """在本地模型锁内完成失败片段回退、正文回填与视觉素材。"""
from .formulas import optimize_hybrid_formula_number_blocks
+ from .inline_recognition import finish_inline_recognition
from .ocr import _apply_seal_ocr
- from .tables import _restore_native_high_table_blocks
images_list = state.images_list
window_pages = state.window_pages
- images_pil_list = state.images_pil_list
np_images = state.np_images
images_layout_res = state.images_layout_res
vl_style_layout_blocks = state.vl_style_layout_blocks
page_text_geometries = state.page_text_geometries
window = state.window
local_model_context = hybrid_model
- if parse_mode == "txt" and effort == "high":
- window_model_list = _restore_native_high_table_blocks(window_model_list, state.accepted_native_tables)
- if effort in {"high", "xhigh"}:
- window_model_list = _convert_vlm_results_to_model_list(window_model_list)
- if effort == "xhigh":
- _normalize_xhigh_vlm_blocks(window_model_list)
- _apply_layout_title_split(
- window_model_list,
- images_layout_res,
- [_normalize_page_size(image) for image in images_pil_list],
+ inline_options: dict[str, list[list[dict[str, Any]]]] = {}
+ if state.inline_recognition is not None:
+ if state.inline_outputs is None:
+ raise ValueError("Missing window inline recognition outputs")
+ inline_options["recognized_inline_formulas"] = finish_inline_recognition(
+ state.inline_recognition, state.inline_outputs, hybrid_model, np_images, page_offset=window.start
)
if effort == "flash":
@@ -542,6 +580,7 @@ def _finish_pdf_window(
page_vector_geometries=state.page_vector_geometries,
**({"page_snapshots": state.page_snapshots} if state.page_snapshots is not None else {}),
np_images=np_images,
+ **inline_options,
)
if effort in {"medium", "high"}:
@@ -612,6 +651,10 @@ def _process_pdf_window(
result = vlm_predictor.batch_extract_with_layout(**options)
else:
result = vlm_predictor.batch_two_step_extract(**options)
+ result = _prepare_window_result(state, result, effort=effort, parse_mode=parse_mode)
+ if state.inline_recognition is not None and state.inline_recognition.crops:
+ with stage_timer("pdf.inline_vlm"):
+ state.inline_outputs = vlm_predictor.batch_recognize_text(state.inline_recognition.images)
with local_model_stage(hybrid_model.device):
return _finish_pdf_window(state, result, effort=effort, parse_mode=parse_mode, hybrid_model=hybrid_model)
finally:
@@ -719,6 +762,10 @@ async def aio_process_pdf_windows(
result = await vlm_predictor.aio_batch_extract_with_layout(**options)
else:
result = await vlm_predictor.aio_batch_two_step_extract(**options)
+ result = await run_sync(_prepare_window_result, state, result, effort=effort, parse_mode=parse_mode)
+ if state.inline_recognition is not None and state.inline_recognition.crops:
+ with stage_timer("pdf.inline_vlm"):
+ state.inline_outputs = await vlm_predictor.aio_batch_recognize_text(state.inline_recognition.images)
model_list.extend(
await run_sync(
_finish_locked_window,
diff --git a/mineru/model/vlm/async_runtime.py b/mineru/model/vlm/async_runtime.py
index 38cfe49e..eabcdd62 100644
--- a/mineru/model/vlm/async_runtime.py
+++ b/mineru/model/vlm/async_runtime.py
@@ -13,6 +13,7 @@ from ...utils.async_utils import drain_future, run_sync
from ..runtime import ownership
from .errors import EngineDeadError, is_fatal_engine_error
from .handles import RuntimeHandle
+from .text_recognition import TextRecognitionResult, raise_unrecoverable_text_error, validate_text_result
if TYPE_CHECKING:
from mineru_vl_utils.structs import ContentBlock, ExtractResult
@@ -193,6 +194,40 @@ class AsyncVlmPredictor:
pass
raise
+ async def _recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """逐请求隔离识别异常,跨文档共享并发额度,取消后排空所有切片。"""
+ async def recognize(image: Image) -> TextRecognitionResult:
+ """单片失败不会取消同批成功请求,程序错误和引擎故障仍向上传播。"""
+ try:
+ output = await self._predictor.aio_content_extract(image, type="text", semaphore=self._semaphore)
+ except Exception as error:
+ raise_unrecoverable_text_error(error)
+ return error
+ return validate_text_result(output)
+
+ tasks = [asyncio.create_task(recognize(image)) for image in images]
+ pending = set(tasks)
+ try:
+ # wait 不会把父任务取消隐式转发给子任务,避免二次 cancel 打断清理。
+ while pending:
+ done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)
+ for task in done:
+ task.result()
+ return [task.result() for task in tasks]
+ except BaseException:
+ for task in pending:
+ task.cancel()
+ await drain_future(asyncio.gather(*tasks, return_exceptions=True))
+ raise
+
+ def batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """同步调用方通过常驻循环复用原生异步切片识别。"""
+ return self._call(lambda: self._recognize_text(images))
+
+ async def aio_batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """异步切片识别沿用运行时取消确认和关闭屏障。"""
+ return await self._acall(lambda: self._recognize_text(images))
+
async def _extract_with_layout(
self,
images: list[Image],
diff --git a/mineru/model/vlm/contracts.py b/mineru/model/vlm/contracts.py
index b9797b7d..c2fd72e9 100644
--- a/mineru/model/vlm/contracts.py
+++ b/mineru/model/vlm/contracts.py
@@ -8,11 +8,20 @@ from collections.abc import Sequence
if TYPE_CHECKING:
from PIL.Image import Image
from mineru_vl_utils.structs import ContentBlock, ExtractResult
+ from .text_recognition import TextRecognitionResult
class VlmPredictor(Protocol):
"""同步客户端与原生异步代理共同实现的最小文档抽取接口。"""
+ def batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """按输入顺序返回切片文本或可局部回退的异常。"""
+ ...
+
+ async def aio_batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """异步切片识别复用文档推理的并发、取消和资源生命周期。"""
+ ...
+
def batch_extract_with_layout(
self,
images: list[Image],
diff --git a/mineru/model/vlm/handles.py b/mineru/model/vlm/handles.py
index 0dfbc3b7..df8a196f 100644
--- a/mineru/model/vlm/handles.py
+++ b/mineru/model/vlm/handles.py
@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING
from ...utils.async_utils import drain_future, run_sync
from ..runtime.closing import runtime_close_scope
from ..runtime.ownership import RuntimeOwner
+from .text_recognition import TextRecognitionResult, recognize_text_batch
if TYPE_CHECKING:
from mineru_vl_utils import MinerUClient
@@ -91,6 +92,16 @@ class RuntimeHandle:
if lock is not None:
lock.release()
+ def batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """切片合批及必要的单片补试共享同一次运行时租约与执行锁。"""
+ with self._use() as predictor:
+ return recognize_text_batch(predictor, images)
+
+ async def aio_batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:
+ """同步引擎在线程内识别切片,取消时等待推理退出再归还图片。"""
+ async with self._async_use() as predictor:
+ return await run_sync(recognize_text_batch, predictor, images)
+
def batch_extract_with_layout(
self,
images: list[Image],
diff --git a/mineru/model/vlm/text_recognition.py b/mineru/model/vlm/text_recognition.py
new file mode 100644
index 00000000..8fd58bd1
--- /dev/null
+++ b/mineru/model/vlm/text_recognition.py
@@ -0,0 +1,63 @@
+"""行内切片文本识别的结果契约与局部失败隔离。"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, TypeAlias
+
+from mineru_vl_utils.vlm_client.base_client import ClientClosedError, ServerError
+
+from .errors import is_fatal_engine_error
+
+if TYPE_CHECKING:
+ from mineru_vl_utils import MinerUClient
+ from PIL.Image import Image
+
+TextRecognitionResult: TypeAlias = str | None | Exception
+
+
+def raise_unrecoverable_text_error(error: Exception) -> None:
+ """关闭、引擎死亡及程序契约错误必须传播,不能降级成单片识别失败。"""
+ if (
+ is_fatal_engine_error(error)
+ or isinstance(
+ error,
+ (ClientClosedError, MemoryError, AssertionError, AttributeError, TypeError, LookupError, NotImplementedError),
+ )
+ or (isinstance(error, RuntimeError) and not isinstance(error, ServerError))
+ ):
+ raise error
+
+
+def validate_text_result(value: object) -> str | None:
+ """保留合法空响应供宿主回退,拒绝无法保持输入输出契约的非文本结果。"""
+ if value is None:
+ return None
+ if not isinstance(value, str):
+ raise TypeError(f"VLM text recognition returned {type(value).__name__}, expected str or None")
+ return str(value)
+
+
+def recognize_text_batch(client: MinerUClient, images: list[Image]) -> list[TextRecognitionResult]:
+ """正常路径保留本地合批;可恢复批次异常时逐片补试一次,保留成功结果。"""
+ if not images:
+ return []
+ try:
+ outputs = client.batch_content_extract(images, types="text")
+ except Exception as error:
+ raise_unrecoverable_text_error(error)
+ results: list[TextRecognitionResult] = []
+ for image in images:
+ try:
+ output = client.content_extract(image, type="text")
+ except Exception as item_error:
+ raise_unrecoverable_text_error(item_error)
+ results.append(item_error)
+ else:
+ results.append(validate_text_result(output))
+ return results
+ if len(outputs) != len(images):
+ raise ValueError(f"VLM text recognition count mismatch: inputs={len(images)}, outputs={len(outputs)}")
+ return [validate_text_result(output) for output in outputs]
+
+
+__all__ = ["TextRecognitionResult", "raise_unrecoverable_text_error", "recognize_text_batch", "validate_text_result"]
diff --git a/tests/unittest/test_native_pdf_table_pipeline.py b/tests/unittest/test_native_pdf_table_pipeline.py
index ce88ddd0..6e4a2231 100644
--- a/tests/unittest/test_native_pdf_table_pipeline.py
+++ b/tests/unittest/test_native_pdf_table_pipeline.py
@@ -551,6 +551,7 @@ def test_high_txt_window_excludes_native_table_from_vlm(
*,
page_vector_geometries: object,
np_images: object,
+ recognized_inline_formulas: object = None,
) -> list[list[dict[str, object]]]:
"""跳过与本测试无关的正文和公式回填。"""
diff --git a/tests/unittest/test_pdf_analyze_block_content.py b/tests/unittest/test_pdf_analyze_block_content.py
index 3396089b..bdad1803 100644
--- a/tests/unittest/test_pdf_analyze_block_content.py
+++ b/tests/unittest/test_pdf_analyze_block_content.py
@@ -186,12 +186,14 @@ def test_doc_analyze_converts_vlm_results_before_downstream_processing(
*,
page_vector_geometries: object,
np_images: object,
+ recognized_inline_formulas: object = None,
) -> list[list[dict[str, object]]]:
"""原样返回窗口结果,并在后处理入口校验精确容器类型。"""
assert type(window_model_list) is list
assert all(type(page) is list for page in window_model_list)
assert all(type(block) is dict for page in window_model_list for block in page)
if parse_mode == "txt":
+ assert recognized_inline_formulas == [[]]
assert isinstance(_page_text_geometries, list)
assert len(_page_text_geometries) == 1
else:
diff --git a/tests/unittest/test_pdf_inline_vlm.py b/tests/unittest/test_pdf_inline_vlm.py
new file mode 100644
index 00000000..7c43de24
--- /dev/null
+++ b/tests/unittest/test_pdf_inline_vlm.py
@@ -0,0 +1,224 @@
+"""验证 VLM 行内切片、混合文本保护和单片 MFR 回退。"""
+
+from __future__ import annotations
+
+from copy import deepcopy
+from io import BytesIO
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+from docvortex.document.pdf import PDFDocument
+from PIL import Image
+from reportlab.pdfgen.canvas import Canvas
+
+from mineru.backend.analysis.pdf.inline_recognition import (
+ finish_inline_recognition,
+ normalize_inline_text,
+ prepare_inline_recognition,
+)
+from mineru.backend.analysis.pdf.normalization import _natural_language_spans
+from mineru.backend.analysis.pdf.text.content import (
+ _apply_window_post_ocr,
+ _build_page_text_formula_spans,
+ _fill_native_pdf_text_spans,
+ _lines_to_block_content,
+)
+from mineru.backend.analysis.pdf.text.lines import group_spans_to_lines
+from mineru.model.vlm.errors import EngineDeadError
+from mineru.integrations.docvortex import build_metadata
+from mineru.parser.base import ParseResult
+from mineru.render import render_html
+from mineru.types import MiddleJson, PageInfo, Producer, TextBlock
+
+
+ ("source", "expected"),
+ [
+ ("α ≤ 5", "α ≤ 5"),
+ (r"前 \(x_1\),后 \[y^2\]。", r"前 \(x_1\),后 \(y^2\)。"),
+ (r"\( (m_1, m_n) \) . (D)", r"\((m_1, m_n)\) . (D)"),
+ (r"US$5 and $10", r"US$5 and $10"),
+ (r"\$4.48 ± \$1.02", "$4.48 ± $1.02"),
+ (r"price \$4.48; \(x+\$1\); \[y\] then \$1.02", r"price $4.48; \(x+\$1\); \(y\) then $1.02"),
+ (r"literal \\$5 and \\\$6", r"literal \\$5 and \\$6"),
+ (r"C:\data\file.txt \alpha \% \_ \$5", r"C:\data\file.txt \alpha \% \_ $5"),
+ (r"\(\text{\$4.48}\) and \$1.02", r"\(\text{\$4.48}\) and $1.02"),
+ (r"literal \\(x\\) and \(y\)", r"literal \\(x\\) and \(y\)"),
+ ("(p =", "(p ="),
+ (r"\((p =\)", r"\((p =\)"),
+ ],
+)
+def test_inline_text_normalization(source: str, expected: str) -> None:
+ """仅处理模型公式定界符,正文括号、Unicode、美元及转义文字不猜测补全。"""
+ assert normalize_inline_text(source) == expected
+
+
[email protected]("source", [None, "", " ", "[Non-Text]", r"\(\)", r"\(x", r"x\)", r"\[x\)", r"\(a\(b\)\)"])
+def test_invalid_inline_response_requests_fallback(source: str | None) -> None:
+ """空结果及残缺定界符必须触发回退,不能在遮罩后静默丢字。"""
+ with pytest.raises(ValueError):
+ normalize_inline_text(source)
+
+
+def test_semantic_formula_split_respects_backslash_parity() -> None:
+ """转义定界符保留文字,公式边界前的成对反斜杠保留在原内容中。"""
+ assert _natural_language_spans(r"literal \\(x\\) and \(y\)") == [
+ {"type": "text", "content": r"literal \\(x\\) and "},
+ {"type": "equation_inline", "content": "y"},
+ ]
+ assert _natural_language_spans(r"\\\(x\\\)") == [
+ {"type": "text", "content": r"\\"},
+ {"type": "equation_inline", "content": "x\\\\"},
+ ]
+
+
+def test_vlm_currency_roundtrip_keeps_semantic_text_and_renderer_escaping_separate() -> None:
+ """真实回填与序列化后金额为纯文本,Markdown 只转义一次,HTML 不泄漏反斜杠。"""
+ raw = r"\$4.48 ± \$1.02"
+ with Image.new("RGB", (100, 30)) as image:
+ batch = prepare_inline_recognition([[_formula([1, 1, 99, 29])]], [[]], [image])
+ try:
+ formulas = finish_inline_recognition(batch, [raw], SimpleNamespace(), [], page_offset=0)
+ spans = _build_page_text_formula_spans(formulas[0], [], (100, 30), 1.0)
+ content = _lines_to_block_content(group_spans_to_lines(spans), "text")
+ finally:
+ batch.close()
+ semantic = _natural_language_spans(content)
+ assert semantic == [{"type": "text", "content": "$4.48 ± $1.02"}]
+ middle = MiddleJson(
+ pages=[PageInfo(page_idx=0, blocks=[TextBlock(type="text", index=0, bbox=(0, 0, 1, 1), content=semantic)])],
+ is_full_document=True,
+ metadata={"file_suffix": "pdf", "producer": Producer(name="mineru", version="test")},
+ extensions=build_metadata(effort="high", parse_mode="txt"),
+ )
+ result = ParseResult.from_json(ParseResult(middle).to_json())
+ assert result.pages[0].blocks[0].content[0].content == "$4.48 ± $1.02"
+ assert result.markdown() == raw
+ rendered = render_html(result.middle_json)
+ assert "$4.48 ± $1.02" in rendered
+ assert r"\$" not in rendered
+
+
+def test_currency_unescaping_does_not_change_native_text_or_mfr_latex() -> None:
+ """修复限定在 VLM 文本入口,原生文本与 MFR 公式的美元转义保持原义。"""
+ assert _natural_language_spans(r"literal \$5") == [{"type": "text", "content": r"literal \$5"}]
+ spans = _build_page_text_formula_spans([{**_formula([1, 1, 20, 10]), "latex": r"\$5"}], [], (100, 30), 1.0)
+ content = _lines_to_block_content(group_spans_to_lines(spans), "text")
+ assert _natural_language_spans(content) == [{"type": "equation_inline", "content": r"\$5"}]
+
+
+def _formula(bbox: list[float]) -> dict[str, Any]:
+ """构造可追踪的原始检测框。"""
+ return {"label": "inline_formula", "bbox": bbox, "score": 0.9}
+
+
+def test_crop_selection_and_mixed_results_preserve_original_masks() -> None:
+ """以最终表格过滤切片,保持 MFR 裁边规则且不改原始检测数据。"""
+ layouts = [[_formula([1.2, 2.8, 9.1, 8.2]), _formula([30, 30, 40, 40]), _formula([200, 0, 210, 10])], []]
+ before = deepcopy(layouts)
+ with Image.new("RGB", (100, 100)) as image:
+ batch = prepare_inline_recognition(layouts, [[{"type": "table", "bbox": [0.2, 0.2, 0.5, 0.5]}], []], [image, image])
+ try:
+ assert len(batch.crops) == 1
+ assert batch.images[0].size == (9, 7)
+ result = finish_inline_recognition(batch, [r"\(x\) . (D)"], SimpleNamespace(), [], page_offset=0)
+ assert result[0][0]["content"] == r"\(x\) . (D)"
+ assert result[0][0]["content_format"] == "text_with_inline_math"
+ assert result[1] == []
+ assert layouts == before
+ crop = batch.images[0]
+ finally:
+ batch.close()
+ with pytest.raises(ValueError, match="closed image"):
+ crop.getpixel((0, 0))
+
+
+def test_only_failed_crops_reach_mfr_in_page_order() -> None:
+ """只把不同页的失败切片交给 MFR,识别相同文本也不能错配几何落点。"""
+ layout = [[_formula([1, 1, 10, 10]), _formula([12, 1, 20, 10])], [_formula([1, 1, 10, 10])]]
+ with Image.new("RGB", (40, 40)) as image:
+ batch = prepare_inline_recognition(layout, [[], []], [image, image])
+ mfr = MagicMock(return_value=[[{**layout[0][1], "latex": "x_2"}], [{**layout[1][0], "latex": "x_3"}]])
+ try:
+ result = finish_inline_recognition(
+ batch,
+ ["α", None, ValueError("request failed")],
+ SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr)),
+ [np.zeros((40, 40, 3))] * 2,
+ page_offset=64,
+ )
+ inputs = mfr.call_args.args[0]
+ assert [[f["bbox"] for f in page] for page in inputs] == [[layout[0][1]["bbox"]], [layout[1][0]["bbox"]]]
+ assert mfr.call_args.kwargs["interline_enable"] is False
+ assert result[0][0]["content"] == "α"
+ assert result[0][1]["latex"] == "x_2"
+ assert result[1][0]["latex"] == "x_3"
+ assert "content_format" not in result[0][1]
+ finally:
+ batch.close()
+
+
[email protected]("failure", ["empty", "count", "identity", "engine", "vlm_count"])
+def test_fallback_failure_and_contract_errors_are_not_swallowed(failure: str) -> None:
+ """MFR 二次失败、返回错配及引擎死亡必须显式报错。"""
+ formula = _formula([1, 1, 10, 10])
+ with Image.new("RGB", (20, 20)) as image:
+ batch = prepare_inline_recognition([[formula]], [[]], [image])
+ answer = {**formula, "latex": "" if failure == "empty" else "x"}
+ if failure == "identity":
+ answer["bbox"] = [2, 2, 3, 3]
+ mfr = MagicMock(return_value=[] if failure == "count" else [[answer]])
+ outputs = [] if failure == "vlm_count" else [EngineDeadError("dead") if failure == "engine" else None]
+ try:
+ with pytest.raises((ValueError, EngineDeadError)):
+ finish_inline_recognition(
+ batch, outputs, SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr)), [], page_offset=0
+ )
+ if failure in {"engine", "vlm_count"}:
+ mfr.assert_not_called()
+ finally:
+ batch.close()
+
+
[email protected]("recognized", ["α ≤ 5", r"\(m_n\) . (D)", r"literal \\(x\\) and \(y\)"])
+def test_native_and_post_ocr_cannot_overwrite_vlm_text(recognized: str) -> None:
+ """用真实 PDF 原生字符验证几何占位保护,最终才拆出没有几何字段的语义 span。"""
+ stream = BytesIO()
+ canvas = Canvas(stream, pagesize=(200, 80))
+ canvas.setFont("Helvetica", 12)
+ canvas.drawString(10, 40, "before WRONG after")
+ canvas.save()
+ with PDFDocument(stream.getvalue()) as document, Image.new("RGB", (200, 80)) as image:
+ page = document[0]
+ chars = [char for char in page.get_chars() if char["char"] not in "\r\n"]
+
+ def bbox(selected: list[dict[str, Any]]) -> list[float]:
+ """由真实字符框计算三段互不重叠的检测位置。"""
+ return [
+ min(c["bbox"][0] for c in selected),
+ min(c["bbox"][1] for c in selected),
+ max(c["bbox"][2] for c in selected),
+ max(c["bbox"][3] for c in selected),
+ ]
+
+ formula = {**_formula(bbox(chars[7:12])), "content": recognized, "content_format": "text_with_inline_math"}
+ ocr = [{"bbox": bbox(chars[:6]), "text": ""}, {"bbox": bbox(chars[13:]), "text": ""}]
+ spans = _build_page_text_formula_spans([formula], ocr, page.size, 1.0)
+ spans = _fill_native_pdf_text_spans(page, spans, image, 1.0, page.size)
+ lines = group_spans_to_lines(spans)
+ model = SimpleNamespace(ocr_model=SimpleNamespace(ocr=MagicMock()))
+ _apply_window_post_ocr(model, [{0: lines}])
+ model.ocr_model.ocr.assert_not_called()
+ content = _lines_to_block_content(lines, "text")
+ assert "WRONG" not in content
+ assert content == f"before {recognized} after"
+ semantic = _natural_language_spans(content)
+ assert all(set(span) <= {"type", "content"} for span in semantic)
+ if recognized == "α ≤ 5":
+ assert semantic == [{"type": "text", "content": "before α ≤ 5 after"}]
+ else:
+ assert sum(span["type"] == "equation_inline" for span in semantic) == 1
+ assert semantic[-1]["content"].endswith(" after")
diff --git a/tests/unittest/test_pdf_inline_vlm_window.py b/tests/unittest/test_pdf_inline_vlm_window.py
new file mode 100644
index 00000000..3269530f
--- /dev/null
+++ b/tests/unittest/test_pdf_inline_vlm_window.py
@@ -0,0 +1,133 @@
+"""验证真实窗口编排的最终表格归属、模型锁和切片生命周期。"""
+
+from __future__ import annotations
+
+import asyncio
+from collections.abc import Iterator
+from contextlib import contextmanager
+from copy import deepcopy
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import MagicMock
+
+import numpy as np
+import pytest
+from PIL import Image
+
+from mineru.backend.analysis.pdf import ocr, window
+
+
[email protected]("effort", ["high", "xhigh"])
[email protected]("is_async", [False, True])
[email protected]("fallback", [False, True])
+def test_window_inline_inference_runs_outside_local_lock(
+ monkeypatch: pytest.MonkeyPatch, effort: str, is_async: bool, fallback: bool
+) -> None:
+ """同步与异步入口在最终版面过滤后识别,仅失败片段在本地模型锁内回退。"""
+ locked = []
+ observed_crops = []
+ observed_results = []
+ layouts = [[{"label": "inline_formula", "bbox": box} for box in [[50, 50, 60, 60], [10, 10, 15, 15], [20, 10, 25, 15]]]]
+ # 本地 table 故意覆盖外侧公式,实际归属必须由最终 VLM 表格决定。
+ layouts[0].append({"label": "table", "bbox": [0, 0, 30, 30]})
+ final_blocks = [[{"type": "text", "bbox": [0, 0, 0.3, 0.3]}, {"type": "table", "bbox": [0.4, 0.4, 0.8, 0.8]}]]
+
+ @contextmanager
+ def stage(_device: str) -> Iterator[None]:
+ """显式观察本地模型锁,不依赖线程重入锁的实现细节。"""
+ assert not locked
+ locked.append(True)
+ try:
+ yield
+ finally:
+ locked.pop()
+
+ def prepare(*args: Any, **kwargs: Any) -> window._WindowInputs:
+ """只替代渲染与 layout,继续使用真实窗口转换、推理和资源关闭。"""
+ assert locked
+ picture = Image.new("RGB", (100, 100))
+ return window._WindowInputs(
+ window._ProcessingWindow(0, 1, 0, 0),
+ [{"img_pil": picture, "scale": 1}],
+ [object()],
+ [picture],
+ [np.asarray(picture).copy()],
+ deepcopy(layouts),
+ [],
+ [],
+ [[]],
+ None,
+ )
+
+ def main_infer(**kwargs: Any) -> list[list[dict[str, Any]]]:
+ """主 VLM 也必须在本地模型锁之外执行。"""
+ assert not locked
+ return deepcopy(final_blocks)
+
+ def inline_infer(images: list[Image.Image]) -> list[str | None]:
+ """最终表格内公式不得进入 VLM,并保存切片引用核对关闭时机。"""
+ assert not locked
+ assert len(images) == 2
+ observed_crops.extend(images)
+ assert all(image.getpixel((0, 0)) == (0, 0, 0) for image in images)
+ return ["α", None if fallback else r"\(x\) . (D)"]
+
+ async def async_main(**kwargs: Any) -> list[list[dict[str, Any]]]:
+ """异步主推理复用同一版面及锁断言。"""
+ return main_infer(**kwargs)
+
+ async def async_inline(images: list[Image.Image]) -> list[str | None]:
+ """让真实异步窗口经过一次等待,再验证图片仍可读取。"""
+ await asyncio.sleep(0)
+ return inline_infer(images)
+
+ def mfr_predict(inputs: list[list[dict[str, Any]]], *args: Any, **kwargs: Any) -> list[list[dict[str, Any]]]:
+ """回退必须持有本地模型锁,且只接收第二个表外切片。"""
+ assert locked
+ assert [item["bbox"] for item in inputs[0]] == [[20, 10, 25, 15]]
+ return [[{**item, "latex": "x"} for item in inputs[0]]]
+
+ def fill(*args: Any, **kwargs: Any) -> list[list[dict[str, Any]]]:
+ """正文阶段观察按原顺序合并后的 VLM 与 MFR 结果。"""
+ assert locked
+ observed_results.extend(kwargs["recognized_inline_formulas"][0])
+ return args[2]
+
+ model = SimpleNamespace(device="cpu")
+ if fallback:
+ model.mfr_model = SimpleNamespace(batch_predict=mfr_predict)
+ predictor = SimpleNamespace(
+ batch_extract_with_layout=main_infer,
+ batch_two_step_extract=main_infer,
+ aio_batch_extract_with_layout=async_main,
+ aio_batch_two_step_extract=async_main,
+ batch_recognize_text=inline_infer,
+ aio_batch_recognize_text=async_inline,
+ )
+ monkeypatch.setattr(window, "local_model_stage", stage)
+ monkeypatch.setattr(window, "_prepare_pdf_window", prepare)
+ monkeypatch.setattr(window, "_process_text_and_formulas", fill)
+ monkeypatch.setattr(window, "_attach_visual_block_images", MagicMock())
+ monkeypatch.setattr(window, "_supplement_missing_image_block_containers", MagicMock())
+ monkeypatch.setattr(window, "get_document_render_session", lambda document: None)
+ monkeypatch.setattr(ocr, "_apply_seal_ocr", MagicMock())
+ options = {
+ "effort": effort,
+ "parse_mode": "txt",
+ "image_analysis": False,
+ "hybrid_model": model,
+ "vlm_predictor": predictor,
+ }
+ if is_async:
+ result = asyncio.run(window.aio_process_pdf_windows(b"", SimpleNamespace(page_count=1), **options))
+ else:
+ result = window._process_pdf_window(
+ b"", SimpleNamespace(page_count=1), window._ProcessingWindow(0, 1, 0, 0), page_count=1, **options
+ )
+ assert len(result) == 1
+ assert observed_results[0]["content"] == "α"
+ assert observed_results[1].get("latex") == ("x" if fallback else "")
+ assert [item["bbox"] for item in observed_results] == [[10, 10, 15, 15], [20, 10, 25, 15]]
+ for image in observed_crops:
+ with pytest.raises(ValueError, match="closed image"):
+ image.getpixel((0, 0))
diff --git a/tests/unittest/test_pdf_memory_lifecycle.py b/tests/unittest/test_pdf_memory_lifecycle.py
index 21321032..8def5dee 100644
--- a/tests/unittest/test_pdf_memory_lifecycle.py
+++ b/tests/unittest/test_pdf_memory_lifecycle.py
@@ -353,6 +353,7 @@ def test_async_window_keeps_64_page_boundary_and_order(
# 此处只验证窗口顺序,图片及文档均为替身,不启动真实渲染会话。
monkeypatch.setattr(window, "get_document_render_session", lambda _document: None)
monkeypatch.setattr(window, "_prepare_locked_window", prepare)
+ monkeypatch.setattr(window, "_prepare_window_result", lambda state, result, **kwargs: result)
monkeypatch.setattr(window, "_finish_locked_window", lambda state, result, **kwargs: result)
result = asyncio.run(
window.aio_process_pdf_windows(
diff --git a/tests/unittest/test_pdf_mfr_table_routing.py b/tests/unittest/test_pdf_mfr_table_routing.py
index de2b10a2..22ddd6ad 100644
--- a/tests/unittest/test_pdf_mfr_table_routing.py
+++ b/tests/unittest/test_pdf_mfr_table_routing.py
@@ -14,6 +14,7 @@ from PIL import Image
from mineru.backend.analysis.pdf import window
from mineru.backend.analysis.pdf.formulas import _build_formula_inputs
+from mineru.backend.analysis.pdf.inline_recognition import finish_inline_recognition, prepare_inline_recognition
from mineru.backend.analysis.pdf.tables import _build_table_external_mfr_inputs
from mineru.backend.analysis.pdf import ocr as stage_ocr
@@ -85,7 +86,7 @@ def test_window_mfr_routing_and_original_ocr_masks(
blocks = [[{"type": "table", "bbox": [0.1, 0.1, 0.9, 0.9], "content": content}]]
original = deepcopy(layout)
mfr = MagicMock(side_effect=lambda inputs, *_args, **_kwargs: deepcopy(inputs))
- enabled = effort == "medium" or parse_mode == "txt"
+ enabled = effort == "medium"
context = SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr)) if enabled else SimpleNamespace()
det = MagicMock(return_value=[[]])
fill = MagicMock(return_value=blocks)
@@ -107,16 +108,16 @@ def test_window_mfr_routing_and_original_ocr_masks(
effort,
context,
layout,
+ recognized_inline_formulas=[[layout[0][1]]] if effort != "medium" and parse_mode == "txt" else None,
)
if enabled:
- expected = _build_formula_inputs(layout) if effort == "medium" else [[layout[0][1]]]
- assert mfr.call_args.args[0] == expected
+ assert mfr.call_args.args[0] == _build_formula_inputs(layout)
assert mfr.call_args.kwargs["interline_enable"] is (effort == "medium")
else:
mfr.assert_not_called()
assert det.call_args.args[3] == _build_formula_inputs(layout)
assert layout == original
- expected_inline = [layout[0][:2]] if effort == "medium" or not enabled else [[layout[0][1]]]
+ expected_inline = [layout[0][:2]] if effort == "medium" or parse_mode == "ocr" else [[layout[0][1]]]
assert fill.call_args.args[3] == expected_inline
@@ -134,12 +135,15 @@ def test_final_layout_controls_filter_and_empty_results_skip_model_loading(
layout = [[formula, {"label": "table", "bbox": local_table_bbox}], [_formula([10, 10, 20, 20], "display_formula")]]
blocks = [[{"type": "table", "bbox": [0.1, 0.1, 0.5, 0.5]}] if final_table else [], []]
mfr = MagicMock(side_effect=lambda inputs, *_args, **_kwargs: deepcopy(inputs))
- context = SimpleNamespace() if final_table else SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr))
+ context = SimpleNamespace()
fill = MagicMock(return_value=blocks)
det = MagicMock(return_value=[[], []])
monkeypatch.setattr(stage_ocr, "_ocr_det", det)
monkeypatch.setattr(stage_text_content, "_fill_window_block_content_and_lines", fill)
with Image.new("RGB", (100, 100)) as image:
+ batch = prepare_inline_recognition(layout, blocks, [image, image])
+ recognized = finish_inline_recognition(batch, ["α"] * len(batch.crops), context, [], page_offset=0)
+ batch.close()
window._process_text_and_formulas(
[{"img_pil": image, "scale": 1.0}] * 2,
[MagicMock()] * 2,
@@ -148,10 +152,13 @@ def test_final_layout_controls_filter_and_empty_results_skip_model_loading(
effort,
context,
layout,
+ recognized_inline_formulas=recognized,
)
- assert fill.call_args.args[3] == ([[], []] if final_table else [[formula], []])
+ assert fill.call_args.args[3] == recognized
+ expected = [[], []] if final_table else [[{**formula, "content": "α", "content_format": "text_with_inline_math"}], []]
+ assert recognized == expected
assert det.call_args.args[3] == _build_formula_inputs(layout)
- assert mfr.call_count == (0 if final_table else 1)
+ mfr.assert_not_called()
def test_demo1_frozen_mfr_inputs_exclude_only_29_table_formulas() -> None:
diff --git a/tests/unittest/test_vlm_remote.py b/tests/unittest/test_vlm_remote.py
index 29252f53..05b818bb 100644
--- a/tests/unittest/test_vlm_remote.py
+++ b/tests/unittest/test_vlm_remote.py
@@ -117,6 +117,29 @@ def _settings(server: _OpenAIServer, **overrides: Any) -> VlmConfig:
return VlmConfig.model_validate({"server_url": server.url, "api_key": server.api_key, **overrides})
[email protected]("is_async", [False, True])
+def test_remote_inline_text_uses_default_prompt_and_preserves_failures(
+ openai_server: _OpenAIServer, is_async: bool
+) -> None:
+ """真实 HTTP 适配器用文本提示识别切片,并将请求失败交给宿主按片回退。"""
+ predictor, _ = get_vlm_predictor(_settings(openai_server))
+ with Image.new("RGB", (32, 16), "white") as image:
+ if is_async:
+ result = asyncio.run(predictor.aio_batch_recognize_text([image, image]))
+ else:
+ result = predictor.batch_recognize_text([image, image])
+ assert result == ["Remote VLM text", "Remote VLM text"]
+ bodies = [body for _, _, body in openai_server.requests if body is not None]
+ assert len(bodies) == 2
+ for body in bodies:
+ content = body["messages"][-1]["content"]
+ assert [part["text"] for part in content if part["type"] == "text"] == ["\nText Recognition:"]
+ openai_server.chat_status = 400
+ failures = predictor.batch_recognize_text([image, image])
+ assert len(failures) == 2
+ assert all(isinstance(item, Exception) for item in failures)
+
+
def test_remote_client_protocol_and_cache(openai_server: _OpenAIServer, monkeypatch: pytest.MonkeyPatch) -> None:
"""真实 HTTP 验证模型发现、显式模型、认证、代理路径、参数及缓存隔离。"""
from mineru.model.vlm import runtime, selector
@@ -263,6 +286,7 @@ def hybrid_stub(monkeypatch: pytest.MonkeyPatch) -> None:
page_vector_geometries: object,
np_images: object,
page_snapshots: object = None,
+ recognized_inline_formulas: object = None,
) -> list[list[dict[str, Any]]]:
"""保留远程正文并模拟 Hybrid 提供的行框,满足 PDF 文本块校验合同。"""
for page in blocks:
diff --git a/tests/unittest/test_vlm_text_recognition.py b/tests/unittest/test_vlm_text_recognition.py
new file mode 100644
index 00000000..cea764b5
--- /dev/null
+++ b/tests/unittest/test_vlm_text_recognition.py
@@ -0,0 +1,173 @@
+"""验证行内 VLM 文本识别的错误隔离、共享并发和取消屏障。"""
+
+from __future__ import annotations
+
+import asyncio
+import threading
+from concurrent.futures import ThreadPoolExecutor
+from types import SimpleNamespace
+from typing import Any
+from unittest.mock import MagicMock
+
+import pytest
+from mineru_vl_utils.vlm_client.base_client import ClientClosedError, RequestError
+
+from mineru.model.vlm.async_runtime import AsyncVlmPredictor
+from mineru.model.vlm.errors import EngineDeadError
+from mineru.model.vlm.handles import RuntimeHandle
+from mineru.model.vlm.text_recognition import recognize_text_batch
+
+
+def test_local_batch_keeps_successes_and_only_retries_after_batch_failure() -> None:
+ """本地正常合批不拆单;批次失败后只逐片补试一次并保留异常落点。"""
+ failure = RequestError("bad crop")
+ client = SimpleNamespace(batch_content_extract=MagicMock(return_value=["α", None]), content_extract=MagicMock())
+ assert recognize_text_batch(client, [1, 2]) == ["α", None]
+ client.content_extract.assert_not_called()
+ client.batch_content_extract.assert_called_once_with([1, 2], types="text")
+ client.batch_content_extract.side_effect = RequestError("batch failed")
+ client.content_extract.side_effect = ["success", failure, None]
+ assert recognize_text_batch(client, [1, 2, 3]) == ["success", failure, None]
+ assert client.content_extract.call_count == 3
+ assert [call.kwargs for call in client.content_extract.call_args_list] == [{"type": "text"}] * 3
+
+
+ "error", [EngineDeadError("dead"), ClientClosedError("closed"), RuntimeError("engine stopped"), TypeError("bug")]
+)
+def test_local_fatal_errors_never_retry(error: Exception) -> None:
+ """引擎失效、关闭及程序错误不得重试后伪装成 MFR 回退。"""
+ client = SimpleNamespace(batch_content_extract=MagicMock(side_effect=error), content_extract=MagicMock())
+ with pytest.raises(type(error)):
+ recognize_text_batch(client, [1])
+ client.content_extract.assert_not_called()
+
+
[email protected]("outputs", [[], ["x", "y"], [object()]])
+def test_local_output_contract_is_strict(outputs: list[Any]) -> None:
+ """长度和类型错配立即报错,不能触发掩盖适配器缺陷的逐片补试。"""
+ client = SimpleNamespace(batch_content_extract=MagicMock(return_value=outputs), content_extract=MagicMock())
+ with pytest.raises((ValueError, TypeError)):
+ recognize_text_batch(client, [1])
+ client.content_extract.assert_not_called()
+
+
+class _TextClient:
+ """异步文本替身在真实运行时循环里使用共享信号量并延迟清理。"""
+
+ def __init__(self) -> None:
+ """记录请求和清理事件,便于验证多个文档及不同入口共享运行时。"""
+ self.loop = asyncio.get_running_loop()
+ self.active = 0
+ self.peak = 0
+ self.block = False
+ self.entered = threading.Event()
+ self.closed = False
+
+ async def aio_content_extract(self, image: int, *, type: str, semaphore: asyncio.Semaphore) -> str:
+ """指定切片失败仍释放请求额度,取消时模拟异步资源清理。"""
+ assert asyncio.get_running_loop() is self.loop
+ assert type == "text"
+ async with semaphore:
+ self.active += 1
+ self.peak = max(self.peak, self.active)
+ self.entered.set()
+ try:
+ await asyncio.sleep(60 if self.block else 0.01)
+ if image == -1:
+ raise RequestError("bad crop")
+ if image == -2:
+ raise EngineDeadError("dead")
+ return str(image)
+ finally:
+ await asyncio.sleep(0.03)
+ self.active -= 1
+
+ async def aclose(self) -> None:
+ """只有所有切片完成清理之后才允许关闭客户端。"""
+ assert self.active == 0
+ self.closed = True
+
+
+def text_runtime() -> Any:
+ """创建原生异步宿主并保证测试结束后关闭线程及客户端。"""
+ runtime = AsyncVlmPredictor(
+ lambda: RuntimeHandle(_TextClient(), backend="http-client"), backend="http-client", max_concurrency=2
+ )
+ runtime.wait_ready()
+ try:
+ yield runtime
+ finally:
+ runtime.shutdown()
+
+
+def test_native_text_results_keep_order_errors_and_global_limit(text_runtime: AsyncVlmPredictor) -> None:
+ """跨线程同步及异步入口共享额度,单片失败不会取消其他切片。"""
+
+ def invoke(index: int) -> list[Any]:
+ """交错同步和异步调用,模拟多文档同时提交。"""
+ if index % 2:
+ return asyncio.run(text_runtime.aio_batch_recognize_text([index, -1]))
+ return text_runtime.batch_recognize_text([index, -1])
+
+ with ThreadPoolExecutor(max_workers=4) as pool:
+ results = list(pool.map(invoke, range(4)))
+ assert [result[0] for result in results] == ["0", "1", "2", "3"]
+ assert all(isinstance(result[1], RequestError) for result in results)
+ assert text_runtime._predictor.peak == 2
+ assert text_runtime._predictor.active == 0
+
+
+def test_native_repeated_cancellation_drains_all_crops(text_runtime: AsyncVlmPredictor) -> None:
+ """重复取消不得使调用方在任何切片仍读取图片时退出。"""
+ client = text_runtime._predictor
+ client.block = True
+
+ async def run() -> None:
+ """发出两次取消并等待跨循环的清理确认。"""
+ task = asyncio.create_task(text_runtime.aio_batch_recognize_text([1, 2, 3]))
+ assert await asyncio.to_thread(client.entered.wait, 2)
+ task.cancel()
+ await asyncio.sleep(0.005)
+ task.cancel()
+ assert not task.done()
+ with pytest.raises(asyncio.CancelledError):
+ await task
+ assert client.active == 0
+
+ asyncio.run(run())
+ client.block = False
+ assert text_runtime.batch_recognize_text([4]) == ["4"]
+
+
+def test_native_engine_failure_drains_other_requests(text_runtime: AsyncVlmPredictor) -> None:
+ """引擎死亡向上传播,同时等待同批其他请求清理。"""
+ with pytest.raises(EngineDeadError):
+ text_runtime.batch_recognize_text([-2, 1, 2])
+ assert text_runtime._predictor.active == 0
+
+
+def test_local_shutdown_waits_for_text_request() -> None:
+ """本地切片识别在途时关闭必须等待,关闭后拒绝新请求。"""
+ entered, release, closed = threading.Event(), threading.Event(), threading.Event()
+
+ def infer(images: list[Any], *, types: str) -> list[str]:
+ """模拟持有图片的同步推理,显式释放前不得关闭引擎。"""
+ entered.set()
+ assert release.wait(2)
+ assert not closed.is_set()
+ return ["x"]
+
+ handle = RuntimeHandle(SimpleNamespace(batch_content_extract=infer), backend="test", close_engine=closed.set)
+ with ThreadPoolExecutor(max_workers=2) as pool:
+ work = pool.submit(handle.batch_recognize_text, [1])
+ assert entered.wait(2)
+ shutting_down = pool.submit(handle.shutdown)
+ assert not closed.wait(0.02)
+ release.set()
+ assert work.result() == ["x"]
+ shutting_down.result()
+ assert closed.is_set()
+ with pytest.raises(RuntimeError, match="shutting down"):
+ handle.batch_recognize_text([1])