Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
feat(pdf): recognize inline formulas with VLM text extraction
myhloli115693124d27
diff --git a/docs/next/tiers.md b/docs/next/tiers.mdindex 7a9f450e..43dbfefb 100644--- a/docs/next/tiers.md+++ b/docs/next/tiers.md@@ -13,6 +13,14 @@基础包已经提供 ONNX 与 llama.cpp;Basic 不要求额外安装 Torch。非 PDF 原生格式使用 DocVortex,与 PDF 推理档位不是同一种实现;不承诺 Flash 对原生格式的质量总是最低。+Standard / Advanced 在 PDF 文本模式(`txt`,包括 `auto` 分类为文本的文档)中,+使用 VLM 的 Text Recognition 识别最终表格外的行内公式切片。纯文本直接回填正文,+带公式定界符的响应拆成文本与行内公式;不会强制把可用 Unicode 表达的内容转成 LaTeX。+单个切片识别失败、返回空内容或残缺公式定界符时,仅该切片回退 MFR,并记录告警;+回退仍无有效结果时报告错误。取消、引擎死亡和程序契约错误不触发此回退。+Basic 继续使用 MFR;Standard / Advanced 的 OCR 模式继续由 VLM 识别正文,+表内内容和行间公式沿用原有识别流程。跨行检测框及字体样式不由此切片识别策略修复。+## 各入口的默认选择| 场景 | 当前规则 |diff --git a/mineru/backend/analysis/pdf/inline_recognition.py b/mineru/backend/analysis/pdf/inline_recognition.pynew file mode 100644index 00000000..a681a395--- /dev/null+++ b/mineru/backend/analysis/pdf/inline_recognition.py@@ -0,0 +1,201 @@+"""表外行内切片的 VLM 文本识别准备、定界符校验及局部 MFR 回退。"""++from __future__ import annotations++import math+import re+from dataclasses import dataclass, field+from typing import TYPE_CHECKING, Any++import numpy as np+from loguru import logger++from ....model.vlm.text_recognition import TextRecognitionResult, raise_unrecoverable_text_error+from ....utils.timing import stage_timer+from .constants import BATCH_RATIO, MFR_BASE_BATCH_SIZE+from .formulas import _build_formula_inputs, _normalize_hybrid_formula_bbox+from .tables import _build_table_external_mfr_inputs++if TYPE_CHECKING:+ from PIL.Image import Image++ from ....model.runtime.hybrid import HybridLocalModelContext++_DELIMITER = re.compile(r"\\[()\[\]]")+_TEXT_DOLLAR_ESCAPE = re.compile(r"(?<!\\)(?P<pairs>(?:\\\\)*)\\\$")+++@dataclass+class _InlineCrop:+ """显式保存切片在窗口内的落点,不用识别文本反查公式。"""++ page_index: int+ formula_index: int+ image: Image+++@dataclass+class InlineRecognitionBatch:+ """窗口拥有公式副本和切片资源;原始 MFD 继续独立服务 OCR 遮罩。"""++ formulas: list[list[dict[str, Any]]]+ crops: list[_InlineCrop] = field(default_factory=list)++ @property+ def images(self) -> list[Image]:+ """按稳定任务顺序交付图片,不向模型传递页面资源或裸 PDF 句柄。"""+ return [crop.image for crop in self.crops]++ def close(self) -> None:+ """在识别及回填退出后关闭当前窗口持有的全部切片。"""+ for crop in self.crops:+ crop.image.close()+ self.crops.clear()+++def prepare_inline_recognition(+ images_layout_res: list[list[dict[str, Any]]],+ model_list: list[list[dict[str, Any]]],+ images: list[Image],+) -> InlineRecognitionBatch:+ """依据最终表格归属选框,按现有 MFR 的取整和裁边规则创建独立切片。"""+ if not (len(images_layout_res) == len(model_list) == len(images)):+ raise ValueError("Inline recognition page count mismatch")+ inputs = _build_table_external_mfr_inputs(+ _build_formula_inputs(images_layout_res), model_list, [image.size for image in images]+ )+ batch = InlineRecognitionBatch([[] for _ in images])+ try:+ for page_index, (formulas, image) in enumerate(zip(inputs, images, strict=True)):+ for formula in formulas:+ bbox = _normalize_hybrid_formula_bbox(formula.get("bbox"))+ if bbox is None:+ continue+ x0, y0, x1, y1 = bbox+ box = (+ max(0, min(image.width, math.floor(x0))),+ max(0, min(image.height, math.floor(y0))),+ max(0, min(image.width, math.ceil(x1))),+ max(0, min(image.height, math.ceil(y1))),+ )+ if box[2] <= box[0] or box[3] <= box[1]:+ continue+ crop = image.crop(box)+ batch.crops.append(_InlineCrop(page_index, len(batch.formulas[page_index]), crop))+ batch.formulas[page_index].append(formula)+ return batch+ except BaseException:+ batch.close()+ raise+++def _unescape_text_dollars(text: str) -> str:+ """仅解除 VLM 普通文本中美元前的单层转义,保留成对反斜杠和其他命令。"""+ return _TEXT_DOLLAR_ESCAPE.sub(r"\g<pairs>$", text)+++def normalize_inline_text(text: str | None) -> str:+ """规范公式定界符和公式外的美元转义,拒绝空或残缺响应。"""+ if text is None or not text.strip() or text.strip() == "[Non-Text]":+ raise ValueError("empty VLM inline response")+ parts: list[str] = []+ cursor = 0+ opening: re.Match[str] | None = None+ for match in _DELIMITER.finditer(text):+ # 偶数个连续反斜杠表示转义文字,不作为公式边界。+ preceding = match.start() - 1+ while preceding >= 0 and text[preceding] == "\\":+ preceding -= 1+ if (match.start() - preceding - 1) % 2:+ continue+ token = match.group()+ if token in {r"\(", r"\["}:+ if opening is not None:+ raise ValueError("nested VLM inline delimiter")+ opening = match+ else:+ expected = {r"\)": r"\(", r"\]": r"\["}[token]+ if opening is None or opening.group() != expected:+ raise ValueError("unmatched VLM inline delimiter")+ latex = text[opening.end() : match.start()].strip()+ if not latex:+ raise ValueError("empty VLM inline formula")+ parts.extend((_unescape_text_dollars(text[cursor : opening.start()]), f"\\({latex}\\)"))+ cursor = match.end()+ opening = None+ if opening is not None:+ raise ValueError("unclosed VLM inline delimiter")+ parts.append(_unescape_text_dollars(text[cursor:]))+ return "".join(parts).strip()+++def finish_inline_recognition(+ batch: InlineRecognitionBatch,+ outputs: list[TextRecognitionResult],+ context: HybridLocalModelContext,+ np_images: list[np.ndarray],+ *,+ page_offset: int,+) -> list[list[dict[str, Any]]]:+ """在本地模型锁内仅对失败片段合批回退,任何二次失败均显式终止。"""+ if len(outputs) != len(batch.crops):+ raise ValueError(f"Inline recognition count mismatch: inputs={len(batch.crops)}, outputs={len(outputs)}")+ fallback_inputs: list[list[dict[str, Any]]] = [[] for _ in batch.formulas]+ failed_crops: list[list[_InlineCrop]] = [[] for _ in batch.formulas]+ for crop, output in zip(batch.crops, outputs, strict=True):+ formula = batch.formulas[crop.page_index][crop.formula_index]+ try:+ if isinstance(output, Exception):+ raise_unrecoverable_text_error(output)+ raise output+ content = normalize_inline_text(output)+ except Exception as error:+ raise_unrecoverable_text_error(error)+ logger.warning(+ "Inline VLM fallback to MFR: page={}, formula_index={}, bbox={}, reason={}: {}",+ page_offset + crop.page_index + 1,+ crop.formula_index,+ formula["bbox"],+ type(error).__name__,+ error,+ )+ fallback_inputs[crop.page_index].append(dict(formula))+ failed_crops[crop.page_index].append(crop)+ else:+ formula["content"] = content+ formula["content_format"] = "text_with_inline_math"+ fallback_count = sum(len(page) for page in failed_crops)+ if fallback_count:+ with stage_timer("pdf.inline_mfr_fallback"):+ fallback_results = context.mfr_model.batch_predict(+ fallback_inputs,+ np_images,+ batch_size=BATCH_RATIO * MFR_BASE_BATCH_SIZE,+ interline_enable=False,+ )+ if len(fallback_results) != len(failed_crops):+ raise ValueError("Inline MFR fallback page count mismatch")+ for crops, results in zip(failed_crops, fallback_results, strict=True):+ if len(crops) != len(results):+ raise ValueError("Inline MFR fallback result count mismatch")+ for crop, result in zip(crops, results, strict=True):+ formula = batch.formulas[crop.page_index][crop.formula_index]+ if result.get("bbox") != formula["bbox"] or result.get("label") != "inline_formula":+ raise ValueError("Inline MFR fallback result identity mismatch")+ latex = result.get("latex")+ if not isinstance(latex, str) or not latex.strip():+ raise ValueError(+ f"Empty inline MFR fallback: page={page_offset + crop.page_index + 1}, bbox={formula['bbox']}"+ )+ formula["latex"] = latex.strip()+ if batch.crops:+ logger.info(+ "Inline recognition: total={}, vlm_success={}, mfr_fallback={}",+ len(batch.crops),+ len(batch.crops) - fallback_count,+ fallback_count,+ )+ return batch.formulas+++__all__ = ["InlineRecognitionBatch", "prepare_inline_recognition", "finish_inline_recognition", "normalize_inline_text"]diff --git a/mineru/backend/analysis/pdf/normalization.py b/mineru/backend/analysis/pdf/normalization.pyindex 13b4a067..0e0d04f8 100644--- a/mineru/backend/analysis/pdf/normalization.py+++ b/mineru/backend/analysis/pdf/normalization.py@@ -98,13 +98,22 @@ def _is_valid_pdf_text_block(block: dict[str, Any]) -> bool:def _natural_language_spans(content: str) -> list[dict[str, Any]]:"""把 PDF 自然语言字符串及圆括号公式定界符直接转换为 Span。"""- pattern = re.compile(r"\\\((?P<round>.*?)\\\)", re.DOTALL)+ # 偶数个前导反斜杠属于内容,后面的奇数位反斜杠才可作为模型公式定界符。+ pattern = re.compile(r"(?<!\\)(?:\\\\)*(?P<delimiter>\\[()])")spans: list[dict[str, Any]] = []cursor = 0+ opening: tuple[int, int] | None = Nonefor match in pattern.finditer(content):- append_text_span(spans, content[cursor : match.start()])- append_equation_span(spans, match.group("round"))+ if match.group("delimiter") == r"\(":+ if opening is None:+ opening = (match.start("delimiter"), match.end())+ continue+ if opening is None:+ continue+ append_text_span(spans, content[cursor : opening[0]])+ append_equation_span(spans, content[opening[1] : match.start("delimiter")])cursor = match.end()+ opening = Noneappend_text_span(spans, content[cursor:])return strip_span_dicts(spans)diff --git a/mineru/backend/analysis/pdf/ocr.py b/mineru/backend/analysis/pdf/ocr.pyindex f3257cc1..7be5b6d8 100644--- a/mineru/backend/analysis/pdf/ocr.py+++ b/mineru/backend/analysis/pdf/ocr.py@@ -54,7 +54,7 @@ def _build_ocr_det_type_and_mfr_enable(if effort == "medium":return PIPELINE_DET_TYPE, Trueif parse_mode == "txt":- return VLM_TXT_DET_TYPE, True+ return VLM_TXT_DET_TYPE, Falsereturn VLM_OCR_DET_TYPE, Falsediff --git a/mineru/backend/analysis/pdf/text/content.py b/mineru/backend/analysis/pdf/text/content.pyindex 7bf2d639..b6469d3a 100644--- a/mineru/backend/analysis/pdf/text/content.py+++ b/mineru/backend/analysis/pdf/text/content.py@@ -121,8 +121,15 @@ def _build_page_text_formula_spans(_AnalyzeSpan(type=ContentType.INLINE_EQUATION,bbox=bbox,- content=str(formula.get("latex", "") or "").strip(),+ content=str(+ formula.get("content", "")+ if formula.get("content_format") == "text_with_inline_math"+ else formula.get("latex", "") or ""+ ).strip(),score=float(formula.get("score", 0.0) or 0.0),+ content_format=(+ "text_with_inline_math" if formula.get("content_format") == "text_with_inline_math" else "default"+ ),))@@ -400,7 +407,7 @@ def _line_content_parts(line: _AnalyzeLine) -> list[tuple[str, str]]:content = str(span.content or "").strip()elif span.type == ContentType.INLINE_EQUATION:latex = str(span.content or "").strip()- content = f"\\({latex}\\)" if latex else ""+ content = latex if span.content_format == "text_with_inline_math" else (f"\\({latex}\\)" if latex else "")else:continueif content:diff --git a/mineru/backend/analysis/pdf/text/models.py b/mineru/backend/analysis/pdf/text/models.pyindex 7ae00765..f4c6e7ce 100644--- a/mineru/backend/analysis/pdf/text/models.py+++ b/mineru/backend/analysis/pdf/text/models.py@@ -4,7 +4,7 @@from __future__ import annotationsfrom dataclasses import dataclass, field-from typing import Any+from typing import Any, Literalfrom .....types import BBox@@ -19,6 +19,7 @@ class _AnalyzeSpan:score: float = 0.0image: Any = Nonemetadata: dict[str, Any] = field(default_factory=dict)+ content_format: Literal["default", "text_with_inline_math"] = "default"@dataclass(slots=True)diff --git a/mineru/backend/analysis/pdf/window.py b/mineru/backend/analysis/pdf/window.pyindex 21c8e845..79417b13 100644--- a/mineru/backend/analysis/pdf/window.py+++ b/mineru/backend/analysis/pdf/window.py@@ -12,6 +12,8 @@ if TYPE_CHECKING:from docvortex.document.pdf import PDFRenderSessionfrom PIL.Image import Imagefrom ....model.vlm.contracts import VlmPredictor+ from ....model.vlm.text_recognition import TextRecognitionResult+ from .inline_recognition import InlineRecognitionBatchimport numpy as npfrom docvortex.assets import image_size as _normalize_page_size@@ -196,6 +198,7 @@ def _process_text_and_formulas(page_vector_geometries: list[PDFPageVectorGeometry | None] | None = None,page_snapshots: PageSnapshotCache | None = None,np_images: list[np.ndarray] | None = None,+ recognized_inline_formulas: list[list[dict[str, Any]]] | None = None,) -> list[list[dict[str, Any]]]:"""在当前窗口内完成 OCR、公式、原生文本及 block 行信息回填。"""from .formulas import (@@ -206,7 +209,7 @@ def _process_text_and_formulas(optimize_hybrid_formula_number_blocks,)from .ocr import _apply_ocr_rec_results, _build_ocr_det_type_and_mfr_enable, _ocr_det- from .tables import _apply_medium_table_recognition, _build_table_external_mfr_inputs+ from .tables import _apply_medium_table_recognitionfrom .text.content import _fill_window_block_content_and_lines, _validate_text_formula_window_inputs@@ -233,13 +236,15 @@ def _process_text_and_formulas(images_formula_list = mfd_resinterline_enable = effort == "medium"- # medium 保留未被原生表格消费的公式;high/xhigh 只识别最终表格外的行内公式。+ # high/xhigh TXT 的 VLM 与局部 MFR 回退已在窗口阶段完成,不能重新识别成功切片。+ if effort in {"high", "xhigh"} and parse_mode == "txt":+ if recognized_inline_formulas is None:+ raise ValueError("High/Xhigh TXT requires window inline recognition results")+ images_formula_list = recognized_inline_formulas++ # 只有 medium 在此阶段运行完整 MFR;OCR 遮罩始终使用原始 mfd_res。if mfr_enable:- mfr_inputs = (- mfd_res- if effort == "medium"- else _build_table_external_mfr_inputs(mfd_res, model_list, [image.size for image in images_pil_list])- )+ mfr_inputs = mfd_res# 全部被过滤时保持逐页空结果,不能让原始表内公式重新进入正文 sidecar。images_formula_list = [[] for _ in mfd_res]if any(mfr_inputs):@@ -329,12 +334,20 @@ class _WindowInputs:page_text_geometries: list[PDFPageTextGeometry | None] | Nonepage_vector_geometries: list[PDFPageVectorGeometry | None] | None = Nonepage_snapshots: PageSnapshotCache | None = None+ inline_recognition: InlineRecognitionBatch | None = None+ inline_outputs: list[TextRecognitionResult] | None = Nonedef close(self) -> None:"""推理及回填真正退出后关闭图片,仅清除本层拥有的容器。"""try:- _close_images(self.images_list)+ try:+ if self.inline_recognition is not None:+ self.inline_recognition.close()+ finally:+ _close_images(self.images_list)finally:+ self.inline_outputs = None+ self.inline_recognition = Noneself.images_list.clear()self.images_pil_list.clear()self.np_images.clear()@@ -483,38 +496,63 @@ def _prepare_pdf_window(table_items.clear()-def _finish_pdf_window(+def _prepare_window_result(state: _WindowInputs,window_model_list: list[Any],*,effort: AnalyzeEffort,parse_mode: Literal["txt", "ocr"],+) -> list[list[dict[str, Any]]]:+ """先确定最终版面与原生表格归属,再准备表外行内切片。"""+ from .inline_recognition import prepare_inline_recognition+ from .tables import _restore_native_high_table_blocks++ if parse_mode == "txt" and effort == "high":+ window_model_list = _restore_native_high_table_blocks(window_model_list, state.accepted_native_tables)+ if effort in {"high", "xhigh"}:+ window_model_list = _convert_vlm_results_to_model_list(window_model_list)+ if effort == "xhigh":+ _normalize_xhigh_vlm_blocks(window_model_list)+ _apply_layout_title_split(+ window_model_list,+ state.images_layout_res,+ [_normalize_page_size(image) for image in state.images_pil_list],+ )+ if parse_mode == "txt" and effort in {"high", "xhigh"}:+ state.inline_recognition = prepare_inline_recognition(+ state.images_layout_res, window_model_list, state.images_pil_list+ )+ state.inline_outputs = []+ return window_model_list+++def _finish_pdf_window(+ state: _WindowInputs,+ window_model_list: list[list[dict[str, Any]]],+ *,+ effort: AnalyzeEffort,+ parse_mode: Literal["txt", "ocr"],hybrid_model: HybridLocalModelContext,) -> list[list[dict[str, Any]]]:- """按原有顺序回填推理结果、文本公式与视觉素材。"""+ """在本地模型锁内完成失败片段回退、正文回填与视觉素材。"""from .formulas import optimize_hybrid_formula_number_blocks+ from .inline_recognition import finish_inline_recognitionfrom .ocr import _apply_seal_ocr- from .tables import _restore_native_high_table_blocksimages_list = state.images_listwindow_pages = state.window_pages- images_pil_list = state.images_pil_listnp_images = state.np_imagesimages_layout_res = state.images_layout_resvl_style_layout_blocks = state.vl_style_layout_blockspage_text_geometries = state.page_text_geometrieswindow = state.windowlocal_model_context = hybrid_model- if parse_mode == "txt" and effort == "high":- window_model_list = _restore_native_high_table_blocks(window_model_list, state.accepted_native_tables)- if effort in {"high", "xhigh"}:- window_model_list = _convert_vlm_results_to_model_list(window_model_list)- if effort == "xhigh":- _normalize_xhigh_vlm_blocks(window_model_list)- _apply_layout_title_split(- window_model_list,- images_layout_res,- [_normalize_page_size(image) for image in images_pil_list],+ inline_options: dict[str, list[list[dict[str, Any]]]] = {}+ if state.inline_recognition is not None:+ if state.inline_outputs is None:+ raise ValueError("Missing window inline recognition outputs")+ inline_options["recognized_inline_formulas"] = finish_inline_recognition(+ state.inline_recognition, state.inline_outputs, hybrid_model, np_images, page_offset=window.start)if effort == "flash":@@ -542,6 +580,7 @@ def _finish_pdf_window(page_vector_geometries=state.page_vector_geometries,**({"page_snapshots": state.page_snapshots} if state.page_snapshots is not None else {}),np_images=np_images,+ **inline_options,)if effort in {"medium", "high"}:@@ -612,6 +651,10 @@ def _process_pdf_window(result = vlm_predictor.batch_extract_with_layout(**options)else:result = vlm_predictor.batch_two_step_extract(**options)+ result = _prepare_window_result(state, result, effort=effort, parse_mode=parse_mode)+ if state.inline_recognition is not None and state.inline_recognition.crops:+ with stage_timer("pdf.inline_vlm"):+ state.inline_outputs = vlm_predictor.batch_recognize_text(state.inline_recognition.images)with local_model_stage(hybrid_model.device):return _finish_pdf_window(state, result, effort=effort, parse_mode=parse_mode, hybrid_model=hybrid_model)finally:@@ -719,6 +762,10 @@ async def aio_process_pdf_windows(result = await vlm_predictor.aio_batch_extract_with_layout(**options)else:result = await vlm_predictor.aio_batch_two_step_extract(**options)+ result = await run_sync(_prepare_window_result, state, result, effort=effort, parse_mode=parse_mode)+ if state.inline_recognition is not None and state.inline_recognition.crops:+ with stage_timer("pdf.inline_vlm"):+ state.inline_outputs = await vlm_predictor.aio_batch_recognize_text(state.inline_recognition.images)model_list.extend(await run_sync(_finish_locked_window,diff --git a/mineru/model/vlm/async_runtime.py b/mineru/model/vlm/async_runtime.pyindex 38cfe49e..eabcdd62 100644--- a/mineru/model/vlm/async_runtime.py+++ b/mineru/model/vlm/async_runtime.py@@ -13,6 +13,7 @@ from ...utils.async_utils import drain_future, run_syncfrom ..runtime import ownershipfrom .errors import EngineDeadError, is_fatal_engine_errorfrom .handles import RuntimeHandle+from .text_recognition import TextRecognitionResult, raise_unrecoverable_text_error, validate_text_resultif TYPE_CHECKING:from mineru_vl_utils.structs import ContentBlock, ExtractResult@@ -193,6 +194,40 @@ class AsyncVlmPredictor:passraise+ async def _recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """逐请求隔离识别异常,跨文档共享并发额度,取消后排空所有切片。"""+ async def recognize(image: Image) -> TextRecognitionResult:+ """单片失败不会取消同批成功请求,程序错误和引擎故障仍向上传播。"""+ try:+ output = await self._predictor.aio_content_extract(image, type="text", semaphore=self._semaphore)+ except Exception as error:+ raise_unrecoverable_text_error(error)+ return error+ return validate_text_result(output)++ tasks = [asyncio.create_task(recognize(image)) for image in images]+ pending = set(tasks)+ try:+ # wait 不会把父任务取消隐式转发给子任务,避免二次 cancel 打断清理。+ while pending:+ done, pending = await asyncio.wait(pending, return_when=asyncio.FIRST_COMPLETED)+ for task in done:+ task.result()+ return [task.result() for task in tasks]+ except BaseException:+ for task in pending:+ task.cancel()+ await drain_future(asyncio.gather(*tasks, return_exceptions=True))+ raise++ def batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """同步调用方通过常驻循环复用原生异步切片识别。"""+ return self._call(lambda: self._recognize_text(images))++ async def aio_batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """异步切片识别沿用运行时取消确认和关闭屏障。"""+ return await self._acall(lambda: self._recognize_text(images))+async def _extract_with_layout(self,images: list[Image],diff --git a/mineru/model/vlm/contracts.py b/mineru/model/vlm/contracts.pyindex b9797b7d..c2fd72e9 100644--- a/mineru/model/vlm/contracts.py+++ b/mineru/model/vlm/contracts.py@@ -8,11 +8,20 @@ from collections.abc import Sequenceif TYPE_CHECKING:from PIL.Image import Imagefrom mineru_vl_utils.structs import ContentBlock, ExtractResult+ from .text_recognition import TextRecognitionResultclass VlmPredictor(Protocol):"""同步客户端与原生异步代理共同实现的最小文档抽取接口。"""+ def batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """按输入顺序返回切片文本或可局部回退的异常。"""+ ...++ async def aio_batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """异步切片识别复用文档推理的并发、取消和资源生命周期。"""+ ...+def batch_extract_with_layout(self,images: list[Image],diff --git a/mineru/model/vlm/handles.py b/mineru/model/vlm/handles.pyindex 0dfbc3b7..df8a196f 100644--- a/mineru/model/vlm/handles.py+++ b/mineru/model/vlm/handles.py@@ -12,6 +12,7 @@ from typing import TYPE_CHECKINGfrom ...utils.async_utils import drain_future, run_syncfrom ..runtime.closing import runtime_close_scopefrom ..runtime.ownership import RuntimeOwner+from .text_recognition import TextRecognitionResult, recognize_text_batchif TYPE_CHECKING:from mineru_vl_utils import MinerUClient@@ -91,6 +92,16 @@ class RuntimeHandle:if lock is not None:lock.release()+ def batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """切片合批及必要的单片补试共享同一次运行时租约与执行锁。"""+ with self._use() as predictor:+ return recognize_text_batch(predictor, images)++ async def aio_batch_recognize_text(self, images: list[Image]) -> list[TextRecognitionResult]:+ """同步引擎在线程内识别切片,取消时等待推理退出再归还图片。"""+ async with self._async_use() as predictor:+ return await run_sync(recognize_text_batch, predictor, images)+def batch_extract_with_layout(self,images: list[Image],diff --git a/mineru/model/vlm/text_recognition.py b/mineru/model/vlm/text_recognition.pynew file mode 100644index 00000000..8fd58bd1--- /dev/null+++ b/mineru/model/vlm/text_recognition.py@@ -0,0 +1,63 @@+"""行内切片文本识别的结果契约与局部失败隔离。"""++from __future__ import annotations++from typing import TYPE_CHECKING, TypeAlias++from mineru_vl_utils.vlm_client.base_client import ClientClosedError, ServerError++from .errors import is_fatal_engine_error++if TYPE_CHECKING:+ from mineru_vl_utils import MinerUClient+ from PIL.Image import Image++TextRecognitionResult: TypeAlias = str | None | Exception+++def raise_unrecoverable_text_error(error: Exception) -> None:+ """关闭、引擎死亡及程序契约错误必须传播,不能降级成单片识别失败。"""+ if (+ is_fatal_engine_error(error)+ or isinstance(+ error,+ (ClientClosedError, MemoryError, AssertionError, AttributeError, TypeError, LookupError, NotImplementedError),+ )+ or (isinstance(error, RuntimeError) and not isinstance(error, ServerError))+ ):+ raise error+++def validate_text_result(value: object) -> str | None:+ """保留合法空响应供宿主回退,拒绝无法保持输入输出契约的非文本结果。"""+ if value is None:+ return None+ if not isinstance(value, str):+ raise TypeError(f"VLM text recognition returned {type(value).__name__}, expected str or None")+ return str(value)+++def recognize_text_batch(client: MinerUClient, images: list[Image]) -> list[TextRecognitionResult]:+ """正常路径保留本地合批;可恢复批次异常时逐片补试一次,保留成功结果。"""+ if not images:+ return []+ try:+ outputs = client.batch_content_extract(images, types="text")+ except Exception as error:+ raise_unrecoverable_text_error(error)+ results: list[TextRecognitionResult] = []+ for image in images:+ try:+ output = client.content_extract(image, type="text")+ except Exception as item_error:+ raise_unrecoverable_text_error(item_error)+ results.append(item_error)+ else:+ results.append(validate_text_result(output))+ return results+ if len(outputs) != len(images):+ raise ValueError(f"VLM text recognition count mismatch: inputs={len(images)}, outputs={len(outputs)}")+ return [validate_text_result(output) for output in outputs]+++__all__ = ["TextRecognitionResult", "raise_unrecoverable_text_error", "recognize_text_batch", "validate_text_result"]diff --git a/tests/unittest/test_native_pdf_table_pipeline.py b/tests/unittest/test_native_pdf_table_pipeline.pyindex ce88ddd0..6e4a2231 100644--- a/tests/unittest/test_native_pdf_table_pipeline.py+++ b/tests/unittest/test_native_pdf_table_pipeline.py@@ -551,6 +551,7 @@ def test_high_txt_window_excludes_native_table_from_vlm(*,page_vector_geometries: object,np_images: object,+ recognized_inline_formulas: object = None,) -> list[list[dict[str, object]]]:"""跳过与本测试无关的正文和公式回填。"""diff --git a/tests/unittest/test_pdf_analyze_block_content.py b/tests/unittest/test_pdf_analyze_block_content.pyindex 3396089b..bdad1803 100644--- a/tests/unittest/test_pdf_analyze_block_content.py+++ b/tests/unittest/test_pdf_analyze_block_content.py@@ -186,12 +186,14 @@ def test_doc_analyze_converts_vlm_results_before_downstream_processing(*,page_vector_geometries: object,np_images: object,+ recognized_inline_formulas: object = None,) -> list[list[dict[str, object]]]:"""原样返回窗口结果,并在后处理入口校验精确容器类型。"""assert type(window_model_list) is listassert all(type(page) is list for page in window_model_list)assert all(type(block) is dict for page in window_model_list for block in page)if parse_mode == "txt":+ assert recognized_inline_formulas == [[]]assert isinstance(_page_text_geometries, list)assert len(_page_text_geometries) == 1else:diff --git a/tests/unittest/test_pdf_inline_vlm.py b/tests/unittest/test_pdf_inline_vlm.pynew file mode 100644index 00000000..7c43de24--- /dev/null+++ b/tests/unittest/test_pdf_inline_vlm.py@@ -0,0 +1,224 @@+"""验证 VLM 行内切片、混合文本保护和单片 MFR 回退。"""++from __future__ import annotations++from copy import deepcopy+from io import BytesIO+from types import SimpleNamespace+from typing import Any+from unittest.mock import MagicMock++import numpy as np+import pytest+from docvortex.document.pdf import PDFDocument+from PIL import Image+from reportlab.pdfgen.canvas import Canvas++from mineru.backend.analysis.pdf.inline_recognition import (+ finish_inline_recognition,+ normalize_inline_text,+ prepare_inline_recognition,+)+from mineru.backend.analysis.pdf.normalization import _natural_language_spans+from mineru.backend.analysis.pdf.text.content import (+ _apply_window_post_ocr,+ _build_page_text_formula_spans,+ _fill_native_pdf_text_spans,+ _lines_to_block_content,+)+from mineru.backend.analysis.pdf.text.lines import group_spans_to_lines+from mineru.model.vlm.errors import EngineDeadError+from mineru.integrations.docvortex import build_metadata+from mineru.parser.base import ParseResult+from mineru.render import render_html+from mineru.types import MiddleJson, PageInfo, Producer, TextBlock+++ ("source", "expected"),+ [+ ("α ≤ 5", "α ≤ 5"),+ (r"前 \(x_1\),后 \[y^2\]。", r"前 \(x_1\),后 \(y^2\)。"),+ (r"\( (m_1, m_n) \) . (D)", r"\((m_1, m_n)\) . (D)"),+ (r"US$5 and $10", r"US$5 and $10"),+ (r"\$4.48 ± \$1.02", "$4.48 ± $1.02"),+ (r"price \$4.48; \(x+\$1\); \[y\] then \$1.02", r"price $4.48; \(x+\$1\); \(y\) then $1.02"),+ (r"literal \\$5 and \\\$6", r"literal \\$5 and \\$6"),+ (r"C:\data\file.txt \alpha \% \_ \$5", r"C:\data\file.txt \alpha \% \_ $5"),+ (r"\(\text{\$4.48}\) and \$1.02", r"\(\text{\$4.48}\) and $1.02"),+ (r"literal \\(x\\) and \(y\)", r"literal \\(x\\) and \(y\)"),+ ("(p =", "(p ="),+ (r"\((p =\)", r"\((p =\)"),+ ],+)+def test_inline_text_normalization(source: str, expected: str) -> None:+ """仅处理模型公式定界符,正文括号、Unicode、美元及转义文字不猜测补全。"""+ assert normalize_inline_text(source) == expected++[email protected]("source", [None, "", " ", "[Non-Text]", r"\(\)", r"\(x", r"x\)", r"\[x\)", r"\(a\(b\)\)"])+def test_invalid_inline_response_requests_fallback(source: str | None) -> None:+ """空结果及残缺定界符必须触发回退,不能在遮罩后静默丢字。"""+ with pytest.raises(ValueError):+ normalize_inline_text(source)+++def test_semantic_formula_split_respects_backslash_parity() -> None:+ """转义定界符保留文字,公式边界前的成对反斜杠保留在原内容中。"""+ assert _natural_language_spans(r"literal \\(x\\) and \(y\)") == [+ {"type": "text", "content": r"literal \\(x\\) and "},+ {"type": "equation_inline", "content": "y"},+ ]+ assert _natural_language_spans(r"\\\(x\\\)") == [+ {"type": "text", "content": r"\\"},+ {"type": "equation_inline", "content": "x\\\\"},+ ]+++def test_vlm_currency_roundtrip_keeps_semantic_text_and_renderer_escaping_separate() -> None:+ """真实回填与序列化后金额为纯文本,Markdown 只转义一次,HTML 不泄漏反斜杠。"""+ raw = r"\$4.48 ± \$1.02"+ with Image.new("RGB", (100, 30)) as image:+ batch = prepare_inline_recognition([[_formula([1, 1, 99, 29])]], [[]], [image])+ try:+ formulas = finish_inline_recognition(batch, [raw], SimpleNamespace(), [], page_offset=0)+ spans = _build_page_text_formula_spans(formulas[0], [], (100, 30), 1.0)+ content = _lines_to_block_content(group_spans_to_lines(spans), "text")+ finally:+ batch.close()+ semantic = _natural_language_spans(content)+ assert semantic == [{"type": "text", "content": "$4.48 ± $1.02"}]+ middle = MiddleJson(+ pages=[PageInfo(page_idx=0, blocks=[TextBlock(type="text", index=0, bbox=(0, 0, 1, 1), content=semantic)])],+ is_full_document=True,+ metadata={"file_suffix": "pdf", "producer": Producer(name="mineru", version="test")},+ extensions=build_metadata(effort="high", parse_mode="txt"),+ )+ result = ParseResult.from_json(ParseResult(middle).to_json())+ assert result.pages[0].blocks[0].content[0].content == "$4.48 ± $1.02"+ assert result.markdown() == raw+ rendered = render_html(result.middle_json)+ assert "$4.48 ± $1.02" in rendered+ assert r"\$" not in rendered+++def test_currency_unescaping_does_not_change_native_text_or_mfr_latex() -> None:+ """修复限定在 VLM 文本入口,原生文本与 MFR 公式的美元转义保持原义。"""+ assert _natural_language_spans(r"literal \$5") == [{"type": "text", "content": r"literal \$5"}]+ spans = _build_page_text_formula_spans([{**_formula([1, 1, 20, 10]), "latex": r"\$5"}], [], (100, 30), 1.0)+ content = _lines_to_block_content(group_spans_to_lines(spans), "text")+ assert _natural_language_spans(content) == [{"type": "equation_inline", "content": r"\$5"}]+++def _formula(bbox: list[float]) -> dict[str, Any]:+ """构造可追踪的原始检测框。"""+ return {"label": "inline_formula", "bbox": bbox, "score": 0.9}+++def test_crop_selection_and_mixed_results_preserve_original_masks() -> None:+ """以最终表格过滤切片,保持 MFR 裁边规则且不改原始检测数据。"""+ layouts = [[_formula([1.2, 2.8, 9.1, 8.2]), _formula([30, 30, 40, 40]), _formula([200, 0, 210, 10])], []]+ before = deepcopy(layouts)+ with Image.new("RGB", (100, 100)) as image:+ batch = prepare_inline_recognition(layouts, [[{"type": "table", "bbox": [0.2, 0.2, 0.5, 0.5]}], []], [image, image])+ try:+ assert len(batch.crops) == 1+ assert batch.images[0].size == (9, 7)+ result = finish_inline_recognition(batch, [r"\(x\) . (D)"], SimpleNamespace(), [], page_offset=0)+ assert result[0][0]["content"] == r"\(x\) . (D)"+ assert result[0][0]["content_format"] == "text_with_inline_math"+ assert result[1] == []+ assert layouts == before+ crop = batch.images[0]+ finally:+ batch.close()+ with pytest.raises(ValueError, match="closed image"):+ crop.getpixel((0, 0))+++def test_only_failed_crops_reach_mfr_in_page_order() -> None:+ """只把不同页的失败切片交给 MFR,识别相同文本也不能错配几何落点。"""+ layout = [[_formula([1, 1, 10, 10]), _formula([12, 1, 20, 10])], [_formula([1, 1, 10, 10])]]+ with Image.new("RGB", (40, 40)) as image:+ batch = prepare_inline_recognition(layout, [[], []], [image, image])+ mfr = MagicMock(return_value=[[{**layout[0][1], "latex": "x_2"}], [{**layout[1][0], "latex": "x_3"}]])+ try:+ result = finish_inline_recognition(+ batch,+ ["α", None, ValueError("request failed")],+ SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr)),+ [np.zeros((40, 40, 3))] * 2,+ page_offset=64,+ )+ inputs = mfr.call_args.args[0]+ assert [[f["bbox"] for f in page] for page in inputs] == [[layout[0][1]["bbox"]], [layout[1][0]["bbox"]]]+ assert mfr.call_args.kwargs["interline_enable"] is False+ assert result[0][0]["content"] == "α"+ assert result[0][1]["latex"] == "x_2"+ assert result[1][0]["latex"] == "x_3"+ assert "content_format" not in result[0][1]+ finally:+ batch.close()++[email protected]("failure", ["empty", "count", "identity", "engine", "vlm_count"])+def test_fallback_failure_and_contract_errors_are_not_swallowed(failure: str) -> None:+ """MFR 二次失败、返回错配及引擎死亡必须显式报错。"""+ formula = _formula([1, 1, 10, 10])+ with Image.new("RGB", (20, 20)) as image:+ batch = prepare_inline_recognition([[formula]], [[]], [image])+ answer = {**formula, "latex": "" if failure == "empty" else "x"}+ if failure == "identity":+ answer["bbox"] = [2, 2, 3, 3]+ mfr = MagicMock(return_value=[] if failure == "count" else [[answer]])+ outputs = [] if failure == "vlm_count" else [EngineDeadError("dead") if failure == "engine" else None]+ try:+ with pytest.raises((ValueError, EngineDeadError)):+ finish_inline_recognition(+ batch, outputs, SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr)), [], page_offset=0+ )+ if failure in {"engine", "vlm_count"}:+ mfr.assert_not_called()+ finally:+ batch.close()++[email protected]("recognized", ["α ≤ 5", r"\(m_n\) . (D)", r"literal \\(x\\) and \(y\)"])+def test_native_and_post_ocr_cannot_overwrite_vlm_text(recognized: str) -> None:+ """用真实 PDF 原生字符验证几何占位保护,最终才拆出没有几何字段的语义 span。"""+ stream = BytesIO()+ canvas = Canvas(stream, pagesize=(200, 80))+ canvas.setFont("Helvetica", 12)+ canvas.drawString(10, 40, "before WRONG after")+ canvas.save()+ with PDFDocument(stream.getvalue()) as document, Image.new("RGB", (200, 80)) as image:+ page = document[0]+ chars = [char for char in page.get_chars() if char["char"] not in "\r\n"]++ def bbox(selected: list[dict[str, Any]]) -> list[float]:+ """由真实字符框计算三段互不重叠的检测位置。"""+ return [+ min(c["bbox"][0] for c in selected),+ min(c["bbox"][1] for c in selected),+ max(c["bbox"][2] for c in selected),+ max(c["bbox"][3] for c in selected),+ ]++ formula = {**_formula(bbox(chars[7:12])), "content": recognized, "content_format": "text_with_inline_math"}+ ocr = [{"bbox": bbox(chars[:6]), "text": ""}, {"bbox": bbox(chars[13:]), "text": ""}]+ spans = _build_page_text_formula_spans([formula], ocr, page.size, 1.0)+ spans = _fill_native_pdf_text_spans(page, spans, image, 1.0, page.size)+ lines = group_spans_to_lines(spans)+ model = SimpleNamespace(ocr_model=SimpleNamespace(ocr=MagicMock()))+ _apply_window_post_ocr(model, [{0: lines}])+ model.ocr_model.ocr.assert_not_called()+ content = _lines_to_block_content(lines, "text")+ assert "WRONG" not in content+ assert content == f"before {recognized} after"+ semantic = _natural_language_spans(content)+ assert all(set(span) <= {"type", "content"} for span in semantic)+ if recognized == "α ≤ 5":+ assert semantic == [{"type": "text", "content": "before α ≤ 5 after"}]+ else:+ assert sum(span["type"] == "equation_inline" for span in semantic) == 1+ assert semantic[-1]["content"].endswith(" after")diff --git a/tests/unittest/test_pdf_inline_vlm_window.py b/tests/unittest/test_pdf_inline_vlm_window.pynew file mode 100644index 00000000..3269530f--- /dev/null+++ b/tests/unittest/test_pdf_inline_vlm_window.py@@ -0,0 +1,133 @@+"""验证真实窗口编排的最终表格归属、模型锁和切片生命周期。"""++from __future__ import annotations++import asyncio+from collections.abc import Iterator+from contextlib import contextmanager+from copy import deepcopy+from types import SimpleNamespace+from typing import Any+from unittest.mock import MagicMock++import numpy as np+import pytest+from PIL import Image++from mineru.backend.analysis.pdf import ocr, window++[email protected]("effort", ["high", "xhigh"])[email protected]("is_async", [False, True])[email protected]("fallback", [False, True])+def test_window_inline_inference_runs_outside_local_lock(+ monkeypatch: pytest.MonkeyPatch, effort: str, is_async: bool, fallback: bool+) -> None:+ """同步与异步入口在最终版面过滤后识别,仅失败片段在本地模型锁内回退。"""+ locked = []+ observed_crops = []+ observed_results = []+ layouts = [[{"label": "inline_formula", "bbox": box} for box in [[50, 50, 60, 60], [10, 10, 15, 15], [20, 10, 25, 15]]]]+ # 本地 table 故意覆盖外侧公式,实际归属必须由最终 VLM 表格决定。+ layouts[0].append({"label": "table", "bbox": [0, 0, 30, 30]})+ final_blocks = [[{"type": "text", "bbox": [0, 0, 0.3, 0.3]}, {"type": "table", "bbox": [0.4, 0.4, 0.8, 0.8]}]]++ @contextmanager+ def stage(_device: str) -> Iterator[None]:+ """显式观察本地模型锁,不依赖线程重入锁的实现细节。"""+ assert not locked+ locked.append(True)+ try:+ yield+ finally:+ locked.pop()++ def prepare(*args: Any, **kwargs: Any) -> window._WindowInputs:+ """只替代渲染与 layout,继续使用真实窗口转换、推理和资源关闭。"""+ assert locked+ picture = Image.new("RGB", (100, 100))+ return window._WindowInputs(+ window._ProcessingWindow(0, 1, 0, 0),+ [{"img_pil": picture, "scale": 1}],+ [object()],+ [picture],+ [np.asarray(picture).copy()],+ deepcopy(layouts),+ [],+ [],+ [[]],+ None,+ )++ def main_infer(**kwargs: Any) -> list[list[dict[str, Any]]]:+ """主 VLM 也必须在本地模型锁之外执行。"""+ assert not locked+ return deepcopy(final_blocks)++ def inline_infer(images: list[Image.Image]) -> list[str | None]:+ """最终表格内公式不得进入 VLM,并保存切片引用核对关闭时机。"""+ assert not locked+ assert len(images) == 2+ observed_crops.extend(images)+ assert all(image.getpixel((0, 0)) == (0, 0, 0) for image in images)+ return ["α", None if fallback else r"\(x\) . (D)"]++ async def async_main(**kwargs: Any) -> list[list[dict[str, Any]]]:+ """异步主推理复用同一版面及锁断言。"""+ return main_infer(**kwargs)++ async def async_inline(images: list[Image.Image]) -> list[str | None]:+ """让真实异步窗口经过一次等待,再验证图片仍可读取。"""+ await asyncio.sleep(0)+ return inline_infer(images)++ def mfr_predict(inputs: list[list[dict[str, Any]]], *args: Any, **kwargs: Any) -> list[list[dict[str, Any]]]:+ """回退必须持有本地模型锁,且只接收第二个表外切片。"""+ assert locked+ assert [item["bbox"] for item in inputs[0]] == [[20, 10, 25, 15]]+ return [[{**item, "latex": "x"} for item in inputs[0]]]++ def fill(*args: Any, **kwargs: Any) -> list[list[dict[str, Any]]]:+ """正文阶段观察按原顺序合并后的 VLM 与 MFR 结果。"""+ assert locked+ observed_results.extend(kwargs["recognized_inline_formulas"][0])+ return args[2]++ model = SimpleNamespace(device="cpu")+ if fallback:+ model.mfr_model = SimpleNamespace(batch_predict=mfr_predict)+ predictor = SimpleNamespace(+ batch_extract_with_layout=main_infer,+ batch_two_step_extract=main_infer,+ aio_batch_extract_with_layout=async_main,+ aio_batch_two_step_extract=async_main,+ batch_recognize_text=inline_infer,+ aio_batch_recognize_text=async_inline,+ )+ monkeypatch.setattr(window, "local_model_stage", stage)+ monkeypatch.setattr(window, "_prepare_pdf_window", prepare)+ monkeypatch.setattr(window, "_process_text_and_formulas", fill)+ monkeypatch.setattr(window, "_attach_visual_block_images", MagicMock())+ monkeypatch.setattr(window, "_supplement_missing_image_block_containers", MagicMock())+ monkeypatch.setattr(window, "get_document_render_session", lambda document: None)+ monkeypatch.setattr(ocr, "_apply_seal_ocr", MagicMock())+ options = {+ "effort": effort,+ "parse_mode": "txt",+ "image_analysis": False,+ "hybrid_model": model,+ "vlm_predictor": predictor,+ }+ if is_async:+ result = asyncio.run(window.aio_process_pdf_windows(b"", SimpleNamespace(page_count=1), **options))+ else:+ result = window._process_pdf_window(+ b"", SimpleNamespace(page_count=1), window._ProcessingWindow(0, 1, 0, 0), page_count=1, **options+ )+ assert len(result) == 1+ assert observed_results[0]["content"] == "α"+ assert observed_results[1].get("latex") == ("x" if fallback else "")+ assert [item["bbox"] for item in observed_results] == [[10, 10, 15, 15], [20, 10, 25, 15]]+ for image in observed_crops:+ with pytest.raises(ValueError, match="closed image"):+ image.getpixel((0, 0))diff --git a/tests/unittest/test_pdf_memory_lifecycle.py b/tests/unittest/test_pdf_memory_lifecycle.pyindex 21321032..8def5dee 100644--- a/tests/unittest/test_pdf_memory_lifecycle.py+++ b/tests/unittest/test_pdf_memory_lifecycle.py@@ -353,6 +353,7 @@ def test_async_window_keeps_64_page_boundary_and_order(# 此处只验证窗口顺序,图片及文档均为替身,不启动真实渲染会话。monkeypatch.setattr(window, "get_document_render_session", lambda _document: None)monkeypatch.setattr(window, "_prepare_locked_window", prepare)+ monkeypatch.setattr(window, "_prepare_window_result", lambda state, result, **kwargs: result)monkeypatch.setattr(window, "_finish_locked_window", lambda state, result, **kwargs: result)result = asyncio.run(window.aio_process_pdf_windows(diff --git a/tests/unittest/test_pdf_mfr_table_routing.py b/tests/unittest/test_pdf_mfr_table_routing.pyindex de2b10a2..22ddd6ad 100644--- a/tests/unittest/test_pdf_mfr_table_routing.py+++ b/tests/unittest/test_pdf_mfr_table_routing.py@@ -14,6 +14,7 @@ from PIL import Imagefrom mineru.backend.analysis.pdf import windowfrom mineru.backend.analysis.pdf.formulas import _build_formula_inputs+from mineru.backend.analysis.pdf.inline_recognition import finish_inline_recognition, prepare_inline_recognitionfrom mineru.backend.analysis.pdf.tables import _build_table_external_mfr_inputsfrom mineru.backend.analysis.pdf import ocr as stage_ocr@@ -85,7 +86,7 @@ def test_window_mfr_routing_and_original_ocr_masks(blocks = [[{"type": "table", "bbox": [0.1, 0.1, 0.9, 0.9], "content": content}]]original = deepcopy(layout)mfr = MagicMock(side_effect=lambda inputs, *_args, **_kwargs: deepcopy(inputs))- enabled = effort == "medium" or parse_mode == "txt"+ enabled = effort == "medium"context = SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr)) if enabled else SimpleNamespace()det = MagicMock(return_value=[[]])fill = MagicMock(return_value=blocks)@@ -107,16 +108,16 @@ def test_window_mfr_routing_and_original_ocr_masks(effort,context,layout,+ recognized_inline_formulas=[[layout[0][1]]] if effort != "medium" and parse_mode == "txt" else None,)if enabled:- expected = _build_formula_inputs(layout) if effort == "medium" else [[layout[0][1]]]- assert mfr.call_args.args[0] == expected+ assert mfr.call_args.args[0] == _build_formula_inputs(layout)assert mfr.call_args.kwargs["interline_enable"] is (effort == "medium")else:mfr.assert_not_called()assert det.call_args.args[3] == _build_formula_inputs(layout)assert layout == original- expected_inline = [layout[0][:2]] if effort == "medium" or not enabled else [[layout[0][1]]]+ expected_inline = [layout[0][:2]] if effort == "medium" or parse_mode == "ocr" else [[layout[0][1]]]assert fill.call_args.args[3] == expected_inline@@ -134,12 +135,15 @@ def test_final_layout_controls_filter_and_empty_results_skip_model_loading(layout = [[formula, {"label": "table", "bbox": local_table_bbox}], [_formula([10, 10, 20, 20], "display_formula")]]blocks = [[{"type": "table", "bbox": [0.1, 0.1, 0.5, 0.5]}] if final_table else [], []]mfr = MagicMock(side_effect=lambda inputs, *_args, **_kwargs: deepcopy(inputs))- context = SimpleNamespace() if final_table else SimpleNamespace(mfr_model=SimpleNamespace(batch_predict=mfr))+ context = SimpleNamespace()fill = MagicMock(return_value=blocks)det = MagicMock(return_value=[[], []])monkeypatch.setattr(stage_ocr, "_ocr_det", det)monkeypatch.setattr(stage_text_content, "_fill_window_block_content_and_lines", fill)with Image.new("RGB", (100, 100)) as image:+ batch = prepare_inline_recognition(layout, blocks, [image, image])+ recognized = finish_inline_recognition(batch, ["α"] * len(batch.crops), context, [], page_offset=0)+ batch.close()window._process_text_and_formulas([{"img_pil": image, "scale": 1.0}] * 2,[MagicMock()] * 2,@@ -148,10 +152,13 @@ def test_final_layout_controls_filter_and_empty_results_skip_model_loading(effort,context,layout,+ recognized_inline_formulas=recognized,)- assert fill.call_args.args[3] == ([[], []] if final_table else [[formula], []])+ assert fill.call_args.args[3] == recognized+ expected = [[], []] if final_table else [[{**formula, "content": "α", "content_format": "text_with_inline_math"}], []]+ assert recognized == expectedassert det.call_args.args[3] == _build_formula_inputs(layout)- assert mfr.call_count == (0 if final_table else 1)+ mfr.assert_not_called()def test_demo1_frozen_mfr_inputs_exclude_only_29_table_formulas() -> None:diff --git a/tests/unittest/test_vlm_remote.py b/tests/unittest/test_vlm_remote.pyindex 29252f53..05b818bb 100644--- a/tests/unittest/test_vlm_remote.py+++ b/tests/unittest/test_vlm_remote.py@@ -117,6 +117,29 @@ def _settings(server: _OpenAIServer, **overrides: Any) -> VlmConfig:return VlmConfig.model_validate({"server_url": server.url, "api_key": server.api_key, **overrides})[email protected]("is_async", [False, True])+def test_remote_inline_text_uses_default_prompt_and_preserves_failures(+ openai_server: _OpenAIServer, is_async: bool+) -> None:+ """真实 HTTP 适配器用文本提示识别切片,并将请求失败交给宿主按片回退。"""+ predictor, _ = get_vlm_predictor(_settings(openai_server))+ with Image.new("RGB", (32, 16), "white") as image:+ if is_async:+ result = asyncio.run(predictor.aio_batch_recognize_text([image, image]))+ else:+ result = predictor.batch_recognize_text([image, image])+ assert result == ["Remote VLM text", "Remote VLM text"]+ bodies = [body for _, _, body in openai_server.requests if body is not None]+ assert len(bodies) == 2+ for body in bodies:+ content = body["messages"][-1]["content"]+ assert [part["text"] for part in content if part["type"] == "text"] == ["\nText Recognition:"]+ openai_server.chat_status = 400+ failures = predictor.batch_recognize_text([image, image])+ assert len(failures) == 2+ assert all(isinstance(item, Exception) for item in failures)++def test_remote_client_protocol_and_cache(openai_server: _OpenAIServer, monkeypatch: pytest.MonkeyPatch) -> None:"""真实 HTTP 验证模型发现、显式模型、认证、代理路径、参数及缓存隔离。"""from mineru.model.vlm import runtime, selector@@ -263,6 +286,7 @@ def hybrid_stub(monkeypatch: pytest.MonkeyPatch) -> None:page_vector_geometries: object,np_images: object,page_snapshots: object = None,+ recognized_inline_formulas: object = None,) -> list[list[dict[str, Any]]]:"""保留远程正文并模拟 Hybrid 提供的行框,满足 PDF 文本块校验合同。"""for page in blocks:diff --git a/tests/unittest/test_vlm_text_recognition.py b/tests/unittest/test_vlm_text_recognition.pynew file mode 100644index 00000000..cea764b5--- /dev/null+++ b/tests/unittest/test_vlm_text_recognition.py@@ -0,0 +1,173 @@+"""验证行内 VLM 文本识别的错误隔离、共享并发和取消屏障。"""++from __future__ import annotations++import asyncio+import threading+from concurrent.futures import ThreadPoolExecutor+from types import SimpleNamespace+from typing import Any+from unittest.mock import MagicMock++import pytest+from mineru_vl_utils.vlm_client.base_client import ClientClosedError, RequestError++from mineru.model.vlm.async_runtime import AsyncVlmPredictor+from mineru.model.vlm.errors import EngineDeadError+from mineru.model.vlm.handles import RuntimeHandle+from mineru.model.vlm.text_recognition import recognize_text_batch+++def test_local_batch_keeps_successes_and_only_retries_after_batch_failure() -> None:+ """本地正常合批不拆单;批次失败后只逐片补试一次并保留异常落点。"""+ failure = RequestError("bad crop")+ client = SimpleNamespace(batch_content_extract=MagicMock(return_value=["α", None]), content_extract=MagicMock())+ assert recognize_text_batch(client, [1, 2]) == ["α", None]+ client.content_extract.assert_not_called()+ client.batch_content_extract.assert_called_once_with([1, 2], types="text")+ client.batch_content_extract.side_effect = RequestError("batch failed")+ client.content_extract.side_effect = ["success", failure, None]+ assert recognize_text_batch(client, [1, 2, 3]) == ["success", failure, None]+ assert client.content_extract.call_count == 3+ assert [call.kwargs for call in client.content_extract.call_args_list] == [{"type": "text"}] * 3+++ "error", [EngineDeadError("dead"), ClientClosedError("closed"), RuntimeError("engine stopped"), TypeError("bug")]+)+def test_local_fatal_errors_never_retry(error: Exception) -> None:+ """引擎失效、关闭及程序错误不得重试后伪装成 MFR 回退。"""+ client = SimpleNamespace(batch_content_extract=MagicMock(side_effect=error), content_extract=MagicMock())+ with pytest.raises(type(error)):+ recognize_text_batch(client, [1])+ client.content_extract.assert_not_called()++[email protected]("outputs", [[], ["x", "y"], [object()]])+def test_local_output_contract_is_strict(outputs: list[Any]) -> None:+ """长度和类型错配立即报错,不能触发掩盖适配器缺陷的逐片补试。"""+ client = SimpleNamespace(batch_content_extract=MagicMock(return_value=outputs), content_extract=MagicMock())+ with pytest.raises((ValueError, TypeError)):+ recognize_text_batch(client, [1])+ client.content_extract.assert_not_called()+++class _TextClient:+ """异步文本替身在真实运行时循环里使用共享信号量并延迟清理。"""++ def __init__(self) -> None:+ """记录请求和清理事件,便于验证多个文档及不同入口共享运行时。"""+ self.loop = asyncio.get_running_loop()+ self.active = 0+ self.peak = 0+ self.block = False+ self.entered = threading.Event()+ self.closed = False++ async def aio_content_extract(self, image: int, *, type: str, semaphore: asyncio.Semaphore) -> str:+ """指定切片失败仍释放请求额度,取消时模拟异步资源清理。"""+ assert asyncio.get_running_loop() is self.loop+ assert type == "text"+ async with semaphore:+ self.active += 1+ self.peak = max(self.peak, self.active)+ self.entered.set()+ try:+ await asyncio.sleep(60 if self.block else 0.01)+ if image == -1:+ raise RequestError("bad crop")+ if image == -2:+ raise EngineDeadError("dead")+ return str(image)+ finally:+ await asyncio.sleep(0.03)+ self.active -= 1++ async def aclose(self) -> None:+ """只有所有切片完成清理之后才允许关闭客户端。"""+ assert self.active == 0+ self.closed = True+++def text_runtime() -> Any:+ """创建原生异步宿主并保证测试结束后关闭线程及客户端。"""+ runtime = AsyncVlmPredictor(+ lambda: RuntimeHandle(_TextClient(), backend="http-client"), backend="http-client", max_concurrency=2+ )+ runtime.wait_ready()+ try:+ yield runtime+ finally:+ runtime.shutdown()+++def test_native_text_results_keep_order_errors_and_global_limit(text_runtime: AsyncVlmPredictor) -> None:+ """跨线程同步及异步入口共享额度,单片失败不会取消其他切片。"""++ def invoke(index: int) -> list[Any]:+ """交错同步和异步调用,模拟多文档同时提交。"""+ if index % 2:+ return asyncio.run(text_runtime.aio_batch_recognize_text([index, -1]))+ return text_runtime.batch_recognize_text([index, -1])++ with ThreadPoolExecutor(max_workers=4) as pool:+ results = list(pool.map(invoke, range(4)))+ assert [result[0] for result in results] == ["0", "1", "2", "3"]+ assert all(isinstance(result[1], RequestError) for result in results)+ assert text_runtime._predictor.peak == 2+ assert text_runtime._predictor.active == 0+++def test_native_repeated_cancellation_drains_all_crops(text_runtime: AsyncVlmPredictor) -> None:+ """重复取消不得使调用方在任何切片仍读取图片时退出。"""+ client = text_runtime._predictor+ client.block = True++ async def run() -> None:+ """发出两次取消并等待跨循环的清理确认。"""+ task = asyncio.create_task(text_runtime.aio_batch_recognize_text([1, 2, 3]))+ assert await asyncio.to_thread(client.entered.wait, 2)+ task.cancel()+ await asyncio.sleep(0.005)+ task.cancel()+ assert not task.done()+ with pytest.raises(asyncio.CancelledError):+ await task+ assert client.active == 0++ asyncio.run(run())+ client.block = False+ assert text_runtime.batch_recognize_text([4]) == ["4"]+++def test_native_engine_failure_drains_other_requests(text_runtime: AsyncVlmPredictor) -> None:+ """引擎死亡向上传播,同时等待同批其他请求清理。"""+ with pytest.raises(EngineDeadError):+ text_runtime.batch_recognize_text([-2, 1, 2])+ assert text_runtime._predictor.active == 0+++def test_local_shutdown_waits_for_text_request() -> None:+ """本地切片识别在途时关闭必须等待,关闭后拒绝新请求。"""+ entered, release, closed = threading.Event(), threading.Event(), threading.Event()++ def infer(images: list[Any], *, types: str) -> list[str]:+ """模拟持有图片的同步推理,显式释放前不得关闭引擎。"""+ entered.set()+ assert release.wait(2)+ assert not closed.is_set()+ return ["x"]++ handle = RuntimeHandle(SimpleNamespace(batch_content_extract=infer), backend="test", close_engine=closed.set)+ with ThreadPoolExecutor(max_workers=2) as pool:+ work = pool.submit(handle.batch_recognize_text, [1])+ assert entered.wait(2)+ shutting_down = pool.submit(handle.shutdown)+ assert not closed.wait(0.02)+ release.set()+ assert work.result() == ["x"]+ shutting_down.result()+ assert closed.is_set()+ with pytest.raises(RuntimeError, match="shutting down"):+ handle.batch_recognize_text([1])