Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix(table): preserve atomic image and formula tokens
myhlolidb468264f476
diff --git a/mineru/backend/analysis/pdf/tables.py b/mineru/backend/analysis/pdf/tables.pyindex 07eb663d..8532c6ca 100644--- a/mineru/backend/analysis/pdf/tables.py+++ b/mineru/backend/analysis/pdf/tables.py@@ -797,6 +797,7 @@ def _collect_medium_table_tasks("table_img": rotated_crop,"wired_table_img": rotated_crop,"ocr_result": [],+ "atomic_token_indices": frozenset(),"table_res": {},"inline_objects": prepared_objects,}@@ -804,15 +805,17 @@ def _collect_medium_table_tasks(return table_tasks-def _sort_medium_table_ocr_result(ocr_result: list[list[Any]]) -> None:- """按 token 顶边和左边坐标排序,保证表格 OCR 输入顺序稳定。"""+def _sort_medium_table_ocr_result(ocr_result: list[list[Any]], token_kinds: list[str]) -> frozenset[int]:+ """同步排序 OCR token 和对象类型,返回不可拆分对象在三元组列表中的索引。"""- def sort_key(item: list[Any]) -> tuple[float, float]:+ def sort_key(item: tuple[list[Any], str]) -> tuple[float, float]:"""提取四点框的最小 y/x,兼容普通列表和 numpy 数组。"""- box = np.asarray(item[0], dtype=np.float32).reshape(-1, 2)+ box = np.asarray(item[0][0], dtype=np.float32).reshape(-1, 2)return float(np.min(box[:, 1])), float(np.min(box[:, 0]))- ocr_result.sort(key=sort_key)+ sorted_items = sorted(zip(ocr_result, token_kinds, strict=True), key=sort_key)+ ocr_result[:] = [token for token, _ in sorted_items]+ return frozenset(index for index, (_, kind) in enumerate(sorted_items) if kind in {"image", "formula"})def _prepare_medium_table_ocr_results(@@ -834,6 +837,7 @@ def _prepare_medium_table_ocr_results(for table_task in table_tasks:ocr_result: list[list[Any]] = []+ token_kinds: list[str] = []bgr_image = rgb_to_bgr(table_task["table_img"])mask_boxes = [{"bbox": item["mask_bbox"]} for item in table_task["inline_objects"]]masked_image = mask_formula_regions_for_ocr_det(bgr_image, mask_boxes)@@ -859,6 +863,7 @@ def _prepare_medium_table_ocr_results(float(score or 0.0),])+ token_kinds.append("text")except Exception as exc:logger.warning(f"Hybrid medium table OCR failed: page_idx={table_task['page_idx']}, error={exc}")@@ -872,7 +877,8 @@ def _prepare_medium_table_ocr_results(inline_object["score"],])- _sort_medium_table_ocr_result(ocr_result)+ token_kinds.append(inline_object["kind"])+ table_task["atomic_token_indices"] = _sort_medium_table_ocr_result(ocr_result, token_kinds)table_task["ocr_result"] = ocr_result@@ -929,6 +935,7 @@ def _apply_medium_table_recognition(table_task["wired_table_img"],table_task["ocr_result"],wireless_html,+ atomic_token_indices=table_task["atomic_token_indices"],)if wired_html is not None:table_task["table_res"]["html"] = wired_htmldiff --git a/mineru/model/table/rec/unet_table/main.py b/mineru/model/table/rec/unet_table/main.pyindex bc7b4c89..aaf90c23 100644--- a/mineru/model/table/rec/unet_table/main.py+++ b/mineru/model/table/rec/unet_table/main.py@@ -4,7 +4,7 @@ import loggingimport timeimport tracebackfrom dataclasses import asdict, dataclass-from typing import Any, Dict, List, Optional, Union+from typing import Any, Dict, List, Optionalimport numpy as npfrom bs4 import BeautifulSoup@@ -66,9 +66,12 @@ class WiredTableRecognition:def __call__(self,img: InputType,- ocr_result: Optional[List[Union[List[List[float]], str, str]]] = None,- **kwargs,+ ocr_result: list[list[Any]] | None = None,+ *,+ atomic_token_indices: frozenset[int] = frozenset(),+ **kwargs: Any,) -> WiredTableOutput:+ """恢复有线表格结构,并在匹配时保持指定图片和公式 token 完整。"""s = time.perf_counter()need_ocr = Truecol_threshold = 15@@ -98,7 +101,9 @@ class WiredTableRecognition:logi_points[idx_list],time.perf_counter() - s,)- cell_box_det_map, not_match_orc_boxes = match_ocr_cell(ocr_result, polygons)+ cell_box_det_map, not_match_orc_boxes = match_ocr_cell(+ ocr_result, polygons, atomic_token_indices=atomic_token_indices+ )# 如果有识别框没有ocr结果,直接进行rec补充cell_box_det_map = self.fill_blank_rec(img, polygons, cell_box_det_map)# 转换为中间格式,修正识别框坐标,将物理识别框,逻辑识别框,ocr识别框整合为dict,方便后续处理@@ -269,7 +274,16 @@ class UnetTableModel:self.wired_table_model = WiredTableRecognition(wired_input_args, ocr_engine)self.ocr_engine = ocr_engine- def predict(self, input_img, ocr_result, wireless_html_code, return_metadata: bool = False):+ def predict(+ self,+ input_img: Image.Image | np.ndarray,+ ocr_result: list[list[Any]] | None,+ wireless_html_code: str,+ return_metadata: bool = False,+ *,+ atomic_token_indices: frozenset[int] = frozenset(),+ ) -> str | dict[str, Any]:+ """传递不可拆分对象索引,并按现有规则选择有线或无线表格结果。"""if isinstance(input_img, Image.Image):np_img = np.asarray(input_img)elif isinstance(input_img, np.ndarray):@@ -287,7 +301,7 @@ class UnetTableModel:]try:- wired_table_results = self.wired_table_model(np_img, ocr_result)+ wired_table_results = self.wired_table_model(np_img, ocr_result, atomic_token_indices=atomic_token_indices)wired_structure_results = self.wired_table_model(np_img, need_ocr=False) if return_metadata else None# viser = VisTable()diff --git a/mineru/model/table/rec/unet_table/utils_table_recover.py b/mineru/model/table/rec/unet_table/utils_table_recover.pyindex af3fb6ee..95651b0f 100644--- a/mineru/model/table/rec/unet_table/utils_table_recover.py+++ b/mineru/model/table/rec/unet_table/utils_table_recover.py@@ -419,8 +419,51 @@ def _select_clear_best_cell(return None-def match_ocr_cell(dt_rec_boxes: List[List[Union[Any, str]]], pred_bboxes: np.ndarray):+def _select_atomic_cell(ocr_box: np.ndarray, pred_boxes: np.ndarray, coverage_scores: np.ndarray) -> int | None:+ """按中心包含、覆盖率和最近距离,为图片或公式确定唯一的有效单元格。"""+ valid_indices = np.flatnonzero(+ np.isfinite(pred_boxes).all(axis=1) & (pred_boxes[:, 2] > pred_boxes[:, 0]) & (pred_boxes[:, 3] > pred_boxes[:, 1])+ )+ if not len(valid_indices):+ return None++ boxes = pred_boxes[valid_indices]+ center = (ocr_box[:2] + ocr_box[2:]) / 2+ # 两轴均采用左闭右开区间,公共边界稳定归到右侧或下侧单元格。+ candidates = ((boxes[:, :2] <= center) & (center < boxes[:, 2:])).all(axis=1)+ primary_scores = -coverage_scores[valid_indices]+ if not candidates.any():+ candidates = coverage_scores[valid_indices] > 0+ if not candidates.any():+ candidates = np.ones(len(boxes), dtype=bool)+ distances = np.maximum(np.maximum(boxes[:, :2] - center, center - boxes[:, 2:]), 0)+ primary_scores = np.sum(distances**2, axis=1)+ center_distances = np.sum(((boxes[:, :2] + boxes[:, 2:]) / 2 - center) ** 2, axis=1)++ def rank(position: int) -> tuple[float, float, float, float, float, float, int]:+ """用中心距离、几何坐标和原始索引消除平局,避免归属依赖遍历顺序。"""+ return (+ float(primary_scores[position]),+ float(center_distances[position]),+ float(boxes[position, 0]),+ float(boxes[position, 1]),+ float(boxes[position, 2]),+ float(boxes[position, 3]),+ int(valid_indices[position]),+ )++ best_position = min(np.flatnonzero(candidates).tolist(), key=rank)+ return int(valid_indices[best_position])+++def match_ocr_cell(+ dt_rec_boxes: list[list[Any]],+ pred_bboxes: np.ndarray,+ *,+ atomic_token_indices: frozenset[int] = frozenset(),+) -> tuple[dict[int, list[list[Any]]], list[list[Any]]]:"""+ 将 OCR 匹配到单元格;标记的图片和公式整体归属,其余文字保留跨格拆分。:param dt_rec_boxes: [[(4.2), text, score]]:param pred_bboxes: shap (4,2):return:@@ -494,6 +537,14 @@ def match_ocr_cell(dt_rec_boxes: List[List[Union[Any, str]]], pred_bboxes: np.ndmatched = {}not_match_orc_boxes = []for i, gt_box in enumerate(dt_rec_boxes):+ if i in atomic_token_indices:+ cell_idx = _select_atomic_cell(ocr_boxes[i], pred_boxes, coverage[i])+ if cell_idx is not None:+ matched.setdefault(cell_idx, []).append(gt_box)+ else:+ not_match_orc_boxes.append(gt_box)+ continue+projection_indices = np.flatnonzero(projection_mask[i])if len(projection_indices) > 1:min_segment_width = _min_projection_segment_width(str(gt_box[1]), ocr_boxes[i])diff --git a/tests/unittest/test_table_atomic_tokens.py b/tests/unittest/test_table_atomic_tokens.pynew file mode 100644index 00000000..86a508d5--- /dev/null+++ b/tests/unittest/test_table_atomic_tokens.py@@ -0,0 +1,255 @@+"""验证表内图片与公式在跨格、旋转和导出过程中完整且唯一归属。"""++from __future__ import annotations++import base64+import html+import io+import zipfile+from pathlib import Path+from types import SimpleNamespace+from typing import Any+from unittest.mock import Mock++import numpy as np+import pytest+from bs4 import BeautifulSoup+from docvortex.assets import parse_image_data_uri_strict+from PIL import Image++from mineru.backend.analysis.pdf import tables+from mineru.model.runtime.contracts import AtomicModelName+from mineru.model.table.rec.slanet_plus.main import PaddleTable+from mineru.model.table.rec.unet_table import main as unet+from mineru.model.table.rec.unet_table.utils_table_recover import match_ocr_cell+from mineru.parser.base import ParseResult+from mineru.parser.task_outputs import build_task_output+from mineru.parser.writer import FileBasedDataWriter+from mineru.types import BlockType, MiddleJson++Rect = tuple[float, float, float, float]+TWO_CELLS = [(0, 0, 100, 50), (100, 0, 200, 50)]+FOUR_CELLS = [*TWO_CELLS, (0, 50, 100, 100), (100, 50, 200, 100)]+LATEX = r"\frac{1}{2} \times 36 = 18"+++def _quad(rect: Rect) -> list[list[float]]:+ """把测试矩形转换为现有匹配器使用的顺时针四点框。"""+ x0, y0, x1, y1 = rect+ return [[x0, y0], [x1, y0], [x1, y1], [x0, y1]]++[email protected](params=["image", "formula"])+def atomic_content(request: pytest.FixtureRequest) -> str:+ """提供真实 JPEG token 与包含反斜杠命令的公式 token。"""+ if request.param == "formula":+ return f"<eq>{LATEX}</eq>"+ buffer = io.BytesIO()+ with Image.new("RGB", (64, 64), (200, 30, 30)) as image:+ image.save(buffer, "JPEG")+ return '<img src="data:image/jpeg;base64,' + base64.b64encode(buffer.getvalue()).decode() + '"/>'+++ ("bbox", "cells", "expected"),+ [+ pytest.param((20, 10, 40, 30), TWO_CELLS, 0, id="inside"),+ pytest.param((95, 20, 105, 30), TWO_CELLS, 1, id="shared-column-edge"),+ pytest.param((95, 45, 105, 55), FOUR_CELLS, 3, id="four-cell-corner"),+ pytest.param((10, 10, 290, 30), [*TWO_CELLS, (200, 0, 300, 50)], 1, id="three-columns"),+ pytest.param((95, 20, 105, 30), [(0, 0, 100, 50), (100, 0, 300, 50)], 1, id="unequal-cells"),+ pytest.param((0, 0, 100, 20), [(0, 0, 45, 20), (48, 0, 52, 20)], 1, id="center-before-coverage"),+ pytest.param((0, 0, 100, 10), [(0, 0, 100, 1000), (40, 0, 60, 10)], 0, id="coverage-before-distance"),+ pytest.param((60, 10, 120, 30), [(0, 0, 100, 50), (80, 0, 200, 50)], 0, id="overlap-center-tie"),+ pytest.param((95, 20, 105, 30), [(0, 0, 200, 50), (50, 0, 150, 50)], 0, id="geometry-tie"),+ pytest.param((95, 20, 105, 30), [(0, 0, 200, 50)] * 2, 0, id="identical-cells"),+ pytest.param((60, 10, 150, 30), [(0, 0, 80, 50), (120, 0, 200, 50)], 1, id="gap-most-overlap"),+ pytest.param((95, 20, 105, 30), [(0, 0, 90, 50), (110, 0, 200, 50)], 0, id="gap-no-overlap-tie"),+ pytest.param((-30, 90, -10, 110), [(0, 0, 200, 200), (5, 90, 15, 110)], 0, id="nearest-rectangle"),+ pytest.param((-30, 90, -10, 110), [(0, 0, 200, 200), (0, 90, 10, 110)], 1, id="nearest-center-tie"),+ pytest.param(+ (95, 20, 105, 30),+ [(100, 0, 100, 50), (0, 0, float("inf"), 50), (float("nan"), 0, 200, 50), (0, 0, 200, 50)],+ 3,+ id="ignore-invalid-cells",+ ),+ ],+)+def test_atomic_token_has_one_complete_owner(atomic_content: str, bbox: Rect, cells: list[Rect], expected: int) -> None:+ """所有几何分支只接收原始 token 一次,且预测框排列不影响非重复格的归属。"""+ token = [_quad(bbox), atomic_content, 0.83]+ for order in [list(range(len(cells))), list(reversed(range(len(cells))))]:+ polygons = np.asarray([_quad(cells[index]) for index in order], dtype=float)+ matched, unmatched = match_ocr_cell([token], polygons, atomic_token_indices=frozenset({0}))+ assert unmatched == []+ assert len(matched) == 1+ cell_idx, values = next(iter(matched.items()))+ assert cells[order[cell_idx]] == cells[expected]+ if len(set(cells)) == 1:+ assert cell_idx == 0+ assert len(values) == 1 and values[0] is token+ assert token[0] == _quad(bbox) and token[1:] == [atomic_content, 0.83]+++def test_atomic_token_without_valid_cells_stays_unmatched(atomic_content: str) -> None:+ """无有效结构时不向退化格分配对象,并保留空结构的原有返回语义。"""+ token = [_quad((95, 20, 105, 30)), atomic_content, 1.0]+ invalid = np.asarray([_quad((100, 0, 100, 50)), _quad((200, 0, 100, 50))], dtype=float)+ matched, unmatched = match_ocr_cell([token], invalid, atomic_token_indices=frozenset({0}))+ assert matched == {} and len(unmatched) == 1 and unmatched[0] is token+ assert match_ocr_cell([token], np.empty((0, 4, 2)), atomic_token_indices=frozenset({0})) == ({}, [])+++def test_identical_object_payloads_are_not_deduplicated(atomic_content: str) -> None:+ """内容相同的两个对象仍分别保留,不能用文本或对象集合去重。"""+ tokens = [[_quad((95, y, 105, y + 10)), atomic_content, 1.0] for y in (10, 30)]+ matched, unmatched = match_ocr_cell(+ tokens, np.asarray([_quad(cell) for cell in TWO_CELLS]), atomic_token_indices=frozenset({0, 1})+ )+ assert unmatched == [] and len(matched[1]) == 2+ assert all(actual is expected for actual, expected in zip(matched[1], tokens, strict=True))++[email protected]("text", ["123 456", "1,234.56 7,890.12", "2026-01-01 2026-02-02"])+def test_plain_text_keeps_cross_cell_splitting(text: str) -> None:+ """未标记文字继续按普通规则拆分,标记行为由索引而非 HTML 内容决定。"""+ token = [_quad((60, 15, 140, 35)), text, 1.0]+ polygons = np.asarray([_quad(cell) for cell in TWO_CELLS])+ matched, unmatched = match_ocr_cell([token], polygons)+ assert unmatched == []+ assert [matched[index][0][1] for index in (0, 1)] == text.split()+ atomic_matched, _ = match_ocr_cell([token], polygons, atomic_token_indices=frozenset({0}))+ assert list(atomic_matched) == [1] and atomic_matched[1][0] is token+++def _result_for_table(markup: str) -> ParseResult:+ """构造包含正常页和实际表格 HTML 的两页结果,贯通完整文件导出。"""+ pages = []+ for index, content in enumerate(["<table><tr><td>normal page</td></tr></table>", markup]):+ pages.append(+ {+ "page_idx": index,+ "blocks": [+ {+ "type": "table",+ "index": 0,+ "bbox": [0.0, 0.0, 1.0, 1.0],+ "content": [{"type": "table_body", "index": 0, "bbox": [0.0, 0.0, 1.0, 1.0], "content": content}],+ }+ ],+ }+ )+ return ParseResult(+ MiddleJson.from_dict(+ {+ "schema": "docvortex.middle",+ "schema_version": "2.0",+ "is_full_document": True,+ "metadata": {"file_suffix": "pdf", "producer": {"name": "mineru", "version": "test"}},+ "extensions": {"mineru": {"tier": "basic", "parse_mode": "ocr"}},+ "pages": pages,+ }+ )+ )++[email protected](("angle", "image_cell", "formula_cell"), [(0, 1, 3), (90, 2, 3), (180, 3, 1), (270, 3, 2)])+def test_atomic_objects_survive_table_pipeline_and_exports(+ monkeypatch: pytest.MonkeyPatch, tmp_path: Path, angle: int, image_cell: int, formula_cell: int+) -> None:+ """只替换模型预测,真实执行对象构建、旋转、有线匹配、HTML 拼接及两种导出。"""+ page = np.full((100, 200, 3), 255, dtype=np.uint8)+ page[10:40, 80:120] = [200, 30, 30]+ table = {"type": BlockType.TABLE, "bbox": [0.0, 0.0, 1.0, 1.0], "angle": angle}+ blocks = [[table, {"type": BlockType.IMAGE, "bbox": [0.4, 0.1, 0.6, 0.4]}]]+ formulas = [[{"bbox": [0.3, 0.6, 0.7, 0.8], "latex": LATEX, "score": 0.93}]]+ height, width = (200, 100) if angle in (90, 270) else (100, 200)+ polygons = np.asarray(+ [+ _quad((column * width / 2, row * height / 2, (column + 1) * width / 2, (row + 1) * height / 2))+ for row in range(2)+ for column in range(2)+ ]+ )+ # 有线结构模型返回逆时针框;真实 TableRecover 和后续方向转换继续执行。+ structure = Mock(return_value=(polygons[:, [0, 3, 2, 1]].copy(), polygons[:, [0, 3, 2, 1]].copy()))+ monkeypatch.setattr(unet, "TSRUnet", Mock(return_value=structure))++ def recognize(image: np.ndarray | list[np.ndarray], *, det: bool = True) -> list[Any]:+ """构造故意乱序的普通 OCR 结果,空格补识别返回空文本。"""+ if not det:+ return [[("", 0.0) for _ in image]]+ return [+ [+ [_quad((5, height - 10, 20, height - 5)), ("tail", 0.9)],+ [_quad((5, 2, 20, 8)), ("<eq>literal</eq>", 0.9)],+ ]+ ]++ ocr = SimpleNamespace(ocr=Mock(side_effect=recognize))+ captured: list[dict[str, Any]] = []++ def classify(tasks: list[dict[str, Any]]) -> None:+ """固定选择有线路径,同时检查模型输入未被新增结构遮罩改变。"""+ for task in tasks:+ task["table_res"].update(cls_label=AtomicModelName.WiredTable, cls_score=1.0)+ assert np.any(task["wired_table_img"] != 255)+ np.testing.assert_array_equal(task["wired_table_img"], task["table_img"])++ def wireless_predict(tasks: list[dict[str, Any]]) -> None:+ """用真实无线输入适配器验证三元组契约,并保留有线识别的验收输入。"""+ for task in tasks:+ dt_boxes, rec_res = PaddleTable.get_boxes_recs(None, task["ocr_result"], height, width)+ assert len(dt_boxes) == len(rec_res) == 4+ assert all(len(token) == 3 for token in task["ocr_result"])+ captured.append(task)+ task["table_res"]["html"] = ""++ context = SimpleNamespace(+ get_ocr_model=Mock(return_value=ocr),+ table_cls_model=SimpleNamespace(batch_predict=classify),+ wireless_table_model=SimpleNamespace(batch_predict=wireless_predict),+ wired_table_model=unet.UnetTableModel(ocr, model_path="unused"),+ )+ tables._apply_medium_table_recognition(context, blocks, formulas, [page])+ task = captured[0]+ tokens = task["ocr_result"]+ atomic_tokens = [tokens[index] for index in sorted(task["atomic_token_indices"])]+ assert len(atomic_tokens) == 2+ assert {token[1] for token in atomic_tokens} == {item["content"] for item in task["inline_objects"]}+ assert tokens[0][1] == html.escape("<eq>literal</eq>") and 0 not in task["atomic_token_indices"]+ coordinates = [(min(point[1] for point in token[0]), min(point[0] for point in token[0])) for token in tokens]+ assert coordinates == sorted(coordinates)+ for inline_object in task["inline_objects"]:+ x0, y0, x1, y1 = map(int, inline_object["mask_bbox"])+ assert np.all(ocr.ocr.call_args_list[0].args[0][y0:y1, x0:x1] == 255)+ assert "atomic_token_indices" not in structure.call_args.kwargs++ markup = table["content"]+ soup = BeautifulSoup(markup, "html.parser")+ cells = soup.find_all("td")+ assert len(cells) == 4 and len(soup.find_all("img")) == len(soup.find_all("eq")) == 1+ assert cells[image_cell].find("img") is not None+ assert cells[formula_cell].find("eq").get_text() == LATEX+ original_src = BeautifulSoup(+ next(item["content"] for item in task["inline_objects"] if item["kind"] == "image"), "html.parser"+ ).img["src"]+ expected_bytes, _ = parse_image_data_uri_strict(original_src)+ assert soup.img["src"] == original_src+ assert all(markup.count(token[1]) == 1 for token in atomic_tokens)++ result = _result_for_table(markup)+ before = result.middle_json.to_json()+ directory = tmp_path / "sdk"+ result.save(FileBasedDataWriter(str(directory)))+ archive_path = build_task_output(result, ["default"], tmp_path / "task")+ with zipfile.ZipFile(archive_path) as archive:+ entries = {name: archive.read(name) for name in archive.namelist() if not name.endswith("/")}+ assert entries == {str(path.relative_to(directory)): path.read_bytes() for path in directory.rglob("*") if path.is_file()}+ image_entries = [value for name, value in entries.items() if name.startswith("images/")]+ assert image_entries == [expected_bytes]+ saved = ParseResult.from_json(entries["middle_json.json"].decode())+ saved_soup = BeautifulSoup(saved.pages[1].blocks[0].content[0].content, "html.parser")+ assert len(saved.pages) == 2 and saved_soup.eq.get_text() == LATEX+ assert len(saved_soup.find_all("img")) == 1 and entries[saved_soup.img["src"]] == expected_bytes+ assert result.middle_json.to_json() == before