RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
feat(MinerU):Add support for MinerU table and image chunk classification (#16703)
Aeovyb27623a0c2b7
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.pyindex cd69cc1770..71f688f1d0 100644--- a/deepdoc/parser/mineru_parser.py+++ b/deepdoc/parser/mineru_parser.py@@ -831,13 +831,21 @@ class MinerUParser(RAGFlowPdfParser):def _transfer_to_sections(self, outputs: list[dict[str, Any]], parse_method: str = None, table_enable: bool = False):sections = []+ parse_method = (parse_method or "raw").lower()for output in outputs:- match output.get("type"):+ output_type = output.get("type")+ # These chunkers consume tables and images separately, so exclude+ # media from their text sections. Raw consumers keep legacy sections.+ if parse_method in {"naive", "manual", "paper"} and (output_type == MinerUContentType.IMAGE or (output_type == MinerUContentType.TABLE and table_enable)):+ continue++ match output_type:case MinerUContentType.TEXT:section = output.get("text", "")case MinerUContentType.TABLE:section = output.get("table_body", "") + "\n".join(output.get("table_caption", [])) + "\n".join(output.get("table_footnote", []))if not section.strip():+ self.logger.warning("[MinerU] Empty table content at page_idx=%s; using fallback text.", output.get("page_idx"))section = "FAILED TO PARSE TABLE"case MinerUContentType.IMAGE:section = "".join(output.get("image_caption", [])) + "\n" + "".join(output.get("image_footnote", []))@@ -859,7 +867,9 @@ class MinerUParser(RAGFlowPdfParser):self.logger.debug("[MinerU] Skip unsupported section type=%s", output.get("type"))continue- if not table_enable:+ # Only flatten table HTML when table extraction is disabled; the+ # same sanitization would corrupt valid text, code, and equations.+ if output_type == MinerUContentType.TABLE and not table_enable:section = self._sanitize_section_text(section)if not section:self.logger.debug("[MinerU] Skip section after sanitization: type=%s", output.get("type"))@@ -873,8 +883,52 @@ class MinerUParser(RAGFlowPdfParser):sections.append((section, self._line_tag(output)))return sections- def _transfer_to_tables(self, outputs: list[dict[str, Any]]):- return []+ def _transfer_to_tables(self, outputs: list[dict[str, Any]], table_enable: bool = True):+ """Convert MinerU media blocks to RAGFlow table/image tuples.++ Args:+ outputs: MinerU content-list blocks in source order.+ table_enable: Whether table blocks should be emitted.+ """+ tables = []+ for output in outputs:+ output_type = output.get("type")+ if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE}:+ continue+ if output_type == MinerUContentType.TABLE and not table_enable:+ continue++ position_tag = self._line_tag(output) if "page_idx" in output and "bbox" in output else ""+ positions = [(page, left, right, top, bottom) for pages, left, right, top, bottom in self.extract_positions(position_tag) for page in pages]++ if output_type == MinerUContentType.TABLE:+ text = output.get("table_body", "") + "\n".join(output.get("table_caption", [])) + "\n".join(output.get("table_footnote", []))+ if not text.strip():+ self.logger.warning("[MinerU] Empty table content at page_idx=%s; using fallback text.", output.get("page_idx"))+ text = "FAILED TO PARSE TABLE"+ tables.append(((None, text), positions))+ continue++ texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]+ vlm_description = (output.get("vlm_description") or "").strip()+ if vlm_description:+ texts.append(vlm_description)++ image = None+ image_path = output.get("img_path")+ if image_path:+ try:+ with Image.open(image_path) as source:+ source.load()+ image = source.copy()+ except Exception as e:+ self.logger.warning(f"[MinerU] Failed to load image '{image_path}': {e}")+ if image is None:+ self.logger.warning("[MinerU] Skip image without a readable resource: %s", image_path)+ continue++ tables.append(((image, texts or [""]), positions))+ return tablesdef _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"):"""Generate semantic descriptions for image blocks via the tenant's@@ -1002,7 +1056,9 @@ class MinerUParser(RAGFlowPdfParser):except Exception as e:self.logger.warning(f"[MinerU] VLM image enhancement failed: {e}. Continuing without descriptions.")- return self._transfer_to_sections(outputs, parse_method, enable_table), self._transfer_to_tables(outputs)+ sections = self._transfer_to_sections(outputs, parse_method, enable_table)+ tables = self._transfer_to_tables(outputs, enable_table) if (parse_method or "raw").lower() in {"naive", "manual", "paper"} else []+ return sections, tablesfinally:if temp_pdf and temp_pdf.exists():try:diff --git a/rag/app/manual.py b/rag/app/manual.pyindex 50b614b033..1ada2131a6 100644--- a/rag/app/manual.py+++ b/rag/app/manual.py@@ -240,10 +240,11 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=sec_ids.append(sid)sections = [(txt, sec_ids[i], poss) for i, (txt, _, poss) in enumerate(sections)]- for (img, rows), poss in tbls:- if not rows:- continue- sections.append((rows if isinstance(rows, str) else rows[0], -1, [(p[0] + 1 - from_page, p[1], p[2], p[3], p[4]) for p in poss]))+ if name != "mineru":+ for (img, rows), poss in tbls:+ if not rows:+ continue+ sections.append((rows if isinstance(rows, str) else rows[0], -1, [(p[0] + 1 - from_page, p[1], p[2], p[3], p[4]) for p in poss]))def tag(pn, left, right, top, bottom):if pn + left + right + top + bottom == 0:@@ -264,13 +265,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=tk_cnt = num_tokens_from_string(txt)if sec_id > -1:last_sid = sec_id- tbls = vision_figure_parser_pdf_wrapper(- tbls=tbls,- sections=sections,- callback=callback,- lang=lang,- **kwargs,- )+ if name != "mineru":+ tbls = vision_figure_parser_pdf_wrapper(+ tbls=tbls,+ sections=sections,+ callback=callback,+ lang=lang,+ **kwargs,+ )res = tokenize_table(tbls, doc, eng, language=lang)res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang))table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0))diff --git a/rag/app/naive.py b/rag/app/naive.pyindex aac579b4f4..7479b2dc32 100644--- a/rag/app/naive.py+++ b/rag/app/naive.py@@ -1068,6 +1068,8 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=name = layout_recognizer.strip().lower()parser = PARSERS.get(name, by_plaintext)callback(0.1, "Start to parse.")+ if name == "mineru":+ kwargs["parse_method"] = "naive"sections, tables, pdf_parser = parser(filename=filename,diff --git a/rag/app/paper.py b/rag/app/paper.pyindex 19aeac46fa..c0ffb29032 100644--- a/rag/app/paper.py+++ b/rag/app/paper.py@@ -180,13 +180,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=paper = {"title": filename, "authors": " ", "abstract": "", "sections": sections, "tables": tables}tbls = paper["tables"]- tbls = vision_figure_parser_pdf_wrapper(- tbls=tbls,- sections=sections,- callback=callback,- lang=lang,- **kwargs,- )+ if name != "mineru":+ tbls = vision_figure_parser_pdf_wrapper(+ tbls=tbls,+ sections=sections,+ callback=callback,+ lang=lang,+ **kwargs,+ )paper["tables"] = tblselse:raise NotImplementedError("file type not supported yet(pdf supported)")diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.pyindex 4e80745813..eda668727d 100644--- a/rag/flow/parser/parser.py+++ b/rag/flow/parser/parser.py@@ -786,7 +786,9 @@ class Parser(ProcessBase):for b in bboxes:if b.get("layout_type", "") == "title":mkdn += "\n## "- if b.get("layout_type", "") == "figure":+ # The current frontend uses JSON for PDF output. Keep this+ # defensive guard for imported or API-authored Markdown flows.+ if b.get("layout_type", "") == "figure" and b.get("image") is not None:mkdn += "\n".format(VLM.image2base64(b["image"]))continuemkdn += b.get("text", "") + "\n"diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.pyindex cbcc381797..f257bbecec 100644--- a/rag/nlp/__init__.py+++ b/rag/nlp/__init__.py@@ -477,15 +477,15 @@ def tokenize_table(tbls, doc, eng, batch_size=10, language="English"):for (img, rows), poss in tbls:if not rows:continue+ # Media producers use strings for tables and lists for figures. Keep+ # that contract explicit instead of guessing the type from HTML tags.if isinstance(rows, str):d = copy.deepcopy(doc)tokenize(d, rows, eng, language=language)d["content_with_weight"] = rowsd["doc_type_kwd"] = "table"- if img:+ if img is not None:d["image"] = img- if d["content_with_weight"].find("<tr>") < 0:- d["doc_type_kwd"] = "image"if poss:add_positions(d, poss)res.append(d)@@ -496,11 +496,9 @@ def tokenize_table(tbls, doc, eng, batch_size=10, language="English"):d = copy.deepcopy(doc)r = de.join(rows[i : i + batch_size])tokenize(d, r, eng, language=language)- d["doc_type_kwd"] = "table"- if img:+ d["doc_type_kwd"] = "image"+ if img is not None:d["image"] = img- if d["content_with_weight"].find("<tr>") < 0:- d["doc_type_kwd"] = "image"add_positions(d, poss)res.append(d)return res@@ -884,6 +882,14 @@ def append_context2table_image4pdf(sections: list, tabls: list, table_context_sires = []contexts = []for (img, tb), poss in tabls:+ # MinerU is expected to provide page_idx and bbox, but production+ # fallbacks may still yield media without positions. Preserve it.+ if not poss:+ res.append(((img, tb), poss))+ if return_context:+ contexts.append(("", ""))+ continue+page, left, right, top, bott = poss[0]_page, _left, _right, _top, _bott = poss[-1]if isinstance(tb, list):diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.pyindex bf7090c4cf..cbddd45ba8 100644--- a/test/unit_test/deepdoc/parser/test_mineru_parser.py+++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py@@ -125,8 +125,10 @@ def test_transfer_to_sections_logs_sections_dropped_after_sanitization(monkeypatparser = module.MinerUParser()outputs = [{- "type": module.MinerUContentType.TEXT,- "text": "<td></td>",+ "type": module.MinerUContentType.TABLE,+ "table_body": "<td></td>",+ "table_caption": [],+ "table_footnote": [],"page_idx": 0,"bbox": (0, 0, 1, 1),}@@ -137,7 +139,52 @@ def test_transfer_to_sections_logs_sections_dropped_after_sanitization(monkeypatassert sections == []assert "Skip section after sanitization" in caplog.text- assert f"type={module.MinerUContentType.TEXT}" in caplog.text+ assert f"type={module.MinerUContentType.TABLE}" in caplog.text+++ ("output", "expected"),+ [+ ({"type": "text", "text": "Use List<String> and a<b. 5 < 6"}, "Use List<String> and a<b. 5 < 6"),+ ({"type": "code", "code_body": "template<typename T> void f();", "code_caption": []}, "template<typename T> void f();"),+ ({"type": "equation", "text": "x < y > z"}, "x < y > z"),+ ],+)+def test_transfer_to_sections_only_sanitizes_tables(monkeypatch, output, expected):+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ output = {**output, "page_idx": 0, "bbox": (0, 0, 1, 1)}++ sections = parser._transfer_to_sections([output], parse_method="raw", table_enable=False)++ assert sections[0][0] == expected++[email protected]("transfer", ["sections", "tables"])+def test_empty_table_fallback_is_logged(monkeypatch, caplog, transfer):+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ output = {+ "type": module.MinerUContentType.TABLE,+ "table_body": "",+ "table_caption": [],+ "table_footnote": [],+ "page_idx": 2,+ "bbox": (0, 0, 1, 1),+ }++ with caplog.at_level(logging.WARNING, logger=parser.logger.name):+ if transfer == "sections":+ result = parser._transfer_to_sections([output], parse_method="raw", table_enable=True)+ fallback = result[0][0]+ else:+ result = parser._transfer_to_tables([output])+ fallback = result[0][0][1]++ assert fallback == "FAILED TO PARSE TABLE"+ assert "Empty table content at page_idx=2; using fallback text." in caplog.textdef test_transfer_to_sections_skips_page_chrome_without_duplicating_text(monkeypatch):@@ -187,6 +234,121 @@ def test_transfer_to_sections_skips_unknown_types_without_duplicating_text(monkeassert "Skip unsupported section type=sidebar" in caplog.text+def test_transfer_to_tables_emits_ordered_typed_media(monkeypatch, tmp_path):+ from rag.nlp import tokenize_table++ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ image_path = tmp_path / "figure.png"+ module.Image.new("RGB", (2, 2), "red").save(image_path)+ outputs = [+ {+ "type": module.MinerUContentType.TABLE,+ "table_body": "<table><tr><td>first</td></tr></table>",+ "table_caption": [],+ "table_footnote": [],+ "page_idx": 0,+ "bbox": (1, 2, 3, 4),+ },+ {+ "type": module.MinerUContentType.IMAGE,+ "img_path": str(image_path),+ "image_caption": ["Figure 1"],+ "image_footnote": ["Source"],+ "vlm_description": "A red square",+ "page_idx": 0,+ "bbox": (5, 6, 7, 8),+ },+ {+ "type": module.MinerUContentType.TABLE,+ "table_body": "second table",+ "table_caption": [],+ "table_footnote": [],+ "page_idx": 1,+ "bbox": (9, 10, 11, 12),+ },+ {+ "type": module.MinerUContentType.IMAGE,+ "image_caption": ["Caption without image"],+ "image_footnote": [],+ },+ ]++ media = parser._transfer_to_tables(outputs)++ assert len(media) == 3+ assert media[0][0] == (None, "<table><tr><td>first</td></tr></table>")+ assert media[2][0] == (None, "second table")+ image, texts = media[1][0]+ image_path.unlink()+ assert isinstance(image, module.Image.Image)+ assert image.getpixel((0, 0)) == (255, 0, 0)+ assert texts == ["Figure 1", "Source", "A red square"]+ assert [chunk["doc_type_kwd"] for chunk in tokenize_table(media, {}, False)] == ["table", "image", "table"]+++def test_tokenize_table_uses_payload_type_instead_of_html_content():+ from PIL import Image++ from rag.nlp import tokenize_table++ image = Image.new("RGB", (1, 1))+ media = [((image, "plain table"), []), ((image, ["caption with <tr> text"]), [])]++ chunks = tokenize_table(media, {}, False)++ assert [chunk["doc_type_kwd"] for chunk in chunks] == ["table", "image"]+ assert [chunk["image"] for chunk in chunks] == [image, image]+++def test_media_context_preserves_media_without_positions(monkeypatch):+ from rag.nlp import append_context2table_image4pdf++ parser_module = ModuleType("deepdoc.parser")+ parser_module.PdfParser = Mock()+ monkeypatch.setitem(sys.modules, "deepdoc.parser", parser_module)++ image = object()+ media = [((None, "table"), []), ((image, ["figure"]), [])]++ assert append_context2table_image4pdf([], media, 1) == media+ assert append_context2table_image4pdf([], media, 1, return_context=True) == [("", ""), ("", "")]++[email protected]("parse_method", ["naive", "manual", "paper"])+def test_transfer_to_sections_routes_app_media_separately(monkeypatch, parse_method):+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ outputs = [+ {"type": module.MinerUContentType.TEXT, "text": "Body", "page_idx": 0, "bbox": (0, 0, 1, 1)},+ {+ "type": module.MinerUContentType.TABLE,+ "table_body": "table",+ "table_caption": [],+ "table_footnote": [],+ "page_idx": 0,+ "bbox": (0, 1, 1, 2),+ },+ {+ "type": module.MinerUContentType.IMAGE,+ "image_caption": ["figure"],+ "image_footnote": [],+ "page_idx": 0,+ "bbox": (0, 2, 1, 3),+ },+ ]++ sections = parser._transfer_to_sections(outputs, parse_method=parse_method, table_enable=True)++ assert len(sections) == 1+ assert sections[0][0].startswith("Body")+ assert len(parser._transfer_to_sections(outputs, parse_method="raw", table_enable=True)) == 3++class _FakeZipResponse:"""Stand-in for the streaming response returned by requests.post.diff --git a/test/unit_test/rag/flow/parser/test_markdown_output.py b/test/unit_test/rag/flow/parser/test_markdown_output.pynew file mode 100644index 0000000000..aeb2f9833f--- /dev/null+++ b/test/unit_test/rag/flow/parser/test_markdown_output.py@@ -0,0 +1,68 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import importlib+import sys+from types import SimpleNamespace+from unittest.mock import Mock+++def _load_parser_module(monkeypatch):+ for module_name in ("rag.flow.parser.parser", "deepdoc.parser.pdf_parser", "deepdoc.parser", "deepdoc"):+ monkeypatch.delitem(sys.modules, module_name, raising=False)+ importlib.invalidate_caches()+ importlib.import_module("deepdoc.parser.pdf_parser")+ return importlib.import_module("rag.flow.parser.parser")+++class _FakeProcess:+ def __init__(self):+ self._param = SimpleNamespace(+ setups={+ "pdf": {+ "parse_method": "docling",+ "output_format": "markdown",+ "flatten_media_to_text": False,+ "remove_toc": False,+ "remove_header_footer": False,+ }+ }+ )+ self._canvas = SimpleNamespace(_tenant_id="tenant", _language="English")+ self.callback = Mock()+ self.outputs = {}++ def set_output(self, name, value):+ self.outputs[name] = value+++def test_pdf_markdown_preserves_figure_text_when_image_is_missing(monkeypatch):+ parser_module = _load_parser_module(monkeypatch)+ pdf_parser = SimpleNamespace(+ outlines=[],+ parse_pdf=Mock(return_value=([("caption survives", "image", "")], [])),+ )+ process = _FakeProcess()+ image2base64 = Mock()++ monkeypatch.setattr(parser_module.TenantModelService, "get_by_id", Mock(return_value=(False, None)))+ monkeypatch.setattr(parser_module, "DoclingParser", Mock(return_value=pdf_parser))+ monkeypatch.setattr(parser_module, "enhance_media_sections_with_vision", Mock())+ monkeypatch.setattr(parser_module.VLM, "image2base64", image2base64)++ parser_module.Parser._pdf(process, "document.pdf", b"pdf")++ assert process.outputs["markdown"] == "caption survives\n"+ image2base64.assert_not_called()