Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

feat(MinerU):Add support for MinerU table and image chunk classification (#16703)

Aeovyb27623a0c2b7
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py
index cd69cc1770..71f688f1d0 100644
--- a/deepdoc/parser/mineru_parser.py
+++ b/deepdoc/parser/mineru_parser.py
@@ -831,13 +831,21 @@ class MinerUParser(RAGFlowPdfParser):
def _transfer_to_sections(self, outputs: list[dict[str, Any]], parse_method: str = None, table_enable: bool = False):
sections = []
+ parse_method = (parse_method or "raw").lower()
for output in outputs:
- match output.get("type"):
+ output_type = output.get("type")
+ # These chunkers consume tables and images separately, so exclude
+ # media from their text sections. Raw consumers keep legacy sections.
+ if parse_method in {"naive", "manual", "paper"} and (output_type == MinerUContentType.IMAGE or (output_type == MinerUContentType.TABLE and table_enable)):
+ continue
+
+ match output_type:
case MinerUContentType.TEXT:
section = output.get("text", "")
case MinerUContentType.TABLE:
section = output.get("table_body", "") + "\n".join(output.get("table_caption", [])) + "\n".join(output.get("table_footnote", []))
if not section.strip():
+ self.logger.warning("[MinerU] Empty table content at page_idx=%s; using fallback text.", output.get("page_idx"))
section = "FAILED TO PARSE TABLE"
case MinerUContentType.IMAGE:
section = "".join(output.get("image_caption", [])) + "\n" + "".join(output.get("image_footnote", []))
@@ -859,7 +867,9 @@ class MinerUParser(RAGFlowPdfParser):
self.logger.debug("[MinerU] Skip unsupported section type=%s", output.get("type"))
continue
- if not table_enable:
+ # Only flatten table HTML when table extraction is disabled; the
+ # same sanitization would corrupt valid text, code, and equations.
+ if output_type == MinerUContentType.TABLE and not table_enable:
section = self._sanitize_section_text(section)
if not section:
self.logger.debug("[MinerU] Skip section after sanitization: type=%s", output.get("type"))
@@ -873,8 +883,52 @@ class MinerUParser(RAGFlowPdfParser):
sections.append((section, self._line_tag(output)))
return sections
- def _transfer_to_tables(self, outputs: list[dict[str, Any]]):
- return []
+ def _transfer_to_tables(self, outputs: list[dict[str, Any]], table_enable: bool = True):
+ """Convert MinerU media blocks to RAGFlow table/image tuples.
+
+ Args:
+ outputs: MinerU content-list blocks in source order.
+ table_enable: Whether table blocks should be emitted.
+ """
+ tables = []
+ for output in outputs:
+ output_type = output.get("type")
+ if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE}:
+ continue
+ if output_type == MinerUContentType.TABLE and not table_enable:
+ continue
+
+ position_tag = self._line_tag(output) if "page_idx" in output and "bbox" in output else ""
+ positions = [(page, left, right, top, bottom) for pages, left, right, top, bottom in self.extract_positions(position_tag) for page in pages]
+
+ if output_type == MinerUContentType.TABLE:
+ text = output.get("table_body", "") + "\n".join(output.get("table_caption", [])) + "\n".join(output.get("table_footnote", []))
+ if not text.strip():
+ self.logger.warning("[MinerU] Empty table content at page_idx=%s; using fallback text.", output.get("page_idx"))
+ text = "FAILED TO PARSE TABLE"
+ tables.append(((None, text), positions))
+ continue
+
+ texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]
+ vlm_description = (output.get("vlm_description") or "").strip()
+ if vlm_description:
+ texts.append(vlm_description)
+
+ image = None
+ image_path = output.get("img_path")
+ if image_path:
+ try:
+ with Image.open(image_path) as source:
+ source.load()
+ image = source.copy()
+ except Exception as e:
+ self.logger.warning(f"[MinerU] Failed to load image '{image_path}': {e}")
+ if image is None:
+ self.logger.warning("[MinerU] Skip image without a readable resource: %s", image_path)
+ continue
+
+ tables.append(((image, texts or [""]), positions))
+ return tables
def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"):
"""Generate semantic descriptions for image blocks via the tenant's
@@ -1002,7 +1056,9 @@ class MinerUParser(RAGFlowPdfParser):
except Exception as e:
self.logger.warning(f"[MinerU] VLM image enhancement failed: {e}. Continuing without descriptions.")
- return self._transfer_to_sections(outputs, parse_method, enable_table), self._transfer_to_tables(outputs)
+ sections = self._transfer_to_sections(outputs, parse_method, enable_table)
+ tables = self._transfer_to_tables(outputs, enable_table) if (parse_method or "raw").lower() in {"naive", "manual", "paper"} else []
+ return sections, tables
finally:
if temp_pdf and temp_pdf.exists():
try:
diff --git a/rag/app/manual.py b/rag/app/manual.py
index 50b614b033..1ada2131a6 100644
--- a/rag/app/manual.py
+++ b/rag/app/manual.py
@@ -240,10 +240,11 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
sec_ids.append(sid)
sections = [(txt, sec_ids[i], poss) for i, (txt, _, poss) in enumerate(sections)]
- for (img, rows), poss in tbls:
- if not rows:
- continue
- sections.append((rows if isinstance(rows, str) else rows[0], -1, [(p[0] + 1 - from_page, p[1], p[2], p[3], p[4]) for p in poss]))
+ if name != "mineru":
+ for (img, rows), poss in tbls:
+ if not rows:
+ continue
+ sections.append((rows if isinstance(rows, str) else rows[0], -1, [(p[0] + 1 - from_page, p[1], p[2], p[3], p[4]) for p in poss]))
def tag(pn, left, right, top, bottom):
if pn + left + right + top + bottom == 0:
@@ -264,13 +265,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
tk_cnt = num_tokens_from_string(txt)
if sec_id > -1:
last_sid = sec_id
- tbls = vision_figure_parser_pdf_wrapper(
- tbls=tbls,
- sections=sections,
- callback=callback,
- lang=lang,
- **kwargs,
- )
+ if name != "mineru":
+ tbls = vision_figure_parser_pdf_wrapper(
+ tbls=tbls,
+ sections=sections,
+ callback=callback,
+ lang=lang,
+ **kwargs,
+ )
res = tokenize_table(tbls, doc, eng, language=lang)
res.extend(tokenize_chunks(chunks, doc, eng, pdf_parser, language=lang))
table_ctx = max(0, int(parser_config.get("table_context_size", 0) or 0))
diff --git a/rag/app/naive.py b/rag/app/naive.py
index aac579b4f4..7479b2dc32 100644
--- a/rag/app/naive.py
+++ b/rag/app/naive.py
@@ -1068,6 +1068,8 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
name = layout_recognizer.strip().lower()
parser = PARSERS.get(name, by_plaintext)
callback(0.1, "Start to parse.")
+ if name == "mineru":
+ kwargs["parse_method"] = "naive"
sections, tables, pdf_parser = parser(
filename=filename,
diff --git a/rag/app/paper.py b/rag/app/paper.py
index 19aeac46fa..c0ffb29032 100644
--- a/rag/app/paper.py
+++ b/rag/app/paper.py
@@ -180,13 +180,14 @@ def chunk(filename, binary=None, from_page=0, to_page=MAXIMUM_PAGE_NUMBER, lang=
paper = {"title": filename, "authors": " ", "abstract": "", "sections": sections, "tables": tables}
tbls = paper["tables"]
- tbls = vision_figure_parser_pdf_wrapper(
- tbls=tbls,
- sections=sections,
- callback=callback,
- lang=lang,
- **kwargs,
- )
+ if name != "mineru":
+ tbls = vision_figure_parser_pdf_wrapper(
+ tbls=tbls,
+ sections=sections,
+ callback=callback,
+ lang=lang,
+ **kwargs,
+ )
paper["tables"] = tbls
else:
raise NotImplementedError("file type not supported yet(pdf supported)")
diff --git a/rag/flow/parser/parser.py b/rag/flow/parser/parser.py
index 4e80745813..eda668727d 100644
--- a/rag/flow/parser/parser.py
+++ b/rag/flow/parser/parser.py
@@ -786,7 +786,9 @@ class Parser(ProcessBase):
for b in bboxes:
if b.get("layout_type", "") == "title":
mkdn += "\n## "
- if b.get("layout_type", "") == "figure":
+ # The current frontend uses JSON for PDF output. Keep this
+ # defensive guard for imported or API-authored Markdown flows.
+ if b.get("layout_type", "") == "figure" and b.get("image") is not None:
mkdn += "\n![Image]({})".format(VLM.image2base64(b["image"]))
continue
mkdn += b.get("text", "") + "\n"
diff --git a/rag/nlp/__init__.py b/rag/nlp/__init__.py
index cbcc381797..f257bbecec 100644
--- a/rag/nlp/__init__.py
+++ b/rag/nlp/__init__.py
@@ -477,15 +477,15 @@ def tokenize_table(tbls, doc, eng, batch_size=10, language="English"):
for (img, rows), poss in tbls:
if not rows:
continue
+ # Media producers use strings for tables and lists for figures. Keep
+ # that contract explicit instead of guessing the type from HTML tags.
if isinstance(rows, str):
d = copy.deepcopy(doc)
tokenize(d, rows, eng, language=language)
d["content_with_weight"] = rows
d["doc_type_kwd"] = "table"
- if img:
+ if img is not None:
d["image"] = img
- if d["content_with_weight"].find("<tr>") < 0:
- d["doc_type_kwd"] = "image"
if poss:
add_positions(d, poss)
res.append(d)
@@ -496,11 +496,9 @@ def tokenize_table(tbls, doc, eng, batch_size=10, language="English"):
d = copy.deepcopy(doc)
r = de.join(rows[i : i + batch_size])
tokenize(d, r, eng, language=language)
- d["doc_type_kwd"] = "table"
- if img:
+ d["doc_type_kwd"] = "image"
+ if img is not None:
d["image"] = img
- if d["content_with_weight"].find("<tr>") < 0:
- d["doc_type_kwd"] = "image"
add_positions(d, poss)
res.append(d)
return res
@@ -884,6 +882,14 @@ def append_context2table_image4pdf(sections: list, tabls: list, table_context_si
res = []
contexts = []
for (img, tb), poss in tabls:
+ # MinerU is expected to provide page_idx and bbox, but production
+ # fallbacks may still yield media without positions. Preserve it.
+ if not poss:
+ res.append(((img, tb), poss))
+ if return_context:
+ contexts.append(("", ""))
+ continue
+
page, left, right, top, bott = poss[0]
_page, _left, _right, _top, _bott = poss[-1]
if isinstance(tb, list):
diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.py
index bf7090c4cf..cbddd45ba8 100644
--- a/test/unit_test/deepdoc/parser/test_mineru_parser.py
+++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py
@@ -125,8 +125,10 @@ def test_transfer_to_sections_logs_sections_dropped_after_sanitization(monkeypat
parser = module.MinerUParser()
outputs = [
{
- "type": module.MinerUContentType.TEXT,
- "text": "&lt;td&gt;&lt;/td&gt;",
+ "type": module.MinerUContentType.TABLE,
+ "table_body": "&lt;td&gt;&lt;/td&gt;",
+ "table_caption": [],
+ "table_footnote": [],
"page_idx": 0,
"bbox": (0, 0, 1, 1),
}
@@ -137,7 +139,52 @@ def test_transfer_to_sections_logs_sections_dropped_after_sanitization(monkeypat
assert sections == []
assert "Skip section after sanitization" in caplog.text
- assert f"type={module.MinerUContentType.TEXT}" in caplog.text
+ assert f"type={module.MinerUContentType.TABLE}" in caplog.text
+
+
+ ("output", "expected"),
+ [
+ ({"type": "text", "text": "Use List<String> and a<b. 5 &lt; 6"}, "Use List<String> and a<b. 5 &lt; 6"),
+ ({"type": "code", "code_body": "template<typename T> void f();", "code_caption": []}, "template<typename T> void f();"),
+ ({"type": "equation", "text": "x < y > z"}, "x < y > z"),
+ ],
+)
+def test_transfer_to_sections_only_sanitizes_tables(monkeypatch, output, expected):
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ output = {**output, "page_idx": 0, "bbox": (0, 0, 1, 1)}
+
+ sections = parser._transfer_to_sections([output], parse_method="raw", table_enable=False)
+
+ assert sections[0][0] == expected
+
+
[email protected]("transfer", ["sections", "tables"])
+def test_empty_table_fallback_is_logged(monkeypatch, caplog, transfer):
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ output = {
+ "type": module.MinerUContentType.TABLE,
+ "table_body": "",
+ "table_caption": [],
+ "table_footnote": [],
+ "page_idx": 2,
+ "bbox": (0, 0, 1, 1),
+ }
+
+ with caplog.at_level(logging.WARNING, logger=parser.logger.name):
+ if transfer == "sections":
+ result = parser._transfer_to_sections([output], parse_method="raw", table_enable=True)
+ fallback = result[0][0]
+ else:
+ result = parser._transfer_to_tables([output])
+ fallback = result[0][0][1]
+
+ assert fallback == "FAILED TO PARSE TABLE"
+ assert "Empty table content at page_idx=2; using fallback text." in caplog.text
def test_transfer_to_sections_skips_page_chrome_without_duplicating_text(monkeypatch):
@@ -187,6 +234,121 @@ def test_transfer_to_sections_skips_unknown_types_without_duplicating_text(monke
assert "Skip unsupported section type=sidebar" in caplog.text
+def test_transfer_to_tables_emits_ordered_typed_media(monkeypatch, tmp_path):
+ from rag.nlp import tokenize_table
+
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ image_path = tmp_path / "figure.png"
+ module.Image.new("RGB", (2, 2), "red").save(image_path)
+ outputs = [
+ {
+ "type": module.MinerUContentType.TABLE,
+ "table_body": "<table><tr><td>first</td></tr></table>",
+ "table_caption": [],
+ "table_footnote": [],
+ "page_idx": 0,
+ "bbox": (1, 2, 3, 4),
+ },
+ {
+ "type": module.MinerUContentType.IMAGE,
+ "img_path": str(image_path),
+ "image_caption": ["Figure 1"],
+ "image_footnote": ["Source"],
+ "vlm_description": "A red square",
+ "page_idx": 0,
+ "bbox": (5, 6, 7, 8),
+ },
+ {
+ "type": module.MinerUContentType.TABLE,
+ "table_body": "second table",
+ "table_caption": [],
+ "table_footnote": [],
+ "page_idx": 1,
+ "bbox": (9, 10, 11, 12),
+ },
+ {
+ "type": module.MinerUContentType.IMAGE,
+ "image_caption": ["Caption without image"],
+ "image_footnote": [],
+ },
+ ]
+
+ media = parser._transfer_to_tables(outputs)
+
+ assert len(media) == 3
+ assert media[0][0] == (None, "<table><tr><td>first</td></tr></table>")
+ assert media[2][0] == (None, "second table")
+ image, texts = media[1][0]
+ image_path.unlink()
+ assert isinstance(image, module.Image.Image)
+ assert image.getpixel((0, 0)) == (255, 0, 0)
+ assert texts == ["Figure 1", "Source", "A red square"]
+ assert [chunk["doc_type_kwd"] for chunk in tokenize_table(media, {}, False)] == ["table", "image", "table"]
+
+
+def test_tokenize_table_uses_payload_type_instead_of_html_content():
+ from PIL import Image
+
+ from rag.nlp import tokenize_table
+
+ image = Image.new("RGB", (1, 1))
+ media = [((image, "plain table"), []), ((image, ["caption with <tr> text"]), [])]
+
+ chunks = tokenize_table(media, {}, False)
+
+ assert [chunk["doc_type_kwd"] for chunk in chunks] == ["table", "image"]
+ assert [chunk["image"] for chunk in chunks] == [image, image]
+
+
+def test_media_context_preserves_media_without_positions(monkeypatch):
+ from rag.nlp import append_context2table_image4pdf
+
+ parser_module = ModuleType("deepdoc.parser")
+ parser_module.PdfParser = Mock()
+ monkeypatch.setitem(sys.modules, "deepdoc.parser", parser_module)
+
+ image = object()
+ media = [((None, "table"), []), ((image, ["figure"]), [])]
+
+ assert append_context2table_image4pdf([], media, 1) == media
+ assert append_context2table_image4pdf([], media, 1, return_context=True) == [("", ""), ("", "")]
+
+
[email protected]("parse_method", ["naive", "manual", "paper"])
+def test_transfer_to_sections_routes_app_media_separately(monkeypatch, parse_method):
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ outputs = [
+ {"type": module.MinerUContentType.TEXT, "text": "Body", "page_idx": 0, "bbox": (0, 0, 1, 1)},
+ {
+ "type": module.MinerUContentType.TABLE,
+ "table_body": "table",
+ "table_caption": [],
+ "table_footnote": [],
+ "page_idx": 0,
+ "bbox": (0, 1, 1, 2),
+ },
+ {
+ "type": module.MinerUContentType.IMAGE,
+ "image_caption": ["figure"],
+ "image_footnote": [],
+ "page_idx": 0,
+ "bbox": (0, 2, 1, 3),
+ },
+ ]
+
+ sections = parser._transfer_to_sections(outputs, parse_method=parse_method, table_enable=True)
+
+ assert len(sections) == 1
+ assert sections[0][0].startswith("Body")
+ assert len(parser._transfer_to_sections(outputs, parse_method="raw", table_enable=True)) == 3
+
+
class _FakeZipResponse:
"""Stand-in for the streaming response returned by requests.post.
diff --git a/test/unit_test/rag/flow/parser/test_markdown_output.py b/test/unit_test/rag/flow/parser/test_markdown_output.py
new file mode 100644
index 0000000000..aeb2f9833f
--- /dev/null
+++ b/test/unit_test/rag/flow/parser/test_markdown_output.py
@@ -0,0 +1,68 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import importlib
+import sys
+from types import SimpleNamespace
+from unittest.mock import Mock
+
+
+def _load_parser_module(monkeypatch):
+ for module_name in ("rag.flow.parser.parser", "deepdoc.parser.pdf_parser", "deepdoc.parser", "deepdoc"):
+ monkeypatch.delitem(sys.modules, module_name, raising=False)
+ importlib.invalidate_caches()
+ importlib.import_module("deepdoc.parser.pdf_parser")
+ return importlib.import_module("rag.flow.parser.parser")
+
+
+class _FakeProcess:
+ def __init__(self):
+ self._param = SimpleNamespace(
+ setups={
+ "pdf": {
+ "parse_method": "docling",
+ "output_format": "markdown",
+ "flatten_media_to_text": False,
+ "remove_toc": False,
+ "remove_header_footer": False,
+ }
+ }
+ )
+ self._canvas = SimpleNamespace(_tenant_id="tenant", _language="English")
+ self.callback = Mock()
+ self.outputs = {}
+
+ def set_output(self, name, value):
+ self.outputs[name] = value
+
+
+def test_pdf_markdown_preserves_figure_text_when_image_is_missing(monkeypatch):
+ parser_module = _load_parser_module(monkeypatch)
+ pdf_parser = SimpleNamespace(
+ outlines=[],
+ parse_pdf=Mock(return_value=([("caption survives", "image", "")], [])),
+ )
+ process = _FakeProcess()
+ image2base64 = Mock()
+
+ monkeypatch.setattr(parser_module.TenantModelService, "get_by_id", Mock(return_value=(False, None)))
+ monkeypatch.setattr(parser_module, "DoclingParser", Mock(return_value=pdf_parser))
+ monkeypatch.setattr(parser_module, "enhance_media_sections_with_vision", Mock())
+ monkeypatch.setattr(parser_module.VLM, "image2base64", image2base64)
+
+ parser_module.Parser._pdf(process, "document.pdf", b"pdf")
+
+ assert process.outputs["markdown"] == "caption survives\n"
+ image2base64.assert_not_called()