RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(parser): handle MinerU chart blocks instead of silently dropping them (#19096)
Paul Yao0c28d59ea1d3
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.ymlindex 996ce0ce67..b5f3d706d3 100644--- a/.github/workflows/sep-tests.yml+++ b/.github/workflows/sep-tests.yml@@ -281,6 +281,84 @@ jobs:uv sync --python 3.13 --group test --frozenuv pip install -e sdk/python+ - name: Provision NLTK data for unit tests+ if: steps.detect_changes.outputs.has_python_changes == 'true'+ run: |+ set -euo pipefail++ # The self-hosted runner stores NLTK packages as root-owned zip files.+ # Reading them directly as the runner user fails with EACCES, while+ # downloading replacements is blocked by the runner's SSRF policy.+ # Copy the three required archives into the workspace with sudo, then+ # unpack and validate them as the runner user.+ NLTK_TARGET="${GITHUB_WORKSPACE}/nltk_data"+ rm -rf "${NLTK_TARGET}"+ mkdir -p "${NLTK_TARGET}/tokenizers" "${NLTK_TARGET}/corpora"++ copy_nltk_archive() {+ local relative_path="$1"+ local source=""+ for root in /usr/share/nltk_data /usr/local/share/nltk_data; do+ if sudo test -f "${root}/${relative_path}"; then+ source="${root}/${relative_path}"+ break+ fi+ done+ if [ -z "${source}" ]; then+ echo "Missing required NLTK archive: ${relative_path}" >&2+ return 1+ fi+ echo "Copying ${source}"+ sudo cp "${source}" "${NLTK_TARGET}/${relative_path}"+ sudo chown "$(id -u):$(id -g)" "${NLTK_TARGET}/${relative_path}"+ chmod 0644 "${NLTK_TARGET}/${relative_path}"+ }++ copy_nltk_archive tokenizers/punkt_tab.zip+ copy_nltk_archive tokenizers/punkt.zip+ copy_nltk_archive corpora/wordnet.zip++ NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'+ import os+ import zipfile++ root = os.environ["NLTK_DATA"]+ archives = (+ ("tokenizers", "punkt_tab"),+ ("tokenizers", "punkt"),+ ("corpora", "wordnet"),+ )+ for category, package in archives:+ archive = os.path.join(root, category, f"{package}.zip")+ destination = os.path.join(root, category)+ with zipfile.ZipFile(archive) as package_zip:+ bad_member = package_zip.testzip()+ if bad_member is not None:+ raise RuntimeError(f"Corrupt NLTK archive {archive}: {bad_member}")+ members = [name for name in package_zip.namelist() if not name.startswith("__MACOSX/")]+ prefix = f"{package}/"+ if not members or not all(name == package or name.startswith(prefix) for name in members):+ raise RuntimeError(f"Unexpected layout in {archive}: expected entries below {prefix}")+ package_zip.extractall(destination)+ print(f"Unpacked {archive} -> {os.path.join(destination, package)}")+ PYEOF++ echo "NLTK_DATA=${NLTK_TARGET}" >> "${GITHUB_ENV}"++ # Validate real tokenizer/lemmatizer calls, not only directory names.+ # The command exits non-zero if any resource is absent or malformed.+ NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'+ import nltk+ from nltk.corpus import wordnet+ from nltk.tokenize import word_tokenize++ for resource in ("tokenizers/punkt_tab", "tokenizers/punkt", "corpora/wordnet"):+ print(f"[OK] {resource} -> {nltk.data.find(resource)}")+ assert word_tokenize("NLTK data is ready.")+ assert wordnet.synsets("document")+ print("NLTK tokenizer and WordNet smoke checks passed")+ PYEOF+- name: Run unit testif: steps.detect_changes.outputs.has_python_changes == 'true'run: |diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.pyindex 373d0e2c6d..cf40876820 100644--- a/deepdoc/parser/mineru_parser.py+++ b/deepdoc/parser/mineru_parser.py@@ -56,6 +56,11 @@ class MinerUContentType(StrEnum):FOOTER = "footer"PAGE_NUMBER = "page_number"DISCARDED = "discarded"+ # MinerU 3.4.x VLM backend emits chart blocks for figures whose visual is a+ # chart rather than a plain image. They carry chart_caption / chart_footnote+ # / sub_type / img_path / bbox / page_idx — the same shape as IMAGE blocks —+ # so they are routed through the image pipeline instead of being dropped.+ CHART = "chart"# Mapping from language names to MinerU language codes@@ -848,7 +853,9 @@ class MinerUParser(RAGFlowPdfParser):output_type = output.get("type")# These chunkers consume tables and images separately, so exclude# media from their text sections. Raw consumers keep legacy sections.- if parse_method in {"naive", "manual", "paper"} and (output_type == MinerUContentType.IMAGE or (output_type == MinerUContentType.TABLE and table_enable)):+ # Charts are routed the same way as images: MinerU emits them as+ # visual blocks that _transfer_to_tables turns into image chunks.+ if parse_method in {"naive", "manual", "paper"} and (output_type in {MinerUContentType.IMAGE, MinerUContentType.CHART} or (output_type == MinerUContentType.TABLE and table_enable)):continuematch output_type:@@ -867,6 +874,11 @@ class MinerUParser(RAGFlowPdfParser):vlm_description = (output.get("vlm_description") or "").strip()if vlm_description:section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description+ case MinerUContentType.CHART:+ section = "".join(output.get("chart_caption", [])) + "\n" + "".join(output.get("chart_footnote", []))+ vlm_description = (output.get("vlm_description") or "").strip()+ if vlm_description:+ section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_descriptioncase MinerUContentType.EQUATION:section = output.get("text", "")case MinerUContentType.CODE:@@ -886,7 +898,7 @@ class MinerUParser(RAGFlowPdfParser):case MinerUContentType.HEADER | MinerUContentType.FOOTER | MinerUContentType.PAGE_NUMBER | MinerUContentType.DISCARDED:continuecase _:- self.logger.debug("[MinerU] Skip unsupported section type=%s", output.get("type"))+ self.logger.warning("[MinerU] Skip unsupported section type=%s", output.get("type"))continue# Only flatten table HTML when table extraction is disabled; the@@ -915,7 +927,7 @@ class MinerUParser(RAGFlowPdfParser):tables = []for output in outputs:output_type = output.get("type")- if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE}:+ if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE, MinerUContentType.CHART}:continueif output_type == MinerUContentType.TABLE and not table_enable:continue@@ -933,7 +945,12 @@ class MinerUParser(RAGFlowPdfParser):tables.append(((None, text), positions))continue- texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]+ # IMAGE and CHART share the same visual pipeline; only the caption+ # field names differ (image_caption/image_footnote vs chart_*).+ if output_type == MinerUContentType.CHART:+ texts = [*output.get("chart_caption", []), *output.get("chart_footnote", [])]+ else:+ texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]vlm_description = (output.get("vlm_description") or "").strip()if vlm_description:texts.append(vlm_description)@@ -957,15 +974,17 @@ class MinerUParser(RAGFlowPdfParser):def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"):"""Generate semantic descriptions for image blocks via the tenant'sVISION model, mirroring deepdoc's VisionFigureParser. Each- IMAGE block with a readable img_path gets a ``vlm_description``- field that ``_transfer_to_sections`` then folds into the chunk- text — closing issue #14869.+ IMAGE or CHART block with a readable img_path gets a+ ``vlm_description`` field that ``_transfer_to_sections`` then folds+ into the chunk text — closing issue #14869."""from concurrent.futures import ThreadPoolExecutor, as_completedfrom rag.app.picture import vision_llm_chunkfrom rag.prompts.generator import vision_llm_figure_describe_prompt- image_jobs = [(idx, item) for idx, item in enumerate(outputs) if item.get("type") == MinerUContentType.IMAGE and item.get("img_path") and os.path.exists(item["img_path"])]+ image_jobs = [+ (idx, item) for idx, item in enumerate(outputs) if item.get("type") in {MinerUContentType.IMAGE, MinerUContentType.CHART} and item.get("img_path") and os.path.exists(item["img_path"])+ ]if not image_jobs:returndiff --git a/test/unit_test/conftest.py b/test/unit_test/conftest.pyindex 512398c591..19a57c8471 100644--- a/test/unit_test/conftest.py+++ b/test/unit_test/conftest.py@@ -49,13 +49,10 @@ if _LOCAL_NLTK_DATA not in nltk.data.path:nltk.data.path.insert(0, _LOCAL_NLTK_DATA)# (download name, resource path used by nltk.data.find)-# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data-# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises-# LookupError at load time; `omw-1.4` must also be present._REQUIRED_NLTK_DATA = (("punkt_tab", "tokenizers/punkt_tab"),+ ("punkt", "tokenizers/punkt"),("wordnet", "corpora/wordnet"),- ("omw-1.4", "corpora/omw-1.4"),)for _name, _find_path in _REQUIRED_NLTK_DATA:try:diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.pyindex 3a8e206f54..fc974a87b6 100644--- a/test/unit_test/deepdoc/parser/test_mineru_parser.py+++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py@@ -412,6 +412,86 @@ def test_transfer_to_tables_emits_ordered_typed_media(monkeypatch, tmp_path):assert [chunk["page_num_int"] for chunk in chunks] == [[13, 14], [13], [14]]+def test_transfer_to_tables_emits_chart_as_image_chunk(monkeypatch, tmp_path):+ """MinerU 3.4.x VLM chart blocks (chart_caption/chart_footnote/img_path)+ must surface as image chunks instead of being silently dropped (#19080)."""+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ parser.page_from = 0+ chart_path = tmp_path / "chart.png"+ module.Image.new("RGB", (2, 2), "blue").save(chart_path)+ outputs = [+ {+ "type": module.MinerUContentType.CHART,+ "img_path": str(chart_path),+ "chart_caption": ["Figure 3"],+ "chart_footnote": ["Source: dataset"],+ "sub_type": "line",+ "vlm_description": "A blue square",+ "page_idx": 0,+ "bbox": (1, 2, 3, 4),+ },+ {+ "type": module.MinerUContentType.CHART,+ "chart_caption": ["Caption without image"],+ "chart_footnote": [],+ },+ ]++ media = parser._transfer_to_tables(outputs)++ # The chart with a readable image becomes an image chunk; the chart without+ # an img_path is skipped (mirrors how IMAGE blocks behave).+ assert len(media) == 1+ image, texts = media[0][0]+ chart_path.unlink()+ assert isinstance(image, module.Image.Image)+ assert image.getpixel((0, 0)) == (0, 0, 255)+ assert texts == ["Figure 3", "Source: dataset", "A blue square"]+++def test_transfer_to_sections_routes_chart_like_image_per_parse_method(monkeypatch):+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ outputs = [+ {"type": module.MinerUContentType.TEXT, "text": "Body", "page_idx": 0, "bbox": (0, 0, 1, 1)},+ {+ "type": module.MinerUContentType.CHART,+ "chart_caption": ["figure"],+ "chart_footnote": [],+ "page_idx": 0,+ "bbox": (0, 2, 1, 3),+ },+ ]++ # app chunkers consume media separately: the chart is excluded from text sections.+ for app_method in ("naive", "manual", "paper"):+ sections = parser._transfer_to_sections(outputs, parse_method=app_method, table_enable=True)+ assert len(sections) == 1+ assert sections[0][0].startswith("Body")++ # raw consumers keep the chart as a text section (caption/footnote), like IMAGE.+ raw_sections = parser._transfer_to_sections(outputs, parse_method="raw", table_enable=True)+ assert len(raw_sections) == 2+ assert raw_sections[1][0].strip() == "figure"+++def test_transfer_to_sections_warns_on_unknown_type(monkeypatch, caplog):+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser()+ outputs = [+ {"type": "sidebar", "text": "ignored", "page_idx": 0, "bbox": (0, 0, 1, 1)},+ ]++ with caplog.at_level(logging.WARNING, logger=parser.logger.name):+ parser._transfer_to_sections(outputs, parse_method="raw")++ assert "Skip unsupported section type=sidebar" in caplog.text++@pytest.mark.p1def test_tokenize_table_uses_payload_type_instead_of_html_content():from PIL import Image