Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(parser): handle MinerU chart blocks instead of silently dropping them (#19096)

Paul Yao0c28d59ea1d3
diff --git a/.github/workflows/sep-tests.yml b/.github/workflows/sep-tests.yml
index 996ce0ce67..b5f3d706d3 100644
--- a/.github/workflows/sep-tests.yml
+++ b/.github/workflows/sep-tests.yml
@@ -281,6 +281,84 @@ jobs:
uv sync --python 3.13 --group test --frozen
uv pip install -e sdk/python
+ - name: Provision NLTK data for unit tests
+ if: steps.detect_changes.outputs.has_python_changes == 'true'
+ run: |
+ set -euo pipefail
+
+ # The self-hosted runner stores NLTK packages as root-owned zip files.
+ # Reading them directly as the runner user fails with EACCES, while
+ # downloading replacements is blocked by the runner's SSRF policy.
+ # Copy the three required archives into the workspace with sudo, then
+ # unpack and validate them as the runner user.
+ NLTK_TARGET="${GITHUB_WORKSPACE}/nltk_data"
+ rm -rf "${NLTK_TARGET}"
+ mkdir -p "${NLTK_TARGET}/tokenizers" "${NLTK_TARGET}/corpora"
+
+ copy_nltk_archive() {
+ local relative_path="$1"
+ local source=""
+ for root in /usr/share/nltk_data /usr/local/share/nltk_data; do
+ if sudo test -f "${root}/${relative_path}"; then
+ source="${root}/${relative_path}"
+ break
+ fi
+ done
+ if [ -z "${source}" ]; then
+ echo "Missing required NLTK archive: ${relative_path}" >&2
+ return 1
+ fi
+ echo "Copying ${source}"
+ sudo cp "${source}" "${NLTK_TARGET}/${relative_path}"
+ sudo chown "$(id -u):$(id -g)" "${NLTK_TARGET}/${relative_path}"
+ chmod 0644 "${NLTK_TARGET}/${relative_path}"
+ }
+
+ copy_nltk_archive tokenizers/punkt_tab.zip
+ copy_nltk_archive tokenizers/punkt.zip
+ copy_nltk_archive corpora/wordnet.zip
+
+ NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'
+ import os
+ import zipfile
+
+ root = os.environ["NLTK_DATA"]
+ archives = (
+ ("tokenizers", "punkt_tab"),
+ ("tokenizers", "punkt"),
+ ("corpora", "wordnet"),
+ )
+ for category, package in archives:
+ archive = os.path.join(root, category, f"{package}.zip")
+ destination = os.path.join(root, category)
+ with zipfile.ZipFile(archive) as package_zip:
+ bad_member = package_zip.testzip()
+ if bad_member is not None:
+ raise RuntimeError(f"Corrupt NLTK archive {archive}: {bad_member}")
+ members = [name for name in package_zip.namelist() if not name.startswith("__MACOSX/")]
+ prefix = f"{package}/"
+ if not members or not all(name == package or name.startswith(prefix) for name in members):
+ raise RuntimeError(f"Unexpected layout in {archive}: expected entries below {prefix}")
+ package_zip.extractall(destination)
+ print(f"Unpacked {archive} -> {os.path.join(destination, package)}")
+ PYEOF
+
+ echo "NLTK_DATA=${NLTK_TARGET}" >> "${GITHUB_ENV}"
+
+ # Validate real tokenizer/lemmatizer calls, not only directory names.
+ # The command exits non-zero if any resource is absent or malformed.
+ NLTK_DATA="${NLTK_TARGET}" uv run python - <<'PYEOF'
+ import nltk
+ from nltk.corpus import wordnet
+ from nltk.tokenize import word_tokenize
+
+ for resource in ("tokenizers/punkt_tab", "tokenizers/punkt", "corpora/wordnet"):
+ print(f"[OK] {resource} -> {nltk.data.find(resource)}")
+ assert word_tokenize("NLTK data is ready.")
+ assert wordnet.synsets("document")
+ print("NLTK tokenizer and WordNet smoke checks passed")
+ PYEOF
+
- name: Run unit test
if: steps.detect_changes.outputs.has_python_changes == 'true'
run: |
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py
index 373d0e2c6d..cf40876820 100644
--- a/deepdoc/parser/mineru_parser.py
+++ b/deepdoc/parser/mineru_parser.py
@@ -56,6 +56,11 @@ class MinerUContentType(StrEnum):
FOOTER = "footer"
PAGE_NUMBER = "page_number"
DISCARDED = "discarded"
+ # MinerU 3.4.x VLM backend emits chart blocks for figures whose visual is a
+ # chart rather than a plain image. They carry chart_caption / chart_footnote
+ # / sub_type / img_path / bbox / page_idx — the same shape as IMAGE blocks —
+ # so they are routed through the image pipeline instead of being dropped.
+ CHART = "chart"
# Mapping from language names to MinerU language codes
@@ -848,7 +853,9 @@ class MinerUParser(RAGFlowPdfParser):
output_type = output.get("type")
# These chunkers consume tables and images separately, so exclude
# media from their text sections. Raw consumers keep legacy sections.
- if parse_method in {"naive", "manual", "paper"} and (output_type == MinerUContentType.IMAGE or (output_type == MinerUContentType.TABLE and table_enable)):
+ # Charts are routed the same way as images: MinerU emits them as
+ # visual blocks that _transfer_to_tables turns into image chunks.
+ if parse_method in {"naive", "manual", "paper"} and (output_type in {MinerUContentType.IMAGE, MinerUContentType.CHART} or (output_type == MinerUContentType.TABLE and table_enable)):
continue
match output_type:
@@ -867,6 +874,11 @@ class MinerUParser(RAGFlowPdfParser):
vlm_description = (output.get("vlm_description") or "").strip()
if vlm_description:
section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description
+ case MinerUContentType.CHART:
+ section = "".join(output.get("chart_caption", [])) + "\n" + "".join(output.get("chart_footnote", []))
+ vlm_description = (output.get("vlm_description") or "").strip()
+ if vlm_description:
+ section = (section.strip("\n") + "\n" + vlm_description).strip("\n") if section.strip() else vlm_description
case MinerUContentType.EQUATION:
section = output.get("text", "")
case MinerUContentType.CODE:
@@ -886,7 +898,7 @@ class MinerUParser(RAGFlowPdfParser):
case MinerUContentType.HEADER | MinerUContentType.FOOTER | MinerUContentType.PAGE_NUMBER | MinerUContentType.DISCARDED:
continue
case _:
- self.logger.debug("[MinerU] Skip unsupported section type=%s", output.get("type"))
+ self.logger.warning("[MinerU] Skip unsupported section type=%s", output.get("type"))
continue
# Only flatten table HTML when table extraction is disabled; the
@@ -915,7 +927,7 @@ class MinerUParser(RAGFlowPdfParser):
tables = []
for output in outputs:
output_type = output.get("type")
- if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE}:
+ if output_type not in {MinerUContentType.TABLE, MinerUContentType.IMAGE, MinerUContentType.CHART}:
continue
if output_type == MinerUContentType.TABLE and not table_enable:
continue
@@ -933,7 +945,12 @@ class MinerUParser(RAGFlowPdfParser):
tables.append(((None, text), positions))
continue
- texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]
+ # IMAGE and CHART share the same visual pipeline; only the caption
+ # field names differ (image_caption/image_footnote vs chart_*).
+ if output_type == MinerUContentType.CHART:
+ texts = [*output.get("chart_caption", []), *output.get("chart_footnote", [])]
+ else:
+ texts = [*output.get("image_caption", []), *output.get("image_footnote", [])]
vlm_description = (output.get("vlm_description") or "").strip()
if vlm_description:
texts.append(vlm_description)
@@ -957,15 +974,17 @@ class MinerUParser(RAGFlowPdfParser):
def _enhance_images_with_vlm(self, outputs: list[dict[str, Any]], vision_model, callback: Optional[Callable] = None, language: str = "English"):
"""Generate semantic descriptions for image blocks via the tenant's
VISION model, mirroring deepdoc's VisionFigureParser. Each
- IMAGE block with a readable img_path gets a ``vlm_description``
- field that ``_transfer_to_sections`` then folds into the chunk
- text — closing issue #14869.
+ IMAGE or CHART block with a readable img_path gets a
+ ``vlm_description`` field that ``_transfer_to_sections`` then folds
+ into the chunk text — closing issue #14869.
"""
from concurrent.futures import ThreadPoolExecutor, as_completed
from rag.app.picture import vision_llm_chunk
from rag.prompts.generator import vision_llm_figure_describe_prompt
- image_jobs = [(idx, item) for idx, item in enumerate(outputs) if item.get("type") == MinerUContentType.IMAGE and item.get("img_path") and os.path.exists(item["img_path"])]
+ image_jobs = [
+ (idx, item) for idx, item in enumerate(outputs) if item.get("type") in {MinerUContentType.IMAGE, MinerUContentType.CHART} and item.get("img_path") and os.path.exists(item["img_path"])
+ ]
if not image_jobs:
return
diff --git a/test/unit_test/conftest.py b/test/unit_test/conftest.py
index 512398c591..19a57c8471 100644
--- a/test/unit_test/conftest.py
+++ b/test/unit_test/conftest.py
@@ -49,13 +49,10 @@ if _LOCAL_NLTK_DATA not in nltk.data.path:
nltk.data.path.insert(0, _LOCAL_NLTK_DATA)
# (download name, resource path used by nltk.data.find)
-# NOTE: NLTK >=3.8.2 gates the `wordnet` corpus behind the `omw-1.4` data
-# package. Downloading `wordnet` alone leaves a stub `wordnet.zip` that raises
-# LookupError at load time; `omw-1.4` must also be present.
_REQUIRED_NLTK_DATA = (
("punkt_tab", "tokenizers/punkt_tab"),
+ ("punkt", "tokenizers/punkt"),
("wordnet", "corpora/wordnet"),
- ("omw-1.4", "corpora/omw-1.4"),
)
for _name, _find_path in _REQUIRED_NLTK_DATA:
try:
diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.py
index 3a8e206f54..fc974a87b6 100644
--- a/test/unit_test/deepdoc/parser/test_mineru_parser.py
+++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py
@@ -412,6 +412,86 @@ def test_transfer_to_tables_emits_ordered_typed_media(monkeypatch, tmp_path):
assert [chunk["page_num_int"] for chunk in chunks] == [[13, 14], [13], [14]]
+def test_transfer_to_tables_emits_chart_as_image_chunk(monkeypatch, tmp_path):
+ """MinerU 3.4.x VLM chart blocks (chart_caption/chart_footnote/img_path)
+ must surface as image chunks instead of being silently dropped (#19080)."""
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ parser.page_from = 0
+ chart_path = tmp_path / "chart.png"
+ module.Image.new("RGB", (2, 2), "blue").save(chart_path)
+ outputs = [
+ {
+ "type": module.MinerUContentType.CHART,
+ "img_path": str(chart_path),
+ "chart_caption": ["Figure 3"],
+ "chart_footnote": ["Source: dataset"],
+ "sub_type": "line",
+ "vlm_description": "A blue square",
+ "page_idx": 0,
+ "bbox": (1, 2, 3, 4),
+ },
+ {
+ "type": module.MinerUContentType.CHART,
+ "chart_caption": ["Caption without image"],
+ "chart_footnote": [],
+ },
+ ]
+
+ media = parser._transfer_to_tables(outputs)
+
+ # The chart with a readable image becomes an image chunk; the chart without
+ # an img_path is skipped (mirrors how IMAGE blocks behave).
+ assert len(media) == 1
+ image, texts = media[0][0]
+ chart_path.unlink()
+ assert isinstance(image, module.Image.Image)
+ assert image.getpixel((0, 0)) == (0, 0, 255)
+ assert texts == ["Figure 3", "Source: dataset", "A blue square"]
+
+
+def test_transfer_to_sections_routes_chart_like_image_per_parse_method(monkeypatch):
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ outputs = [
+ {"type": module.MinerUContentType.TEXT, "text": "Body", "page_idx": 0, "bbox": (0, 0, 1, 1)},
+ {
+ "type": module.MinerUContentType.CHART,
+ "chart_caption": ["figure"],
+ "chart_footnote": [],
+ "page_idx": 0,
+ "bbox": (0, 2, 1, 3),
+ },
+ ]
+
+ # app chunkers consume media separately: the chart is excluded from text sections.
+ for app_method in ("naive", "manual", "paper"):
+ sections = parser._transfer_to_sections(outputs, parse_method=app_method, table_enable=True)
+ assert len(sections) == 1
+ assert sections[0][0].startswith("Body")
+
+ # raw consumers keep the chart as a text section (caption/footnote), like IMAGE.
+ raw_sections = parser._transfer_to_sections(outputs, parse_method="raw", table_enable=True)
+ assert len(raw_sections) == 2
+ assert raw_sections[1][0].strip() == "figure"
+
+
+def test_transfer_to_sections_warns_on_unknown_type(monkeypatch, caplog):
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser()
+ outputs = [
+ {"type": "sidebar", "text": "ignored", "page_idx": 0, "bbox": (0, 0, 1, 1)},
+ ]
+
+ with caplog.at_level(logging.WARNING, logger=parser.logger.name):
+ parser._transfer_to_sections(outputs, parse_method="raw")
+
+ assert "Skip unsupported section type=sidebar" in caplog.text
+
+
@pytest.mark.p1
def test_tokenize_table_uses_payload_type_instead_of_html_content():
from PIL import Image