Skip to content

RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.

fix(mineru): upload the PDF once when several page ranges are configured (#19183)

Yash Raj Pandeyafe404e6d32d
diff --git a/api/db/services/task_service.py b/api/db/services/task_service.py
index 8132657d86..387a1436f7 100644
--- a/api/db/services/task_service.py
+++ b/api/db/services/task_service.py
@@ -459,7 +459,9 @@ def queue_tasks(doc: dict, bucket: str, name: str, priority: int, user_id: str |
(Task rows have no ``user_id`` column).
Note:
- - For PDF documents, tasks are created per page range based on configuration
+ - For PDF documents, tasks are created per page range based on configuration.
+ A resume PDF and a MinerU PDF are the two exceptions. Each gets one task that
+ covers every configured range
- For Excel documents, tasks are created per row range
- Task digests are calculated for optimization and reuse
- Previous task chunks may be reused if available
@@ -504,7 +506,16 @@ def queue_tasks(doc: dict, bucket: str, name: str, priority: int, user_id: str |
if doc["parser_id"] == "resume":
# The resume parser ignores from_page and to_page, so every task parses the whole
# file. Collapse the configured ranges into one range to keep a resume document at a single task.
+ logging.info("Document %s uses the resume parser, so one task covers the whole file instead of the ranges %s", doc["id"], page_ranges)
page_ranges = [(1, MAXIMUM_PAGE_NUMBER)]
+ if is_mineru and len(page_ranges) > 1:
+ # One task per configured range means one full upload per range, which is
+ # the cost unsplit-task mode exists to remove. Cover every range with a
+ # single span instead. MinerUParser._select_configured_pages then drops
+ # the blocks that fall in the gaps between the ranges.
+ merged = [(min(s for s, _ in page_ranges), max(e for _, e in page_ranges))]
+ logging.info("Document %s merged MinerU page ranges %s into one task span %s", doc["id"], page_ranges, merged)
+ page_ranges = merged
for s, e in page_ranges:
s -= 1
s = max(0, s)
diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.py
index 2b0d799dc6..708992d6fa 100644
--- a/deepdoc/parser/mineru_parser.py
+++ b/deepdoc/parser/mineru_parser.py
@@ -868,6 +868,46 @@ class MinerUParser(RAGFlowPdfParser):
item[key] = str((subdir / item[key]).resolve())
return data
+ def _select_configured_pages(self, outputs: list[dict[str, Any]], page_ranges, page_from: int) -> list[dict[str, Any]]:
+ """Keep only the blocks whose page falls inside ``page_ranges``.
+
+ The dispatcher covers every configured range with one request, so the PDF is
+ uploaded once. MinerU then also returns the pages between the ranges.
+
+ ``page_idx`` counts from ``page_from``. A configured range ``(s, e)`` selects
+ the 1-based pages ``s`` to ``e - 1``, because ``queue_tasks`` builds each task
+ stop with ``e = min(e - 1, pages)``. The end stays exclusive here so the merged
+ span selects the same pages that one task per range selected.
+
+ A cross-page table is one block whose ``page_idx`` is its first page and whose
+ ``_mineru_positions`` hold every page it covers. A table that starts on a gap
+ page and continues into a configured range must stay, so the block is kept when
+ any of its pages is in a range, not only the first one.
+ """
+ if not page_ranges:
+ return outputs
+
+ spans = [(int(s), int(e)) for s, e in page_ranges]
+ kept = []
+ for output in outputs:
+ pages = [output.get("page_idx")]
+ pages += [pos.get("page_idx") for pos in output.get("_mineru_positions") or []]
+ pages = [int(page_idx) for page_idx in pages if page_idx is not None]
+ if not pages:
+ # A block with no page cannot be placed on a gap page either. The rest of
+ # the parser already treats it as unplaced: _line_tag gives it no position
+ # and _middle_positions_for_output returns nothing for it. One task per
+ # range kept it too, so dropping it here would lose content.
+ kept.append(output)
+ continue
+ if any(s <= page_from + page_idx + 1 < e for page_idx in pages for s, e in spans):
+ kept.append(output)
+
+ dropped = len(outputs) - len(kept)
+ if dropped:
+ self.logger.info("[MinerU] Dropped %d block(s) outside the configured page ranges %s.", dropped, spans)
+ return kept
+
def _transfer_to_sections(self, outputs: list[dict[str, Any]], parse_method: str = None, table_enable: bool = False):
sections = []
parse_method = (parse_method or "raw").lower()
@@ -1194,6 +1234,7 @@ class MinerUParser(RAGFlowPdfParser):
)
final_out_dir = self._run_mineru(pdf, out_dir, options, callback=callback, page_from=page_from, page_to=page_to)
outputs = self._read_output(final_out_dir, pdf.stem, method=mineru_method_raw_str, backend=backend)
+ outputs = self._select_configured_pages(outputs, parser_cfg.get("pages"), page_from)
self.logger.info(f"[MinerU] Parsed {len(outputs)} blocks from PDF.")
if callback:
callback(0.75, f"[MinerU] Parsed {len(outputs)} blocks from PDF.")
diff --git a/test/unit_test/api/db/services/test_task_service_mineru_page_ranges.py b/test/unit_test/api/db/services/test_task_service_mineru_page_ranges.py
new file mode 100644
index 0000000000..a1a8154ff1
--- /dev/null
+++ b/test/unit_test/api/db/services/test_task_service_mineru_page_ranges.py
@@ -0,0 +1,75 @@
+#
+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+import pytest
+
+from api.db.services import task_service as ts
+
+
+class _Storage:
+ @staticmethod
+ def get(_bucket, _name):
+ return b"%PDF-1.4"
+
+
+def _queued_page_spans(monkeypatch, layout_model_name, pages, total_pages=30):
+ """Run queue_tasks on a PDF and return the (from_page, to_page) of each task."""
+ captured = []
+ parser_config = {"layout_recognize": "a" * 32}
+ if pages is not None:
+ parser_config["pages"] = pages
+ doc = {
+ "id": "doc1",
+ "name": "manual.pdf",
+ "type": ts.FileType.PDF.value,
+ "parser_id": "naive",
+ "parser_config": parser_config,
+ }
+
+ monkeypatch.setattr(ts.settings, "STORAGE_IMPL", _Storage, raising=False)
+ monkeypatch.setattr(ts.PdfParser, "total_page_number", staticmethod(lambda *_a, **_k: total_pages))
+ monkeypatch.setattr(ts, "get_composite_model_name_by_id", lambda _id: layout_model_name)
+ monkeypatch.setattr(ts.DocumentService, "get_chunking_config", staticmethod(lambda _id: {"tenant_id": "t", "kb_id": "k", "parser_config": {}}))
+ monkeypatch.setattr(ts.TaskService, "get_tasks", staticmethod(lambda _id: []))
+ monkeypatch.setattr(ts.DocumentService, "update_by_id", staticmethod(lambda *_a, **_k: None))
+ monkeypatch.setattr(ts.DocumentService, "begin2parse", staticmethod(lambda *_a, **_k: None))
+ monkeypatch.setattr(ts, "bulk_insert_into_db", lambda _model, rows, _replace: captured.extend(rows))
+ monkeypatch.setattr(ts, "seed_doc_chunking_counter", lambda *_a, **_k: True)
+ monkeypatch.setattr(ts.REDIS_CONN, "queue_product", lambda *_a, **_k: True)
+
+ ts.queue_tasks(doc, "bucket", "manual.pdf", 0)
+ return [(task["from_page"], task["to_page"]) for task in captured]
+
+
+class TestQueueTasksMinerUPageRanges:
+ def test_four_ranges_become_one_task(self, monkeypatch):
+ # Every task uploads the whole PDF to the MinerU API server, so four
+ # configured ranges must still produce a single request.
+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", [(1, 5), (8, 12), (15, 20), (25, 30)]) == [(0, 29)]
+
+ def test_two_ranges_span_the_gap(self, monkeypatch):
+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", [(1, 10), (20, 30)]) == [(0, 29)]
+
+ def test_one_range_is_unchanged(self, monkeypatch):
+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", [(1, 10)]) == [(0, 9)]
+
+ def test_no_range_is_unchanged(self, monkeypatch):
+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", None) == [(0, 30)]
+
+ def test_other_parsers_keep_one_task_per_range(self, monkeypatch):
+ # The collapse is MinerU-only. DeepDOC reads the file once per task, so
+ # it keeps a task per range and splits each range by task_page_size.
+ assert _queued_page_spans(monkeypatch, "vlm@DeepDOC", [(1, 10), (20, 30)]) == [(0, 9), (19, 29)]
diff --git a/test/unit_test/api/db/services/test_task_service_queue_tasks.py b/test/unit_test/api/db/services/test_task_service_queue_tasks.py
index ec28bc222e..7cf573ceb2 100644
--- a/test/unit_test/api/db/services/test_task_service_queue_tasks.py
+++ b/test/unit_test/api/db/services/test_task_service_queue_tasks.py
@@ -62,3 +62,20 @@ def test_queue_tasks_keeps_multiple_page_ranges_for_naive(monkeypatch):
queued_tasks = _queue_tasks(monkeypatch, "naive", {"pages": [[1, 5], [10, 15]], "task_page_size": 12})
assert len(queued_tasks) > 1
+
+
+def test_queue_tasks_logs_the_resume_collapse(monkeypatch, caplog):
+ with caplog.at_level("INFO"):
+ _queue_tasks(monkeypatch, "resume", {"pages": [[1, 5], [10, 15]]})
+
+ assert "Document doc-id uses the resume parser" in caplog.text
+ assert "[[1, 5], [10, 15]]" in caplog.text
+
+
+def test_queue_tasks_does_not_log_the_resume_collapse_for_naive(monkeypatch, caplog):
+ with caplog.at_level("INFO"):
+ _queue_tasks(monkeypatch, "naive", {"pages": [[1, 5], [10, 15]], "task_page_size": 12})
+
+ assert "uses the resume parser" not in caplog.text
diff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.py
index f8ed12784a..440e93aa1a 100644
--- a/test/unit_test/deepdoc/parser/test_mineru_parser.py
+++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py
@@ -1689,3 +1689,137 @@ def test_initial_last_image_coverage_has_vlm_configured_key(monkeypatch):
parser = module.MinerUParser()
assert "vlm_configured" in parser.last_image_coverage
assert parser.last_image_coverage["vlm_configured"] is False
+
+
+def _page_filter_parser(monkeypatch):
+ module = _load_mineru_parser(monkeypatch)
+ return module.MinerUParser(mineru_api="http://mineru.local")
+
+
+def test_select_configured_pages_drops_blocks_between_the_ranges(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ # One request covers the merged span, so MinerU also returns pages 10..19.
+ outputs = [{"page_idx": i, "text": f"p{i + 1}"} for i in range(29)]
+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)
+
+ assert [o["text"] for o in kept] == [f"p{n}" for n in list(range(1, 10)) + list(range(20, 30))]
+
+
+def test_select_configured_pages_matches_one_task_per_range(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ # queue_tasks turns a range (s, e) into a task stop of e - 1, so one task per
+ # range covered 1..9 and 20..29. The merged span must select the same pages.
+ outputs = [{"page_idx": i, "text": f"p{i + 1}"} for i in range(29)]
+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)
+
+ assert [o["text"] for o in kept][-1] == "p29"
+ assert "p10" not in [o["text"] for o in kept]
+
+
+def test_select_configured_pages_offsets_page_idx_by_page_from(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ # The task starts at page_from=5, so page_idx 0 is document page 6.
+ outputs = [{"page_idx": i, "text": f"p{5 + i + 1}"} for i in range(10)]
+ kept = parser._select_configured_pages(outputs, [(6, 7), (12, 14)], 5)
+
+ assert [o["text"] for o in kept] == ["p6", "p12", "p13"]
+
+
+def test_select_configured_pages_keeps_everything_without_ranges(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ outputs = [{"page_idx": 0, "text": "a"}, {"page_idx": 9, "text": "b"}]
+
+ assert parser._select_configured_pages(outputs, None, 0) == outputs
+ assert parser._select_configured_pages(outputs, [], 0) == outputs
+
+
+def test_select_configured_pages_keeps_a_block_with_no_page(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ outputs = [{"page_idx": 0, "text": "in"}, {"text": "unplaced"}, {"page_idx": 14, "text": "gap"}]
+ kept = parser._select_configured_pages(outputs, [(1, 10)], 0)
+
+ assert [o["text"] for o in kept] == ["in", "unplaced"]
+
+
+def test_parse_pdf_sends_one_merged_span_to_the_mineru_api(monkeypatch, tmp_path):
+ module = _load_mineru_parser(monkeypatch)
+ parser = module.MinerUParser(mineru_api="http://mineru.local")
+ pdf_path = tmp_path / "document.pdf"
+ pdf_path.write_bytes(b"%PDF-1.4 fake")
+ output_dir = tmp_path / "output"
+ output_dir.mkdir()
+
+ posted = {}
+
+ def fake_post(**kwargs):
+ posted.update(kwargs["data"])
+ return _FakePostContext(_FakeZipResponse(), posted)
+
+ monkeypatch.setattr(module, "extract_pdf_outlines", Mock(return_value=[]))
+ monkeypatch.setattr(parser, "__images__", Mock())
+ monkeypatch.setattr(module.requests, "post", fake_post)
+ monkeypatch.setattr(parser, "_extract_zip_no_root", Mock())
+ # MinerU parses the whole merged span, so it returns the gap pages as well.
+ monkeypatch.setattr(
+ parser,
+ "_read_output",
+ Mock(return_value=[{"type": module.MinerUContentType.TEXT, "page_idx": i, "bbox": [0, 0, 1, 1], "text": f"p{i + 1}"} for i in range(29)]),
+ )
+
+ sections, _tables = parser.parse_pdf(
+ filepath=pdf_path,
+ binary=None,
+ output_dir=str(output_dir),
+ delete_output=False,
+ page_from=0,
+ page_to=29,
+ parser_config={"pages": [(1, 10), (20, 30)]},
+ )
+
+ # One request for the merged span. start_page_id is 0-based, end_page_id is
+ # 0-based inclusive, so pages 1 to 29 become 0 and 28.
+ assert posted["start_page_id"] == 0
+ assert posted["end_page_id"] == 28
+
+ kept = [section for section, _tag in sections]
+ assert kept == [f"p{n}" for n in list(range(1, 10)) + list(range(20, 30))]
+
+
+def test_select_configured_pages_keeps_a_table_that_reaches_into_a_range(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ # _enrich_outputs_with_middle_positions gives a cross-page table every page it
+ # covers. This one starts on gap page 19 and continues onto configured page 20.
+ outputs = [
+ {"page_idx": 18, "text": "gap only"},
+ {
+ "page_idx": 18,
+ "text": "table 19-20",
+ "_mineru_positions": [{"page_idx": 18}, {"page_idx": 19}],
+ },
+ ]
+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)
+
+ assert [o["text"] for o in kept] == ["table 19-20"]
+
+
+def test_select_configured_pages_keeps_a_table_that_reaches_out_of_a_range(monkeypatch):
+ parser = _page_filter_parser(monkeypatch)
+
+ # The mirror case: the table starts on configured page 9 and runs onto gap page 10.
+ outputs = [
+ {
+ "page_idx": 8,
+ "text": "table 9-10",
+ "_mineru_positions": [{"page_idx": 8}, {"page_idx": 9}],
+ },
+ {"page_idx": 9, "text": "gap only"},
+ ]
+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)
+
+ assert [o["text"] for o in kept] == ["table 9-10"]