RAGFlow is an agent-native database: a Retrieval-Augmented Generation (RAG) engine that grew into context infrastructure.
Go agent-harnessagent-native-databaseagentic-aiagentic-retrievalagentic-searchaiai-agentscontext-enginecontext-engineeringcontext-managementharness-engineeringknowledge-compilationragretrieval-augmented-generationretrieval-harnesssearch-harness
fix(mineru): upload the PDF once when several page ranges are configured (#19183)
Yash Raj Pandeyafe404e6d32d
diff --git a/api/db/services/task_service.py b/api/db/services/task_service.pyindex 8132657d86..387a1436f7 100644--- a/api/db/services/task_service.py+++ b/api/db/services/task_service.py@@ -459,7 +459,9 @@ def queue_tasks(doc: dict, bucket: str, name: str, priority: int, user_id: str |(Task rows have no ``user_id`` column).Note:- - For PDF documents, tasks are created per page range based on configuration+ - For PDF documents, tasks are created per page range based on configuration.+ A resume PDF and a MinerU PDF are the two exceptions. Each gets one task that+ covers every configured range- For Excel documents, tasks are created per row range- Task digests are calculated for optimization and reuse- Previous task chunks may be reused if available@@ -504,7 +506,16 @@ def queue_tasks(doc: dict, bucket: str, name: str, priority: int, user_id: str |if doc["parser_id"] == "resume":# The resume parser ignores from_page and to_page, so every task parses the whole# file. Collapse the configured ranges into one range to keep a resume document at a single task.+ logging.info("Document %s uses the resume parser, so one task covers the whole file instead of the ranges %s", doc["id"], page_ranges)page_ranges = [(1, MAXIMUM_PAGE_NUMBER)]+ if is_mineru and len(page_ranges) > 1:+ # One task per configured range means one full upload per range, which is+ # the cost unsplit-task mode exists to remove. Cover every range with a+ # single span instead. MinerUParser._select_configured_pages then drops+ # the blocks that fall in the gaps between the ranges.+ merged = [(min(s for s, _ in page_ranges), max(e for _, e in page_ranges))]+ logging.info("Document %s merged MinerU page ranges %s into one task span %s", doc["id"], page_ranges, merged)+ page_ranges = mergedfor s, e in page_ranges:s -= 1s = max(0, s)diff --git a/deepdoc/parser/mineru_parser.py b/deepdoc/parser/mineru_parser.pyindex 2b0d799dc6..708992d6fa 100644--- a/deepdoc/parser/mineru_parser.py+++ b/deepdoc/parser/mineru_parser.py@@ -868,6 +868,46 @@ class MinerUParser(RAGFlowPdfParser):item[key] = str((subdir / item[key]).resolve())return data+ def _select_configured_pages(self, outputs: list[dict[str, Any]], page_ranges, page_from: int) -> list[dict[str, Any]]:+ """Keep only the blocks whose page falls inside ``page_ranges``.++ The dispatcher covers every configured range with one request, so the PDF is+ uploaded once. MinerU then also returns the pages between the ranges.++ ``page_idx`` counts from ``page_from``. A configured range ``(s, e)`` selects+ the 1-based pages ``s`` to ``e - 1``, because ``queue_tasks`` builds each task+ stop with ``e = min(e - 1, pages)``. The end stays exclusive here so the merged+ span selects the same pages that one task per range selected.++ A cross-page table is one block whose ``page_idx`` is its first page and whose+ ``_mineru_positions`` hold every page it covers. A table that starts on a gap+ page and continues into a configured range must stay, so the block is kept when+ any of its pages is in a range, not only the first one.+ """+ if not page_ranges:+ return outputs++ spans = [(int(s), int(e)) for s, e in page_ranges]+ kept = []+ for output in outputs:+ pages = [output.get("page_idx")]+ pages += [pos.get("page_idx") for pos in output.get("_mineru_positions") or []]+ pages = [int(page_idx) for page_idx in pages if page_idx is not None]+ if not pages:+ # A block with no page cannot be placed on a gap page either. The rest of+ # the parser already treats it as unplaced: _line_tag gives it no position+ # and _middle_positions_for_output returns nothing for it. One task per+ # range kept it too, so dropping it here would lose content.+ kept.append(output)+ continue+ if any(s <= page_from + page_idx + 1 < e for page_idx in pages for s, e in spans):+ kept.append(output)++ dropped = len(outputs) - len(kept)+ if dropped:+ self.logger.info("[MinerU] Dropped %d block(s) outside the configured page ranges %s.", dropped, spans)+ return kept+def _transfer_to_sections(self, outputs: list[dict[str, Any]], parse_method: str = None, table_enable: bool = False):sections = []parse_method = (parse_method or "raw").lower()@@ -1194,6 +1234,7 @@ class MinerUParser(RAGFlowPdfParser):)final_out_dir = self._run_mineru(pdf, out_dir, options, callback=callback, page_from=page_from, page_to=page_to)outputs = self._read_output(final_out_dir, pdf.stem, method=mineru_method_raw_str, backend=backend)+ outputs = self._select_configured_pages(outputs, parser_cfg.get("pages"), page_from)self.logger.info(f"[MinerU] Parsed {len(outputs)} blocks from PDF.")if callback:callback(0.75, f"[MinerU] Parsed {len(outputs)} blocks from PDF.")diff --git a/test/unit_test/api/db/services/test_task_service_mineru_page_ranges.py b/test/unit_test/api/db/services/test_task_service_mineru_page_ranges.pynew file mode 100644index 0000000000..a1a8154ff1--- /dev/null+++ b/test/unit_test/api/db/services/test_task_service_mineru_page_ranges.py@@ -0,0 +1,75 @@+#+# Copyright 2026 The InfiniFlow Authors. All Rights Reserved.+#+# Licensed under the Apache License, Version 2.0 (the "License");+# you may not use this file except in compliance with the License.+# You may obtain a copy of the License at+#+# http://www.apache.org/licenses/LICENSE-2.0+#+# Unless required by applicable law or agreed to in writing, software+# distributed under the License is distributed on an "AS IS" BASIS,+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+# See the License for the specific language governing permissions and+# limitations under the License.+#+import pytest++from api.db.services import task_service as ts+++class _Storage:+ @staticmethod+ def get(_bucket, _name):+ return b"%PDF-1.4"+++def _queued_page_spans(monkeypatch, layout_model_name, pages, total_pages=30):+ """Run queue_tasks on a PDF and return the (from_page, to_page) of each task."""+ captured = []+ parser_config = {"layout_recognize": "a" * 32}+ if pages is not None:+ parser_config["pages"] = pages+ doc = {+ "id": "doc1",+ "name": "manual.pdf",+ "type": ts.FileType.PDF.value,+ "parser_id": "naive",+ "parser_config": parser_config,+ }++ monkeypatch.setattr(ts.settings, "STORAGE_IMPL", _Storage, raising=False)+ monkeypatch.setattr(ts.PdfParser, "total_page_number", staticmethod(lambda *_a, **_k: total_pages))+ monkeypatch.setattr(ts, "get_composite_model_name_by_id", lambda _id: layout_model_name)+ monkeypatch.setattr(ts.DocumentService, "get_chunking_config", staticmethod(lambda _id: {"tenant_id": "t", "kb_id": "k", "parser_config": {}}))+ monkeypatch.setattr(ts.TaskService, "get_tasks", staticmethod(lambda _id: []))+ monkeypatch.setattr(ts.DocumentService, "update_by_id", staticmethod(lambda *_a, **_k: None))+ monkeypatch.setattr(ts.DocumentService, "begin2parse", staticmethod(lambda *_a, **_k: None))+ monkeypatch.setattr(ts, "bulk_insert_into_db", lambda _model, rows, _replace: captured.extend(rows))+ monkeypatch.setattr(ts, "seed_doc_chunking_counter", lambda *_a, **_k: True)+ monkeypatch.setattr(ts.REDIS_CONN, "queue_product", lambda *_a, **_k: True)++ ts.queue_tasks(doc, "bucket", "manual.pdf", 0)+ return [(task["from_page"], task["to_page"]) for task in captured]+++class TestQueueTasksMinerUPageRanges:+ def test_four_ranges_become_one_task(self, monkeypatch):+ # Every task uploads the whole PDF to the MinerU API server, so four+ # configured ranges must still produce a single request.+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", [(1, 5), (8, 12), (15, 20), (25, 30)]) == [(0, 29)]++ def test_two_ranges_span_the_gap(self, monkeypatch):+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", [(1, 10), (20, 30)]) == [(0, 29)]++ def test_one_range_is_unchanged(self, monkeypatch):+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", [(1, 10)]) == [(0, 9)]++ def test_no_range_is_unchanged(self, monkeypatch):+ assert _queued_page_spans(monkeypatch, "vlm@MinerU", None) == [(0, 30)]++ def test_other_parsers_keep_one_task_per_range(self, monkeypatch):+ # The collapse is MinerU-only. DeepDOC reads the file once per task, so+ # it keeps a task per range and splits each range by task_page_size.+ assert _queued_page_spans(monkeypatch, "vlm@DeepDOC", [(1, 10), (20, 30)]) == [(0, 9), (19, 29)]diff --git a/test/unit_test/api/db/services/test_task_service_queue_tasks.py b/test/unit_test/api/db/services/test_task_service_queue_tasks.pyindex ec28bc222e..7cf573ceb2 100644--- a/test/unit_test/api/db/services/test_task_service_queue_tasks.py+++ b/test/unit_test/api/db/services/test_task_service_queue_tasks.py@@ -62,3 +62,20 @@ def test_queue_tasks_keeps_multiple_page_ranges_for_naive(monkeypatch):queued_tasks = _queue_tasks(monkeypatch, "naive", {"pages": [[1, 5], [10, 15]], "task_page_size": 12})assert len(queued_tasks) > 1+++def test_queue_tasks_logs_the_resume_collapse(monkeypatch, caplog):+ with caplog.at_level("INFO"):+ _queue_tasks(monkeypatch, "resume", {"pages": [[1, 5], [10, 15]]})++ assert "Document doc-id uses the resume parser" in caplog.text+ assert "[[1, 5], [10, 15]]" in caplog.text+++def test_queue_tasks_does_not_log_the_resume_collapse_for_naive(monkeypatch, caplog):+ with caplog.at_level("INFO"):+ _queue_tasks(monkeypatch, "naive", {"pages": [[1, 5], [10, 15]], "task_page_size": 12})++ assert "uses the resume parser" not in caplog.textdiff --git a/test/unit_test/deepdoc/parser/test_mineru_parser.py b/test/unit_test/deepdoc/parser/test_mineru_parser.pyindex f8ed12784a..440e93aa1a 100644--- a/test/unit_test/deepdoc/parser/test_mineru_parser.py+++ b/test/unit_test/deepdoc/parser/test_mineru_parser.py@@ -1689,3 +1689,137 @@ def test_initial_last_image_coverage_has_vlm_configured_key(monkeypatch):parser = module.MinerUParser()assert "vlm_configured" in parser.last_image_coverageassert parser.last_image_coverage["vlm_configured"] is False+++def _page_filter_parser(monkeypatch):+ module = _load_mineru_parser(monkeypatch)+ return module.MinerUParser(mineru_api="http://mineru.local")+++def test_select_configured_pages_drops_blocks_between_the_ranges(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ # One request covers the merged span, so MinerU also returns pages 10..19.+ outputs = [{"page_idx": i, "text": f"p{i + 1}"} for i in range(29)]+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)++ assert [o["text"] for o in kept] == [f"p{n}" for n in list(range(1, 10)) + list(range(20, 30))]+++def test_select_configured_pages_matches_one_task_per_range(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ # queue_tasks turns a range (s, e) into a task stop of e - 1, so one task per+ # range covered 1..9 and 20..29. The merged span must select the same pages.+ outputs = [{"page_idx": i, "text": f"p{i + 1}"} for i in range(29)]+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)++ assert [o["text"] for o in kept][-1] == "p29"+ assert "p10" not in [o["text"] for o in kept]+++def test_select_configured_pages_offsets_page_idx_by_page_from(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ # The task starts at page_from=5, so page_idx 0 is document page 6.+ outputs = [{"page_idx": i, "text": f"p{5 + i + 1}"} for i in range(10)]+ kept = parser._select_configured_pages(outputs, [(6, 7), (12, 14)], 5)++ assert [o["text"] for o in kept] == ["p6", "p12", "p13"]+++def test_select_configured_pages_keeps_everything_without_ranges(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ outputs = [{"page_idx": 0, "text": "a"}, {"page_idx": 9, "text": "b"}]++ assert parser._select_configured_pages(outputs, None, 0) == outputs+ assert parser._select_configured_pages(outputs, [], 0) == outputs+++def test_select_configured_pages_keeps_a_block_with_no_page(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ outputs = [{"page_idx": 0, "text": "in"}, {"text": "unplaced"}, {"page_idx": 14, "text": "gap"}]+ kept = parser._select_configured_pages(outputs, [(1, 10)], 0)++ assert [o["text"] for o in kept] == ["in", "unplaced"]+++def test_parse_pdf_sends_one_merged_span_to_the_mineru_api(monkeypatch, tmp_path):+ module = _load_mineru_parser(monkeypatch)+ parser = module.MinerUParser(mineru_api="http://mineru.local")+ pdf_path = tmp_path / "document.pdf"+ pdf_path.write_bytes(b"%PDF-1.4 fake")+ output_dir = tmp_path / "output"+ output_dir.mkdir()++ posted = {}++ def fake_post(**kwargs):+ posted.update(kwargs["data"])+ return _FakePostContext(_FakeZipResponse(), posted)++ monkeypatch.setattr(module, "extract_pdf_outlines", Mock(return_value=[]))+ monkeypatch.setattr(parser, "__images__", Mock())+ monkeypatch.setattr(module.requests, "post", fake_post)+ monkeypatch.setattr(parser, "_extract_zip_no_root", Mock())+ # MinerU parses the whole merged span, so it returns the gap pages as well.+ monkeypatch.setattr(+ parser,+ "_read_output",+ Mock(return_value=[{"type": module.MinerUContentType.TEXT, "page_idx": i, "bbox": [0, 0, 1, 1], "text": f"p{i + 1}"} for i in range(29)]),+ )++ sections, _tables = parser.parse_pdf(+ filepath=pdf_path,+ binary=None,+ output_dir=str(output_dir),+ delete_output=False,+ page_from=0,+ page_to=29,+ parser_config={"pages": [(1, 10), (20, 30)]},+ )++ # One request for the merged span. start_page_id is 0-based, end_page_id is+ # 0-based inclusive, so pages 1 to 29 become 0 and 28.+ assert posted["start_page_id"] == 0+ assert posted["end_page_id"] == 28++ kept = [section for section, _tag in sections]+ assert kept == [f"p{n}" for n in list(range(1, 10)) + list(range(20, 30))]+++def test_select_configured_pages_keeps_a_table_that_reaches_into_a_range(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ # _enrich_outputs_with_middle_positions gives a cross-page table every page it+ # covers. This one starts on gap page 19 and continues onto configured page 20.+ outputs = [+ {"page_idx": 18, "text": "gap only"},+ {+ "page_idx": 18,+ "text": "table 19-20",+ "_mineru_positions": [{"page_idx": 18}, {"page_idx": 19}],+ },+ ]+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)++ assert [o["text"] for o in kept] == ["table 19-20"]+++def test_select_configured_pages_keeps_a_table_that_reaches_out_of_a_range(monkeypatch):+ parser = _page_filter_parser(monkeypatch)++ # The mirror case: the table starts on configured page 9 and runs onto gap page 10.+ outputs = [+ {+ "page_idx": 8,+ "text": "table 9-10",+ "_mineru_positions": [{"page_idx": 8}, {"page_idx": 9}],+ },+ {"page_idx": 9, "text": "gap only"},+ ]+ kept = parser._select_configured_pages(outputs, [(1, 10), (20, 30)], 0)++ assert [o["text"] for o in kept] == ["table 9-10"]