Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix(cli): preserve full-document output after parsing

myhloli7ce2f073029a
diff --git a/mineru/cli/commands/parse.py b/mineru/cli/commands/parse.py
index dbfd7a7a..6733dbed 100644
--- a/mineru/cli/commands/parse.py
+++ b/mineru/cli/commands/parse.py
@@ -21,7 +21,8 @@ from ...doclib.types import (
TelemetryObservationsRequest,
)
from ...errors import MineruError, error_response
-from ...parser.page_range import normalize_page_range_input
+from ...filetypes import is_page_range_parse_extension
+from ...parser.page_range import format_page_range, normalize_page_range_input, parse_page_range_set
from ...types import Tier
from ...utils.i18n import t
from ..contracts import CliContext, CliResult
@@ -236,6 +237,11 @@ def _parse(
if st == "done":
_record_parse_wait(client, wait_parse_ids, "succeeded", wait_started_at)
done_result = result.model_copy(update={"status": "done"})
+ if not is_page_range_parse_extension(file_path):
+ # 非 PDF 的初始页数只是估计,完成后以整本任务的实际逻辑页范围为准。
+ done_result.page_range = format_page_range(
+ page_no for row in parse_rows for page_no in parse_page_range_set(row.page_range)
+ )
return _prepare_parse_result(
client,
done_result,
@@ -313,7 +319,8 @@ def _prepare_parse_result(
"""Fetch and prepare parsed content for CLI output."""
sha256 = parse_result.sha256
tier = parse_result.tier
- page_range = parse_result.page_range
+ # 非 PDF 始终读取整本,避免把调度时的预估页数当成输出过滤条件。
+ page_range = parse_result.page_range if source_path is None or is_page_range_parse_extension(source_path) else None
output_path = _ensure_output_parent(output)
if output_path and output_path != "-":
exported = client.export_doc_content(
diff --git a/mineru/doclib/server.py b/mineru/doclib/server.py
index 350a21c7..bcb9f42a 100644
--- a/mineru/doclib/server.py
+++ b/mineru/doclib/server.py
@@ -21,7 +21,7 @@ from pydantic import ValidationError
from ..config import config
from ..errors import InvalidRequestError, MineruError, NotFoundError, error_response, http_status_for
-from ..filetypes import IMAGE_EXTENSIONS, TEXT_EXTENSIONS, TEXT_FILE_TYPES
+from ..filetypes import IMAGE_EXTENSIONS, TEXT_EXTENSIONS, TEXT_FILE_TYPES, is_page_range_parse_extension
from ..parser.page_range import (
count_pages_in_range,
expand_page_range as _expand_page_range,
@@ -683,7 +683,9 @@ class DoclibServer(AsyncDoclibInterface):
limit = max(1, limit)
doc = await self._doc_for_ref(doc_ref, param="doc_ref")
- normalized_page_range = _normalize_content_page_range(page_range, after, doc)
+ # 非 PDF 的省略范围表示整本,逻辑分节数不应触发 PDF 的前十页默认限制。
+ full_document = not is_page_range_parse_extension(doc["file_type"]) and not normalize_page_range_input(page_range)
+ normalized_page_range = None if full_document else _normalize_content_page_range(page_range, after, doc)
after_cursor = _parse_after_cursor(after)
if after_cursor:
_validate_cursor_for_doc(after_cursor, doc, tier, normalized_page_range)
@@ -1353,6 +1355,14 @@ class DoclibServer(AsyncDoclibInterface):
page_count = doc["page_count"] if doc else None
paginated = _is_paginated_doc(doc) if doc else True
+ if (
+ plan.next_mode == "parse"
+ and plan.page_range is None
+ and doc
+ and not is_page_range_parse_extension(doc["file_type"])
+ ):
+ # 整本格式仅用内容游标续读,避免生成它不接受的 --pages 解析参数。
+ paginated = False
if plan.next_mode == "range":
next_request = _next_range_read_request(rendered, plan.locator)
elif plan.next_mode == "read":
diff --git a/pyproject.toml b/pyproject.toml
index 65e4a24d..12c2dd05 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -38,7 +38,7 @@ classifiers = [
"Programming Language :: Python :: 3.14",
]
dependencies = [
- "docvortex>=0.5.17,<1",
+ "docvortex>=0.5.18,<1",
"gradio>=6.8,<7",
"mineru[torch] ; sys_platform == 'darwin' and platform_machine == 'arm64'",
"mineru-vl-utils>=2.0.7,<3",
diff --git a/tests/unittest/test_cli_parse_full_document.py b/tests/unittest/test_cli_parse_full_document.py
new file mode 100644
index 00000000..7b1af0d1
--- /dev/null
+++ b/tests/unittest/test_cli_parse_full_document.py
@@ -0,0 +1,412 @@
+"""验证整本格式从首次解析到 CLI 输出的范围不会被预估页数截断。"""
+
+from __future__ import annotations
+
+import asyncio
+import json
+import re
+import shlex
+from collections.abc import Coroutine, Iterator
+from pathlib import Path
+from types import SimpleNamespace
+from typing import Any, TypeVar
+
+import pytest
+from docx import Document
+from docx.enum.section import WD_SECTION_START
+from typer.testing import CliRunner
+
+from mineru.backend import analyze
+from mineru import parser
+from mineru.cli.commands import parse as parse_cmd
+from mineru.cli.main import app
+from mineru.doclib.core.db import DatabaseManager
+from mineru.doclib.core.fts import FTSManager
+from mineru.doclib.server import DoclibServer
+from mineru.doclib.services.config_svc import ConfigService
+from mineru.doclib.services.parse_svc import ParseService
+from mineru.doclib.types import (
+ ContentRequestScope,
+ DocContentExportRequest,
+ DocContentExportResponse,
+ DocContentResponse,
+ ListParsesResponse,
+ ParseInfo,
+ ParseRequest,
+ ParseResponse,
+ TelemetryObservationsRequest,
+)
+from mineru.parser import mineru_parser
+from mineru.types import Tier
+
+_T = TypeVar("_T")
+
+
+class _DoclibHarness:
+ """用临时数据库连接真实解析与导出服务,避免访问常驻服务。"""
+
+ def __init__(self, root: Path, section_count: int = 3) -> None:
+ """创建多分节真实 DOCX 和独立的数据库、任务队列。"""
+ self.source = root / "multi_section.docx"
+ document = Document()
+ for section in range(1, section_count + 1):
+ if section > 1:
+ document.add_section(WD_SECTION_START.NEW_PAGE)
+ document.add_paragraph(f"SECTION-{section}-MARKER")
+ for paragraph in range(20):
+ document.add_paragraph(f"Section {section} paragraph {paragraph}: 这是测试文本,用于验证多分节解析完整性。")
+ document.save(self.source)
+ self.loop = asyncio.new_event_loop()
+ self.db = DatabaseManager(str(root / "doclib.db"))
+ self.run(self.db.initialize())
+ self.service = ParseService(
+ self.db,
+ FTSManager(self.db),
+ ConfigService(self.db),
+ str(root / "data"),
+ parse_lock_timeout_sec=1800,
+ )
+ self.server = DoclibServer(SimpleNamespace(db=self.db, data_dir=str(root / "data"), parse_svc=self.service))
+ self.output_ranges: list[str | None] = []
+
+ def run(self, operation: Coroutine[Any, Any, _T]) -> _T:
+ """在同一事件循环中执行 CLI 适配器发起的服务调用。"""
+ return self.loop.run_until_complete(operation)
+
+ async def process_pending(self) -> None:
+ """执行隔离队列中的任务,保留真实 DOCX 解析与结果持久化。"""
+ while (task := await self.service.acquire_task()) is not None:
+ assert await self.service.process_doc(task)
+
+ def close(self) -> None:
+ """关闭解析使用的线程池和事件循环。"""
+ self.run(self.loop.shutdown_default_executor())
+ self.loop.close()
+
+
+def doclib_harness(tmp_path: Path, monkeypatch: pytest.MonkeyPatch, request: pytest.FixtureRequest) -> Iterator[_DoclibHarness]:
+ """只替换客户端传输和轮询等待,其余路径均使用真实实现。"""
+
+ def _keep_process_logging() -> None:
+ """避免后台解析把全局日志 sink 绑定到短命的 CliRunner 捕获流。"""
+
+ monkeypatch.setattr(analyze, "configure_global_log_level", _keep_process_logging)
+ monkeypatch.setattr(parser, "configure_global_log_level", _keep_process_logging)
+ monkeypatch.setattr(mineru_parser, "configure_global_log_level", _keep_process_logging)
+ harness = _DoclibHarness(tmp_path, section_count=getattr(request, "param", 3))
+
+ class _Client:
+ """把 CLI 的同步调用连接到隔离的异步 Doclib 服务。"""
+
+ def __init__(self, *, timeout: int) -> None:
+ """接受生产客户端超时配置,不建立网络连接。"""
+
+ def ensure_parse(self, request: ParseRequest) -> ParseResponse:
+ """使用真实入库、缓存查找和任务调度。"""
+ return harness.run(
+ harness.service.request_parse(
+ request.path,
+ tier=request.tier,
+ page_range=request.page_range,
+ force=request.force,
+ remote=request.remote,
+ )
+ )
+
+ def list_parses(self, *, ids: list[int]) -> ListParsesResponse:
+ """在状态轮询前完成后台解析并读取实际持久化范围。"""
+ harness.run(harness.process_pending())
+ return harness.run(harness.server.list_parses(ids=ids))
+
+ def get_doc_content(
+ self,
+ doc_ref: str,
+ *,
+ tier: Tier,
+ page_range: str | None,
+ after: str | None,
+ limit: int,
+ format: str,
+ no_marker: bool,
+ ) -> DocContentResponse:
+ """记录范围并调用真实的内容分页和 Markdown 渲染。"""
+ harness.output_ranges.append(page_range)
+ return harness.run(
+ harness.server.get_doc_content(
+ doc_ref,
+ tier=tier,
+ page_range=page_range,
+ after=after,
+ limit=limit,
+ format=format,
+ no_marker=no_marker,
+ )
+ )
+
+ def export_doc_content(self, doc_ref: str, request: DocContentExportRequest) -> DocContentExportResponse:
+ """记录范围并将真实缓存内容导出到测试目录。"""
+ harness.output_ranges.append(request.page_range)
+ return harness.run(harness.server.export_doc_content(doc_ref, request))
+
+ def record_observations(self, request: TelemetryObservationsRequest) -> None:
+ """隔离回归不发送使用统计。"""
+
+ def _skip_wait(seconds: float) -> None:
+ """后台任务由测试主动驱动,不需要真实轮询延迟。"""
+
+ monkeypatch.setenv("CI", "1")
+ monkeypatch.setattr(parse_cmd, "DoclibClient", _Client)
+ monkeypatch.setattr(parse_cmd.time, "sleep", _skip_wait)
+ try:
+ yield harness
+ finally:
+ harness.close()
+
+
[email protected]("scenario", ["first", "cache", "force", "reenter"])
[email protected]("output_mode", ["stdout", "file", "json", "json_file"])
+def test_multisection_docx_cli_keeps_full_document(
+ doclib_harness: _DoclibHarness,
+ tmp_path: Path,
+ scenario: str,
+ output_mode: str,
+) -> None:
+ """首次、缓存、重解析和异步重入均应按序保留三节全部段落。"""
+ harness = doclib_harness
+ runner = CliRunner()
+ args = ["parse", str(harness.source), "--tier", "flash", "--no-marker"]
+ if scenario in {"cache", "force"}:
+ harness.run(harness.service.request_parse(str(harness.source), tier="flash"))
+ harness.run(harness.process_pending())
+ elif scenario == "reenter":
+ pending = runner.invoke(app, [*args, "--no-wait", "--json"])
+ assert pending.exit_code == 0, pending.output
+ assert json.loads(pending.stdout)["parse"]["status"] == "pending"
+ if scenario == "force":
+ args.append("--force")
+ output = tmp_path / "out.md"
+ if "file" in output_mode:
+ args.extend(["-o", str(output)])
+ if "json" in output_mode:
+ args.append("--json")
+
+ result = runner.invoke(app, args)
+
+ assert result.exit_code == 0, result.output
+ assert harness.output_ranges == [None]
+ payload = json.loads(result.stdout) if "json" in output_mode else None
+ if payload is not None:
+ assert payload["parse"]["status"] == "done"
+ assert payload["parse"]["page_range"] == "1-3"
+ if "file" in output_mode:
+ content = output.read_text(encoding="utf-8")
+ elif payload is not None:
+ content = payload["content"]["content"]
+ assert payload["content"]["request_scope"]["page_range"] is None
+ assert payload["content"]["truncated"] is False
+ else:
+ content = result.stdout
+ expected = [
+ text
+ for section in (1, 2, 3)
+ for text in [f"SECTION-{section}-MARKER", *[f"Section {section} paragraph {paragraph}:" for paragraph in range(20)]]
+ ]
+ assert all(content.count(text) == 1 for text in expected)
+ positions = [content.index(text) for text in expected]
+ assert positions == sorted(positions)
+
+
+def test_multisection_docx_cli_cursor_reaches_last_section(doclib_harness: _DoclibHarness) -> None:
+ """整本读取仍遵守 stdout 限额,按返回游标续读时不漏段或重复。"""
+ harness = doclib_harness
+ runner = CliRunner()
+ args = ["parse", str(harness.source), "--tier", "flash", "--no-marker", "--limit", "200", "--json"]
+ chunks: list[str] = []
+ after: str | None = None
+ for _ in range(100):
+ result = runner.invoke(app, args + (["--after", after] if after else []))
+ assert result.exit_code == 0, result.output
+ payload = json.loads(result.stdout)
+ assert payload["parse"]["page_range"] == "1-3"
+ content = payload["content"]
+ assert content["request_scope"]["limit"] == 200
+ assert content["request_scope"]["page_range"] is None
+ chunks.append(content["content"])
+ next_request = content["next_request"]
+ if next_request is None:
+ assert content["truncated"] is False
+ break
+ assert content["truncated"] is True
+ assert next_request.get("page_range") is None
+ assert next_request["after"] != after
+ after = next_request["after"]
+ else:
+ pytest.fail("DOCX 内容游标未能在有限次续读后结束")
+ assert len(chunks) > 1
+ combined = "\n".join(chunks)
+ for section in (1, 2, 3):
+ assert combined.count(f"SECTION-{section}-MARKER") == 1
+ for paragraph in range(20):
+ assert combined.count(f"Section {section} paragraph {paragraph}:") == 1
+
+
[email protected]("doclib_harness", [11], indirect=True)
+def test_docx_cli_does_not_apply_pdf_default_page_limit(doclib_harness: _DoclibHarness) -> None:
+ """十一分节 DOCX 的默认 stdout 输出不能套用 PDF 前十页限制。"""
+ result = CliRunner().invoke(app, ["parse", str(doclib_harness.source), "--tier", "flash", "--json"])
+ assert result.exit_code == 0, result.output
+ payload = json.loads(result.stdout)
+ assert payload["parse"]["page_range"] == "1-11"
+ assert payload["content"]["truncated"] is False
+ assert payload["content"]["next_request"] is None
+ assert payload["content"]["request_scope"]["page_range"] is None
+ content = payload["content"]["content"]
+ for section in range(1, 12):
+ assert content.count(f"SECTION-{section}-MARKER") == 1
+ assert content.count(f"Section {section} paragraph 19:") == 1
+
+
[email protected]("doclib_harness", [11], indirect=True)
+def test_docx_cli_next_commands_read_all_sections(doclib_harness: _DoclibHarness) -> None:
+ """执行十一节 DOCX 实际输出的 Next 命令,验证续读不带不合法的 --pages。"""
+ runner = CliRunner()
+ args = ["parse", str(doclib_harness.source), "--tier", "flash", "--limit", "800"]
+ chunks: list[str] = []
+ commands: set[tuple[str, ...]] = set()
+ for _ in range(100):
+ result = runner.invoke(app, args)
+ assert result.exit_code == 0, result.output
+ marker = re.search(r"<!-- Next: (.*?) -->", result.stdout)
+ chunks.append(result.stdout[: marker.start()] if marker else result.stdout)
+ if marker is None:
+ break
+ command = shlex.split(marker.group(1))
+ assert command[0] == "mineru"
+ assert "--pages" not in command
+ assert "--after" in command
+ assert tuple(command) not in commands
+ commands.add(tuple(command))
+ args = command[1:]
+ else:
+ pytest.fail("DOCX 输出的 Next 命令未能读完整本")
+ assert commands
+ combined = "\n".join(chunks)
+ for section in range(1, 12):
+ assert combined.count(f"SECTION-{section}-MARKER") == 1
+ assert combined.count(f"Section {section} paragraph 19:") == 1
+
+
[email protected]("locator_read", [False, True])
+def test_docx_explicit_logical_page_read_stays_scoped(doclib_harness: _DoclibHarness, locator_read: bool) -> None:
+ """整本默认读取不改变内容接口显式范围及定位器的逻辑页选择。"""
+ harness = doclib_harness
+ submitted = harness.run(harness.service.request_parse(str(harness.source), tier="flash"))
+ harness.run(harness.process_pending())
+ if locator_read:
+ response = harness.run(harness.server.read_content(f"doc:{submitted.short_id}/tier:flash/page:2", no_marker=True))
+ else:
+ response = harness.run(harness.server.get_doc_content(submitted.sha256, tier="flash", page_range="2", no_marker=True))
+ assert "SECTION-2-MARKER" in response.content
+ assert "SECTION-1-MARKER" not in response.content
+ assert "SECTION-3-MARKER" not in response.content
+ assert response.request_scope.page_range == "2"
+
+
[email protected]("cache_hit", [False, True])
[email protected]("output_file", [False, True])
+def test_pdf_cli_preserves_requested_range_when_batch_is_wider(
+ tmp_path: Path,
+ monkeypatch: pytest.MonkeyPatch,
+ cache_hit: bool,
+ output_file: bool,
+) -> None:
+ """PDF 请求 2-3 页复用 1-5 页任务时,输出范围仍须保持 2-3。"""
+ source = tmp_path / "document.pdf"
+ source.write_bytes(b"%PDF-1.7\n")
+ observed: list[str | None] = []
+
+ class _Client:
+ """模拟已完成或正在等待的更宽 PDF 缓存任务。"""
+
+ def __init__(self, *, timeout: int) -> None:
+ """接受 CLI 的客户端初始化参数。"""
+
+ def ensure_parse(self, request: ParseRequest) -> ParseResponse:
+ """保留用户范围并关联覆盖五页的已有任务。"""
+ assert request.page_range == "2-3"
+ return ParseResponse(
+ sha256="a" * 64,
+ tier="flash",
+ page_range="2-3",
+ status="done" if cache_hit else "pending",
+ cache_hit=cache_hit,
+ wait_parse_ids=[] if cache_hit else [1],
+ )
+
+ def list_parses(self, *, ids: list[int]) -> ListParsesResponse:
+ """返回比当前请求范围更宽的完成任务。"""
+ return ListParsesResponse(
+ parses=[
+ ParseInfo(
+ id=1,
+ sha256="a" * 64,
+ short_id="aaaaaaa",
+ tier="flash",
+ page_range="1-5",
+ status="done",
+ privacy="local",
+ created_at=1,
+ updated_at=2,
+ )
+ ],
+ total=1,
+ limit=50,
+ )
+
+ def get_doc_content(
+ self,
+ doc_ref: str,
+ *,
+ tier: Tier,
+ page_range: str | None,
+ after: str | None,
+ limit: int,
+ format: str,
+ no_marker: bool,
+ ) -> DocContentResponse:
+ """检查 stdout 读取使用的范围与限额。"""
+ observed.append(page_range)
+ assert after is None
+ assert limit == 30000
+ return DocContentResponse(
+ sha256=doc_ref,
+ short_id="aaaaaaa",
+ tier=tier,
+ format="markdown",
+ content="selected pages",
+ request_scope=ContentRequestScope(page_range=page_range, limit=limit),
+ )
+
+ def export_doc_content(self, doc_ref: str, request: DocContentExportRequest) -> DocContentExportResponse:
+ """检查文件导出继续携带用户指定的 PDF 范围。"""
+ observed.append(request.page_range)
+ Path(request.output).write_text("selected pages", encoding="utf-8")
+ return DocContentExportResponse(sha256=doc_ref, short_id="aaaaaaa", tier=request.tier, output=request.output)
+
+ def record_observations(self, request: TelemetryObservationsRequest) -> None:
+ """隔离测试不发送统计。"""
+
+ def _skip_wait(seconds: float) -> None:
+ """无需为完成任务执行真实轮询延迟。"""
+
+ monkeypatch.setenv("CI", "1")
+ monkeypatch.setattr(parse_cmd, "DoclibClient", _Client)
+ monkeypatch.setattr(parse_cmd.time, "sleep", _skip_wait)
+ args = ["parse", str(source), "--tier", "flash", "--pages", "2-3", "--json"]
+ if output_file:
+ args.extend(["-o", str(tmp_path / "selected.md")])
+ result = CliRunner().invoke(app, args)
+ assert result.exit_code == 0, result.output
+ assert json.loads(result.stdout)["parse"]["page_range"] == "2-3"
+ assert observed == ["2-3"]