Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix(doclib): preserve content across truncated locator reads
crunz-ailabfa6dee7766e7
diff --git a/docs/next/cli/mineru-read.md b/docs/next/cli/mineru-read.mdindex 01dbfff3..4c9ca04d 100644--- a/docs/next/cli/mineru-read.md+++ b/docs/next/cli/mineru-read.md@@ -70,7 +70,7 @@ mineru read doc:ab12cd3/tier:flash/page:all| `--format`, `-f` | `markdown` / `image` | `markdown` | 输出 content format || `--json` | bool | false | 将整个 CLI 响应包装为 JSON || `--limit` | int | `30000` | 文本输出软字符上限 |-| `--after` | locator | 不传 | 使用范围读取返回的实际页码游标继续读取;要求 Markdown 页选择且 context=0 |+| `--after` | locator | 不传 | 使用返回的内容游标续读;要求 Markdown 文档定位或页选择,且 context=0 || `--context` | int | `0` | 按 locator 周围扩展上下文 || `--output`, `-o` | path | 不传 | 将返回内容或 image asset 写入本地路径;image 只支持 `.png`、`.jpg`、`.jpeg`、`.webp` || `--no-marker` | bool | false | 关闭 continuation marker |@@ -159,7 +159,9 @@ locator-first 读取时:- `request_scope.locator` 为规范化 locator。- `request_scope.context` 为实际生效的 context。-- 单点读取的 `next_request` 使用 `locator`;范围内截断时同时返回 `locator` 和 `after`。+- 页或文档读取在页内截断时,`next_request` 使用文档 `locator` 与 `after`;读完当前页后恢复下一页建议。+- 单块读取保留块/字符定位的续读方式,不自动扩大到同页其他块。+- 范围内截断时同时返回原范围 `locator` 和 `after`。- 多页选择只读取范围内的已缓存内容,读完后 `next_request=null`;不会创建新解析或继续推荐范围外页面。## 8. Continuation@@ -172,7 +174,18 @@ locator-first 读取时:<!-- Next: mineru read doc:ab12cd3/tier:basic/page:5 -->```-`--json` 时,通过 `next_request.locator` 表示下一次建议读取的位置。+`--json` 时,`next_request` 表示下一次建议读取的请求;续读时应同时传递+其中的 `locator` 和非空 `after`,而不是只使用 `locator`。++页或文档读取因长度限制在页内停止时,续读使用文档定位符和已消费位置:++```text+<!-- Next: mineru read doc:ab12cd3/tier:flash --after doc:ab12cd3/tier:flash/page:1/block:2 -->+```++这会读取第 1 页第 2 块之后的剩余内容;若游标带有 `char`,从对应字符偏移+继续。当前页读完后,仍按单点阅读规则建议下一页。页级 `after` 游标表示该页+已经读完,从下一页开始。它不会重新解析文档,也不会扩大直接单块读取的范围。范围读取的续读保留原选择,并以实际页码的块/字符游标推进。例如:diff --git a/mineru/doclib/services/content_read_svc.py b/mineru/doclib/services/content_read_svc.pyindex 514de1e0..5e4c5e5c 100644--- a/mineru/doclib/services/content_read_svc.py+++ b/mineru/doclib/services/content_read_svc.py@@ -272,14 +272,19 @@ class ContentReadService:page_range = _locator_page_range(cursor, doc, context)if after is not None:- if cursor.page_range is None or cursor.block_no is not None or context or format != "markdown":+ if cursor.block_no is not None or context or format != "markdown":raise InvalidRequestError(- "invalid_request", "after requires a Markdown page selection without context.", "after"+ "invalid_request", "after requires a Markdown document or page selection without context.", "after")after_cursor = _parse_after_cursor(after)if after_cursor is None:raise InvalidRequestError("invalid_locator", "after must be a non-empty content cursor.", "after")- _validate_cursor_for_doc(after_cursor, doc, tier, page_range)+ if cursor.page_range is None:+ _validate_cursor_for_doc(after_cursor, doc, tier, None)+ # 文档续读先读完当前页;页级游标表示该页已完成,应从下一页开始。+ page_range = str(after_cursor.page_no + (after_cursor.block_no is None))+ else:+ _validate_cursor_for_doc(after_cursor, doc, tier, page_range)return _ReadPlan(sha256=doc["sha256"],short_id=doc["short_id"],@@ -302,7 +307,9 @@ class ContentReadService:if cursor.page_no is not Noneelse None,next_mode="range"- if (cursor.range_selection and cursor.block_no is None and not context) or after is not None+ # 页选择携带 after 时仍有界;文档续读读完当前页后保留下一页建议。+ if (cursor.range_selection and cursor.block_no is None and not context)+ or (after is not None and cursor.page_range is not None)else "read",)@@ -409,7 +416,7 @@ class ContentReadService:if plan.next_mode == "range":next_request = _next_range_read_request(rendered, plan.locator)elif plan.next_mode == "read":- next_request = _next_read_request(rendered, plan.short_id, plan.tier, page_count)+ next_request = _next_read_request(rendered, plan.short_id, plan.tier, page_count, target=plan.target)else:next_request = _next_content_request(rendered=rendered,@@ -1053,12 +1060,20 @@ def _next_read_request(short_id: str,tier: Tier,page_count: int | None,+ *,+ target: ContentCursor | None = None,) -> ContentNextRequest | None:"""为定位符读取生成后续块或下一页请求。"""if not rendered.content_ranges:return Noneif rendered.truncated:- return ContentNextRequest(locator=rendered.content_ranges[-1].end)+ # 单块读取保留块内续读;页/文档读取不能把已消费的尾部块变成新的单块目标。+ if target is not None and target.block_no is not None:+ return ContentNextRequest(locator=rendered.content_ranges[-1].end)+ if rendered.cut_inside_page:+ return ContentNextRequest(locator=f"doc:{short_id}/tier:{tier}", after=rendered.content_ranges[-1].end)+ next_page_no = rendered.next_page_no or (rendered.last_page_no + 1)+ return ContentNextRequest(locator=page_ref(short_id, tier, next_page_no))total_page_count = page_count or rendered.last_page_nonext_page_no = rendered.last_page_no + 1if next_page_no > total_page_count:diff --git a/tests/unittest/test_doclib_read_continuation.py b/tests/unittest/test_doclib_read_continuation.pynew file mode 100644index 00000000..f07347ad--- /dev/null+++ b/tests/unittest/test_doclib_read_continuation.py@@ -0,0 +1,249 @@+"""验证单点续读不漏块,并保留单块读取、范围读取及正常翻页语义。"""++from __future__ import annotations++import asyncio+import re+import shlex+from pathlib import Path++import httpx+import pytest+from doclib_test_utils import DOCLIB_AUTH_HEADERS+from test_doclib_page_selection import _DOC_REF, _cached_library, _page++from mineru.cli.commands.read import _read_next_marker+from mineru.doclib.server import DoclibServer+from mineru.doclib.types import DocContentResponse+from mineru.errors import InvalidRequestError+++def _body(content: str) -> str:+ """忽略页标记和块间空白,保留所有测试字符用于查重和漏读检查。"""+ content = re.sub(r"<!-- page [0-9]+(?: of [0-9]+)? -->", "", content)+ return re.sub(r"\s+", "", content)+++async def _follow(+ server: DoclibServer,+ locator: str,+ *,+ limit: int,+ no_marker: bool = True,+ cli_markers: bool = False,+) -> list[DocContentResponse]:+ """只使用服务返回的请求推进,不自行推测下一个块或下一页。"""+ results: list[DocContentResponse] = []+ after = None+ visited: set[tuple[str, str | None]] = set()+ for _ in range(100):+ state = (locator, after)+ assert state not in visited, state+ visited.add(state)+ result = await server.read_content(+ locator,+ after=after,+ limit=30000 if cli_markers and results else limit,+ no_marker=no_marker,+ )+ results.append(result)+ if result.next_request is None:+ return results+ if cli_markers:+ marker = _read_next_marker(result.next_request)+ assert marker is not None+ command = shlex.split(marker.removeprefix("<!-- Next: ").removesuffix(" -->"))+ assert command[:2] == ["mineru", "read"]+ locator = command[2]+ after = command[command.index("--after") + 1] if "--after" in command else None+ else:+ assert result.next_request.locator is not None+ locator = result.next_request.locator+ after = result.next_request.after+ pytest.fail("Locator continuation did not terminate")++[email protected]("suffix", ["", "/page:1"])[email protected]("limit", [3, 8, 21])[email protected]("no_marker", [True, False])+def test_locator_continuation_preserves_all_blocks(tmp_path: Path, suffix: str, limit: int, no_marker: bool) -> None:+ """字符内、块间及页间截断后,原样续读必须不重、不漏地到达下一页。"""++ async def run() -> None:+ db, server = await _cached_library(+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]+ )+ try:+ results = await _follow(server, _DOC_REF + suffix, limit=limit, no_marker=no_marker)+ assert "".join(_body(result.content) for result in results) == "ALPHABRAVOCHARLIEDELTA"+ finally:+ await db.close()++ asyncio.run(run())+++ "suffix,expected",+ [+ ("/block:2", "BRAVODELTA"),+ ("/block:2/char:0", "BRAVODELTA"),+ ("/block:2/char:2", "AVODELTA"),+ ],+)[email protected]("limit", [3, 30000])+def test_direct_block_continuation_does_not_expand_to_neighbours(+ tmp_path: Path, suffix: str, expected: str, limit: int+) -> None:+ """直接选择一个块时保留块内字符续读,不把同页相邻块纳入读取范围。"""++ async def run() -> None:+ db, server = await _cached_library(+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]+ )+ try:+ results = await _follow(server, f"{_DOC_REF}/page:1{suffix}", limit=limit)+ assert "".join(_body(result.content) for result in results) == expected+ finally:+ await db.close()++ asyncio.run(run())+++ "suffix,expected,next_page",+ [+ ("/block:1", "BRAVOCHARLIE", 2),+ ("/block:1/char:2", "PHABRAVOCHARLIE", 2),+ ("", "DELTA", 3),+ ],+)+def test_document_after_preserves_page_turning(tmp_path: Path, suffix: str, expected: str, next_page: int) -> None:+ """文档续读恢复游标所在页;页游标表示已完成该页,读完后保留下一页建议。"""++ async def run() -> None:+ db, server = await _cached_library(+ tmp_path,+ page_count=3,+ pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA"), _page(2, "LAST")],+ )+ try:+ result = await server.read_content(_DOC_REF, after=f"{_DOC_REF}/page:1{suffix}", no_marker=True)+ assert _body(result.content) == expected+ assert result.next_request is not None+ assert result.next_request.locator == f"{_DOC_REF}/page:{next_page}"+ assert result.next_request.after is None+ finally:+ await db.close()++ asyncio.run(run())+++ "after",+ [+ "doc:bbbbbbb/tier:flash/page:1/block:1",+ "doc:aaaaaaa/tier:standard/page:1/block:1",+ ],+)+def test_document_after_rejects_other_documents_and_tiers(tmp_path: Path, after: str) -> None:+ """文档级续读仍必须拒绝跨文档和跨档位游标。"""++ async def run() -> None:+ db, server = await _cached_library(tmp_path)+ try:+ with pytest.raises(InvalidRequestError) as error:+ await server.read_content(_DOC_REF, after=after)+ assert error.value.param == "after"+ finally:+ await db.close()++ asyncio.run(run())+++def test_locator_continuation_skips_empty_blocks_and_index_gaps(tmp_path: Path) -> None:+ """使用实际消费游标,不假定块号连续或每个块都可渲染。"""++ async def run() -> None:+ page = _page(0, "ALPHA", " ", "CHARLIE")+ page.blocks[-1].index = 7+ db, server = await _cached_library(tmp_path, page_count=2, pages=[page, _page(1, "DELTA")])+ try:+ results = await _follow(server, f"{_DOC_REF}/page:1", limit=5)+ assert "".join(_body(result.content) for result in results) == "ALPHACHARLIEDELTA"+ finally:+ await db.close()++ asyncio.run(run())++[email protected]("limit", [5, 30000])+def test_document_continuation_reaches_pages_after_default_window(tmp_path: Path, limit: int) -> None:+ """默认十页窗口之后仍能读到后续页面,且每页只出现一次。"""++ async def run() -> None:+ db, server = await _cached_library(tmp_path)+ try:+ results = await _follow(server, _DOC_REF, limit=limit)+ assert "".join(_body(result.content) for result in results) == "".join(+ f"PAGE{number:02}" for number in range(1, 13)+ )+ finally:+ await db.close()++ asyncio.run(run())+++def test_following_cli_markers_with_default_budget_preserves_content(tmp_path: Path) -> None:+ """CLI 续读命令不携带 --limit;恢复默认预算后仍不能遗漏同页剩余块。"""++ async def run() -> None:+ db, server = await _cached_library(+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]+ )+ try:+ results = await _follow(server, f"{_DOC_REF}/page:1", limit=8, no_marker=False, cli_markers=True)+ assert "".join(_body(result.content) for result in results) == "ALPHABRAVOCHARLIEDELTA"+ finally:+ await db.close()++ asyncio.run(run())+++def test_locator_continuation_http_round_trip(tmp_path: Path) -> None:+ """真实路由返回的续读请求可直接回传,不依赖调用者推测正确起点。"""++ async def run() -> None:+ db, server = await _cached_library(+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]+ )+ try:+ async with httpx.AsyncClient(+ transport=httpx.ASGITransport(app=server.app), base_url="http://test", headers=DOCLIB_AUTH_HEADERS+ ) as client:+ locator = f"{_DOC_REF}/page:1"+ after = None+ chunks: list[str] = []+ for _ in range(20):+ response = await client.get(+ "/api/v1/content",+ params={+ "locator": locator,+ "limit": 8,+ "no_marker": True,+ **({"after": after} if after is not None else {}),+ },+ )+ assert response.status_code == 200, response.text+ data = response.json()+ chunks.append(_body(data["content"]))+ if data["next_request"] is None:+ break+ locator = data["next_request"]["locator"]+ after = data["next_request"]["after"]+ else:+ pytest.fail("HTTP continuation did not terminate")+ assert "".join(chunks) == "ALPHABRAVOCHARLIEDELTA"+ finally:+ await db.close()++ asyncio.run(run())