Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix(doclib): preserve content across truncated locator reads

crunz-ailabfa6dee7766e7
diff --git a/docs/next/cli/mineru-read.md b/docs/next/cli/mineru-read.md
index 01dbfff3..4c9ca04d 100644
--- a/docs/next/cli/mineru-read.md
+++ b/docs/next/cli/mineru-read.md
@@ -70,7 +70,7 @@ mineru read doc:ab12cd3/tier:flash/page:all
| `--format`, `-f` | `markdown` / `image` | `markdown` | 输出 content format |
| `--json` | bool | false | 将整个 CLI 响应包装为 JSON |
| `--limit` | int | `30000` | 文本输出软字符上限 |
-| `--after` | locator | 不传 | 使用范围读取返回的实际页码游标继续读取;要求 Markdown 页选择且 context=0 |
+| `--after` | locator | 不传 | 使用返回的内容游标续读;要求 Markdown 文档定位或页选择,且 context=0 |
| `--context` | int | `0` | 按 locator 周围扩展上下文 |
| `--output`, `-o` | path | 不传 | 将返回内容或 image asset 写入本地路径;image 只支持 `.png`、`.jpg`、`.jpeg`、`.webp` |
| `--no-marker` | bool | false | 关闭 continuation marker |
@@ -159,7 +159,9 @@ locator-first 读取时:
- `request_scope.locator` 为规范化 locator。
- `request_scope.context` 为实际生效的 context。
-- 单点读取的 `next_request` 使用 `locator`;范围内截断时同时返回 `locator` 和 `after`。
+- 页或文档读取在页内截断时,`next_request` 使用文档 `locator` 与 `after`;读完当前页后恢复下一页建议。
+- 单块读取保留块/字符定位的续读方式,不自动扩大到同页其他块。
+- 范围内截断时同时返回原范围 `locator` 和 `after`。
- 多页选择只读取范围内的已缓存内容,读完后 `next_request=null`;不会创建新解析或继续推荐范围外页面。
## 8. Continuation
@@ -172,7 +174,18 @@ locator-first 读取时:
<!-- Next: mineru read doc:ab12cd3/tier:basic/page:5 -->
```
-`--json` 时,通过 `next_request.locator` 表示下一次建议读取的位置。
+`--json` 时,`next_request` 表示下一次建议读取的请求;续读时应同时传递
+其中的 `locator` 和非空 `after`,而不是只使用 `locator`。
+
+页或文档读取因长度限制在页内停止时,续读使用文档定位符和已消费位置:
+
+```text
+<!-- Next: mineru read doc:ab12cd3/tier:flash --after doc:ab12cd3/tier:flash/page:1/block:2 -->
+```
+
+这会读取第 1 页第 2 块之后的剩余内容;若游标带有 `char`,从对应字符偏移
+继续。当前页读完后,仍按单点阅读规则建议下一页。页级 `after` 游标表示该页
+已经读完,从下一页开始。它不会重新解析文档,也不会扩大直接单块读取的范围。
范围读取的续读保留原选择,并以实际页码的块/字符游标推进。例如:
diff --git a/mineru/doclib/services/content_read_svc.py b/mineru/doclib/services/content_read_svc.py
index 514de1e0..5e4c5e5c 100644
--- a/mineru/doclib/services/content_read_svc.py
+++ b/mineru/doclib/services/content_read_svc.py
@@ -272,14 +272,19 @@ class ContentReadService:
page_range = _locator_page_range(cursor, doc, context)
if after is not None:
- if cursor.page_range is None or cursor.block_no is not None or context or format != "markdown":
+ if cursor.block_no is not None or context or format != "markdown":
raise InvalidRequestError(
- "invalid_request", "after requires a Markdown page selection without context.", "after"
+ "invalid_request", "after requires a Markdown document or page selection without context.", "after"
)
after_cursor = _parse_after_cursor(after)
if after_cursor is None:
raise InvalidRequestError("invalid_locator", "after must be a non-empty content cursor.", "after")
- _validate_cursor_for_doc(after_cursor, doc, tier, page_range)
+ if cursor.page_range is None:
+ _validate_cursor_for_doc(after_cursor, doc, tier, None)
+ # 文档续读先读完当前页;页级游标表示该页已完成,应从下一页开始。
+ page_range = str(after_cursor.page_no + (after_cursor.block_no is None))
+ else:
+ _validate_cursor_for_doc(after_cursor, doc, tier, page_range)
return _ReadPlan(
sha256=doc["sha256"],
short_id=doc["short_id"],
@@ -302,7 +307,9 @@ class ContentReadService:
if cursor.page_no is not None
else None,
next_mode="range"
- if (cursor.range_selection and cursor.block_no is None and not context) or after is not None
+ # 页选择携带 after 时仍有界;文档续读读完当前页后保留下一页建议。
+ if (cursor.range_selection and cursor.block_no is None and not context)
+ or (after is not None and cursor.page_range is not None)
else "read",
)
@@ -409,7 +416,7 @@ class ContentReadService:
if plan.next_mode == "range":
next_request = _next_range_read_request(rendered, plan.locator)
elif plan.next_mode == "read":
- next_request = _next_read_request(rendered, plan.short_id, plan.tier, page_count)
+ next_request = _next_read_request(rendered, plan.short_id, plan.tier, page_count, target=plan.target)
else:
next_request = _next_content_request(
rendered=rendered,
@@ -1053,12 +1060,20 @@ def _next_read_request(
short_id: str,
tier: Tier,
page_count: int | None,
+ *,
+ target: ContentCursor | None = None,
) -> ContentNextRequest | None:
"""为定位符读取生成后续块或下一页请求。"""
if not rendered.content_ranges:
return None
if rendered.truncated:
- return ContentNextRequest(locator=rendered.content_ranges[-1].end)
+ # 单块读取保留块内续读;页/文档读取不能把已消费的尾部块变成新的单块目标。
+ if target is not None and target.block_no is not None:
+ return ContentNextRequest(locator=rendered.content_ranges[-1].end)
+ if rendered.cut_inside_page:
+ return ContentNextRequest(locator=f"doc:{short_id}/tier:{tier}", after=rendered.content_ranges[-1].end)
+ next_page_no = rendered.next_page_no or (rendered.last_page_no + 1)
+ return ContentNextRequest(locator=page_ref(short_id, tier, next_page_no))
total_page_count = page_count or rendered.last_page_no
next_page_no = rendered.last_page_no + 1
if next_page_no > total_page_count:
diff --git a/tests/unittest/test_doclib_read_continuation.py b/tests/unittest/test_doclib_read_continuation.py
new file mode 100644
index 00000000..f07347ad
--- /dev/null
+++ b/tests/unittest/test_doclib_read_continuation.py
@@ -0,0 +1,249 @@
+"""验证单点续读不漏块,并保留单块读取、范围读取及正常翻页语义。"""
+
+from __future__ import annotations
+
+import asyncio
+import re
+import shlex
+from pathlib import Path
+
+import httpx
+import pytest
+from doclib_test_utils import DOCLIB_AUTH_HEADERS
+from test_doclib_page_selection import _DOC_REF, _cached_library, _page
+
+from mineru.cli.commands.read import _read_next_marker
+from mineru.doclib.server import DoclibServer
+from mineru.doclib.types import DocContentResponse
+from mineru.errors import InvalidRequestError
+
+
+def _body(content: str) -> str:
+ """忽略页标记和块间空白,保留所有测试字符用于查重和漏读检查。"""
+ content = re.sub(r"<!-- page [0-9]+(?: of [0-9]+)? -->", "", content)
+ return re.sub(r"\s+", "", content)
+
+
+async def _follow(
+ server: DoclibServer,
+ locator: str,
+ *,
+ limit: int,
+ no_marker: bool = True,
+ cli_markers: bool = False,
+) -> list[DocContentResponse]:
+ """只使用服务返回的请求推进,不自行推测下一个块或下一页。"""
+ results: list[DocContentResponse] = []
+ after = None
+ visited: set[tuple[str, str | None]] = set()
+ for _ in range(100):
+ state = (locator, after)
+ assert state not in visited, state
+ visited.add(state)
+ result = await server.read_content(
+ locator,
+ after=after,
+ limit=30000 if cli_markers and results else limit,
+ no_marker=no_marker,
+ )
+ results.append(result)
+ if result.next_request is None:
+ return results
+ if cli_markers:
+ marker = _read_next_marker(result.next_request)
+ assert marker is not None
+ command = shlex.split(marker.removeprefix("<!-- Next: ").removesuffix(" -->"))
+ assert command[:2] == ["mineru", "read"]
+ locator = command[2]
+ after = command[command.index("--after") + 1] if "--after" in command else None
+ else:
+ assert result.next_request.locator is not None
+ locator = result.next_request.locator
+ after = result.next_request.after
+ pytest.fail("Locator continuation did not terminate")
+
+
[email protected]("suffix", ["", "/page:1"])
[email protected]("limit", [3, 8, 21])
[email protected]("no_marker", [True, False])
+def test_locator_continuation_preserves_all_blocks(tmp_path: Path, suffix: str, limit: int, no_marker: bool) -> None:
+ """字符内、块间及页间截断后,原样续读必须不重、不漏地到达下一页。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(
+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
+ )
+ try:
+ results = await _follow(server, _DOC_REF + suffix, limit=limit, no_marker=no_marker)
+ assert "".join(_body(result.content) for result in results) == "ALPHABRAVOCHARLIEDELTA"
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
+ "suffix,expected",
+ [
+ ("/block:2", "BRAVODELTA"),
+ ("/block:2/char:0", "BRAVODELTA"),
+ ("/block:2/char:2", "AVODELTA"),
+ ],
+)
[email protected]("limit", [3, 30000])
+def test_direct_block_continuation_does_not_expand_to_neighbours(
+ tmp_path: Path, suffix: str, expected: str, limit: int
+) -> None:
+ """直接选择一个块时保留块内字符续读,不把同页相邻块纳入读取范围。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(
+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
+ )
+ try:
+ results = await _follow(server, f"{_DOC_REF}/page:1{suffix}", limit=limit)
+ assert "".join(_body(result.content) for result in results) == expected
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
+ "suffix,expected,next_page",
+ [
+ ("/block:1", "BRAVOCHARLIE", 2),
+ ("/block:1/char:2", "PHABRAVOCHARLIE", 2),
+ ("", "DELTA", 3),
+ ],
+)
+def test_document_after_preserves_page_turning(tmp_path: Path, suffix: str, expected: str, next_page: int) -> None:
+ """文档续读恢复游标所在页;页游标表示已完成该页,读完后保留下一页建议。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(
+ tmp_path,
+ page_count=3,
+ pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA"), _page(2, "LAST")],
+ )
+ try:
+ result = await server.read_content(_DOC_REF, after=f"{_DOC_REF}/page:1{suffix}", no_marker=True)
+ assert _body(result.content) == expected
+ assert result.next_request is not None
+ assert result.next_request.locator == f"{_DOC_REF}/page:{next_page}"
+ assert result.next_request.after is None
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
+ "after",
+ [
+ "doc:bbbbbbb/tier:flash/page:1/block:1",
+ "doc:aaaaaaa/tier:standard/page:1/block:1",
+ ],
+)
+def test_document_after_rejects_other_documents_and_tiers(tmp_path: Path, after: str) -> None:
+ """文档级续读仍必须拒绝跨文档和跨档位游标。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(tmp_path)
+ try:
+ with pytest.raises(InvalidRequestError) as error:
+ await server.read_content(_DOC_REF, after=after)
+ assert error.value.param == "after"
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
+def test_locator_continuation_skips_empty_blocks_and_index_gaps(tmp_path: Path) -> None:
+ """使用实际消费游标,不假定块号连续或每个块都可渲染。"""
+
+ async def run() -> None:
+ page = _page(0, "ALPHA", " ", "CHARLIE")
+ page.blocks[-1].index = 7
+ db, server = await _cached_library(tmp_path, page_count=2, pages=[page, _page(1, "DELTA")])
+ try:
+ results = await _follow(server, f"{_DOC_REF}/page:1", limit=5)
+ assert "".join(_body(result.content) for result in results) == "ALPHACHARLIEDELTA"
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
[email protected]("limit", [5, 30000])
+def test_document_continuation_reaches_pages_after_default_window(tmp_path: Path, limit: int) -> None:
+ """默认十页窗口之后仍能读到后续页面,且每页只出现一次。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(tmp_path)
+ try:
+ results = await _follow(server, _DOC_REF, limit=limit)
+ assert "".join(_body(result.content) for result in results) == "".join(
+ f"PAGE{number:02}" for number in range(1, 13)
+ )
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
+def test_following_cli_markers_with_default_budget_preserves_content(tmp_path: Path) -> None:
+ """CLI 续读命令不携带 --limit;恢复默认预算后仍不能遗漏同页剩余块。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(
+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
+ )
+ try:
+ results = await _follow(server, f"{_DOC_REF}/page:1", limit=8, no_marker=False, cli_markers=True)
+ assert "".join(_body(result.content) for result in results) == "ALPHABRAVOCHARLIEDELTA"
+ finally:
+ await db.close()
+
+ asyncio.run(run())
+
+
+def test_locator_continuation_http_round_trip(tmp_path: Path) -> None:
+ """真实路由返回的续读请求可直接回传,不依赖调用者推测正确起点。"""
+
+ async def run() -> None:
+ db, server = await _cached_library(
+ tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
+ )
+ try:
+ async with httpx.AsyncClient(
+ transport=httpx.ASGITransport(app=server.app), base_url="http://test", headers=DOCLIB_AUTH_HEADERS
+ ) as client:
+ locator = f"{_DOC_REF}/page:1"
+ after = None
+ chunks: list[str] = []
+ for _ in range(20):
+ response = await client.get(
+ "/api/v1/content",
+ params={
+ "locator": locator,
+ "limit": 8,
+ "no_marker": True,
+ **({"after": after} if after is not None else {}),
+ },
+ )
+ assert response.status_code == 200, response.text
+ data = response.json()
+ chunks.append(_body(data["content"]))
+ if data["next_request"] is None:
+ break
+ locator = data["next_request"]["locator"]
+ after = data["next_request"]["after"]
+ else:
+ pytest.fail("HTTP continuation did not terminate")
+ assert "".join(chunks) == "ALPHABRAVOCHARLIEDELTA"
+ finally:
+ await db.close()
+
+ asyncio.run(run())