Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
"""验证单点续读不漏块,并保留单块读取、范围读取及正常翻页语义。"""
from __future__ import annotations
import asyncio
import re
import shlex
from pathlib import Path
import httpx
import pytest
from doclib_test_utils import DOCLIB_AUTH_HEADERS
from test_doclib_page_selection import _DOC_REF, _cached_library, _page
from mineru.cli.commands.read import _read_next_marker
from mineru.doclib.server import DoclibServer
from mineru.doclib.types import DocContentResponse
from mineru.errors import InvalidRequestError
def _body(content: str) -> str:
"""忽略页标记和块间空白,保留所有测试字符用于查重和漏读检查。"""
content = re.sub(r"<!-- page [0-9]+(?: of [0-9]+)? -->", "", content)
return re.sub(r"\s+", "", content)
async def _follow(
server: DoclibServer,
locator: str,
*,
limit: int,
after: str | None = None,
no_marker: bool = True,
cli_markers: bool = False,
) -> list[DocContentResponse]:
"""只使用服务返回的请求推进,不自行推测下一个块或下一页。"""
results: list[DocContentResponse] = []
visited: set[tuple[str, str | None]] = set()
for _ in range(100):
state = (locator, after)
assert state not in visited, state
visited.add(state)
result = await server.read_content(
locator,
after=after,
limit=30000 if cli_markers and results else limit,
no_marker=no_marker,
)
results.append(result)
if result.next_request is None:
return results
if cli_markers:
marker = _read_next_marker(result.next_request)
assert marker is not None
command = shlex.split(marker.removeprefix("<!-- Next: ").removesuffix(" -->"))
assert command[:2] == ["mineru", "read"]
locator = command[2]
after = command[command.index("--after") + 1] if "--after" in command else None
else:
assert result.next_request.locator is not None
locator = result.next_request.locator
after = result.next_request.after
pytest.fail("Locator continuation did not terminate")
@pytest.mark.parametrize("suffix", ["", "/page:1"])
@pytest.mark.parametrize("limit", [3, 8, 21])
@pytest.mark.parametrize("no_marker", [True, False])
def test_locator_continuation_preserves_all_blocks(tmp_path: Path, suffix: str, limit: int, no_marker: bool) -> None:
"""字符内、块间及页间截断后,原样续读必须不重、不漏地到达下一页。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(
tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
)
try:
results = await _follow(server, _DOC_REF + suffix, limit=limit, no_marker=no_marker)
assert "".join(_body(result.content) for result in results) == "ALPHABRAVOCHARLIEDELTA"
finally:
await db.close()
asyncio.run(run())
def test_default_budget_continuation_preserves_all_blocks(tmp_path: Path) -> None:
"""默认三万字符预算截断长页面后,续读必须按顺序消费所有块且没有重复。"""
async def run() -> None:
"""省略首个请求的 limit,原样回传其文档定位符和消费游标。"""
first_block = "A" * 12000
second_block = "B" * 12000
third_block = "C" * 12000
last_page = "D" * 10
db, server = await _cached_library(
tmp_path,
page_count=2,
pages=[_page(0, first_block, second_block, third_block), _page(1, last_page)],
)
try:
first = await server.read_content(f"{_DOC_REF}/page:1", no_marker=True)
assert first.request_scope.limit == 30000
assert first.truncated
assert first.next_request is not None
assert first.next_request.locator == _DOC_REF
assert first.next_request.after == f"{_DOC_REF}/page:1/block:2"
remaining = await _follow(
server,
first.next_request.locator,
after=first.next_request.after,
limit=30000,
)
assert [_body(result.content) for result in [first, *remaining]] == [
first_block + second_block,
third_block,
last_page,
]
finally:
await db.close()
asyncio.run(run())
@pytest.mark.parametrize(
"suffix,expected",
[
("/block:2", "BRAVODELTA"),
("/block:2/char:0", "BRAVODELTA"),
("/block:2/char:2", "AVODELTA"),
],
)
@pytest.mark.parametrize("limit", [3, 30000])
def test_direct_block_continuation_does_not_expand_to_neighbours(
tmp_path: Path, suffix: str, expected: str, limit: int
) -> None:
"""直接选择一个块时保留块内字符续读,不把同页相邻块纳入读取范围。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(
tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
)
try:
results = await _follow(server, f"{_DOC_REF}/page:1{suffix}", limit=limit)
assert "".join(_body(result.content) for result in results) == expected
finally:
await db.close()
asyncio.run(run())
@pytest.mark.parametrize(
"suffix,expected,next_page",
[
("/block:1", "BRAVOCHARLIE", 2),
("/block:1/char:2", "PHABRAVOCHARLIE", 2),
("", "DELTA", 3),
],
)
def test_document_after_preserves_page_turning(tmp_path: Path, suffix: str, expected: str, next_page: int) -> None:
"""文档续读恢复游标所在页;页游标表示已完成该页,读完后保留下一页建议。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(
tmp_path,
page_count=3,
pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA"), _page(2, "LAST")],
)
try:
result = await server.read_content(_DOC_REF, after=f"{_DOC_REF}/page:1{suffix}", no_marker=True)
assert _body(result.content) == expected
assert result.next_request is not None
assert result.next_request.locator == f"{_DOC_REF}/page:{next_page}"
assert result.next_request.after is None
finally:
await db.close()
asyncio.run(run())
@pytest.mark.parametrize(
"after",
[
"doc:bbbbbbb/tier:flash/page:1/block:1",
"doc:aaaaaaa/tier:standard/page:1/block:1",
],
)
def test_document_after_rejects_other_documents_and_tiers(tmp_path: Path, after: str) -> None:
"""文档级续读仍必须拒绝跨文档和跨档位游标。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(tmp_path)
try:
with pytest.raises(InvalidRequestError) as error:
await server.read_content(_DOC_REF, after=after)
assert error.value.param == "after"
finally:
await db.close()
asyncio.run(run())
def test_locator_continuation_skips_empty_blocks_and_index_gaps(tmp_path: Path) -> None:
"""使用实际消费游标,不假定块号连续或每个块都可渲染。"""
async def run() -> None:
"""通过空块和不连续块号检查实际消费游标。"""
page = _page(0, "ALPHA", " ", "CHARLIE")
page.blocks[-1].index = 7
db, server = await _cached_library(tmp_path, page_count=2, pages=[page, _page(1, "DELTA")])
try:
results = await _follow(server, f"{_DOC_REF}/page:1", limit=5)
assert "".join(_body(result.content) for result in results) == "ALPHACHARLIEDELTA"
finally:
await db.close()
asyncio.run(run())
@pytest.mark.parametrize("limit", [5, 30000])
def test_document_continuation_reaches_pages_after_default_window(tmp_path: Path, limit: int) -> None:
"""默认十页窗口之后仍能读到后续页面,且每页只出现一次。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(tmp_path)
try:
results = await _follow(server, _DOC_REF, limit=limit)
assert "".join(_body(result.content) for result in results) == "".join(
f"PAGE{number:02}" for number in range(1, 13)
)
finally:
await db.close()
asyncio.run(run())
def test_following_cli_markers_with_default_budget_preserves_content(tmp_path: Path) -> None:
"""CLI 续读命令不携带 --limit;恢复默认预算后仍不能遗漏同页剩余块。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(
tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
)
try:
results = await _follow(server, f"{_DOC_REF}/page:1", limit=8, no_marker=False, cli_markers=True)
assert "".join(_body(result.content) for result in results) == "ALPHABRAVOCHARLIEDELTA"
finally:
await db.close()
asyncio.run(run())
def test_locator_continuation_http_round_trip(tmp_path: Path) -> None:
"""真实路由返回的续读请求可直接回传,不依赖调用者推测正确起点。"""
async def run() -> None:
"""通过真实缓存和服务核对当前场景的续读结果。"""
db, server = await _cached_library(
tmp_path, page_count=2, pages=[_page(0, "ALPHA", "BRAVO", "CHARLIE"), _page(1, "DELTA")]
)
try:
async with httpx.AsyncClient(
transport=httpx.ASGITransport(app=server.app), base_url="http://test", headers=DOCLIB_AUTH_HEADERS
) as client:
locator = f"{_DOC_REF}/page:1"
after = None
chunks: list[str] = []
for _ in range(20):
response = await client.get(
"/api/v1/content",
params={
"locator": locator,
"limit": 8,
"no_marker": True,
**({"after": after} if after is not None else {}),
},
)
assert response.status_code == 200, response.text
data = response.json()
chunks.append(_body(data["content"]))
if data["next_request"] is None:
break
locator = data["next_request"]["locator"]
after = data["next_request"]["after"]
else:
pytest.fail("HTTP continuation did not terminate")
assert "".join(chunks) == "ALPHABRAVOCHARLIEDELTA"
finally:
await db.close()
asyncio.run(run())