Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
from docvortex.options import LatexDelimitersConfig
from _span_test_utils import inline as _inline
import pytest
from mineru.doclib.services.content_read_svc import (
_next_content_request,
_normalize_content_page_range,
_page_markdown_blocks,
_render_progressive_markdown,
)
from mineru.doclib.types import ContentRange
from mineru.errors import InvalidRequestError
from mineru.types import BlockType, ImageBlock, ImageBodyBlock, PageInfo, TableBlock, TableBodyBlock, TextBlock
def _page(page_idx: int, *texts: str) -> PageInfo:
blocks = [
TextBlock(type=BlockType.TEXT, index=index, bbox=(0.0, 0.0, 0.1, 0.1), content=_inline(text))
for index, text in enumerate(texts)
]
return PageInfo(page_idx=page_idx, blocks=blocks)
def test_paginated_default_content_pages_are_first_ten() -> None:
doc = {"page_count": 38}
assert _normalize_content_page_range(None, None, doc) == "1-10"
assert _normalize_content_page_range("all", None, doc) == "1-38"
assert _normalize_content_page_range("1-5,r5-r1", None, doc) == "1-5,34-38"
def test_content_page_range_uses_available_subset_and_merges_ranges() -> None:
doc = {"page_count": 5}
assert _normalize_content_page_range("1-10,3,4-5", None, doc) == "1-5"
def test_content_page_range_rejects_empty_available_subset() -> None:
doc = {"page_count": 5}
with pytest.raises(InvalidRequestError) as exc_info:
_normalize_content_page_range("6-10", None, doc)
assert exc_info.value.code == "page_range_invalid"
def test_paginated_page_boundary_truncation_suggests_pages_only() -> None:
pages = [_page(index, f"page {index + 1}") for index in range(10)]
rendered = _render_progressive_markdown(
pages,
short_id="ab12cd3",
tier="standard",
after=None,
limit=40,
add_markers=False,
latex_delimiters=LatexDelimitersConfig(),
)
next_request = _next_content_request(
rendered=rendered,
request_page_range="1-10",
after=None,
page_count=38,
paginated=True,
)
assert rendered.truncated is True
assert rendered.content_ranges[0].page_range == "1-5"
assert next_request is not None
assert next_request.page_range == "6-10"
assert next_request.after is None
def test_paginated_long_single_page_truncation_keeps_pages_with_after() -> None:
pages = [_page(6, "word " * 100)]
rendered = _render_progressive_markdown(
pages,
short_id="ab12cd3",
tier="standard",
after=None,
limit=40,
add_markers=False,
latex_delimiters=LatexDelimitersConfig(),
)
next_request = _next_content_request(
rendered=rendered,
request_page_range="7-10",
after=None,
page_count=38,
paginated=True,
)
assert rendered.truncated is True
assert rendered.content_ranges[-1].end.startswith("doc:ab12cd3/tier:standard/page:7/block:1/char:")
assert next_request is not None
assert next_request.page_range == "7-10"
assert next_request.after == rendered.content_ranges[-1].end
def test_non_paginated_truncation_can_use_after_only() -> None:
rendered = _render_progressive_markdown(
[_page(0, "block 1", "block 2", "block 3")],
short_id="ab12cd3",
tier="standard",
after=None,
limit=15,
add_markers=False,
latex_delimiters=LatexDelimitersConfig(),
)
next_request = _next_content_request(
rendered=rendered,
request_page_range=None,
after=None,
page_count=1,
paginated=False,
)
assert rendered.truncated is True
assert next_request is not None
assert next_request.page_range is None
assert next_request.after == rendered.content_ranges[-1].end
def test_non_contiguous_pages_suggest_after_last_requested_page() -> None:
rendered = _render_progressive_markdown(
[_page(0, "page 1"), _page(4, "page 5"), _page(19, "page 20"), _page(24, "page 25")],
short_id="ab12cd3",
tier="standard",
after=None,
limit=30000,
add_markers=False,
latex_delimiters=LatexDelimitersConfig(),
)
next_request = _next_content_request(
rendered=rendered,
request_page_range="1-5,20-25",
after=None,
page_count=38,
paginated=True,
)
assert next_request is not None
assert next_request.page_range == "26-35"
def test_truncated_false_can_still_have_next_request() -> None:
rendered = _render_progressive_markdown(
[_page(index, f"page {index + 1}") for index in range(10)],
short_id="ab12cd3",
tier="standard",
after=None,
limit=30000,
add_markers=False,
latex_delimiters=LatexDelimitersConfig(),
)
next_request = _next_content_request(
rendered=rendered,
request_page_range="1-10",
after=None,
page_count=38,
paginated=True,
)
assert rendered.truncated is False
assert next_request is not None
assert next_request.page_range == "11-20"
def test_paginated_next_request_is_never_after_only() -> None:
next_request = _next_content_request(
rendered=type(
"Rendered",
(),
{
"content_ranges": [
ContentRange(
page_range="7",
start="doc:ab12cd3/tier:standard/page:7/block:1/char:0",
end="doc:ab12cd3/tier:standard/page:7/block:1/char:40",
)
],
"truncated": True,
"last_page_no": 7,
"next_page_no": 7,
"cut_inside_page": True,
},
)(),
request_page_range="7-10",
after=None,
page_count=38,
paginated=True,
)
assert next_request is not None
assert next_request.page_range == "7-10"
assert next_request.after is not None
@pytest.mark.parametrize(
("page_count", "expected_marker"),
[(None, "<!-- page 1 -->"), (38, "<!-- page 1 of 38 -->")],
)
def test_paginated_empty_page_renders_page_marker_and_content_range(
page_count: int | None,
expected_marker: str,
) -> None:
rendered = _render_progressive_markdown(
[PageInfo(page_idx=0)],
short_id="ab12cd3",
tier="flash",
after=None,
limit=30000,
add_markers=True,
page_count=page_count,
latex_delimiters=LatexDelimitersConfig(),
)
assert rendered.content == expected_marker
assert rendered.content_ranges == [
ContentRange(page_range="1", start="doc:ab12cd3/tier:flash/page:1", end="doc:ab12cd3/tier:flash/page:1")
]
assert rendered.truncated is False
def test_page_markdown_blocks_prefers_markdown_table_in_doclib() -> None:
html = """
<table>
<tr><th>Name</th><th>Score</th></tr>
<tr><td>Alice</td><td>90</td></tr>
</table>
""".strip()
page = PageInfo(
page_idx=0,
blocks=[
TableBlock(
type=BlockType.TABLE,
index=0,
content=[
TableBodyBlock(
type=BlockType.TABLE_BODY,
index=0,
content=html,
)
],
)
],
)
assert _page_markdown_blocks(page, latex_delimiters=LatexDelimitersConfig()) == [
(
0,
"\n".join(
[
"| Name | Score |",
"| --- | --- |",
"| Alice | 90 |",
]
),
)
]
def test_page_markdown_blocks_preserves_remote_only_image() -> None:
"""验证 Doclib Markdown 在没有 bbox/sidecar 时仍输出安全编码的远程图片 URL。"""
page = PageInfo(
page_idx=0,
blocks=[
ImageBlock(
type=BlockType.IMAGE,
index=0,
content=[
ImageBodyBlock(
type=BlockType.IMAGE_BODY,
index=0,
content="",
image_url="https://cdn.example.com/remote image(1).png",
)
],
)
],
)
assert _page_markdown_blocks(page, latex_delimiters=LatexDelimitersConfig()) == [
(0, "")
]