Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
from __future__ import annotations
import json
import math
import statistics
from dataclasses import dataclass
from pathlib import Path
import pytest
from docvortex.analyzers.pdf import PDF_NATIVE_SCRIPT_MARKUP_KEY
from docvortex.document.pdf import Bbox, Char, PDFDocument, get_lines_from_chars
from mineru.backend.analysis.pdf.text import native
from mineru.backend.analysis.pdf.text.models import _AnalyzeSpan
from mineru.types import BBox, ContentType
_PROJECT_ROOT = Path(__file__).resolve().parents[2]
_DEMO_PDF_DIR = _PROJECT_ROOT / "demo" / "pdfs"
_ZH_2_SOURCE = "中文论文2.pdf"
_PDF_CONTROL_CHARS = {"\r", "\n", "\x02", "\ufffe", "\uffff"}
@dataclass(frozen=True, slots=True)
class _ScriptFixture:
"""保存测试字符及对应 tight/origin side maps。"""
chars: list[Char]
tight_bboxes: dict[int, BBox]
origins: dict[int, tuple[float, float]]
def _lines_chars_containing(pdf_path: Path, page_index: int, probe: str) -> list[_ScriptFixture]:
"""从真实 PDF 指定页中返回全部包含探针的物理行字符。"""
with PDFDocument(str(pdf_path)) as document:
geometry = document.get_page_chars_with_geometry(page_index)
matching_lines = []
for line in get_lines_from_chars(geometry.chars):
line_text = "".join(span["text"] for span in line["spans"])
line_text = "".join(char for char in line_text if char not in _PDF_CONTROL_CHARS)
if probe in line_text:
matching_lines.append(
_ScriptFixture(
chars=[char for span in line["spans"] for char in span.get("chars", [])],
tight_bboxes=geometry.tight_bboxes,
origins=geometry.origins,
)
)
return matching_lines
def _line_chars_containing(pdf_path: Path, page_index: int, probe: str) -> _ScriptFixture:
"""从真实 PDF 指定页中返回唯一包含探针的物理行字符。"""
matching_lines = _lines_chars_containing(pdf_path, page_index, probe)
assert len(matching_lines) == 1, (pdf_path, page_index, probe, len(matching_lines))
return matching_lines[0]
def _contiguous_chars_containing(pdf_path: Path, page_index: int, probe: str) -> _ScriptFixture:
"""忽略 PDF 控制字符后截取唯一连续探针,覆盖视觉同行但内部带换行的文本。"""
with PDFDocument(str(pdf_path)) as document:
geometry = document.get_page_chars_with_geometry(page_index)
visible_chars = [char for char in geometry.chars if str(char.get("char", "")) not in _PDF_CONTROL_CHARS]
page_text = "".join(str(char.get("char", "")) for char in visible_chars)
start_index = page_text.find(probe)
assert start_index >= 0, (pdf_path, page_index, probe)
assert page_text.find(probe, start_index + 1) < 0, (pdf_path, page_index, probe)
return _ScriptFixture(
chars=visible_chars[start_index : start_index + len(probe)],
tight_bboxes=geometry.tight_bboxes,
origins=geometry.origins,
)
def _synthetic_fixture(
chars: list[Char],
*,
tight_heights: dict[int, float] | None = None,
origin_ys: dict[int, float] | None = None,
) -> _ScriptFixture:
"""从 loose 测试字符构造可覆盖 tight 高度和 origin 的几何 fixture。"""
tight_heights = tight_heights or {}
origin_ys = origin_ys or {}
tight_bboxes = {}
origins = {}
for char in chars:
char_idx = int(char["char_idx"])
bbox = tuple(float(item) for item in char["bbox"])
center_x = (bbox[0] + bbox[2]) / 2
center_y = (bbox[1] + bbox[3]) / 2
tight_height = tight_heights.get(char_idx, bbox[3] - bbox[1])
tight_bboxes[char_idx] = (
bbox[0],
center_y - tight_height / 2,
bbox[2],
center_y + tight_height / 2,
)
origins[char_idx] = (center_x, origin_ys.get(char_idx, center_y))
return _ScriptFixture(
chars=chars,
tight_bboxes=tight_bboxes,
origins=origins,
)
def _tight_only_span_bbox(fixture: _ScriptFixture) -> BBox:
"""构造真实 tight 字形可命中、异常 loose 字体框完全无法命中的窄 Span。"""
tight_boxes = [fixture.tight_bboxes[int(char["char_idx"])] for char in fixture.chars]
center_y = statistics.median((bbox[1] + bbox[3]) / 2 for bbox in tight_boxes)
x0 = min(bbox[0] for bbox in tight_boxes) - 0.5
x1 = max(bbox[2] for bbox in tight_boxes) + 0.5
for quarter_points in range(16, 161):
height = quarter_points / 4
span_bbox = (x0, center_y - height / 2, x1, center_y + height / 2)
tight_matches = [
native.calculate_char_in_span(
tight_bbox,
span_bbox,
str(char["char"]),
)
for char, tight_bbox in zip(fixture.chars, tight_boxes)
]
loose_matches = [
native.calculate_char_in_span(
tuple(float(value) for value in char["bbox"]),
span_bbox,
str(char["char"]),
)
for char in fixture.chars
]
if all(tight_matches) and not any(loose_matches):
return span_bbox
raise AssertionError("未找到可复现 loose 字体框偏移的 tight-only Span")
def _render_chars(value: list[Char] | _ScriptFixture) -> str:
"""使用生产 chars_to_content 路径重建字符内容和上下标标签。"""
fixture = value if isinstance(value, _ScriptFixture) else _synthetic_fixture(value)
chars = fixture.chars
span = _AnalyzeSpan(
type=ContentType.TEXT,
bbox=(0.0, 0.0, 1.0, 1.0),
metadata={"chars": chars},
)
native.chars_to_content(
span,
tight_bboxes=fixture.tight_bboxes,
origins=fixture.origins,
)
return span.content
def _demo_chars(source_name: str, page_index: int, probe: str, *, selection: str = "line") -> _ScriptFixture:
"""读取保留 PDF 或已冻结的最小字符输入,不依赖其他仓库与网络。"""
if source_name in {"demo1.pdf", "demo2.pdf"}:
path = _DEMO_PDF_DIR / source_name
if selection == "contiguous":
return _contiguous_chars_containing(path, page_index, probe)
return _line_chars_containing(path, page_index, probe)
payload = json.loads((_PROJECT_ROOT / "tests/fixtures/hybrid_native_script_inputs.json").read_text(encoding="utf-8"))
case = next(
case
for case in payload["cases"]
if (case["source"], case["page_index"], case["probe"], case["selection"]) == (source_name, page_index, probe, selection)
)
assert len(case["fragments"]) == 1
fragment = case["fragments"][0]
return _ScriptFixture(
chars=[{**char, "bbox": Bbox(char["bbox"])} for char in fragment["chars"]],
tight_bboxes={int(key): tuple(value) for key, value in fragment["tight_bboxes"].items()},
origins={int(key): tuple(value) for key, value in fragment["origins"].items()},
)
def _render_demo_line(source_name: str, page_index: int, probe: str) -> str:
"""使用同一生产回填路径验证真实文档物理行的文本和上下标。"""
return _render_chars(_demo_chars(source_name, page_index, probe))
def _residual_script_case(source_name: str, probe: str = "") -> tuple[dict, list[_ScriptFixture]]:
"""读取残留 PDF 的原始字符快照,保持 Hybrid 输入与几何证据不变。"""
payload = json.loads((_PROJECT_ROOT / "tests/fixtures/hybrid_pdf_script_inputs.json").read_text(encoding="utf-8"))
case = next(case for case in payload["cases"] if Path(case["source"]).name == source_name and case["probe"] == probe)
fragments = [
_ScriptFixture(
chars=[{**char, "bbox": Bbox(char["bbox"])} for char in fragment["chars"]],
tight_bboxes={int(key): tuple(value) for key, value in fragment["tight_bboxes"].items()},
origins={int(key): tuple(value) for key, value in fragment["origins"].items()},
)
for fragment in case["fragments"]
]
return case, fragments
def _script_char(
index: int,
text: str,
*,
height: float,
center_y: float,
font_name: str = "Body",
) -> Char:
"""构造最小字符几何,用于验证组件证据和边界不变量。"""
return {
"bbox": Bbox(
[
float(index),
center_y - height / 2,
float(index + 1),
center_y + height / 2,
]
),
"char": text,
"rotation": 0.0,
"font": {"name": font_name, "flags": 0, "size": 1.0, "weight": 400},
"char_idx": index,
}
def _rotated_char(index: int, text: str, rotation_degrees: float) -> Char:
"""构造带指定字符方向的最小字符,用于 span 回填角度测试。"""
char = _script_char(index, text, height=10.0, center_y=10.0)
char["rotation"] = math.radians(rotation_degrees)
return char
@pytest.mark.parametrize(
("degrees", "expected"),
[
(0.0, True),
(90.0, True),
(180.0, True),
(270.0, True),
(0.05, True),
(0.1, False),
(359.5, False),
(359.899, False),
(359.9, False),
(359.95, True),
(360.0, True),
(360.00001001791264, True),
(45.0, False),
(315.0, False),
],
)
def test_supported_pdfium_rotations_include_zero_wrap(degrees: float, expected: bool) -> None:
"""覆盖 PDFium 的整圈边界及单精度上溢,同时保持严格小于 0.1 度的容差。"""
assert native._is_supported_rotation(math.radians(degrees)) is expected
@pytest.mark.parametrize("degrees", [0.05, 359.95, 360.00001001791264])
def test_span_fill_keeps_short_labels_near_zero(degrees: float) -> None:
"""单字符标签无需借助多字符基线恢复即可通过近零方向筛选。"""
char = _rotated_char(0, "A", degrees)
assert native._get_chars_for_span_fill([char]) == [char]
def test_span_fill_does_not_restore_315_degree_watermark(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""验证标准正文粗行中的 315 度水印字符不会被仿斜体分支重新放行。"""
body = _rotated_char(0, "A", 0.0)
watermark = _rotated_char(1, "水", 315.0)
line = {"rotation": 0.0, "spans": [{"chars": [body, watermark]}]}
monkeypatch.setattr(native, "get_lines_from_chars", lambda _chars: [line])
assert native._get_chars_for_span_fill([body, watermark]) == [body]
def test_span_fill_restores_19_degree_sheared_text(
monkeypatch: pytest.MonkeyPatch,
) -> None:
"""验证标准正文粗行中的约 19 度仿斜体字符仍可参与 span 回填。"""
body = _rotated_char(0, "A", 0.0)
sheared = _rotated_char(1, "B", 19.2)
line = {"rotation": 0.0, "spans": [{"chars": [body, sheared]}]}
monkeypatch.setattr(native, "get_lines_from_chars", lambda _chars: [line])
assert native._get_chars_for_span_fill([body, sheared]) == [body, sheared]
def _fillable_span(bbox: tuple[float, float, float, float]) -> _AnalyzeSpan:
"""构造已初始化字符容器的可回填文本 Span。"""
return _AnalyzeSpan(
type=ContentType.TEXT,
bbox=bbox,
metadata={
"chars": [],
"height": bbox[3] - bbox[1],
"width": bbox[2] - bbox[0],
},
)
def test_span_fill_prefers_tight_bbox_over_inflated_loose_bbox() -> None:
"""验证异常 loose 中心落入下一行时仍按 tight bbox 归属正确行。"""
first = _fillable_span((10.0, 10.0, 80.0, 20.0))
second = _fillable_span((10.0, 22.0, 80.0, 32.0))
chars = [
{
"char": text,
"bbox": Bbox([12.0 + index * 6.0, 13.0, 18.0 + index * 6.0, 35.0]),
"rotation": 0.0,
"font": {},
"char_idx": index,
}
for index, text in enumerate("gpySource")
]
tight_bboxes = {index: (12.0 + index * 6.0, 12.0, 18.0 + index * 6.0, 19.0) for index in range(len(chars))}
origins = {index: (15.0 + index * 6.0, 19.0) for index in range(len(chars))}
native.fill_char_in_spans(
[first, second],
chars, # type: ignore[arg-type]
10.0,
tight_bboxes=tight_bboxes,
origins=origins,
)
assert first.content == "gpySource"
assert second.content == ""
@pytest.mark.parametrize(
("page_index", "probe"),
[
pytest.param(0, "providing", id="page-1-providing"),
pytest.param(10, "Source", id="page-11-source"),
pytest.param(10, "Hypothesis", id="page-11-hypothesis"),
pytest.param(10, "Reference", id="page-11-reference"),
pytest.param(10, "BLEU", id="page-11-bleu"),
pytest.param(10, "ROUGE", id="page-11-rouge"),
pytest.param(16, "generalisation", id="page-17-generalisation"),
],
)
def test_zh2_tight_bbox_restores_words_rejected_by_loose_bbox(
page_index: int,
probe: str,
) -> None:
"""验证中文论文2真实字体框中 loose 全部失配时 tight-first 仍恢复完整英文词。"""
fixture = _demo_chars(_ZH_2_SOURCE, page_index, probe, selection="contiguous")
span_bbox = _tight_only_span_bbox(fixture)
loose_matches = [
native.calculate_char_in_span(
tuple(float(value) for value in char["bbox"]),
span_bbox,
str(char["char"]),
)
for char in fixture.chars
]
span = _fillable_span(span_bbox)
native.fill_char_in_spans(
[span],
fixture.chars,
span_bbox[3] - span_bbox[1],
tight_bboxes=fixture.tight_bboxes,
origins=fixture.origins,
)
assert not any(loose_matches)
assert span.content == probe
def test_span_fill_uses_loose_bbox_when_tight_punctuation_does_not_match() -> None:
"""验证 tight 标点未命中时继续使用原首尾标点 loose 回退。"""
span = _fillable_span((10.0, 10.0, 30.0, 20.0))
char = {
"char": ".",
"bbox": Bbox([27.0, 12.0, 31.0, 18.0]),
"rotation": 0.0,
"font": {},
"char_idx": 0,
}
native.fill_char_in_spans(
[span],
[char], # type: ignore[list-item]
10.0,
tight_bboxes={0: (40.0, 12.0, 41.0, 18.0)},
origins={0: (27.0, 18.0)},
)
assert span.content == "."
def test_zh2_heading_number_bridges_low_internal_periods() -> None:
"""验证中文论文2窄标题框可恢复同行数字之间偏低的句点。"""
probe = "1.3.6 评估结果缺乏可解释性"
fixture = _demo_chars(_ZH_2_SOURCE, 6, probe, selection="contiguous")
span_bbox = (308.16, 628.56, 444.96, 637.56)
periods = [char for char in fixture.chars if char["char"] == "."]
assert len(periods) == 2
for period in periods:
char_idx = int(period["char_idx"])
assert not native.calculate_char_in_span(fixture.tight_bboxes[char_idx], span_bbox, ".")
assert not native.calculate_char_in_span(tuple(float(value) for value in period["bbox"]), span_bbox, ".")
span = _fillable_span(span_bbox)
native.fill_char_in_spans(
[span],
fixture.chars,
8.64,
tight_bboxes=fixture.tight_bboxes,
origins=fixture.origins,
detect_scripts=False,
)
assert span.content == probe
def test_span_fill_bridges_consecutive_internal_punctuation() -> None:
"""验证连续内部标点在同行两侧同属一个 Span 时均可恢复。"""
span = _fillable_span((10.0, 10.0, 40.0, 20.0))
chars = [
_script_char(0, "1", height=6.0, center_y=15.0),
_script_char(1, ".", height=2.4, center_y=18.4),
_script_char(2, ".", height=2.4, center_y=18.4),
_script_char(3, "3", height=6.0, center_y=15.0),
]
loose_bboxes = [
Bbox([12.0, 12.0, 15.0, 18.0]),
Bbox([15.0, 17.2, 16.0, 19.6]),
Bbox([16.0, 17.2, 17.0, 19.6]),
Bbox([17.0, 12.0, 20.0, 18.0]),
]
for char, bbox in zip(chars, loose_bboxes, strict=True):
char["bbox"] = bbox
tight_bboxes = {index: tuple(float(value) for value in bbox) for index, bbox in enumerate(loose_bboxes)}
native.fill_char_in_spans(
[span],
chars,
10.0,
tight_bboxes=tight_bboxes,
origins={index: (float(index), 18.0) for index in range(len(chars))},
detect_scripts=False,
)
assert span.content == "1..3"
def test_span_fill_does_not_bridge_punctuation_between_different_spans() -> None:
"""验证前后字符归属不同 Span 时不会吸收中间标点。"""
first = _fillable_span((10.0, 10.0, 20.0, 20.0))
second = _fillable_span((20.0, 10.0, 30.0, 20.0))
chars = [
_script_char(0, "A", height=6.0, center_y=15.0),
_script_char(1, ".", height=2.4, center_y=18.4),
_script_char(2, "B", height=6.0, center_y=15.0),
]
loose_bboxes = [
Bbox([12.0, 12.0, 15.0, 18.0]),
Bbox([19.5, 17.2, 20.5, 19.6]),
Bbox([22.0, 12.0, 25.0, 18.0]),
]
for char, bbox in zip(chars, loose_bboxes, strict=True):
char["bbox"] = bbox
native.fill_char_in_spans(
[first, second],
chars,
10.0,
tight_bboxes={index: tuple(float(value) for value in bbox) for index, bbox in enumerate(loose_bboxes)},
origins={index: (float(index), 18.0) for index in range(len(chars))},
detect_scripts=False,
)
assert first.content == "A"
assert second.content == "B"
def test_span_fill_does_not_bridge_punctuation_across_line_break_or_outside_span() -> None:
"""验证跨物理行或中心位于 Span 外部的标点不会被桥接。"""
line_break_span = _fillable_span((10.0, 10.0, 40.0, 20.0))
line_break_chars = [
_script_char(0, "A", height=6.0, center_y=15.0),
_script_char(1, "\r", height=0.0, center_y=15.0),
_script_char(2, ".", height=2.4, center_y=18.4),
_script_char(3, "B", height=6.0, center_y=15.0),
]
line_break_bboxes = [
Bbox([12.0, 12.0, 15.0, 18.0]),
Bbox([15.0, 15.0, 15.0, 15.0]),
Bbox([15.0, 17.2, 16.0, 19.6]),
Bbox([16.0, 12.0, 19.0, 18.0]),
]
for char, bbox in zip(line_break_chars, line_break_bboxes, strict=True):
char["bbox"] = bbox
native.fill_char_in_spans(
[line_break_span],
line_break_chars,
10.0,
tight_bboxes={
0: (12.0, 12.0, 15.0, 18.0),
2: (15.0, 17.2, 16.0, 19.6),
3: (16.0, 12.0, 19.0, 18.0),
},
origins={index: (float(index), 18.0) for index in range(len(line_break_chars))},
detect_scripts=False,
)
outside_span = _fillable_span((10.0, 10.0, 40.0, 20.0))
outside_chars = [
_script_char(0, "A", height=6.0, center_y=15.0),
_script_char(1, ".", height=2.0, center_y=22.0),
_script_char(2, "B", height=6.0, center_y=15.0),
]
outside_bboxes = [
Bbox([12.0, 12.0, 15.0, 18.0]),
Bbox([15.0, 21.0, 16.0, 23.0]),
Bbox([16.0, 12.0, 19.0, 18.0]),
]
for char, bbox in zip(outside_chars, outside_bboxes, strict=True):
char["bbox"] = bbox
native.fill_char_in_spans(
[outside_span],
outside_chars,
10.0,
tight_bboxes={index: tuple(float(value) for value in bbox) for index, bbox in enumerate(outside_bboxes)},
origins={index: (float(index), 18.0) for index in range(len(outside_chars))},
detect_scripts=False,
)
assert line_break_span.content == "AB"
assert outside_span.content == "A B"
def test_span_fill_attaches_whitespace_to_tight_owned_neighbors() -> None:
"""验证空格跟随两侧 tight 字符进入同一 Span,而不是留在旧的重叠 Span。"""
first = _fillable_span((0.0, 0.0, 12.0, 20.0))
closer = _fillable_span((0.0, 8.0, 12.0, 16.0))
chars = [
_script_char(0, "A", height=14.0, center_y=10.0),
_script_char(1, " ", height=6.0, center_y=6.0),
_script_char(2, "C", height=14.0, center_y=10.0),
]
chars[0]["bbox"] = Bbox([0.0, 3.0, 5.0, 17.0])
chars[1]["bbox"] = Bbox([4.8, 6.0, 4.8, 6.0])
chars[2]["bbox"] = Bbox([5.0, 3.0, 10.0, 17.0])
native.fill_char_in_spans(
[first, closer],
chars,
8.0,
tight_bboxes={
0: (0.0, 10.0, 5.0, 14.0),
1: (8.0, 10.0, 9.0, 14.0),
2: (5.0, 10.0, 10.0, 14.0),
},
origins={index: (float(index), 14.0) for index in range(3)},
)
assert first.content == ""
assert closer.content == "A C"
def test_span_fill_chooses_closest_tight_candidate_before_span_order() -> None:
"""验证 tight bbox 同时命中重叠 Span 时按中心距离而非列表顺序归属。"""
farther = _fillable_span((10.0, 8.0, 30.0, 20.0))
closer = _fillable_span((10.0, 11.0, 30.0, 19.0))
char = {
"char": "A",
"bbox": Bbox([15.0, 8.0, 20.0, 24.0]),
"rotation": 0.0,
"font": {},
"char_idx": 0,
}
native.fill_char_in_spans(
[farther, closer],
[char], # type: ignore[list-item]
10.0,
tight_bboxes={0: (15.0, 13.0, 20.0, 17.0)},
origins={0: (15.0, 17.0)},
)
assert farther.content == ""
assert closer.content == "A"
def test_weak_offset_component_without_seed_stays_body() -> None:
"""验证只有连续级弱偏移、没有高置信证据的组件不会产生标签。"""
chars = [
_script_char(0, "A", height=10.0, center_y=10.0),
_script_char(1, "x", height=9.0, center_y=10.9),
]
assert _render_chars(chars) == "Ax"
def test_mixed_font_run_with_weak_dual_bbox_displacement_stays_body() -> None:
"""验证字体框高度突变但 tight/origin 中心偏移不足时仍保持正文。"""
chars = [
_script_char(0, "中", height=10.0, center_y=10.0, font_name="CJK"),
_script_char(1, "S", height=20.0, center_y=12.4, font_name="Latin"),
_script_char(2, "o", height=20.0, center_y=12.4, font_name="Latin"),
]
fixture = _ScriptFixture(
chars=chars,
tight_bboxes={
0: (0.0, 5.0, 1.0, 15.0),
1: (1.0, 8.8, 2.0, 13.8),
2: (2.0, 8.8, 3.0, 13.8),
},
origins={0: (0.0, 10.0), 1: (1.0, 11.2), 2: (2.0, 11.2)},
)
assert _render_chars(fixture) == "中So"
def test_isolated_punctuation_with_opposed_origin_stays_body() -> None:
"""验证标点的 origin 与 bbox 中心方向冲突时不会独立成为角标种子。"""
chars = [
_script_char(0, "A", height=10.0, center_y=10.0),
_script_char(1, ":", height=5.0, center_y=13.0),
_script_char(2, "B", height=10.0, center_y=10.0),
]
fixture = _ScriptFixture(
chars=chars,
tight_bboxes={
0: (0.0, 5.0, 1.0, 15.0),
1: (1.0, 11.0, 2.0, 15.0),
2: (2.0, 5.0, 3.0, 15.0),
},
origins={0: (0.0, 10.0), 1: (1.0, 8.0), 2: (2.0, 10.0)},
)
assert _render_chars(fixture) == "A:B"
def test_strong_isolated_symbol_with_three_consistent_signals_is_script() -> None:
"""验证三类几何均强位移的孤立符号仍可独立识别为角标。"""
chars = [
_script_char(0, "A", height=10.0, center_y=10.0),
_script_char(1, "*", height=5.0, center_y=6.0),
_script_char(2, "B", height=10.0, center_y=10.0),
]
assert _render_chars(chars) == "A<sup>*</sup>B"
def test_strong_geometry_seed_absorbs_same_side_punctuation() -> None:
"""验证强几何种子会把同侧连续标点纳入同一个上标组件。"""
chars = [
_script_char(0, "A", height=10.0, center_y=10.0),
_script_char(1, "2", height=7.0, center_y=7.0),
_script_char(2, ",", height=9.0, center_y=8.5),
]
assert _render_chars(chars) == "A<sup>2,</sup>"
def test_script_markup_records_detector_ownership() -> None:
"""验证只有脚本检测实际生成标签时才写入私有 ownership 标记。"""
script_span = _AnalyzeSpan(
type=ContentType.TEXT,
bbox=(0.0, 0.0, 1.0, 1.0),
metadata={
"chars": [
_script_char(0, "A", height=10.0, center_y=10.0),
_script_char(1, "2", height=7.0, center_y=7.0),
]
},
)
body_span = _AnalyzeSpan(
type=ContentType.TEXT,
bbox=(0.0, 0.0, 1.0, 1.0),
metadata={
"chars": [
_script_char(0, "A", height=10.0, center_y=10.0),
_script_char(1, "B", height=10.0, center_y=10.0),
]
},
)
script_fixture = _synthetic_fixture(script_span.metadata["chars"])
body_fixture = _synthetic_fixture(body_span.metadata["chars"])
native.chars_to_content(
script_span,
tight_bboxes=script_fixture.tight_bboxes,
origins=script_fixture.origins,
)
native.chars_to_content(
body_span,
tight_bboxes=body_fixture.tight_bboxes,
origins=body_fixture.origins,
)
assert script_span.content == "A<sup>2</sup>"
assert script_span.metadata[PDF_NATIVE_SCRIPT_MARKUP_KEY] is True
assert PDF_NATIVE_SCRIPT_MARKUP_KEY not in body_span.metadata
def test_missing_extended_geometry_keeps_all_characters_body() -> None:
"""验证缺少 tight/origin 时宁可不标 style,也不恢复字体或文本猜测。"""
span = _AnalyzeSpan(
type=ContentType.TEXT,
bbox=(0.0, 0.0, 1.0, 1.0),
metadata={
"chars": [
_script_char(0, "R", height=10.0, center_y=10.0, font_name="Base"),
_script_char(1, "2", height=7.0, center_y=7.0, font_name="Suffix"),
]
},
)
native.chars_to_content(span)
assert span.content == "R2"
assert PDF_NATIVE_SCRIPT_MARKUP_KEY not in span.metadata
def test_overlapping_spacing_ogonek_composes_demo2_author_name() -> None:
"""验证 demo2 原生字符流中与 e 重叠的 spacing ogonek 合成为 ę。"""
content = _render_demo_line("demo2.pdf", 0, "Kowalczuk")
assert content.startswith("Jędrzej Kowalczuk")
assert "\u02db" not in content
def test_non_overlapping_spacing_ogonek_remains_literal() -> None:
"""验证没有几何重叠的 spacing ogonek 不会被字符规范化误合并。"""
chars = [
_script_char(0, "˛", height=10.0, center_y=10.0),
_script_char(1, "e", height=10.0, center_y=10.0),
]
merged = native._merge_overlapping_spacing_diacritics(chars)
assert [char["char"] for char in merged] == ["˛", "e"]
def test_tight_geometry_keeps_same_size_loose_base_character_body() -> None:
"""验证 loose 高度接近时仍只根据 tight/origin 标记后缀。"""
chars = [
_script_char(0, "9", height=20.0, center_y=12.0),
_script_char(1, " ", height=0.0, center_y=12.0),
_script_char(2, "R", height=11.0, center_y=10.0, font_name="Base"),
_script_char(3, "2", height=10.5, center_y=8.5, font_name="Suffix"),
]
fixture = _synthetic_fixture(
chars,
tight_heights={3: 7.0},
origin_ys={2: 10.0, 3: 8.5},
)
assert _render_chars(fixture) == "9 R<sup>2</sup>"
@pytest.mark.parametrize(
("chars", "expected"),
[
pytest.param(
[
_script_char(0, "N", height=10.0, center_y=10.0, font_name="Latin"),
_script_char(1, "P", height=10.0, center_y=10.0, font_name="Latin"),
_script_char(2, "U", height=10.0, center_y=10.0, font_name="Latin"),
_script_char(3, "开", height=10.0, center_y=8.5, font_name="CJK"),
_script_char(4, "发", height=10.0, center_y=8.5, font_name="CJK"),
],
"NPU开发",
id="latin-to-cjk",
),
pytest.param(
[
_script_char(0, "2", height=10.0, center_y=10.0, font_name="Digit"),
_script_char(1, "4", height=10.0, center_y=10.0, font_name="Digit"),
_script_char(2, "年", height=10.0, center_y=8.5, font_name="CJK"),
],
"24年",
id="digit-to-cjk",
),
],
)
def test_cjk_font_boundary_does_not_seed_script_component(
chars: list[Char],
expected: str,
) -> None:
"""验证正常的 CJK 与拉丁或数字字体切换不会被解释为上下标后缀。"""
assert _render_chars(chars) == expected
def test_shifted_reference_geometry_marks_the_complete_component() -> None:
"""验证不读取括号语义也会把同基线引用整体标记为上标。"""
chars = [
_script_char(0, "依", height=10.0, center_y=10.0),
_script_char(1, "[", height=7.0, center_y=7.0),
_script_char(2, "1", height=10.2, center_y=7.5, font_name="Reference"),
_script_char(3, "]", height=7.0, center_y=7.0),
_script_char(4, "。", height=10.0, center_y=10.0),
]
fixture = _synthetic_fixture(
chars,
tight_heights={1: 6.0, 2: 6.5, 3: 6.0},
origin_ys={0: 10.0, 1: 7.0, 2: 7.0, 3: 7.0, 4: 10.0},
)
assert _render_chars(fixture) == "依<sup>[1]</sup>。"
@pytest.mark.parametrize(
("pdf_name", "page_index", "probe", "expected_fragments"),
[
pytest.param(
"demo1.pdf",
0,
"Patrick N.J. Lanea,c,*",
("Lane<sup>a,c,*</sup>", "Best<sup>b,c,d</sup>"),
id="author-affiliations",
),
pytest.param(
"demo3.pdf",
1,
"where segi",
("seg<sub>i</sub>",),
id="variable-subscript",
),
pytest.param(
"demo4.pdf",
0,
"Innate [18F]",
("[<sup>18</sup>F]",),
id="isotope-superscript",
),
pytest.param(
"mixed_elements_pages_03_06.pdf",
0,
"vacuum of 10–3 Pa",
("10<sup>–3</sup> Pa",),
id="negative-exponent",
),
pytest.param(
"mixed_elements_pages_03_06.pdf",
0,
"to consist of ScFeGe2",
("ScFeGe<sub>2</sub>", "Sc<sub>4</sub>Fe<sub>4</sub>Ge<sub>6.6</sub>"),
id="chemical-subscripts",
),
pytest.param(
"mixed_elements_pages_11_15.pdf",
0,
"H2-antagonist",
("H<sub>2</sub>-antagonist",),
id="medical-subscript",
),
pytest.param(
"2407.00079v4_origi-10.pdf",
0,
"7: Tqueue ←",
("T<sub>queue</sub>",),
id="algorithm-variable-subscript",
),
pytest.param(
"demo2.pdf",
1,
"where ∆g(p, q) is",
("∆<sub>g</sub>(p, q)", "∆<sub>c</sub>(p, q)"),
id="delta-subscripts",
),
pytest.param(
"demo3.pdf",
4,
"same as BERTBASE and BERTLARGE",
("BERT<sub>BASE</sub>", "BERT<sub>LARGE</sub>"),
id="model-name-subscripts",
),
pytest.param(
"demo4.pdf",
1,
"body mass index (BMI) were 71.75 kg and 25.3 kg/m2",
("kg/m<sup>2</sup>",),
id="unit-superscript",
),
pytest.param(
"中文论文.pdf",
0,
"其中,季节等[1]",
("季节等<sup>[1]</sup>",),
id="chinese-reference",
),
pytest.param(
"中文论文.pdf",
0,
"等[7]",
("等<sup>[7]</sup>",),
id="chinese-reference-7",
),
pytest.param(
"中文论文.pdf",
0,
"Zhao等[8]",
("Zhao等<sup>[8]</sup>",),
id="chinese-reference-8",
),
pytest.param(
"中文论文.pdf",
0,
"利用决定系数(R2",
("决定系数(R<sup>2</sup>",),
id="same-size-font-suffix",
),
pytest.param(
"中文论文.pdf",
0,
"Voting 模型的性能略低于 Stacking 模型(R2",
("Stacking 模型(R<sup>2</sup>",),
id="same-size-font-suffix-after-tall-digits",
),
pytest.param(
"中文论文.pdf",
1,
"扎兰屯段边坡,经烘干后测得干密度为1. 72 g⋅cm-3",
("g⋅cm<sup>-3</sup>",),
id="cjk-unit-superscript",
),
],
)
def test_real_pdf_scripts_keep_confirmed_markup(
pdf_name: str,
page_index: int,
probe: str,
expected_fragments: tuple[str, ...],
) -> None:
"""验证已有输出中经页面视觉确认的真实上下标继续保留。"""
content = _render_demo_line(pdf_name, page_index, probe)
assert all(expected_fragment in content for expected_fragment in expected_fragments)
@pytest.mark.parametrize(
("probe", "expected_fragment"),
[
pytest.param(
"预测。Meng 等[10]利用机器学习对冻结岩土的力学",
"Meng 等<sup>[10]</sup>",
id="reference-10",
),
pytest.param(
"特性进行了预测。Esmaeili-Falak 等[11]通过机器学习对冻土的轴向抗压强度",
"Esmaeili-Falak 等<sup>[11]</sup>",
id="reference-11",
),
pytest.param(
"Li等[12]采用 ANN 模型对冻土应力应变进行预测和验证。",
"Li等<sup>[12]</sup>",
id="reference-12",
),
],
)
def test_chinese_mixed_font_reference_uses_local_main_font_body(
probe: str,
expected_fragment: str,
) -> None:
"""验证含拉丁姓名的中文 OCR 行使用主字体正文带识别数字引用。"""
chars = _demo_chars("中文论文.pdf", 1, probe, selection="contiguous")
assert expected_fragment in _render_chars(chars)
def test_chinese_reference_line_only_marks_the_reference() -> None:
"""验证短行中数字引用占优时,正文“依据”和句号不会反向误判为下标。"""
chars = _demo_chars("中文论文.pdf", 1, "依据[14-17]。", selection="contiguous")
content = _render_chars(chars)
assert content == "依据<sup>[14-17]</sup>。"
assert "<sub>" not in content
@pytest.mark.parametrize(
("pdf_name", "page_index", "probe", "expected"),
[
pytest.param(
"mixed_elements_pages_03_06.pdf",
0,
"0.26 µm (Fig. 1a)",
"0.26 µm (Fig. 1a)",
id="plain-micrometer-unit",
),
pytest.param(
"demo2.pdf",
2,
") to O(ω).",
") to O(ω).",
id="plain-complexity-notation",
),
],
)
def test_real_pdf_plain_text_does_not_gain_script_markup(
pdf_name: str,
page_index: int,
probe: str,
expected: str,
) -> None:
"""验证字体框或 glyph 形状差异不会给普通单位和复杂度记号加角标。"""
content = _render_demo_line(pdf_name, page_index, probe)
assert expected in content
assert "<sup>" not in content and "<sub>" not in content
@pytest.mark.parametrize(
("page_index", "probe", "expected"),
[
pytest.param(0, "providing", "providing", id="page-1-english-abstract"),
pytest.param(10, "Source", "Source", id="page-11-source"),
pytest.param(10, "Hypothesis", "Hypothesis", id="page-11-hypothesis"),
pytest.param(10, "Reference", "Reference", id="page-11-reference"),
pytest.param(10, "如 BLEU 和 ROUGE", "BLEU 和 ROUGE", id="page-11-metrics"),
pytest.param(16, "generalisation", "generalisation", id="page-17-reference"),
],
)
def test_zh2_normal_english_runs_do_not_gain_script_markup(
page_index: int,
probe: str,
expected: str,
) -> None:
"""验证中文论文2中受字体框影响的普通英文 run 不会被误判成上下标。"""
content = _render_demo_line(_ZH_2_SOURCE, page_index, probe)
assert expected in content
assert "<sup>" not in content and "<sub>" not in content
@pytest.mark.parametrize(
("probe", "minimum_font_count"),
[
pytest.param("• 合成支持工程师 Support Engineer", 2, id="bullet-text"),
pytest.param("• 混合字体工具 Acuity Toolkit", 2, id="mixed-font-bullet-text"),
pytest.param("• 测试系统 Ubuntu 24.04", 2, id="version-bullet-text"),
pytest.param(
"mineru_toolkit_binary_1.0.0_linux_x86_64.tgz",
1,
id="filename-descender",
),
],
)
def test_synthetic_cjk_plain_text_does_not_gain_script_tags(
probe: str,
minimum_font_count: int,
) -> None:
"""验证合成项目符号、混合字体和文件名降部字符不会触发上下标误判。"""
_, matching_lines = _residual_script_case("native_cjk_layout_synthetic.pdf", probe)
contents = [_render_chars(chars) for chars in matching_lines]
font_names = {str((char.get("font") or {}).get("name", "")) for fixture in matching_lines for char in fixture.chars}
assert contents
assert len(font_names) >= minimum_font_count
assert all("<sup>" not in content and "<sub>" not in content for content in contents)
def test_rotated_powerpoint_page_keeps_all_plain_text_on_baseline() -> None:
"""验证 PowerPoint 旋转页的紧字形框不会把普通升部和降部误判为上下标。"""
case, fragments = _residual_script_case("metabolic_pathway_page_3.pdf")
rendered_lines = [_render_chars(fragment) for fragment in fragments]
assert case["page_rotation"] == 90
assert any((char.get("font") or {}).get("name") == "CalifornianFB-Reg" for fragment in fragments for char in fragment.chars)
assert len(rendered_lines) == 33
assert rendered_lines[0] == "Energy Metabolism"
assert all("<sup>" not in content and "<sub>" not in content for content in rendered_lines)
def test_mixed_font_unit_and_citation_use_shared_local_body_reference() -> None:
"""最小真实字符夹具通过 Hybrid 生产路径恢复正文单位和完整引用上标。"""
from copy import deepcopy
payload = json.loads((_PROJECT_ROOT / "tests/fixtures/pdf_mixed_font_script_line.json").read_text(encoding="utf-8"))
chars = []
tight_bboxes = {}
origins = {}
for row in payload["chars"]:
char = dict(row)
tight, origin = char.pop("tight"), char.pop("origin")
char["bbox"] = Bbox(char["bbox"])
chars.append(char)
if tight is not None:
tight_bboxes[char["char_idx"]] = tuple(tight)
if origin is not None:
origins[char["char_idx"]] = tuple(origin)
before = deepcopy(payload)
roles = native._classify_char_script_roles(chars, tight_bboxes=tight_bboxes, origins=origins)
assert {str(c["char_idx"]): role for c, role in zip(chars, roles) if 1353 <= c["char_idx"] <= 1360} == payload["expected"]
text = "".join(_render_chars(_ScriptFixture(chars, tight_bboxes, origins)).split())
assert "12gpm<sup>[8]</sup>" in text
assert "<sub>" not in text
assert payload == before