Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

perf(pdf): consume native span content without character regrouping

myhloli8adde38a948b
diff --git a/.github/workflows/docvortex-boundary.yml b/.github/workflows/docvortex-boundary.yml
index 6f88d2d2..2dd3ee42 100644
--- a/.github/workflows/docvortex-boundary.yml
+++ b/.github/workflows/docvortex-boundary.yml
@@ -167,7 +167,7 @@ jobs:
assert any(path.is_relative_to(root) for root in site_roots), (name, path, site_roots)
info = backend_info()
assert info["backend"] == "rust", info
- assert native.PROTOCOL_VERSION >= 13 and info["protocol"] == native.PROTOCOL_VERSION, info
+ assert native.PROTOCOL_VERSION >= 14 and info["protocol"] == native.PROTOCOL_VERSION, info
assert info["extension_sha256"] == hashlib.sha256(origins["native"].read_bytes()).hexdigest(), info
artifact = json.loads(Path(".boundary/engine-artifact.json").read_text(encoding="utf-8"))
distribution = importlib.metadata.distribution("docvortex")
diff --git a/mineru/backend/analysis/pdf/text/native.py b/mineru/backend/analysis/pdf/text/native.py
index 0134a0c2..9a7b13d6 100644
--- a/mineru/backend/analysis/pdf/text/native.py
+++ b/mineru/backend/analysis/pdf/text/native.py
@@ -516,9 +516,14 @@ def _bridge_unassigned_punctuation(
assigned_span_indices[char_position] = previous_owner
-def _owned_span_assignments(owner: Any, spans: list[_AnalyzeSpan], median_height: float):
+def _owned_span_inputs(owner: Any, spans: list[_AnalyzeSpan], median_height: float) -> tuple | None:
"""内部同源快照直接匹配字符;特殊可执行值和替换后的规则保留原 Python 语义。"""
- if owner is None or calculate_char_in_span is not _STANDARD_CHAR_IN_SPAN:
+ if (
+ owner is None
+ or calculate_char_in_span is not _STANDARD_CHAR_IN_SPAN
+ or type(CONTROL_LINE_BREAK_CHARS) not in (set, frozenset)
+ or CONTROL_LINE_BREAK_CHARS != {"\r", "\n"}
+ ):
return None
defaults = calculate_char_in_span.__defaults__
if type(defaults) is not tuple or len(defaults) != 1:
@@ -539,7 +544,61 @@ def _owned_span_assignments(owner: Any, spans: list[_AnalyzeSpan], median_height
for flags in (LINE_STOP_FLAG, LINE_START_FLAG)
):
return None
- return owner.assign_spans(boxes, median_height, LINE_STOP_FLAG, LINE_START_FLAG, ratio)
+ return boxes, median_height, LINE_STOP_FLAG, LINE_START_FLAG, ratio
+
+
+def _owned_span_assignments(owner: Any, spans: list[_AnalyzeSpan], median_height: float) -> list[int | None] | None:
+ """用同源快照匹配字符,数值或配置不满足既有语义时显式选择参考实现。"""
+ inputs = _owned_span_inputs(owner, spans, median_height)
+ return owner.assign_spans(*inputs) if inputs is not None else None
+
+
+def _owned_span_texts(owner: Any, spans: list[_AnalyzeSpan], median_height: float, detect_scripts: bool) -> list[tuple] | None:
+ """共享脚本侧车已准备时,整段回填留在 Rust;不绕过自定义规则或已有字符内容。"""
+ if detect_scripts or not callable(getattr(owner, "prepare_span_texts", None)):
+ return None
+ functions = (
+ chars_to_content,
+ __replace_unicode,
+ __replace_ligatures,
+ _get_private_use_text_signal,
+ _is_private_use_char,
+ _get_char_bbox_metrics_list,
+ _merge_overlapping_spacing_diacritics_with_protection,
+ _compose_overlapping_spacing_diacritic,
+ _axis_overlap_ratio,
+ _wrap_script_runs,
+ _append_script_wrapped_text,
+ unicodedata.category,
+ unicodedata.normalize,
+ statistics.median,
+ )
+ if any(value is not original for value, original in zip(functions, _STANDARD_CONTENT_FUNCTIONS)):
+ return None
+ if any(
+ type(span.metadata) is not dict or type(span.metadata.get("chars")) is not list or span.metadata["chars"]
+ for span in spans
+ ):
+ return None
+ if type(_SPACING_DIACRITIC_TO_COMBINING) is not dict or any(
+ type(key) is not str or type(value) is not str for key, value in _SPACING_DIACRITIC_TO_COMBINING.items()
+ ):
+ return None
+ threshold = SPACING_DIACRITIC_MIN_OVERLAP_RATIO
+ if type(threshold) not in (int, float) or not math.isfinite(threshold):
+ return None
+ if any(type(value) is not int or not 0 <= value <= 0x10FFFF for value in (PRIVATE_USE_AREA_START, PRIVATE_USE_AREA_END)):
+ return None
+ inputs = _owned_span_inputs(owner, spans, median_height)
+ if inputs is None:
+ return None
+ return owner.prepare_span_texts(
+ *inputs,
+ list(CONTROL_LINE_BREAK_CHARS),
+ _SPACING_DIACRITIC_TO_COMBINING,
+ threshold,
+ (PRIVATE_USE_AREA_START, PRIVATE_USE_AREA_END),
+ )
def fill_char_in_spans(
@@ -557,8 +616,11 @@ def fill_char_in_spans(
tight_bboxes = tight_bboxes or {}
origins = origins or {}
- assigned_span_indices = _owned_span_assignments(_native_text, spans, median_span_height)
- if assigned_span_indices is not None and len(assigned_span_indices) != len(all_chars):
+ native_contents = _owned_span_texts(_native_text, spans, median_span_height, detect_scripts)
+ if native_contents is not None and len(native_contents) != len(spans):
+ raise RuntimeError("Native content count differs from span count")
+ assigned_span_indices = _owned_span_assignments(_native_text, spans, median_span_height) if native_contents is None else []
+ if native_contents is None and assigned_span_indices is not None and len(assigned_span_indices) != len(all_chars):
raise RuntimeError("Native span assignments do not match the owned character sequence")
if assigned_span_indices is None:
grid_size = max(1, median_span_height)
@@ -657,15 +719,24 @@ def fill_char_in_spans(
spans[span_index].metadata["chars"].append(char)
need_ocr_spans = []
- for span in spans:
- private_use_signal = _get_private_use_text_signal(span.metadata["chars"])
- should_post_ocr_private_use = _should_fallback_to_post_ocr_for_private_use_text(private_use_signal)
- chars_to_content(
- span,
- tight_bboxes=tight_bboxes,
- origins=origins,
- detect_scripts=detect_scripts,
- )
+ for span_index, span in enumerate(spans):
+ if native_contents is None:
+ private_use_signal = _get_private_use_text_signal(span.metadata["chars"])
+ should_post_ocr_private_use = _should_fallback_to_post_ocr_for_private_use_text(private_use_signal)
+ chars_to_content(span, tight_bboxes=tight_bboxes, origins=origins, detect_scripts=detect_scripts)
+ else:
+ text, pua_count, text_count, pua_run = native_contents[span_index]
+ private_use_signal = {
+ "pua_count": pua_count,
+ "text_char_count": text_count,
+ "max_pua_run": pua_run,
+ "pua_ratio": pua_count / text_count if text_count else 0.0,
+ }
+ should_post_ocr_private_use = _should_fallback_to_post_ocr_for_private_use_text(private_use_signal)
+ span.metadata.pop(PDF_NATIVE_SCRIPT_MARKUP_KEY, None)
+ if text is not None:
+ span.content = text
+ del span.metadata["chars"]
# 有的span中虽然没有字但有一两个空的占位符,用宽高和content长度过滤
if should_post_ocr_private_use and span.content:
span.metadata[POST_OCR_FALLBACK_CONTENT_KEY] = span.content
@@ -1031,3 +1102,22 @@ def chars_to_content(
span.metadata[PDF_NATIVE_SCRIPT_MARKUP_KEY] = True
del span.metadata["chars"]
+
+
+# 仅原有内置规则启用整段原生化,替换后的函数保留参考调用及异常语义。
+_STANDARD_CONTENT_FUNCTIONS = (
+ chars_to_content,
+ __replace_unicode,
+ __replace_ligatures,
+ _get_private_use_text_signal,
+ _is_private_use_char,
+ _get_char_bbox_metrics_list,
+ _merge_overlapping_spacing_diacritics_with_protection,
+ _compose_overlapping_spacing_diacritic,
+ _axis_overlap_ratio,
+ _wrap_script_runs,
+ _append_script_wrapped_text,
+ unicodedata.category,
+ unicodedata.normalize,
+ statistics.median,
+)
diff --git a/tests/unittest/test_native_span_assignment.py b/tests/unittest/test_native_span_assignment.py
index f72e03e2..bc8010b3 100644
--- a/tests/unittest/test_native_span_assignment.py
+++ b/tests/unittest/test_native_span_assignment.py
@@ -1,5 +1,9 @@
"""真实字符快照的 Rust 归属与原 Python 规则逐字符差分。"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
from copy import deepcopy
from dataclasses import asdict
from io import BytesIO
@@ -14,8 +18,12 @@ from mineru.backend.analysis.pdf.text.models import _AnalyzeSpan
from mineru.types import ContentType
+if TYPE_CHECKING:
+ from docvortex._native import NativeTextSnapshot
+
+
@pytest.fixture
-def owned_text():
+def owned_text() -> NativeTextSnapshot:
"""生成包含边界标点、空白及多行的 PDF,关闭原文档后再验证归属。"""
stream = BytesIO()
canvas = Canvas(stream, pagesize=(320, 100))
@@ -31,7 +39,9 @@ def owned_text():
@pytest.mark.parametrize("seed", range(24))
-def test_owned_assignments_match_python_for_overlapping_spans(owned_text, monkeypatch, seed):
+def test_owned_assignments_match_python_for_overlapping_spans(
+ owned_text: NativeTextSnapshot, monkeypatch: pytest.MonkeyPatch, seed: int
+) -> None:
"""在随机重叠框、负坐标与相同框中比对原始 char_idx 归属、文本及后置 OCR 判定。"""
randomizer = random.Random(seed)
boxes = [(0.0, 0.0, 100.0, 100.0), (0.0, 0.0, 100.0, 100.0)]
@@ -41,7 +51,7 @@ def test_owned_assignments_match_python_for_overlapping_spans(owned_text, monkey
original = native.chars_to_content
observed = []
- def capture(span, **kwargs):
+ def capture(span: _AnalyzeSpan, **kwargs: object) -> None:
"""在原内容组装删除临时字符前记录归属,不替换实际规则。"""
observed.append((span.bbox, [char["char_idx"] for char in span.metadata["chars"]]))
original(span, **kwargs)
@@ -71,10 +81,12 @@ def test_owned_assignments_match_python_for_overlapping_spans(owned_text, monkey
assert results[0] == results[1]
-def test_owned_assignment_keeps_replaced_rules_on_reference_path(owned_text, monkeypatch):
+def test_owned_assignment_keeps_replaced_rules_on_reference_path(
+ owned_text: NativeTextSnapshot, monkeypatch: pytest.MonkeyPatch
+) -> None:
"""调用方替换归属规则时不偷偷使用原生版本,保持参考计算能力。"""
- def reject(*args):
+ def reject(*args: object) -> bool:
"""让所有字符失配,验证规则替换确实被执行。"""
return False
@@ -82,7 +94,7 @@ def test_owned_assignment_keeps_replaced_rules_on_reference_path(owned_text, mon
assert native._owned_span_assignments(owned_text, [_AnalyzeSpan(ContentType.TEXT, (0.0, 0.0, 100.0, 100.0))], 12.0) is None
-def test_rotated_snapshot_explicitly_retains_reference_selection():
+def test_rotated_snapshot_explicitly_retains_reference_selection() -> None:
"""整行倾斜水印仍由原有组行规则裁决,不能把未迁移角度当成可忽略字符。"""
stream = BytesIO()
canvas = Canvas(stream, pagesize=(120, 120))
@@ -95,3 +107,82 @@ def test_rotated_snapshot_explicitly_retains_reference_selection():
pytest.skip("Native span assignment unavailable")
assert not owner.supports_span_matching()
assert owner.assign_spans([(0.0, 0.0, 120.0, 120.0)], 10.0, native.LINE_STOP_FLAG, native.LINE_START_FLAG, 0.33) is None
+
+
[email protected]("seed", range(24))
+def test_owned_content_matches_reference(owned_text: NativeTextSnapshot, seed: int) -> None:
+ """不替换内容函数,比对原生组装后的全部字段、空框原文及 OCR 待处理集合。"""
+ from docvortex import _native
+
+ randomizer = random.Random(seed)
+ boxes = [(0.0, 0.0, 320.0, 100.0), (500.0, 500.0, 520.0, 520.0)]
+ for _ in range(12):
+ x, y = randomizer.uniform(-15, 100), randomizer.uniform(-10, 95)
+ boxes.append((x, y, x + randomizer.uniform(2, 90), y + randomizer.uniform(2, 30)))
+ results = []
+ before = _native.text_snapshot_stats()[2]
+ for owner in [None, owned_text]:
+ geometry = owned_text.materialize_geometry()
+ spans = [
+ _AnalyzeSpan(
+ ContentType.TEXT,
+ box,
+ content="原文",
+ metadata={"chars": [], "height": box[3] - box[1], "width": box[2] - box[0]},
+ )
+ for box in boxes
+ ]
+ pending = native.fill_char_in_spans(
+ spans,
+ geometry.chars,
+ 12.0,
+ tight_bboxes=geometry.tight_bboxes,
+ origins=geometry.origins,
+ detect_scripts=False,
+ _native_text=owner,
+ )
+ results.append(([asdict(span) for span in spans], [spans.index(span) for span in pending]))
+ assert _native.text_snapshot_stats()[2] == before + 1
+ assert results[0] == results[1]
+
+
[email protected]("text", ["e˛", "˛e", "fiflffffifflſtst", "\ue000\ue001 X", "a\u2003b\u00a0c", "A\x02B"])
+def test_owned_unicode_content_matches_reference(text: str) -> None:
+ """真实嵌入字体保留 Unicode 映射,比对附加符、连字、PUA 与控制字符处理。"""
+ from pathlib import Path
+ import reportlab
+ from reportlab.pdfbase import pdfmetrics
+ from reportlab.pdfbase.ttfonts import TTFont
+ from docvortex import _native
+
+ pdfmetrics.registerFont(TTFont("ContentVera", str(Path(reportlab.__file__).parent / "fonts/Vera.ttf")))
+ stream = BytesIO()
+ canvas = Canvas(stream, pagesize=(320, 100))
+ canvas.setFont("ContentVera", 12)
+ if text in {"e˛", "˛e"}:
+ for character in text:
+ canvas.drawString(10, 70, character)
+ else:
+ canvas.drawString(10, 70, text)
+ canvas.save()
+ with PDFDocument(stream.getvalue()) as document:
+ owner = document[0].get_text_snapshot()
+ if owner is None:
+ pytest.skip("Native content unavailable")
+ results = []
+ before = _native.text_snapshot_stats()[2]
+ for current in [None, owner]:
+ geometry = owner.materialize_geometry()
+ span = _AnalyzeSpan(ContentType.TEXT, (0.0, 0.0, 320.0, 100.0), metadata={"chars": [], "height": 100.0, "width": 320.0})
+ pending = native.fill_char_in_spans(
+ [span],
+ geometry.chars,
+ 12.0,
+ tight_bboxes=geometry.tight_bboxes,
+ origins=geometry.origins,
+ detect_scripts=False,
+ _native_text=current,
+ )
+ results.append((asdict(span), bool(pending)))
+ assert _native.text_snapshot_stats()[2] == before + 1
+ assert results[0] == results[1]