Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
perf(pdf): consume native span content without character regrouping
myhloli8adde38a948b
diff --git a/.github/workflows/docvortex-boundary.yml b/.github/workflows/docvortex-boundary.ymlindex 6f88d2d2..2dd3ee42 100644--- a/.github/workflows/docvortex-boundary.yml+++ b/.github/workflows/docvortex-boundary.yml@@ -167,7 +167,7 @@ jobs:assert any(path.is_relative_to(root) for root in site_roots), (name, path, site_roots)info = backend_info()assert info["backend"] == "rust", info- assert native.PROTOCOL_VERSION >= 13 and info["protocol"] == native.PROTOCOL_VERSION, info+ assert native.PROTOCOL_VERSION >= 14 and info["protocol"] == native.PROTOCOL_VERSION, infoassert info["extension_sha256"] == hashlib.sha256(origins["native"].read_bytes()).hexdigest(), infoartifact = json.loads(Path(".boundary/engine-artifact.json").read_text(encoding="utf-8"))distribution = importlib.metadata.distribution("docvortex")diff --git a/mineru/backend/analysis/pdf/text/native.py b/mineru/backend/analysis/pdf/text/native.pyindex 0134a0c2..9a7b13d6 100644--- a/mineru/backend/analysis/pdf/text/native.py+++ b/mineru/backend/analysis/pdf/text/native.py@@ -516,9 +516,14 @@ def _bridge_unassigned_punctuation(assigned_span_indices[char_position] = previous_owner-def _owned_span_assignments(owner: Any, spans: list[_AnalyzeSpan], median_height: float):+def _owned_span_inputs(owner: Any, spans: list[_AnalyzeSpan], median_height: float) -> tuple | None:"""内部同源快照直接匹配字符;特殊可执行值和替换后的规则保留原 Python 语义。"""- if owner is None or calculate_char_in_span is not _STANDARD_CHAR_IN_SPAN:+ if (+ owner is None+ or calculate_char_in_span is not _STANDARD_CHAR_IN_SPAN+ or type(CONTROL_LINE_BREAK_CHARS) not in (set, frozenset)+ or CONTROL_LINE_BREAK_CHARS != {"\r", "\n"}+ ):return Nonedefaults = calculate_char_in_span.__defaults__if type(defaults) is not tuple or len(defaults) != 1:@@ -539,7 +544,61 @@ def _owned_span_assignments(owner: Any, spans: list[_AnalyzeSpan], median_heightfor flags in (LINE_STOP_FLAG, LINE_START_FLAG)):return None- return owner.assign_spans(boxes, median_height, LINE_STOP_FLAG, LINE_START_FLAG, ratio)+ return boxes, median_height, LINE_STOP_FLAG, LINE_START_FLAG, ratio+++def _owned_span_assignments(owner: Any, spans: list[_AnalyzeSpan], median_height: float) -> list[int | None] | None:+ """用同源快照匹配字符,数值或配置不满足既有语义时显式选择参考实现。"""+ inputs = _owned_span_inputs(owner, spans, median_height)+ return owner.assign_spans(*inputs) if inputs is not None else None+++def _owned_span_texts(owner: Any, spans: list[_AnalyzeSpan], median_height: float, detect_scripts: bool) -> list[tuple] | None:+ """共享脚本侧车已准备时,整段回填留在 Rust;不绕过自定义规则或已有字符内容。"""+ if detect_scripts or not callable(getattr(owner, "prepare_span_texts", None)):+ return None+ functions = (+ chars_to_content,+ __replace_unicode,+ __replace_ligatures,+ _get_private_use_text_signal,+ _is_private_use_char,+ _get_char_bbox_metrics_list,+ _merge_overlapping_spacing_diacritics_with_protection,+ _compose_overlapping_spacing_diacritic,+ _axis_overlap_ratio,+ _wrap_script_runs,+ _append_script_wrapped_text,+ unicodedata.category,+ unicodedata.normalize,+ statistics.median,+ )+ if any(value is not original for value, original in zip(functions, _STANDARD_CONTENT_FUNCTIONS)):+ return None+ if any(+ type(span.metadata) is not dict or type(span.metadata.get("chars")) is not list or span.metadata["chars"]+ for span in spans+ ):+ return None+ if type(_SPACING_DIACRITIC_TO_COMBINING) is not dict or any(+ type(key) is not str or type(value) is not str for key, value in _SPACING_DIACRITIC_TO_COMBINING.items()+ ):+ return None+ threshold = SPACING_DIACRITIC_MIN_OVERLAP_RATIO+ if type(threshold) not in (int, float) or not math.isfinite(threshold):+ return None+ if any(type(value) is not int or not 0 <= value <= 0x10FFFF for value in (PRIVATE_USE_AREA_START, PRIVATE_USE_AREA_END)):+ return None+ inputs = _owned_span_inputs(owner, spans, median_height)+ if inputs is None:+ return None+ return owner.prepare_span_texts(+ *inputs,+ list(CONTROL_LINE_BREAK_CHARS),+ _SPACING_DIACRITIC_TO_COMBINING,+ threshold,+ (PRIVATE_USE_AREA_START, PRIVATE_USE_AREA_END),+ )def fill_char_in_spans(@@ -557,8 +616,11 @@ def fill_char_in_spans(tight_bboxes = tight_bboxes or {}origins = origins or {}- assigned_span_indices = _owned_span_assignments(_native_text, spans, median_span_height)- if assigned_span_indices is not None and len(assigned_span_indices) != len(all_chars):+ native_contents = _owned_span_texts(_native_text, spans, median_span_height, detect_scripts)+ if native_contents is not None and len(native_contents) != len(spans):+ raise RuntimeError("Native content count differs from span count")+ assigned_span_indices = _owned_span_assignments(_native_text, spans, median_span_height) if native_contents is None else []+ if native_contents is None and assigned_span_indices is not None and len(assigned_span_indices) != len(all_chars):raise RuntimeError("Native span assignments do not match the owned character sequence")if assigned_span_indices is None:grid_size = max(1, median_span_height)@@ -657,15 +719,24 @@ def fill_char_in_spans(spans[span_index].metadata["chars"].append(char)need_ocr_spans = []- for span in spans:- private_use_signal = _get_private_use_text_signal(span.metadata["chars"])- should_post_ocr_private_use = _should_fallback_to_post_ocr_for_private_use_text(private_use_signal)- chars_to_content(- span,- tight_bboxes=tight_bboxes,- origins=origins,- detect_scripts=detect_scripts,- )+ for span_index, span in enumerate(spans):+ if native_contents is None:+ private_use_signal = _get_private_use_text_signal(span.metadata["chars"])+ should_post_ocr_private_use = _should_fallback_to_post_ocr_for_private_use_text(private_use_signal)+ chars_to_content(span, tight_bboxes=tight_bboxes, origins=origins, detect_scripts=detect_scripts)+ else:+ text, pua_count, text_count, pua_run = native_contents[span_index]+ private_use_signal = {+ "pua_count": pua_count,+ "text_char_count": text_count,+ "max_pua_run": pua_run,+ "pua_ratio": pua_count / text_count if text_count else 0.0,+ }+ should_post_ocr_private_use = _should_fallback_to_post_ocr_for_private_use_text(private_use_signal)+ span.metadata.pop(PDF_NATIVE_SCRIPT_MARKUP_KEY, None)+ if text is not None:+ span.content = text+ del span.metadata["chars"]# 有的span中虽然没有字但有一两个空的占位符,用宽高和content长度过滤if should_post_ocr_private_use and span.content:span.metadata[POST_OCR_FALLBACK_CONTENT_KEY] = span.content@@ -1031,3 +1102,22 @@ def chars_to_content(span.metadata[PDF_NATIVE_SCRIPT_MARKUP_KEY] = Truedel span.metadata["chars"]+++# 仅原有内置规则启用整段原生化,替换后的函数保留参考调用及异常语义。+_STANDARD_CONTENT_FUNCTIONS = (+ chars_to_content,+ __replace_unicode,+ __replace_ligatures,+ _get_private_use_text_signal,+ _is_private_use_char,+ _get_char_bbox_metrics_list,+ _merge_overlapping_spacing_diacritics_with_protection,+ _compose_overlapping_spacing_diacritic,+ _axis_overlap_ratio,+ _wrap_script_runs,+ _append_script_wrapped_text,+ unicodedata.category,+ unicodedata.normalize,+ statistics.median,+)diff --git a/tests/unittest/test_native_span_assignment.py b/tests/unittest/test_native_span_assignment.pyindex f72e03e2..bc8010b3 100644--- a/tests/unittest/test_native_span_assignment.py+++ b/tests/unittest/test_native_span_assignment.py@@ -1,5 +1,9 @@"""真实字符快照的 Rust 归属与原 Python 规则逐字符差分。"""+from __future__ import annotations++from typing import TYPE_CHECKING+from copy import deepcopyfrom dataclasses import asdictfrom io import BytesIO@@ -14,8 +18,12 @@ from mineru.backend.analysis.pdf.text.models import _AnalyzeSpanfrom mineru.types import ContentType+if TYPE_CHECKING:+ from docvortex._native import NativeTextSnapshot++@pytest.fixture-def owned_text():+def owned_text() -> NativeTextSnapshot:"""生成包含边界标点、空白及多行的 PDF,关闭原文档后再验证归属。"""stream = BytesIO()canvas = Canvas(stream, pagesize=(320, 100))@@ -31,7 +39,9 @@ def owned_text():@pytest.mark.parametrize("seed", range(24))-def test_owned_assignments_match_python_for_overlapping_spans(owned_text, monkeypatch, seed):+def test_owned_assignments_match_python_for_overlapping_spans(+ owned_text: NativeTextSnapshot, monkeypatch: pytest.MonkeyPatch, seed: int+) -> None:"""在随机重叠框、负坐标与相同框中比对原始 char_idx 归属、文本及后置 OCR 判定。"""randomizer = random.Random(seed)boxes = [(0.0, 0.0, 100.0, 100.0), (0.0, 0.0, 100.0, 100.0)]@@ -41,7 +51,7 @@ def test_owned_assignments_match_python_for_overlapping_spans(owned_text, monkeyoriginal = native.chars_to_contentobserved = []- def capture(span, **kwargs):+ def capture(span: _AnalyzeSpan, **kwargs: object) -> None:"""在原内容组装删除临时字符前记录归属,不替换实际规则。"""observed.append((span.bbox, [char["char_idx"] for char in span.metadata["chars"]]))original(span, **kwargs)@@ -71,10 +81,12 @@ def test_owned_assignments_match_python_for_overlapping_spans(owned_text, monkeyassert results[0] == results[1]-def test_owned_assignment_keeps_replaced_rules_on_reference_path(owned_text, monkeypatch):+def test_owned_assignment_keeps_replaced_rules_on_reference_path(+ owned_text: NativeTextSnapshot, monkeypatch: pytest.MonkeyPatch+) -> None:"""调用方替换归属规则时不偷偷使用原生版本,保持参考计算能力。"""- def reject(*args):+ def reject(*args: object) -> bool:"""让所有字符失配,验证规则替换确实被执行。"""return False@@ -82,7 +94,7 @@ def test_owned_assignment_keeps_replaced_rules_on_reference_path(owned_text, monassert native._owned_span_assignments(owned_text, [_AnalyzeSpan(ContentType.TEXT, (0.0, 0.0, 100.0, 100.0))], 12.0) is None-def test_rotated_snapshot_explicitly_retains_reference_selection():+def test_rotated_snapshot_explicitly_retains_reference_selection() -> None:"""整行倾斜水印仍由原有组行规则裁决,不能把未迁移角度当成可忽略字符。"""stream = BytesIO()canvas = Canvas(stream, pagesize=(120, 120))@@ -95,3 +107,82 @@ def test_rotated_snapshot_explicitly_retains_reference_selection():pytest.skip("Native span assignment unavailable")assert not owner.supports_span_matching()assert owner.assign_spans([(0.0, 0.0, 120.0, 120.0)], 10.0, native.LINE_STOP_FLAG, native.LINE_START_FLAG, 0.33) is None++[email protected]("seed", range(24))+def test_owned_content_matches_reference(owned_text: NativeTextSnapshot, seed: int) -> None:+ """不替换内容函数,比对原生组装后的全部字段、空框原文及 OCR 待处理集合。"""+ from docvortex import _native++ randomizer = random.Random(seed)+ boxes = [(0.0, 0.0, 320.0, 100.0), (500.0, 500.0, 520.0, 520.0)]+ for _ in range(12):+ x, y = randomizer.uniform(-15, 100), randomizer.uniform(-10, 95)+ boxes.append((x, y, x + randomizer.uniform(2, 90), y + randomizer.uniform(2, 30)))+ results = []+ before = _native.text_snapshot_stats()[2]+ for owner in [None, owned_text]:+ geometry = owned_text.materialize_geometry()+ spans = [+ _AnalyzeSpan(+ ContentType.TEXT,+ box,+ content="原文",+ metadata={"chars": [], "height": box[3] - box[1], "width": box[2] - box[0]},+ )+ for box in boxes+ ]+ pending = native.fill_char_in_spans(+ spans,+ geometry.chars,+ 12.0,+ tight_bboxes=geometry.tight_bboxes,+ origins=geometry.origins,+ detect_scripts=False,+ _native_text=owner,+ )+ results.append(([asdict(span) for span in spans], [spans.index(span) for span in pending]))+ assert _native.text_snapshot_stats()[2] == before + 1+ assert results[0] == results[1]++[email protected]("text", ["e˛", "˛e", "fiflffffifflſtst", "\ue000\ue001 X", "a\u2003b\u00a0c", "A\x02B"])+def test_owned_unicode_content_matches_reference(text: str) -> None:+ """真实嵌入字体保留 Unicode 映射,比对附加符、连字、PUA 与控制字符处理。"""+ from pathlib import Path+ import reportlab+ from reportlab.pdfbase import pdfmetrics+ from reportlab.pdfbase.ttfonts import TTFont+ from docvortex import _native++ pdfmetrics.registerFont(TTFont("ContentVera", str(Path(reportlab.__file__).parent / "fonts/Vera.ttf")))+ stream = BytesIO()+ canvas = Canvas(stream, pagesize=(320, 100))+ canvas.setFont("ContentVera", 12)+ if text in {"e˛", "˛e"}:+ for character in text:+ canvas.drawString(10, 70, character)+ else:+ canvas.drawString(10, 70, text)+ canvas.save()+ with PDFDocument(stream.getvalue()) as document:+ owner = document[0].get_text_snapshot()+ if owner is None:+ pytest.skip("Native content unavailable")+ results = []+ before = _native.text_snapshot_stats()[2]+ for current in [None, owner]:+ geometry = owner.materialize_geometry()+ span = _AnalyzeSpan(ContentType.TEXT, (0.0, 0.0, 320.0, 100.0), metadata={"chars": [], "height": 100.0, "width": 320.0})+ pending = native.fill_char_in_spans(+ [span],+ geometry.chars,+ 12.0,+ tight_bboxes=geometry.tight_bboxes,+ origins=geometry.origins,+ detect_scripts=False,+ _native_text=current,+ )+ results.append((asdict(span), bool(pending)))+ assert _native.text_snapshot_stats()[2] == before + 1+ assert results[0] == results[1]