Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix(pdf): reuse shared CJK space handling for native text
myhloli78379ff635ff
diff --git a/docs/next/middle-json/current-medium.md b/docs/next/middle-json/current-medium.mdindex e012e9ad..b7a874e9 100644--- a/docs/next/middle-json/current-medium.md+++ b/docs/next/middle-json/current-medium.md@@ -101,8 +101,12 @@ TextSpan 中保持原文。renderer 在目标格式边界执行转义,不反- `image/table/chart/code` 的 caption 与 footnote;- `algorithm_body`。-`continues_prev` 只属于顶层 `text/ref_text/list/table`。标题仍使用全局层级:-`doc_title.level=1`,`paragraph_title.level=2..6`。+`continues_prev` 只属于顶层 `text/ref_text/list/table`。++顶层 `text.continues_prev=True` 还可表示前序 `sub_type="text"` 列表最后一个直属 `text` 成员的续文。+续文保留原页、bbox、索引和顶层归属;列表子项不携带该字段。参考文献列表或嵌套末项不参与此类续接。++标题仍使用全局层级:`doc_title.level=1`,`paragraph_title.level=2..6`。## 保持专用字符串的 blockdiff --git a/docs/next/middle-json/rendering.md b/docs/next/middle-json/rendering.mdindex 62a84f80..5dab2078 100644--- a/docs/next/middle-json/rendering.md+++ b/docs/next/middle-json/rendering.md@@ -485,6 +485,7 @@ Structured Content 与 Content List V1/V2 不使用展示型 planner。- 隐藏 `header/footer/page_number/aside_text`,保留独立 `page_footnote`。- 合并页内和跨页 `text/ref_text.continues_prev`;ref_text 可跨过页面脚注与辅助块查找前序 ref_text。+- 正文续文也可接入前序正文列表的最后一个直属 text;连续片段保留行内公式、链接、样式和词界。- 合并页内和跨页 `list.continues_prev`;ref list 可跨过页面脚注与辅助块查找前序列表。- 只合并跨页 `table.continues_prev`。- 页面之间不输出分割线。@@ -494,6 +495,7 @@ Structured Content 与 Content List V1/V2 不使用展示型 planner。- 展示全部顶层 block。- 带 anchor 的 `page_footnote` 在 DEFAULT/FULL 中由 Markdown 输出 span id,HTML 输出元素 id。- 只合并页内 `text/ref_text/list.continues_prev`。+- 正文列表末项的续文遵循相同页边界:同页跨栏合并,跨页保留物理分段。- 不合并跨页 text/ref_text/list/table。- 每两个相邻 `PageInfo` 之间保留格式对应的页面边界:Markdown 输出分隔线,HTML 输出 page section 与分页线。diff --git a/mineru/backend/analysis/pdf/text/native.py b/mineru/backend/analysis/pdf/text/native.pyindex 2b293731..dbbd5cf3 100644--- a/mineru/backend/analysis/pdf/text/native.py+++ b/mineru/backend/analysis/pdf/text/native.py@@ -12,7 +12,7 @@ from typing import Any, Callable, castimport numpy as npfrom docvortex.analyzers.pdf import PDF_NATIVE_SCRIPT_MARKUP_KEY, ScriptRole, classify_char_script_roles-from docvortex.analyzers.pdf import join_tight_text, needs_tight_space+from docvortex.analyzers.pdf import is_generated_cjk_space, join_tight_text, needs_tight_spacefrom docvortex.assets import calculate_contrastfrom docvortex.document.pdf import Char, PDFPage, get_lines_from_charsfrom docvortex.geometry import calculate_overlap_area_in_bbox1_area_ratio@@ -566,6 +566,7 @@ def _owned_span_texts(owner: Any, spans: list[_AnalyzeSpan], median_height: floareturn Nonefunctions = (needs_tight_space,+ is_generated_cjk_space,chars_to_content,__replace_unicode,__replace_ligatures,@@ -1096,6 +1097,13 @@ def chars_to_content(for idx, char1 in enumerate(chars):if char1["char"] in CONTROL_LINE_BREAK_CHARS:continue+ if (+ span.metadata.get("_native_tight_spacing", True)+ and 0 < idx < len(chars) - 1+ and is_generated_cjk_space(chars[idx - 1], char1, chars[idx + 1], origins=origins)+ ):+ # 原始相邻项仍是空格,旧字距分支也不会重新插入被忽略的生成空格。+ continuechar2 = chars[idx + 1] if idx + 1 < len(chars) else Nonerole1 = script_roles[idx]role2 = script_roles[idx + 1] if char2 else None@@ -1131,6 +1139,7 @@ def chars_to_content(# 仅原有内置规则启用整段原生化,替换后的函数保留参考调用及异常语义。_STANDARD_CONTENT_FUNCTIONS = (needs_tight_space,+ is_generated_cjk_space,chars_to_content,__replace_unicode,__replace_ligatures,diff --git a/tests/unittest/test_pdf_tight_spacing.py b/tests/unittest/test_pdf_tight_spacing.pyindex 8df9c24c..c79eb8f1 100644--- a/tests/unittest/test_pdf_tight_spacing.py+++ b/tests/unittest/test_pdf_tight_spacing.py@@ -69,27 +69,31 @@ def test_virtual_page_fill_keeps_actual_code_region() -> None:assert span.metadata["_native_tight_spacing"] is False-def test_vertical_line_fill_uses_tight_spacing(monkeypatch: pytest.MonkeyPatch) -> None:- """强制命中竖向 line/span 回填分支,验证没有绕过共享词界规则。"""[email protected]("text", ["AB", "中 文"])[email protected]("spacing", [True, False])+def test_vertical_line_fill_uses_tight_spacing(monkeypatch: pytest.MonkeyPatch, text: str, spacing: bool) -> None:+ """旋转行回填共用英文词界及中文生成空格规则,代码仍保留原文。"""import mathfrom unittest.mock import Mockfrom PIL import Imagefrom mineru.types import BlockType- chars = [_char("A", 0, 0), _char("B", 1, 8)]- for char, y in zip(chars, [0, 8]):+ chars = [_char(char, i, 0) for i, char in enumerate(text)]+ positions = [0, 8] if text == "AB" else [0, 5, 10]+ for char, y in zip(chars, positions):char.update(bbox=Bbox([40, y, 50, y + 10]),- tight_bbox=(41, y, 49, y + 4),+ tight_bbox=(41, y, 49, y + (4 if text == "AB" else 8)),origin=(40, y),rotation=math.pi / 2,writing_angle=math.pi / 2,+ is_generated=char["char"] == " ",)- line = {"rotation": math.pi / 2, "bbox": Bbox([40, 0, 50, 30]), "spans": [{"text": "AB", "chars": chars}]}+ line = {"rotation": math.pi / 2, "bbox": Bbox([40, 0, 50, 30]), "spans": [{"text": text, "chars": chars}]}monkeypatch.setattr(native, "get_lines_from_chars", Mock(return_value=[line]))page = Mock()- page.get_char_count.return_value = 2- target = _AnalyzeSpan(ContentType.TEXT, (40, 0, 50, 30))+ page.get_char_count.return_value = len(chars)+ target = _AnalyzeSpan(ContentType.TEXT, (40, 0, 50, 30), metadata={"_native_tight_spacing": spacing})spans = [target, _AnalyzeSpan(ContentType.TEXT, (60, 60, 80, 68)), _AnalyzeSpan(ContentType.TEXT, (60, 80, 80, 88))]with Image.new("RGB", (100, 100), "white") as image:native.txt_spans_extract(@@ -102,4 +106,54 @@ def test_vertical_line_fill_uses_tight_spacing(monkeypatch: pytest.MonkeyPatch)page_chars=chars,detect_scripts=False,)- assert target.content == "A B"+ assert target.content == (("A B" if text == "AB" else "中文") if spacing else text)++[email protected]("generated", [True, False, None])[email protected]("spacing", [True, False])[email protected]("scripts", [True, False])+def test_generated_cjk_space_materialization(generated: bool | None, spacing: bool, scripts: bool) -> None:+ """Python 回填只忽略已确认生成的中文空格,旧字距不会重加,代码及未知证据保留。"""+ chars = [_char(char, i, i * 5) for i, char in enumerate("卫 星 InSAR 技 术")]+ for char in chars:+ char["tight_bbox"] = tuple(char["bbox"].bbox)+ if generated is not None:+ char["is_generated"] = generated and char["char"] == " "+ span = _AnalyzeSpan(ContentType.TEXT, (0, 0, 250, 10), metadata={"chars": chars, "_native_tight_spacing": spacing})+ native.chars_to_content(span, detect_scripts=scripts)+ assert span.content == ("卫星 InSAR 技术" if generated is True and spacing else "卫 星 InSAR 技 术")+ assert len(chars) == 13++[email protected]("spacing", [True, False])+def test_generated_cjk_spaces_owned_snapshot_matches_python(spacing: bool) -> None:+ """真实 PDF 关闭后 Rust span 内容物化仍与 Python 一致,同时保留真实中英空格。"""+ from io import BytesIO+ from reportlab.pdfgen.canvas import Canvas+ from reportlab.pdfbase import pdfmetrics+ from reportlab.pdfbase.cidfonts import UnicodeCIDFont+ from docvortex.document.pdf import PDFDocument++ pdfmetrics.registerFont(UnicodeCIDFont("STSong-Light"))+ stream = BytesIO()+ canvas = Canvas(stream, pagesize=(240, 100))+ canvas.setFont("STSong-Light", 12)+ for i, char in enumerate("卫星技术"):+ canvas.drawString(10 + i * 15, 70, char)+ canvas.drawString(10, 30, "中 文 InSAR 技术")+ canvas.save()+ with PDFDocument(stream.getvalue()) as document:+ owner = document[0].get_text_snapshot()+ geometry = document[0].get_chars_with_geometry()+ if owner is None:+ pytest.skip("Python backend")+ spans = [_AnalyzeSpan(ContentType.TEXT, (0.0, 0.0, 240.0, 100.0), metadata={"chars": [], "_native_tight_spacing": spacing})]+ records = native._owned_span_texts(owner, spans, 12.0, False)+ assert records is not None+ reference = _AnalyzeSpan(+ ContentType.TEXT, spans[0].bbox, metadata={"chars": geometry.chars, "_native_tight_spacing": spacing}+ )+ native.chars_to_content(reference, detect_scripts=False)+ assert records[0][0] == reference.content+ assert "中 文 InSAR 技术" in reference.content+ assert ("卫星技术" if spacing else "卫 星 技 术") in reference.content