Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix(pdf): reuse shared CJK space handling for native text

myhloli78379ff635ff
diff --git a/docs/next/middle-json/current-medium.md b/docs/next/middle-json/current-medium.md
index e012e9ad..b7a874e9 100644
--- a/docs/next/middle-json/current-medium.md
+++ b/docs/next/middle-json/current-medium.md
@@ -101,8 +101,12 @@ TextSpan 中保持原文。renderer 在目标格式边界执行转义,不反
- `image/table/chart/code` 的 caption 与 footnote;
- `algorithm_body`。
-`continues_prev` 只属于顶层 `text/ref_text/list/table`。标题仍使用全局层级:
-`doc_title.level=1`,`paragraph_title.level=2..6`。
+`continues_prev` 只属于顶层 `text/ref_text/list/table`。
+
+顶层 `text.continues_prev=True` 还可表示前序 `sub_type="text"` 列表最后一个直属 `text` 成员的续文。
+续文保留原页、bbox、索引和顶层归属;列表子项不携带该字段。参考文献列表或嵌套末项不参与此类续接。
+
+标题仍使用全局层级:`doc_title.level=1`,`paragraph_title.level=2..6`。
## 保持专用字符串的 block
diff --git a/docs/next/middle-json/rendering.md b/docs/next/middle-json/rendering.md
index 62a84f80..5dab2078 100644
--- a/docs/next/middle-json/rendering.md
+++ b/docs/next/middle-json/rendering.md
@@ -485,6 +485,7 @@ Structured Content 与 Content List V1/V2 不使用展示型 planner。
- 隐藏 `header/footer/page_number/aside_text`,保留独立 `page_footnote`。
- 合并页内和跨页 `text/ref_text.continues_prev`;ref_text 可跨过页面脚注与辅助块查找前序 ref_text。
+- 正文续文也可接入前序正文列表的最后一个直属 text;连续片段保留行内公式、链接、样式和词界。
- 合并页内和跨页 `list.continues_prev`;ref list 可跨过页面脚注与辅助块查找前序列表。
- 只合并跨页 `table.continues_prev`。
- 页面之间不输出分割线。
@@ -494,6 +495,7 @@ Structured Content 与 Content List V1/V2 不使用展示型 planner。
- 展示全部顶层 block。
- 带 anchor 的 `page_footnote` 在 DEFAULT/FULL 中由 Markdown 输出 span id,HTML 输出元素 id。
- 只合并页内 `text/ref_text/list.continues_prev`。
+- 正文列表末项的续文遵循相同页边界:同页跨栏合并,跨页保留物理分段。
- 不合并跨页 text/ref_text/list/table。
- 每两个相邻 `PageInfo` 之间保留格式对应的页面边界:Markdown 输出分隔线,HTML 输出 page section 与分页线。
diff --git a/mineru/backend/analysis/pdf/text/native.py b/mineru/backend/analysis/pdf/text/native.py
index 2b293731..dbbd5cf3 100644
--- a/mineru/backend/analysis/pdf/text/native.py
+++ b/mineru/backend/analysis/pdf/text/native.py
@@ -12,7 +12,7 @@ from typing import Any, Callable, cast
import numpy as np
from docvortex.analyzers.pdf import PDF_NATIVE_SCRIPT_MARKUP_KEY, ScriptRole, classify_char_script_roles
-from docvortex.analyzers.pdf import join_tight_text, needs_tight_space
+from docvortex.analyzers.pdf import is_generated_cjk_space, join_tight_text, needs_tight_space
from docvortex.assets import calculate_contrast
from docvortex.document.pdf import Char, PDFPage, get_lines_from_chars
from docvortex.geometry import calculate_overlap_area_in_bbox1_area_ratio
@@ -566,6 +566,7 @@ def _owned_span_texts(owner: Any, spans: list[_AnalyzeSpan], median_height: floa
return None
functions = (
needs_tight_space,
+ is_generated_cjk_space,
chars_to_content,
__replace_unicode,
__replace_ligatures,
@@ -1096,6 +1097,13 @@ def chars_to_content(
for idx, char1 in enumerate(chars):
if char1["char"] in CONTROL_LINE_BREAK_CHARS:
continue
+ if (
+ span.metadata.get("_native_tight_spacing", True)
+ and 0 < idx < len(chars) - 1
+ and is_generated_cjk_space(chars[idx - 1], char1, chars[idx + 1], origins=origins)
+ ):
+ # 原始相邻项仍是空格,旧字距分支也不会重新插入被忽略的生成空格。
+ continue
char2 = chars[idx + 1] if idx + 1 < len(chars) else None
role1 = script_roles[idx]
role2 = script_roles[idx + 1] if char2 else None
@@ -1131,6 +1139,7 @@ def chars_to_content(
# 仅原有内置规则启用整段原生化,替换后的函数保留参考调用及异常语义。
_STANDARD_CONTENT_FUNCTIONS = (
needs_tight_space,
+ is_generated_cjk_space,
chars_to_content,
__replace_unicode,
__replace_ligatures,
diff --git a/tests/unittest/test_pdf_tight_spacing.py b/tests/unittest/test_pdf_tight_spacing.py
index 8df9c24c..c79eb8f1 100644
--- a/tests/unittest/test_pdf_tight_spacing.py
+++ b/tests/unittest/test_pdf_tight_spacing.py
@@ -69,27 +69,31 @@ def test_virtual_page_fill_keeps_actual_code_region() -> None:
assert span.metadata["_native_tight_spacing"] is False
-def test_vertical_line_fill_uses_tight_spacing(monkeypatch: pytest.MonkeyPatch) -> None:
- """强制命中竖向 line/span 回填分支,验证没有绕过共享词界规则。"""
[email protected]("text", ["AB", "中 文"])
[email protected]("spacing", [True, False])
+def test_vertical_line_fill_uses_tight_spacing(monkeypatch: pytest.MonkeyPatch, text: str, spacing: bool) -> None:
+ """旋转行回填共用英文词界及中文生成空格规则,代码仍保留原文。"""
import math
from unittest.mock import Mock
from PIL import Image
from mineru.types import BlockType
- chars = [_char("A", 0, 0), _char("B", 1, 8)]
- for char, y in zip(chars, [0, 8]):
+ chars = [_char(char, i, 0) for i, char in enumerate(text)]
+ positions = [0, 8] if text == "AB" else [0, 5, 10]
+ for char, y in zip(chars, positions):
char.update(
bbox=Bbox([40, y, 50, y + 10]),
- tight_bbox=(41, y, 49, y + 4),
+ tight_bbox=(41, y, 49, y + (4 if text == "AB" else 8)),
origin=(40, y),
rotation=math.pi / 2,
writing_angle=math.pi / 2,
+ is_generated=char["char"] == " ",
)
- line = {"rotation": math.pi / 2, "bbox": Bbox([40, 0, 50, 30]), "spans": [{"text": "AB", "chars": chars}]}
+ line = {"rotation": math.pi / 2, "bbox": Bbox([40, 0, 50, 30]), "spans": [{"text": text, "chars": chars}]}
monkeypatch.setattr(native, "get_lines_from_chars", Mock(return_value=[line]))
page = Mock()
- page.get_char_count.return_value = 2
- target = _AnalyzeSpan(ContentType.TEXT, (40, 0, 50, 30))
+ page.get_char_count.return_value = len(chars)
+ target = _AnalyzeSpan(ContentType.TEXT, (40, 0, 50, 30), metadata={"_native_tight_spacing": spacing})
spans = [target, _AnalyzeSpan(ContentType.TEXT, (60, 60, 80, 68)), _AnalyzeSpan(ContentType.TEXT, (60, 80, 80, 88))]
with Image.new("RGB", (100, 100), "white") as image:
native.txt_spans_extract(
@@ -102,4 +106,54 @@ def test_vertical_line_fill_uses_tight_spacing(monkeypatch: pytest.MonkeyPatch)
page_chars=chars,
detect_scripts=False,
)
- assert target.content == "A B"
+ assert target.content == (("A B" if text == "AB" else "中文") if spacing else text)
+
+
[email protected]("generated", [True, False, None])
[email protected]("spacing", [True, False])
[email protected]("scripts", [True, False])
+def test_generated_cjk_space_materialization(generated: bool | None, spacing: bool, scripts: bool) -> None:
+ """Python 回填只忽略已确认生成的中文空格,旧字距不会重加,代码及未知证据保留。"""
+ chars = [_char(char, i, i * 5) for i, char in enumerate("卫 星 InSAR 技 术")]
+ for char in chars:
+ char["tight_bbox"] = tuple(char["bbox"].bbox)
+ if generated is not None:
+ char["is_generated"] = generated and char["char"] == " "
+ span = _AnalyzeSpan(ContentType.TEXT, (0, 0, 250, 10), metadata={"chars": chars, "_native_tight_spacing": spacing})
+ native.chars_to_content(span, detect_scripts=scripts)
+ assert span.content == ("卫星 InSAR 技术" if generated is True and spacing else "卫 星 InSAR 技 术")
+ assert len(chars) == 13
+
+
[email protected]("spacing", [True, False])
+def test_generated_cjk_spaces_owned_snapshot_matches_python(spacing: bool) -> None:
+ """真实 PDF 关闭后 Rust span 内容物化仍与 Python 一致,同时保留真实中英空格。"""
+ from io import BytesIO
+ from reportlab.pdfgen.canvas import Canvas
+ from reportlab.pdfbase import pdfmetrics
+ from reportlab.pdfbase.cidfonts import UnicodeCIDFont
+ from docvortex.document.pdf import PDFDocument
+
+ pdfmetrics.registerFont(UnicodeCIDFont("STSong-Light"))
+ stream = BytesIO()
+ canvas = Canvas(stream, pagesize=(240, 100))
+ canvas.setFont("STSong-Light", 12)
+ for i, char in enumerate("卫星技术"):
+ canvas.drawString(10 + i * 15, 70, char)
+ canvas.drawString(10, 30, "中 文 InSAR 技术")
+ canvas.save()
+ with PDFDocument(stream.getvalue()) as document:
+ owner = document[0].get_text_snapshot()
+ geometry = document[0].get_chars_with_geometry()
+ if owner is None:
+ pytest.skip("Python backend")
+ spans = [_AnalyzeSpan(ContentType.TEXT, (0.0, 0.0, 240.0, 100.0), metadata={"chars": [], "_native_tight_spacing": spacing})]
+ records = native._owned_span_texts(owner, spans, 12.0, False)
+ assert records is not None
+ reference = _AnalyzeSpan(
+ ContentType.TEXT, spans[0].bbox, metadata={"chars": geometry.chars, "_native_tight_spacing": spacing}
+ )
+ native.chars_to_content(reference, detect_scripts=False)
+ assert records[0][0] == reference.content
+ assert "中 文 InSAR 技术" in reference.content
+ assert ("卫星技术" if spacing else "卫 星 技 术") in reference.content