Skip to content

Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.

fix: tighten Flash header and text evidence handling

myhlolib7fffdf94679
diff --git a/mineru/model/flash/pdf/pipeline.py b/mineru/model/flash/pdf/pipeline.py
index b73a81c9..7f1a2c5a 100644
--- a/mineru/model/flash/pdf/pipeline.py
+++ b/mineru/model/flash/pdf/pipeline.py
@@ -20,7 +20,7 @@ from .text_styles import (
detect_pdf_text_link_lines,
detect_pdf_text_style_lines,
materialize_pdf_inline_spans,
- _partition_resplit_text_evidence,
+ _realign_repaired_text_evidence,
)
from .models import (
@@ -28,6 +28,7 @@ from .models import (
_DocumentBodyProfile,
_DocumentTitleProfile,
_LineItem,
+ _MarginalCandidate,
_PageSource,
_PreparedPage,
)
@@ -65,6 +66,7 @@ from .line_merging import (
)
from .index_blocks import _extract_index_blocks
from .tables import (
+ _connected_horizontal_rule_bboxes,
_detect_table_candidates,
_materialize_table_blocks,
)
@@ -87,6 +89,7 @@ from .code_blocks import (
_build_rule_delimited_code_blocks,
)
from .auxiliary_text import (
+ _build_marginal_candidate,
_classify_deferred_image_footnotes,
_classify_isolated_first_page_footer,
_classify_page_footnote_trailing_footers,
@@ -99,6 +102,8 @@ from .auxiliary_text import (
_classify_repeated_page_marginals,
_classify_repeated_visual_headers,
_classify_single_page_compound_headers,
+ _marginal_geometry_matches,
+ _marginal_text_matches,
)
from .titles import (
_classify_body_height_section_titles,
@@ -207,7 +212,7 @@ def _table_detection_drawing_lines(
def _detect_repeated_header_separator_bboxes(
sources: list[_PageSource],
) -> list[set[BBox]]:
- """用至少三页重复的页首几何和上方刊头文本确认页眉分隔线。"""
+ """用重复刊头文本和非表格横线共同确认跨页页眉分隔线。"""
candidates_by_signature: dict[
tuple[float, float, float],
@@ -217,6 +222,7 @@ def _detect_repeated_header_separator_bboxes(
page_width, page_height = source.page_size
if page_width <= 0 or page_height <= 0:
continue
+ connected_table_rules = _connected_horizontal_rule_bboxes(source)
for drawing_line in source.drawing_lines:
bbox = drawing_line.bbox
center_y = _bbox_center_y(bbox)
@@ -225,6 +231,7 @@ def _detect_repeated_header_separator_bboxes(
or center_y > 0.15 * page_height
or bbox[2] - bbox[0] < 0.6 * page_width
or sum(line.angle == 0 and line.bbox[3] <= center_y for line in source.lines) < 2
+ or bbox in connected_table_rules
):
continue
signature = (
@@ -241,11 +248,51 @@ def _detect_repeated_header_separator_bboxes(
for members in candidates_by_signature.values():
if len({page_index for page_index, _bbox in members}) < 3:
continue
+ if len(_repeated_header_evidence_pages(sources, members)) < 2:
+ continue
for page_index, bbox in members:
output[page_index].add(bbox)
return output
+def _repeated_header_evidence_pages(
+ sources: list[_PageSource],
+ separator_members: list[tuple[int, BBox]],
+) -> set[int]:
+ """返回具有已分类或跨页重复刊头文本证据的页号集合。"""
+
+ supported_pages: set[int] = set()
+ candidates: list[tuple[int, _MarginalCandidate]] = []
+ for page_index, separator_bbox in separator_members:
+ center_y = _bbox_center_y(separator_bbox)
+ for line in sources[page_index].lines:
+ if line.angle != 0 or line.bbox[3] > center_y:
+ continue
+ if line.semantic_type == "header":
+ supported_pages.add(page_index)
+ continue
+ candidate = _build_marginal_candidate(
+ page_index,
+ line,
+ sources[page_index].page_size,
+ )
+ if candidate is not None and candidate.region == "header":
+ candidates.append((page_index, candidate))
+
+ for left_index, (left_page, left) in enumerate(candidates):
+ for right_page, right in candidates[left_index + 1 :]:
+ page_delta = right_page - left_page
+ if page_delta > 2:
+ break
+ if (
+ page_delta > 0
+ and _marginal_geometry_matches(left, right)
+ and _marginal_text_matches(left.line.text, right.line.text)
+ ):
+ supported_pages.update((left_page, right_page))
+ return supported_pages
+
+
def _analyze_native_document(
pdf_doc: PDFDocument,
*,
@@ -534,6 +581,7 @@ def _prepare_page_source(
+ 1
)
if geometry_plan is not None:
+ repaired_line_bboxes = {line.source_index: line.bbox for line in source.lines}
repaired_char_bboxes = {
char_idx: repair.layout_bbox
for (repair_page_index, char_idx), repair in geometry_plan.char_repairs.items()
@@ -545,16 +593,16 @@ def _prepare_page_source(
repaired_char_bboxes,
source_index_start=next_source_index,
)
- if resplits:
- partitioned_styles, partitioned_links = _partition_resplit_text_evidence(
- style_lines or [],
- link_lines or [],
- resplits,
- )
- if style_lines is not None:
- style_lines[:] = partitioned_styles
- if link_lines is not None:
- link_lines[:] = partitioned_links
+ aligned_styles, aligned_links = _realign_repaired_text_evidence(
+ style_lines or [],
+ link_lines or [],
+ repaired_line_bboxes,
+ resplits,
+ )
+ if style_lines is not None and aligned_styles is not style_lines:
+ style_lines[:] = aligned_styles
+ if link_lines is not None and aligned_links is not link_lines:
+ link_lines[:] = aligned_links
next_source_index = max(
next_source_index,
max(
@@ -728,18 +776,10 @@ def _apply_post_aggregation_tight_bboxes(
"_output_bbox_repaired",
False,
)
- if (
- output_bbox_repaired is True
- and isinstance(output_line_bboxes, list)
- and output_line_bboxes
- ):
- local_bboxes = [
- _coerce_bbox(value) for value in output_line_bboxes
- ]
+ if output_bbox_repaired is True and isinstance(output_line_bboxes, list) and output_line_bboxes:
+ local_bboxes = [_coerce_bbox(value) for value in output_line_bboxes]
if all(value is not None for value in local_bboxes):
- resolved_local_bboxes = [
- value for value in local_bboxes if value is not None
- ]
+ resolved_local_bboxes = [value for value in local_bboxes if value is not None]
block["_local_line_bboxes"] = resolved_local_bboxes
angle = int(block.get("angle", 0) or 0) % 360
candidate_bbox = _bbox_union_many(
@@ -883,11 +923,7 @@ def _finalize_prepared_page(
prepared.drawing_lines,
page_footnote_groups=prepared.page_footnote_groups,
page_index=page_index,
- visual_bboxes=[
- block["bbox"]
- for block in prepared.fixed_blocks
- if block.get("type") == "image"
- ],
+ visual_bboxes=[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],
)
text_blocks = _merge_multiline_title_blocks(text_blocks)
text_blocks = _merge_front_matter_column_blocks(
diff --git a/mineru/model/flash/pdf/tables.py b/mineru/model/flash/pdf/tables.py
index 0455901f..7d415aab 100644
--- a/mineru/model/flash/pdf/tables.py
+++ b/mineru/model/flash/pdf/tables.py
@@ -1026,6 +1026,31 @@ def _connected_rule_grid_components(
return output
+def _connected_horizontal_rule_bboxes(
+ source: _PageSource,
+) -> set[BBox]:
+ """返回参与常规横排闭合网格的原始水平线框,供上游避免删除真实表格边界。"""
+
+ angle_lines = [line for line in source.lines if line.angle == 0]
+ fragments = _build_fragments(angle_lines, source.page_size)
+ if not fragments:
+ return set()
+ local_axis_lines = _transform_axis_lines(
+ source.drawing_lines,
+ source.page_size,
+ 0,
+ )
+ return {
+ rule.original_bbox
+ for component in _connected_rule_grid_components(
+ local_axis_lines,
+ _median_fragment_height(fragments),
+ )
+ for rule in component
+ if rule.orientation == "horizontal"
+ }
+
+
def _rule_bands_share_grid_tracks(
top_rule: _LocalAxisLine,
bottom_rule: _LocalAxisLine,
diff --git a/mineru/model/flash/pdf/text_styles.py b/mineru/model/flash/pdf/text_styles.py
index 9dcbd332..700cbdeb 100644
--- a/mineru/model/flash/pdf/text_styles.py
+++ b/mineru/model/flash/pdf/text_styles.py
@@ -515,6 +515,49 @@ def _partition_resplit_text_evidence(
return partitioned_styles, partitioned_links
+def _realign_repaired_text_evidence(
+ style_lines: list[PDFTextStyleLine],
+ link_lines: list[PDFTextLinkLine],
+ line_bboxes: dict[int, BBox],
+ resplits: dict[int, _NativeVisualResplit],
+) -> tuple[list[PDFTextStyleLine], list[PDFTextLinkLine]]:
+ """同步未重切修复行的 evidence 框,再按字符身份切分发生重切的样式与链接。"""
+
+ aligned_styles = style_lines
+ for index, line in enumerate(style_lines):
+ bbox = line_bboxes.get(line.source_index)
+ if line.source_index in resplits or bbox is None or bbox == line.bbox:
+ continue
+ if aligned_styles is style_lines:
+ aligned_styles = list(style_lines)
+ aligned_styles[index] = PDFTextStyleLine(
+ bbox=bbox,
+ text=line.text,
+ style_ranges=line.style_ranges,
+ source_index=line.source_index,
+ )
+
+ aligned_links = link_lines
+ for index, line in enumerate(link_lines):
+ bbox = line_bboxes.get(line.source_index)
+ if line.source_index in resplits or bbox is None or bbox == line.bbox:
+ continue
+ if aligned_links is link_lines:
+ aligned_links = list(link_lines)
+ aligned_links[index] = PDFTextLinkLine(
+ bbox=bbox,
+ text=line.text,
+ link_ranges=line.link_ranges,
+ source_index=line.source_index,
+ )
+
+ return _partition_resplit_text_evidence(
+ aligned_styles,
+ aligned_links,
+ resplits,
+ )
+
+
def _canonical_styles(styles: Iterable[str]) -> tuple[PDFTextStyle, ...]:
"""按公开富文本协议顺序过滤、去重并规范样式集合。"""
diff --git a/tests/unittest/test_flash_pdf_pipeline.py b/tests/unittest/test_flash_pdf_pipeline.py
index cc3ddd0e..1210f899 100644
--- a/tests/unittest/test_flash_pdf_pipeline.py
+++ b/tests/unittest/test_flash_pdf_pipeline.py
@@ -329,6 +329,129 @@ def test_prepare_page_resplits_repaired_cross_column_row_before_text_classificat
} == {0: [5], 1: [6]}
+def test_prepare_page_realigns_unsplit_repaired_text_evidence() -> None:
+ """验证普通修复行未重切时,样式和链接 evidence 仍使用最终布局框。"""
+
+ line = _text_line("linked", (0.0, 80.0, 200.0, 100.0), 5)
+ source = models._PageSource(
+ page_size=(200.0, 200.0),
+ lines=[line],
+ chars=[],
+ drawing_lines=[],
+ )
+ repaired_bbox = (20.0, 80.0, 60.0, 100.0)
+ plan = char_geometry.DocumentGeometryPlan(
+ line_repairs={
+ (0, 5): char_geometry.LineGeometryRepair(
+ source_bbox=line.bbox,
+ layout_bbox=repaired_bbox,
+ ink_bbox=repaired_bbox,
+ baseline=100.0,
+ em_height=20.0,
+ state="repair_x",
+ )
+ }
+ )
+ style_lines = [
+ text_styles.PDFTextStyleLine(
+ bbox=line.bbox,
+ text=line.text,
+ style_ranges=(text_styles.PDFTextStyleRange(0, len(line.text), ("bold",)),),
+ source_index=line.source_index,
+ )
+ ]
+ link_lines = [
+ text_styles.PDFTextLinkLine(
+ bbox=line.bbox,
+ text=line.text,
+ link_ranges=(text_styles.PDFTextLinkRange(0, len(line.text), "https://example.test/repaired"),),
+ source_index=line.source_index,
+ )
+ ]
+
+ prepared = pipeline._prepare_page_source(
+ source,
+ geometry_plan=plan,
+ page_index=0,
+ style_lines=style_lines,
+ link_lines=link_lines,
+ )
+
+ assert [item.bbox for item in prepared.remaining_lines] == [repaired_bbox]
+ assert [item.bbox for item in style_lines] == [repaired_bbox]
+ assert [item.bbox for item in link_lines] == [repaired_bbox]
+ block = {"type": "text", "bbox": repaired_bbox, "content": line.text}
+ assert text_styles._assign_lines_to_blocks([block], style_lines, source.page_size)
+ assert text_styles._assign_lines_to_blocks([block], link_lines, source.page_size)
+
+
+def _repeated_separator_source(
+ header_text: str,
+ *,
+ connected_grid: bool = False,
+) -> models._PageSource:
+ """构造重复页首横线,并可把该横线作为闭合表格的真实顶边。"""
+
+ lines = [
+ _text_line(header_text, (20.0, 15.0, 180.0, 25.0), 0),
+ _text_line(f"{header_text} detail", (20.0, 30.0, 180.0, 40.0), 1),
+ ]
+ drawing_lines = [
+ models._AxisLine(
+ bbox=(10.0, 50.0, 190.0, 50.1),
+ width=0.1,
+ orientation="horizontal",
+ )
+ ]
+ if connected_grid:
+ lines.extend(
+ [
+ _text_line("left cell", (20.0, 65.0, 80.0, 75.0), 2),
+ _text_line("right cell", (110.0, 65.0, 170.0, 75.0), 3),
+ ]
+ )
+ drawing_lines.extend(
+ [
+ models._AxisLine((10.0, 90.0, 190.0, 90.1), 0.1, "horizontal"),
+ models._AxisLine((10.0, 50.0, 10.1, 90.0), 0.1, "vertical"),
+ models._AxisLine((100.0, 50.0, 100.1, 90.0), 0.1, "vertical"),
+ models._AxisLine((189.9, 50.0, 190.0, 90.0), 0.1, "vertical"),
+ ]
+ )
+ return models._PageSource(
+ page_size=(200.0, 400.0),
+ lines=lines,
+ chars=[],
+ drawing_lines=drawing_lines,
+ )
+
+
+def test_repeated_header_separator_requires_repeated_header_text() -> None:
+ """验证只有横线重复、而上方普通文本不重复时不会移除表格规则。"""
+
+ sources = [_repeated_separator_source(text) for text in ("alpha banner", "beta notice", "gamma heading")]
+
+ assert pipeline._detect_repeated_header_separator_bboxes(sources) == [set(), set(), set()]
+
+
+def test_repeated_header_separator_supports_alternating_headers() -> None:
+ """验证奇偶页各自重复的刊头可以共同确认同一页眉分隔线。"""
+
+ sources = [_repeated_separator_source("even journal" if page_index % 2 == 0 else "odd article") for page_index in range(4)]
+ expected = {(10.0, 50.0, 190.0, 50.1)}
+
+ assert pipeline._detect_repeated_header_separator_bboxes(sources) == [expected] * 4
+
+
+def test_repeated_table_top_rule_is_not_header_separator() -> None:
+ """验证重复表单的闭合网格顶边不会因上方重复标题而被删除。"""
+
+ sources = [_repeated_separator_source("repeated form", connected_grid=True) for _page_index in range(3)]
+
+ assert pipeline._detect_repeated_header_separator_bboxes(sources) == [set(), set(), set()]
+ assert [candidate.bbox for candidate in tables._detect_table_candidates(sources[0])] == [(10.0, 50.0, 190.0, 90.1)]
+
+
def test_table_detection_excludes_confirmed_masthead_separator() -> None:
"""验证页首通栏分隔线不与下方真表格边界组成巨型候选。"""
diff --git a/tests/unittest/test_pdf_text_styles.py b/tests/unittest/test_pdf_text_styles.py
index 8ade14c3..28ce79bb 100644
--- a/tests/unittest/test_pdf_text_styles.py
+++ b/tests/unittest/test_pdf_text_styles.py
@@ -46,6 +46,7 @@ from mineru.model.flash.pdf.text_styles import (
detect_pdf_text_style_lines,
materialize_pdf_inline_spans,
_partition_resplit_text_evidence,
+ _realign_repaired_text_evidence,
)
from _span_test_utils import inline_text, inline_urls
@@ -391,6 +392,94 @@ def test_resplit_text_evidence_preserves_identity_and_unsafe_mapping(
assert all("unsafe resplit mapping" in warning for warning in warnings)
+def test_repaired_text_evidence_realigns_only_changed_unsplit_lines() -> None:
+ """验证 X/Y 修复后的普通行更新 bbox,未修复 evidence 保持对象身份和顺序。"""
+
+ untouched_style = PDFTextStyleLine((10.0, 210.0, 50.0, 220.0), "plain", (), 0)
+ x_style = PDFTextStyleLine(
+ (0.0, 80.0, 200.0, 100.0),
+ "bold",
+ (PDFTextStyleRange(0, 4, ("bold",)),),
+ 1,
+ )
+ y_style = PDFTextStyleLine(
+ (20.0, 0.0, 60.0, 200.0),
+ "under",
+ (PDFTextStyleRange(0, 5, ("underline",)),),
+ 2,
+ )
+ untouched_link = PDFTextLinkLine(
+ (10.0, 210.0, 50.0, 220.0),
+ "plain",
+ (PDFTextLinkRange(0, 5, "https://example.test/plain"),),
+ 0,
+ )
+ x_link = PDFTextLinkLine(
+ x_style.bbox,
+ x_style.text,
+ (PDFTextLinkRange(0, 4, "https://example.test/x"),),
+ 1,
+ )
+ y_link = PDFTextLinkLine(
+ y_style.bbox,
+ y_style.text,
+ (PDFTextLinkRange(0, 5, "https://example.test/y"),),
+ 2,
+ )
+ style_lines = [untouched_style, x_style, y_style]
+ link_lines = [untouched_link, x_link, y_link]
+ repaired_bboxes = {
+ 0: untouched_style.bbox,
+ 1: (20.0, 80.0, 60.0, 100.0),
+ 2: (100.0, 80.0, 140.0, 100.0),
+ }
+
+ aligned_styles, aligned_links = _realign_repaired_text_evidence(
+ style_lines,
+ link_lines,
+ repaired_bboxes,
+ {},
+ )
+
+ assert aligned_styles[0] is untouched_style
+ assert aligned_links[0] is untouched_link
+ assert [line.source_index for line in aligned_styles] == [0, 1, 2]
+ assert [line.source_index for line in aligned_links] == [0, 1, 2]
+ assert [line.bbox for line in aligned_styles] == [untouched_style.bbox, repaired_bboxes[1], repaired_bboxes[2]]
+ assert [line.bbox for line in aligned_links] == [untouched_link.bbox, repaired_bboxes[1], repaired_bboxes[2]]
+ blocks = [
+ {"type": "text", "bbox": repaired_bboxes[1], "content": "bold"},
+ {"type": "text", "bbox": repaired_bboxes[2], "content": "under"},
+ ]
+ assert {
+ block_index: [line.source_index for line in lines]
+ for block_index, lines in flash_text_styles._assign_lines_to_blocks(
+ blocks,
+ aligned_styles[1:],
+ (200.0, 240.0),
+ ).items()
+ } == {0: [1], 1: [2]}
+ assert {
+ block_index: [line.source_index for line in lines]
+ for block_index, lines in flash_text_styles._assign_lines_to_blocks(
+ blocks,
+ aligned_links[1:],
+ (200.0, 240.0),
+ ).items()
+ } == {0: [1], 1: [2]}
+
+ original_untouched_styles = [untouched_style]
+ original_untouched_links = [untouched_link]
+ unchanged_styles, unchanged_links = _realign_repaired_text_evidence(
+ original_untouched_styles,
+ original_untouched_links,
+ {0: untouched_style.bbox},
+ {},
+ )
+ assert unchanged_styles is original_untouched_styles
+ assert unchanged_links is original_untouched_links
+
+
@pytest.mark.parametrize("angle", [0, 90, 180, 270])
def test_detect_and_apply_pdf_text_link_ranges_with_styles(angle: int) -> None:
"""验证标准方向链接映射、内部空格、URL 转义、样式顺序和幂等性。"""