Transforms complex documents like PDFs and Office docs into LLM-ready markdown/JSON for your Agentic workflows.
Python ai4sciencedocument-analysisdocxextract-datalayout-analysisocrparserpdfpdf-converterpdf-extractor-llmpdf-extractor-pretrainpdf-extractor-ragpdf-parserpptxpythonxlsx
fix: tighten Flash header and text evidence handling
myhlolib7fffdf94679
diff --git a/mineru/model/flash/pdf/pipeline.py b/mineru/model/flash/pdf/pipeline.pyindex b73a81c9..7f1a2c5a 100644--- a/mineru/model/flash/pdf/pipeline.py+++ b/mineru/model/flash/pdf/pipeline.py@@ -20,7 +20,7 @@ from .text_styles import (detect_pdf_text_link_lines,detect_pdf_text_style_lines,materialize_pdf_inline_spans,- _partition_resplit_text_evidence,+ _realign_repaired_text_evidence,)from .models import (@@ -28,6 +28,7 @@ from .models import (_DocumentBodyProfile,_DocumentTitleProfile,_LineItem,+ _MarginalCandidate,_PageSource,_PreparedPage,)@@ -65,6 +66,7 @@ from .line_merging import ()from .index_blocks import _extract_index_blocksfrom .tables import (+ _connected_horizontal_rule_bboxes,_detect_table_candidates,_materialize_table_blocks,)@@ -87,6 +89,7 @@ from .code_blocks import (_build_rule_delimited_code_blocks,)from .auxiliary_text import (+ _build_marginal_candidate,_classify_deferred_image_footnotes,_classify_isolated_first_page_footer,_classify_page_footnote_trailing_footers,@@ -99,6 +102,8 @@ from .auxiliary_text import (_classify_repeated_page_marginals,_classify_repeated_visual_headers,_classify_single_page_compound_headers,+ _marginal_geometry_matches,+ _marginal_text_matches,)from .titles import (_classify_body_height_section_titles,@@ -207,7 +212,7 @@ def _table_detection_drawing_lines(def _detect_repeated_header_separator_bboxes(sources: list[_PageSource],) -> list[set[BBox]]:- """用至少三页重复的页首几何和上方刊头文本确认页眉分隔线。"""+ """用重复刊头文本和非表格横线共同确认跨页页眉分隔线。"""candidates_by_signature: dict[tuple[float, float, float],@@ -217,6 +222,7 @@ def _detect_repeated_header_separator_bboxes(page_width, page_height = source.page_sizeif page_width <= 0 or page_height <= 0:continue+ connected_table_rules = _connected_horizontal_rule_bboxes(source)for drawing_line in source.drawing_lines:bbox = drawing_line.bboxcenter_y = _bbox_center_y(bbox)@@ -225,6 +231,7 @@ def _detect_repeated_header_separator_bboxes(or center_y > 0.15 * page_heightor bbox[2] - bbox[0] < 0.6 * page_widthor sum(line.angle == 0 and line.bbox[3] <= center_y for line in source.lines) < 2+ or bbox in connected_table_rules):continuesignature = (@@ -241,11 +248,51 @@ def _detect_repeated_header_separator_bboxes(for members in candidates_by_signature.values():if len({page_index for page_index, _bbox in members}) < 3:continue+ if len(_repeated_header_evidence_pages(sources, members)) < 2:+ continuefor page_index, bbox in members:output[page_index].add(bbox)return output+def _repeated_header_evidence_pages(+ sources: list[_PageSource],+ separator_members: list[tuple[int, BBox]],+) -> set[int]:+ """返回具有已分类或跨页重复刊头文本证据的页号集合。"""++ supported_pages: set[int] = set()+ candidates: list[tuple[int, _MarginalCandidate]] = []+ for page_index, separator_bbox in separator_members:+ center_y = _bbox_center_y(separator_bbox)+ for line in sources[page_index].lines:+ if line.angle != 0 or line.bbox[3] > center_y:+ continue+ if line.semantic_type == "header":+ supported_pages.add(page_index)+ continue+ candidate = _build_marginal_candidate(+ page_index,+ line,+ sources[page_index].page_size,+ )+ if candidate is not None and candidate.region == "header":+ candidates.append((page_index, candidate))++ for left_index, (left_page, left) in enumerate(candidates):+ for right_page, right in candidates[left_index + 1 :]:+ page_delta = right_page - left_page+ if page_delta > 2:+ break+ if (+ page_delta > 0+ and _marginal_geometry_matches(left, right)+ and _marginal_text_matches(left.line.text, right.line.text)+ ):+ supported_pages.update((left_page, right_page))+ return supported_pages++def _analyze_native_document(pdf_doc: PDFDocument,*,@@ -534,6 +581,7 @@ def _prepare_page_source(+ 1)if geometry_plan is not None:+ repaired_line_bboxes = {line.source_index: line.bbox for line in source.lines}repaired_char_bboxes = {char_idx: repair.layout_bboxfor (repair_page_index, char_idx), repair in geometry_plan.char_repairs.items()@@ -545,16 +593,16 @@ def _prepare_page_source(repaired_char_bboxes,source_index_start=next_source_index,)- if resplits:- partitioned_styles, partitioned_links = _partition_resplit_text_evidence(- style_lines or [],- link_lines or [],- resplits,- )- if style_lines is not None:- style_lines[:] = partitioned_styles- if link_lines is not None:- link_lines[:] = partitioned_links+ aligned_styles, aligned_links = _realign_repaired_text_evidence(+ style_lines or [],+ link_lines or [],+ repaired_line_bboxes,+ resplits,+ )+ if style_lines is not None and aligned_styles is not style_lines:+ style_lines[:] = aligned_styles+ if link_lines is not None and aligned_links is not link_lines:+ link_lines[:] = aligned_linksnext_source_index = max(next_source_index,max(@@ -728,18 +776,10 @@ def _apply_post_aggregation_tight_bboxes("_output_bbox_repaired",False,)- if (- output_bbox_repaired is True- and isinstance(output_line_bboxes, list)- and output_line_bboxes- ):- local_bboxes = [- _coerce_bbox(value) for value in output_line_bboxes- ]+ if output_bbox_repaired is True and isinstance(output_line_bboxes, list) and output_line_bboxes:+ local_bboxes = [_coerce_bbox(value) for value in output_line_bboxes]if all(value is not None for value in local_bboxes):- resolved_local_bboxes = [- value for value in local_bboxes if value is not None- ]+ resolved_local_bboxes = [value for value in local_bboxes if value is not None]block["_local_line_bboxes"] = resolved_local_bboxesangle = int(block.get("angle", 0) or 0) % 360candidate_bbox = _bbox_union_many(@@ -883,11 +923,7 @@ def _finalize_prepared_page(prepared.drawing_lines,page_footnote_groups=prepared.page_footnote_groups,page_index=page_index,- visual_bboxes=[- block["bbox"]- for block in prepared.fixed_blocks- if block.get("type") == "image"- ],+ visual_bboxes=[block["bbox"] for block in prepared.fixed_blocks if block.get("type") == "image"],)text_blocks = _merge_multiline_title_blocks(text_blocks)text_blocks = _merge_front_matter_column_blocks(diff --git a/mineru/model/flash/pdf/tables.py b/mineru/model/flash/pdf/tables.pyindex 0455901f..7d415aab 100644--- a/mineru/model/flash/pdf/tables.py+++ b/mineru/model/flash/pdf/tables.py@@ -1026,6 +1026,31 @@ def _connected_rule_grid_components(return output+def _connected_horizontal_rule_bboxes(+ source: _PageSource,+) -> set[BBox]:+ """返回参与常规横排闭合网格的原始水平线框,供上游避免删除真实表格边界。"""++ angle_lines = [line for line in source.lines if line.angle == 0]+ fragments = _build_fragments(angle_lines, source.page_size)+ if not fragments:+ return set()+ local_axis_lines = _transform_axis_lines(+ source.drawing_lines,+ source.page_size,+ 0,+ )+ return {+ rule.original_bbox+ for component in _connected_rule_grid_components(+ local_axis_lines,+ _median_fragment_height(fragments),+ )+ for rule in component+ if rule.orientation == "horizontal"+ }++def _rule_bands_share_grid_tracks(top_rule: _LocalAxisLine,bottom_rule: _LocalAxisLine,diff --git a/mineru/model/flash/pdf/text_styles.py b/mineru/model/flash/pdf/text_styles.pyindex 9dcbd332..700cbdeb 100644--- a/mineru/model/flash/pdf/text_styles.py+++ b/mineru/model/flash/pdf/text_styles.py@@ -515,6 +515,49 @@ def _partition_resplit_text_evidence(return partitioned_styles, partitioned_links+def _realign_repaired_text_evidence(+ style_lines: list[PDFTextStyleLine],+ link_lines: list[PDFTextLinkLine],+ line_bboxes: dict[int, BBox],+ resplits: dict[int, _NativeVisualResplit],+) -> tuple[list[PDFTextStyleLine], list[PDFTextLinkLine]]:+ """同步未重切修复行的 evidence 框,再按字符身份切分发生重切的样式与链接。"""++ aligned_styles = style_lines+ for index, line in enumerate(style_lines):+ bbox = line_bboxes.get(line.source_index)+ if line.source_index in resplits or bbox is None or bbox == line.bbox:+ continue+ if aligned_styles is style_lines:+ aligned_styles = list(style_lines)+ aligned_styles[index] = PDFTextStyleLine(+ bbox=bbox,+ text=line.text,+ style_ranges=line.style_ranges,+ source_index=line.source_index,+ )++ aligned_links = link_lines+ for index, line in enumerate(link_lines):+ bbox = line_bboxes.get(line.source_index)+ if line.source_index in resplits or bbox is None or bbox == line.bbox:+ continue+ if aligned_links is link_lines:+ aligned_links = list(link_lines)+ aligned_links[index] = PDFTextLinkLine(+ bbox=bbox,+ text=line.text,+ link_ranges=line.link_ranges,+ source_index=line.source_index,+ )++ return _partition_resplit_text_evidence(+ aligned_styles,+ aligned_links,+ resplits,+ )++def _canonical_styles(styles: Iterable[str]) -> tuple[PDFTextStyle, ...]:"""按公开富文本协议顺序过滤、去重并规范样式集合。"""diff --git a/tests/unittest/test_flash_pdf_pipeline.py b/tests/unittest/test_flash_pdf_pipeline.pyindex cc3ddd0e..1210f899 100644--- a/tests/unittest/test_flash_pdf_pipeline.py+++ b/tests/unittest/test_flash_pdf_pipeline.py@@ -329,6 +329,129 @@ def test_prepare_page_resplits_repaired_cross_column_row_before_text_classificat} == {0: [5], 1: [6]}+def test_prepare_page_realigns_unsplit_repaired_text_evidence() -> None:+ """验证普通修复行未重切时,样式和链接 evidence 仍使用最终布局框。"""++ line = _text_line("linked", (0.0, 80.0, 200.0, 100.0), 5)+ source = models._PageSource(+ page_size=(200.0, 200.0),+ lines=[line],+ chars=[],+ drawing_lines=[],+ )+ repaired_bbox = (20.0, 80.0, 60.0, 100.0)+ plan = char_geometry.DocumentGeometryPlan(+ line_repairs={+ (0, 5): char_geometry.LineGeometryRepair(+ source_bbox=line.bbox,+ layout_bbox=repaired_bbox,+ ink_bbox=repaired_bbox,+ baseline=100.0,+ em_height=20.0,+ state="repair_x",+ )+ }+ )+ style_lines = [+ text_styles.PDFTextStyleLine(+ bbox=line.bbox,+ text=line.text,+ style_ranges=(text_styles.PDFTextStyleRange(0, len(line.text), ("bold",)),),+ source_index=line.source_index,+ )+ ]+ link_lines = [+ text_styles.PDFTextLinkLine(+ bbox=line.bbox,+ text=line.text,+ link_ranges=(text_styles.PDFTextLinkRange(0, len(line.text), "https://example.test/repaired"),),+ source_index=line.source_index,+ )+ ]++ prepared = pipeline._prepare_page_source(+ source,+ geometry_plan=plan,+ page_index=0,+ style_lines=style_lines,+ link_lines=link_lines,+ )++ assert [item.bbox for item in prepared.remaining_lines] == [repaired_bbox]+ assert [item.bbox for item in style_lines] == [repaired_bbox]+ assert [item.bbox for item in link_lines] == [repaired_bbox]+ block = {"type": "text", "bbox": repaired_bbox, "content": line.text}+ assert text_styles._assign_lines_to_blocks([block], style_lines, source.page_size)+ assert text_styles._assign_lines_to_blocks([block], link_lines, source.page_size)+++def _repeated_separator_source(+ header_text: str,+ *,+ connected_grid: bool = False,+) -> models._PageSource:+ """构造重复页首横线,并可把该横线作为闭合表格的真实顶边。"""++ lines = [+ _text_line(header_text, (20.0, 15.0, 180.0, 25.0), 0),+ _text_line(f"{header_text} detail", (20.0, 30.0, 180.0, 40.0), 1),+ ]+ drawing_lines = [+ models._AxisLine(+ bbox=(10.0, 50.0, 190.0, 50.1),+ width=0.1,+ orientation="horizontal",+ )+ ]+ if connected_grid:+ lines.extend(+ [+ _text_line("left cell", (20.0, 65.0, 80.0, 75.0), 2),+ _text_line("right cell", (110.0, 65.0, 170.0, 75.0), 3),+ ]+ )+ drawing_lines.extend(+ [+ models._AxisLine((10.0, 90.0, 190.0, 90.1), 0.1, "horizontal"),+ models._AxisLine((10.0, 50.0, 10.1, 90.0), 0.1, "vertical"),+ models._AxisLine((100.0, 50.0, 100.1, 90.0), 0.1, "vertical"),+ models._AxisLine((189.9, 50.0, 190.0, 90.0), 0.1, "vertical"),+ ]+ )+ return models._PageSource(+ page_size=(200.0, 400.0),+ lines=lines,+ chars=[],+ drawing_lines=drawing_lines,+ )+++def test_repeated_header_separator_requires_repeated_header_text() -> None:+ """验证只有横线重复、而上方普通文本不重复时不会移除表格规则。"""++ sources = [_repeated_separator_source(text) for text in ("alpha banner", "beta notice", "gamma heading")]++ assert pipeline._detect_repeated_header_separator_bboxes(sources) == [set(), set(), set()]+++def test_repeated_header_separator_supports_alternating_headers() -> None:+ """验证奇偶页各自重复的刊头可以共同确认同一页眉分隔线。"""++ sources = [_repeated_separator_source("even journal" if page_index % 2 == 0 else "odd article") for page_index in range(4)]+ expected = {(10.0, 50.0, 190.0, 50.1)}++ assert pipeline._detect_repeated_header_separator_bboxes(sources) == [expected] * 4+++def test_repeated_table_top_rule_is_not_header_separator() -> None:+ """验证重复表单的闭合网格顶边不会因上方重复标题而被删除。"""++ sources = [_repeated_separator_source("repeated form", connected_grid=True) for _page_index in range(3)]++ assert pipeline._detect_repeated_header_separator_bboxes(sources) == [set(), set(), set()]+ assert [candidate.bbox for candidate in tables._detect_table_candidates(sources[0])] == [(10.0, 50.0, 190.0, 90.1)]++def test_table_detection_excludes_confirmed_masthead_separator() -> None:"""验证页首通栏分隔线不与下方真表格边界组成巨型候选。"""diff --git a/tests/unittest/test_pdf_text_styles.py b/tests/unittest/test_pdf_text_styles.pyindex 8ade14c3..28ce79bb 100644--- a/tests/unittest/test_pdf_text_styles.py+++ b/tests/unittest/test_pdf_text_styles.py@@ -46,6 +46,7 @@ from mineru.model.flash.pdf.text_styles import (detect_pdf_text_style_lines,materialize_pdf_inline_spans,_partition_resplit_text_evidence,+ _realign_repaired_text_evidence,)from _span_test_utils import inline_text, inline_urls@@ -391,6 +392,94 @@ def test_resplit_text_evidence_preserves_identity_and_unsafe_mapping(assert all("unsafe resplit mapping" in warning for warning in warnings)+def test_repaired_text_evidence_realigns_only_changed_unsplit_lines() -> None:+ """验证 X/Y 修复后的普通行更新 bbox,未修复 evidence 保持对象身份和顺序。"""++ untouched_style = PDFTextStyleLine((10.0, 210.0, 50.0, 220.0), "plain", (), 0)+ x_style = PDFTextStyleLine(+ (0.0, 80.0, 200.0, 100.0),+ "bold",+ (PDFTextStyleRange(0, 4, ("bold",)),),+ 1,+ )+ y_style = PDFTextStyleLine(+ (20.0, 0.0, 60.0, 200.0),+ "under",+ (PDFTextStyleRange(0, 5, ("underline",)),),+ 2,+ )+ untouched_link = PDFTextLinkLine(+ (10.0, 210.0, 50.0, 220.0),+ "plain",+ (PDFTextLinkRange(0, 5, "https://example.test/plain"),),+ 0,+ )+ x_link = PDFTextLinkLine(+ x_style.bbox,+ x_style.text,+ (PDFTextLinkRange(0, 4, "https://example.test/x"),),+ 1,+ )+ y_link = PDFTextLinkLine(+ y_style.bbox,+ y_style.text,+ (PDFTextLinkRange(0, 5, "https://example.test/y"),),+ 2,+ )+ style_lines = [untouched_style, x_style, y_style]+ link_lines = [untouched_link, x_link, y_link]+ repaired_bboxes = {+ 0: untouched_style.bbox,+ 1: (20.0, 80.0, 60.0, 100.0),+ 2: (100.0, 80.0, 140.0, 100.0),+ }++ aligned_styles, aligned_links = _realign_repaired_text_evidence(+ style_lines,+ link_lines,+ repaired_bboxes,+ {},+ )++ assert aligned_styles[0] is untouched_style+ assert aligned_links[0] is untouched_link+ assert [line.source_index for line in aligned_styles] == [0, 1, 2]+ assert [line.source_index for line in aligned_links] == [0, 1, 2]+ assert [line.bbox for line in aligned_styles] == [untouched_style.bbox, repaired_bboxes[1], repaired_bboxes[2]]+ assert [line.bbox for line in aligned_links] == [untouched_link.bbox, repaired_bboxes[1], repaired_bboxes[2]]+ blocks = [+ {"type": "text", "bbox": repaired_bboxes[1], "content": "bold"},+ {"type": "text", "bbox": repaired_bboxes[2], "content": "under"},+ ]+ assert {+ block_index: [line.source_index for line in lines]+ for block_index, lines in flash_text_styles._assign_lines_to_blocks(+ blocks,+ aligned_styles[1:],+ (200.0, 240.0),+ ).items()+ } == {0: [1], 1: [2]}+ assert {+ block_index: [line.source_index for line in lines]+ for block_index, lines in flash_text_styles._assign_lines_to_blocks(+ blocks,+ aligned_links[1:],+ (200.0, 240.0),+ ).items()+ } == {0: [1], 1: [2]}++ original_untouched_styles = [untouched_style]+ original_untouched_links = [untouched_link]+ unchanged_styles, unchanged_links = _realign_repaired_text_evidence(+ original_untouched_styles,+ original_untouched_links,+ {0: untouched_style.bbox},+ {},+ )+ assert unchanged_styles is original_untouched_styles+ assert unchanged_links is original_untouched_links++@pytest.mark.parametrize("angle", [0, 90, 180, 270])def test_detect_and_apply_pdf_text_link_ranges_with_styles(angle: int) -> None:"""验证标准方向链接映射、内部空格、URL 转义、样式顺序和幂等性。"""