Python tool for converting files and office documents to Markdown.
Fix: PDF parsing doesn't support partially numbered lists (#1525)
lesyk7fdaefb724d0
diff --git a/packages/markitdown/src/markitdown/__about__.py b/packages/markitdown/src/markitdown/__about__.pyindex 4c8b68f6..659cf6ba 100644--- a/packages/markitdown/src/markitdown/__about__.py+++ b/packages/markitdown/src/markitdown/__about__.py@@ -1,4 +1,4 @@# SPDX-FileCopyrightText: 2024-present Adam Fourney <[email protected]>## SPDX-License-Identifier: MIT-__version__ = "0.1.4"+__version__ = "0.1.5b1"diff --git a/packages/markitdown/src/markitdown/converters/_pdf_converter.py b/packages/markitdown/src/markitdown/converters/_pdf_converter.pyindex b692f169..76d17a67 100644--- a/packages/markitdown/src/markitdown/converters/_pdf_converter.py+++ b/packages/markitdown/src/markitdown/converters/_pdf_converter.py@@ -1,11 +1,62 @@import sysimport io+import refrom typing import BinaryIO, Anyfrom .._base_converter import DocumentConverter, DocumentConverterResultfrom .._stream_info import StreamInfofrom .._exceptions import MissingDependencyException, MISSING_DEPENDENCY_MESSAGE+# Pattern for MasterFormat-style partial numbering (e.g., ".1", ".2", ".10")+PARTIAL_NUMBERING_PATTERN = re.compile(r"^\.\d+$")+++def _merge_partial_numbering_lines(text: str) -> str:+ """+ Post-process extracted text to merge MasterFormat-style partial numbering+ with the following text line.++ MasterFormat documents use partial numbering like:+ .1 The intent of this Request for Proposal...+ .2 Available information relative to...++ Some PDF extractors split these into separate lines:+ .1+ The intent of this Request for Proposal...++ This function merges them back together.+ """+ lines = text.split("\n")+ result_lines: list[str] = []+ i = 0++ while i < len(lines):+ line = lines[i]+ stripped = line.strip()++ # Check if this line is ONLY a partial numbering+ if PARTIAL_NUMBERING_PATTERN.match(stripped):+ # Look for the next non-empty line to merge with+ j = i + 1+ while j < len(lines) and not lines[j].strip():+ j += 1++ if j < len(lines):+ # Merge the partial numbering with the next line+ next_line = lines[j].strip()+ result_lines.append(f"{stripped} {next_line}")+ i = j + 1 # Skip past the merged line+ else:+ # No next line to merge with, keep as is+ result_lines.append(line)+ i += 1+ else:+ result_lines.append(line)+ i += 1++ return "\n".join(result_lines)++# Load dependencies_dependency_exc_info = Nonetry:@@ -117,6 +168,14 @@ def _extract_form_content_from_words(page: Any) -> str | None:# Determine row typeis_paragraph = line_width > page_width * 0.55 and len(combined_text) > 60+ # Check for MasterFormat-style partial numbering (e.g., ".1", ".2")+ # These should be treated as list items, not table rows+ has_partial_numbering = False+ if row_words:+ first_word = row_words[0]["text"].strip()+ if PARTIAL_NUMBERING_PATTERN.match(first_word):+ has_partial_numbering = True+row_info.append({"y_key": y_key,@@ -125,6 +184,7 @@ def _extract_form_content_from_words(page: Any) -> str | None:"x_groups": x_groups,"is_paragraph": is_paragraph,"num_columns": len(x_groups),+ "has_partial_numbering": has_partial_numbering,})@@ -156,6 +216,11 @@ def _extract_form_content_from_words(page: Any) -> str | None:info["is_table_row"] = Falsecontinue+ # Rows with partial numbering (e.g., ".1", ".2") are list items, not table rows+ if info["has_partial_numbering"]:+ info["is_table_row"] = False+ continue+# Count how many global columns this row's words align withaligned_columns: set[int] = set()for word in info["words"]:@@ -469,4 +534,7 @@ class PdfConverter(DocumentConverter):pdf_bytes.seek(0)markdown = pdfminer.high_level.extract_text(pdf_bytes)+ # Post-process to merge MasterFormat-style partial numbering with following text+ markdown = _merge_partial_numbering_lines(markdown)+return DocumentConverterResult(markdown=markdown)diff --git a/packages/markitdown/tests/test_files/masterformat_partial_numbering.pdf b/packages/markitdown/tests/test_files/masterformat_partial_numbering.pdfnew file mode 100644index 00000000..246639a8--- /dev/null+++ b/packages/markitdown/tests/test_files/masterformat_partial_numbering.pdf@@ -0,0 +1,74 @@+%PDF-1.3+%���� ReportLab Generated PDF document http://www.reportlab.com+1 0 obj+<<+/F1 2 0 R /F2 3 0 R+>>+endobj+2 0 obj+<<+/BaseFont /Helvetica /Encoding /WinAnsiEncoding /Name /F1 /Subtype /Type1 /Type /Font+>>+endobj+3 0 obj+<<+/BaseFont /Helvetica-Bold /Encoding /WinAnsiEncoding /Name /F2 /Subtype /Type1 /Type /Font+>>+endobj+4 0 obj+<<+/Contents 8 0 R /MediaBox [ 0 0 612 792 ] /Parent 7 0 R /Resources <<+/Font 1 0 R /ProcSet [ /PDF /Text /ImageB /ImageC /ImageI ]+>> /Rotate 0 /Trans <<++>>+ /Type /Page+>>+endobj+5 0 obj+<<+/PageMode /UseNone /Pages 7 0 R /Type /Catalog+>>+endobj+6 0 obj+<<+/Author (anonymous) /CreationDate (D:20260108192537+01'00') /Creator (ReportLab PDF Library - www.reportlab.com) /Keywords () /ModDate (D:20260108192537+01'00') /Producer (ReportLab PDF Library - www.reportlab.com)+ /Subject (unspecified) /Title (untitled) /Trapped /False+>>+endobj+7 0 obj+<<+/Count 1 /Kids [ 4 0 R ] /Type /Pages+>>+endobj+8 0 obj+<<+/Filter [ /ASCII85Decode /FlateDecode ] /Length 670+>>+stream+Gat$td;IYl'Rf-pcJpsZ/27V[H_WEoW#\5sVS2I3Jt]?;R+`$Ms*f.>6<=3APUNhTmQL<9F,pFup'KGk=TR,7^>/u!#kAE+l;?UQ8Fg(+-O>;^54HWJ*kXdl'VdsI]Y^$-G(GWPR)iGMeWbg3)F'+jfWpCb"rU?d?8?q_r!E2N'0sM)J>=XD.jgunBuga\Wi4MX$WV/b)1F@bC8Nj8(0*)"ZK06BSqlu1$[^37A;/aK=mfgqg$&i),2OH&%^\"B1%B\dd_V>$5OtPri4rcEe3LoBUeL6QAPnpQr+R-t0f]ZSYc?BTAKQ?A&+J#J*N*=6;'?@Cp*>auj0",hDS3bH4[hVs3O="&bk&U@>+8c1&c2iDg6R*%q%iEZq'-!FNSB8#C*'po69R8$S(:.=-$N6'!_[1/jV<$@V3Z_"gd!g!MJMT)mTUN4cWjUQQj]HT_m]0*R=YgTmcl@k>*b/SBce9?.m,bEi#?PI:=r_6G.auM&FtP,>O7T%Z<$f#=g6(2+d@;8?"$8cdI38ZZ>hq5b2_pQY:M\.Kod,pl)ZX7a7Gc'Mf_'SB1X3*L[-51a8`h4)KjJQjLfm/3TIeQY?2+?^.r^HNafjHp<5,1M=W'N>8sb=dB#FC5M`7L91"BC@CfEckPe`M5O:#!Fj$K]s(Gs8rW$>H7gK~>endstream+endobj+xref+0 9+0000000000 65535 f+0000000073 00000 n+0000000114 00000 n+0000000221 00000 n+0000000333 00000 n+0000000526 00000 n+0000000594 00000 n+0000000890 00000 n+0000000949 00000 n+trailer+<<+/ID+[<5467fcd5093f18002be6af3fb13ce6c3><5467fcd5093f18002be6af3fb13ce6c3>]+% ReportLab generated PDF document -- digest (http://www.reportlab.com)++/Info 6 0 R+/Root 5 0 R+/Size 9+>>+startxref+1709+%%EOFdiff --git a/packages/markitdown/tests/test_pdf_masterformat.py b/packages/markitdown/tests/test_pdf_masterformat.pynew file mode 100644index 00000000..8d3eb073--- /dev/null+++ b/packages/markitdown/tests/test_pdf_masterformat.py@@ -0,0 +1,171 @@+#!/usr/bin/env python3 -m pytest+"""Tests for MasterFormat-style partial numbering in PDF conversion."""++import os+import re+import pytest++from markitdown import MarkItDown+from markitdown.converters._pdf_converter import PARTIAL_NUMBERING_PATTERN++TEST_FILES_DIR = os.path.join(os.path.dirname(__file__), "test_files")+++class TestMasterFormatPartialNumbering:+ """Test handling of MasterFormat-style partial numbering (.1, .2, etc.)."""++ def test_partial_numbering_pattern_regex(self):+ """Test that the partial numbering regex pattern correctly matches."""++ # Should match partial numbering patterns+ assert PARTIAL_NUMBERING_PATTERN.match(".1") is not None+ assert PARTIAL_NUMBERING_PATTERN.match(".2") is not None+ assert PARTIAL_NUMBERING_PATTERN.match(".10") is not None+ assert PARTIAL_NUMBERING_PATTERN.match(".99") is not None++ # Should NOT match other patterns+ assert PARTIAL_NUMBERING_PATTERN.match("1.") is None+ assert PARTIAL_NUMBERING_PATTERN.match("1.2") is None+ assert PARTIAL_NUMBERING_PATTERN.match(".1.2") is None+ assert PARTIAL_NUMBERING_PATTERN.match("text") is None+ assert PARTIAL_NUMBERING_PATTERN.match(".a") is None+ assert PARTIAL_NUMBERING_PATTERN.match("") is None++ def test_masterformat_partial_numbering_not_split(self):+ """Test that MasterFormat partial numbering stays with associated text.++ MasterFormat documents use partial numbering like:+ .1 The intent of this Request for Proposal...+ .2 Available information relative to...++ These should NOT be split into separate table columns, but kept+ as coherent text lines with the number followed by its description.+ """+ pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")++ markitdown = MarkItDown()+ result = markitdown.convert(pdf_path)+ text_content = result.text_content++ # Partial numberings should NOT appear isolated on their own lines+ # If they're isolated, it means the parser incorrectly split them from their text+ lines = text_content.split("\n")+ isolated_numberings = []+ for line in lines:+ stripped = line.strip()+ # Check if line contains ONLY a partial numbering (with possible whitespace/pipes)+ cleaned = stripped.replace("|", "").strip()+ if cleaned in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8", ".9", ".10"]:+ isolated_numberings.append(stripped)++ assert len(isolated_numberings) == 0, (+ f"Partial numberings should not be isolated from their text. "+ f"Found isolated: {isolated_numberings}"+ )++ # Verify that partial numberings appear WITH following text on the same line+ # Look for patterns like ".1 The intent" or ".1 Some text"+ partial_with_text = re.findall(r"\.\d+\s+\w+", text_content)+ assert (+ len(partial_with_text) > 0+ ), "Expected to find partial numberings followed by text on the same line"++ def test_masterformat_content_preserved(self):+ """Test that MasterFormat document content is fully preserved."""+ pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")++ markitdown = MarkItDown()+ result = markitdown.convert(pdf_path)+ text_content = result.text_content++ # Verify key content from the MasterFormat document is preserved+ expected_content = [+ "RFP for Construction Management Services",+ "Section 00 00 43",+ "Instructions to Respondents",+ "Ken Sargent House",+ "INTENT",+ "Request for Proposal",+ "KEN SARGENT HOUSE",+ "GRANDE PRAIRIE, ALBERTA",+ "Section 00 00 45",+ ]++ for content in expected_content:+ assert (+ content in text_content+ ), f"Expected content '{content}' not found in extracted text"++ # Verify partial numbering is followed by text on the same line+ # .1 should be followed by "The intent" on the same line+ assert re.search(+ r"\.1\s+The intent", text_content+ ), "Partial numbering .1 should be followed by 'The intent' text"++ # .2 should be followed by "Available information" on the same line+ assert re.search(+ r"\.2\s+Available information", text_content+ ), "Partial numbering .2 should be followed by 'Available information' text"++ # Ensure text content is not empty and has reasonable length+ assert (+ len(text_content.strip()) > 100+ ), "MasterFormat document should have substantial text content"++ def test_merge_partial_numbering_with_empty_lines_between(self):+ """Test that partial numberings merge correctly even with empty lines between.++ When PDF extractors produce output like:+ .1++ The intent of this Request...++ The merge logic should still combine them properly.+ """+ pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")++ markitdown = MarkItDown()+ result = markitdown.convert(pdf_path)+ text_content = result.text_content++ # The merged result should have .1 and .2 followed by text+ # Check that we don't have patterns like ".1\n\nThe intent" (unmerged)+ lines = text_content.split("\n")++ for i, line in enumerate(lines):+ stripped = line.strip()+ # If we find an isolated partial numbering, the merge failed+ if stripped in [".1", ".2", ".3", ".4", ".5", ".6", ".7", ".8"]:+ # Check if next non-empty line exists and wasn't merged+ for j in range(i + 1, min(i + 3, len(lines))):+ if lines[j].strip():+ pytest.fail(+ f"Partial numbering '{stripped}' on line {i} was not "+ f"merged with following text '{lines[j].strip()[:30]}...'"+ )+ break++ def test_multiple_partial_numberings_all_merged(self):+ """Test that all partial numberings in a document are properly merged."""+ pdf_path = os.path.join(TEST_FILES_DIR, "masterformat_partial_numbering.pdf")++ markitdown = MarkItDown()+ result = markitdown.convert(pdf_path)+ text_content = result.text_content++ # Count occurrences of merged partial numberings (number followed by text)+ merged_count = len(re.findall(r"\.\d+\s+[A-Za-z]", text_content))++ # Count isolated partial numberings (number alone on a line)+ isolated_count = 0+ for line in text_content.split("\n"):+ stripped = line.strip()+ if re.match(r"^\.\d+$", stripped):+ isolated_count += 1++ assert (+ merged_count >= 2+ ), f"Expected at least 2 merged partial numberings, found {merged_count}"+ assert (+ isolated_count == 0+ ), f"Found {isolated_count} isolated partial numberings that weren't merged"