Python tool for converting files and office documents to Markdown.
perf(csv): Batch-remove blank rows to eliminate the quadratic overhead in table conversion
dickbownf0c5d01fbe90
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.pyindex 82fd2974..1aee0ffb 100644--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py@@ -30,17 +30,27 @@ def _escape_table_cell(value: str) -> str:def _trim_outer_blank_rows(rows: list[list[str]]) -> None:"""Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""- # Pop empty rows from the beginning- while len(rows) > 0 and not rows[0]:- rows.pop(0)-- # Pop empty rows after the header- while len(rows) > 1 and not rows[1]:- rows.pop(1)-- # Pop empty rows from the end- while len(rows) > 0 and not rows[-1]:- rows.pop(-1)+ start = 0+ while start < len(rows) and not rows[start]:+ start += 1++ if start == len(rows):+ rows.clear()+ return++ header_index = start+ start += 1+ while start < len(rows) and not rows[start]:+ start += 1++ end = len(rows)+ while end > start and not rows[end - 1]:+ end -= 1++ # Remove each blank run at once, rather than shifting the list per row.+ del rows[end:]+ del rows[header_index + 1 : start]+ del rows[:header_index]class CsvConverter(DocumentConverter):diff --git a/packages/markitdown/tests/test_csv_blank_runs.py b/packages/markitdown/tests/test_csv_blank_runs.pynew file mode 100644index 00000000..357af591--- /dev/null+++ b/packages/markitdown/tests/test_csv_blank_runs.py@@ -0,0 +1,35 @@+"""CSV conversion preserves table contents when trimming long blank runs."""++import io++import pytest++from markitdown import MarkItDown, StreamInfo++[email protected]("position", ["leading", "after_header", "trailing", "all"])+def test_csv_long_blank_runs(position: str) -> None:+ blank = b"\n" * 100_000+ header = b"name,value\n"+ # An internal blank row and a wider data row must survive trimming.+ data = b"Alice,1\n\nBob,2,extra\n"+ content = {+ "leading": blank + header + data,+ "after_header": header + blank + data,+ "trailing": header + data + blank,+ "all": blank,+ }[position]++ result = MarkItDown(enable_plugins=False).convert_stream(+ io.BytesIO(content),+ stream_info=StreamInfo(extension=".csv", charset="utf-8"),+ )++ expected = (+ "| name | value | |\n"+ "| --- | --- | --- |\n"+ "| Alice | 1 | |\n"+ "| | | |\n"+ "| Bob | 2 | extra |"+ )+ assert result.markdown == ("" if position == "all" else expected)