Skip to content

Python tool for converting files and office documents to Markdown.

perf(csv): Batch-remove blank rows to eliminate the quadratic overhead in table conversion

dickbownf0c5d01fbe90
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py
index 82fd2974..1aee0ffb 100644
--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py
@@ -30,17 +30,27 @@ def _escape_table_cell(value: str) -> str:
def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
"""Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""
- # Pop empty rows from the beginning
- while len(rows) > 0 and not rows[0]:
- rows.pop(0)
-
- # Pop empty rows after the header
- while len(rows) > 1 and not rows[1]:
- rows.pop(1)
-
- # Pop empty rows from the end
- while len(rows) > 0 and not rows[-1]:
- rows.pop(-1)
+ start = 0
+ while start < len(rows) and not rows[start]:
+ start += 1
+
+ if start == len(rows):
+ rows.clear()
+ return
+
+ header_index = start
+ start += 1
+ while start < len(rows) and not rows[start]:
+ start += 1
+
+ end = len(rows)
+ while end > start and not rows[end - 1]:
+ end -= 1
+
+ # Remove each blank run at once, rather than shifting the list per row.
+ del rows[end:]
+ del rows[header_index + 1 : start]
+ del rows[:header_index]
class CsvConverter(DocumentConverter):
diff --git a/packages/markitdown/tests/test_csv_blank_runs.py b/packages/markitdown/tests/test_csv_blank_runs.py
new file mode 100644
index 00000000..357af591
--- /dev/null
+++ b/packages/markitdown/tests/test_csv_blank_runs.py
@@ -0,0 +1,35 @@
+"""CSV conversion preserves table contents when trimming long blank runs."""
+
+import io
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+
+
[email protected]("position", ["leading", "after_header", "trailing", "all"])
+def test_csv_long_blank_runs(position: str) -> None:
+ blank = b"\n" * 100_000
+ header = b"name,value\n"
+ # An internal blank row and a wider data row must survive trimming.
+ data = b"Alice,1\n\nBob,2,extra\n"
+ content = {
+ "leading": blank + header + data,
+ "after_header": header + blank + data,
+ "trailing": header + data + blank,
+ "all": blank,
+ }[position]
+
+ result = MarkItDown(enable_plugins=False).convert_stream(
+ io.BytesIO(content),
+ stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+ )
+
+ expected = (
+ "| name | value | |\n"
+ "| --- | --- | --- |\n"
+ "| Alice | 1 | |\n"
+ "| | | |\n"
+ "| Bob | 2 | extra |"
+ )
+ assert result.markdown == ("" if position == "all" else expected)