Skip to content

Python tool for converting files and office documents to Markdown.

fix(csv): strip the UTF-8 BOM and skip blank rows before building the table (#2303)

Ruiming Zhaod04418ec9a50
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py
index 7e9631e1..82691cd4 100644
--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py
@@ -12,6 +12,21 @@ ACCEPTED_MIME_TYPE_PREFIXES = [
ACCEPTED_FILE_EXTENSIONS = [".csv"]
+def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
+ """Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""
+ # Pop empty rows from the beginning
+ while len(rows) > 0 and not rows[0]:
+ rows.pop(0)
+
+ # Pop empty rows after the header
+ while len(rows) > 1 and not rows[1]:
+ rows.pop(1)
+
+ # Pop empty rows from the end
+ while len(rows) > 0 and not rows[-1]:
+ rows.pop(-1)
+
+
class CsvConverter(DocumentConverter):
"""
Converts CSV files to Markdown tables.
@@ -47,9 +62,14 @@ class CsvConverter(DocumentConverter):
else:
content = str(from_bytes(file_stream.read()).best())
+ # Excel and other tools prepend a UTF-8 BOM to CSV exports; strip it so
+ # it does not end up inside the first header cell.
+ content = content.lstrip("\ufeff")
+
# Parse CSV content
reader = csv.reader(io.StringIO(content))
rows = list(reader)
+ _trim_outer_blank_rows(rows)
if not rows:
return DocumentConverterResult(markdown="")
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index a263801d..2cd6129b 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -748,6 +748,61 @@ def test_json_with_late_non_ascii_character(tmp_path) -> None:
assert "non-ASCII character: è" in result.text_content
+###############################################################################
+# CSV converter: BOM handling and blank-row handling
+###############################################################################
+
+
+def _convert_csv(data: bytes, charset: str | None = None) -> str:
+ stream_info = StreamInfo(extension=".csv", charset=charset)
+ return (
+ MarkItDown()
+ .convert_stream(io.BytesIO(data), stream_info=stream_info)
+ .text_content
+ )
+
+
+def test_csv_utf8_bom_is_stripped_from_the_header() -> None:
+ result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n")
+
+ assert "\ufeff" not in result
+ assert result.startswith("| name | age |")
+ assert "| Alice | 30 |" in result
+
+
+def test_csv_utf8_bom_is_stripped_when_charset_is_known() -> None:
+ result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n", charset="utf-8")
+
+ assert "\ufeff" not in result
+ assert result.startswith("| name | age |")
+
+
+def test_csv_leading_blank_line_does_not_destroy_the_table() -> None:
+ result = _convert_csv(b"\nname,age\nAlice,30\n")
+
+ assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"
+
+
+def test_csv_trailing_blank_lines_are_skipped() -> None:
+ result = _convert_csv(b"name,age\nAlice,30\n\n\n")
+
+ assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"
+
+
+def test_csv_blank_lines_between_rows_are_kept() -> None:
+ result = _convert_csv(b"name,age\n\nAlice,30\n\nBob,40\n")
+
+ assert (
+ result == "| name | age |\n| --- | --- |\n| Alice | 30 |\n| | |\n| Bob | 40 |"
+ )
+
+
+def test_csv_all_blank_input_returns_empty_markdown() -> None:
+ result = _convert_csv(b"\n\n\n")
+
+ assert result == ""
+
+
if __name__ == "__main__":
"""Runs this file's tests from the command line."""
for test in [