Python tool for converting files and office documents to Markdown.
fix(csv): strip the UTF-8 BOM and skip blank rows before building the table (#2303)
Ruiming Zhaod04418ec9a50
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.pyindex 7e9631e1..82691cd4 100644--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py@@ -12,6 +12,21 @@ ACCEPTED_MIME_TYPE_PREFIXES = [ACCEPTED_FILE_EXTENSIONS = [".csv"]+def _trim_outer_blank_rows(rows: list[list[str]]) -> None:+ """Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""+ # Pop empty rows from the beginning+ while len(rows) > 0 and not rows[0]:+ rows.pop(0)++ # Pop empty rows after the header+ while len(rows) > 1 and not rows[1]:+ rows.pop(1)++ # Pop empty rows from the end+ while len(rows) > 0 and not rows[-1]:+ rows.pop(-1)++class CsvConverter(DocumentConverter):"""Converts CSV files to Markdown tables.@@ -47,9 +62,14 @@ class CsvConverter(DocumentConverter):else:content = str(from_bytes(file_stream.read()).best())+ # Excel and other tools prepend a UTF-8 BOM to CSV exports; strip it so+ # it does not end up inside the first header cell.+ content = content.lstrip("\ufeff")+# Parse CSV contentreader = csv.reader(io.StringIO(content))rows = list(reader)+ _trim_outer_blank_rows(rows)if not rows:return DocumentConverterResult(markdown="")diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex a263801d..2cd6129b 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -748,6 +748,61 @@ def test_json_with_late_non_ascii_character(tmp_path) -> None:assert "non-ASCII character: è" in result.text_content+###############################################################################+# CSV converter: BOM handling and blank-row handling+###############################################################################+++def _convert_csv(data: bytes, charset: str | None = None) -> str:+ stream_info = StreamInfo(extension=".csv", charset=charset)+ return (+ MarkItDown()+ .convert_stream(io.BytesIO(data), stream_info=stream_info)+ .text_content+ )+++def test_csv_utf8_bom_is_stripped_from_the_header() -> None:+ result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n")++ assert "\ufeff" not in result+ assert result.startswith("| name | age |")+ assert "| Alice | 30 |" in result+++def test_csv_utf8_bom_is_stripped_when_charset_is_known() -> None:+ result = _convert_csv(b"\xef\xbb\xbfname,age\nAlice,30\n", charset="utf-8")++ assert "\ufeff" not in result+ assert result.startswith("| name | age |")+++def test_csv_leading_blank_line_does_not_destroy_the_table() -> None:+ result = _convert_csv(b"\nname,age\nAlice,30\n")++ assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"+++def test_csv_trailing_blank_lines_are_skipped() -> None:+ result = _convert_csv(b"name,age\nAlice,30\n\n\n")++ assert result == "| name | age |\n| --- | --- |\n| Alice | 30 |"+++def test_csv_blank_lines_between_rows_are_kept() -> None:+ result = _convert_csv(b"name,age\n\nAlice,30\n\nBob,40\n")++ assert (+ result == "| name | age |\n| --- | --- |\n| Alice | 30 |\n| | |\n| Bob | 40 |"+ )+++def test_csv_all_blank_input_returns_empty_markdown() -> None:+ result = _convert_csv(b"\n\n\n")++ assert result == ""++if __name__ == "__main__":"""Runs this file's tests from the command line."""for test in [