Skip to content

Python tool for converting files and office documents to Markdown.

Fix CSV parsing for CR-only line endings (#2412)

Kim Do Yeonab6792caca7f
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py
index 649c9e28..f028cfb2 100644
--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py
@@ -83,7 +83,7 @@ class CsvConverter(DocumentConverter):
content = content.lstrip("\ufeff")
# Parse CSV content
- reader = csv.reader(io.StringIO(content))
+ reader = csv.reader(io.StringIO(content, newline=""))
rows = list(reader)
_trim_outer_blank_rows(rows)
diff --git a/packages/markitdown/tests/test_csv_line_endings.py b/packages/markitdown/tests/test_csv_line_endings.py
new file mode 100644
index 00000000..71d73b54
--- /dev/null
+++ b/packages/markitdown/tests/test_csv_line_endings.py
@@ -0,0 +1,53 @@
+"""Regression tests for CR-only CSV record separators (issue #2411)."""
+
+import io
+
+import pytest
+
+from markitdown import MarkItDown, StreamInfo
+
+
[email protected](scope="module")
+def converter() -> MarkItDown:
+ return MarkItDown(enable_plugins=False)
+
+
[email protected]("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"])
[email protected]("final_newline", [False, True])
+def test_csv_record_separators_produce_a_table(
+ converter: MarkItDown, newline: str, final_newline: bool
+) -> None:
+ content = newline.join(["name,age", "Alice,30"])
+ if final_newline:
+ content += newline
+
+ result = converter.convert_stream(
+ io.BytesIO(content.encode("utf-8")),
+ stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+ )
+
+ assert result.markdown == "| name | age |\n| --- | --- |\n| Alice | 30 |"
+
+
[email protected]("newline", ["\n", "\r\n", "\r"], ids=["LF", "CRLF", "CR"])
+ "cell_newline", ["\n", "\r\n", "\r"], ids=["cell-LF", "cell-CRLF", "cell-CR"]
+)
+def test_csv_quoted_line_break_stays_inside_its_cell(
+ converter: MarkItDown, newline: str, cell_newline: str
+) -> None:
+ content = newline.join(
+ ["name,notes", f'Alice,"first{cell_newline}second"', "Bob,plain"]
+ )
+
+ result = converter.convert_stream(
+ io.BytesIO(content.encode("utf-8")),
+ stream_info=StreamInfo(extension=".csv", charset="utf-8"),
+ )
+
+ assert result.markdown == (
+ "| name | notes |\n"
+ "| --- | --- |\n"
+ "| Alice | first second |\n"
+ "| Bob | plain |"
+ )