Python tool for converting files and office documents to Markdown.
fix: escape pipes and newlines in CSV values (#2266)
Asjad Abbasaf56495dbbf5
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.pyindex 82691cd4..ee1a0527 100644--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py@@ -1,5 +1,6 @@import csvimport io+import refrom typing import BinaryIO, Anyfrom charset_normalizer import from_bytesfrom .._base_converter import DocumentConverter, DocumentConverterResult@@ -12,6 +13,21 @@ ACCEPTED_MIME_TYPE_PREFIXES = [ACCEPTED_FILE_EXTENSIONS = [".csv"]+# Matches a pipe together with the (possibly empty) run of backslashes in front+# of it, so that run can be doubled before the pipe is escaped.+_PIPE_ESCAPE_RE = re.compile(r"(\\*)\|")+++def _escape_table_cell(value: str) -> str:+ r"""Escape a CSV value so it is safe inside a Markdown table cell.++ A pipe is a column separator, so it must be escaped.+ Line breaks would end the row early, so they collapse to a single space.+ """+ value = _PIPE_ESCAPE_RE.sub(lambda m: m.group(1) * 2 + r"\|", value)+ return value.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")++def _trim_outer_blank_rows(rows: list[list[str]]) -> None:"""Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""# Pop empty rows from the beginning@@ -78,7 +94,8 @@ class CsvConverter(DocumentConverter):markdown_table = []# Add header row- markdown_table.append("| " + " | ".join(rows[0]) + " |")+ header = [_escape_table_cell(cell) for cell in rows[0]]+ markdown_table.append("| " + " | ".join(header) + " |")# Add separator rowmarkdown_table.append("| " + " | ".join(["---"] * len(rows[0])) + " |")@@ -90,7 +107,9 @@ class CsvConverter(DocumentConverter):row.append("")# Truncate if row has more columns than headerrow = row[: len(rows[0])]- markdown_table.append("| " + " | ".join(row) + " |")+ markdown_table.append(+ "| " + " | ".join(_escape_table_cell(cell) for cell in row) + " |"+ )result = "\n".join(markdown_table)diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex 6804c806..036da249 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -858,6 +858,89 @@ def test_csv_all_blank_input_returns_empty_markdown() -> None:assert result == ""+###############################################################################+# CSV converter: escaping values that would otherwise break the table+#+# The converter builds its table by joining cells with `" | "`. Two characters+# that are perfectly legal inside a CSV field corrupt that structure if they+# are passed through untouched:+#+# * a pipe is read as a column separator, so the row gains columns the header+# never declared -- a renderer discards the surplus and the data is lost;+# * a newline inside a quoted field ends the row early, leaving the remainder+# as stray text after the table.+#+# Escaping the pipe is itself subtle: `\|` only escapes when the run of+# backslashes in front of the pipe has odd length, so a value that already+# contains a backslash before the pipe has to be handled explicitly.+#+# Neither case raises, so both fail silently.+###############################################################################+++def test_csv_pipe_in_cell_is_escaped() -> None:+ result = _convert_csv(b'name,description\nWidget,"cheap | fast"\n')++ # The pipe must be escaped rather than emitted raw, or it splits the row.+ assert "| Widget | cheap \\| fast |" in result+++def test_csv_pipe_preceded_by_backslash_is_still_escaped() -> None:+ # `left\|right` must not become `left\\|right`: the doubled backslash is a+ # literal backslash, which would leave the pipe acting as a delimiter and+ # let a renderer drop `right`. The run of backslashes is doubled instead,+ # so the escaping `\|` survives.+ result = _convert_csv(b'name,description\nWidget,"left\\|right"\n')++ assert r"| Widget | left\\\|right |" in result+++def test_csv_pipe_preceded_by_two_backslashes_is_still_escaped() -> None:+ # An even-length run is just as dangerous once the naive `\|` is appended,+ # so check a longer run as well.+ result = _convert_csv(b'name,description\nWidget,"left\\\\|right"\n')++ assert r"| Widget | left\\\\\|right |" in result+++def test_csv_newline_in_quoted_cell_does_not_split_the_row() -> None:+ result = _convert_csv(b'name,notes\nWidget,"line one\nline two"\n')++ # The table must stay on one line per record; the embedded break collapses+ # to a space.+ assert len(result.splitlines()) == 3+ assert "| Widget | line one line two |" in result+++def test_csv_carriage_returns_in_quoted_cell_collapse_to_spaces() -> None:+ result = _convert_csv(b'name,notes\nWidget,"line one\r\nline two\rline three"\n')++ assert len(result.splitlines()) == 3+ assert "| Widget | line one line two line three |" in result+++def test_csv_pipe_in_header_is_escaped() -> None:+ result = _convert_csv(b'"a | b",c\n1,2\n')++ assert "| a \\| b | c |" in result+++def test_csv_plain_values_are_unchanged() -> None:+ # Guards against over-escaping ordinary content.+ result = _convert_csv(b"name,description\nWidget,cheap and fast\n")++ assert "| Widget | cheap and fast |" in result+ assert "\\" not in result+++def test_csv_backslash_without_a_pipe_is_left_alone() -> None:+ # Only backslashes that guard a pipe are doubled; a Windows path stays+ # readable.+ result = _convert_csv(b"name,path\nWidget,C:\\temp\\file.txt\n")++ assert r"| Widget | C:\temp\file.txt |" in result++if __name__ == "__main__":"""Runs this file's tests from the command line."""for test in [