Skip to content

Python tool for converting files and office documents to Markdown.

fix: escape pipes and newlines in CSV values (#2266)

Asjad Abbasaf56495dbbf5
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py
index 82691cd4..ee1a0527 100644
--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py
@@ -1,5 +1,6 @@
import csv
import io
+import re
from typing import BinaryIO, Any
from charset_normalizer import from_bytes
from .._base_converter import DocumentConverter, DocumentConverterResult
@@ -12,6 +13,21 @@ ACCEPTED_MIME_TYPE_PREFIXES = [
ACCEPTED_FILE_EXTENSIONS = [".csv"]
+# Matches a pipe together with the (possibly empty) run of backslashes in front
+# of it, so that run can be doubled before the pipe is escaped.
+_PIPE_ESCAPE_RE = re.compile(r"(\\*)\|")
+
+
+def _escape_table_cell(value: str) -> str:
+ r"""Escape a CSV value so it is safe inside a Markdown table cell.
+
+ A pipe is a column separator, so it must be escaped.
+ Line breaks would end the row early, so they collapse to a single space.
+ """
+ value = _PIPE_ESCAPE_RE.sub(lambda m: m.group(1) * 2 + r"\|", value)
+ return value.replace("\r\n", " ").replace("\n", " ").replace("\r", " ")
+
+
def _trim_outer_blank_rows(rows: list[list[str]]) -> None:
"""Remove empty rows from the beginning and end, and immediately after the header. This operation is performed in-place."""
# Pop empty rows from the beginning
@@ -78,7 +94,8 @@ class CsvConverter(DocumentConverter):
markdown_table = []
# Add header row
- markdown_table.append("| " + " | ".join(rows[0]) + " |")
+ header = [_escape_table_cell(cell) for cell in rows[0]]
+ markdown_table.append("| " + " | ".join(header) + " |")
# Add separator row
markdown_table.append("| " + " | ".join(["---"] * len(rows[0])) + " |")
@@ -90,7 +107,9 @@ class CsvConverter(DocumentConverter):
row.append("")
# Truncate if row has more columns than header
row = row[: len(rows[0])]
- markdown_table.append("| " + " | ".join(row) + " |")
+ markdown_table.append(
+ "| " + " | ".join(_escape_table_cell(cell) for cell in row) + " |"
+ )
result = "\n".join(markdown_table)
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index 6804c806..036da249 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -858,6 +858,89 @@ def test_csv_all_blank_input_returns_empty_markdown() -> None:
assert result == ""
+###############################################################################
+# CSV converter: escaping values that would otherwise break the table
+#
+# The converter builds its table by joining cells with `" | "`. Two characters
+# that are perfectly legal inside a CSV field corrupt that structure if they
+# are passed through untouched:
+#
+# * a pipe is read as a column separator, so the row gains columns the header
+# never declared -- a renderer discards the surplus and the data is lost;
+# * a newline inside a quoted field ends the row early, leaving the remainder
+# as stray text after the table.
+#
+# Escaping the pipe is itself subtle: `\|` only escapes when the run of
+# backslashes in front of the pipe has odd length, so a value that already
+# contains a backslash before the pipe has to be handled explicitly.
+#
+# Neither case raises, so both fail silently.
+###############################################################################
+
+
+def test_csv_pipe_in_cell_is_escaped() -> None:
+ result = _convert_csv(b'name,description\nWidget,"cheap | fast"\n')
+
+ # The pipe must be escaped rather than emitted raw, or it splits the row.
+ assert "| Widget | cheap \\| fast |" in result
+
+
+def test_csv_pipe_preceded_by_backslash_is_still_escaped() -> None:
+ # `left\|right` must not become `left\\|right`: the doubled backslash is a
+ # literal backslash, which would leave the pipe acting as a delimiter and
+ # let a renderer drop `right`. The run of backslashes is doubled instead,
+ # so the escaping `\|` survives.
+ result = _convert_csv(b'name,description\nWidget,"left\\|right"\n')
+
+ assert r"| Widget | left\\\|right |" in result
+
+
+def test_csv_pipe_preceded_by_two_backslashes_is_still_escaped() -> None:
+ # An even-length run is just as dangerous once the naive `\|` is appended,
+ # so check a longer run as well.
+ result = _convert_csv(b'name,description\nWidget,"left\\\\|right"\n')
+
+ assert r"| Widget | left\\\\\|right |" in result
+
+
+def test_csv_newline_in_quoted_cell_does_not_split_the_row() -> None:
+ result = _convert_csv(b'name,notes\nWidget,"line one\nline two"\n')
+
+ # The table must stay on one line per record; the embedded break collapses
+ # to a space.
+ assert len(result.splitlines()) == 3
+ assert "| Widget | line one line two |" in result
+
+
+def test_csv_carriage_returns_in_quoted_cell_collapse_to_spaces() -> None:
+ result = _convert_csv(b'name,notes\nWidget,"line one\r\nline two\rline three"\n')
+
+ assert len(result.splitlines()) == 3
+ assert "| Widget | line one line two line three |" in result
+
+
+def test_csv_pipe_in_header_is_escaped() -> None:
+ result = _convert_csv(b'"a | b",c\n1,2\n')
+
+ assert "| a \\| b | c |" in result
+
+
+def test_csv_plain_values_are_unchanged() -> None:
+ # Guards against over-escaping ordinary content.
+ result = _convert_csv(b"name,description\nWidget,cheap and fast\n")
+
+ assert "| Widget | cheap and fast |" in result
+ assert "\\" not in result
+
+
+def test_csv_backslash_without_a_pipe_is_left_alone() -> None:
+ # Only backslashes that guard a pipe are doubled; a Windows path stays
+ # readable.
+ result = _convert_csv(b"name,path\nWidget,C:\\temp\\file.txt\n")
+
+ assert r"| Widget | C:\temp\file.txt |" in result
+
+
if __name__ == "__main__":
"""Runs this file's tests from the command line."""
for test in [