Skip to content

Python tool for converting files and office documents to Markdown.

Improve efficiency of escaping pipes in CSVs (#2464)

afourney2277fff52d91
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.py
index 1aee0ffb..331b0712 100644
--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py
+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py
@@ -15,7 +15,8 @@ ACCEPTED_FILE_EXTENSIONS = [".csv"]
# Matches a pipe together with the (possibly empty) run of backslashes in front
# of it, so that run can be doubled before the pipe is escaped.
-_PIPE_ESCAPE_RE = re.compile(r"(\\*)\|")
+# The lookbehind avoids retrying from each position inside a backslash run.
+_PIPE_ESCAPE_RE = re.compile(r"(?<!\\)(\\*)\|")
def _escape_table_cell(value: str) -> str:
diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.py
index 84035c4b..bc031246 100644
--- a/packages/markitdown/tests/test_module_misc.py
+++ b/packages/markitdown/tests/test_module_misc.py
@@ -1800,6 +1800,20 @@ def test_csv_backslash_without_a_pipe_is_left_alone() -> None:
assert r"| Widget | C:\temp\file.txt |" in result
+ "suffix,escaped_suffix",
+ [("", ""), ("|", r"\|"), ("x|", r"x\|")],
+)
+def test_csv_long_backslash_runs(suffix: str, escaped_suffix: str) -> None:
+ backslashes = "\\" * 65_536
+ value = backslashes + suffix
+ expected = backslashes * (2 if suffix == "|" else 1) + escaped_suffix
+
+ result = _convert_csv(f"{value}\n{value}\n".encode("utf-8"), charset="utf-8")
+
+ assert result == f"| {expected} |\n| --- |\n| {expected} |"
+
+
# ---------------------------------------------------------------------------
# Regression test for issue #1960:
# exiftool_path pointing to a nonexistent binary used to leak a raw