Python tool for converting files and office documents to Markdown.
Improve efficiency of escaping pipes in CSVs (#2464)
afourney2277fff52d91
diff --git a/packages/markitdown/src/markitdown/converters/_csv_converter.py b/packages/markitdown/src/markitdown/converters/_csv_converter.pyindex 1aee0ffb..331b0712 100644--- a/packages/markitdown/src/markitdown/converters/_csv_converter.py+++ b/packages/markitdown/src/markitdown/converters/_csv_converter.py@@ -15,7 +15,8 @@ ACCEPTED_FILE_EXTENSIONS = [".csv"]# Matches a pipe together with the (possibly empty) run of backslashes in front# of it, so that run can be doubled before the pipe is escaped.-_PIPE_ESCAPE_RE = re.compile(r"(\\*)\|")+# The lookbehind avoids retrying from each position inside a backslash run.+_PIPE_ESCAPE_RE = re.compile(r"(?<!\\)(\\*)\|")def _escape_table_cell(value: str) -> str:diff --git a/packages/markitdown/tests/test_module_misc.py b/packages/markitdown/tests/test_module_misc.pyindex 84035c4b..bc031246 100644--- a/packages/markitdown/tests/test_module_misc.py+++ b/packages/markitdown/tests/test_module_misc.py@@ -1800,6 +1800,20 @@ def test_csv_backslash_without_a_pipe_is_left_alone() -> None:assert r"| Widget | C:\temp\file.txt |" in result+ "suffix,escaped_suffix",+ [("", ""), ("|", r"\|"), ("x|", r"x\|")],+)+def test_csv_long_backslash_runs(suffix: str, escaped_suffix: str) -> None:+ backslashes = "\\" * 65_536+ value = backslashes + suffix+ expected = backslashes * (2 if suffix == "|" else 1) + escaped_suffix++ result = _convert_csv(f"{value}\n{value}\n".encode("utf-8"), charset="utf-8")++ assert result == f"| {expected} |\n| --- |\n| {expected} |"++# ---------------------------------------------------------------------------# Regression test for issue #1960:# exiftool_path pointing to a nonexistent binary used to leak a raw