Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
markdown: Fix escaping non-ASCII chars (#55782)
alkinunc0596fade730
diff --git a/crates/markdown/src/markdown.rs b/crates/markdown/src/markdown.rsindex 69feee416da..44e87f55677 100644--- a/crates/markdown/src/markdown.rs+++ b/crates/markdown/src/markdown.rs@@ -401,12 +401,12 @@ enum EscapeAction {}impl EscapeAction {- fn output_len(&self) -> usize {+ fn output_len(&self, c: char) -> usize {match self {- Self::PassThrough => 1,+ Self::PassThrough => c.len_utf8(),Self::Nbsp(count) => count * '\u{00A0}'.len_utf8(),Self::DoubleNewline => 2,- Self::PrefixBackslash => 2,+ Self::PrefixBackslash => '\\'.len_utf8() + c.len_utf8(),}}@@ -431,8 +431,6 @@ impl EscapeAction {}}-// Valid to operate on raw bytes since multi-byte UTF-8-// sequences never contain ASCII-range bytes.struct MarkdownEscaper {in_leading_whitespace: bool,}@@ -446,21 +444,21 @@ impl MarkdownEscaper {}}- fn next(&mut self, byte: u8) -> EscapeAction {- let action = if self.in_leading_whitespace && byte == b'\t' {+ fn next(&mut self, c: char) -> EscapeAction {+ let action = if self.in_leading_whitespace && c == '\t' {EscapeAction::Nbsp(Self::TAB_SIZE)- } else if self.in_leading_whitespace && byte == b' ' {+ } else if self.in_leading_whitespace && c == ' ' {EscapeAction::Nbsp(1)- } else if byte == b'\n' {+ } else if c == '\n' {EscapeAction::DoubleNewline- } else if byte.is_ascii_punctuation() {+ } else if c.is_ascii_punctuation() {EscapeAction::PrefixBackslash} else {EscapeAction::PassThrough};self.in_leading_whitespace =- byte == b'\n' || (self.in_leading_whitespace && (byte == b' ' || byte == b'\t'));+ c == '\n' || (self.in_leading_whitespace && (c == ' ' || c == '\t'));action}}@@ -675,7 +673,7 @@ impl Markdown {pub fn escape(s: &str) -> Cow<'_, str> {let output_len: usize = {let mut escaper = MarkdownEscaper::new();- s.bytes().map(|byte| escaper.next(byte).output_len()).sum()+ s.chars().map(|c| escaper.next(c).output_len(c)).sum()};if output_len == s.len() {@@ -685,7 +683,7 @@ impl Markdown {let mut escaper = MarkdownEscaper::new();let mut output = String::with_capacity(output_len);for c in s.chars() {- escaper.next(c as u8).write_to(c, &mut output);+ escaper.next(c).write_to(c, &mut output);}output.into()}@@ -3931,6 +3929,30 @@ mod tests {);}+ #[test]+ fn test_escape_non_ascii() {+ // Cyrillic characters should not have backslashes added before them,+ // but ASCII punctuation should still be escaped.+ assert_eq!(Markdown::escape("Привет, мир"), r"Привет\, мир");+ // Test with markdown special characters mixed in+ assert_eq!(Markdown::escape("Привет, *мир*"), r"Привет\, \*мир\*");+ // Test with the exact example from the issue (single quotes are also ASCII punctuation)+ assert_eq!(+ Markdown::escape("Отсутствует пробел справа от ','"),+ r"Отсутствует пробел справа от \'\,\'"+ );+ // Test more non-ASCII scripts+ assert_eq!(+ Markdown::escape("こんにちは *world*"),+ r"こんにちは \*world\*"+ );+ assert_eq!(Markdown::escape("العربيّة [link]"), r"العربيّة \[link\]");+ assert_eq!(Markdown::escape("Ελληνικά _text_"), r"Ελληνικά \_text\_");+ assert_eq!(Markdown::escape("עברית `code`"), r"עברית \`code\`");+ // Non-ASCII followed by ASCII punctuation+ assert_eq!(Markdown::escape("Test: тест"), r"Test\: тест");+ }+fn has_code_block(markdown: &str) -> bool {let parsed_data = parse_markdown_with_options(markdown, false, false);parsed_data@@ -3959,12 +3981,12 @@ mod tests {];for input in cases {let mut escaper = MarkdownEscaper::new();- let precomputed: usize = input.bytes().map(|b| escaper.next(b).output_len()).sum();+ let precomputed: usize = input.chars().map(|c| escaper.next(c).output_len(c)).sum();let mut escaper = MarkdownEscaper::new();let mut output = String::new();for c in input.chars() {- escaper.next(c as u8).write_to(c, &mut output);+ escaper.next(c).write_to(c, &mut output);}assert_eq!(precomputed, output.len(), "length mismatch for {:?}", input);