Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

markdown: Fix escaping non-ASCII chars (#55782)

alkinunc0596fade730
diff --git a/crates/markdown/src/markdown.rs b/crates/markdown/src/markdown.rs
index 69feee416da..44e87f55677 100644
--- a/crates/markdown/src/markdown.rs
+++ b/crates/markdown/src/markdown.rs
@@ -401,12 +401,12 @@ enum EscapeAction {
}
impl EscapeAction {
- fn output_len(&self) -> usize {
+ fn output_len(&self, c: char) -> usize {
match self {
- Self::PassThrough => 1,
+ Self::PassThrough => c.len_utf8(),
Self::Nbsp(count) => count * '\u{00A0}'.len_utf8(),
Self::DoubleNewline => 2,
- Self::PrefixBackslash => 2,
+ Self::PrefixBackslash => '\\'.len_utf8() + c.len_utf8(),
}
}
@@ -431,8 +431,6 @@ impl EscapeAction {
}
}
-// Valid to operate on raw bytes since multi-byte UTF-8
-// sequences never contain ASCII-range bytes.
struct MarkdownEscaper {
in_leading_whitespace: bool,
}
@@ -446,21 +444,21 @@ impl MarkdownEscaper {
}
}
- fn next(&mut self, byte: u8) -> EscapeAction {
- let action = if self.in_leading_whitespace && byte == b'\t' {
+ fn next(&mut self, c: char) -> EscapeAction {
+ let action = if self.in_leading_whitespace && c == '\t' {
EscapeAction::Nbsp(Self::TAB_SIZE)
- } else if self.in_leading_whitespace && byte == b' ' {
+ } else if self.in_leading_whitespace && c == ' ' {
EscapeAction::Nbsp(1)
- } else if byte == b'\n' {
+ } else if c == '\n' {
EscapeAction::DoubleNewline
- } else if byte.is_ascii_punctuation() {
+ } else if c.is_ascii_punctuation() {
EscapeAction::PrefixBackslash
} else {
EscapeAction::PassThrough
};
self.in_leading_whitespace =
- byte == b'\n' || (self.in_leading_whitespace && (byte == b' ' || byte == b'\t'));
+ c == '\n' || (self.in_leading_whitespace && (c == ' ' || c == '\t'));
action
}
}
@@ -675,7 +673,7 @@ impl Markdown {
pub fn escape(s: &str) -> Cow<'_, str> {
let output_len: usize = {
let mut escaper = MarkdownEscaper::new();
- s.bytes().map(|byte| escaper.next(byte).output_len()).sum()
+ s.chars().map(|c| escaper.next(c).output_len(c)).sum()
};
if output_len == s.len() {
@@ -685,7 +683,7 @@ impl Markdown {
let mut escaper = MarkdownEscaper::new();
let mut output = String::with_capacity(output_len);
for c in s.chars() {
- escaper.next(c as u8).write_to(c, &mut output);
+ escaper.next(c).write_to(c, &mut output);
}
output.into()
}
@@ -3931,6 +3929,30 @@ mod tests {
);
}
+ #[test]
+ fn test_escape_non_ascii() {
+ // Cyrillic characters should not have backslashes added before them,
+ // but ASCII punctuation should still be escaped.
+ assert_eq!(Markdown::escape("Привет, мир"), r"Привет\, мир");
+ // Test with markdown special characters mixed in
+ assert_eq!(Markdown::escape("Привет, *мир*"), r"Привет\, \*мир\*");
+ // Test with the exact example from the issue (single quotes are also ASCII punctuation)
+ assert_eq!(
+ Markdown::escape("Отсутствует пробел справа от ','"),
+ r"Отсутствует пробел справа от \'\,\'"
+ );
+ // Test more non-ASCII scripts
+ assert_eq!(
+ Markdown::escape("こんにちは *world*"),
+ r"こんにちは \*world\*"
+ );
+ assert_eq!(Markdown::escape("العربيّة [link]"), r"العربيّة \[link\]");
+ assert_eq!(Markdown::escape("Ελληνικά _text_"), r"Ελληνικά \_text\_");
+ assert_eq!(Markdown::escape("עברית `code`"), r"עברית \`code\`");
+ // Non-ASCII followed by ASCII punctuation
+ assert_eq!(Markdown::escape("Test: тест"), r"Test\: тест");
+ }
+
fn has_code_block(markdown: &str) -> bool {
let parsed_data = parse_markdown_with_options(markdown, false, false);
parsed_data
@@ -3959,12 +3981,12 @@ mod tests {
];
for input in cases {
let mut escaper = MarkdownEscaper::new();
- let precomputed: usize = input.bytes().map(|b| escaper.next(b).output_len()).sum();
+ let precomputed: usize = input.chars().map(|c| escaper.next(c).output_len(c)).sum();
let mut escaper = MarkdownEscaper::new();
let mut output = String::new();
for c in input.chars() {
- escaper.next(c as u8).write_to(c, &mut output);
+ escaper.next(c).write_to(c, &mut output);
}
assert_eq!(precomputed, output.len(), "length mismatch for {:?}", input);