Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Fix UTF-8 boundary crashes in syntax parsing (#65273)
Ben Brandte5ff5550e4ec
diff --git a/crates/language/src/language.rs b/crates/language/src/language.rsindex dd299c0381f..496f71efe35 100644--- a/crates/language/src/language.rs+++ b/crates/language/src/language.rs@@ -1470,7 +1470,7 @@ pub(crate) fn parse_text(grammar: &Grammar, text: &Rope, old_tree: Option<Tree>).parse_with_options(&mut move |offset, _| {chunks.seek(offset);- chunks.next().unwrap_or("").as_bytes()+ chunks.peek_bytes().unwrap_or_default()},old_tree.as_ref(),None,diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rsindex 26a53bfd5ba..0e151f74f3e 100644--- a/crates/language/src/syntax_map.rs+++ b/crates/language/src/syntax_map.rs@@ -349,8 +349,6 @@ struct ChangeRegionSet(Vec<ChangedRegion>);struct TextProvider<'a>(&'a Rope);-struct ByteChunks<'a>(text::Chunks<'a>);-pub(crate) struct QueryCursorHandle(Option<QueryCursor>);impl SyntaxMap {@@ -1642,7 +1640,8 @@ fn parse_text(.parse_with_options(&mut move |offset, _| {chunks.seek(start_byte + offset);- chunks.next().unwrap_or("").as_bytes()+ // Tree-sitter can request bytes inside a UTF-8 character.+ chunks.peek_bytes().unwrap_or_default()},old_tree,progress_callback@@ -2248,18 +2247,10 @@ impl std::fmt::Debug for SyntaxLayerEntry {}impl<'a> tree_sitter::TextProvider<&'a [u8]> for TextProvider<'a> {- type I = ByteChunks<'a>;+ type I = text::Bytes<'a>;fn text(&mut self, node: tree_sitter::Node) -> Self::I {- ByteChunks(self.0.chunks_in_range(node.byte_range()))- }-}--impl<'a> Iterator for ByteChunks<'a> {- type Item = &'a [u8];-- fn next(&mut self) -> Option<Self::Item> {- self.0.next().map(str::as_bytes)+ self.0.bytes_in_range(node.byte_range())}}diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rsindex c86338e7bf0..2dd6723ec69 100644--- a/crates/language/src/syntax_map/syntax_map_tests.rs+++ b/crates/language/src/syntax_map/syntax_map_tests.rs@@ -14,6 +14,87 @@ use tree_sitter::Node;use unindent::Unindent as _;use util::test::marked_text_ranges;+#[test]+fn test_parse_text_inside_multibyte_character() -> anyhow::Result<()> {+ let language = rust_lang();+ let grammar = language.grammar().expect("Rust grammar");+ for prefix in ["", "prefix "] {+ for character in ["и", "中", "🧘"] {+ let source = format!("// {character}\nfn main() {{}}");+ let text = Rope::from(format!("{prefix}{source}").as_str());+ for character_offset in 1..character.len() {+ let range = tree_sitter::Range {+ start_byte: 3 + character_offset,+ start_point: tree_sitter::Point {+ row: 0,+ column: 3 + character_offset,+ },+ end_byte: source.len(),+ end_point: text.max_point().to_ts_point(),+ };+ let tree = parse_text(grammar, &text, prefix.len(), &[range], None, &mut None)?;++ let mut parser = tree_sitter::Parser::new();+ parser.set_language(&grammar.ts_language)?;+ parser.set_included_ranges(&[range])?;+ let expected_tree = parser+ .parse(source.as_bytes(), None)+ .expect("parse contiguous bytes");+ assert_eq!(+ tree.root_node().to_sexp(),+ expected_tree.root_node().to_sexp()+ );+ assert_eq!(+ tree.root_node().byte_range(),+ expected_tree.root_node().byte_range()+ );+ assert_eq!(tree.root_node().end_byte(), source.len());+ }+ }+ }+ Ok(())+}++#[test]+fn test_query_text_inside_multibyte_character() -> anyhow::Result<()> {+ let language = rust_lang();+ let grammar = language.grammar().expect("Rust grammar");+ let query = Query::new(+ &grammar.ts_language,+ "((ERROR) @error (#match? @error \".*\"))",+ )?;+ for character in ["и", "中", "🧘"] {+ let source = format!("// {character}\nfn main() {{}}");+ let text = Rope::from(source.as_str());+ for character_offset in 1..character.len() {+ let range = tree_sitter::Range {+ start_byte: 3 + character_offset,+ start_point: tree_sitter::Point {+ row: 0,+ column: 3 + character_offset,+ },+ end_byte: text.len(),+ end_point: text.max_point().to_ts_point(),+ };+ let tree = parse_text(grammar, &text, 0, &[range], None, &mut None)?;+ let mut cursor = QueryCursor::new();+ let mut matches = cursor.matches(&query, tree.root_node(), TextProvider(&text));+ let query_match = matches.next().expect("predicate matches the error node");+ let node = query_match.captures.first().expect("error capture").node;+ assert_eq!(node.start_byte(), range.start_byte);+ assert!(!source.is_char_boundary(node.start_byte()));+ assert_eq!(+ tree_sitter::TextProvider::text(&mut TextProvider(&text), node)+ .flatten()+ .copied()+ .collect::<Vec<_>>(),+ &source.as_bytes()[node.byte_range()],+ );+ }+ }+ Ok(())+}+#[test]fn test_splice_included_ranges() {let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];diff --git a/crates/rope/src/rope.rs b/crates/rope/src/rope.rsindex 4d5aa410ae7..d31a5dbbb37 100644--- a/crates/rope/src/rope.rs+++ b/crates/rope/src/rope.rs@@ -962,6 +962,18 @@ impl<'a> Chunks<'a> {}pub fn peek(&self) -> Option<&'a str> {+ let (chunk, slice_range) = self.current_chunk()?;+ Some(&chunk.text[slice_range])+ }++ /// Unlike `peek`, this can return bytes starting or ending inside a UTF-8 character.+ /// The initial offset passed to `new` must still be a character boundary.+ pub fn peek_bytes(&self) -> Option<&'a [u8]> {+ let (chunk, slice_range) = self.current_chunk()?;+ Some(&chunk.text.as_bytes()[slice_range])+ }++ fn current_chunk(&self) -> Option<(&'a Chunk, Range<usize>)> {if !self.offset_is_valid() {return None;}@@ -978,26 +990,12 @@ impl<'a> Chunks<'a> {slice_start..slice_end};- Some(&chunk.text[slice_range])+ Some((chunk, slice_range))}/// Returns bitmaps that represent character positions and tab positionspub fn peek_with_bitmaps(&self) -> Option<ChunkBitmaps<'a>> {- if !self.offset_is_valid() {- return None;- }-- let chunk = self.chunks.item()?;- let chunk_start = *self.chunks.start();- let slice_range = if self.reversed {- let slice_start = cmp::max(chunk_start, self.range.start) - chunk_start;- let slice_end = self.offset - chunk_start;- slice_start..slice_end- } else {- let slice_start = self.offset - chunk_start;- let slice_end = cmp::min(self.chunks.end(), self.range.end) - chunk_start;- slice_start..slice_end- };+ let (chunk, slice_range) = self.current_chunk()?;let chunk_start_offset = slice_range.start;let slice_text = &chunk.text[slice_range];@@ -1745,6 +1743,42 @@ mod tests {assert_eq!(rope.text(), text);}+ #[test]+ fn test_chunks_peek_bytes() {+ let text = "aи中🧘z".repeat(128);+ let rope = Rope::from(text.as_str());+ for range in [0..text.len(), 1..text.len() - 1, 0..2] {+ let mut chunks = rope.chunks_in_range(range.clone());+ for offset in (range.clone()).chain(range.clone().rev()) {+ chunks.seek(offset);+ let bytes = chunks.peek_bytes().expect("chunk bytes");+ assert!(!bytes.is_empty());+ assert_eq!(bytes, &text.as_bytes()[offset..offset + bytes.len()]);+ assert!(offset + bytes.len() <= range.end);+ if text.is_char_boundary(offset) && text.is_char_boundary(offset + bytes.len()) {+ assert_eq!(chunks.peek().map(str::as_bytes), Some(bytes));+ }+ }+ chunks.seek(range.end);+ assert_eq!(chunks.peek_bytes(), None);+ }++ for range in [0..text.len(), 2..text.len() - 1, 2..3] {+ let mut chunks = rope.reversed_chunks_in_range(range.clone());+ let offsets = range.start + 1..=range.end;+ for offset in offsets.clone().rev().chain(offsets) {+ chunks.seek(offset);+ let bytes = chunks.peek_bytes().expect("reversed chunk bytes");+ assert!(!bytes.is_empty());+ assert_eq!(bytes, &text.as_bytes()[offset - bytes.len()..offset]);+ assert!(offset - bytes.len() >= range.start);+ }+ chunks.seek(range.start);+ assert_eq!(chunks.peek_bytes(), None);+ }+ assert_eq!(Rope::new().chunks().peek_bytes(), None);+ }+#[test]fn test_clip() {let rope = Rope::from("🧘");