Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Fix UTF-8 boundary crashes in syntax parsing (#65273)

Ben Brandte5ff5550e4ec
diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs
index dd299c0381f..496f71efe35 100644
--- a/crates/language/src/language.rs
+++ b/crates/language/src/language.rs
@@ -1470,7 +1470,7 @@ pub(crate) fn parse_text(grammar: &Grammar, text: &Rope, old_tree: Option<Tree>)
.parse_with_options(
&mut move |offset, _| {
chunks.seek(offset);
- chunks.next().unwrap_or("").as_bytes()
+ chunks.peek_bytes().unwrap_or_default()
},
old_tree.as_ref(),
None,
diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs
index 26a53bfd5ba..0e151f74f3e 100644
--- a/crates/language/src/syntax_map.rs
+++ b/crates/language/src/syntax_map.rs
@@ -349,8 +349,6 @@ struct ChangeRegionSet(Vec<ChangedRegion>);
struct TextProvider<'a>(&'a Rope);
-struct ByteChunks<'a>(text::Chunks<'a>);
-
pub(crate) struct QueryCursorHandle(Option<QueryCursor>);
impl SyntaxMap {
@@ -1642,7 +1640,8 @@ fn parse_text(
.parse_with_options(
&mut move |offset, _| {
chunks.seek(start_byte + offset);
- chunks.next().unwrap_or("").as_bytes()
+ // Tree-sitter can request bytes inside a UTF-8 character.
+ chunks.peek_bytes().unwrap_or_default()
},
old_tree,
progress_callback
@@ -2248,18 +2247,10 @@ impl std::fmt::Debug for SyntaxLayerEntry {
}
impl<'a> tree_sitter::TextProvider<&'a [u8]> for TextProvider<'a> {
- type I = ByteChunks<'a>;
+ type I = text::Bytes<'a>;
fn text(&mut self, node: tree_sitter::Node) -> Self::I {
- ByteChunks(self.0.chunks_in_range(node.byte_range()))
- }
-}
-
-impl<'a> Iterator for ByteChunks<'a> {
- type Item = &'a [u8];
-
- fn next(&mut self) -> Option<Self::Item> {
- self.0.next().map(str::as_bytes)
+ self.0.bytes_in_range(node.byte_range())
}
}
diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs
index c86338e7bf0..2dd6723ec69 100644
--- a/crates/language/src/syntax_map/syntax_map_tests.rs
+++ b/crates/language/src/syntax_map/syntax_map_tests.rs
@@ -14,6 +14,87 @@ use tree_sitter::Node;
use unindent::Unindent as _;
use util::test::marked_text_ranges;
+#[test]
+fn test_parse_text_inside_multibyte_character() -> anyhow::Result<()> {
+ let language = rust_lang();
+ let grammar = language.grammar().expect("Rust grammar");
+ for prefix in ["", "prefix "] {
+ for character in ["и", "中", "🧘"] {
+ let source = format!("// {character}\nfn main() {{}}");
+ let text = Rope::from(format!("{prefix}{source}").as_str());
+ for character_offset in 1..character.len() {
+ let range = tree_sitter::Range {
+ start_byte: 3 + character_offset,
+ start_point: tree_sitter::Point {
+ row: 0,
+ column: 3 + character_offset,
+ },
+ end_byte: source.len(),
+ end_point: text.max_point().to_ts_point(),
+ };
+ let tree = parse_text(grammar, &text, prefix.len(), &[range], None, &mut None)?;
+
+ let mut parser = tree_sitter::Parser::new();
+ parser.set_language(&grammar.ts_language)?;
+ parser.set_included_ranges(&[range])?;
+ let expected_tree = parser
+ .parse(source.as_bytes(), None)
+ .expect("parse contiguous bytes");
+ assert_eq!(
+ tree.root_node().to_sexp(),
+ expected_tree.root_node().to_sexp()
+ );
+ assert_eq!(
+ tree.root_node().byte_range(),
+ expected_tree.root_node().byte_range()
+ );
+ assert_eq!(tree.root_node().end_byte(), source.len());
+ }
+ }
+ }
+ Ok(())
+}
+
+#[test]
+fn test_query_text_inside_multibyte_character() -> anyhow::Result<()> {
+ let language = rust_lang();
+ let grammar = language.grammar().expect("Rust grammar");
+ let query = Query::new(
+ &grammar.ts_language,
+ "((ERROR) @error (#match? @error \".*\"))",
+ )?;
+ for character in ["и", "中", "🧘"] {
+ let source = format!("// {character}\nfn main() {{}}");
+ let text = Rope::from(source.as_str());
+ for character_offset in 1..character.len() {
+ let range = tree_sitter::Range {
+ start_byte: 3 + character_offset,
+ start_point: tree_sitter::Point {
+ row: 0,
+ column: 3 + character_offset,
+ },
+ end_byte: text.len(),
+ end_point: text.max_point().to_ts_point(),
+ };
+ let tree = parse_text(grammar, &text, 0, &[range], None, &mut None)?;
+ let mut cursor = QueryCursor::new();
+ let mut matches = cursor.matches(&query, tree.root_node(), TextProvider(&text));
+ let query_match = matches.next().expect("predicate matches the error node");
+ let node = query_match.captures.first().expect("error capture").node;
+ assert_eq!(node.start_byte(), range.start_byte);
+ assert!(!source.is_char_boundary(node.start_byte()));
+ assert_eq!(
+ tree_sitter::TextProvider::text(&mut TextProvider(&text), node)
+ .flatten()
+ .copied()
+ .collect::<Vec<_>>(),
+ &source.as_bytes()[node.byte_range()],
+ );
+ }
+ }
+ Ok(())
+}
+
#[test]
fn test_splice_included_ranges() {
let ranges = vec![ts_range(20..30), ts_range(50..60), ts_range(80..90)];
diff --git a/crates/rope/src/rope.rs b/crates/rope/src/rope.rs
index 4d5aa410ae7..d31a5dbbb37 100644
--- a/crates/rope/src/rope.rs
+++ b/crates/rope/src/rope.rs
@@ -962,6 +962,18 @@ impl<'a> Chunks<'a> {
}
pub fn peek(&self) -> Option<&'a str> {
+ let (chunk, slice_range) = self.current_chunk()?;
+ Some(&chunk.text[slice_range])
+ }
+
+ /// Unlike `peek`, this can return bytes starting or ending inside a UTF-8 character.
+ /// The initial offset passed to `new` must still be a character boundary.
+ pub fn peek_bytes(&self) -> Option<&'a [u8]> {
+ let (chunk, slice_range) = self.current_chunk()?;
+ Some(&chunk.text.as_bytes()[slice_range])
+ }
+
+ fn current_chunk(&self) -> Option<(&'a Chunk, Range<usize>)> {
if !self.offset_is_valid() {
return None;
}
@@ -978,26 +990,12 @@ impl<'a> Chunks<'a> {
slice_start..slice_end
};
- Some(&chunk.text[slice_range])
+ Some((chunk, slice_range))
}
/// Returns bitmaps that represent character positions and tab positions
pub fn peek_with_bitmaps(&self) -> Option<ChunkBitmaps<'a>> {
- if !self.offset_is_valid() {
- return None;
- }
-
- let chunk = self.chunks.item()?;
- let chunk_start = *self.chunks.start();
- let slice_range = if self.reversed {
- let slice_start = cmp::max(chunk_start, self.range.start) - chunk_start;
- let slice_end = self.offset - chunk_start;
- slice_start..slice_end
- } else {
- let slice_start = self.offset - chunk_start;
- let slice_end = cmp::min(self.chunks.end(), self.range.end) - chunk_start;
- slice_start..slice_end
- };
+ let (chunk, slice_range) = self.current_chunk()?;
let chunk_start_offset = slice_range.start;
let slice_text = &chunk.text[slice_range];
@@ -1745,6 +1743,42 @@ mod tests {
assert_eq!(rope.text(), text);
}
+ #[test]
+ fn test_chunks_peek_bytes() {
+ let text = "aи中🧘z".repeat(128);
+ let rope = Rope::from(text.as_str());
+ for range in [0..text.len(), 1..text.len() - 1, 0..2] {
+ let mut chunks = rope.chunks_in_range(range.clone());
+ for offset in (range.clone()).chain(range.clone().rev()) {
+ chunks.seek(offset);
+ let bytes = chunks.peek_bytes().expect("chunk bytes");
+ assert!(!bytes.is_empty());
+ assert_eq!(bytes, &text.as_bytes()[offset..offset + bytes.len()]);
+ assert!(offset + bytes.len() <= range.end);
+ if text.is_char_boundary(offset) && text.is_char_boundary(offset + bytes.len()) {
+ assert_eq!(chunks.peek().map(str::as_bytes), Some(bytes));
+ }
+ }
+ chunks.seek(range.end);
+ assert_eq!(chunks.peek_bytes(), None);
+ }
+
+ for range in [0..text.len(), 2..text.len() - 1, 2..3] {
+ let mut chunks = rope.reversed_chunks_in_range(range.clone());
+ let offsets = range.start + 1..=range.end;
+ for offset in offsets.clone().rev().chain(offsets) {
+ chunks.seek(offset);
+ let bytes = chunks.peek_bytes().expect("reversed chunk bytes");
+ assert!(!bytes.is_empty());
+ assert_eq!(bytes, &text.as_bytes()[offset - bytes.len()..offset]);
+ assert!(offset - bytes.len() >= range.start);
+ }
+ chunks.seek(range.start);
+ assert_eq!(chunks.peek_bytes(), None);
+ }
+ assert_eq!(Rope::new().chunks().peek_bytes(), None);
+ }
+
#[test]
fn test_clip() {
let rope = Rope::from("🧘");