Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
language: Fix JSON rainbow bracket colors across chunks (#60741)
saberoueslati7cdf2ae6b62c
diff --git a/Cargo.lock b/Cargo.lockindex c231723525f..6b4c6950b3f 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -9706,6 +9706,7 @@ dependencies = ["fuzzy_nucleo","globset","gpui",+ "grammars","http_client","imara-diff","indoc",diff --git a/crates/editor/src/bracket_colorization.rs b/crates/editor/src/bracket_colorization.rsindex 993fb8f8e6b..a0ae9022b32 100644--- a/crates/editor/src/bracket_colorization.rs+++ b/crates/editor/src/bracket_colorization.rs@@ -1981,7 +1981,7 @@ mod foo «1{assert_eq!(indoc! {r#"-⋯1»+⋯1»2»1»fn small_function«1()1» «1{let x = «2(1, «3(2, 3)3»)2»;diff --git a/crates/language/Cargo.toml b/crates/language/Cargo.tomlindex df493c393e7..69789da21ec 100644--- a/crates/language/Cargo.toml+++ b/crates/language/Cargo.toml@@ -16,11 +16,13 @@ doctest = falsetest-support = ["rand","collections/test-support",+ "grammars","lsp/test-support","text/test-support","tree-sitter-rust","tree-sitter-python","tree-sitter-typescript",+ "tree-sitter-json","tree-sitter-md","settings/test-support","util/test-support",@@ -64,6 +66,8 @@ text.workspace = truetheme.workspace = truetoml.workspace = truetracing.workspace = true+grammars = { workspace = true, optional = true }+tree-sitter-json = { workspace = true, optional = true }tree-sitter-md = { workspace = true, optional = true }tree-sitter-python = { workspace = true, optional = true }tree-sitter-rust = { workspace = true, optional = true }@@ -79,6 +83,7 @@ diffy = "0.4.2"[dev-dependencies]collections = { workspace = true, features = ["test-support"] }ctor.workspace = true+grammars.workspace = truegpui = { workspace = true, features = ["test-support"] }indoc.workspace = truelsp = { workspace = true, features = ["test-support"] }diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rsindex e568a5db4f5..d63e588a364 100644--- a/crates/language/src/buffer.rs+++ b/crates/language/src/buffer.rs@@ -4847,75 +4847,112 @@ impl BufferSnapshot {let mut opens = Vec::new();let mut color_pairs = Vec::new();- let mut matches = self.syntax.matches_with_options(+ let bounded_query = (chunk_range.clone(),- &self.text,TreeSitterOptions {max_bytes_to_query: Some(MAX_BYTES_TO_QUERY),max_start_depth: None,},- |grammar| grammar.brackets_config.as_ref().map(|c| &c.query),);- let configs = matches- .grammars()- .iter()- .map(|grammar| grammar.brackets_config.as_ref().unwrap())- .collect::<Vec<_>>();+ // The bounded query drops any pair spanning more than `MAX_BYTES_TO_QUERY`+ // (tree-sitter's containing byte range requires full containment), which+ // resets bracket depth at chunk boundaries. Every such pair encloses a+ // chunk edge, so recover them with two point-sized unbounded queries that+ // only traverse the syntax tree spine around each edge.+ let boundary_queries = [chunk_range.start, chunk_range.end].map(|offset| {+ (+ self.clip_offset(offset.saturating_sub(1), Bias::Left)+ ..self.clip_offset(offset.saturating_add(1), Bias::Right),+ TreeSitterOptions::default(),+ )+ });+ let mut grammar_ids = Vec::new();+ let mut configs = Vec::new();+ let mut seen_delimiters = HashSet::default();// Group matches by open delimiter so we can either trust grammar output// or repair it by picking a single closest close per open.let mut close_delimiters_by_open = BTreeMap::new();let mut bogus_patterns = HashSet::default();- while let Some(mat) = matches.peek() {- let mut open = None;- let mut close = None;- let syntax_layer_depth = mat.depth;- let pattern_key = BracketPatternKey {- grammar_index: mat.grammar_index,- pattern_index: mat.pattern_index,- };- let config = configs[mat.grammar_index];- let pattern = &config.patterns[mat.pattern_index];- for capture in mat.captures {- if capture.index == config.open_capture_ix {- open = Some(capture.node.byte_range());- } else if capture.index == config.close_capture_ix {- close = Some(capture.node.byte_range());+ for (query_range, options) in iter::once(bounded_query).chain(boundary_queries) {+ let mut matches =+ self.syntax+ .matches_with_options(query_range, &self.text, options, |grammar| {+ grammar.brackets_config.as_ref().map(|c| &c.query)+ });+ let grammar_indices = matches+ .grammars()+ .iter()+ .map(|grammar| {+ grammar_ids+ .iter()+ .position(|&id| id == grammar.id())+ .unwrap_or_else(|| {+ grammar_ids.push(grammar.id());+ configs.push(grammar.brackets_config.as_ref().unwrap());+ grammar_ids.len() - 1+ })+ })+ .collect::<Vec<_>>();++ while let Some(mat) = matches.peek() {+ let mut open = None;+ let mut close = None;+ let syntax_layer_depth = mat.depth;+ let grammar_index = grammar_indices[mat.grammar_index];+ let pattern_key = BracketPatternKey {+ grammar_index,+ pattern_index: mat.pattern_index,+ };+ let config = configs[grammar_index];+ let pattern = &config.patterns[mat.pattern_index];+ for capture in mat.captures {+ if capture.index == config.open_capture_ix {+ open = Some(capture.node.byte_range());+ } else if capture.index == config.close_capture_ix {+ close = Some(capture.node.byte_range());+ }}- }- matches.advance();+ matches.advance();- let Some((open_range, close_range)) = open.zip(close) else {- continue;- };+ let Some((open_range, close_range)) = open.zip(close) else {+ continue;+ };- let bracket_range = open_range.start..=close_range.end;- if !bracket_range.overlaps(&chunk_range) {- continue;- }+ let bracket_range = open_range.start..=close_range.end;+ if !bracket_range.overlaps(&chunk_range) {+ continue;+ }- let candidate = BracketMatchCandidate {- bracket_match: BracketMatch {- open_range,- close_range,- syntax_layer_depth,- newline_only: pattern.newline_only,- color_index: None,- },- pattern: pattern_key,- rainbow_exclude: pattern.rainbow_exclude,- };+ let candidate = BracketMatchCandidate {+ bracket_match: BracketMatch {+ open_range,+ close_range,+ syntax_layer_depth,+ newline_only: pattern.newline_only,+ color_index: None,+ },+ pattern: pattern_key,+ rainbow_exclude: pattern.rainbow_exclude,+ };- let close_delimiters = close_delimiters_by_open- .entry(candidate.open_delimiter())- .or_insert_with(BTreeSet::new);- close_delimiters.insert(candidate.close_delimiter());- if close_delimiters.len() > 1 {- bogus_patterns.insert(pattern_key);- }+ if !seen_delimiters+ .insert((candidate.open_delimiter(), candidate.close_delimiter()))+ {+ continue;+ }++ let close_delimiters = close_delimiters_by_open+ .entry(candidate.open_delimiter())+ .or_insert_with(BTreeSet::new);+ close_delimiters.insert(candidate.close_delimiter());+ if close_delimiters.len() > 1 {+ bogus_patterns.insert(pattern_key);+ }- all_brackets.push(candidate);+ all_brackets.push(candidate);+ }}if !bogus_patterns.is_empty() {diff --git a/crates/language/src/buffer_tests.rs b/crates/language/src/buffer_tests.rsindex 5c4bf46a11f..2203bd30f7a 100644--- a/crates/language/src/buffer_tests.rs+++ b/crates/language/src/buffer_tests.rs@@ -20,7 +20,7 @@ use std::{sync::LazyLock,time::{Duration, Instant},};-use syntax_map::TreeSitterOptions;+use syntax_map::{MAX_BYTES_TO_QUERY, TreeSitterOptions};use text::network::Network;use text::{BufferId, LineEnding};use text::{Point, ToPoint};@@ -768,9 +768,7 @@ async fn test_resetting_language(cx: &mut gpui::TestAppContext) {"(source_file (expression_statement (block)))");- buffer.update(cx, |buffer, cx| {- buffer.set_language(Some(Arc::new(json_lang())), cx)- });+ buffer.update(cx, |buffer, cx| buffer.set_language(Some(json_lang()), cx));cx.executor().run_until_parked();assert_eq!(get_tree_sexp(&buffer, cx), "(document (object))");}@@ -1419,6 +1417,51 @@ fn test_enclosing_bracket_ranges(cx: &mut App) {);}+#[gpui::test]+fn test_bracket_colorization_indices_remain_stable_across_row_chunks(cx: &mut App) {+ let mut text = String::from("{\n \"theme\": {\n");+ let mut property_object_open_offsets = Vec::new();+ for index in 0..500 {+ text.push_str(&format!(" \"scope_{index:03}\": "));+ property_object_open_offsets.push(text.len());+ text.push_str("{\n \"color\": \"#ffffff\"\n },\n");+ }+ text.push_str(" \"last\": {}\n }\n}\n");+ assert!(+ text.len() > MAX_BYTES_TO_QUERY,+ "fixture should exceed the bounded tree-sitter query window"+ );++ let buffer = cx.new(|cx| Buffer::local(text.clone(), cx).with_language(json_lang(), cx));+ let snapshot = buffer.update(cx, |buffer, _| buffer.snapshot());++ let late_open_offset = property_object_open_offsets[400];+ let late_matches = snapshot.fetch_bracket_ranges(late_open_offset..late_open_offset + 1, None);+ let late_color_index = color_index_for_open(&late_matches, late_open_offset);++ assert_eq!(+ late_color_index,+ Some(2),+ "Jumping directly into a later row chunk should preserve enclosing JSON object depth"+ );++ let first_open_offset = property_object_open_offsets[0];+ let all_matches = snapshot.fetch_bracket_ranges(0..snapshot.len(), None);+ assert_eq!(+ color_index_for_open(&all_matches, first_open_offset),+ late_color_index,+ "Sibling object braces should keep the same color across row chunks"+ );++ for open_offset in property_object_open_offsets {+ assert_eq!(+ color_index_for_open(&all_matches, open_offset),+ late_color_index,+ "All generated sibling object braces should share the same depth color"+ );+ }+}+#[gpui::test]fn test_enclosing_bracket_ranges_where_brackets_are_not_outermost_children(cx: &mut App) {let mut assert = |selection_text, bracket_pair_texts| {@@ -4393,18 +4436,15 @@ fn erb_lang() -> Language {.unwrap()}-fn json_lang() -> Language {- Language::new(- LanguageConfig {- name: "Json".into(),- matcher: LanguageMatcher {- path_suffixes: vec!["js".to_string()],- ..Default::default()- },- ..Default::default()- },- Some(tree_sitter_json::LANGUAGE.into()),- )+fn color_index_for_open(+ matches: &HashMap<Range<BufferRow>, Vec<BracketMatch<usize>>>,+ open_offset: usize,+) -> Option<usize> {+ matches+ .values()+ .flatten()+ .find(|bracket_match| bracket_match.open_range.start == open_offset)+ .and_then(|bracket_match| bracket_match.color_index)}fn javascript_lang() -> Language {diff --git a/crates/language/src/language.rs b/crates/language/src/language.rsindex e88e6a4d1ef..08712de4415 100644--- a/crates/language/src/language.rs+++ b/crates/language/src/language.rs@@ -1596,130 +1596,28 @@ pub fn range_from_lsp(range: lsp::Range) -> Range<Unclipped<PointUtf16>> {#[doc(hidden)]#[cfg(any(test, feature = "test-support"))]pub fn rust_lang() -> Arc<Language> {- use std::borrow::Cow;+ test_language("rust", tree_sitter_rust::LANGUAGE.into())+}- let language = Language::new(- LanguageConfig {- name: "Rust".into(),- matcher: LanguageMatcher {- path_suffixes: vec!["rs".to_string()],- ..Default::default()- },- line_comments: vec!["// ".into(), "/// ".into(), "//! ".into()],- brackets: BracketPairConfig {- pairs: vec![- BracketPair {- start: "{".into(),- end: "}".into(),- close: true,- surround: false,- newline: true,- },- BracketPair {- start: "[".into(),- end: "]".into(),- close: true,- surround: false,- newline: true,- },- BracketPair {- start: "(".into(),- end: ")".into(),- close: true,- surround: false,- newline: true,- },- BracketPair {- start: "<".into(),- end: ">".into(),- close: false,- surround: false,- newline: true,- },- BracketPair {- start: "\"".into(),- end: "\"".into(),- close: true,- surround: false,- newline: false,- },- ],- ..Default::default()- },- ..Default::default()- },- Some(tree_sitter_rust::LANGUAGE.into()),- )- .with_queries(LanguageQueries {- outline: Some(Cow::from(include_str!(- "../../grammars/src/rust/outline.scm"- ))),- indents: Some(Cow::from(include_str!(- "../../grammars/src/rust/indents.scm"- ))),- brackets: Some(Cow::from(include_str!(- "../../grammars/src/rust/brackets.scm"- ))),- text_objects: Some(Cow::from(include_str!(- "../../grammars/src/rust/textobjects.scm"- ))),- highlights: Some(Cow::from(include_str!(- "../../grammars/src/rust/highlights.scm"- ))),- injections: Some(Cow::from(include_str!(- "../../grammars/src/rust/injections.scm"- ))),- overrides: Some(Cow::from(include_str!(- "../../grammars/src/rust/overrides.scm"- ))),- redactions: None,- runnables: Some(Cow::from(include_str!(- "../../grammars/src/rust/runnables.scm"- ))),- debugger: Some(Cow::from(include_str!(- "../../grammars/src/rust/debugger.scm"- ))),- })- .expect("Could not parse queries");- Arc::new(language)+#[doc(hidden)]+#[cfg(any(test, feature = "test-support"))]+pub fn json_lang() -> Arc<Language> {+ test_language("json", tree_sitter_json::LANGUAGE.into())}#[doc(hidden)]#[cfg(any(test, feature = "test-support"))]pub fn markdown_lang() -> Arc<Language> {- use std::borrow::Cow;+ test_language("markdown", tree_sitter_md::LANGUAGE.into())+}- let language = Language::new(- LanguageConfig {- name: "Markdown".into(),- matcher: LanguageMatcher {- path_suffixes: vec!["md".into()],- ..Default::default()- },- ..LanguageConfig::default()- },- Some(tree_sitter_md::LANGUAGE.into()),+#[cfg(any(test, feature = "test-support"))]+fn test_language(name: &str, grammar: tree_sitter::Language) -> Arc<Language> {+ Arc::new(+ Language::new(grammars::load_config(name), Some(grammar))+ .with_queries(grammars::load_queries(name))+ .unwrap_or_else(|error| panic!("could not parse queries for language {name}: {error}")),)- .with_queries(LanguageQueries {- brackets: Some(Cow::from(include_str!(- "../../grammars/src/markdown/brackets.scm"- ))),- injections: Some(Cow::from(include_str!(- "../../grammars/src/markdown/injections.scm"- ))),- highlights: Some(Cow::from(include_str!(- "../../grammars/src/markdown/highlights.scm"- ))),- indents: Some(Cow::from(include_str!(- "../../grammars/src/markdown/indents.scm"- ))),- outline: Some(Cow::from(include_str!(- "../../grammars/src/markdown/outline.scm"- ))),- ..LanguageQueries::default()- })- .expect("Could not parse markdown queries");- Arc::new(language)}#[cfg(test)]