Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
Fix language injections containing interpolations (#49265)
Jeff Brennan72b02bf1bb2f
diff --git a/crates/grammars/src/python/injections.scm b/crates/grammars/src/python/injections.scmindex bc47469dc87..8981caffac4 100644--- a/crates/grammars/src/python/injections.scm+++ b/crates/grammars/src/python/injections.scm@@ -13,14 +13,14 @@arguments: (argument_list(comment) @_comment(string- (string_content) @injection.content)))+ (string_content) @injection.content) @injection.host)); string variables((comment) @_comment.(expression_statement(assignmentright: (string- (string_content) @injection.content))))+ (string_content) @injection.content) @injection.host)))](#match? @_comment "^(#|#\\s+)(?i:sql)\\s*$")(#set! injection.language "sql"))diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rsindex 7f9d377e6d0..9ff17ce2ded 100644--- a/crates/language/src/syntax_map.rs+++ b/crates/language/src/syntax_map.rs@@ -253,6 +253,21 @@ enum ParseMode {},}+/// Identifies a set of injection matches that should share a single syntax layer.+#[derive(PartialEq, Eq, Hash)]+enum InjectionGroupKey {+ /// Every match of an `injection.combined` pattern for a given language shares one+ /// layer spanning the whole parent layer.+ Combined(LanguageId),+ /// Every match whose `@injection.host` capture resolves to the same node shares one+ /// layer spanning that node. This keeps interpolated strings such as Python+ /// f-strings, which produce one match per string fragment, in a single layer.+ Host {+ language: LanguageId,+ host_range: Range<usize>,+ },+}+#[derive(Debug, PartialEq, Eq)]struct ChangedRegion {depth: usize,@@ -561,7 +576,7 @@ impl SyntaxSnapshot {let mut changed_regions = ChangeRegionSet::default();let mut queue = BinaryHeap::new();- let mut combined_injection_ranges = HashMap::default();+ let mut injection_groups = HashMap::default();queue.push(ParseStep {depth: 0,language: ParseStepLanguage::Loaded {@@ -860,25 +875,29 @@ impl SyntaxSnapshot {registry,step.depth + 1,&expanded_ranges,- &mut combined_injection_ranges,+ &mut injection_groups,&mut queue,);}- let included_sub_ranges: Option<Vec<Range<Anchor>>> = if is_combined {- Some(- included_ranges- .into_iter()- .filter(|r| r.start_byte < r.end_byte)- .map(|r| {- text.anchor_before(r.start_byte + step_start_byte)- ..text.anchor_after(r.end_byte + step_start_byte)- })- .collect(),- )- } else {- None- };+ // Layers built from more than one included range don't cover their+ // whole span, so record the sub-ranges to let callers tell which+ // offsets actually belong to this layer.+ let included_sub_ranges: Option<Vec<Range<Anchor>>> =+ if is_combined || included_ranges.len() > 1 {+ Some(+ included_ranges+ .into_iter()+ .filter(|r| r.start_byte < r.end_byte)+ .map(|r| {+ text.anchor_before(r.start_byte + step_start_byte)+ ..text.anchor_after(r.end_byte + step_start_byte)+ })+ .collect(),+ )+ } else {+ None+ };SyntaxLayerContent::Parsed {tree,language,@@ -1583,7 +1602,7 @@ fn get_injections(language_registry: &Arc<LanguageRegistry>,depth: usize,changed_ranges: &[Range<usize>],- combined_injection_ranges: &mut HashMap<LanguageId, (Arc<Language>, Vec<tree_sitter::Range>)>,+ injection_groups: &mut HashMap<InjectionGroupKey, (Arc<Language>, Vec<tree_sitter::Range>)>,queue: &mut BinaryHeap<ParseStep>,) {let mut query_cursor = QueryCursorHandle::new();@@ -1591,7 +1610,7 @@ fn get_injections(// Ensure that a `ParseStep` is created for every combined injection language, even// if there currently no matches for that injection.- combined_injection_ranges.clear();+ injection_groups.clear();for pattern in &config.patterns {if let (Some(language_name), true) = (pattern.language.as_ref(), pattern.combined)&& let Some(language) = language_registry@@ -1599,7 +1618,10 @@ fn get_injections(.now_or_never().and_then(|language| language.ok()){- combined_injection_ranges.insert(language.id, (language, Vec::new()));+ injection_groups.insert(+ InjectionGroupKey::Combined(language.id),+ (language, Vec::new()),+ );}}@@ -1629,6 +1651,11 @@ fn get_injections(prev_match = Some((mat.pattern_index, content_range.clone()));let combined = config.patterns[mat.pattern_index].combined;+ let host_range = config+ .host_capture_ix+ .and_then(|ix| mat.nodes_for_capture_index(ix).next())+ .map(|host_node| host_node.byte_range());+let mut step_range = content_range.clone();let language_name =if let Some(name) = config.patterns[mat.pattern_index].language.as_ref() {@@ -1658,11 +1685,23 @@ fn get_injections(.now_or_never().and_then(|language| language.ok());let range = text.anchor_before(step_range.start)..text.anchor_after(step_range.end);+if let Some(language) = language {- if combined {- combined_injection_ranges- .entry(language.id)- .or_insert_with(|| (language.clone(), vec![]))+ let group_key = if let Some(host_range) = host_range {+ Some(InjectionGroupKey::Host {+ language: language.id,+ host_range,+ })+ } else if combined {+ Some(InjectionGroupKey::Combined(language.id))+ } else {+ None+ };++ if let Some(group_key) = group_key {+ injection_groups+ .entry(group_key)+ .or_insert_with(|| (language.clone(), Vec::new())).1.extend(content_ranges);} else {@@ -1689,19 +1728,34 @@ fn get_injections(}}- for (_, (language, mut included_ranges)) in combined_injection_ranges.drain() {+ for (group_key, (language, mut included_ranges)) in injection_groups.drain() {included_ranges.sort_unstable_by(|a, b| {Ord::cmp(&a.start_byte, &b.start_byte).then_with(|| Ord::cmp(&a.end_byte, &b.end_byte))});+ // Overlapping changed ranges can yield the same match more than once, and+ // `set_included_ranges` rejects overlapping ranges.+ included_ranges.dedup();++ let (range, mode) = match group_key {+ InjectionGroupKey::Combined(_) => (+ outer_range.clone(),+ ParseMode::Combined {+ parent_layer_range: outer_range.to_offset(text),+ parent_layer_changed_ranges: changed_ranges.to_vec(),+ },+ ),+ InjectionGroupKey::Host { host_range, .. } => (+ text.anchor_before(host_range.start)..text.anchor_after(host_range.end),+ ParseMode::Single,+ ),+ };+queue.push(ParseStep {depth,language: ParseStepLanguage::Loaded { language },- range: outer_range.clone(),+ range,included_ranges,- mode: ParseMode::Combined {- parent_layer_range: outer_range.to_offset(text),- parent_layer_changed_ranges: changed_ranges.to_vec(),- },+ mode,})}}diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rsindex 442e6890fe7..d4eecbe04c1 100644--- a/crates/language/src/syntax_map/syntax_map_tests.rs+++ b/crates/language/src/syntax_map/syntax_map_tests.rs@@ -4,6 +4,7 @@ use crate::{buffer_tests::markdown_inline_lang, markdown_lang, rust_lang,};use gpui::App;+use indoc::indoc;use pretty_assertions::assert_eq;use rand::rngs::StdRng;use std::borrow::Cow;@@ -1049,6 +1050,99 @@ fn test_comment_triggered_injection_toggle(cx: &mut App) {);}+#[gpui::test]+fn test_injection_grouped_by_host(cx: &mut App) {+ // Each SQL layer is described by the buffer text of the fragments it is parsed from,+ // so that interpolated strings are asserted to produce a single layer covering every+ // fragment of the string.+ let cases: &[(&str, &[&[&str]])] = &[+ (+ indoc! {r#"+ # sql+ cmd = "SELECT col1, col2 FROM tbl"+ "#},+ &[&["SELECT col1, col2 FROM tbl"]],+ ),+ (+ indoc! {r#"+ # sql+ cmd = f"SELECT col1 FROM tbl WHERE col2 = '{my_var}'"+ "#},+ &[&["SELECT col1 FROM tbl WHERE col2 = '", "'"]],+ ),+ (+ indoc! {r#"+ # sql+ cmd = f"SELECT {col1}, {col2} FROM {tbl}"+ "#},+ &[&["SELECT ", ", ", " FROM "]],+ ),+ (+ indoc! {r#"+ cursor.execute(+ # sql+ f"SELECT col1 FROM tbl WHERE col2 = '{my_var}'"+ )+ "#},+ &[&["SELECT col1 FROM tbl WHERE col2 = '", "'"]],+ ),+ (+ indoc! {r#"+ cursor.execute(+ # sql+ f"SELECT col1 FROM tbl WHERE col2 = '{my_var}'"+ )++ cursor.execute(+ # sql+ f"INSERT INTO tbl VALUES ('{val1}')"+ )+ "#},+ &[+ &["SELECT col1 FROM tbl WHERE col2 = '", "'"],+ &["INSERT INTO tbl VALUES ('", "')"],+ ],+ ),+ ];++ let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));+ let python = Arc::new(python_lang());+ registry.add(python.clone());+ registry.add(Arc::new(comment_lang()));+ registry.add(Arc::new(sql_lang()));++ for (buffer_index, (text, expected_layers)) in cases.iter().enumerate() {+ let buffer = Buffer::new(+ ReplicaId::LOCAL,+ BufferId::new(buffer_index as u64 + 1).unwrap(),+ text.to_string(),+ );+ let mut syntax_map = SyntaxMap::new(&buffer);+ syntax_map.set_language_registry(registry.clone());+ syntax_map.reparse(python.clone(), &buffer);++ let sql_layers = syntax_map+ .layers_for_range(0..buffer.len(), &buffer, true)+ .filter(|layer| layer.language.name() == "SQL")+ .map(|layer| match layer.included_sub_ranges {+ Some(sub_ranges) => sub_ranges+ .iter()+ .map(|range| buffer.text_for_range(range.clone()).collect::<String>())+ .collect::<Vec<_>>(),+ None => {+ vec![+ buffer+ .text_for_range(layer.node().byte_range())+ .collect::<String>(),+ ]+ }+ })+ .collect::<Vec<_>>();++ assert_eq!(sql_layers, *expected_layers, "buffer text:\n{text}");+ }+}+#[gpui::test]fn test_syntax_map_languages_loading_with_erb(cx: &mut App) {let text = r#"@@ -1188,6 +1282,32 @@ fn test_random_syntax_map_edits_with_erb(rng: StdRng, cx: &mut App) {test_random_edits(text, registry, language, rng);}+#[gpui::test(iterations = 50)]+fn test_random_syntax_map_edits_with_python_sql(rng: StdRng, cx: &mut App) {+ let text = r#"+ # sql+ query = f"SELECT {col} FROM tbl WHERE name = '{name}' AND id = {id}"++ cursor.execute(+ # sql+ f"""+ INSERT INTO tbl (col1, col2)+ VALUES ('{one}', '{two}')+ """+ )+ "#+ .unindent()+ .repeat(5);++ let registry = Arc::new(LanguageRegistry::test(cx.background_executor().clone()));+ let language = Arc::new(python_lang());+ registry.add(language.clone());+ registry.add(Arc::new(comment_lang()));+ registry.add(Arc::new(sql_lang()));++ test_random_edits(text, registry, language, rng);+}+#[gpui::test(iterations = 50)]fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) {let text = r#"@@ -1630,6 +1750,18 @@ fn comment_lang() -> Language {)}+/// Stands in for the real SQL language, which this crate doesn't depend on. The grammar+/// is irrelevant here; only the layers Python's injection query produces are asserted on.+fn sql_lang() -> Language {+ Language::new(+ LanguageConfig {+ name: "SQL".into(),+ ..Default::default()+ },+ Some(tree_sitter_json::LANGUAGE.into()),+ )+}+fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {let start = buffer.as_rope().to_string().find(text).unwrap();start..start + text.len()diff --git a/crates/language_core/src/grammar.rs b/crates/language_core/src/grammar.rsindex 42c398d3cd0..ff4dc0c8850 100644--- a/crates/language_core/src/grammar.rs+++ b/crates/language_core/src/grammar.rs@@ -183,6 +183,7 @@ pub struct InjectionConfig {pub query: Query,pub content_capture_ix: u32,pub language_capture_ix: Option<u32>,+ pub host_capture_ix: Option<u32>,pub patterns: Vec<InjectionPatternConfig>,}@@ -687,6 +688,7 @@ impl Grammar {let mut injection_language_capture_ix = None;let mut content_capture_ix = None;let mut injection_content_capture_ix = None;+ let mut host_capture_ix = None;if populate_capture_indices(&query,language_name,@@ -697,6 +699,7 @@ impl Grammar {Capture::Optional("injection.language", &mut injection_language_capture_ix),Capture::Optional("content", &mut content_capture_ix),Capture::Optional("injection.content", &mut injection_content_capture_ix),+ Capture::Optional("injection.host", &mut host_capture_ix),],) {language_capture_ix = match (language_capture_ix, injection_language_capture_ix) {@@ -735,6 +738,7 @@ impl Grammar {query,language_capture_ix,content_capture_ix,+ host_capture_ix,patterns,});} else {