The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.
TypeScript code-editorcross-platformdapdatabase-clientdebuggerdesktop-appdeveloper-toolsgithttp-clientideintellijlsprusttauriterminal
feat(editor): Tree-sitter highlighting for Zig, Nix, Make, GraphQL and HCL
l0ng-aifde075457bdf
diff --git a/crates/ide-language/queries/graphql/highlights.scm b/crates/ide-language/queries/graphql/highlights.scmnew file mode 100644index 000000000..bcea7da94--- /dev/null+++ b/crates/ide-language/queries/graphql/highlights.scm@@ -0,0 +1,49 @@+; GraphQL highlighting (the grammar crate ships no query). Later patterns win.++(comment) @comment+(description (string_value) @comment.doc)+(string_value) @string+(int_value) @number+(float_value) @number+[(boolean_value) (null_value)] @keyword+(enum_value (name) @constant)++(variable) @variable+(argument (name) @variable.parameter)+(input_value_definition (name) @variable.parameter)+(object_field (name) @property)++(field (name) @property)+(alias (name) @property)+(field_definition (name) @property)+(enum_value_definition (enum_value (name) @constant))++(named_type (name) @type)+(object_type_definition (name) @type)+(interface_type_definition (name) @type)+(union_type_definition (name) @type)+(enum_type_definition (name) @type)+(input_object_type_definition (name) @type)+(scalar_type_definition (name) @type)+(object_type_extension (name) @type)+(interface_type_extension (name) @type)+(union_type_extension (name) @type)+(enum_type_extension (name) @type)+(input_object_type_extension (name) @type)+(scalar_type_extension (name) @type)++(operation_definition (name) @function)+(fragment_name (name) @function)+(directive_definition (name) @attribute)+(directive (name) @attribute)+"@" @attribute+(directive_location) @constant++(operation_type) @keyword+[+ "query" "mutation" "subscription" "fragment" "on" "type" "interface" "union"+ "enum" "input" "scalar" "schema" "extend" "implements" "directive" "repeatable"+] @keyword++["(" ")" "[" "]" "{" "}"] @punctuation.bracket+[":" "=" "|" "&" "!" "..."] @punctuation.delimiterdiff --git a/crates/ide-language/queries/hcl/highlights.scm b/crates/ide-language/queries/hcl/highlights.scmnew file mode 100644index 000000000..372f3b1da--- /dev/null+++ b/crates/ide-language/queries/hcl/highlights.scm@@ -0,0 +1,29 @@+; HCL / Terraform highlighting (the grammar crate ships no query). Later patterns win.++(comment) @comment+(string_lit) @string+(quoted_template) @string+(heredoc_template) @string+[(heredoc_start) (heredoc_identifier)] @string.special+(template_literal) @string+(numeric_lit) @number+[(bool_lit) (null_lit)] @keyword++(identifier) @variable+(attribute (identifier) @property)+(object_elem key: (expression (variable_expr (identifier) @property)))+(get_attr (identifier) @property)+(function_call (identifier) @function)+; `resource "aws_instance" "web" {`+(block (identifier) @keyword)+(block (string_lit (template_literal) @type))++(template_interpolation) @embedded+[(template_interpolation_start) (template_interpolation_end)] @punctuation.special+[(template_directive_start) (template_directive_end) (strip_marker)] @punctuation.special+["for" "in" "if" "else" "endfor" "endif"] @keyword++(operation) @operator+["!" "!=" "%" "&&" "*" "+" "-" "/" "<" "<=" "==" ">" ">=" "||" "?" "=>"] @operator+["=" ":" "," "."] @punctuation.delimiter+["(" ")" "[" "]" "{" "}" "[*]" ".*"] @punctuation.bracketdiff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rsindex fad2d5c51..2045bde4a 100644--- a/crates/ide-language/src/syntax/grammars.rs+++ b/crates/ide-language/src/syntax/grammars.rs@@ -10,7 +10,7 @@ pub(crate) struct Spec {/// `queries/<name>/outline.scm`; see `queries/README.md` for the capture vocabulary.pub outline: &'static str,/// The grammar's `highlights.scm`, for languages without a CodeMirror mode.- pub highlights: Option<&'static str>,+ pub highlights: Option<Highlights>,/// Node kinds folded when they span several lines (in addition to every/// bracket-delimited node, which is folded generically).pub fold_nodes: &'static [&'static str],@@ -22,6 +22,21 @@ pub(crate) struct Spec {pub constructors: &'static [&'static str],}+/// A highlights query and how its patterns rank against each other.+pub(crate) struct Highlights {+ pub query: &'static str,+ /// For the same node, the first matching pattern wins (tree-sitter CLI /+ /// GitHub queries) instead of the last one (nvim / Zed queries).+ pub first_wins: bool,+}++/// Our own `queries/<name>/highlights.scm`, for grammars that ship none.+macro_rules! highlights {+ ($name:literal) => {+ include_str!(concat!("../../queries/", $name, "/highlights.scm"))+ };+}+macro_rules! outline {($name:literal) => {include_str!(concat!("../../queries/", $name, "/outline.scm"))@@ -240,7 +255,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "zig",language: tree_sitter_zig::LANGUAGE,outline: outline!("zig"),- highlights: None,+ highlights: Some(Highlights { query: tree_sitter_zig::HIGHLIGHTS_QUERY, first_wins: false }),fold_nodes: NONE,fold_bodies: NONE,imports: NONE,@@ -250,7 +265,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "elixir",language: tree_sitter_elixir::LANGUAGE,outline: outline!("elixir"),- highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY),+ highlights: Some(Highlights { query: tree_sitter_elixir::HIGHLIGHTS_QUERY, first_wins: false }),fold_nodes: &["do_block", "anonymous_function", "string", "sigil"],fold_bodies: NONE,imports: NONE,@@ -379,7 +394,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "hcl",language: tree_sitter_hcl::LANGUAGE,outline: outline!("hcl"),- highlights: None,+ highlights: Some(Highlights { query: highlights!("hcl"), first_wins: false }),fold_nodes: &["block", "heredoc_template"],fold_bodies: NONE,imports: NONE,@@ -389,7 +404,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "nix",language: tree_sitter_nix::LANGUAGE,outline: outline!("nix"),- highlights: None,+ highlights: Some(Highlights { query: tree_sitter_nix::HIGHLIGHTS_QUERY, first_wins: true }),fold_nodes: &["let_expression", "indented_string_expression"],fold_bodies: NONE,imports: NONE,@@ -409,7 +424,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "graphql",language: tree_sitter_graphql::LANGUAGE,outline: outline!("graphql"),- highlights: None,+ highlights: Some(Highlights { query: highlights!("graphql"), first_wins: false }),fold_nodes: NONE,fold_bodies: NONE,imports: NONE,@@ -419,7 +434,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "make",language: tree_sitter_make::LANGUAGE,outline: outline!("make"),- highlights: None,+ highlights: Some(Highlights { query: tree_sitter_make::HIGHLIGHTS_QUERY, first_wins: false }),fold_nodes: &["rule", "conditional", "define_directive"],fold_bodies: NONE,imports: &["include_directive"],diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rsindex 629c081ce..2daab82fd 100644--- a/crates/ide-language/src/syntax/highlight.rs+++ b/crates/ide-language/src/syntax/highlight.rs@@ -5,13 +5,14 @@//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by//! their longest known dotted prefix. Overlaps resolve the way those editors//! do: a smaller node wins over the node containing it, and for the same node-//! the later pattern wins. `@embedded` (string interpolation) clears the-//! surrounding colour.+//! the later pattern wins (the first one for tree-sitter CLI style queries). `@embedded` (string interpolation) clears the+//! surrounding colour. Besides tree-sitter's own text predicates, nvim's+//! `#lua-match?` is understood; patterns with other predicates never match.use super::lines::LineIndex;use serde::{Deserialize, Serialize};use specta::Type;-use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree};+use tree_sitter::{Language, Query, QueryCursor, QueryMatch, QueryPredicateArg, StreamingIterator, Tree};/// Highlight classes sent to the UI, which maps each onto a Lezer tag.#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]@@ -169,36 +170,135 @@ fn paint_for(name: &str) -> Option<Paint> {}}+/// A predicate tree-sitter leaves to the caller.+enum Check {+ /// `#lua-match?` / `#not-lua-match?` on a capture's text.+ Matches {+ capture: u32,+ regex: regex_lite::Regex,+ negate: bool,+ },+ Unsupported,+}+pub(crate) struct HighlightQuery {query: Query,/// By capture index.paints: Vec<Option<Paint>>,+ /// By pattern index.+ checks: Vec<Vec<Check>>,+ first_wins: bool,}impl HighlightQuery {- pub(crate) fn new(language: &Language, source: &str) -> Result<Self, tree_sitter::QueryError> {+ pub(crate) fn new(language: &Language, source: &str, first_wins: bool) -> Result<Self, tree_sitter::QueryError> {let query = Query::new(language, source)?;let paints = query.capture_names().iter().map(|n| paint_for(n)).collect();- Ok(Self { query, paints })+ let checks = (0..query.pattern_count())+ .map(|i| {+ query+ .general_predicates(i)+ .iter()+ .filter_map(|p| match (&*p.operator, &*p.args) {+ // Locals are not tracked: builtins are assumed not to be shadowed.+ ("is-not?", [QueryPredicateArg::String(s)]) if &**s == "local" => None,+ (+ op @ ("lua-match?" | "not-lua-match?"),+ [QueryPredicateArg::Capture(c), QueryPredicateArg::String(pat)],+ ) => Some(match regex_lite::Regex::new(&lua_pattern_to_regex(pat)) {+ Ok(regex) => Check::Matches { capture: *c, regex, negate: op.starts_with("not-") },+ Err(_) => Check::Unsupported,+ }),+ _ => Some(Check::Unsupported),+ })+ .collect()+ })+ .collect();+ Ok(Self { query, paints, checks, first_wins })+ }++ fn accepts(&self, m: &QueryMatch<'_, '_>, text: &str) -> bool {+ self.checks[m.pattern_index].iter().all(|check| match check {+ Check::Matches { capture, regex, negate } => m+ .captures+ .iter()+ .filter(|c| c.index == *capture)+ .all(|c| regex.is_match(&text[c.node.byte_range()]) != *negate),+ Check::Unsupported => false,+ })}}+/// Translates the Lua patterns used by nvim queries (`^[A-Z][A-Z_0-9]+$`, `^//!`) to regex syntax.+fn lua_pattern_to_regex(pat: &str) -> String {+ let class = |c: char| match c {+ 'a' => Some("A-Za-z"),+ 'd' => Some("0-9"),+ 'l' => Some("a-z"),+ 'u' => Some("A-Z"),+ 'w' => Some("A-Za-z0-9"),+ 's' => Some("\\s"),+ 'x' => Some("0-9A-Fa-f"),+ 'p' => Some("[:punct:]"),+ _ => None,+ };+ let mut out = String::with_capacity(pat.len() + 8);+ let mut in_set = false;+ let mut chars = pat.chars();+ while let Some(c) = chars.next() {+ match c {+ '%' => {+ let Some(next) = chars.next() else { break };+ match class(next) {+ Some(set) if in_set => out.push_str(set),+ Some(set) => {+ out.push('[');+ out.push_str(set);+ out.push(']');+ }+ None => out.push_str(®ex_lite::escape(&next.to_string())),+ }+ }+ '[' if !in_set => {+ in_set = true;+ out.push(c);+ }+ ']' if in_set => {+ in_set = false;+ out.push(c);+ }+ // Lua's lazy `*` (inside a set, `-` is a range).+ '-' if !in_set => out.push_str("*?"),+ '\\' | '{' | '}' | '|' => {+ out.push('\\');+ out.push(c);+ }+ _ => out.push(c),+ }+ }+ out+}+/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`.const NONE: u8 = 0;/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`].pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec<u32> {- // (byte len, pattern, start, end, paint): painted largest first, then in- // pattern order, so inner nodes and later patterns overwrite.+ // (byte len, rank, start, end, paint): painted largest first, then by rank,+ // so inner nodes and winning patterns overwrite.let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new();let mut cursor = QueryCursor::new();let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes());while let Some(m) = matches.next() {+ if !q.accepts(m, text) {+ continue;+ }for c in m.captures {if let Some(paint) = q.paints[c.index as usize] {let r = c.node.byte_range();if r.start < r.end {- captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint));+ let rank = if q.first_wins { usize::MAX - m.pattern_index } else { m.pattern_index };+ captures.push((r.end - r.start, rank, r.start, r.end, paint));}}}@@ -282,11 +382,98 @@ mod tests {assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0]));}+ #[test]+ fn highlights_other_grammar_only_languages() {+ use SyntaxHighlightKind as K;+ type Expect<'a> = &'a [(&'a str, SyntaxHighlightKind)];+ let cases: &[(&str, &str, Expect)] = &[+ (+ "zig",+ "const std = @import(\"std\");\n// hi\npub fn main() void {\n var x: u32 = 42;\n _ = x;\n}\n",+ &[+ ("const", K::Keyword),+ ("\"std\"", K::String),+ ("// hi", K::Comment),+ ("42", K::Number),+ ("u32", K::Builtin),+ ],+ ),+ (+ "nix",+ "{ pkgs ? import <nixpkgs> {} }:\n# hi\nlet x = 1; in pkgs.mkShell { name = \"dev\"; }\n",+ &[+ ("let", K::Keyword),+ ("# hi", K::Comment),+ ("1", K::Number),+ ("\"dev\"", K::String),+ ("import", K::Builtin),+ ],+ ),+ (+ "graphql",+ "# hi\nquery Hero($id: ID!) @cached {\n hero(id: $id) { name ...Friends }\n}\nenum Episode { NEWHOPE }\n",+ &[+ ("# hi", K::Comment),+ ("query", K::Keyword),+ ("Hero", K::FunctionCall),+ ("$id", K::Variable),+ ("ID", K::Type),+ ("@cached", K::Attribute),+ ("name", K::Property),+ ("Friends", K::FunctionCall),+ ("NEWHOPE", K::Constant),+ ],+ ),+ (+ "hcl",+ "# hi\nresource \"aws_instance\" \"web\" {\n ami = var.ami\n count = 2\n tags = { Name = \"web-${var.env}\" }\n name = upper(\"x\")\n}\n",+ &[+ ("# hi", K::Comment),+ ("resource", K::Keyword),+ ("aws_instance", K::Type),+ ("ami", K::Property),+ ("2", K::Number),+ ("upper", K::FunctionCall),+ ("\"web-", K::String),+ ("env", K::Property),+ ("var", K::Variable),+ ],+ ),+ ("make", "# hi\nCC := gcc\nall: main.o\n\t$(CC) -o app main.o\n", &[("# hi", K::Comment)]),+ ];+ for (lang, src, expect) in cases {+ let got = spans(lang, src);+ for (text, kind) in *expect {+ assert!(got.iter().any(|(s, k)| s == text && k == kind), "{lang}: {text:?} as {kind:?} in {got:?}");+ }+ }+ }+#[test]fn languages_without_a_query_have_none() {assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None);}+ #[test]+ fn every_highlights_query_compiles() {+ for spec in super::super::grammars::SPECS {+ if let Some(h) = &spec.highlights {+ let lang = Language::new(spec.language);+ if let Err(e) = HighlightQuery::new(&lang, h.query, h.first_wins) {+ panic!("{}: {e}", spec.name);+ }+ }+ }+ }++ #[test]+ fn lua_patterns_translate() {+ assert_eq!(lua_pattern_to_regex("^[A-Z][A-Z_0-9]+$"), "^[A-Z][A-Z_0-9]+$");+ assert_eq!(lua_pattern_to_regex("^//!"), "^//!");+ assert_eq!(lua_pattern_to_regex("^%u[%w_]*$"), "^[A-Z][A-Za-z0-9_]*$");+ assert_eq!(lua_pattern_to_regex("a.-b%."), "a.*?b\\.");+ }+#[test]fn legend_matches_discriminants() {for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() {diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rsindex 783dfe9c4..fed5514fe 100644--- a/crates/ide-language/src/syntax/mod.rs+++ b/crates/ide-language/src/syntax/mod.rs@@ -57,8 +57,8 @@ impl Grammar {fn highlight_query(&self) -> Option<&highlight::HighlightQuery> {self.highlights.get_or_init(|| {- let source = self.spec.highlights?;- highlight::HighlightQuery::new(&self.language, source)+ let h = self.spec.highlights.as_ref()?;+ highlight::HighlightQuery::new(&self.language, h.query, h.first_wins).inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name)).ok()})