Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

feat(editor): Tree-sitter highlighting for Zig, Nix, Make, GraphQL and HCL

l0ng-aifde075457bdf
diff --git a/crates/ide-language/queries/graphql/highlights.scm b/crates/ide-language/queries/graphql/highlights.scm
new file mode 100644
index 000000000..bcea7da94
--- /dev/null
+++ b/crates/ide-language/queries/graphql/highlights.scm
@@ -0,0 +1,49 @@
+; GraphQL highlighting (the grammar crate ships no query). Later patterns win.
+
+(comment) @comment
+(description (string_value) @comment.doc)
+(string_value) @string
+(int_value) @number
+(float_value) @number
+[(boolean_value) (null_value)] @keyword
+(enum_value (name) @constant)
+
+(variable) @variable
+(argument (name) @variable.parameter)
+(input_value_definition (name) @variable.parameter)
+(object_field (name) @property)
+
+(field (name) @property)
+(alias (name) @property)
+(field_definition (name) @property)
+(enum_value_definition (enum_value (name) @constant))
+
+(named_type (name) @type)
+(object_type_definition (name) @type)
+(interface_type_definition (name) @type)
+(union_type_definition (name) @type)
+(enum_type_definition (name) @type)
+(input_object_type_definition (name) @type)
+(scalar_type_definition (name) @type)
+(object_type_extension (name) @type)
+(interface_type_extension (name) @type)
+(union_type_extension (name) @type)
+(enum_type_extension (name) @type)
+(input_object_type_extension (name) @type)
+(scalar_type_extension (name) @type)
+
+(operation_definition (name) @function)
+(fragment_name (name) @function)
+(directive_definition (name) @attribute)
+(directive (name) @attribute)
+"@" @attribute
+(directive_location) @constant
+
+(operation_type) @keyword
+[
+ "query" "mutation" "subscription" "fragment" "on" "type" "interface" "union"
+ "enum" "input" "scalar" "schema" "extend" "implements" "directive" "repeatable"
+] @keyword
+
+["(" ")" "[" "]" "{" "}"] @punctuation.bracket
+[":" "=" "|" "&" "!" "..."] @punctuation.delimiter
diff --git a/crates/ide-language/queries/hcl/highlights.scm b/crates/ide-language/queries/hcl/highlights.scm
new file mode 100644
index 000000000..372f3b1da
--- /dev/null
+++ b/crates/ide-language/queries/hcl/highlights.scm
@@ -0,0 +1,29 @@
+; HCL / Terraform highlighting (the grammar crate ships no query). Later patterns win.
+
+(comment) @comment
+(string_lit) @string
+(quoted_template) @string
+(heredoc_template) @string
+[(heredoc_start) (heredoc_identifier)] @string.special
+(template_literal) @string
+(numeric_lit) @number
+[(bool_lit) (null_lit)] @keyword
+
+(identifier) @variable
+(attribute (identifier) @property)
+(object_elem key: (expression (variable_expr (identifier) @property)))
+(get_attr (identifier) @property)
+(function_call (identifier) @function)
+; `resource "aws_instance" "web" {`
+(block (identifier) @keyword)
+(block (string_lit (template_literal) @type))
+
+(template_interpolation) @embedded
+[(template_interpolation_start) (template_interpolation_end)] @punctuation.special
+[(template_directive_start) (template_directive_end) (strip_marker)] @punctuation.special
+["for" "in" "if" "else" "endfor" "endif"] @keyword
+
+(operation) @operator
+["!" "!=" "%" "&&" "*" "+" "-" "/" "<" "<=" "==" ">" ">=" "||" "?" "=>"] @operator
+["=" ":" "," "."] @punctuation.delimiter
+["(" ")" "[" "]" "{" "}" "[*]" ".*"] @punctuation.bracket
diff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rs
index fad2d5c51..2045bde4a 100644
--- a/crates/ide-language/src/syntax/grammars.rs
+++ b/crates/ide-language/src/syntax/grammars.rs
@@ -10,7 +10,7 @@ pub(crate) struct Spec {
/// `queries/<name>/outline.scm`; see `queries/README.md` for the capture vocabulary.
pub outline: &'static str,
/// The grammar's `highlights.scm`, for languages without a CodeMirror mode.
- pub highlights: Option<&'static str>,
+ pub highlights: Option<Highlights>,
/// Node kinds folded when they span several lines (in addition to every
/// bracket-delimited node, which is folded generically).
pub fold_nodes: &'static [&'static str],
@@ -22,6 +22,21 @@ pub(crate) struct Spec {
pub constructors: &'static [&'static str],
}
+/// A highlights query and how its patterns rank against each other.
+pub(crate) struct Highlights {
+ pub query: &'static str,
+ /// For the same node, the first matching pattern wins (tree-sitter CLI /
+ /// GitHub queries) instead of the last one (nvim / Zed queries).
+ pub first_wins: bool,
+}
+
+/// Our own `queries/<name>/highlights.scm`, for grammars that ship none.
+macro_rules! highlights {
+ ($name:literal) => {
+ include_str!(concat!("../../queries/", $name, "/highlights.scm"))
+ };
+}
+
macro_rules! outline {
($name:literal) => {
include_str!(concat!("../../queries/", $name, "/outline.scm"))
@@ -240,7 +255,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "zig",
language: tree_sitter_zig::LANGUAGE,
outline: outline!("zig"),
- highlights: None,
+ highlights: Some(Highlights { query: tree_sitter_zig::HIGHLIGHTS_QUERY, first_wins: false }),
fold_nodes: NONE,
fold_bodies: NONE,
imports: NONE,
@@ -250,7 +265,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "elixir",
language: tree_sitter_elixir::LANGUAGE,
outline: outline!("elixir"),
- highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY),
+ highlights: Some(Highlights { query: tree_sitter_elixir::HIGHLIGHTS_QUERY, first_wins: false }),
fold_nodes: &["do_block", "anonymous_function", "string", "sigil"],
fold_bodies: NONE,
imports: NONE,
@@ -379,7 +394,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "hcl",
language: tree_sitter_hcl::LANGUAGE,
outline: outline!("hcl"),
- highlights: None,
+ highlights: Some(Highlights { query: highlights!("hcl"), first_wins: false }),
fold_nodes: &["block", "heredoc_template"],
fold_bodies: NONE,
imports: NONE,
@@ -389,7 +404,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "nix",
language: tree_sitter_nix::LANGUAGE,
outline: outline!("nix"),
- highlights: None,
+ highlights: Some(Highlights { query: tree_sitter_nix::HIGHLIGHTS_QUERY, first_wins: true }),
fold_nodes: &["let_expression", "indented_string_expression"],
fold_bodies: NONE,
imports: NONE,
@@ -409,7 +424,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "graphql",
language: tree_sitter_graphql::LANGUAGE,
outline: outline!("graphql"),
- highlights: None,
+ highlights: Some(Highlights { query: highlights!("graphql"), first_wins: false }),
fold_nodes: NONE,
fold_bodies: NONE,
imports: NONE,
@@ -419,7 +434,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "make",
language: tree_sitter_make::LANGUAGE,
outline: outline!("make"),
- highlights: None,
+ highlights: Some(Highlights { query: tree_sitter_make::HIGHLIGHTS_QUERY, first_wins: false }),
fold_nodes: &["rule", "conditional", "define_directive"],
fold_bodies: NONE,
imports: &["include_directive"],
diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rs
index 629c081ce..2daab82fd 100644
--- a/crates/ide-language/src/syntax/highlight.rs
+++ b/crates/ide-language/src/syntax/highlight.rs
@@ -5,13 +5,14 @@
//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by
//! their longest known dotted prefix. Overlaps resolve the way those editors
//! do: a smaller node wins over the node containing it, and for the same node
-//! the later pattern wins. `@embedded` (string interpolation) clears the
-//! surrounding colour.
+//! the later pattern wins (the first one for tree-sitter CLI style queries). `@embedded` (string interpolation) clears the
+//! surrounding colour. Besides tree-sitter's own text predicates, nvim's
+//! `#lua-match?` is understood; patterns with other predicates never match.
use super::lines::LineIndex;
use serde::{Deserialize, Serialize};
use specta::Type;
-use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree};
+use tree_sitter::{Language, Query, QueryCursor, QueryMatch, QueryPredicateArg, StreamingIterator, Tree};
/// Highlight classes sent to the UI, which maps each onto a Lezer tag.
#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
@@ -169,36 +170,135 @@ fn paint_for(name: &str) -> Option<Paint> {
}
}
+/// A predicate tree-sitter leaves to the caller.
+enum Check {
+ /// `#lua-match?` / `#not-lua-match?` on a capture's text.
+ Matches {
+ capture: u32,
+ regex: regex_lite::Regex,
+ negate: bool,
+ },
+ Unsupported,
+}
+
pub(crate) struct HighlightQuery {
query: Query,
/// By capture index.
paints: Vec<Option<Paint>>,
+ /// By pattern index.
+ checks: Vec<Vec<Check>>,
+ first_wins: bool,
}
impl HighlightQuery {
- pub(crate) fn new(language: &Language, source: &str) -> Result<Self, tree_sitter::QueryError> {
+ pub(crate) fn new(language: &Language, source: &str, first_wins: bool) -> Result<Self, tree_sitter::QueryError> {
let query = Query::new(language, source)?;
let paints = query.capture_names().iter().map(|n| paint_for(n)).collect();
- Ok(Self { query, paints })
+ let checks = (0..query.pattern_count())
+ .map(|i| {
+ query
+ .general_predicates(i)
+ .iter()
+ .filter_map(|p| match (&*p.operator, &*p.args) {
+ // Locals are not tracked: builtins are assumed not to be shadowed.
+ ("is-not?", [QueryPredicateArg::String(s)]) if &**s == "local" => None,
+ (
+ op @ ("lua-match?" | "not-lua-match?"),
+ [QueryPredicateArg::Capture(c), QueryPredicateArg::String(pat)],
+ ) => Some(match regex_lite::Regex::new(&lua_pattern_to_regex(pat)) {
+ Ok(regex) => Check::Matches { capture: *c, regex, negate: op.starts_with("not-") },
+ Err(_) => Check::Unsupported,
+ }),
+ _ => Some(Check::Unsupported),
+ })
+ .collect()
+ })
+ .collect();
+ Ok(Self { query, paints, checks, first_wins })
+ }
+
+ fn accepts(&self, m: &QueryMatch<'_, '_>, text: &str) -> bool {
+ self.checks[m.pattern_index].iter().all(|check| match check {
+ Check::Matches { capture, regex, negate } => m
+ .captures
+ .iter()
+ .filter(|c| c.index == *capture)
+ .all(|c| regex.is_match(&text[c.node.byte_range()]) != *negate),
+ Check::Unsupported => false,
+ })
}
}
+/// Translates the Lua patterns used by nvim queries (`^[A-Z][A-Z_0-9]+$`, `^//!`) to regex syntax.
+fn lua_pattern_to_regex(pat: &str) -> String {
+ let class = |c: char| match c {
+ 'a' => Some("A-Za-z"),
+ 'd' => Some("0-9"),
+ 'l' => Some("a-z"),
+ 'u' => Some("A-Z"),
+ 'w' => Some("A-Za-z0-9"),
+ 's' => Some("\\s"),
+ 'x' => Some("0-9A-Fa-f"),
+ 'p' => Some("[:punct:]"),
+ _ => None,
+ };
+ let mut out = String::with_capacity(pat.len() + 8);
+ let mut in_set = false;
+ let mut chars = pat.chars();
+ while let Some(c) = chars.next() {
+ match c {
+ '%' => {
+ let Some(next) = chars.next() else { break };
+ match class(next) {
+ Some(set) if in_set => out.push_str(set),
+ Some(set) => {
+ out.push('[');
+ out.push_str(set);
+ out.push(']');
+ }
+ None => out.push_str(&regex_lite::escape(&next.to_string())),
+ }
+ }
+ '[' if !in_set => {
+ in_set = true;
+ out.push(c);
+ }
+ ']' if in_set => {
+ in_set = false;
+ out.push(c);
+ }
+ // Lua's lazy `*` (inside a set, `-` is a range).
+ '-' if !in_set => out.push_str("*?"),
+ '\\' | '{' | '}' | '|' => {
+ out.push('\\');
+ out.push(c);
+ }
+ _ => out.push(c),
+ }
+ }
+ out
+}
+
/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`.
const NONE: u8 = 0;
/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`].
pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec<u32> {
- // (byte len, pattern, start, end, paint): painted largest first, then in
- // pattern order, so inner nodes and later patterns overwrite.
+ // (byte len, rank, start, end, paint): painted largest first, then by rank,
+ // so inner nodes and winning patterns overwrite.
let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new();
let mut cursor = QueryCursor::new();
let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes());
while let Some(m) = matches.next() {
+ if !q.accepts(m, text) {
+ continue;
+ }
for c in m.captures {
if let Some(paint) = q.paints[c.index as usize] {
let r = c.node.byte_range();
if r.start < r.end {
- captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint));
+ let rank = if q.first_wins { usize::MAX - m.pattern_index } else { m.pattern_index };
+ captures.push((r.end - r.start, rank, r.start, r.end, paint));
}
}
}
@@ -282,11 +382,98 @@ mod tests {
assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0]));
}
+ #[test]
+ fn highlights_other_grammar_only_languages() {
+ use SyntaxHighlightKind as K;
+ type Expect<'a> = &'a [(&'a str, SyntaxHighlightKind)];
+ let cases: &[(&str, &str, Expect)] = &[
+ (
+ "zig",
+ "const std = @import(\"std\");\n// hi\npub fn main() void {\n var x: u32 = 42;\n _ = x;\n}\n",
+ &[
+ ("const", K::Keyword),
+ ("\"std\"", K::String),
+ ("// hi", K::Comment),
+ ("42", K::Number),
+ ("u32", K::Builtin),
+ ],
+ ),
+ (
+ "nix",
+ "{ pkgs ? import <nixpkgs> {} }:\n# hi\nlet x = 1; in pkgs.mkShell { name = \"dev\"; }\n",
+ &[
+ ("let", K::Keyword),
+ ("# hi", K::Comment),
+ ("1", K::Number),
+ ("\"dev\"", K::String),
+ ("import", K::Builtin),
+ ],
+ ),
+ (
+ "graphql",
+ "# hi\nquery Hero($id: ID!) @cached {\n hero(id: $id) { name ...Friends }\n}\nenum Episode { NEWHOPE }\n",
+ &[
+ ("# hi", K::Comment),
+ ("query", K::Keyword),
+ ("Hero", K::FunctionCall),
+ ("$id", K::Variable),
+ ("ID", K::Type),
+ ("@cached", K::Attribute),
+ ("name", K::Property),
+ ("Friends", K::FunctionCall),
+ ("NEWHOPE", K::Constant),
+ ],
+ ),
+ (
+ "hcl",
+ "# hi\nresource \"aws_instance\" \"web\" {\n ami = var.ami\n count = 2\n tags = { Name = \"web-${var.env}\" }\n name = upper(\"x\")\n}\n",
+ &[
+ ("# hi", K::Comment),
+ ("resource", K::Keyword),
+ ("aws_instance", K::Type),
+ ("ami", K::Property),
+ ("2", K::Number),
+ ("upper", K::FunctionCall),
+ ("\"web-", K::String),
+ ("env", K::Property),
+ ("var", K::Variable),
+ ],
+ ),
+ ("make", "# hi\nCC := gcc\nall: main.o\n\t$(CC) -o app main.o\n", &[("# hi", K::Comment)]),
+ ];
+ for (lang, src, expect) in cases {
+ let got = spans(lang, src);
+ for (text, kind) in *expect {
+ assert!(got.iter().any(|(s, k)| s == text && k == kind), "{lang}: {text:?} as {kind:?} in {got:?}");
+ }
+ }
+ }
+
#[test]
fn languages_without_a_query_have_none() {
assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None);
}
+ #[test]
+ fn every_highlights_query_compiles() {
+ for spec in super::super::grammars::SPECS {
+ if let Some(h) = &spec.highlights {
+ let lang = Language::new(spec.language);
+ if let Err(e) = HighlightQuery::new(&lang, h.query, h.first_wins) {
+ panic!("{}: {e}", spec.name);
+ }
+ }
+ }
+ }
+
+ #[test]
+ fn lua_patterns_translate() {
+ assert_eq!(lua_pattern_to_regex("^[A-Z][A-Z_0-9]+$"), "^[A-Z][A-Z_0-9]+$");
+ assert_eq!(lua_pattern_to_regex("^//!"), "^//!");
+ assert_eq!(lua_pattern_to_regex("^%u[%w_]*$"), "^[A-Z][A-Za-z0-9_]*$");
+ assert_eq!(lua_pattern_to_regex("a.-b%."), "a.*?b\\.");
+ }
+
#[test]
fn legend_matches_discriminants() {
for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() {
diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rs
index 783dfe9c4..fed5514fe 100644
--- a/crates/ide-language/src/syntax/mod.rs
+++ b/crates/ide-language/src/syntax/mod.rs
@@ -57,8 +57,8 @@ impl Grammar {
fn highlight_query(&self) -> Option<&highlight::HighlightQuery> {
self.highlights
.get_or_init(|| {
- let source = self.spec.highlights?;
- highlight::HighlightQuery::new(&self.language, source)
+ let h = self.spec.highlights.as_ref()?;
+ highlight::HighlightQuery::new(&self.language, h.query, h.first_wins)
.inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name))
.ok()
})