The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.
TypeScript code-editorcross-platformdapdatabase-clientdebuggerdesktop-appdeveloper-toolsgithttp-clientideintellijlsprusttauriterminal
feat(editor): Highlight languages without a CodeMirror mode from tree-sitter
l0ng-ai1572fc36ea00
diff --git a/crates/ide-language/src/lib.rs b/crates/ide-language/src/lib.rsindex 67f736cde..97b1b3520 100644--- a/crates/ide-language/src/lib.rs+++ b/crates/ide-language/src/lib.rs@@ -12,7 +12,10 @@ use std::collections::BTreeMap;mod registry;mod syntax;pub use registry::LanguageRegistry;-pub use syntax::{MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, TreeError};+pub use syntax::{+ MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, SyntaxHighlightKind,+ SyntaxHighlights, TreeError,+};/// Parsed form of `languages/<id>.toml`. Backend-only; the UI gets [`LanguageInfo`].#[derive(Clone, Debug, Serialize, Deserialize)]diff --git a/crates/ide-language/src/syntax/cache.rs b/crates/ide-language/src/syntax/cache.rsindex fef3cb9be..0969fe7df 100644--- a/crates/ide-language/src/syntax/cache.rs+++ b/crates/ide-language/src/syntax/cache.rs@@ -5,7 +5,10 @@//! Structure / folding / Extend Selection on a large file cost an incremental//! parse instead of a full one.-use super::{Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, TreeError, folding, outline, parse_cancellable, selection};+use super::{+ Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxHighlights, TreeError, folding, highlight, outline,+ parse_cancellable, selection,+};use crate::FoldRange;use ide_core::{DocId, LanguageId, OutlineItem, Range};use ide_text::ByteEdit;@@ -59,6 +62,7 @@ struct Memo {key: Option<(u32, LanguageId)>,outline: Option<Arc<Vec<OutlineItem>>>,folding: Option<Arc<Vec<FoldRange>>>,+ highlights: Option<Arc<Vec<u32>>>,}impl Memo {@@ -211,6 +215,21 @@ impl SyntaxCache {)}+ /// Highlighting from the grammar's own query; empty without one.+ pub fn highlights(&self, doc: DocId, lang: &LanguageId, version: u32, text: &str) -> SyntaxHighlights {+ SyntaxHighlights::new(self.memoized(+ doc,+ lang,+ version,+ |m| &mut m.highlights,+ || {+ self.with_tree(doc, lang, version, text, |g, tree, text| {+ Some(highlight::highlights(g.highlight_query()?, tree, text))+ })+ },+ ))+ }+/// `compute()` once per document version; `None` (no grammar / query) is not memoized.fn memoized<T: Clone>(&self,diff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rsindex beb969ea7..fad2d5c51 100644--- a/crates/ide-language/src/syntax/grammars.rs+++ b/crates/ide-language/src/syntax/grammars.rs@@ -9,6 +9,8 @@ pub(crate) struct Spec {pub language: LanguageFn,/// `queries/<name>/outline.scm`; see `queries/README.md` for the capture vocabulary.pub outline: &'static str,+ /// The grammar's `highlights.scm`, for languages without a CodeMirror mode.+ pub highlights: Option<&'static str>,/// Node kinds folded when they span several lines (in addition to every/// bracket-delimited node, which is folded generically).pub fold_nodes: &'static [&'static str],@@ -33,6 +35,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "rust",language: tree_sitter_rust::LANGUAGE,outline: outline!("rust"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: &["use_declaration", "extern_crate_declaration"],@@ -42,6 +45,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "typescript",language: tree_sitter_typescript::LANGUAGE_TYPESCRIPT,outline: outline!("typescript"),+ highlights: None,fold_nodes: &["template_string"],fold_bodies: NONE,imports: &["import_statement"],@@ -51,6 +55,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "tsx",language: tree_sitter_typescript::LANGUAGE_TSX,outline: outline!("typescript"),+ highlights: None,fold_nodes: &["template_string", "jsx_element"],fold_bodies: NONE,imports: &["import_statement"],@@ -60,6 +65,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "javascript",language: tree_sitter_javascript::LANGUAGE,outline: outline!("javascript"),+ highlights: None,fold_nodes: &["template_string", "jsx_element"],fold_bodies: NONE,imports: &["import_statement"],@@ -69,6 +75,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "python",language: tree_sitter_python::LANGUAGE,outline: outline!("python"),+ highlights: None,fold_nodes: &["string"],fold_bodies: &["block"],imports: &["import_statement", "import_from_statement", "future_import_statement"],@@ -78,6 +85,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "go",language: tree_sitter_go::LANGUAGE,outline: outline!("go"),+ highlights: None,fold_nodes: &["raw_string_literal"],fold_bodies: NONE,imports: &["import_declaration"],@@ -87,6 +95,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "java",language: tree_sitter_java::LANGUAGE,outline: outline!("java"),+ highlights: None,fold_nodes: &["text_block"],fold_bodies: NONE,imports: &["import_declaration"],@@ -96,6 +105,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "kotlin",language: tree_sitter_kotlin_ng::LANGUAGE,outline: outline!("kotlin"),+ highlights: None,fold_nodes: &["string_literal"],fold_bodies: NONE,imports: &["import"],@@ -105,6 +115,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "scala",language: tree_sitter_scala::LANGUAGE,outline: outline!("scala"),+ highlights: None,fold_nodes: &["string"],fold_bodies: &["indented_block", "template_body"],imports: &["import_declaration"],@@ -114,6 +125,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "c",language: tree_sitter_c::LANGUAGE,outline: outline!("c"),+ highlights: None,fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def"],fold_bodies: NONE,imports: &["preproc_include"],@@ -123,6 +135,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "cpp",language: tree_sitter_cpp::LANGUAGE,outline: outline!("cpp"),+ highlights: None,fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def", "raw_string_literal"],fold_bodies: NONE,imports: &["preproc_include", "using_declaration"],@@ -132,6 +145,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "csharp",language: tree_sitter_c_sharp::LANGUAGE,outline: outline!("csharp"),+ highlights: None,fold_nodes: &["preproc_region", "raw_string_literal", "verbatim_string_literal"],fold_bodies: NONE,imports: &["using_directive"],@@ -141,6 +155,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "swift",language: tree_sitter_swift::LANGUAGE,outline: outline!("swift"),+ highlights: None,// Control-flow statements hold `{ statements }` without a block node of their own.fold_nodes: &["multi_line_string_literal",@@ -160,6 +175,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "ruby",language: tree_sitter_ruby::LANGUAGE,outline: outline!("ruby"),+ highlights: None,fold_nodes: &["method","singleton_method",@@ -185,6 +201,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "php",language: tree_sitter_php::LANGUAGE_PHP,outline: outline!("php"),+ highlights: None,fold_nodes: &["heredoc", "nowdoc"],fold_bodies: NONE,imports: &["namespace_use_declaration"],@@ -194,6 +211,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "lua",language: tree_sitter_lua::LANGUAGE,outline: outline!("lua"),+ highlights: None,fold_nodes: &["function_declaration","function_definition",@@ -212,6 +230,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "dart",language: tree_sitter_dart::LANGUAGE,outline: outline!("dart"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: &["import_or_export", "library_name", "part_directive", "part_of_directive"],@@ -221,6 +240,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "zig",language: tree_sitter_zig::LANGUAGE,outline: outline!("zig"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: NONE,@@ -230,6 +250,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "elixir",language: tree_sitter_elixir::LANGUAGE,outline: outline!("elixir"),+ highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY),fold_nodes: &["do_block", "anonymous_function", "string", "sigil"],fold_bodies: NONE,imports: NONE,@@ -239,6 +260,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "haskell",language: tree_sitter_haskell::LANGUAGE,outline: outline!("haskell"),+ highlights: None,// `bind`: equations without arguments (`main = do …`).fold_nodes: &["function", "bind", "data_type", "newtype", "class", "instance", "signature"],fold_bodies: NONE,@@ -249,6 +271,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "ocaml",language: tree_sitter_ocaml::LANGUAGE_OCAML,outline: outline!("ocaml"),+ highlights: None,fold_nodes: &["value_definition","type_definition",@@ -266,6 +289,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "ocaml_interface",language: tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE,outline: outline!("ocaml_interface"),+ highlights: None,fold_nodes: &["module_definition", "module_type_definition", "signature", "type_definition"],fold_bodies: NONE,imports: &["open_module"],@@ -275,6 +299,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "bash",language: tree_sitter_bash::LANGUAGE,outline: outline!("bash"),+ highlights: None,fold_nodes: &["do_group", "if_statement", "case_statement", "heredoc_body"],fold_bodies: NONE,imports: NONE,@@ -284,6 +309,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "html",language: tree_sitter_html::LANGUAGE,outline: outline!("html"),+ highlights: None,fold_nodes: &["element", "script_element", "style_element"],fold_bodies: NONE,imports: NONE,@@ -293,6 +319,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "xml",language: tree_sitter_xml::LANGUAGE_XML,outline: outline!("xml"),+ highlights: None,fold_nodes: &["element", "CDSect"],fold_bodies: NONE,imports: NONE,@@ -302,6 +329,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "css",language: tree_sitter_css::LANGUAGE,outline: outline!("css"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: &["import_statement"],@@ -311,6 +339,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "json",language: tree_sitter_json::LANGUAGE,outline: outline!("json"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: NONE,@@ -320,6 +349,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "yaml",language: tree_sitter_yaml::LANGUAGE,outline: outline!("yaml"),+ highlights: None,fold_nodes: &["block_mapping_pair", "block_sequence_item", "block_scalar"],fold_bodies: NONE,imports: NONE,@@ -329,6 +359,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "toml",language: tree_sitter_toml_ng::LANGUAGE,outline: outline!("toml"),+ highlights: None,fold_nodes: &["table", "table_array_element", "string"],fold_bodies: NONE,imports: NONE,@@ -338,6 +369,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "markdown",language: tree_sitter_md::LANGUAGE,outline: outline!("markdown"),+ highlights: None,fold_nodes: &["section", "fenced_code_block", "list", "block_quote", "html_block"],fold_bodies: NONE,imports: NONE,@@ -347,6 +379,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "hcl",language: tree_sitter_hcl::LANGUAGE,outline: outline!("hcl"),+ highlights: None,fold_nodes: &["block", "heredoc_template"],fold_bodies: NONE,imports: NONE,@@ -356,6 +389,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "nix",language: tree_sitter_nix::LANGUAGE,outline: outline!("nix"),+ highlights: None,fold_nodes: &["let_expression", "indented_string_expression"],fold_bodies: NONE,imports: NONE,@@ -365,6 +399,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "proto",language: tree_sitter_proto::LANGUAGE,outline: outline!("proto"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: &["import"],@@ -374,6 +409,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "graphql",language: tree_sitter_graphql::LANGUAGE,outline: outline!("graphql"),+ highlights: None,fold_nodes: NONE,fold_bodies: NONE,imports: NONE,@@ -383,6 +419,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "make",language: tree_sitter_make::LANGUAGE,outline: outline!("make"),+ highlights: None,fold_nodes: &["rule", "conditional", "define_directive"],fold_bodies: NONE,imports: &["include_directive"],@@ -392,6 +429,7 @@ pub(crate) static SPECS: &[Spec] = &[name: "cmake",language: tree_sitter_cmake::LANGUAGE,outline: outline!("cmake"),+ highlights: None,fold_nodes: &["function_def", "macro_def", "if_condition", "foreach_loop", "while_loop", "block_def"],fold_bodies: NONE,imports: NONE,diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rsnew file mode 100644index 000000000..629c081ce--- /dev/null+++ b/crates/ide-language/src/syntax/highlight.rs@@ -0,0 +1,296 @@+//! Syntax highlighting from a grammar's own `highlights.scm`, for languages the+//! UI has no CodeMirror mode for.+//!+//! Capture names follow the nvim-treesitter / Zed conventions (`@keyword`,+//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by+//! their longest known dotted prefix. Overlaps resolve the way those editors+//! do: a smaller node wins over the node containing it, and for the same node+//! the later pattern wins. `@embedded` (string interpolation) clears the+//! surrounding colour.++use super::lines::LineIndex;+use serde::{Deserialize, Serialize};+use specta::Type;+use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree};++/// Highlight classes sent to the UI, which maps each onto a Lezer tag.+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]+#[serde(rename_all = "camelCase")]+pub enum SyntaxHighlightKind {+ Keyword,+ String,+ StringSpecial,+ Escape,+ Regexp,+ Atom,+ Number,+ Constant,+ Comment,+ DocComment,+ Function,+ FunctionCall,+ Builtin,+ Type,+ Namespace,+ Variable,+ SelfVariable,+ Property,+ Attribute,+ Macro,+ Label,+ Tag,+ Operator,+ Punctuation,+}++impl SyntaxHighlightKind {+ /// Every kind, in `as u32` order: the legend for [`SyntaxHighlights::data`].+ pub const ALL: [Self; 24] = [+ Self::Keyword,+ Self::String,+ Self::StringSpecial,+ Self::Escape,+ Self::Regexp,+ Self::Atom,+ Self::Number,+ Self::Constant,+ Self::Comment,+ Self::DocComment,+ Self::Function,+ Self::FunctionCall,+ Self::Builtin,+ Self::Type,+ Self::Namespace,+ Self::Variable,+ Self::SelfVariable,+ Self::Property,+ Self::Attribute,+ Self::Macro,+ Self::Label,+ Self::Tag,+ Self::Operator,+ Self::Punctuation,+ ];+}++/// Highlighted ranges of a whole document, packed like LSP semantic tokens.+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)]+#[serde(rename_all = "camelCase")]+pub struct SyntaxHighlights {+ /// Legend: `data` kind `i` is `kinds[i]`.+ pub kinds: Vec<SyntaxHighlightKind>,+ /// `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets.+ pub data: Vec<u32>,+}++impl SyntaxHighlights {+ pub(crate) fn new(data: Vec<u32>) -> Self {+ Self { kinds: SyntaxHighlightKind::ALL.to_vec(), data }+ }+}++/// What a capture paints over its node.+#[derive(Clone, Copy, Debug, PartialEq, Eq)]+enum Paint {+ Kind(SyntaxHighlightKind),+ /// `@embedded`: code inside a string, uncoloured unless captured itself.+ Clear,+}++/// Capture name prefixes, most specific first within each family.+const CAPTURES: &[(&str, Paint)] = {+ use Paint::{Clear, Kind};+ use SyntaxHighlightKind as K;+ &[+ ("embedded", Clear),+ ("keyword", Kind(K::Keyword)),+ ("include", Kind(K::Keyword)),+ ("import", Kind(K::Keyword)),+ ("conditional", Kind(K::Keyword)),+ ("repeat", Kind(K::Keyword)),+ ("exception", Kind(K::Keyword)),+ ("boolean", Kind(K::Keyword)),+ ("string.escape", Kind(K::Escape)),+ ("escape", Kind(K::Escape)),+ ("string.regex", Kind(K::Regexp)),+ ("string.regexp", Kind(K::Regexp)),+ ("string.special.symbol", Kind(K::Atom)),+ ("symbol", Kind(K::Atom)),+ ("string.special", Kind(K::StringSpecial)),+ ("string", Kind(K::String)),+ ("character", Kind(K::String)),+ ("number", Kind(K::Number)),+ ("float", Kind(K::Number)),+ ("constant.builtin", Kind(K::Constant)),+ ("constant.macro", Kind(K::Macro)),+ ("constant", Kind(K::Constant)),+ // nvim greys out unused bindings (`_acc`); IDEA shows them as variables.+ ("comment.unused", Kind(K::Variable)),+ ("comment.doc", Kind(K::DocComment)),+ ("comment.documentation", Kind(K::DocComment)),+ ("comment", Kind(K::Comment)),+ ("function.builtin", Kind(K::Builtin)),+ ("function.macro", Kind(K::Macro)),+ ("function.definition", Kind(K::Function)),+ ("function", Kind(K::FunctionCall)),+ ("method", Kind(K::FunctionCall)),+ ("constructor", Kind(K::Type)),+ ("type.builtin", Kind(K::Builtin)),+ ("type", Kind(K::Type)),+ ("module", Kind(K::Namespace)),+ ("namespace", Kind(K::Namespace)),+ ("variable.builtin", Kind(K::SelfVariable)),+ ("variable.member", Kind(K::Property)),+ ("variable.field", Kind(K::Property)),+ ("variable", Kind(K::Variable)),+ ("parameter", Kind(K::Variable)),+ ("property", Kind(K::Property)),+ ("field", Kind(K::Property)),+ ("attribute", Kind(K::Attribute)),+ ("label", Kind(K::Label)),+ ("tag", Kind(K::Tag)),+ ("operator", Kind(K::Operator)),+ ("punctuation", Kind(K::Punctuation)),+ ]+};++/// The paint for a capture name: its longest prefix (on `.` boundaries) in+/// [`CAPTURES`]. Private captures (`@__name__`) and unknown names paint nothing.+fn paint_for(name: &str) -> Option<Paint> {+ if name.starts_with('_') {+ return None;+ }+ let mut prefix = name;+ loop {+ if let Some((_, p)) = CAPTURES.iter().find(|(n, _)| *n == prefix) {+ return Some(*p);+ }+ prefix = &prefix[..prefix.rfind('.')?];+ }+}++pub(crate) struct HighlightQuery {+ query: Query,+ /// By capture index.+ paints: Vec<Option<Paint>>,+}++impl HighlightQuery {+ pub(crate) fn new(language: &Language, source: &str) -> Result<Self, tree_sitter::QueryError> {+ let query = Query::new(language, source)?;+ let paints = query.capture_names().iter().map(|n| paint_for(n)).collect();+ Ok(Self { query, paints })+ }+}++/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`.+const NONE: u8 = 0;++/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`].+pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec<u32> {+ // (byte len, pattern, start, end, paint): painted largest first, then in+ // pattern order, so inner nodes and later patterns overwrite.+ let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new();+ let mut cursor = QueryCursor::new();+ let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes());+ while let Some(m) = matches.next() {+ for c in m.captures {+ if let Some(paint) = q.paints[c.index as usize] {+ let r = c.node.byte_range();+ if r.start < r.end {+ captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint));+ }+ }+ }+ }+ captures.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1)));++ let mut painted = vec![NONE; text.len()];+ for &(_, _, start, end, paint) in &captures {+ let v = match paint {+ Paint::Kind(k) => 1 + k as u8,+ Paint::Clear => NONE,+ };+ painted[start..end].fill(v);+ }++ let lines = LineIndex::new(text);+ let mut data = Vec::new();+ let mut i = 0;+ while i < painted.len() {+ let v = painted[i];+ let start = i;+ while i < painted.len() && painted[i] == v {+ i += 1;+ }+ // Node boundaries are char boundaries; whitespace-only runs stay uncoloured.+ if v != NONE && text.is_char_boundary(start) && text.is_char_boundary(i) && !text[start..i].trim().is_empty() {+ data.extend([lines.utf16_at(start) as u32, lines.utf16_at(i) as u32, (v - 1) as u32]);+ }+ }+ data+}++#[cfg(test)]+mod tests {+ use super::*;++ #[test]+ fn capture_names_fold_to_the_longest_known_prefix() {+ use SyntaxHighlightKind as K;+ assert_eq!(paint_for("keyword.function"), Some(Paint::Kind(K::Keyword)));+ assert_eq!(paint_for("string.special.symbol"), Some(Paint::Kind(K::Atom)));+ assert_eq!(paint_for("string.special.path"), Some(Paint::Kind(K::StringSpecial)));+ assert_eq!(paint_for("comment.doc.__attribute__"), Some(Paint::Kind(K::DocComment)));+ assert_eq!(paint_for("punctuation.bracket"), Some(Paint::Kind(K::Punctuation)));+ assert_eq!(paint_for("embedded"), Some(Paint::Clear));+ assert_eq!(paint_for("__name__"), None);+ assert_eq!(paint_for("spell"), None);+ }++ /// `(text, kind)` of every highlighted range.+ fn spans(lang: &str, text: &str) -> Vec<(String, SyntaxHighlightKind)> {+ let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new(lang), text).unwrap();+ let units: Vec<u16> = text.encode_utf16().collect();+ h.data+ .chunks(3)+ .map(|t| (String::from_utf16(&units[t[0] as usize..t[1] as usize]).unwrap(), h.kinds[t[2] as usize]))+ .collect()+ }++ #[test]+ fn highlights_elixir() {+ use SyntaxHighlightKind as K;+ let src = "defmodule Greeter do\n @moduledoc \"Hi\"\n # 😀 note\n def hello(name, _opts) do\n IO.puts(\"Hello #{name}\")\n :ok\n end\nend\n";+ let got = spans("elixir", src);+ let kind = |t: &str| got.iter().find(|(s, _)| s == t).map(|(_, k)| *k);+ assert_eq!(kind("defmodule"), Some(K::Keyword));+ assert_eq!(kind("Greeter"), Some(K::Namespace));+ assert_eq!(kind("\"Hi\""), Some(K::DocComment));+ assert_eq!(kind("# 😀 note"), Some(K::Comment));+ assert_eq!(kind("def"), Some(K::Keyword));+ assert_eq!(kind("hello"), Some(K::FunctionCall));+ assert_eq!(kind("_opts"), Some(K::Variable));+ assert_eq!(kind("puts"), Some(K::FunctionCall));+ // Interpolation clears the string colour; its contents are coloured on their own.+ assert_eq!(kind("\"Hello "), Some(K::String));+ assert_eq!(kind("#{"), Some(K::Punctuation));+ assert_eq!(kind(":ok"), Some(K::Atom));+ assert_eq!(kind("end"), Some(K::Keyword));+ // Sorted and non-overlapping.+ let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new("elixir"), src).unwrap();+ assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0]));+ }++ #[test]+ fn languages_without_a_query_have_none() {+ assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None);+ }++ #[test]+ fn legend_matches_discriminants() {+ for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() {+ assert_eq!(*k as usize, i);+ }+ }+}diff --git a/crates/ide-language/src/syntax/lines.rs b/crates/ide-language/src/syntax/lines.rsindex 7b3291d23..5b8de2062 100644--- a/crates/ide-language/src/syntax/lines.rs+++ b/crates/ide-language/src/syntax/lines.rs@@ -40,7 +40,7 @@ impl<'a> LineIndex<'a> {}/// UTF-16 offset of a char-boundary byte offset.- fn utf16_at(&self, byte: usize) -> usize {+ pub(crate) fn utf16_at(&self, byte: usize) -> usize {if self.checkpoints.is_empty() {return byte;}diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rsindex e6633bb2a..783dfe9c4 100644--- a/crates/ide-language/src/syntax/mod.rs+++ b/crates/ide-language/src/syntax/mod.rs@@ -1,8 +1,9 @@-//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges.+//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges, highlighting.mod cache;mod folding;mod grammars;+mod highlight;pub(crate) mod lines;mod outline;mod selection;@@ -10,6 +11,7 @@ mod selection;use crate::{FoldRange, LanguageConfig, LanguageRegistry, SymbolTag};pub use cache::{MAX_CACHED_TREES, SyntaxCache, SyntaxCacheStats};use grammars::{SPECS, Spec};+pub use highlight::{SyntaxHighlightKind, SyntaxHighlights};use ide_core::{LanguageId, OutlineItem, Range};use std::cell::RefCell;use std::collections::HashMap;@@ -31,6 +33,7 @@ pub(crate) struct Grammar {pub(crate) spec: &'static Spec,pub(crate) language: Language,outline: OnceLock<Option<outline::OutlineQuery>>,+ highlights: OnceLock<Option<highlight::HighlightQuery>>,kinds: OnceLock<folding::KindTable>,}@@ -50,6 +53,17 @@ impl Grammar {}).as_ref()}++ fn highlight_query(&self) -> Option<&highlight::HighlightQuery> {+ self.highlights+ .get_or_init(|| {+ let source = self.spec.highlights?;+ highlight::HighlightQuery::new(&self.language, source)+ .inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name))+ .ok()+ })+ .as_ref()+ }}thread_local! {@@ -72,6 +86,7 @@ impl Syntax {spec,language: Language::new(spec.language),outline: OnceLock::new(),+ highlights: OnceLock::new(),kinds: OnceLock::new(),}).collect();@@ -150,6 +165,13 @@ impl Syntax {.unwrap_or_default()}+ /// Highlighting from the grammar's own query; `None` without one.+ pub fn highlights(&self, lang: &LanguageId, text: &str) -> Option<SyntaxHighlights> {+ self.analyze(lang, text, |g, tree, text| {+ Some(SyntaxHighlights::new(highlight::highlights(g.highlight_query()?, tree, text)))+ })+ }+/// Syntax-aware selection ranges for IDEA "Extend Selection" (⌥↑), innermost first.pub fn selection_ranges(&self, lang: &LanguageId, text: &str, offset_utf16: u32) -> Vec<Range> {match self.analyze(lang, text, |_, tree, text| Some(selection::selection_ranges(tree, text, offset_utf16))) {diff --git a/src-tauri/src/commands/doc.rs b/src-tauri/src/commands/doc.rsindex 59c4f708e..719dcbcad 100644--- a/src-tauri/src/commands/doc.rs+++ b/src-tauri/src/commands/doc.rs@@ -1,6 +1,6 @@use crate::state::AppState;use ide_core::{ChangeSet, DocId, IdeError, IdeResult, OutlineItem, Range};-use ide_language::FoldRange;+use ide_language::{FoldRange, SyntaxHighlights};use ide_text::LineEnding;use ide_workspace::{DocumentInfo, HistoryEntry};use std::path::PathBuf;@@ -136,6 +136,22 @@ pub async fn doc_folding(window: Window, state: State<'_, AppState>, doc: DocId).await}+/// Tree-sitter highlighting for languages the UI has no CodeMirror mode for;+/// empty when the grammar has no highlights query or the document is large.+#[tauri::command]+#[specta::specta]+pub async fn doc_highlights(window: Window, state: State<'_, AppState>, doc: DocId) -> IdeResult<SyntaxHighlights> {+ let project = state.project(window.label())?;+ blocking(move || {+ let snap = project.documents.text_snapshot(doc)?;+ if snap.large {+ return Ok(SyntaxHighlights::default());+ }+ Ok(project.syntax.highlights(doc, &snap.language, snap.version, &snap.text))+ })+ .await+}+/// Extend Selection (⌥↑): syntax ranges around `offset`, innermost first./// Large documents fall back to word / line / document ranges.#[tauri::command]diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rsindex 4c4f60fab..dc0d79f65 100644--- a/src-tauri/src/commands/mod.rs+++ b/src-tauri/src/commands/mod.rs@@ -74,6 +74,7 @@ pub fn builder() -> tauri_specta::Builder<tauri::Wry> {doc::doc_set_line_ending,doc::doc_outline,doc::doc_folding,+ doc::doc_highlights,doc::doc_selection_ranges,syntax::syntax_support,syntax::syntax_node_at,diff --git a/src/bindings.ts b/src/bindings.tsindex 2afc23058..22f7dc893 100644--- a/src/bindings.ts+++ b/src/bindings.ts@@ -92,6 +92,11 @@ export const commands = {*/docOutline: (doc: DocId) => typedError<OutlineItem[], IdeError>(__TAURI_INVOKE("doc_outline", { doc })),docFolding: (doc: DocId) => typedError<FoldRange[], IdeError>(__TAURI_INVOKE("doc_folding", { doc })),+ /**+ * Tree-sitter highlighting for languages the UI has no CodeMirror mode for;+ * empty when the grammar has no highlights query or the document is large.+ */+ docHighlights: (doc: DocId) => typedError<SyntaxHighlights, IdeError>(__TAURI_INVOKE("doc_highlights", { doc })),/*** Extend Selection (⌥↑): syntax ranges around `offset`, innermost first.* Large documents fall back to word / line / document ranges.@@ -3683,6 +3688,17 @@ export type SyntaxErrors = {truncated: boolean,};+/** Highlight classes sent to the UI, which maps each onto a Lezer tag. */+export type SyntaxHighlightKind = "keyword" | "string" | "stringSpecial" | "escape" | "regexp" | "atom" | "number" | "constant" | "comment" | "docComment" | "function" | "functionCall" | "builtin" | "type" | "namespace" | "variable" | "selfVariable" | "property" | "attribute" | "macro" | "label" | "tag" | "operator" | "punctuation";++/** Highlighted ranges of a whole document, packed like LSP semantic tokens. */+export type SyntaxHighlights = {+ /** Legend: `data` kind `i` is `kinds[i]`. */+ kinds: SyntaxHighlightKind[],+ /** `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets. */+ data: number[],+};+/** One tree-sitter node. */export type SyntaxNode = {/** Grammar node kind, e.g. `method_invocation`, `identifier`, `;`. */diff --git a/src/editor/core/backend.ts b/src/editor/core/backend.tsindex 75de63a25..e075ec6e9 100644--- a/src/editor/core/backend.ts+++ b/src/editor/core/backend.ts@@ -18,6 +18,7 @@ import {type LineEnding,type OutlineItem,type Range,+ type SyntaxHighlights,type TextChange,} from "@/platform/ipc";@@ -43,6 +44,8 @@ export interface DocBackend {setEncoding(doc: DocId, encoding: string, reload: boolean): Promise<string>;outline(doc: DocId): Promise<OutlineItem[]>;folding(doc: DocId): Promise<FoldRange[]>;+ /** Tree-sitter highlighting, for languages without a CodeMirror mode. */+ highlights(doc: DocId): Promise<SyntaxHighlights>;selectionRanges(doc: DocId, offset: number): Promise<Range[]>;languages(): Promise<LanguageInfo[]>;onPatched(handler: (e: DocPatchedEvent) => void): () => void;@@ -62,6 +65,7 @@ export const ipcBackend: DocBackend = {setEncoding: (doc, encoding, reload) => call(commands.docSetEncoding(doc, encoding, reload)),outline: (doc) => call(commands.docOutline(doc)),folding: (doc) => call(commands.docFolding(doc)),+ highlights: (doc) => call(commands.docHighlights(doc)),selectionRanges: (doc, offset) => call(commands.docSelectionRanges(doc, offset)),languages: () => commands.languages(),onPatched: (h) => onIdeEvent("docPatched", (e) => h(e)),@@ -343,6 +347,9 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB}return items;},+ async highlights() {+ return { kinds: [], data: [] };+ },async folding(doc) {const d = get(doc);const out: FoldRange[] = [];diff --git a/src/editor/core/cm/setup.ts b/src/editor/core/cm/setup.tsindex 62cb91df2..e5618dde9 100644--- a/src/editor/core/cm/setup.ts+++ b/src/editor/core/cm/setup.ts@@ -25,6 +25,7 @@ import { openideaTheme } from "./theme";import { caretRowHighlight } from "./caretRow";import { changeLane, overviewLane } from "./lanes";import { foldingExtension } from "./folding";+import { syntaxHighlightField } from "./syntaxHighlight";import { indentGuides } from "./indentGuides";import { stickyMaxLines, stickyScroll } from "./stickyScroll";import { findExtension } from "./find";@@ -267,6 +268,7 @@ export function baseExtensions(o: SetupOptions): Extension {clipboardHistoryExtension,indentOnInput(),syntaxHighlighting(openideaHighlightStyle),+ syntaxHighlightField,closeBrackets(),caretRowHighlight,fontZoom,diff --git a/src/editor/core/cm/syntaxHighlight.test.ts b/src/editor/core/cm/syntaxHighlight.test.tsnew file mode 100644index 000000000..a3a1ecbbb--- /dev/null+++ b/src/editor/core/cm/syntaxHighlight.test.ts@@ -0,0 +1,37 @@+import { describe, expect, it } from "vitest";+import { EditorState } from "@codemirror/state";+import { highlightingFor, syntaxHighlighting } from "@codemirror/language";+import { tags as t } from "@lezer/highlight";+import { openideaHighlightStyle } from "./highlight";+import { setSyntaxHighlights, syntaxHighlightField } from "./syntaxHighlight";++function marks(state: EditorState) {+ const out: { text: string; cls: string }[] = [];+ state.field(syntaxHighlightField).between(0, state.doc.length, (from, to, d) => {+ out.push({ text: state.doc.sliceString(from, to), cls: d.spec.class as string });+ });+ return out;+}++describe("syntaxHighlightField", () => {+ const create = () =>+ EditorState.create({ doc: "def hi, do: :ok", extensions: [syntaxHighlighting(openideaHighlightStyle), syntaxHighlightField] });++ it("styles backend ranges with the active highlight style and maps them through edits", () => {+ let state = create();+ state = state.update({+ effects: setSyntaxHighlights.of({ kinds: ["keyword", "atom", "functionCall"], data: [0, 3, 0, 4, 6, 2, 12, 15, 1, 99, 120, 0] }),+ }).state;+ const keyword = highlightingFor(state, [t.keyword]);+ const atom = highlightingFor(state, [t.atom]);+ expect(keyword).toBeTruthy();+ expect(marks(state)).toEqual([+ { text: "def", cls: keyword },+ { text: "hi", cls: highlightingFor(state, [t.function(t.variableName)]) },+ { text: ":ok", cls: atom },+ ]);++ state = state.update({ changes: { from: 0, insert: " " } }).state;+ expect(marks(state).map((m) => m.text)).toEqual(["def", "hi", ":ok"]);+ });+});diff --git a/src/editor/core/cm/syntaxHighlight.ts b/src/editor/core/cm/syntaxHighlight.tsnew file mode 100644index 000000000..8ced4899d--- /dev/null+++ b/src/editor/core/cm/syntaxHighlight.ts@@ -0,0 +1,66 @@+/**+ * Tree-sitter highlighting (`doc_highlights`) for languages without a+ * CodeMirror mode (Elixir, ...): backend kinds → Lezer tags, so the colours+ * come from the same highlight style and Color Scheme settings as Lezer+ * highlighting. Refreshed with folding; in between, marks are mapped through+ * the changes.+ */+import { StateEffect, StateField, type EditorState } from "@codemirror/state";+import { Decoration, EditorView, type DecorationSet } from "@codemirror/view";+import { highlightingFor } from "@codemirror/language";+import { tags as t, type Tag } from "@lezer/highlight";+import type { SyntaxHighlightKind, SyntaxHighlights } from "@/platform/ipc";++const TAGS: Record<SyntaxHighlightKind, Tag> = {+ keyword: t.keyword,+ string: t.string,+ stringSpecial: t.special(t.string),+ escape: t.escape,+ regexp: t.regexp,+ atom: t.atom,+ number: t.number,+ constant: t.constant(t.variableName),+ comment: t.comment,+ docComment: t.docComment,+ function: t.function(t.definition(t.variableName)),+ functionCall: t.function(t.variableName),+ builtin: t.standard(t.variableName),+ type: t.typeName,+ namespace: t.namespace,+ variable: t.variableName,+ selfVariable: t.self,+ property: t.propertyName,+ attribute: t.annotation,+ macro: t.macroName,+ label: t.labelName,+ tag: t.tagName,+ operator: t.operator,+ punctuation: t.punctuation,+};++export const setSyntaxHighlights = StateEffect.define<SyntaxHighlights>();++function decorations(state: EditorState, h: SyntaxHighlights): DecorationSet {+ const marks = h.kinds.map((k) => {+ const cls = highlightingFor(state, [TAGS[k]]);+ return cls ? Decoration.mark({ class: cls }) : null;+ });+ const len = state.doc.length;+ const ranges = [];+ for (let i = 0; i + 2 < h.data.length; i += 3) {+ const from = h.data[i]!;+ const to = Math.min(h.data[i + 1]!, len);+ const mark = marks[h.data[i + 2]!];+ if (mark && from < to) ranges.push(mark.range(from, to));+ }+ return Decoration.set(ranges, true);+}++export const syntaxHighlightField = StateField.define<DecorationSet>({+ create: () => Decoration.none,+ update(deco, tr) {+ for (const e of tr.effects) if (e.is(setSyntaxHighlights)) return decorations(tr.state, e.value);+ return tr.docChanged ? deco.map(tr.changes) : deco;+ },+ provide: (f) => EditorView.decorations.from(f),+});diff --git a/src/editor/core/model/document.ts b/src/editor/core/model/document.tsindex 799e6e6c9..981289c97 100644--- a/src/editor/core/model/document.ts+++ b/src/editor/core/model/document.ts@@ -7,7 +7,7 @@ import { EditorView } from "@codemirror/view";import type { LanguageDescription, LanguageSupport } from "@codemirror/language";import { create } from "zustand";import type { EditorConfigProps } from "@/bindings";-import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings } from "@/platform/ipc";+import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings, SyntaxHighlights } from "@/platform/ipc";import {_emitEditorEvent,_saveParticipants,@@ -22,6 +22,7 @@ import type { DocBackend, LanguageInfo } from "../backend";import { findLanguageDescription, languageInfo, loadLanguageInfos, loadSupport, loadedSupport, commentTokens } from "../cm/languages";import { foldEffect } from "@codemirror/language";import { backendFolds, setBackendFolds } from "../cm/folding";+import { setSyntaxHighlights } from "../cm/syntaxHighlight";import {caretFor,compartments,@@ -265,6 +266,7 @@ export class DocModel {addSlot(slot: EditorSlot) {this.slots.add(slot);+ if (this.highlights?.counter === this.changeCounter) slot.reconfigure([setSyntaxHighlights.of(this.highlights.value)]);}removeSlot(slot: EditorSlot) {@@ -332,6 +334,8 @@ export class DocModel {indentOverride: { tabSize: number; insertSpaces: boolean } | null = null;/** Indentation detected in the file's text at open time. */detectedIndent: { insertSpaces: boolean; size?: number } | null = null;+ /** Last tree-sitter highlighting, for slots opened before the next refresh. */+ private highlights: { counter: number; value: SyntaxHighlights } | null = null;/** Fold-by-default has been applied (first folding ranges of the document). */private initialFolds = false;/** Signature of settings that require contributions to be rebuilt. */@@ -423,7 +427,12 @@ export class DocModel {await this.sync.whenIdle();const counter = this.changeCounter;const backend = this.manager.backend;- const [folds, outline] = await Promise.allSettled([backend.folding(this.doc), backend.outline(this.doc)]);+ const [folds, outline, highlights] = await Promise.allSettled([+ backend.folding(this.doc),+ backend.outline(this.doc),+ // Without a CodeMirror mode, colours come from the grammar's tree-sitter query.+ this.desc ? null : backend.highlights(this.doc),+ ]);if (this.closed) return;if (counter !== this.changeCounter) {this.scheduleRefresh();@@ -436,6 +445,10 @@ export class DocModel {this.foldByDefault();}}+ if (highlights.status === "fulfilled" && highlights.value) {+ this.highlights = { counter, value: highlights.value };+ for (const s of this.slots) s.reconfigure([setSyntaxHighlights.of(highlights.value)]);+ }if (outline.status === "fulfilled") {useOutline.setState({ [this.path]: outline.value });const lines = methodLines(outline.value);