Skip to content

The IDE you know, rebuilt in Rust. IntelliJ IDEA–style keymap, tool windows and search-everywhere in a fast native app: editor, code insight, debugger, Git, terminal, database and HTTP clients for 60+ languages.

feat(editor): Highlight languages without a CodeMirror mode from tree-sitter

l0ng-ai1572fc36ea00
diff --git a/crates/ide-language/src/lib.rs b/crates/ide-language/src/lib.rs
index 67f736cde..97b1b3520 100644
--- a/crates/ide-language/src/lib.rs
+++ b/crates/ide-language/src/lib.rs
@@ -12,7 +12,10 @@ use std::collections::BTreeMap;
mod registry;
mod syntax;
pub use registry::LanguageRegistry;
-pub use syntax::{MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, TreeError};
+pub use syntax::{
+ MAX_CACHED_TREES, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxCache, SyntaxCacheStats, SyntaxHighlightKind,
+ SyntaxHighlights, TreeError,
+};
/// Parsed form of `languages/<id>.toml`. Backend-only; the UI gets [`LanguageInfo`].
#[derive(Clone, Debug, Serialize, Deserialize)]
diff --git a/crates/ide-language/src/syntax/cache.rs b/crates/ide-language/src/syntax/cache.rs
index fef3cb9be..0969fe7df 100644
--- a/crates/ide-language/src/syntax/cache.rs
+++ b/crates/ide-language/src/syntax/cache.rs
@@ -5,7 +5,10 @@
//! Structure / folding / Extend Selection on a large file cost an incremental
//! parse instead of a full one.
-use super::{Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, TreeError, folding, outline, parse_cancellable, selection};
+use super::{
+ Grammar, MAX_PARSE_BYTES, ParsedTree, Syntax, SyntaxHighlights, TreeError, folding, highlight, outline,
+ parse_cancellable, selection,
+};
use crate::FoldRange;
use ide_core::{DocId, LanguageId, OutlineItem, Range};
use ide_text::ByteEdit;
@@ -59,6 +62,7 @@ struct Memo {
key: Option<(u32, LanguageId)>,
outline: Option<Arc<Vec<OutlineItem>>>,
folding: Option<Arc<Vec<FoldRange>>>,
+ highlights: Option<Arc<Vec<u32>>>,
}
impl Memo {
@@ -211,6 +215,21 @@ impl SyntaxCache {
)
}
+ /// Highlighting from the grammar's own query; empty without one.
+ pub fn highlights(&self, doc: DocId, lang: &LanguageId, version: u32, text: &str) -> SyntaxHighlights {
+ SyntaxHighlights::new(self.memoized(
+ doc,
+ lang,
+ version,
+ |m| &mut m.highlights,
+ || {
+ self.with_tree(doc, lang, version, text, |g, tree, text| {
+ Some(highlight::highlights(g.highlight_query()?, tree, text))
+ })
+ },
+ ))
+ }
+
/// `compute()` once per document version; `None` (no grammar / query) is not memoized.
fn memoized<T: Clone>(
&self,
diff --git a/crates/ide-language/src/syntax/grammars.rs b/crates/ide-language/src/syntax/grammars.rs
index beb969ea7..fad2d5c51 100644
--- a/crates/ide-language/src/syntax/grammars.rs
+++ b/crates/ide-language/src/syntax/grammars.rs
@@ -9,6 +9,8 @@ pub(crate) struct Spec {
pub language: LanguageFn,
/// `queries/<name>/outline.scm`; see `queries/README.md` for the capture vocabulary.
pub outline: &'static str,
+ /// The grammar's `highlights.scm`, for languages without a CodeMirror mode.
+ pub highlights: Option<&'static str>,
/// Node kinds folded when they span several lines (in addition to every
/// bracket-delimited node, which is folded generically).
pub fold_nodes: &'static [&'static str],
@@ -33,6 +35,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "rust",
language: tree_sitter_rust::LANGUAGE,
outline: outline!("rust"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: &["use_declaration", "extern_crate_declaration"],
@@ -42,6 +45,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "typescript",
language: tree_sitter_typescript::LANGUAGE_TYPESCRIPT,
outline: outline!("typescript"),
+ highlights: None,
fold_nodes: &["template_string"],
fold_bodies: NONE,
imports: &["import_statement"],
@@ -51,6 +55,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "tsx",
language: tree_sitter_typescript::LANGUAGE_TSX,
outline: outline!("typescript"),
+ highlights: None,
fold_nodes: &["template_string", "jsx_element"],
fold_bodies: NONE,
imports: &["import_statement"],
@@ -60,6 +65,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "javascript",
language: tree_sitter_javascript::LANGUAGE,
outline: outline!("javascript"),
+ highlights: None,
fold_nodes: &["template_string", "jsx_element"],
fold_bodies: NONE,
imports: &["import_statement"],
@@ -69,6 +75,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "python",
language: tree_sitter_python::LANGUAGE,
outline: outline!("python"),
+ highlights: None,
fold_nodes: &["string"],
fold_bodies: &["block"],
imports: &["import_statement", "import_from_statement", "future_import_statement"],
@@ -78,6 +85,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "go",
language: tree_sitter_go::LANGUAGE,
outline: outline!("go"),
+ highlights: None,
fold_nodes: &["raw_string_literal"],
fold_bodies: NONE,
imports: &["import_declaration"],
@@ -87,6 +95,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "java",
language: tree_sitter_java::LANGUAGE,
outline: outline!("java"),
+ highlights: None,
fold_nodes: &["text_block"],
fold_bodies: NONE,
imports: &["import_declaration"],
@@ -96,6 +105,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "kotlin",
language: tree_sitter_kotlin_ng::LANGUAGE,
outline: outline!("kotlin"),
+ highlights: None,
fold_nodes: &["string_literal"],
fold_bodies: NONE,
imports: &["import"],
@@ -105,6 +115,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "scala",
language: tree_sitter_scala::LANGUAGE,
outline: outline!("scala"),
+ highlights: None,
fold_nodes: &["string"],
fold_bodies: &["indented_block", "template_body"],
imports: &["import_declaration"],
@@ -114,6 +125,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "c",
language: tree_sitter_c::LANGUAGE,
outline: outline!("c"),
+ highlights: None,
fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def"],
fold_bodies: NONE,
imports: &["preproc_include"],
@@ -123,6 +135,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "cpp",
language: tree_sitter_cpp::LANGUAGE,
outline: outline!("cpp"),
+ highlights: None,
fold_nodes: &["preproc_if", "preproc_ifdef", "preproc_function_def", "raw_string_literal"],
fold_bodies: NONE,
imports: &["preproc_include", "using_declaration"],
@@ -132,6 +145,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "csharp",
language: tree_sitter_c_sharp::LANGUAGE,
outline: outline!("csharp"),
+ highlights: None,
fold_nodes: &["preproc_region", "raw_string_literal", "verbatim_string_literal"],
fold_bodies: NONE,
imports: &["using_directive"],
@@ -141,6 +155,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "swift",
language: tree_sitter_swift::LANGUAGE,
outline: outline!("swift"),
+ highlights: None,
// Control-flow statements hold `{ statements }` without a block node of their own.
fold_nodes: &[
"multi_line_string_literal",
@@ -160,6 +175,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "ruby",
language: tree_sitter_ruby::LANGUAGE,
outline: outline!("ruby"),
+ highlights: None,
fold_nodes: &[
"method",
"singleton_method",
@@ -185,6 +201,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "php",
language: tree_sitter_php::LANGUAGE_PHP,
outline: outline!("php"),
+ highlights: None,
fold_nodes: &["heredoc", "nowdoc"],
fold_bodies: NONE,
imports: &["namespace_use_declaration"],
@@ -194,6 +211,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "lua",
language: tree_sitter_lua::LANGUAGE,
outline: outline!("lua"),
+ highlights: None,
fold_nodes: &[
"function_declaration",
"function_definition",
@@ -212,6 +230,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "dart",
language: tree_sitter_dart::LANGUAGE,
outline: outline!("dart"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: &["import_or_export", "library_name", "part_directive", "part_of_directive"],
@@ -221,6 +240,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "zig",
language: tree_sitter_zig::LANGUAGE,
outline: outline!("zig"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: NONE,
@@ -230,6 +250,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "elixir",
language: tree_sitter_elixir::LANGUAGE,
outline: outline!("elixir"),
+ highlights: Some(tree_sitter_elixir::HIGHLIGHTS_QUERY),
fold_nodes: &["do_block", "anonymous_function", "string", "sigil"],
fold_bodies: NONE,
imports: NONE,
@@ -239,6 +260,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "haskell",
language: tree_sitter_haskell::LANGUAGE,
outline: outline!("haskell"),
+ highlights: None,
// `bind`: equations without arguments (`main = do …`).
fold_nodes: &["function", "bind", "data_type", "newtype", "class", "instance", "signature"],
fold_bodies: NONE,
@@ -249,6 +271,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "ocaml",
language: tree_sitter_ocaml::LANGUAGE_OCAML,
outline: outline!("ocaml"),
+ highlights: None,
fold_nodes: &[
"value_definition",
"type_definition",
@@ -266,6 +289,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "ocaml_interface",
language: tree_sitter_ocaml::LANGUAGE_OCAML_INTERFACE,
outline: outline!("ocaml_interface"),
+ highlights: None,
fold_nodes: &["module_definition", "module_type_definition", "signature", "type_definition"],
fold_bodies: NONE,
imports: &["open_module"],
@@ -275,6 +299,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "bash",
language: tree_sitter_bash::LANGUAGE,
outline: outline!("bash"),
+ highlights: None,
fold_nodes: &["do_group", "if_statement", "case_statement", "heredoc_body"],
fold_bodies: NONE,
imports: NONE,
@@ -284,6 +309,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "html",
language: tree_sitter_html::LANGUAGE,
outline: outline!("html"),
+ highlights: None,
fold_nodes: &["element", "script_element", "style_element"],
fold_bodies: NONE,
imports: NONE,
@@ -293,6 +319,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "xml",
language: tree_sitter_xml::LANGUAGE_XML,
outline: outline!("xml"),
+ highlights: None,
fold_nodes: &["element", "CDSect"],
fold_bodies: NONE,
imports: NONE,
@@ -302,6 +329,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "css",
language: tree_sitter_css::LANGUAGE,
outline: outline!("css"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: &["import_statement"],
@@ -311,6 +339,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "json",
language: tree_sitter_json::LANGUAGE,
outline: outline!("json"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: NONE,
@@ -320,6 +349,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "yaml",
language: tree_sitter_yaml::LANGUAGE,
outline: outline!("yaml"),
+ highlights: None,
fold_nodes: &["block_mapping_pair", "block_sequence_item", "block_scalar"],
fold_bodies: NONE,
imports: NONE,
@@ -329,6 +359,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "toml",
language: tree_sitter_toml_ng::LANGUAGE,
outline: outline!("toml"),
+ highlights: None,
fold_nodes: &["table", "table_array_element", "string"],
fold_bodies: NONE,
imports: NONE,
@@ -338,6 +369,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "markdown",
language: tree_sitter_md::LANGUAGE,
outline: outline!("markdown"),
+ highlights: None,
fold_nodes: &["section", "fenced_code_block", "list", "block_quote", "html_block"],
fold_bodies: NONE,
imports: NONE,
@@ -347,6 +379,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "hcl",
language: tree_sitter_hcl::LANGUAGE,
outline: outline!("hcl"),
+ highlights: None,
fold_nodes: &["block", "heredoc_template"],
fold_bodies: NONE,
imports: NONE,
@@ -356,6 +389,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "nix",
language: tree_sitter_nix::LANGUAGE,
outline: outline!("nix"),
+ highlights: None,
fold_nodes: &["let_expression", "indented_string_expression"],
fold_bodies: NONE,
imports: NONE,
@@ -365,6 +399,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "proto",
language: tree_sitter_proto::LANGUAGE,
outline: outline!("proto"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: &["import"],
@@ -374,6 +409,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "graphql",
language: tree_sitter_graphql::LANGUAGE,
outline: outline!("graphql"),
+ highlights: None,
fold_nodes: NONE,
fold_bodies: NONE,
imports: NONE,
@@ -383,6 +419,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "make",
language: tree_sitter_make::LANGUAGE,
outline: outline!("make"),
+ highlights: None,
fold_nodes: &["rule", "conditional", "define_directive"],
fold_bodies: NONE,
imports: &["include_directive"],
@@ -392,6 +429,7 @@ pub(crate) static SPECS: &[Spec] = &[
name: "cmake",
language: tree_sitter_cmake::LANGUAGE,
outline: outline!("cmake"),
+ highlights: None,
fold_nodes: &["function_def", "macro_def", "if_condition", "foreach_loop", "while_loop", "block_def"],
fold_bodies: NONE,
imports: NONE,
diff --git a/crates/ide-language/src/syntax/highlight.rs b/crates/ide-language/src/syntax/highlight.rs
new file mode 100644
index 000000000..629c081ce
--- /dev/null
+++ b/crates/ide-language/src/syntax/highlight.rs
@@ -0,0 +1,296 @@
+//! Syntax highlighting from a grammar's own `highlights.scm`, for languages the
+//! UI has no CodeMirror mode for.
+//!
+//! Capture names follow the nvim-treesitter / Zed conventions (`@keyword`,
+//! `@string.special.symbol`, ...) and are folded into [`SyntaxHighlightKind`] by
+//! their longest known dotted prefix. Overlaps resolve the way those editors
+//! do: a smaller node wins over the node containing it, and for the same node
+//! the later pattern wins. `@embedded` (string interpolation) clears the
+//! surrounding colour.
+
+use super::lines::LineIndex;
+use serde::{Deserialize, Serialize};
+use specta::Type;
+use tree_sitter::{Language, Query, QueryCursor, StreamingIterator, Tree};
+
+/// Highlight classes sent to the UI, which maps each onto a Lezer tag.
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Serialize, Deserialize, Type)]
+#[serde(rename_all = "camelCase")]
+pub enum SyntaxHighlightKind {
+ Keyword,
+ String,
+ StringSpecial,
+ Escape,
+ Regexp,
+ Atom,
+ Number,
+ Constant,
+ Comment,
+ DocComment,
+ Function,
+ FunctionCall,
+ Builtin,
+ Type,
+ Namespace,
+ Variable,
+ SelfVariable,
+ Property,
+ Attribute,
+ Macro,
+ Label,
+ Tag,
+ Operator,
+ Punctuation,
+}
+
+impl SyntaxHighlightKind {
+ /// Every kind, in `as u32` order: the legend for [`SyntaxHighlights::data`].
+ pub const ALL: [Self; 24] = [
+ Self::Keyword,
+ Self::String,
+ Self::StringSpecial,
+ Self::Escape,
+ Self::Regexp,
+ Self::Atom,
+ Self::Number,
+ Self::Constant,
+ Self::Comment,
+ Self::DocComment,
+ Self::Function,
+ Self::FunctionCall,
+ Self::Builtin,
+ Self::Type,
+ Self::Namespace,
+ Self::Variable,
+ Self::SelfVariable,
+ Self::Property,
+ Self::Attribute,
+ Self::Macro,
+ Self::Label,
+ Self::Tag,
+ Self::Operator,
+ Self::Punctuation,
+ ];
+}
+
+/// Highlighted ranges of a whole document, packed like LSP semantic tokens.
+#[derive(Clone, Debug, Default, PartialEq, Eq, Serialize, Deserialize, Type)]
+#[serde(rename_all = "camelCase")]
+pub struct SyntaxHighlights {
+ /// Legend: `data` kind `i` is `kinds[i]`.
+ pub kinds: Vec<SyntaxHighlightKind>,
+ /// `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets.
+ pub data: Vec<u32>,
+}
+
+impl SyntaxHighlights {
+ pub(crate) fn new(data: Vec<u32>) -> Self {
+ Self { kinds: SyntaxHighlightKind::ALL.to_vec(), data }
+ }
+}
+
+/// What a capture paints over its node.
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+enum Paint {
+ Kind(SyntaxHighlightKind),
+ /// `@embedded`: code inside a string, uncoloured unless captured itself.
+ Clear,
+}
+
+/// Capture name prefixes, most specific first within each family.
+const CAPTURES: &[(&str, Paint)] = {
+ use Paint::{Clear, Kind};
+ use SyntaxHighlightKind as K;
+ &[
+ ("embedded", Clear),
+ ("keyword", Kind(K::Keyword)),
+ ("include", Kind(K::Keyword)),
+ ("import", Kind(K::Keyword)),
+ ("conditional", Kind(K::Keyword)),
+ ("repeat", Kind(K::Keyword)),
+ ("exception", Kind(K::Keyword)),
+ ("boolean", Kind(K::Keyword)),
+ ("string.escape", Kind(K::Escape)),
+ ("escape", Kind(K::Escape)),
+ ("string.regex", Kind(K::Regexp)),
+ ("string.regexp", Kind(K::Regexp)),
+ ("string.special.symbol", Kind(K::Atom)),
+ ("symbol", Kind(K::Atom)),
+ ("string.special", Kind(K::StringSpecial)),
+ ("string", Kind(K::String)),
+ ("character", Kind(K::String)),
+ ("number", Kind(K::Number)),
+ ("float", Kind(K::Number)),
+ ("constant.builtin", Kind(K::Constant)),
+ ("constant.macro", Kind(K::Macro)),
+ ("constant", Kind(K::Constant)),
+ // nvim greys out unused bindings (`_acc`); IDEA shows them as variables.
+ ("comment.unused", Kind(K::Variable)),
+ ("comment.doc", Kind(K::DocComment)),
+ ("comment.documentation", Kind(K::DocComment)),
+ ("comment", Kind(K::Comment)),
+ ("function.builtin", Kind(K::Builtin)),
+ ("function.macro", Kind(K::Macro)),
+ ("function.definition", Kind(K::Function)),
+ ("function", Kind(K::FunctionCall)),
+ ("method", Kind(K::FunctionCall)),
+ ("constructor", Kind(K::Type)),
+ ("type.builtin", Kind(K::Builtin)),
+ ("type", Kind(K::Type)),
+ ("module", Kind(K::Namespace)),
+ ("namespace", Kind(K::Namespace)),
+ ("variable.builtin", Kind(K::SelfVariable)),
+ ("variable.member", Kind(K::Property)),
+ ("variable.field", Kind(K::Property)),
+ ("variable", Kind(K::Variable)),
+ ("parameter", Kind(K::Variable)),
+ ("property", Kind(K::Property)),
+ ("field", Kind(K::Property)),
+ ("attribute", Kind(K::Attribute)),
+ ("label", Kind(K::Label)),
+ ("tag", Kind(K::Tag)),
+ ("operator", Kind(K::Operator)),
+ ("punctuation", Kind(K::Punctuation)),
+ ]
+};
+
+/// The paint for a capture name: its longest prefix (on `.` boundaries) in
+/// [`CAPTURES`]. Private captures (`@__name__`) and unknown names paint nothing.
+fn paint_for(name: &str) -> Option<Paint> {
+ if name.starts_with('_') {
+ return None;
+ }
+ let mut prefix = name;
+ loop {
+ if let Some((_, p)) = CAPTURES.iter().find(|(n, _)| *n == prefix) {
+ return Some(*p);
+ }
+ prefix = &prefix[..prefix.rfind('.')?];
+ }
+}
+
+pub(crate) struct HighlightQuery {
+ query: Query,
+ /// By capture index.
+ paints: Vec<Option<Paint>>,
+}
+
+impl HighlightQuery {
+ pub(crate) fn new(language: &Language, source: &str) -> Result<Self, tree_sitter::QueryError> {
+ let query = Query::new(language, source)?;
+ let paints = query.capture_names().iter().map(|n| paint_for(n)).collect();
+ Ok(Self { query, paints })
+ }
+}
+
+/// Byte value for "no colour" in the paint buffer; kinds are `1 + index`.
+const NONE: u8 = 0;
+
+/// `[from, to, kind]` UTF-16 triples for `text`; see [`SyntaxHighlights::data`].
+pub(crate) fn highlights(q: &HighlightQuery, tree: &Tree, text: &str) -> Vec<u32> {
+ // (byte len, pattern, start, end, paint): painted largest first, then in
+ // pattern order, so inner nodes and later patterns overwrite.
+ let mut captures: Vec<(usize, usize, usize, usize, Paint)> = Vec::new();
+ let mut cursor = QueryCursor::new();
+ let mut matches = cursor.matches(&q.query, tree.root_node(), text.as_bytes());
+ while let Some(m) = matches.next() {
+ for c in m.captures {
+ if let Some(paint) = q.paints[c.index as usize] {
+ let r = c.node.byte_range();
+ if r.start < r.end {
+ captures.push((r.end - r.start, m.pattern_index, r.start, r.end, paint));
+ }
+ }
+ }
+ }
+ captures.sort_by(|a, b| b.0.cmp(&a.0).then(a.1.cmp(&b.1)));
+
+ let mut painted = vec![NONE; text.len()];
+ for &(_, _, start, end, paint) in &captures {
+ let v = match paint {
+ Paint::Kind(k) => 1 + k as u8,
+ Paint::Clear => NONE,
+ };
+ painted[start..end].fill(v);
+ }
+
+ let lines = LineIndex::new(text);
+ let mut data = Vec::new();
+ let mut i = 0;
+ while i < painted.len() {
+ let v = painted[i];
+ let start = i;
+ while i < painted.len() && painted[i] == v {
+ i += 1;
+ }
+ // Node boundaries are char boundaries; whitespace-only runs stay uncoloured.
+ if v != NONE && text.is_char_boundary(start) && text.is_char_boundary(i) && !text[start..i].trim().is_empty() {
+ data.extend([lines.utf16_at(start) as u32, lines.utf16_at(i) as u32, (v - 1) as u32]);
+ }
+ }
+ data
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn capture_names_fold_to_the_longest_known_prefix() {
+ use SyntaxHighlightKind as K;
+ assert_eq!(paint_for("keyword.function"), Some(Paint::Kind(K::Keyword)));
+ assert_eq!(paint_for("string.special.symbol"), Some(Paint::Kind(K::Atom)));
+ assert_eq!(paint_for("string.special.path"), Some(Paint::Kind(K::StringSpecial)));
+ assert_eq!(paint_for("comment.doc.__attribute__"), Some(Paint::Kind(K::DocComment)));
+ assert_eq!(paint_for("punctuation.bracket"), Some(Paint::Kind(K::Punctuation)));
+ assert_eq!(paint_for("embedded"), Some(Paint::Clear));
+ assert_eq!(paint_for("__name__"), None);
+ assert_eq!(paint_for("spell"), None);
+ }
+
+ /// `(text, kind)` of every highlighted range.
+ fn spans(lang: &str, text: &str) -> Vec<(String, SyntaxHighlightKind)> {
+ let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new(lang), text).unwrap();
+ let units: Vec<u16> = text.encode_utf16().collect();
+ h.data
+ .chunks(3)
+ .map(|t| (String::from_utf16(&units[t[0] as usize..t[1] as usize]).unwrap(), h.kinds[t[2] as usize]))
+ .collect()
+ }
+
+ #[test]
+ fn highlights_elixir() {
+ use SyntaxHighlightKind as K;
+ let src = "defmodule Greeter do\n @moduledoc \"Hi\"\n # 😀 note\n def hello(name, _opts) do\n IO.puts(\"Hello #{name}\")\n :ok\n end\nend\n";
+ let got = spans("elixir", src);
+ let kind = |t: &str| got.iter().find(|(s, _)| s == t).map(|(_, k)| *k);
+ assert_eq!(kind("defmodule"), Some(K::Keyword));
+ assert_eq!(kind("Greeter"), Some(K::Namespace));
+ assert_eq!(kind("\"Hi\""), Some(K::DocComment));
+ assert_eq!(kind("# 😀 note"), Some(K::Comment));
+ assert_eq!(kind("def"), Some(K::Keyword));
+ assert_eq!(kind("hello"), Some(K::FunctionCall));
+ assert_eq!(kind("_opts"), Some(K::Variable));
+ assert_eq!(kind("puts"), Some(K::FunctionCall));
+ // Interpolation clears the string colour; its contents are coloured on their own.
+ assert_eq!(kind("\"Hello "), Some(K::String));
+ assert_eq!(kind("#{"), Some(K::Punctuation));
+ assert_eq!(kind(":ok"), Some(K::Atom));
+ assert_eq!(kind("end"), Some(K::Keyword));
+ // Sorted and non-overlapping.
+ let h = crate::Syntax::new().highlights(&ide_core::LanguageId::new("elixir"), src).unwrap();
+ assert!(h.data.chunks(3).zip(h.data.chunks(3).skip(1)).all(|(a, b)| a[0] < a[1] && a[1] <= b[0]));
+ }
+
+ #[test]
+ fn languages_without_a_query_have_none() {
+ assert_eq!(crate::Syntax::new().highlights(&ide_core::LanguageId::new("rust"), "fn a() {}"), None);
+ }
+
+ #[test]
+ fn legend_matches_discriminants() {
+ for (i, k) in SyntaxHighlightKind::ALL.iter().enumerate() {
+ assert_eq!(*k as usize, i);
+ }
+ }
+}
diff --git a/crates/ide-language/src/syntax/lines.rs b/crates/ide-language/src/syntax/lines.rs
index 7b3291d23..5b8de2062 100644
--- a/crates/ide-language/src/syntax/lines.rs
+++ b/crates/ide-language/src/syntax/lines.rs
@@ -40,7 +40,7 @@ impl<'a> LineIndex<'a> {
}
/// UTF-16 offset of a char-boundary byte offset.
- fn utf16_at(&self, byte: usize) -> usize {
+ pub(crate) fn utf16_at(&self, byte: usize) -> usize {
if self.checkpoints.is_empty() {
return byte;
}
diff --git a/crates/ide-language/src/syntax/mod.rs b/crates/ide-language/src/syntax/mod.rs
index e6633bb2a..783dfe9c4 100644
--- a/crates/ide-language/src/syntax/mod.rs
+++ b/crates/ide-language/src/syntax/mod.rs
@@ -1,8 +1,9 @@
-//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges.
+//! Tree-sitter driven structure: outline, folding, symbol tags, selection ranges, highlighting.
mod cache;
mod folding;
mod grammars;
+mod highlight;
pub(crate) mod lines;
mod outline;
mod selection;
@@ -10,6 +11,7 @@ mod selection;
use crate::{FoldRange, LanguageConfig, LanguageRegistry, SymbolTag};
pub use cache::{MAX_CACHED_TREES, SyntaxCache, SyntaxCacheStats};
use grammars::{SPECS, Spec};
+pub use highlight::{SyntaxHighlightKind, SyntaxHighlights};
use ide_core::{LanguageId, OutlineItem, Range};
use std::cell::RefCell;
use std::collections::HashMap;
@@ -31,6 +33,7 @@ pub(crate) struct Grammar {
pub(crate) spec: &'static Spec,
pub(crate) language: Language,
outline: OnceLock<Option<outline::OutlineQuery>>,
+ highlights: OnceLock<Option<highlight::HighlightQuery>>,
kinds: OnceLock<folding::KindTable>,
}
@@ -50,6 +53,17 @@ impl Grammar {
})
.as_ref()
}
+
+ fn highlight_query(&self) -> Option<&highlight::HighlightQuery> {
+ self.highlights
+ .get_or_init(|| {
+ let source = self.spec.highlights?;
+ highlight::HighlightQuery::new(&self.language, source)
+ .inspect_err(|e| tracing::error!("highlights query for {} is invalid: {e}", self.spec.name))
+ .ok()
+ })
+ .as_ref()
+ }
}
thread_local! {
@@ -72,6 +86,7 @@ impl Syntax {
spec,
language: Language::new(spec.language),
outline: OnceLock::new(),
+ highlights: OnceLock::new(),
kinds: OnceLock::new(),
})
.collect();
@@ -150,6 +165,13 @@ impl Syntax {
.unwrap_or_default()
}
+ /// Highlighting from the grammar's own query; `None` without one.
+ pub fn highlights(&self, lang: &LanguageId, text: &str) -> Option<SyntaxHighlights> {
+ self.analyze(lang, text, |g, tree, text| {
+ Some(SyntaxHighlights::new(highlight::highlights(g.highlight_query()?, tree, text)))
+ })
+ }
+
/// Syntax-aware selection ranges for IDEA "Extend Selection" (⌥↑), innermost first.
pub fn selection_ranges(&self, lang: &LanguageId, text: &str, offset_utf16: u32) -> Vec<Range> {
match self.analyze(lang, text, |_, tree, text| Some(selection::selection_ranges(tree, text, offset_utf16))) {
diff --git a/src-tauri/src/commands/doc.rs b/src-tauri/src/commands/doc.rs
index 59c4f708e..719dcbcad 100644
--- a/src-tauri/src/commands/doc.rs
+++ b/src-tauri/src/commands/doc.rs
@@ -1,6 +1,6 @@
use crate::state::AppState;
use ide_core::{ChangeSet, DocId, IdeError, IdeResult, OutlineItem, Range};
-use ide_language::FoldRange;
+use ide_language::{FoldRange, SyntaxHighlights};
use ide_text::LineEnding;
use ide_workspace::{DocumentInfo, HistoryEntry};
use std::path::PathBuf;
@@ -136,6 +136,22 @@ pub async fn doc_folding(window: Window, state: State<'_, AppState>, doc: DocId)
.await
}
+/// Tree-sitter highlighting for languages the UI has no CodeMirror mode for;
+/// empty when the grammar has no highlights query or the document is large.
+#[tauri::command]
+#[specta::specta]
+pub async fn doc_highlights(window: Window, state: State<'_, AppState>, doc: DocId) -> IdeResult<SyntaxHighlights> {
+ let project = state.project(window.label())?;
+ blocking(move || {
+ let snap = project.documents.text_snapshot(doc)?;
+ if snap.large {
+ return Ok(SyntaxHighlights::default());
+ }
+ Ok(project.syntax.highlights(doc, &snap.language, snap.version, &snap.text))
+ })
+ .await
+}
+
/// Extend Selection (⌥↑): syntax ranges around `offset`, innermost first.
/// Large documents fall back to word / line / document ranges.
#[tauri::command]
diff --git a/src-tauri/src/commands/mod.rs b/src-tauri/src/commands/mod.rs
index 4c4f60fab..dc0d79f65 100644
--- a/src-tauri/src/commands/mod.rs
+++ b/src-tauri/src/commands/mod.rs
@@ -74,6 +74,7 @@ pub fn builder() -> tauri_specta::Builder<tauri::Wry> {
doc::doc_set_line_ending,
doc::doc_outline,
doc::doc_folding,
+ doc::doc_highlights,
doc::doc_selection_ranges,
syntax::syntax_support,
syntax::syntax_node_at,
diff --git a/src/bindings.ts b/src/bindings.ts
index 2afc23058..22f7dc893 100644
--- a/src/bindings.ts
+++ b/src/bindings.ts
@@ -92,6 +92,11 @@ export const commands = {
*/
docOutline: (doc: DocId) => typedError<OutlineItem[], IdeError>(__TAURI_INVOKE("doc_outline", { doc })),
docFolding: (doc: DocId) => typedError<FoldRange[], IdeError>(__TAURI_INVOKE("doc_folding", { doc })),
+ /**
+ * Tree-sitter highlighting for languages the UI has no CodeMirror mode for;
+ * empty when the grammar has no highlights query or the document is large.
+ */
+ docHighlights: (doc: DocId) => typedError<SyntaxHighlights, IdeError>(__TAURI_INVOKE("doc_highlights", { doc })),
/**
* Extend Selection (⌥↑): syntax ranges around `offset`, innermost first.
* Large documents fall back to word / line / document ranges.
@@ -3683,6 +3688,17 @@ export type SyntaxErrors = {
truncated: boolean,
};
+/** Highlight classes sent to the UI, which maps each onto a Lezer tag. */
+export type SyntaxHighlightKind = "keyword" | "string" | "stringSpecial" | "escape" | "regexp" | "atom" | "number" | "constant" | "comment" | "docComment" | "function" | "functionCall" | "builtin" | "type" | "namespace" | "variable" | "selfVariable" | "property" | "attribute" | "macro" | "label" | "tag" | "operator" | "punctuation";
+
+/** Highlighted ranges of a whole document, packed like LSP semantic tokens. */
+export type SyntaxHighlights = {
+ /** Legend: `data` kind `i` is `kinds[i]`. */
+ kinds: SyntaxHighlightKind[],
+ /** `[from, to, kind]` triples: sorted, non-overlapping UTF-16 offsets. */
+ data: number[],
+};
+
/** One tree-sitter node. */
export type SyntaxNode = {
/** Grammar node kind, e.g. `method_invocation`, `identifier`, `;`. */
diff --git a/src/editor/core/backend.ts b/src/editor/core/backend.ts
index 75de63a25..e075ec6e9 100644
--- a/src/editor/core/backend.ts
+++ b/src/editor/core/backend.ts
@@ -18,6 +18,7 @@ import {
type LineEnding,
type OutlineItem,
type Range,
+ type SyntaxHighlights,
type TextChange,
} from "@/platform/ipc";
@@ -43,6 +44,8 @@ export interface DocBackend {
setEncoding(doc: DocId, encoding: string, reload: boolean): Promise<string>;
outline(doc: DocId): Promise<OutlineItem[]>;
folding(doc: DocId): Promise<FoldRange[]>;
+ /** Tree-sitter highlighting, for languages without a CodeMirror mode. */
+ highlights(doc: DocId): Promise<SyntaxHighlights>;
selectionRanges(doc: DocId, offset: number): Promise<Range[]>;
languages(): Promise<LanguageInfo[]>;
onPatched(handler: (e: DocPatchedEvent) => void): () => void;
@@ -62,6 +65,7 @@ export const ipcBackend: DocBackend = {
setEncoding: (doc, encoding, reload) => call(commands.docSetEncoding(doc, encoding, reload)),
outline: (doc) => call(commands.docOutline(doc)),
folding: (doc) => call(commands.docFolding(doc)),
+ highlights: (doc) => call(commands.docHighlights(doc)),
selectionRanges: (doc, offset) => call(commands.docSelectionRanges(doc, offset)),
languages: () => commands.languages(),
onPatched: (h) => onIdeEvent("docPatched", (e) => h(e)),
@@ -343,6 +347,9 @@ export function createMemoryBackend(files: Record<string, string> = {}): MemoryB
}
return items;
},
+ async highlights() {
+ return { kinds: [], data: [] };
+ },
async folding(doc) {
const d = get(doc);
const out: FoldRange[] = [];
diff --git a/src/editor/core/cm/setup.ts b/src/editor/core/cm/setup.ts
index 62cb91df2..e5618dde9 100644
--- a/src/editor/core/cm/setup.ts
+++ b/src/editor/core/cm/setup.ts
@@ -25,6 +25,7 @@ import { openideaTheme } from "./theme";
import { caretRowHighlight } from "./caretRow";
import { changeLane, overviewLane } from "./lanes";
import { foldingExtension } from "./folding";
+import { syntaxHighlightField } from "./syntaxHighlight";
import { indentGuides } from "./indentGuides";
import { stickyMaxLines, stickyScroll } from "./stickyScroll";
import { findExtension } from "./find";
@@ -267,6 +268,7 @@ export function baseExtensions(o: SetupOptions): Extension {
clipboardHistoryExtension,
indentOnInput(),
syntaxHighlighting(openideaHighlightStyle),
+ syntaxHighlightField,
closeBrackets(),
caretRowHighlight,
fontZoom,
diff --git a/src/editor/core/cm/syntaxHighlight.test.ts b/src/editor/core/cm/syntaxHighlight.test.ts
new file mode 100644
index 000000000..a3a1ecbbb
--- /dev/null
+++ b/src/editor/core/cm/syntaxHighlight.test.ts
@@ -0,0 +1,37 @@
+import { describe, expect, it } from "vitest";
+import { EditorState } from "@codemirror/state";
+import { highlightingFor, syntaxHighlighting } from "@codemirror/language";
+import { tags as t } from "@lezer/highlight";
+import { openideaHighlightStyle } from "./highlight";
+import { setSyntaxHighlights, syntaxHighlightField } from "./syntaxHighlight";
+
+function marks(state: EditorState) {
+ const out: { text: string; cls: string }[] = [];
+ state.field(syntaxHighlightField).between(0, state.doc.length, (from, to, d) => {
+ out.push({ text: state.doc.sliceString(from, to), cls: d.spec.class as string });
+ });
+ return out;
+}
+
+describe("syntaxHighlightField", () => {
+ const create = () =>
+ EditorState.create({ doc: "def hi, do: :ok", extensions: [syntaxHighlighting(openideaHighlightStyle), syntaxHighlightField] });
+
+ it("styles backend ranges with the active highlight style and maps them through edits", () => {
+ let state = create();
+ state = state.update({
+ effects: setSyntaxHighlights.of({ kinds: ["keyword", "atom", "functionCall"], data: [0, 3, 0, 4, 6, 2, 12, 15, 1, 99, 120, 0] }),
+ }).state;
+ const keyword = highlightingFor(state, [t.keyword]);
+ const atom = highlightingFor(state, [t.atom]);
+ expect(keyword).toBeTruthy();
+ expect(marks(state)).toEqual([
+ { text: "def", cls: keyword },
+ { text: "hi", cls: highlightingFor(state, [t.function(t.variableName)]) },
+ { text: ":ok", cls: atom },
+ ]);
+
+ state = state.update({ changes: { from: 0, insert: " " } }).state;
+ expect(marks(state).map((m) => m.text)).toEqual(["def", "hi", ":ok"]);
+ });
+});
diff --git a/src/editor/core/cm/syntaxHighlight.ts b/src/editor/core/cm/syntaxHighlight.ts
new file mode 100644
index 000000000..8ced4899d
--- /dev/null
+++ b/src/editor/core/cm/syntaxHighlight.ts
@@ -0,0 +1,66 @@
+/**
+ * Tree-sitter highlighting (`doc_highlights`) for languages without a
+ * CodeMirror mode (Elixir, ...): backend kinds → Lezer tags, so the colours
+ * come from the same highlight style and Color Scheme settings as Lezer
+ * highlighting. Refreshed with folding; in between, marks are mapped through
+ * the changes.
+ */
+import { StateEffect, StateField, type EditorState } from "@codemirror/state";
+import { Decoration, EditorView, type DecorationSet } from "@codemirror/view";
+import { highlightingFor } from "@codemirror/language";
+import { tags as t, type Tag } from "@lezer/highlight";
+import type { SyntaxHighlightKind, SyntaxHighlights } from "@/platform/ipc";
+
+const TAGS: Record<SyntaxHighlightKind, Tag> = {
+ keyword: t.keyword,
+ string: t.string,
+ stringSpecial: t.special(t.string),
+ escape: t.escape,
+ regexp: t.regexp,
+ atom: t.atom,
+ number: t.number,
+ constant: t.constant(t.variableName),
+ comment: t.comment,
+ docComment: t.docComment,
+ function: t.function(t.definition(t.variableName)),
+ functionCall: t.function(t.variableName),
+ builtin: t.standard(t.variableName),
+ type: t.typeName,
+ namespace: t.namespace,
+ variable: t.variableName,
+ selfVariable: t.self,
+ property: t.propertyName,
+ attribute: t.annotation,
+ macro: t.macroName,
+ label: t.labelName,
+ tag: t.tagName,
+ operator: t.operator,
+ punctuation: t.punctuation,
+};
+
+export const setSyntaxHighlights = StateEffect.define<SyntaxHighlights>();
+
+function decorations(state: EditorState, h: SyntaxHighlights): DecorationSet {
+ const marks = h.kinds.map((k) => {
+ const cls = highlightingFor(state, [TAGS[k]]);
+ return cls ? Decoration.mark({ class: cls }) : null;
+ });
+ const len = state.doc.length;
+ const ranges = [];
+ for (let i = 0; i + 2 < h.data.length; i += 3) {
+ const from = h.data[i]!;
+ const to = Math.min(h.data[i + 1]!, len);
+ const mark = marks[h.data[i + 2]!];
+ if (mark && from < to) ranges.push(mark.range(from, to));
+ }
+ return Decoration.set(ranges, true);
+}
+
+export const syntaxHighlightField = StateField.define<DecorationSet>({
+ create: () => Decoration.none,
+ update(deco, tr) {
+ for (const e of tr.effects) if (e.is(setSyntaxHighlights)) return decorations(tr.state, e.value);
+ return tr.docChanged ? deco.map(tr.changes) : deco;
+ },
+ provide: (f) => EditorView.decorations.from(f),
+});
diff --git a/src/editor/core/model/document.ts b/src/editor/core/model/document.ts
index 799e6e6c9..981289c97 100644
--- a/src/editor/core/model/document.ts
+++ b/src/editor/core/model/document.ts
@@ -7,7 +7,7 @@ import { EditorView } from "@codemirror/view";
import type { LanguageDescription, LanguageSupport } from "@codemirror/language";
import { create } from "zustand";
import type { EditorConfigProps } from "@/bindings";
-import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings } from "@/platform/ipc";
+import type { DocId, DocumentInfo, EditorSettings, FileSettings, LineEnding, OutlineItem, Settings, SyntaxHighlights } from "@/platform/ipc";
import {
_emitEditorEvent,
_saveParticipants,
@@ -22,6 +22,7 @@ import type { DocBackend, LanguageInfo } from "../backend";
import { findLanguageDescription, languageInfo, loadLanguageInfos, loadSupport, loadedSupport, commentTokens } from "../cm/languages";
import { foldEffect } from "@codemirror/language";
import { backendFolds, setBackendFolds } from "../cm/folding";
+import { setSyntaxHighlights } from "../cm/syntaxHighlight";
import {
caretFor,
compartments,
@@ -265,6 +266,7 @@ export class DocModel {
addSlot(slot: EditorSlot) {
this.slots.add(slot);
+ if (this.highlights?.counter === this.changeCounter) slot.reconfigure([setSyntaxHighlights.of(this.highlights.value)]);
}
removeSlot(slot: EditorSlot) {
@@ -332,6 +334,8 @@ export class DocModel {
indentOverride: { tabSize: number; insertSpaces: boolean } | null = null;
/** Indentation detected in the file's text at open time. */
detectedIndent: { insertSpaces: boolean; size?: number } | null = null;
+ /** Last tree-sitter highlighting, for slots opened before the next refresh. */
+ private highlights: { counter: number; value: SyntaxHighlights } | null = null;
/** Fold-by-default has been applied (first folding ranges of the document). */
private initialFolds = false;
/** Signature of settings that require contributions to be rebuilt. */
@@ -423,7 +427,12 @@ export class DocModel {
await this.sync.whenIdle();
const counter = this.changeCounter;
const backend = this.manager.backend;
- const [folds, outline] = await Promise.allSettled([backend.folding(this.doc), backend.outline(this.doc)]);
+ const [folds, outline, highlights] = await Promise.allSettled([
+ backend.folding(this.doc),
+ backend.outline(this.doc),
+ // Without a CodeMirror mode, colours come from the grammar's tree-sitter query.
+ this.desc ? null : backend.highlights(this.doc),
+ ]);
if (this.closed) return;
if (counter !== this.changeCounter) {
this.scheduleRefresh();
@@ -436,6 +445,10 @@ export class DocModel {
this.foldByDefault();
}
}
+ if (highlights.status === "fulfilled" && highlights.value) {
+ this.highlights = { counter, value: highlights.value };
+ for (const s of this.slots) s.reconfigure([setSyntaxHighlights.of(highlights.value)]);
+ }
if (outline.status === "fulfilled") {
useOutline.setState({ [this.path]: outline.value });
const lines = methodLines(outline.value);