Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.
editor: Resolve semantic token overlaps by configured server order (#61735)
Drew Ridleybbd198f57ba9
diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rsindex 7594c7d0951..95a58b0cf95 100644--- a/crates/editor/src/display_map.rs+++ b/crates/editor/src/display_map.rs@@ -162,7 +162,7 @@ pub enum HighlightKey {// Note we want semantic tokens > colorized brackets// to allow language server highlights to work over brackets.ColorizeBracket(usize),- SemanticToken,+ SemanticToken(u32),// below is sorted lexicographically, as there is no relevant ordering for these aside from coming after the aboveBufferSearchHighlights,ConsoleAnsiHighlight(usize),@@ -360,6 +360,7 @@ pub struct SemanticTokenHighlight {pub token_type: TokenType,pub token_modifiers: u32,pub server_id: lsp::LanguageServerId,+ pub precedence: u32,}impl DisplayMap {diff --git a/crates/editor/src/display_map/custom_highlights.rs b/crates/editor/src/display_map/custom_highlights.rsindex 8ca43fe3cd2..828d86edd50 100644--- a/crates/editor/src/display_map/custom_highlights.rs+++ b/crates/editor/src/display_map/custom_highlights.rs@@ -185,12 +185,12 @@ fn create_highlight_endpoints(}highlight_endpoints.push(HighlightEndpoint {offset: *start,- tag: HighlightKey::SemanticToken,+ tag: HighlightKey::SemanticToken(token.precedence),style: Some(interner[token.style]),});highlight_endpoints.push(HighlightEndpoint {offset: end,- tag: HighlightKey::SemanticToken,+ tag: HighlightKey::SemanticToken(token.precedence),style: None,});},@@ -249,11 +249,22 @@ impl<'a> Iterator for CustomHighlightsChunks<'a> {chunk.newlines = chunk.newlines.unbounded_shr(split_idx as u32);chunk.text = suffix;if !self.active_highlights.is_empty() {- prefix.highlight_style = self- .active_highlights- .values()- .copied()- .reduce(|acc, active_highlight| acc.highlight(active_highlight));+ let mut active_highlights = self.active_highlights.iter().peekable();+ while let Some((tag, style)) = active_highlights.next() {+ let overridden_by_later_semantic_token =+ matches!(tag, HighlightKey::SemanticToken(_))+ && matches!(+ active_highlights.peek(),+ Some((HighlightKey::SemanticToken(_), _))+ );+ if overridden_by_later_semantic_token {+ continue;+ }+ prefix.highlight_style = Some(match prefix.highlight_style {+ Some(active_highlight) => active_highlight.highlight(*style),+ None => *style,+ });+ }}Some(prefix)}diff --git a/crates/editor/src/semantic_tokens.rs b/crates/editor/src/semantic_tokens.rsindex 65d22d18673..b15ec323a34 100644--- a/crates/editor/src/semantic_tokens.rs+++ b/crates/editor/src/semantic_tokens.rs@@ -6,8 +6,10 @@ use gpui::{App, Context, FontStyle, FontWeight, HighlightStyle, StrikethroughStyle, Task, UnderlineStyle,};use itertools::Itertools;-use language::language_settings::LanguageSettings;+use language::{LanguageName, LanguageRegistry, language_settings::LanguageSettings};+use lsp::LanguageServerId;use project::{+ LspStore,lsp_store::{BufferSemanticToken, BufferSemanticTokens, SemanticTokenStylizer, TokenType},project_settings::ProjectSettings,};@@ -286,6 +288,15 @@ impl Editor {let Some(project) = project.upgrade() else {return;};+ let precedences = {+ let project = project.read(cx);+ server_precedences(+ project.lsp_store().read(cx),+ project.languages(),+ language_name.as_ref(),+ &tokens,+ )+ };editor.display_map.update(cx, |display_map, cx| {project.read(cx).lsp_store().update(cx, |lsp_store, cx| {let mut token_highlights = Vec::new();@@ -303,6 +314,7 @@ impl Editor {token_highlights.extend(buffer_into_editor_highlights(&server_tokens,stylizer,+ precedences.get(&server_id).copied().unwrap_or_default(),&multi_buffer_snapshot,&mut interner,theme,@@ -310,7 +322,10 @@ impl Editor {}token_highlights.sort_by(|a, b| {- a.range.start.cmp(&b.range.start, &multi_buffer_snapshot)+ a.range+ .start+ .cmp(&b.range.start, &multi_buffer_snapshot)+ .then_with(|| a.precedence.cmp(&b.precedence))});Arc::make_mut(&mut display_map.semantic_token_highlights).insert(buffer_id,@@ -327,9 +342,44 @@ impl Editor {}}+fn server_precedences(+ lsp_store: &LspStore,+ languages: &LanguageRegistry,+ language_name: Option<&LanguageName>,+ tokens: &HashMap<LanguageServerId, Arc<[BufferSemanticToken]>>,+) -> HashMap<LanguageServerId, u32> {+ let ordered_adapters = language_name+ .map(|language_name| languages.lsp_adapters(language_name))+ .unwrap_or_default();++ // We base this on the ids that are present rather than the adapter count,+ // otherwise a server we can't resolve an adapter for outranks the+ // configured ones and overrides them+ let configured_base = tokens.keys().map(|id| id.0).max().map_or(0, |max| max + 1);++ tokens+ .keys()+ .map(|&server_id| {+ let configured_precedence = lsp_store+ .language_server_adapter_for_id(server_id)+ .and_then(|adapter| {+ ordered_adapters+ .iter()+ .position(|ordered_adapter| ordered_adapter.name == adapter.name)+ });+ let precedence = match configured_precedence {+ Some(index) => configured_base + index,+ None => server_id.0,+ };+ (server_id, precedence as u32)+ })+ .collect()+}+fn buffer_into_editor_highlights<'a, 'b>(buffer_tokens: &'a [BufferSemanticToken],stylizer: &'a SemanticTokenStylizer,+ precedence: u32,multi_buffer_snapshot: &'a multi_buffer::MultiBufferSnapshot,interner: &'b mut HighlightStyleInterner,theme: &'a SyntaxTheme,@@ -343,7 +393,7 @@ fn buffer_into_editor_highlights<'a, 'b>(.into_iter().tuples::<(_, _)>().zip(buffer_tokens)- .filter_map(|((multi_buffer_start, multi_buffer_end), token)| {+ .filter_map(move |((multi_buffer_start, multi_buffer_end), token)| {let range = multi_buffer_start?..multi_buffer_end?;let style = convert_token(stylizer, theme, token.token_type, token.token_modifiers)?;let style = interner.intern(style);@@ -353,6 +403,7 @@ fn buffer_into_editor_highlights<'a, 'b>(token_type: token.token_type,token_modifiers: token.token_modifiers,server_id: stylizer.server_id(),+ precedence,})})}@@ -464,6 +515,7 @@ mod tests {use futures::StreamExt as _;use gpui::{AppContext as _, Entity, Focusable as _, HighlightStyle, TestAppContext, UpdateGlobal as _,+ VisualTestContext,};use language::{Diagnostic, DiagnosticEntry, DiagnosticSet, Language, LanguageAwareStyling, LanguageConfig,@@ -478,8 +530,8 @@ mod tests {use rope::{Point, PointUtf16};use serde_json::json;use settings::{- GlobalLspSettingsContent, LanguageSettingsContent, SemanticTokenRule, SemanticTokenRules,- SemanticTokens, SettingsStore,+ ConfiguredLanguageServer, GlobalLspSettingsContent, LanguageSettingsContent,+ SemanticTokenRule, SemanticTokenRules, SemanticTokens, SettingsStore,};use workspace::{MultiWorkspace, WorkspaceHandle as _};@@ -1063,6 +1115,285 @@ mod tests {assert_eq!(full_counter_toml_2.load(atomic::Ordering::Acquire), 1);}+ #[gpui::test]+ async fn lsp_semantic_tokens_multiserver_precedence(cx: &mut TestAppContext) {+ use gpui::Rgba;++ init_test(cx, |_| {});++ let toml_settings = |language_servers: [&str; 2]| LanguageSettingsContent {+ semantic_tokens: Some(SemanticTokens::Full),+ language_servers: Some(+ language_servers+ .into_iter()+ .map(ConfiguredLanguageServer::new)+ .collect(),+ ),+ ..LanguageSettingsContent::default()+ };++ update_test_language_settings(cx, &|language_settings| {+ language_settings+ .languages+ .0+ .insert("TOML".into(), toml_settings(["toml1", "toml2"]));+ });++ let red = Rgba {+ r: 1.0,+ g: 0.0,+ b: 0.0,+ a: 1.0,+ };+ let blue = Rgba {+ r: 0.0,+ g: 0.0,+ b: 1.0,+ a: 1.0,+ };+ cx.update(|cx| {+ SettingsStore::update_global(cx, |store, cx| {+ store.update_user_settings(cx, |settings| {+ settings.global_lsp_settings = Some(GlobalLspSettingsContent {+ semantic_token_rules: Some(SemanticTokenRules {+ rules: Vec::from([+ SemanticTokenRule {+ token_type: Some("property".to_string()),+ foreground_color: Some(red),+ font_style: Some(SemanticTokenFontStyle::Italic),+ ..SemanticTokenRule::default()+ },+ SemanticTokenRule {+ token_type: Some("number".to_string()),+ foreground_color: Some(blue),+ ..SemanticTokenRule::default()+ },+ ]),+ }),+ ..GlobalLspSettingsContent::default()+ });+ });+ });+ });++ let toml_language = Arc::new(Language::new(+ LanguageConfig {+ name: "TOML".into(),+ matcher: (LanguageMatcher {+ path_suffixes: vec!["toml".into()],+ ..LanguageMatcher::default()+ })+ .into(),+ ..LanguageConfig::default()+ },+ None,+ ));++ let app_state = cx.update(workspace::AppState::test);++ cx.update(|cx| {+ assets::Assets.load_test_fonts(cx);+ crate::init(cx);+ workspace::init(app_state.clone(), cx);+ });++ let project = Project::test(app_state.fs.clone(), [], cx).await;+ let language_registry = project.read_with(cx, |project, _| project.languages().clone());++ let mut toml_server_1 = language_registry.register_fake_lsp(+ toml_language.name(),+ FakeLspAdapter {+ name: "toml1",+ capabilities: lsp::ServerCapabilities {+ semantic_tokens_provider: Some(+ lsp::SemanticTokensServerCapabilities::SemanticTokensOptions(+ lsp::SemanticTokensOptions {+ legend: lsp::SemanticTokensLegend {+ token_types: vec!["property".into()],+ token_modifiers: Vec::new(),+ },+ full: Some(lsp::SemanticTokensFullOptions::Delta { delta: None }),+ ..lsp::SemanticTokensOptions::default()+ },+ ),+ ),+ ..lsp::ServerCapabilities::default()+ },+ initializer: Some(Box::new(move |fake_server| {+ fake_server+ .set_request_handler::<lsp::request::SemanticTokensFullRequest, _, _>(+ move |_, _| async move {+ Ok(Some(lsp::SemanticTokensResult::Tokens(+ lsp::SemanticTokens {+ data: vec![+ 0, // delta_line+ 0, // delta_start+ 5, // length+ 0, // token_type+ 0, // token_modifiers_bitset+ ],+ result_id: Some("a".into()),+ },+ )))+ },+ );+ })),+ ..FakeLspAdapter::default()+ },+ );+ let mut toml_server_2 = language_registry.register_fake_lsp(+ toml_language.name(),+ FakeLspAdapter {+ name: "toml2",+ capabilities: lsp::ServerCapabilities {+ semantic_tokens_provider: Some(+ lsp::SemanticTokensServerCapabilities::SemanticTokensOptions(+ lsp::SemanticTokensOptions {+ legend: lsp::SemanticTokensLegend {+ token_types: vec!["number".into()],+ token_modifiers: Vec::new(),+ },+ full: Some(lsp::SemanticTokensFullOptions::Delta { delta: None }),+ ..lsp::SemanticTokensOptions::default()+ },+ ),+ ),+ ..lsp::ServerCapabilities::default()+ },+ initializer: Some(Box::new(move |fake_server| {+ fake_server+ .set_request_handler::<lsp::request::SemanticTokensFullRequest, _, _>(+ move |_, _| async move {+ Ok(Some(lsp::SemanticTokensResult::Tokens(+ lsp::SemanticTokens {+ data: vec![+ 0, // delta_line+ 2, // delta_start+ 1, // length+ 0, // token_type+ 0, // token_modifiers_bitset+ ],+ result_id: Some("a".into()),+ },+ )))+ },+ );+ })),+ ..FakeLspAdapter::default()+ },+ );+ language_registry.add(toml_language.clone());++ app_state+ .fs+ .as_fake()+ .insert_tree(+ EditorLspTestContext::root_path(),+ json!({+ ".git": {},+ "dir": {+ "foo.toml": "a = 1\nb = 2\n",+ }+ }),+ )+ .await;++ let (multi_workspace, cx) =+ cx.add_window_view(|window, cx| MultiWorkspace::test_new(project.clone(), window, cx));+ let workspace = multi_workspace.read_with(cx, |mw, _| mw.workspace().clone());+ project+ .update(cx, |project, cx| {+ project.find_or_create_worktree(EditorLspTestContext::root_path(), true, cx)+ })+ .await+ .unwrap();+ cx.read(|cx| workspace.read(cx).worktree_scans_complete(cx))+ .await;++ let toml_file = cx.read(|cx| workspace.file_project_paths(cx)[0].clone());+ let toml_item = workspace+ .update_in(cx, |workspace, window, cx| {+ workspace.open_path(toml_file, None, true, window, cx)+ })+ .await+ .expect("Could not open test file");++ let editor = cx.update(|_, cx| {+ toml_item+ .act_as::<Editor>(cx)+ .expect("Opened test file wasn't an editor")+ });++ editor.update_in(cx, |editor, window, cx| {+ window.focus(&editor.focus_handle(cx), cx)+ });++ let _toml_server_1 = toml_server_1.next().await.unwrap();+ let _toml_server_2 = toml_server_2.next().await.unwrap();++ let refetch_semantic_tokens = |editor: &Entity<Editor>, cx: &mut VisualTestContext| {+ editor.update_in(cx, |editor, _, cx| {+ editor.edit([(MultiBufferOffset(0)..MultiBufferOffset(1), "a")], cx);+ });+ cx.executor().advance_clock(Duration::from_millis(200));+ let task = editor.update_in(cx, |e, _, _| e.semantic_token_state.take_update_task());+ cx.run_until_parked();+ task+ };++ refetch_semantic_tokens(&editor, cx).await;++ let red_italic = HighlightStyle {+ color: Some(red.into()),+ font_style: Some(FontStyle::Italic),+ ..HighlightStyle::default()+ };+ let blue = HighlightStyle {+ color: Some(blue.into()),+ ..HighlightStyle::default()+ };+ assert_eq!(+ extract_painted_semantic_styles(&editor, cx),+ vec![+ Some(red_italic),+ Some(red_italic),+ Some(blue),+ Some(red_italic),+ Some(red_italic),+ None,+ ],+ "`toml2` is configured last, so it replaces `toml1` where they overlap and `toml1` resumes after"+ );++ cx.update(|_, cx| {+ SettingsStore::update_global(cx, |store, cx| {+ store.update_user_settings(cx, |settings| {+ settings+ .project+ .all_languages+ .languages+ .0+ .insert("TOML".into(), toml_settings(["toml2", "toml1"]));+ });+ });+ });++ refetch_semantic_tokens(&editor, cx).await;++ assert_eq!(+ extract_painted_semantic_styles(&editor, cx),+ vec![+ Some(red_italic),+ Some(red_italic),+ Some(red_italic),+ Some(red_italic),+ Some(red_italic),+ None,+ ],+ "`toml1` is configured last, so it replaces `toml2` across its entire range"+ );+ }+#[gpui::test]async fn lsp_semantic_tokens_multibuffer_part(cx: &mut TestAppContext) {init_test(cx, |_| {});@@ -2568,6 +2899,30 @@ mod tests {);}+ fn extract_painted_semantic_styles(+ editor: &Entity<Editor>,+ cx: &mut VisualTestContext,+ ) -> Vec<Option<HighlightStyle>> {+ editor.update_in(cx, |editor, window, cx| {+ editor+ .snapshot(window, cx)+ .display_snapshot+ .chunks(+ crate::display_map::DisplayRow(0)..crate::display_map::DisplayRow(1),+ LanguageAwareStyling {+ tree_sitter: false,+ diagnostics: false,+ },+ crate::HighlightStyles::default(),+ )+ .flat_map(|chunk| {+ let style = chunk.highlight_style;+ chunk.text.chars().map(move |_| style).collect::<Vec<_>>()+ })+ .collect()+ })+ }+fn extract_semantic_highlight_styles(editor: &Entity<Editor>,cx: &TestAppContext,