Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

language: Cache buffer syntax highlighting per row chunk (#63145)

Kirill Bulatovd7f28899166c
diff --git a/crates/benchmarks/benches/editor_render.rs b/crates/benchmarks/benches/editor_render.rs
index 695cab7e8ff..e4f7492f8d5 100644
--- a/crates/benchmarks/benches/editor_render.rs
+++ b/crates/benchmarks/benches/editor_render.rs
@@ -1,14 +1,20 @@
use std::{path::PathBuf, sync::Arc};
+use benchmarks::bench_utils::random_rust_file;
use editor::{
Editor, EditorMode, MultiBuffer,
actions::{DeleteToPreviousWordStart, SelectAll, SplitSelectionIntoLines},
};
-use gpui::{App, AppContext as _, BenchAppContext, BorrowAppContext as _, Focusable as _};
+use gpui::{
+ App, AppContext as _, BenchAppContext, BorrowAppContext as _, Focusable as _, UpdateGlobal as _,
+};
use indoc::{formatdoc, indoc};
-use language::{Buffer, Capability, DiskState, File, LocalFile};
+use language::{Buffer, Capability, DiskState, File, LocalFile, Rope};
use rand::{Rng as _, SeedableRng as _, rngs::StdRng};
-use settings::{LocalSettingsKind, LocalSettingsPath, SettingsStore, WorktreeId};
+use settings::{
+ DisplayIn, LocalSettingsKind, LocalSettingsPath, SettingsStore, ShowMinimap, WorktreeId,
+};
+use theme::ActiveTheme as _;
use util::{RandomCharIter, paths::PathStyle, rel_path::RelPath};
use zed_actions::editor::{MoveDown, MoveUp};
@@ -352,6 +358,78 @@ fn jetbrains_editorconfig() -> String {
content
}
+#[gpui::bench]
+fn editor_render_highlighted(cx: &mut BenchAppContext) {
+ init_context(cx);
+ render_highlighted_editor(cx);
+}
+
+#[gpui::bench]
+fn editor_render_highlighted_minimap(cx: &mut BenchAppContext) {
+ init_context(cx);
+ cx.update(|cx| {
+ SettingsStore::update_global(cx, |store, cx| {
+ store.update_user_settings(cx, |settings| {
+ let minimap = settings.editor.minimap.get_or_insert_default();
+ minimap.show = Some(ShowMinimap::Always);
+ minimap.display_in = Some(DisplayIn::AllEditors);
+ });
+ });
+ });
+ render_highlighted_editor(cx);
+}
+
+fn render_highlighted_editor(cx: &mut BenchAppContext) {
+ let mut rng = StdRng::seed_from_u64(1);
+ let text = random_rust_file(&mut rng, 10_000).join("\n");
+ let language = language::rust_lang();
+ let syntax_theme = cx.update(|cx| {
+ let syntax_theme = cx.theme().syntax().clone();
+ language.set_theme(&syntax_theme);
+ syntax_theme
+ });
+ let probe = "fn main() {}";
+ assert!(
+ !language
+ .highlight_text(&Rope::from(probe), 0..probe.len())
+ .is_empty(),
+ "the benchmark language must resolve syntax highlights against the theme"
+ );
+ let buffer = cx.update(|cx| {
+ let buffer = cx.new(|cx| Buffer::local(text, cx).with_language(language, cx));
+ cx.new(|cx| MultiBuffer::singleton(buffer, cx))
+ });
+ cx.run_until_idle();
+
+ let mut window = cx.add_empty_window();
+ let editor = window.update(|window, cx| {
+ let editor = window.replace_root(cx, |window, cx| {
+ let mut editor = Editor::new(EditorMode::full(), buffer, None, window, cx);
+ editor.set_style(
+ editor::EditorStyle {
+ syntax: syntax_theme.clone(),
+ ..editor::EditorStyle::default()
+ },
+ window,
+ cx,
+ );
+ editor
+ });
+ window.focus(&editor.focus_handle(cx), cx);
+ editor
+ });
+
+ let mut move_down = true;
+ cx.bench_renderer(editor, move |editor, window, cx| {
+ if move_down {
+ editor.move_down(&MoveDown, window, cx);
+ } else {
+ editor.move_up(&MoveUp, window, cx);
+ }
+ move_down = !move_down;
+ });
+}
+
fn init_context(cx: &mut BenchAppContext) {
cx.update(|cx| {
let store = SettingsStore::test(cx);
@@ -375,6 +453,8 @@ gpui::bench_group!(
editor_multi_cursor_input,
open_editor_with_one_long_line,
editor_render,
- editor_render_with_editorconfig
+ editor_render_with_editorconfig,
+ editor_render_highlighted,
+ editor_render_highlighted_minimap
);
gpui::bench_main!(benches);
diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs
index 246b786ee3f..9dedf63bc2b 100644
--- a/crates/language/src/buffer.rs
+++ b/crates/language/src/buffer.rs
@@ -9,11 +9,12 @@ use crate::{
diagnostic_set::{DiagnosticEntry, DiagnosticEntryRef, DiagnosticGroup},
language_settings::{AutoIndentMode, LanguageSettings},
outline::OutlineItem,
- row_chunk::RowChunks,
+ row_chunk::{RowChunkId, RowChunks},
runnable::{self, RunnableRange},
syntax_map::{
MAX_BYTES_TO_QUERY, SyntaxLayer, SyntaxMap, SyntaxMapCapture, SyntaxMapCaptures,
SyntaxMapMatch, SyntaxMapMatches, SyntaxSnapshot, ToTreeSitterPoint,
+ flattened_highlight_regions,
},
text_diff::text_diff,
unified_diff_with_offsets,
@@ -35,6 +36,7 @@ use gpui::{
App, AppContext as _, Context, Entity, EventEmitter, HighlightStyle, SharedString, StyledText,
Task, TextStyle,
};
+use language_core::highlight_cache::{ChunkHighlightCache, ResolvedHighlights};
use lsp::LanguageServerId;
use parking_lot::Mutex;
@@ -149,21 +151,25 @@ pub struct Buffer {
#[derive(Debug)]
pub struct TreeSitterData {
chunks: RowChunks,
- brackets_by_chunks: Mutex<HashMap<usize, Vec<BracketMatch>>>,
+ brackets_by_chunks: Mutex<HashMap<RowChunkId, Vec<BracketMatch>>>,
+ highlights_by_chunks: ChunkHighlightCache,
}
-const MAX_ROWS_IN_A_CHUNK: u32 = 50;
+pub(crate) const MAX_ROWS_IN_A_CHUNK: u32 = 50;
+pub(crate) const MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK: usize = 4 * MAX_BYTES_TO_QUERY;
impl TreeSitterData {
fn clear(&mut self, snapshot: &text::BufferSnapshot) {
self.chunks = RowChunks::new(snapshot, MAX_ROWS_IN_A_CHUNK);
self.brackets_by_chunks.get_mut().clear();
+ self.highlights_by_chunks.clear();
}
fn new(snapshot: &text::BufferSnapshot) -> Self {
Self {
chunks: RowChunks::new(snapshot, MAX_ROWS_IN_A_CHUNK),
brackets_by_chunks: Mutex::new(HashMap::default()),
+ highlights_by_chunks: ChunkHighlightCache::default(),
}
}
@@ -519,6 +525,13 @@ struct BufferChunkHighlights<'a> {
highlight_maps: Vec<HighlightMap>,
}
+type HighlightRun = (Range<usize>, HighlightId);
+
+struct CachedChunkHighlightsIter {
+ runs: Vec<HighlightRun>,
+ ix: usize,
+}
+
/// An iterator that yields chunks of a buffer's text, along with their
/// syntax highlights and diagnostic status.
pub struct BufferChunks<'a> {
@@ -533,6 +546,7 @@ pub struct BufferChunks<'a> {
unnecessary_depth: usize,
underline: bool,
highlights: Option<BufferChunkHighlights<'a>>,
+ cached_highlights: Option<CachedChunkHighlightsIter>,
}
/// A chunk of a buffer's text, along with its syntax highlight and
@@ -1553,6 +1567,7 @@ impl Buffer {
}
self.non_text_state_update_count += 1;
self.syntax_map.lock().clear(&self.text);
+ Self::invalidate_tree_sitter_data(&mut self.tree_sitter_data, self.text.snapshot());
let old_language = std::mem::replace(&mut self.language, language);
self.refresh_resolved_settings(cx);
self.was_changed();
@@ -4121,7 +4136,18 @@ impl BufferSnapshot {
let mut syntax = None;
if language_aware.tree_sitter {
- syntax = Some(self.get_highlights(range.clone()));
+ match self.cached_highlight_runs(range.clone()) {
+ Some(runs) => {
+ return BufferChunks::with_cached_highlights(
+ self.text.as_rope(),
+ range,
+ runs,
+ language_aware.diagnostics,
+ self,
+ );
+ }
+ None => syntax = Some(self.get_highlights(range.clone())),
+ }
}
BufferChunks::new(
self.text.as_rope(),
@@ -4132,6 +4158,102 @@ impl BufferSnapshot {
)
}
+ pub(crate) fn cached_highlight_runs(&self, range: Range<usize>) -> Option<Vec<HighlightRun>> {
+ #[cfg(any(test, feature = "test-support"))]
+ {
+ static DISABLE_HIGHLIGHT_CACHE: std::sync::LazyLock<bool> =
+ std::sync::LazyLock::new(|| {
+ std::env::var_os("ZED_DISABLE_HIGHLIGHT_CACHE").is_some()
+ });
+ if *DISABLE_HIGHLIGHT_CACHE {
+ return None;
+ }
+ }
+ self.language.as_ref()?.grammar()?;
+ if range.is_empty() {
+ return Some(Vec::new());
+ }
+ let mut runs = Vec::<HighlightRun>::new();
+ for chunk in self
+ .tree_sitter_data
+ .chunks
+ .applicable_chunks(&[range.to_point(self)])
+ {
+ let chunk_range = chunk.anchor_range().to_offset(self);
+ if chunk_range.end <= range.start || chunk_range.start >= range.end {
+ continue;
+ }
+ if chunk_range.len() > MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK {
+ return None;
+ }
+ let chunk_highlights = match self.tree_sitter_data.highlights_by_chunks.get(chunk.id) {
+ Some(chunk_highlights) => chunk_highlights,
+ None => {
+ let chunk_highlights = self.compute_chunk_highlights(chunk_range);
+ self.tree_sitter_data
+ .highlights_by_chunks
+ .insert(chunk.id, chunk_highlights.clone());
+ chunk_highlights
+ }
+ };
+ for (run_range, highlight_id) in chunk_highlights.runs.iter() {
+ if run_range.end <= range.start {
+ continue;
+ }
+ if run_range.start >= range.end {
+ break;
+ }
+ match runs.last_mut() {
+ Some((last_range, last_highlight_id))
+ if last_highlight_id == highlight_id
+ && last_range.end == run_range.start =>
+ {
+ last_range.end = run_range.end;
+ }
+ _ => runs.push((run_range.clone(), *highlight_id)),
+ }
+ }
+ }
+ Some(runs)
+ }
+
+ fn compute_chunk_highlights(&self, range: Range<usize>) -> ResolvedHighlights {
+ let captures = self.syntax.captures(range.clone(), &self.text, |grammar| {
+ grammar
+ .highlights_config
+ .as_ref()
+ .map(|config| &config.query)
+ });
+ let sources = captures
+ .grammars()
+ .iter()
+ .map(|&grammar| (Arc::clone(grammar), grammar.highlight_map()))
+ .collect::<SmallVec<[(Arc<Grammar>, HighlightMap); 2]>>();
+ let mut runs = Vec::<(Range<usize>, HighlightId)>::new();
+ for region in flattened_highlight_regions(captures, range) {
+ let highlight_id = region.stack.iter().rev().find_map(|capture| {
+ let (_, highlight_map) = sources.get(capture.grammar_index)?;
+ highlight_map.get(capture.capture_id)
+ });
+ let Some(highlight_id) = highlight_id else {
+ continue;
+ };
+ match runs.last_mut() {
+ Some((last_range, last_highlight_id))
+ if *last_highlight_id == highlight_id
+ && last_range.end == region.range.start =>
+ {
+ last_range.end = region.range.end;
+ }
+ _ => runs.push((region.range, highlight_id)),
+ }
+ }
+ ResolvedHighlights {
+ sources,
+ runs: runs.into(),
+ }
+ }
+
pub fn highlighted_text_for_range<T: ToOffset>(
&self,
range: Range<T>,
@@ -5508,16 +5630,40 @@ impl<'a> BufferChunks<'a> {
diagnostics: bool,
buffer_snapshot: Option<&'a BufferSnapshot>,
) -> Self {
- let mut highlights = None;
- if let Some((captures, highlight_maps)) = syntax {
- highlights = Some(BufferChunkHighlights {
- captures,
- next_capture: None,
- stack: Default::default(),
- highlight_maps,
- })
- }
+ let highlights = syntax.map(|(captures, highlight_maps)| BufferChunkHighlights {
+ captures,
+ next_capture: None,
+ stack: Vec::new(),
+ highlight_maps,
+ });
+ Self::init(text, range, highlights, None, diagnostics, buffer_snapshot)
+ }
+ fn with_cached_highlights(
+ text: &'a Rope,
+ range: Range<usize>,
+ runs: Vec<HighlightRun>,
+ diagnostics: bool,
+ buffer_snapshot: &'a BufferSnapshot,
+ ) -> Self {
+ Self::init(
+ text,
+ range,
+ None,
+ Some(CachedChunkHighlightsIter { runs, ix: 0 }),
+ diagnostics,
+ Some(buffer_snapshot),
+ )
+ }
+
+ fn init(
+ text: &'a Rope,
+ range: Range<usize>,
+ highlights: Option<BufferChunkHighlights<'a>>,
+ cached_highlights: Option<CachedChunkHighlightsIter>,
+ diagnostics: bool,
+ buffer_snapshot: Option<&'a BufferSnapshot>,
+ ) -> Self {
let diagnostic_endpoints = diagnostics.then(|| Vec::new().into_iter().peekable());
let chunks = text.chunks_in_range(range.clone());
@@ -5533,6 +5679,7 @@ impl<'a> BufferChunks<'a> {
unnecessary_depth: 0,
underline: true,
highlights,
+ cached_highlights,
};
this.initialize_diagnostic_endpoints();
this
@@ -5542,7 +5689,33 @@ impl<'a> BufferChunks<'a> {
pub fn seek(&mut self, range: Range<usize>) {
let old_range = std::mem::replace(&mut self.range, range.clone());
self.chunks.set_range(self.range.clone());
- if let Some(highlights) = self.highlights.as_mut() {
+ if let Some(cached) = self.cached_highlights.as_mut() {
+ if old_range.start <= self.range.start && old_range.end >= self.range.end {
+ cached.ix = cached
+ .runs
+ .partition_point(|(run_range, _)| run_range.end <= range.start);
+ } else if let Some(snapshot) = self.buffer_snapshot {
+ if let Some(runs) = snapshot.cached_highlight_runs(self.range.clone()) {
+ cached.runs = runs;
+ cached.ix = 0;
+ } else {
+ let (captures, highlight_maps) = snapshot.get_highlights(self.range.clone());
+ self.cached_highlights = None;
+ self.highlights = Some(BufferChunkHighlights {
+ captures,
+ next_capture: None,
+ stack: Vec::new(),
+ highlight_maps,
+ });
+ }
+ } else {
+ debug_assert!(
+ false,
+ "Attempted to seek on a language-aware buffer iterator without associated buffer snapshot"
+ );
+ }
+ self.initialize_diagnostic_endpoints();
+ } else if let Some(highlights) = self.highlights.as_mut() {
if old_range.start <= self.range.start && old_range.end >= self.range.end {
// Reuse existing highlights stack, as the new range is a subrange of the old one.
highlights
@@ -5565,7 +5738,7 @@ impl<'a> BufferChunks<'a> {
*highlights = BufferChunkHighlights {
captures,
next_capture: None,
- stack: Default::default(),
+ stack: Vec::new(),
highlight_maps,
};
} else {
@@ -5701,6 +5874,21 @@ impl<'a> Iterator for BufferChunks<'a> {
}
}
+ if let Some(cached) = self.cached_highlights.as_mut() {
+ while cached
+ .runs
+ .get(cached.ix)
+ .is_some_and(|(run_range, _)| run_range.end <= self.range.start)
+ {
+ cached.ix += 1;
+ }
+ if let Some((run_range, _)) = cached.runs.get(cached.ix)
+ && self.range.start < run_range.start
+ {
+ next_capture_start = run_range.start;
+ }
+ }
+
let mut diagnostic_endpoints = std::mem::take(&mut self.diagnostic_endpoints);
if let Some(diagnostic_endpoints) = diagnostic_endpoints.as_mut() {
while let Some(endpoint) = diagnostic_endpoints.peek().copied() {
@@ -5734,6 +5922,13 @@ impl<'a> Iterator for BufferChunks<'a> {
chunk_end = chunk_end.min(*parent_capture_end);
highlight_id = Some(*parent_highlight_id);
}
+ if let Some(cached) = self.cached_highlights.as_ref()
+ && let Some((run_range, run_highlight_id)) = cached.runs.get(cached.ix)
+ && run_range.start <= chunk_start
+ {
+ chunk_end = chunk_end.min(run_range.end);
+ highlight_id = Some(*run_highlight_id);
+ }
let bit_start = chunk_start - self.chunks.offset();
let bit_end = chunk_end - self.chunks.offset();
diff --git a/crates/language/src/buffer/row_chunk.rs b/crates/language/src/buffer/row_chunk.rs
index 605299f67f0..7764067f922 100644
--- a/crates/language/src/buffer/row_chunk.rs
+++ b/crates/language/src/buffer/row_chunk.rs
@@ -10,6 +10,8 @@ use util::RangeExt;
use crate::BufferRow;
+pub type RowChunkId = usize;
+
/// An range of rows, exclusive as [`lsp::Range`] and
/// <https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#range>
/// denote.
@@ -27,7 +29,7 @@ pub struct RowChunks {
buffer_snapshot: text::BufferSnapshot,
last_row: BufferRow,
max_rows_per_chunk: u32,
- computed_chunks: Mutex<HashMap<usize, RowChunk>>,
+ computed_chunks: Mutex<HashMap<RowChunkId, RowChunk>>,
}
impl std::fmt::Debug for RowChunks {
@@ -86,12 +88,18 @@ impl RowChunks {
}
}
- fn chunk_row_range(&self, id: usize) -> Range<BufferRow> {
- let start = id as u32 * self.max_rows_per_chunk;
- start..(start + self.max_rows_per_chunk).min(self.last_row)
+ fn chunk_row_range(&self, id: RowChunkId) -> Range<BufferRow> {
+ let start = u32::try_from(id)
+ .unwrap_or(u32::MAX)
+ .saturating_mul(self.max_rows_per_chunk)
+ .min(self.last_row);
+ start
+ ..start
+ .saturating_add(self.max_rows_per_chunk)
+ .min(self.last_row)
}
- fn chunk(&self, id: usize) -> RowChunk {
+ fn chunk(&self, id: RowChunkId) -> RowChunk {
*self.computed_chunks.lock().entry(id).or_insert_with(|| {
let row_range = self.chunk_row_range(id);
let start = Point::new(row_range.start, 0);
@@ -113,7 +121,7 @@ impl RowChunks {
#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
pub struct RowChunk {
- pub id: usize,
+ pub id: RowChunkId,
pub start: BufferRow,
pub end_exclusive: BufferRow,
pub start_anchor: Anchor,
diff --git a/crates/language/src/buffer_tests.rs b/crates/language/src/buffer_tests.rs
index 80646668b0b..c7b835b4746 100644
--- a/crates/language/src/buffer_tests.rs
+++ b/crates/language/src/buffer_tests.rs
@@ -5387,6 +5387,269 @@ fn test_settings_changed_event(cx: &mut TestAppContext) {
drop(subscription);
}
+#[gpui::test]
+fn test_chunk_highlights_follow_edits_and_theme_changes(cx: &mut TestAppContext) {
+ cx.update(|cx| init_settings(cx, |_| {}));
+
+ let language = keyword_and_function_lang();
+ let theme_with_keyword = keyword_and_function_theme();
+ let theme_without_keyword =
+ SyntaxTheme::new([("function".to_string(), gpui::rgba(0x0000ffff).into())]);
+ language.set_theme(&theme_with_keyword);
+
+ let buffer = cx.new(|cx| {
+ let mut buffer = Buffer::local("fn main() {}", cx);
+ buffer.set_language(Some(language.clone()), cx);
+ buffer
+ });
+ cx.run_until_parked();
+
+ let highlighted_chunks = |snapshot: &BufferSnapshot| {
+ snapshot
+ .chunks(
+ 0..snapshot.len(),
+ LanguageAwareStyling {
+ tree_sitter: true,
+ diagnostics: false,
+ },
+ )
+ .filter_map(|chunk| Some((chunk.text.to_string(), chunk.syntax_highlight_id?)))
+ .collect::<Vec<_>>()
+ };
+
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ let expected_with_keyword = vec![
+ (
+ "fn".to_string(),
+ theme_highlight_id(&theme_with_keyword, "keyword"),
+ ),
+ (
+ "main".to_string(),
+ theme_highlight_id(&theme_with_keyword, "function"),
+ ),
+ ];
+ assert_eq!(highlighted_chunks(&snapshot), expected_with_keyword);
+ assert_eq!(
+ highlighted_chunks(&snapshot),
+ expected_with_keyword,
+ "repeated chunking of the same snapshot must return the same highlights"
+ );
+
+ language.set_theme(&theme_without_keyword);
+ assert_eq!(
+ highlighted_chunks(&snapshot),
+ vec![(
+ "main".to_string(),
+ theme_highlight_id(&theme_without_keyword, "function")
+ )],
+ "a theme change must invalidate highlights of an existing snapshot"
+ );
+
+ buffer.update(cx, |buffer, cx| {
+ buffer.edit([(3..7, "launch")], None, cx);
+ });
+ cx.run_until_parked();
+ let edited_snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ assert_eq!(
+ highlighted_chunks(&edited_snapshot),
+ vec![(
+ "launch".to_string(),
+ theme_highlight_id(&theme_without_keyword, "function")
+ )],
+ "an edit must invalidate the cached highlights"
+ );
+}
+
+#[gpui::test]
+fn test_chunk_highlights_across_row_chunk_seeks(cx: &mut TestAppContext) {
+ cx.update(|cx| init_settings(cx, |_| {}));
+
+ let language = keyword_and_function_lang();
+ let theme = keyword_and_function_theme();
+ language.set_theme(&theme);
+
+ let short_row = "fn short() {}\n";
+ let last_row = "fn omega() {}";
+ let text = format!(
+ "{}{last_row}",
+ short_row.repeat(MAX_ROWS_IN_A_CHUNK as usize)
+ );
+
+ let buffer = cx.new(|cx| {
+ let mut buffer = Buffer::local(text, cx);
+ buffer.set_language(Some(language.clone()), cx);
+ buffer
+ });
+ cx.run_until_parked();
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+
+ let keyword = theme_highlight_id(&theme, "keyword");
+ let function = theme_highlight_id(&theme, "function");
+
+ assert_eq!(
+ merged_highlight_runs(&snapshot, 0..short_row.len()),
+ vec![("fn".to_string(), keyword), ("short".to_string(), function)],
+ );
+
+ let last_row_start = short_row.len() * MAX_ROWS_IN_A_CHUNK as usize;
+ let expected_last_row_runs = vec![("fn".to_string(), keyword), ("omega".to_string(), function)];
+ assert_eq!(
+ merged_highlight_runs(&snapshot, last_row_start..snapshot.len()),
+ expected_last_row_runs,
+ );
+
+ let mut chunks = snapshot.chunks(
+ 0..short_row.len(),
+ LanguageAwareStyling {
+ tree_sitter: true,
+ diagnostics: false,
+ },
+ );
+ chunks.seek(last_row_start..snapshot.len());
+ let mut runs_after_seek = Vec::new();
+ for chunk in chunks {
+ merge_highlighted_chunk(&mut runs_after_seek, &chunk);
+ }
+ assert_eq!(
+ runs_after_seek, expected_last_row_runs,
+ "seeking into another row chunk must refetch that chunk's highlights"
+ );
+}
+
+#[gpui::test]
+fn test_oversized_chunks_bypass_the_highlight_cache(cx: &mut TestAppContext) {
+ if std::env::var_os("ZED_DISABLE_HIGHLIGHT_CACHE").is_some() {
+ return;
+ }
+ cx.update(|cx| init_settings(cx, |_| {}));
+
+ let language = keyword_and_function_lang();
+ let theme = keyword_and_function_theme();
+ language.set_theme(&theme);
+
+ let short_row = "fn short() {}\n";
+ let giant_row = format!(
+ "fn omega() {{}}{}",
+ " ".repeat(MAX_BYTES_TO_HIGHLIGHT_IN_A_CHUNK)
+ );
+ let text = format!(
+ "{}{giant_row}",
+ short_row.repeat(MAX_ROWS_IN_A_CHUNK as usize)
+ );
+
+ let buffer = cx.new(|cx| {
+ let mut buffer = Buffer::local(text, cx);
+ buffer.set_language(Some(language.clone()), cx);
+ buffer
+ });
+ cx.run_until_parked();
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+
+ let keyword = theme_highlight_id(&theme, "keyword");
+ let function = theme_highlight_id(&theme, "function");
+ let expected_giant_row_runs =
+ vec![("fn".to_string(), keyword), ("omega".to_string(), function)];
+
+ let last_row_start = short_row.len() * MAX_ROWS_IN_A_CHUNK as usize;
+ assert_eq!(
+ snapshot.cached_highlight_runs(last_row_start..snapshot.len()),
+ None,
+ "chunks larger than the byte cap must not be cached"
+ );
+ assert!(
+ snapshot.cached_highlight_runs(0..short_row.len()).is_some(),
+ "chunks within the byte cap must still be cached"
+ );
+ assert_eq!(
+ snapshot.cached_highlight_runs(last_row_start + 1..last_row_start + 1),
+ Some(Vec::new()),
+ "an empty range must not compute captures for its oversized chunk"
+ );
+
+ assert_eq!(
+ merged_highlight_runs(&snapshot, last_row_start..snapshot.len()),
+ expected_giant_row_runs,
+ "oversized chunks must fall back to direct highlighting"
+ );
+
+ let mut chunks = snapshot.chunks(
+ 0..short_row.len(),
+ LanguageAwareStyling {
+ tree_sitter: true,
+ diagnostics: false,
+ },
+ );
+ chunks.seek(last_row_start..snapshot.len());
+ let mut runs_after_seek = Vec::new();
+ for chunk in chunks {
+ merge_highlighted_chunk(&mut runs_after_seek, &chunk);
+ }
+ assert_eq!(
+ runs_after_seek, expected_giant_row_runs,
+ "seeking into an oversized chunk must fall back to direct highlighting"
+ );
+}
+
+#[gpui::test]
+fn test_language_change_invalidates_cached_chunk_highlights(cx: &mut TestAppContext) {
+ cx.update(|cx| init_settings(cx, |_| {}));
+
+ let rust = keyword_and_function_lang();
+ let identifiers_only = Arc::new(
+ Language::new(
+ LanguageConfig {
+ name: "Identifiers".into(),
+ ..LanguageConfig::default()
+ },
+ Some(tree_sitter_rust::LANGUAGE.into()),
+ )
+ .with_highlights_query("(identifier) @variable")
+ .unwrap(),
+ );
+ let theme = SyntaxTheme::new([
+ ("keyword".to_string(), gpui::rgba(0xff0000ff).into()),
+ ("function".to_string(), gpui::rgba(0x00ff00ff).into()),
+ ("variable".to_string(), gpui::rgba(0x0000ffff).into()),
+ ]);
+ rust.set_theme(&theme);
+ identifiers_only.set_theme(&theme);
+
+ let buffer = cx.new(|cx| {
+ let mut buffer = Buffer::local("fn main() {}", cx);
+ buffer.set_language(Some(rust), cx);
+ buffer
+ });
+ cx.run_until_parked();
+
+ let keyword = theme_highlight_id(&theme, "keyword");
+ let function = theme_highlight_id(&theme, "function");
+ let variable = theme_highlight_id(&theme, "variable");
+
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ assert_eq!(
+ merged_highlight_runs(&snapshot, 0..snapshot.len()),
+ vec![("fn".to_string(), keyword), ("main".to_string(), function)],
+ );
+
+ buffer.update(cx, |buffer, cx| {
+ buffer.set_sync_parse_timeout(None);
+ buffer.set_language(Some(identifiers_only), cx);
+ });
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ assert_eq!(
+ merged_highlight_runs(&snapshot, 0..snapshot.len()),
+ Vec::new(),
+ "a language change must not serve the old language's cached highlights while the new parse is pending"
+ );
+
+ cx.run_until_parked();
+ let snapshot = buffer.read_with(cx, |buffer, _| buffer.snapshot());
+ assert_eq!(
+ merged_highlight_runs(&snapshot, 0..snapshot.len()),
+ vec![("main".to_string(), variable)],
+ );
+}
+
#[gpui::test(iterations = 100)]
fn test_random_chunk_bitmaps(cx: &mut App, mut rng: StdRng) {
use util::RandomCharIter;
@@ -5514,3 +5777,63 @@ fn test_formatted_chunks(cx: &mut gpui::App) {
}
}
}
+
+fn merged_highlight_runs(
+ snapshot: &BufferSnapshot,
+ range: Range<usize>,
+) -> Vec<(String, HighlightId)> {
+ let chunks = snapshot.chunks(
+ range,
+ LanguageAwareStyling {
+ tree_sitter: true,
+ diagnostics: false,
+ },
+ );
+ let mut runs = Vec::new();
+ for chunk in chunks {
+ merge_highlighted_chunk(&mut runs, &chunk);
+ }
+ runs
+}
+
+fn merge_highlighted_chunk(runs: &mut Vec<(String, HighlightId)>, chunk: &Chunk<'_>) {
+ let Some(highlight_id) = chunk.syntax_highlight_id else {
+ return;
+ };
+ match runs.last_mut() {
+ Some((last_text, last_highlight_id)) if *last_highlight_id == highlight_id => {
+ last_text.push_str(chunk.text);
+ }
+ _ => runs.push((chunk.text.to_string(), highlight_id)),
+ }
+}
+
+fn keyword_and_function_lang() -> Arc<Language> {
+ Arc::new(
+ Language::new(
+ LanguageConfig {
+ name: "Rust".into(),
+ ..LanguageConfig::default()
+ },
+ Some(tree_sitter_rust::LANGUAGE.into()),
+ )
+ .with_highlights_query(
+ r#"
+ "fn" @keyword
+ (identifier) @function
+ "#,
+ )
+ .unwrap(),
+ )
+}
+
+fn keyword_and_function_theme() -> SyntaxTheme {
+ SyntaxTheme::new([
+ ("keyword".to_string(), gpui::rgba(0xff0000ff).into()),
+ ("function".to_string(), gpui::rgba(0x00ff00ff).into()),
+ ])
+}
+
+fn theme_highlight_id(theme: &SyntaxTheme, capture_name: &str) -> HighlightId {
+ HighlightId::new(theme.highlight_id(capture_name).unwrap())
+}
diff --git a/crates/language_core/src/highlight_cache.rs b/crates/language_core/src/highlight_cache.rs
index b820787f716..86741ec4d43 100644
--- a/crates/language_core/src/highlight_cache.rs
+++ b/crates/language_core/src/highlight_cache.rs
@@ -14,6 +14,7 @@ use std::{
pub const MAX_TEXT_HIGHLIGHT_ENTRY_BYTES: usize = MAX_TEXT_HIGHLIGHT_CACHE_BYTES / 8;
const MAX_TEXT_HIGHLIGHT_CACHE_BYTES: usize = 4 * 1024 * 1024;
+const MAX_CHUNK_HIGHLIGHT_CACHE_BYTES: usize = 10 * 1024 * 1024;
const APPROXIMATE_LRU_NODE_BYTES: usize = 4 * size_of::<usize>();
#[derive(Clone, Debug, PartialEq, Eq, Hash)]
@@ -109,6 +110,44 @@ impl TextHighlightCache {
}
}
+pub struct ChunkHighlightCache(Mutex<CostBudgetedLru<usize, ResolvedHighlights>>);
+
+impl Default for ChunkHighlightCache {
+ fn default() -> Self {
+ Self(Mutex::new(CostBudgetedLru::new(
+ MAX_CHUNK_HIGHLIGHT_CACHE_BYTES,
+ MAX_CHUNK_HIGHLIGHT_CACHE_BYTES,
+ )))
+ }
+}
+
+impl fmt::Debug for ChunkHighlightCache {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("ChunkHighlightCache")
+ .finish_non_exhaustive()
+ }
+}
+
+impl ChunkHighlightCache {
+ pub fn get(&self, chunk_id: usize) -> Option<ResolvedHighlights> {
+ let mut lru = self.0.lock();
+ let highlights = lru.get(&chunk_id)?;
+ if !highlights.is_current() {
+ return None;
+ }
+ Some(highlights.clone())
+ }
+
+ pub fn insert(&self, chunk_id: usize, highlights: ResolvedHighlights) {
+ let cost = highlights.cost_bytes();
+ self.0.lock().insert(chunk_id, highlights, cost);
+ }
+
+ pub fn clear(&mut self) {
+ self.0.get_mut().clear();
+ }
+}
+
struct CostBudgetedLru<K: Hash + Eq, V> {
entries: LruCache<K, (V, usize)>,
total_cost: usize,
@@ -149,6 +188,11 @@ impl<K: Hash + Eq, V> CostBudgetedLru<K, V> {
self.total_cost -= evicted_cost;
}
}
+
+ fn clear(&mut self) {
+ self.entries.clear();
+ self.total_cost = 0;
+ }
}
struct TextHighlightEntry {
@@ -209,6 +253,17 @@ mod tests {
assert_eq!(cache.total_cost, 40 + overhead);
}
+ #[test]
+ fn test_clear_resets_the_budget() {
+ let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES;
+ let mut cache = CostBudgetedLru::<&str, u32>::new(90 + overhead, 100);
+ cache.insert("a", 1, 90);
+ cache.clear();
+ assert_eq!(cache.total_cost, 0);
+ cache.insert("b", 2, 90);
+ assert_eq!(cache.get(&"b"), Some(&2));
+ }
+
#[test]
fn test_zero_cost_entries_still_consume_budget() {
let overhead = CostBudgetedLru::<usize, u32>::ENTRY_OVERHEAD_BYTES;