Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

markdown: Cache code block syntax highlighting at parse time instead of re-highlighting every frame (#63138)

Kirill Bulatov318c664ec3e0
diff --git a/Cargo.lock b/Cargo.lock
index 1e74bce4596..38da980bb1b 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -9555,12 +9555,14 @@ dependencies = [
"gpui_shared_string",
"gpui_util",
"log",
+ "lru",
"parking_lot",
"path",
"regex",
"schemars 1.0.4",
"serde",
"serde_json",
+ "smallvec",
"strum 0.28.0",
"toml 0.8.23",
"tree-sitter",
diff --git a/crates/language/src/buffer.rs b/crates/language/src/buffer.rs
index 5bf8e87112c..246b786ee3f 100644
--- a/crates/language/src/buffer.rs
+++ b/crates/language/src/buffer.rs
@@ -19,8 +19,8 @@ use crate::{
unified_diff_with_offsets,
};
pub use crate::{
- Grammar, HighlightId, HighlightMap, Language, LanguageRegistry, diagnostic_set::DiagnosticSet,
- proto,
+ CaptureId, Grammar, HighlightId, HighlightMap, Language, LanguageRegistry,
+ diagnostic_set::DiagnosticSet, proto,
};
use anyhow::{Context as _, Result};
@@ -5553,8 +5553,8 @@ impl<'a> BufferChunks<'a> {
{
let next_capture_end = capture.node.end_byte();
if range.start < next_capture_end
- && let Some(capture_id) =
- highlights.highlight_maps[capture.grammar_index].get(capture.index)
+ && let Some(capture_id) = highlights.highlight_maps[capture.grammar_index]
+ .get(CaptureId(capture.index))
{
highlights.stack.push((next_capture_end, capture_id));
}
@@ -5689,8 +5689,8 @@ impl<'a> Iterator for BufferChunks<'a> {
next_capture_start = capture.node.start_byte();
break;
} else {
- let highlight_id =
- highlights.highlight_maps[capture.grammar_index].get(capture.index);
+ let highlight_id = highlights.highlight_maps[capture.grammar_index]
+ .get(CaptureId(capture.index));
if let Some(highlight_id) = highlight_id {
highlights
.stack
diff --git a/crates/language/src/language.rs b/crates/language/src/language.rs
index a7791c51e79..6dcb998500d 100644
--- a/crates/language/src/language.rs
+++ b/crates/language/src/language.rs
@@ -44,10 +44,12 @@ use http_client::HttpClient;
pub use language_core::{
SymbolKind,
- highlight_map::{HighlightId, HighlightMap},
+ highlight_cache::ResolvedHighlights,
+ highlight_map::{CaptureId, HighlightId, HighlightMap},
};
use futures::future::FutureExt as _;
+use language_core::highlight_cache::{MAX_TEXT_HIGHLIGHT_ENTRY_BYTES, TextHighlightKey};
pub use language_core::{
BlockCommentConfig, BracketPair, BracketPairConfig, BracketPairContent, BracketsConfig,
BracketsPatternConfig, CodeLabel, CodeLabelBuilder, DebugVariablesConfig, DebuggerTextObject,
@@ -83,7 +85,7 @@ use std::{
str,
sync::{Arc, LazyLock},
};
-use syntax_map::{QueryCursorHandle, SyntaxSnapshot};
+use syntax_map::{QueryCursorHandle, SyntaxSnapshot, flattened_highlight_regions};
use task::RunnableTag;
pub use task_context::{ContextLocation, ContextProvider};
pub use text_diff::{
@@ -1120,29 +1122,94 @@ impl Language {
text: &'a Rope,
range: Range<usize>,
) -> Vec<(Range<usize>, HighlightId)> {
- let mut result = Vec::new();
- if let Some(grammar) = &self.grammar {
- let tree = parse_text(grammar, text, None);
- let captures =
- SyntaxSnapshot::single_tree_captures(range.clone(), text, &tree, self, |grammar| {
- grammar
- .highlights_config
- .as_ref()
- .map(|config| &config.query)
- });
- let highlight_maps = vec![grammar.highlight_map()];
- let mut offset = 0;
- for chunk in
- BufferChunks::new(text, range, Some((captures, highlight_maps)), false, None)
+ self.highlight_text_resolved(text, range).runs.to_vec()
+ }
+
+ pub fn highlight_text_resolved(
+ self: &Arc<Self>,
+ text: &Rope,
+ range: Range<usize>,
+ ) -> ResolvedHighlights {
+ let Some(grammar) = &self.grammar else {
+ return ResolvedHighlights::default();
+ };
+ let Some(highlights_config) = &grammar.highlights_config else {
+ return ResolvedHighlights::default();
+ };
+ let highlights = if text.len() > MAX_TEXT_HIGHLIGHT_ENTRY_BYTES {
+ self.compute_resolved_highlights(grammar, text)
+ } else {
+ let key = TextHighlightKey::new(text.chunks(), text.len());
+ match highlights_config
+ .text_highlight_cache
+ .get(&key, text.chunks())
{
- let end_offset = offset + chunk.text.len();
- if let Some(highlight_id) = chunk.syntax_highlight_id {
- result.push((offset..end_offset, highlight_id));
+ Some(highlights) => highlights,
+ None => highlights_config.text_highlight_cache.insert(
+ key,
+ Arc::from(text.chunks().collect::<String>()),
+ self.compute_resolved_highlights(grammar, text),
+ ),
+ }
+ };
+ if range.start == 0 && range.end >= text.len() {
+ return highlights;
+ }
+ ResolvedHighlights {
+ sources: highlights.sources.clone(),
+ runs: highlights
+ .runs
+ .iter()
+ .filter(|(run_range, _)| run_range.start < range.end && run_range.end > range.start)
+ .map(|(run_range, highlight_id)| {
+ (
+ run_range.start.max(range.start) - range.start
+ ..run_range.end.min(range.end) - range.start,
+ *highlight_id,
+ )
+ })
+ .collect(),
+ }
+ }
+
+ fn compute_resolved_highlights(
+ self: &Arc<Self>,
+ grammar: &Arc<Grammar>,
+ text: &Rope,
+ ) -> ResolvedHighlights {
+ let highlight_map = grammar.highlight_map();
+ let tree = parse_text(grammar, text, None);
+ let captures =
+ SyntaxSnapshot::single_tree_captures(0..text.len(), text, &tree, self, |grammar| {
+ grammar
+ .highlights_config
+ .as_ref()
+ .map(|config| &config.query)
+ });
+ let mut runs = Vec::<(Range<usize>, HighlightId)>::new();
+ for region in flattened_highlight_regions(captures, 0..text.len()) {
+ let highlight_id = region
+ .stack
+ .iter()
+ .rev()
+ .find_map(|capture| highlight_map.get(capture.capture_id));
+ let Some(highlight_id) = highlight_id else {
+ continue;
+ };
+ match runs.last_mut() {
+ Some((last_range, last_highlight_id))
+ if *last_highlight_id == highlight_id
+ && last_range.end == region.range.start =>
+ {
+ last_range.end = region.range.end;
}
- offset = end_offset;
+ _ => runs.push((region.range, highlight_id)),
}
}
- result
+ ResolvedHighlights {
+ sources: [(Arc::clone(grammar), highlight_map)].into_iter().collect(),
+ runs: runs.into(),
+ }
}
pub fn path_suffixes(&self) -> &[String] {
@@ -1686,19 +1753,163 @@ mod tests {
let map = build_highlight_map(capture_names, &theme);
assert_eq!(
- theme.get_capture_name(map.get(0).unwrap()),
+ theme.get_capture_name(map.get(CaptureId(0)).unwrap()),
Some("function")
);
assert_eq!(
- theme.get_capture_name(map.get(1).unwrap()),
+ theme.get_capture_name(map.get(CaptureId(1)).unwrap()),
Some("function.async")
);
assert_eq!(
- theme.get_capture_name(map.get(2).unwrap()),
+ theme.get_capture_name(map.get(CaptureId(2)).unwrap()),
Some("variable.builtin")
);
}
+ #[test]
+ fn test_highlight_text_resolves_nested_captures_with_theme_fallback() {
+ let language = Arc::new(
+ Language::new(
+ LanguageConfig {
+ name: "Rust".into(),
+ ..LanguageConfig::default()
+ },
+ Some(tree_sitter_rust::LANGUAGE.into()),
+ )
+ .with_highlights_query(
+ r#"
+ (function_item) @function.definition
+ (identifier) @variable
+ "fn" @keyword
+ "#,
+ )
+ .unwrap(),
+ );
+
+ let theme = SyntaxTheme::new(
+ [
+ ("function", rgba(0x100000ff)),
+ ("keyword", rgba(0x200000ff)),
+ ]
+ .iter()
+ .map(|(name, color)| (name.to_string(), (*color).into())),
+ );
+ language.set_theme(&theme);
+
+ let code = "fn main() {}";
+ let highlights = language.highlight_text_resolved(&Rope::from(code), 0..code.len());
+ assert!(highlights.is_current());
+ let named = highlights
+ .runs
+ .iter()
+ .map(|(range, highlight_id)| {
+ (
+ range.clone(),
+ theme.get_capture_name(*highlight_id).unwrap(),
+ )
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(
+ named,
+ vec![(0..2, "keyword"), (2..12, "function")],
+ "an inner capture missing from the theme must fall back to its outer capture"
+ );
+
+ let memoized = language.highlight_text_resolved(&Rope::from(code), 0..code.len());
+ assert!(
+ Arc::ptr_eq(&highlights.runs, &memoized.runs),
+ "repeated highlighting of the same text must be memoized"
+ );
+ let partial = language.highlight_text_resolved(&Rope::from(code), 0..2);
+ assert!(
+ !Arc::ptr_eq(&highlights.runs, &partial.runs),
+ "a different range must not reuse the memoized highlights"
+ );
+ assert_eq!(partial.runs.as_ref(), &[(0..2, highlights.runs[0].1)]);
+
+ let richer_theme = SyntaxTheme::new(
+ [
+ ("function", rgba(0x100000ff)),
+ ("keyword", rgba(0x200000ff)),
+ ("variable", rgba(0x300000ff)),
+ ]
+ .iter()
+ .map(|(name, color)| (name.to_string(), (*color).into())),
+ );
+ language.set_theme(&richer_theme);
+ assert!(
+ !highlights.is_current(),
+ "a theme change must invalidate previously resolved highlights"
+ );
+ let rethemed = language.highlight_text_resolved(&Rope::from(code), 0..code.len());
+ assert!(rethemed.is_current());
+ let renamed = rethemed
+ .runs
+ .iter()
+ .map(|(range, highlight_id)| {
+ (
+ range.clone(),
+ richer_theme.get_capture_name(*highlight_id).unwrap(),
+ )
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(
+ renamed,
+ vec![
+ (0..2, "keyword"),
+ (2..3, "function"),
+ (3..7, "variable"),
+ (7..12, "function"),
+ ],
+ "a theme change must invalidate memoized highlights and resolve against the new theme"
+ );
+ }
+
+ #[test]
+ fn test_oversized_text_is_highlighted_without_caching() {
+ let language = rust_lang();
+ let theme = SyntaxTheme::new(
+ [
+ ("function", rgba(0x100000ff)),
+ ("keyword", rgba(0x200000ff)),
+ ]
+ .iter()
+ .map(|(name, color)| (name.to_string(), (*color).into())),
+ );
+ language.set_theme(&theme);
+
+ let small_code = "fn main() {}";
+ let small_highlights =
+ language.highlight_text_resolved(&Rope::from(small_code), 0..small_code.len());
+ assert!(
+ !small_highlights.runs.is_empty(),
+ "texts within the size cap must be highlighted"
+ );
+
+ let oversized_code = format!(
+ "fn main() {{}}{}",
+ " ".repeat(MAX_TEXT_HIGHLIGHT_ENTRY_BYTES)
+ );
+ let oversized_rope = Rope::from(oversized_code.as_str());
+ let first_highlights =
+ language.highlight_text_resolved(&oversized_rope, 0..oversized_code.len());
+ assert_eq!(
+ first_highlights.runs.as_ref(),
+ small_highlights.runs.as_ref(),
+ "texts over the cache entry cap must still be highlighted"
+ );
+ let second_highlights =
+ language.highlight_text_resolved(&oversized_rope, 0..oversized_code.len());
+ assert_eq!(
+ second_highlights.runs.as_ref(),
+ first_highlights.runs.as_ref()
+ );
+ assert!(
+ !Arc::ptr_eq(&first_highlights.runs, &second_highlights.runs),
+ "texts over the cache entry cap must not be memoized"
+ );
+ }
+
#[test]
fn test_with_parser_resets_after_cancellation() {
use std::ops::ControlFlow;
diff --git a/crates/language/src/syntax_map.rs b/crates/language/src/syntax_map.rs
index 9ff17ce2ded..26a53bfd5ba 100644
--- a/crates/language/src/syntax_map.rs
+++ b/crates/language/src/syntax_map.rs
@@ -2,11 +2,14 @@
mod syntax_map_tests;
use crate::{
- Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS, with_parser,
+ CaptureId, Grammar, InjectionConfig, Language, LanguageId, LanguageRegistry, QUERY_CURSORS,
+ with_parser,
};
use collections::HashMap;
use futures::FutureExt;
use gpui::SharedString;
+use language_core::highlight_map::HighlightCaptureRef;
+use smallvec::SmallVec;
use std::{
borrow::Cow,
cmp::{self, Ordering, Reverse},
@@ -73,7 +76,7 @@ impl Drop for SyntaxSnapshot {
pub struct SyntaxMapCaptures<'a> {
layers: Vec<SyntaxMapCapturesLayer<'a>>,
active_layer_count: usize,
- grammars: Vec<&'a Grammar>,
+ grammars: Vec<&'a Arc<Grammar>>,
}
#[derive(Default)]
@@ -90,6 +93,73 @@ pub struct SyntaxMapCapture<'a> {
pub grammar_index: usize,
}
+pub(crate) struct HighlightCaptureRegion {
+ pub range: Range<usize>,
+ pub stack: SmallVec<[HighlightCaptureRef; 4]>,
+}
+
+pub(crate) fn flattened_highlight_regions(
+ mut captures: SyntaxMapCaptures<'_>,
+ range: Range<usize>,
+) -> Vec<HighlightCaptureRegion> {
+ let capture_refs = iter::from_fn(move || {
+ let capture = captures.next()?;
+ Some((
+ capture.node.byte_range(),
+ HighlightCaptureRef {
+ grammar_index: capture.grammar_index,
+ capture_id: CaptureId(capture.index),
+ },
+ ))
+ });
+ flatten_capture_regions(range, capture_refs)
+}
+
+fn flatten_capture_regions(
+ range: Range<usize>,
+ mut captures: impl Iterator<Item = (Range<usize>, HighlightCaptureRef)>,
+) -> Vec<HighlightCaptureRegion> {
+ let mut result = Vec::new();
+ let mut stack = Vec::<(Range<usize>, HighlightCaptureRef)>::new();
+ let mut offset = range.start;
+ let mut next_capture = captures.next();
+ loop {
+ stack.retain(|(capture_range, _)| capture_range.end > offset);
+ while let Some((capture_range, capture_ref)) = next_capture.take() {
+ if capture_range.start > offset {
+ next_capture = Some((capture_range, capture_ref));
+ break;
+ }
+ if capture_range.end > offset {
+ stack.push((capture_range, capture_ref));
+ }
+ next_capture = captures.next();
+ }
+ let mut next_boundary = range.end;
+ if let Some(min_end) = stack
+ .iter()
+ .map(|(capture_range, _)| capture_range.end)
+ .min()
+ {
+ next_boundary = next_boundary.min(min_end);
+ }
+ if let Some((capture_range, _)) = &next_capture {
+ next_boundary = next_boundary.min(capture_range.start);
+ }
+ if !stack.is_empty() && next_boundary > offset {
+ result.push(HighlightCaptureRegion {
+ range: offset..next_boundary,
+ stack: stack.iter().map(|(_, capture_ref)| *capture_ref).collect(),
+ });
+ }
+ if next_boundary >= range.end {
+ break;
+ }
+ offset = next_boundary;
+ }
+ result
+}
+
#[derive(Debug)]
pub struct SyntaxMapMatch<'a> {
pub language: Arc<Language>,
@@ -1201,7 +1271,7 @@ impl<'a> SyntaxMapCaptures<'a> {
result
}
- pub fn grammars(&self) -> &[&'a Grammar] {
+ pub fn grammars(&self) -> &[&'a Arc<Grammar>] {
&self.grammars
}
diff --git a/crates/language/src/syntax_map/syntax_map_tests.rs b/crates/language/src/syntax_map/syntax_map_tests.rs
index d4eecbe04c1..4c57b38e6ff 100644
--- a/crates/language/src/syntax_map/syntax_map_tests.rs
+++ b/crates/language/src/syntax_map/syntax_map_tests.rs
@@ -1341,6 +1341,48 @@ fn test_random_syntax_map_edits_with_heex(rng: StdRng, cx: &mut App) {
test_random_edits(text, registry, language, rng);
}
+#[test]
+fn test_flatten_capture_regions_with_nested_captures() {
+ let outer = capture_ref(1);
+ let inner = capture_ref(2);
+ assert_eq!(
+ flattened(0..12, &[(0..10, outer), (2..5, inner)]),
+ vec![
+ (0..2, vec![outer]),
+ (2..5, vec![outer, inner]),
+ (5..10, vec![outer]),
+ ],
+ );
+}
+
+#[test]
+fn test_flatten_capture_regions_with_overlapping_captures() {
+ let first = capture_ref(1);
+ let second = capture_ref(2);
+ assert_eq!(
+ flattened(0..25, &[(0..10, first), (2..20, second)]),
+ vec![
+ (0..2, vec![first]),
+ (2..10, vec![first, second]),
+ (10..20, vec![second]),
+ ],
+ "a capture must not extend past its own end when overlapping another capture"
+ );
+}
+
+#[test]
+fn test_flatten_capture_regions_clips_to_the_requested_range() {
+ let capture = capture_ref(1);
+ assert_eq!(
+ flattened(5..8, &[(0..10, capture)]),
+ vec![(5..8, vec![capture])],
+ );
+ assert_eq!(
+ flattened(0..6, &[(4..10, capture)]),
+ vec![(4..6, vec![capture])],
+ );
+}
+
fn test_random_edits(
text: String,
registry: Arc<LanguageRegistry>,
@@ -1762,6 +1804,23 @@ fn sql_lang() -> Language {
)
}
+fn capture_ref(capture_id: u32) -> HighlightCaptureRef {
+ HighlightCaptureRef {
+ grammar_index: 0,
+ capture_id: CaptureId(capture_id),
+ }
+}
+
+fn flattened(
+ range: Range<usize>,
+ captures: &[(Range<usize>, HighlightCaptureRef)],
+) -> Vec<(Range<usize>, Vec<HighlightCaptureRef>)> {
+ flatten_capture_regions(range, captures.iter().cloned())
+ .into_iter()
+ .map(|region| (region.range, region.stack.to_vec()))
+ .collect()
+}
+
fn range_for_text(buffer: &Buffer, text: &str) -> Range<usize> {
let start = buffer.as_rope().to_string().find(text).unwrap();
start..start + text.len()
diff --git a/crates/language_core/Cargo.toml b/crates/language_core/Cargo.toml
index 672d0dabd50..9941b1f8bf4 100644
--- a/crates/language_core/Cargo.toml
+++ b/crates/language_core/Cargo.toml
@@ -14,12 +14,14 @@ collections.workspace = true
gpui_shared_string.workspace = true
gpui_util.workspace = true
log.workspace = true
+lru.workspace = true
parking_lot.workspace = true
path.workspace = true
regex.workspace = true
schemars.workspace = true
serde.workspace = true
serde_json.workspace = true
+smallvec.workspace = true
strum.workspace = true
toml.workspace = true
tree-sitter.workspace = true
diff --git a/crates/language_core/src/grammar.rs b/crates/language_core/src/grammar.rs
index ff4dc0c8850..fc45de7706e 100644
--- a/crates/language_core/src/grammar.rs
+++ b/crates/language_core/src/grammar.rs
@@ -1,6 +1,6 @@
use crate::{
- HighlightId, HighlightMap, LanguageConfig, LanguageConfigOverride, LanguageName,
- LanguageQueries, language_config::BracketPairConfig,
+ CaptureId, HighlightId, HighlightMap, LanguageConfig, LanguageConfigOverride, LanguageName,
+ LanguageQueries, highlight_cache::TextHighlightCache, language_config::BracketPairConfig,
};
use anyhow::{Context as _, Result};
use collections::HashMap;
@@ -103,6 +103,7 @@ pub struct Grammar {
pub struct HighlightsConfig {
pub query: Query,
pub identifier_capture_indices: Vec<u32>,
+ pub text_highlight_cache: TextHighlightCache,
}
pub struct IndentConfig {
@@ -367,12 +368,16 @@ impl Grammar {
self.highlight_map.lock().clone()
}
+ pub fn highlight_map_matches(&self, highlight_map: &HighlightMap) -> bool {
+ self.highlight_map.lock().same(highlight_map)
+ }
+
pub fn highlight_id_for_name(&self, name: &str) -> Option<HighlightId> {
self.highlights_config
.as_ref()?
.query
.capture_index_for_name(name)
- .and_then(|capture_id| self.highlight_map.lock().get(capture_id))
+ .and_then(|capture_id| self.highlight_map.lock().get(CaptureId(capture_id)))
}
pub fn debug_variables_config(&self) -> Option<&DebugVariablesConfig> {
@@ -469,6 +474,7 @@ impl Grammar {
self.highlights_config = Some(HighlightsConfig {
query,
identifier_capture_indices,
+ text_highlight_cache: TextHighlightCache::default(),
});
Ok(self)
diff --git a/crates/language_core/src/highlight_cache.rs b/crates/language_core/src/highlight_cache.rs
new file mode 100644
index 00000000000..b820787f716
--- /dev/null
+++ b/crates/language_core/src/highlight_cache.rs
@@ -0,0 +1,284 @@
+use crate::grammar::Grammar;
+use crate::highlight_map::{HighlightId, HighlightMap};
+use collections::FxHasher;
+use lru::LruCache;
+use parking_lot::Mutex;
+use smallvec::SmallVec;
+use std::{
+ fmt,
+ hash::{Hash, Hasher as _},
+ ops::Range,
+ sync::Arc,
+};
+
+pub const MAX_TEXT_HIGHLIGHT_ENTRY_BYTES: usize = MAX_TEXT_HIGHLIGHT_CACHE_BYTES / 8;
+
+const MAX_TEXT_HIGHLIGHT_CACHE_BYTES: usize = 4 * 1024 * 1024;
+const APPROXIMATE_LRU_NODE_BYTES: usize = 4 * size_of::<usize>();
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub struct TextHighlightKey {
+ text_hash: u64,
+ text_len: usize,
+}
+
+impl TextHighlightKey {
+ pub fn new<'a>(text_chunks: impl Iterator<Item = &'a str>, text_len: usize) -> Self {
+ let mut hasher = FxHasher::default();
+ for chunk in text_chunks {
+ hasher.write(chunk.as_bytes());
+ }
+ Self {
+ text_hash: hasher.finish(),
+ text_len,
+ }
+ }
+}
+
+#[derive(Clone, Default)]
+pub struct ResolvedHighlights {
+ pub sources: SmallVec<[(Arc<Grammar>, HighlightMap); 2]>,
+ pub runs: Arc<[(Range<usize>, HighlightId)]>,
+}
+
+impl ResolvedHighlights {
+ pub fn is_current(&self) -> bool {
+ self.sources
+ .iter()
+ .all(|(grammar, highlight_map)| grammar.highlight_map_matches(highlight_map))
+ }
+
+ pub fn cost_bytes(&self) -> usize {
+ self.sources.len() * size_of::<(Arc<Grammar>, HighlightMap)>()
+ + self.runs.len() * size_of::<(Range<usize>, HighlightId)>()
+ }
+}
+
+impl fmt::Debug for ResolvedHighlights {
+ fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
+ f.debug_struct("ResolvedHighlights")
+ .field("runs", &self.runs)
+ .finish_non_exhaustive()
+ }
+}
+
+pub struct TextHighlightCache(Mutex<CostBudgetedLru<TextHighlightKey, TextHighlightEntry>>);
+
+impl Default for TextHighlightCache {
+ fn default() -> Self {
+ Self(Mutex::new(CostBudgetedLru::new(
+ MAX_TEXT_HIGHLIGHT_CACHE_BYTES,
+ MAX_TEXT_HIGHLIGHT_ENTRY_BYTES,
+ )))
+ }
+}
+
+impl TextHighlightCache {
+ pub fn get<'a>(
+ &self,
+ key: &TextHighlightKey,
+ text_chunks: impl Iterator<Item = &'a str>,
+ ) -> Option<ResolvedHighlights> {
+ let mut cache = self.0.lock();
+ let entry = cache.get(key)?;
+ if !entry.highlights.is_current() {
+ return None;
+ }
+ if !chunks_match_text(&entry.text, text_chunks) {
+ return None;
+ }
+ Some(entry.highlights.clone())
+ }
+
+ pub fn insert(
+ &self,
+ key: TextHighlightKey,
+ text: Arc<str>,
+ highlights: ResolvedHighlights,
+ ) -> ResolvedHighlights {
+ let cost = text.len() + highlights.cost_bytes();
+ self.0.lock().insert(
+ key,
+ TextHighlightEntry {
+ text,
+ highlights: highlights.clone(),
+ },
+ cost,
+ );
+ highlights
+ }
+}
+
+struct CostBudgetedLru<K: Hash + Eq, V> {
+ entries: LruCache<K, (V, usize)>,
+ total_cost: usize,
+ max_total_cost: usize,
+ max_entry_cost: usize,
+}
+
+impl<K: Hash + Eq, V> CostBudgetedLru<K, V> {
+ const ENTRY_OVERHEAD_BYTES: usize =
+ size_of::<K>() + size_of::<(V, usize)>() + APPROXIMATE_LRU_NODE_BYTES;
+
+ fn new(max_total_cost: usize, max_entry_cost: usize) -> Self {
+ Self {
+ entries: LruCache::unbounded(),
+ total_cost: 0,
+ max_total_cost,
+ max_entry_cost,
+ }
+ }
+
+ fn get(&mut self, key: &K) -> Option<&V> {
+ self.entries.get(key).map(|(value, _)| value)
+ }
+
+ fn insert(&mut self, key: K, value: V, cost: usize) {
+ if cost > self.max_entry_cost {
+ return;
+ }
+ let cost = cost + Self::ENTRY_OVERHEAD_BYTES;
+ if let Some((_, old_cost)) = self.entries.put(key, (value, cost)) {
+ self.total_cost -= old_cost;
+ }
+ self.total_cost += cost;
+ while self.total_cost > self.max_total_cost {
+ let Some((_, (_, evicted_cost))) = self.entries.pop_lru() else {
+ break;
+ };
+ self.total_cost -= evicted_cost;
+ }
+ }
+}
+
+struct TextHighlightEntry {
+ text: Arc<str>,
+ highlights: ResolvedHighlights,
+}
+
+fn chunks_match_text<'a>(text: &str, text_chunks: impl Iterator<Item = &'a str>) -> bool {
+ let mut remaining = text;
+ for chunk in text_chunks {
+ let Some(rest) = remaining.strip_prefix(chunk) else {
+ return false;
+ };
+ remaining = rest;
+ }
+ remaining.is_empty()
+}
+
+#[cfg(test)]
+mod tests {
+ use super::*;
+
+ #[test]
+ fn test_budget_evicts_least_recently_used() {
+ let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES;
+ let mut cache = CostBudgetedLru::<&str, u32>::new(2 * (40 + overhead), 100);
+ cache.insert("a", 1, 40);
+ cache.insert("b", 2, 40);
+ cache.get(&"a");
+ cache.insert("c", 3, 40);
+ assert_eq!(cache.get(&"a"), Some(&1));
+ assert_eq!(cache.get(&"b"), None);
+ assert_eq!(cache.get(&"c"), Some(&3));
+ assert_eq!(cache.total_cost, 2 * (40 + overhead));
+ }
+
+ #[test]
+ fn test_replacing_an_entry_updates_the_budget() {
+ let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES;
+ let mut cache = CostBudgetedLru::<&str, u32>::new(100 + 2 * overhead, 100);
+ cache.insert("a", 1, 60);
+ cache.insert("a", 2, 30);
+ assert_eq!(cache.total_cost, 30 + overhead);
+ assert_eq!(cache.get(&"a"), Some(&2));
+ cache.insert("b", 3, 70);
+ assert_eq!(cache.get(&"a"), Some(&2));
+ assert_eq!(cache.get(&"b"), Some(&3));
+ }
+
+ #[test]
+ fn test_oversized_entries_are_rejected_without_flushing() {
+ let overhead = CostBudgetedLru::<&str, u32>::ENTRY_OVERHEAD_BYTES;
+ let mut cache = CostBudgetedLru::<&str, u32>::new(100 + overhead, 50);
+ cache.insert("a", 1, 40);
+ cache.insert("b", 2, 60);
+ assert_eq!(cache.get(&"a"), Some(&1));
+ assert_eq!(cache.get(&"b"), None);
+ assert_eq!(cache.total_cost, 40 + overhead);
+ }
+
+ #[test]
+ fn test_zero_cost_entries_still_consume_budget() {
+ let overhead = CostBudgetedLru::<usize, u32>::ENTRY_OVERHEAD_BYTES;
+ let mut cache = CostBudgetedLru::<usize, u32>::new(3 * overhead, 100);
+ for key in 0..5_usize {
+ cache.insert(key, key as u32, 0);
+ }
+ assert_eq!(cache.get(&0), None);
+ assert_eq!(cache.get(&1), None);
+ assert_eq!(cache.get(&2), Some(&2));
+ assert_eq!(cache.get(&3), Some(&3));
+ assert_eq!(cache.get(&4), Some(&4));
+ assert_eq!(
+ cache.total_cost,
+ 3 * overhead,
+ "Entries with zero caller-supplied cost must still be bounded by the budget"
+ );
+ }
+
+ #[test]
+ fn test_oversized_highlights_are_returned_but_not_cached() {
+ let text = Arc::<str>::from("fn main() {}");
+ let run_bytes = size_of::<(Range<usize>, HighlightId)>();
+ let max_entry_cost = text.len() + 2 * run_bytes;
+ let cache = TextHighlightCache(Mutex::new(CostBudgetedLru::new(
+ max_entry_cost * 10,
+ max_entry_cost,
+ )));
+ let resolved = |ranges: &[Range<usize>]| ResolvedHighlights {
+ sources: SmallVec::new(),
+ runs: ranges
+ .iter()
+ .map(|range| (range.clone(), HighlightId::new(0)))
+ .collect(),
+ };
+
+ let small_text = Arc::<str>::from("fn f() {}");
+ let small_key = TextHighlightKey::new([small_text.as_ref()].into_iter(), small_text.len());
+ let small_range = 0..2;
+ let small_highlights = cache.insert(
+ small_key.clone(),
+ Arc::clone(&small_text),
+ resolved(std::slice::from_ref(&small_range)),
+ );
+ assert_eq!(
+ small_highlights.runs.as_ref(),
+ resolved(std::slice::from_ref(&small_range)).runs.as_ref()
+ );
+ assert_eq!(
+ cache
+ .get(&small_key, [small_text.as_ref()].into_iter())
+ .map(|highlights| highlights.runs),
+ Some(small_highlights.runs.clone()),
+ "highlights within the entry budget must be cached as-is"
+ );
+
+ let big_key = TextHighlightKey::new([text.as_ref()].into_iter(), text.len());
+ let big_highlights_source = resolved(&[0..2, 3..7, 8..9]);
+ let big_highlights = cache.insert(
+ big_key.clone(),
+ Arc::clone(&text),
+ big_highlights_source.clone(),
+ );
+ assert!(
+ Arc::ptr_eq(&big_highlights.runs, &big_highlights_source.runs),
+ "highlights over the entry budget must be returned unchanged"
+ );
+ assert!(
+ cache.get(&big_key, [text.as_ref()].into_iter()).is_none(),
+ "highlights over the entry budget must not be cached"
+ );
+ }
+}
diff --git a/crates/language_core/src/highlight_map.rs b/crates/language_core/src/highlight_map.rs
index cba5cda6f75..b5ddcf670e1 100644
--- a/crates/language_core/src/highlight_map.rs
+++ b/crates/language_core/src/highlight_map.rs
@@ -3,6 +3,15 @@ use std::{num::NonZeroU32, sync::Arc};
#[derive(Clone, Debug)]
pub struct HighlightMap(Arc<[Option<HighlightId>]>);
+#[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
+pub struct CaptureId(pub u32);
+
+#[derive(Clone, Copy, Debug, PartialEq, Eq)]
+pub struct HighlightCaptureRef {
+ pub grammar_index: usize,
+ pub capture_id: CaptureId,
+}
+
#[derive(Clone, Copy, Debug, PartialEq, Eq)]
pub struct HighlightId(NonZeroU32);
@@ -28,8 +37,13 @@ impl HighlightMap {
}
#[inline]
- pub fn get(&self, capture_id: u32) -> Option<HighlightId> {
- self.0.get(capture_id as usize).copied().flatten()
+ pub fn get(&self, capture_id: CaptureId) -> Option<HighlightId> {
+ self.0.get(capture_id.0 as usize).copied().flatten()
+ }
+
+ #[inline]
+ pub fn same(&self, other: &HighlightMap) -> bool {
+ Arc::ptr_eq(&self.0, &other.0)
}
}
diff --git a/crates/language_core/src/language_core.rs b/crates/language_core/src/language_core.rs
index f3cb3ab51cf..e2050f325e3 100644
--- a/crates/language_core/src/language_core.rs
+++ b/crates/language_core/src/language_core.rs
@@ -2,6 +2,7 @@
// language configuration, and highlight mapping.
pub mod grammar;
+pub mod highlight_cache;
pub mod highlight_map;
pub mod language_config;
@@ -11,7 +12,7 @@ pub use grammar::{
NEXT_GRAMMAR_ID, OutlineConfig, OverrideConfig, OverrideEntry, ParseableLanguage,
RedactionConfig, RunnableCapture, RunnableConfig, TextObject, TextObjectConfig,
};
-pub use highlight_map::{HighlightId, HighlightMap};
+pub use highlight_map::{CaptureId, HighlightId, HighlightMap};
pub use language_config::{
BlockCommentConfig, BracketPair, BracketPairConfig, BracketPairContent, DecreaseIndentConfig,
JsxTagAutoCloseConfig, LanguageConfig, LanguageConfigOverride, LanguageMatcher,
diff --git a/crates/language_tools/src/highlights_tree_view.rs b/crates/language_tools/src/highlights_tree_view.rs
index e63f6e3051c..b3ac451d543 100644
--- a/crates/language_tools/src/highlights_tree_view.rs
+++ b/crates/language_tools/src/highlights_tree_view.rs
@@ -1190,7 +1190,8 @@ fn build_highlight_entries(
.collect::<Vec<_>>();
for capture in captures {
- let Some(highlight_id) = highlight_maps[capture.grammar_index].get(capture.index)
+ let Some(highlight_id) =
+ highlight_maps[capture.grammar_index].get(language::CaptureId(capture.index))
else {
continue;
};
diff --git a/crates/languages/src/rust.rs b/crates/languages/src/rust.rs
index 5bb3089c79a..d0f468a6b27 100644
--- a/crates/languages/src/rust.rs
+++ b/crates/languages/src/rust.rs
@@ -1842,8 +1842,7 @@ mod tests {
vec![
(10..13, HighlightId::TABSTOP_INSERT_ID),
(16..19, HighlightId::TABSTOP_INSERT_ID),
- (0..7, HighlightId::new(2)),
- (7..8, HighlightId::new(2)),
+ (0..8, HighlightId::new(2)),
],
))
);
@@ -1870,8 +1869,7 @@ mod tests {
0..4,
vec![
(5..9, HighlightId::TABSTOP_REPLACE_ID),
- (0..3, HighlightId::new(2)),
- (3..4, HighlightId::new(2)),
+ (0..4, HighlightId::new(2)),
],
))
);
@@ -1956,8 +1954,7 @@ mod tests {
vec![
(15..20, HighlightId::TABSTOP_REPLACE_ID),
(16..19, HighlightId::TABSTOP_INSERT_ID),
- (0..13, HighlightId::new(2)),
- (13..14, HighlightId::new(2)),
+ (0..14, HighlightId::new(2)),
],
))
);
diff --git a/crates/markdown/src/markdown.rs b/crates/markdown/src/markdown.rs
index c8f10d3117b..9dc936e3c77 100644
--- a/crates/markdown/src/markdown.rs
+++ b/crates/markdown/src/markdown.rs
@@ -5,7 +5,6 @@ mod path_range;
mod selection;
use base64::Engine as _;
-
use gpui::EdgesRefinement;
use gpui::HitboxBehavior;
use gpui::UnderlineStyle;
@@ -41,7 +40,7 @@ use gpui::{
TextStyle, TextStyleRefinement, WrappedLineLayout, actions, canvas, img, point, quad, relative,
size,
};
-use language::{CharClassifier, Language, LanguageRegistry, Rope};
+use language::{CharClassifier, Language, LanguageRegistry, ResolvedHighlights, Rope};
use parser::CodeBlockMetadata;
use parser::{
MarkdownEvent, MarkdownTag, MarkdownTagEnd, ParsedMetadataBlock, parse_links_only,
@@ -977,20 +976,7 @@ impl Markdown {
return None;
};
- match kind {
- CodeBlockKind::FencedLang(language) => self
- .parsed_markdown
- .languages_by_name
- .get(language)
- .cloned(),
- CodeBlockKind::FencedSrc(path_range) => self
- .parsed_markdown
- .languages_by_path
- .get(&path_range.path)
- .cloned(),
- CodeBlockKind::Fenced => self.parsed_markdown.fallback_code_block_language.clone(),
- CodeBlockKind::Indented => None,
- }
+ self.parsed_markdown.code_block_language(kind)
})
}
@@ -1258,6 +1244,7 @@ impl Markdown {
footnote_definitions: HashMap::default(),
link_definition_spans: Arc::default(),
fallback_code_block_language: None,
+ code_block_highlights: Arc::default(),
},
Default::default(),
);
@@ -1335,23 +1322,23 @@ impl Markdown {
}
}
- (
- ParsedMarkdown {
- source,
- events: Arc::from(events),
- languages_by_name,
- languages_by_path,
- root_block_starts: Arc::from(root_block_starts),
- html_blocks,
- metadata_blocks,
- mermaid_diagrams,
- heading_slugs,
- footnote_definitions,
- link_definition_spans: Arc::from(link_definition_spans),
- fallback_code_block_language,
- },
- images_by_source_offset,
- )
+ let mut parsed = ParsedMarkdown {
+ source,
+ events: Arc::from(events),
+ languages_by_name,
+ languages_by_path,
+ root_block_starts: Arc::from(root_block_starts),
+ html_blocks,
+ metadata_blocks,
+ mermaid_diagrams,
+ heading_slugs,
+ footnote_definitions,
+ link_definition_spans: Arc::from(link_definition_spans),
+ fallback_code_block_language,
+ code_block_highlights: Arc::default(),
+ };
+ parsed.code_block_highlights = Arc::new(compute_code_block_highlights(&parsed));
+ (parsed, images_by_source_offset)
});
cx.spawn(async move |this, cx| {
@@ -1497,8 +1484,11 @@ pub struct ParsedMarkdown {
pub footnote_definitions: HashMap<SharedString, usize>,
pub(crate) link_definition_spans: Arc<[Range<usize>]>,
pub(crate) fallback_code_block_language: Option<Arc<Language>>,
+ pub(crate) code_block_highlights: Arc<CodeBlockHighlights>,
}
+pub(crate) type CodeBlockHighlights = HashMap<usize, ResolvedHighlights>;
+
impl ParsedMarkdown {
pub fn source(&self) -> &SharedString {
&self.source
@@ -1606,6 +1596,109 @@ impl ParsedMarkdown {
selection,
)
}
+
+ pub(crate) fn code_block_language(&self, kind: &CodeBlockKind) -> Option<Arc<Language>> {
+ match kind {
+ CodeBlockKind::FencedLang(name) => self.languages_by_name.get(name).cloned(),
+ CodeBlockKind::FencedSrc(path_range) => {
+ self.languages_by_path.get(&path_range.path).cloned()
+ }
+ CodeBlockKind::Fenced => self.fallback_code_block_language.clone(),
+ CodeBlockKind::Indented => None,
+ }
+ }
+}
+
+struct PendingCodeBlock<'a> {
+ language: Arc<Language>,
+ texts: Vec<(Range<usize>, &'a str)>,
+}
+
+fn compute_code_block_highlights(parsed: &ParsedMarkdown) -> CodeBlockHighlights {
+ let mut code_block_highlights = CodeBlockHighlights::default();
+ let mut pending_block: Option<PendingCodeBlock> = None;
+ for (range, event) in parsed.events.iter() {
+ match event {
+ MarkdownEvent::Start(MarkdownTag::CodeBlock { kind, .. }) => {
+ if parsed.mermaid_diagrams.contains_key(&range.start) {
+ pending_block = None;
+ continue;
+ }
+ pending_block = parsed
+ .code_block_language(kind)
+ .map(|language| PendingCodeBlock {
+ language,
+ texts: Vec::new(),
+ });
+ }
+ MarkdownEvent::End(MarkdownTagEnd::CodeBlock) => {
+ if let Some(block) = pending_block.take() {
+ highlight_code_block(block, &mut code_block_highlights);
+ }
+ }
+ MarkdownEvent::Text => {
+ if let Some(block) = &mut pending_block {
+ block
+ .texts
+ .push((range.clone(), &parsed.source[range.clone()]));
+ }
+ }
+ MarkdownEvent::SubstitutedText(text) => {
+ if let Some(block) = &mut pending_block {
+ block.texts.push((range.clone(), text.as_str()));
+ }
+ }
+ _ => {}
+ }
+ }
+ code_block_highlights
+}
+
+fn highlight_code_block(block: PendingCodeBlock, code_block_highlights: &mut CodeBlockHighlights) {
+ let mut combined = String::new();
+ let mut text_offsets = Vec::with_capacity(block.texts.len());
+ for (_, text) in &block.texts {
+ text_offsets.push(combined.len());
+ combined.push_str(text);
+ }
+ let resolved = block
+ .language
+ .highlight_text_resolved(&Rope::from(combined.as_str()), 0..combined.len());
+ if resolved.runs.is_empty() {
+ return;
+ }
+ if let [(source_range, _)] = block.texts.as_slice() {
+ code_block_highlights.insert(source_range.start, resolved);
+ return;
+ }
+ let mut runs = resolved.runs.iter().peekable();
+ for ((source_range, text), text_offset) in block.texts.iter().zip(text_offsets) {
+ let text_end = text_offset + text.len();
+ let mut text_runs = Vec::new();
+ while let Some((run_range, highlight_id)) = runs.peek() {
+ if run_range.start >= text_end {
+ break;
+ }
+ let start = run_range.start.max(text_offset);
+ let end = run_range.end.min(text_end);
+ if end > start {
+ text_runs.push((start - text_offset..end - text_offset, *highlight_id));
+ }
+ if run_range.end > text_end {
+ break;
+ }
+ runs.next();
+ }
+ if !text_runs.is_empty() {
+ code_block_highlights.insert(
+ source_range.start,
+ ResolvedHighlights {
+ sources: resolved.sources.clone(),
+ runs: text_runs.into(),
+ },
+ );
+ }
+ }
}
pub enum AutoscrollBehavior {
@@ -2491,12 +2584,7 @@ impl Element for MarkdownElement {
next_search_highlight_ix: 0,
}
};
- let mut builder = MarkdownElementBuilder::new(
- &self.style.container_style,
- self.style.base_text_style.clone(),
- self.style.syntax.clone(),
- highlights,
- );
+
let (parsed_markdown, images, active_root_block, render_mermaid_diagrams, mermaid_state) = {
let markdown = self.markdown.read(cx);
(
@@ -2507,6 +2595,13 @@ impl Element for MarkdownElement {
markdown.mermaid_state.clone(),
)
};
+ let mut builder = MarkdownElementBuilder::new(
+ &self.style.container_style,
+ self.style.base_text_style.clone(),
+ self.style.syntax.clone(),
+ highlights,
+ parsed_markdown.code_block_highlights.clone(),
+ );
let markdown_end = if let Some(last) = parsed_markdown.events.last() {
last.0.end
} else {
@@ -2670,19 +2765,7 @@ impl Element for MarkdownElement {
continue;
}
- let language = match kind {
- CodeBlockKind::Fenced => {
- parsed_markdown.fallback_code_block_language.clone()
- }
- CodeBlockKind::FencedLang(language) => {
- parsed_markdown.languages_by_name.get(language).cloned()
- }
- CodeBlockKind::FencedSrc(path_range) => parsed_markdown
- .languages_by_path
- .get(&path_range.path)
- .cloned(),
- _ => None,
- };
+ let language = parsed_markdown.code_block_language(kind);
let is_indented = matches!(kind, CodeBlockKind::Indented);
let scroll_handle = if self.style.code_block_overflow_x_scroll {
@@ -3587,6 +3670,7 @@ struct MarkdownElementBuilder {
base_text_style: TextStyle,
text_style_stack: Vec<TextStyleRefinement>,
code_block_stack: Vec<Option<Arc<Language>>>,
+ code_block_highlights: Arc<CodeBlockHighlights>,
link_depth: usize,
list_stack: Vec<ListStackEntry>,
table: TableState,
@@ -3688,6 +3772,7 @@ impl MarkdownElementBuilder {
base_text_style: TextStyle,
syntax_theme: Arc<SyntaxTheme>,
highlights: MarkdownHighlights,
+ code_block_highlights: Arc<CodeBlockHighlights>,
) -> Self {
Self {
div_stack: vec![{
@@ -3706,6 +3791,7 @@ impl MarkdownElementBuilder {
base_text_style,
text_style_stack: Vec::new(),
code_block_stack: Vec::new(),
+ code_block_highlights,
link_depth: 0,
list_stack: Vec::new(),
table: TableState::default(),
@@ -3937,24 +4023,34 @@ impl MarkdownElementBuilder {
// Compute the base text style once
let text_style = self.text_style();
- if let Some(Some(language)) = self.code_block_stack.last() {
+ if let Some(language) = self.code_block_stack.last().and_then(Option::as_ref)
+ && let Some(resolved) = self.code_block_highlights.get(&source_range.start)
+ {
+ let runs = if resolved.is_current() {
+ resolved.runs.clone()
+ } else {
+ language
+ .highlight_text_resolved(&Rope::from(text), 0..text.len())
+ .runs
+ };
let mut offset = 0;
- for (range, highlight_id) in language.highlight_text(&Rope::from(text), 0..text.len()) {
- if range.start > offset {
+ for (run_range, highlight_id) in runs.iter() {
+ if run_range.start > offset {
self.pending_line
.runs
- .push(text_style.to_run(range.start - offset));
+ .push(text_style.to_run(run_range.start - offset));
}
- let run_len = range.len();
- if let Some(highlight) = self.syntax_theme.get(highlight_id).cloned() {
+ let run_len = run_range.len();
+ let highlight = self.syntax_theme.get(*highlight_id).cloned();
+ if let Some(highlight) = highlight {
self.pending_line
.runs
.push(text_style.clone().highlight(highlight).to_run(run_len));
} else {
self.pending_line.runs.push(text_style.to_run(run_len));
}
- offset = range.end;
+ offset = run_range.end;
}
if offset < text.len() {
@@ -4080,7 +4176,11 @@ impl MarkdownElementBuilder {
layout: text.layout().clone(),
source_mappings: line.source_mappings,
source_end: self.current_source_index,
- language: self.code_block_stack.last().cloned().flatten(),
+ language: self
+ .code_block_stack
+ .last()
+ .and_then(|entry| entry.as_ref())
+ .cloned(),
text_align,
highlights,
code_chips: line.code_chips.into_iter().collect(),
@@ -5164,9 +5264,118 @@ mod tests {
});
}
+ #[gpui::test]
+ fn test_code_block_highlights_cached_at_parse_time(cx: &mut TestAppContext) {
+ let source = "```rust\nfn main() {}\n```";
+ let (language, markdown) = markdown_with_rust_language(source, cx);
+
+ let code_text = "fn main() {}\n";
+ let code_start = source.find(code_text).unwrap();
+ let cached = cached_code_block_highlights(&markdown, code_start, cx);
+ let expected = language.highlight_text(&Rope::from(code_text), 0..code_text.len());
+ assert_eq!(
+ cached.runs.to_vec(),
+ expected,
+ "highlights cached at parse time must match direct highlighting"
+ );
+ assert!(!expected.is_empty(), "rust code must produce highlights");
+ assert!(cached.is_current());
+
+ language.set_theme(&rust_test_theme());
+ let stale = cached_code_block_highlights(&markdown, code_start, cx);
+ assert!(
+ !stale.is_current(),
+ "a theme change must make parse-time highlights stale so rendering re-resolves them"
+ );
+ }
+
+ #[gpui::test]
+ fn test_code_block_highlights_reused_when_streaming(cx: &mut TestAppContext) {
+ let source = "```rust\nfn main() {}\n```\n\nSome trailing text";
+ let (language, markdown) = markdown_with_rust_language(source, cx);
+
+ let code_start = source.find("fn main").unwrap();
+ let first_parse_highlights = cached_code_block_highlights(&markdown, code_start, cx);
+
+ markdown.update(cx, |markdown, cx| {
+ markdown.append(" that keeps streaming in\n\n```rust\nlet x = 1;\n```", cx)
+ });
+ cx.run_until_parked();
+
+ let second_parse_highlights = cached_code_block_highlights(&markdown, code_start, cx);
+ assert!(
+ Arc::ptr_eq(&first_parse_highlights.runs, &second_parse_highlights.runs),
+ "highlights of an unchanged code block must be reused across parses"
+ );
+
+ markdown.update(cx, |markdown, cx| {
+ markdown.replace("```rust\nfn main() { panic!() }\n```", cx)
+ });
+ cx.run_until_parked();
+
+ let changed_highlights = cached_code_block_highlights(&markdown, code_start, cx);
+ let changed_code = "fn main() { panic!() }\n";
+ assert_eq!(
+ changed_highlights.runs.as_ref(),
+ language
+ .highlight_text_resolved(&Rope::from(changed_code), 0..changed_code.len())
+ .runs
+ .as_ref(),
+ "highlights of a changed code block must be recomputed"
+ );
+ }
+
+ #[gpui::test]
+ fn test_mermaid_blocks_are_not_highlighted(cx: &mut TestAppContext) {
+ ensure_theme_initialized(cx);
+ let mermaid_language = Arc::new(
+ Language::new(
+ LanguageConfig {
+ name: "mermaid".into(),
+ ..LanguageConfig::default()
+ },
+ Some(language::rust_lang().grammar().unwrap().ts_language.clone()),
+ )
+ .with_highlights_query("(identifier) @variable")
+ .unwrap(),
+ );
+ let language_registry = Arc::new(LanguageRegistry::test(cx.executor()));
+ language_registry.add(mermaid_language);
+
+ let source = "```mermaid\ngraph TD;\n```";
+ let markdown = cx.new(|cx| {
+ Markdown::new_with_options(
+ source.into(),
+ Some(language_registry),
+ None,
+ MarkdownOptions {
+ render_mermaid_diagrams: true,
+ ..MarkdownOptions::default()
+ },
+ cx,
+ )
+ });
+ cx.run_until_parked();
+
+ markdown.read_with(cx, |markdown, _| {
+ let parsed = markdown.parsed_markdown();
+ assert_eq!(
+ parsed.mermaid_diagrams.len(),
+ 1,
+ "the block must be recognized as a mermaid diagram"
+ );
+ assert_eq!(
+ parsed.code_block_highlights.len(),
+ 0,
+ "mermaid blocks must not be highlighted"
+ );
+ });
+ }
+
#[gpui::test]
fn test_fallback_language_highlights_untagged_code_blocks(cx: &mut TestAppContext) {
let language = language::rust_lang();
+ language.set_theme(&rust_test_theme());
let language_registry = Arc::new(LanguageRegistry::test(cx.executor()));
language_registry.add(language.clone());
@@ -5186,19 +5395,19 @@ mod tests {
cx.run_until_parked();
markdown.read_with(cx, |markdown, _| {
- let fallback = markdown
- .parsed_markdown()
+ let parsed = markdown.parsed_markdown();
+ let fallback = parsed
.fallback_code_block_language
.as_ref()
.expect("the fallback language must be resolved for untagged code blocks");
assert_eq!(fallback.name(), language.name());
- assert_eq!(
- markdown
- .first_code_block_language()
- .map(|language| language.name()),
- Some(language.name()),
- "untagged code blocks must resolve to the fallback language"
- );
+
+ let code_start = source.find("fn main").unwrap();
+ let cached = parsed
+ .code_block_highlights
+ .get(&code_start)
+ .expect("untagged code blocks must be highlighted with the fallback language");
+ assert!(!cached.runs.is_empty());
});
}
@@ -7167,4 +7376,53 @@ mod tests {
};
(prose.layout.line_height(), code.layout.line_height())
}
+
+ fn rust_test_theme() -> SyntaxTheme {
+ SyntaxTheme::new(
+ [
+ ("keyword", gpui::red()),
+ ("function", gpui::blue()),
+ ("type", gpui::green()),
+ ]
+ .into_iter()
+ .map(|(name, color)| {
+ (
+ name.to_owned(),
+ gpui::HighlightStyle {
+ color: Some(color),
+ ..gpui::HighlightStyle::default()
+ },
+ )
+ }),
+ )
+ }
+
+ fn markdown_with_rust_language(
+ source: &str,
+ cx: &mut TestAppContext,
+ ) -> (Arc<Language>, Entity<Markdown>) {
+ let language = language::rust_lang();
+ language.set_theme(&rust_test_theme());
+ let language_registry = Arc::new(LanguageRegistry::test(cx.executor()));
+ language_registry.add(language.clone());
+ let source = SharedString::from(source.to_owned());
+ let markdown = cx.new(|cx| Markdown::new(source, Some(language_registry), None, cx));
+ cx.run_until_parked();
+ (language, markdown)
+ }
+
+ fn cached_code_block_highlights(
+ markdown: &Entity<Markdown>,
+ code_start: usize,
+ cx: &mut TestAppContext,
+ ) -> ResolvedHighlights {
+ markdown.read_with(cx, |markdown, _| {
+ markdown
+ .parsed_markdown()
+ .code_block_highlights
+ .get(&code_start)
+ .expect("code block highlights must be computed during parse")
+ .clone()
+ })
+ }
}