Skip to content

armgit / zed

Public Imported from GitHub
0

Code at the speed of thought – Zed is a high-performance, multiplayer code editor from the creators of Atom and Tree-sitter.

Properly measure invisible character replacement (#62478)

Kirill Bulatov47825fe00a8f
diff --git a/crates/editor/src/display_map.rs b/crates/editor/src/display_map.rs
index 95a58b0cf95..e633ecc21b3 100644
--- a/crates/editor/src/display_map.rs
+++ b/crates/editor/src/display_map.rs
@@ -2110,7 +2110,7 @@ impl DisplaySnapshot {
});
chars.collect::<String>().graphemes(true).next().map(|s| {
if let Some(invisible) = s.chars().next().filter(|&c| is_invisible(c)) {
- replacement(invisible).unwrap_or(s).to_owned().into()
+ replacement(invisible).map_or_else(|| s.to_owned().into(), SharedString::from)
} else if s == "\n" {
" ".into()
} else {
diff --git a/crates/editor/src/display_map/invisibles.rs b/crates/editor/src/display_map/invisibles.rs
index 90bd54ab280..20f112512f0 100644
--- a/crates/editor/src/display_map/invisibles.rs
+++ b/crates/editor/src/display_map/invisibles.rs
@@ -46,9 +46,9 @@ pub fn is_invisible(c: char) -> bool {
// ASCII control characters have fancy unicode glyphs, everything else
// is replaced by a space - unless it is used in combining characters in
// which case we need to leave it in the string.
-pub fn replacement(c: char) -> Option<&'static str> {
+pub fn replacement(c: char) -> Option<char> {
if c <= '\x1f' {
- Some(C0_SYMBOLS[c as usize])
+ C0_SYMBOLS.get(c as usize).copied()
} else if c == '\x7f' {
Some(DEL)
} else if contains(c, PRESERVE) {
@@ -58,18 +58,18 @@ pub fn replacement(c: char) -> Option<&'static str> {
}
}
-const FIXED_WIDTH_SPACE: &str = "\u{2007}";
+const FIXED_WIDTH_SPACE: char = '\u{2007}';
// IDEOGRAPHIC SPACE is common alongside Chinese and other wide character sets.
// We don't highlight this for now (as it already shows up wide in the editor),
// but could if we tracked state in the classifier.
const IDEOGRAPHIC_SPACE: char = '\u{3000}';
-const C0_SYMBOLS: &[&str] = &[
- "␀", "␁", "␂", "␃", "␄", "␅", "␆", "␇", "␈", "␉", "␊", "␋", "␌", "␍", "␎", "␏", "␐", "␑", "␒",
- "␓", "␔", "␕", "␖", "␗", "␘", "␙", "␚", "␛", "␜", "␝", "␞", "␟",
+const C0_SYMBOLS: &[char] = &[
+ '␀', '␁', '␂', '␃', '␄', '␅', '␆', '␇', '␈', '␉', '␊', '␋', '␌', '␍', '␎', '␏', '␐', '␑', '␒',
+ '␓', '␔', '␕', '␖', '␗', '␘', '␙', '␚', '␛', '␜', '␝', '␞', '␟',
];
-const DEL: &str = "␡";
+const DEL: char = '␡';
// generated using ucd-generate: ucd-generate general-category --include Format --chars ucd-16.0.0
pub const FORMAT: &[(char, char)] = &[
diff --git a/crates/editor/src/display_map/wrap_map.rs b/crates/editor/src/display_map/wrap_map.rs
index 8e01e3ffaeb..7eeb1be3946 100644
--- a/crates/editor/src/display_map/wrap_map.rs
+++ b/crates/editor/src/display_map/wrap_map.rs
@@ -2,16 +2,28 @@ use super::{
Highlights,
dimensions::RowDelta,
fold_map::{Chunk, FoldRows},
+ invisibles::{is_invisible, replacement},
tab_map::{self, TabEdit, TabPoint, TabSnapshot},
};
+use collections::HashMap;
use futures_lite::future::yield_now;
-use gpui::{App, AppContext as _, Context, Entity, Font, LineWrapper, Pixels, Task};
+use gpui::{
+ App, AppContext as _, Context, Entity, Font, FontId, LineWrapper, Pixels, Task, TextSystem,
+};
use language::{LanguageAwareStyling, Point};
use multi_buffer::RowInfo;
-use std::{cmp, collections::VecDeque, mem, ops::Range, sync::LazyLock, time::Duration};
+use std::{
+ cmp,
+ collections::VecDeque,
+ mem,
+ ops::Range,
+ sync::{Arc, LazyLock},
+ time::Duration,
+};
use sum_tree::{Bias, Cursor, Dimensions, SumTree};
use text::Patch;
+use unicode_segmentation::GraphemeCursor;
pub use super::tab_map::TextSummary;
pub type WrapEdit = text::Edit<WrapRow>;
@@ -75,6 +87,70 @@ impl TransformSummary {
#[derive(Copy, Clone, Debug, Default, Eq, Ord, PartialOrd, PartialEq)]
pub struct WrapPoint(pub Point);
+struct LineFragmentBuilder {
+ text_system: Arc<TextSystem>,
+ font_id: FontId,
+ font_size: Pixels,
+ cached_replacement_widths: HashMap<char, Pixels>,
+}
+
+impl LineFragmentBuilder {
+ fn new(text_system: Arc<TextSystem>, font: &Font, font_size: Pixels) -> Self {
+ let font_id = text_system.resolve_font(font);
+ Self {
+ text_system,
+ font_id,
+ font_size,
+ cached_replacement_widths: HashMap::default(),
+ }
+ }
+
+ fn push_fragments<'a>(&mut self, fragments: &mut Vec<gpui::LineFragment<'a>>, text: &'a str) {
+ let mut prefix_start = 0;
+ for (offset, ch) in text.char_indices() {
+ if !is_invisible(ch) {
+ continue;
+ }
+ let ch_end = offset + ch.len_utf8();
+ if !is_standalone_grapheme(text, offset, ch_end) {
+ continue;
+ }
+ let Some(width) = self.replacement_width(ch) else {
+ continue;
+ };
+ if prefix_start < offset {
+ fragments.push(gpui::LineFragment::text(&text[prefix_start..offset]));
+ }
+ fragments.push(gpui::LineFragment::element(width, ch_end - offset));
+ prefix_start = ch_end;
+ }
+ if prefix_start < text.len() || text.is_empty() {
+ fragments.push(gpui::LineFragment::text(&text[prefix_start..]));
+ }
+ }
+
+ fn replacement_width(&mut self, ch: char) -> Option<Pixels> {
+ let replacement_char = replacement(ch)?;
+ let width = *self
+ .cached_replacement_widths
+ .entry(replacement_char)
+ .or_insert_with(|| {
+ self.text_system
+ .layout_width(self.font_id, self.font_size, replacement_char)
+ });
+ Some(width)
+ }
+}
+
+fn is_standalone_grapheme(text: &str, start: usize, end: usize) -> bool {
+ let mut cursor = GraphemeCursor::new(start, text.len(), true);
+ if cursor.is_boundary(text, 0) != Ok(true) {
+ return false;
+ }
+ cursor.set_cursor(end);
+ cursor.is_boundary(text, 0) == Ok(true)
+}
+
pub struct WrapChunks<'a> {
input_chunks: tab_map::TabChunks<'a>,
input_chunk: Chunk<'a>,
@@ -202,6 +278,8 @@ impl WrapMap {
let text_system = cx.text_system();
let (font, font_size) = self.font_with_size.clone();
+ let mut fragment_builder =
+ LineFragmentBuilder::new(text_system.clone(), &font, font_size);
let mut line_wrapper = text_system.line_wrapper(font, font_size);
let tab_snapshot = new_snapshot.tab_snapshot.clone();
let total_rows = tab_snapshot.max_point().row() as usize + 1;
@@ -217,13 +295,20 @@ impl WrapMap {
&tab_edits,
wrap_width,
&mut line_wrapper,
+ &mut fragment_builder,
));
self.snapshot = new_snapshot;
self.edits_since_sync = self.edits_since_sync.compose(&edits);
} else {
let task = cx.background_spawn(async move {
let edits = new_snapshot
- .update(tab_snapshot, &tab_edits, wrap_width, &mut line_wrapper)
+ .update(
+ tab_snapshot,
+ &tab_edits,
+ wrap_width,
+ &mut line_wrapper,
+ &mut fragment_builder,
+ )
.await;
(new_snapshot, edits)
});
@@ -297,6 +382,8 @@ impl WrapMap {
let mut snapshot = self.snapshot.clone();
let text_system = cx.text_system().clone();
let (font, font_size) = self.font_with_size.clone();
+ let mut fragment_builder =
+ LineFragmentBuilder::new(text_system.clone(), &font, font_size);
let mut line_wrapper = text_system.line_wrapper(font, font_size);
if pending_edits.len() == 1
@@ -311,6 +398,7 @@ impl WrapMap {
&tab_edits,
wrap_width,
&mut line_wrapper,
+ &mut fragment_builder,
));
self.snapshot = snapshot;
self.edits_since_sync = self.edits_since_sync.compose(&wrap_edits);
@@ -319,7 +407,13 @@ impl WrapMap {
let mut edits = Patch::default();
for (tab_snapshot, tab_edits) in pending_edits {
let wrap_edits = snapshot
- .update(tab_snapshot, &tab_edits, wrap_width, &mut line_wrapper)
+ .update(
+ tab_snapshot,
+ &tab_edits,
+ wrap_width,
+ &mut line_wrapper,
+ &mut fragment_builder,
+ )
.await;
edits = edits.compose(&wrap_edits);
}
@@ -465,6 +559,7 @@ impl WrapSnapshot {
tab_edits: &[TabEdit],
wrap_width: Pixels,
line_wrapper: &mut LineWrapper,
+ fragment_builder: &mut LineFragmentBuilder,
) -> WrapPatch {
#[derive(Debug)]
struct RowEdit {
@@ -532,7 +627,7 @@ impl WrapSnapshot {
while let Some(chunk) = remaining.take().or_else(|| chunks.next()) {
if let Some(ix) = chunk.text.find('\n') {
let (prefix, suffix) = chunk.text.split_at(ix + 1);
- line_fragments.push(gpui::LineFragment::text(prefix));
+ fragment_builder.push_fragments(&mut line_fragments, prefix);
line.push_str(prefix);
remaining = Some(Chunk {
text: suffix,
@@ -546,7 +641,7 @@ impl WrapSnapshot {
line_fragments
.push(gpui::LineFragment::element(width, chunk.text.len()));
} else {
- line_fragments.push(gpui::LineFragment::text(chunk.text));
+ fragment_builder.push_fragments(&mut line_fragments, chunk.text);
}
line.push_str(chunk.text);
}
@@ -1443,6 +1538,115 @@ mod tests {
assert_eq!(row.0, 3);
}
+ #[gpui::test]
+ async fn test_invisibles_become_width_measured_elements(cx: &mut gpui::TestAppContext) {
+ init_test(cx);
+
+ let text_system = cx.read(|cx| cx.text_system().clone());
+ let font = test_font();
+ let font_size = px(14.0);
+ let mut fragment_builder = LineFragmentBuilder::new(text_system, &font, font_size);
+
+ let mut fragments = Vec::new();
+ fragment_builder.push_fragments(&mut fragments, "ab\u{7f}cd\u{80}\u{81}e");
+ let shapes = fragments
+ .iter()
+ .map(|fragment| match fragment {
+ LineFragment::Text { text } => (false, text.len()),
+ LineFragment::Element { len_utf8, width } => {
+ assert!(
+ *width > px(0.),
+ "invisible characters must carry the measured width of their \
+ replacement glyph so the wrapper accounts for what the renderer \
+ actually draws"
+ );
+ (true, *len_utf8)
+ }
+ })
+ .collect::<Vec<_>>();
+ assert_eq!(
+ shapes,
+ vec![
+ (false, 2),
+ (true, 1),
+ (false, 2),
+ (true, 2),
+ (true, 2),
+ (false, 1),
+ ],
+ "invisible chars must become single-char elements with exact utf8 lengths, \
+ keeping the wrapper's byte offsets aligned with the source text"
+ );
+
+ let mut plain_fragments = Vec::new();
+ fragment_builder.push_fragments(&mut plain_fragments, "plain text");
+ assert_eq!(
+ plain_fragments.len(),
+ 1,
+ "text without invisibles must stay one fragment"
+ );
+ assert_eq!(
+ plain_fragments.first().map(|fragment| match fragment {
+ LineFragment::Text { text } => text.len(),
+ LineFragment::Element { len_utf8, .. } => *len_utf8,
+ }),
+ Some(10),
+ );
+
+ let mut preserved_fragments = Vec::new();
+ fragment_builder.push_fragments(&mut preserved_fragments, "a\u{200d}b");
+ assert_eq!(
+ preserved_fragments.len(),
+ 1,
+ "combining-class invisibles like ZWJ have no replacement and must stay \
+ inside their text fragment"
+ );
+ }
+
+ #[gpui::test]
+ async fn test_invisibles_wrap_at_replacement_glyph_width(cx: &mut gpui::TestAppContext) {
+ init_test(cx);
+
+ let text_system = cx.read(|cx| cx.text_system().clone());
+ let font = test_font();
+ let font_id = text_system.resolve_font(&font);
+ let font_size = px(14.0);
+ let wrap_width = px(140.0);
+
+ let text = "\u{7f}\u{80}\u{81}\u{82}\u{83}\u{84}".repeat(20);
+ let buffer = cx.new(|cx| language::Buffer::local(text, cx));
+ let buffer = cx.new(|cx| MultiBuffer::singleton(buffer, cx));
+ let buffer_snapshot = buffer.read_with(cx, |buffer, cx| buffer.snapshot(cx));
+ let (_inlay_map, inlay_snapshot) = InlayMap::new(buffer_snapshot);
+ let (_fold_map, fold_snapshot) = FoldMap::new(inlay_snapshot);
+ let (mut tab_map, _) = TabMap::new(fold_snapshot, 4.try_into().unwrap());
+ let tabs_snapshot = tab_map.set_max_expansion_column(32);
+ let (_wrap_map, wrap_snapshot) =
+ cx.update(|cx| WrapMap::new(tabs_snapshot, font, font_size, Some(wrap_width), cx));
+
+ let wrapped_text = wrap_snapshot.text();
+ for row in wrapped_text.split('\n') {
+ let rendered_width = row
+ .chars()
+ .map(|ch| {
+ let rendered_char = if is_invisible(ch) {
+ replacement(ch).unwrap_or(ch)
+ } else {
+ ch
+ };
+ text_system.layout_width(font_id, font_size, rendered_char)
+ })
+ .fold(px(0.), |width, char_width| width + char_width);
+ assert!(
+ rendered_width <= wrap_width,
+ "a soft-wrapped row must fit the wrap width when drawn with the \
+ replacement glyphs the renderer substitutes for invisible characters, \
+ but this row renders at {rendered_width:?} against a wrap width of \
+ {wrap_width:?}: {row:?}"
+ );
+ }
+ }
+
#[gpui::test(iterations = 100)]
async fn test_random_wraps(cx: &mut gpui::TestAppContext, mut rng: StdRng) {
// todo this test is flaky
diff --git a/crates/gpui_shared_string/gpui_shared_string.rs b/crates/gpui_shared_string/gpui_shared_string.rs
index 4395065b9fd..8449c065ad2 100644
--- a/crates/gpui_shared_string/gpui_shared_string.rs
+++ b/crates/gpui_shared_string/gpui_shared_string.rs
@@ -1,5 +1,6 @@
use std::{
borrow::{Borrow, Cow},
+ iter,
sync::Arc,
};
@@ -120,6 +121,13 @@ impl From<&str> for SharedString {
}
}
+impl From<char> for SharedString {
+ #[inline]
+ fn from(c: char) -> SharedString {
+ SharedString(SmolStr::from_iter(iter::once(c)))
+ }
+}
+
impl From<&mut str> for SharedString {
#[inline]
fn from(s: &mut str) -> SharedString {