Skip to content

A modern terminal with a built-in multiplexer: close it and your sessions keep running. Run shells and tons of AI agents across your machines, then pick them up from desktop, browser, TUI and more. Written in Rust with no UI framework: interface drawn straight on the GPU. No Electron, GPUI or webview.

thinkterm-font-web: report the gaps in a shaped run

RoversX1ca2ced64759
diff --git a/thinkterm-font-web/src/lib.rs b/thinkterm-font-web/src/lib.rs
index f2439404..2738c685 100644
--- a/thinkterm-font-web/src/lib.rs
+++ b/thinkterm-font-web/src/lib.rs
@@ -12,7 +12,7 @@
use anyhow::{anyhow, Context, Result};
use finl_unicode::grapheme_clusters::Graphemes;
use std::cell::RefCell;
-use std::collections::HashMap;
+use std::collections::{HashMap, HashSet};
use std::ops::Range;
use swash::scale::{image::Content, Render, ScaleContext, Source, StrikeWith};
use swash::zeno::Format;
@@ -31,6 +31,16 @@ pub struct Face {
index: u32,
presentation: Presentation,
scaler: RefCell<ScaleContext>,
+ /// What the cmap said about a character, once asked.
+ ///
+ /// `swash()` rebuilds a `FontRef` and re-reads the table directory on
+ /// every call, and `face_for` calls it for every grapheme against every
+ /// face. A line of Chinese is a *single* cluster -- `CellCluster` only
+ /// breaks on attribute changes and whitespace, and Chinese prose has no
+ /// whitespace -- so its `ShapeKey` differs on every row and the shape
+ /// cache misses on every frame of a scroll. This is what keeps that
+ /// miss cheap.
+ coverage: RefCell<HashMap<char, bool>>,
}
impl std::fmt::Debug for Face {
@@ -42,8 +52,16 @@ impl std::fmt::Debug for Face {
}
}
+/// Characters that join or qualify their neighbours rather than being drawn
+/// themselves: what matters is whether the face can draw what they connect.
+///
+/// Format characters only. A combining mark like U+0301 is a mark, not a
+/// joiner; treating it as one would send every accented letter to the
+/// fallback even where the face draws it perfectly well.
fn is_joiner_or_selector(c: char) -> bool {
- matches!(c, '\u{200d}' | '\u{fe0e}' | '\u{fe0f}') || ('\u{e0100}'..='\u{e01ef}').contains(&c)
+ matches!(c, '\u{200c}' | '\u{200d}')
+ || ('\u{fe00}'..='\u{fe0f}').contains(&c)
+ || ('\u{e0100}'..='\u{e01ef}').contains(&c)
}
impl Face {
@@ -72,6 +90,7 @@ impl Face {
index,
presentation,
scaler: RefCell::new(ScaleContext::new()),
+ coverage: RefCell::new(HashMap::new()),
})
}
@@ -98,16 +117,37 @@ impl Face {
}
}
+ /// Whether this face has a glyph for `c`, remembering the answer.
+ fn has(&self, c: char) -> bool {
+ if let Some(known) = self.coverage.borrow().get(&c) {
+ return *known;
+ }
+ let known = self.swash().charmap().map(c) != 0;
+ let mut coverage = self.coverage.borrow_mut();
+ // Nothing else bounds this: it is keyed by character, and a pane can
+ // emit any of them. Flushed wholesale rather than by LRU -- the
+ // entries cost one cmap lookup to rebuild.
+ if coverage.len() > 65536 {
+ coverage.clear();
+ }
+ coverage.insert(c, known);
+ known
+ }
+
/// Whether every character that matters in `cluster` has a glyph here.
+ ///
+ /// Per character, so it cannot answer whether the face can *join* them:
+ /// a ZWJ sequence whose every piece is in the cmap looks covered here
+ /// and comes out as several separate glyphs. That is why the gap
+ /// decision is finished after shaping, in `FontSet::judge`.
fn covers(&self, cluster: &str) -> bool {
- let charmap = self.swash().charmap();
let mut any = false;
for c in cluster.chars() {
if is_joiner_or_selector(c) {
continue;
}
any = true;
- if charmap.map(c) == 0 {
+ if !self.has(c) {
return false;
}
}
@@ -333,42 +373,366 @@ impl FontSet {
}
}
-impl FontShaper for FontSet {
- fn shape(
+/// Why a grapheme has to be drawn with the platform's own fonts.
+///
+/// The order is part of the design. `.notdef` outranks the rest because it
+/// is the box the user is looking at; colour outranks joining because
+/// `\u{1f468}\u{200d}\u{1f4bb}` asked for emoji presentation is a coverage
+/// problem, not a joining one.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GapReason {
+ /// No bundled face has all of its characters.
+ Uncovered,
+ /// A face took it and the shaper still produced `.notdef`.
+ Notdef,
+ /// Emoji presentation was asked for and the face that answered has no
+ /// colour tables, so all it can draw is a monochrome stand-in.
+ NoColor,
+ /// The face has the pieces but not the join: a ZWJ sequence came out
+ /// as several drawn glyphs.
+ Unjoined,
+}
+
+/// A grapheme the bundled faces cannot draw, handed to the caller whole.
+///
+/// The browser client draws these with the platform's own fonts and keeps
+/// the result in its atlas. Everything the caller needs to do that is here,
+/// and in particular the text is the **whole grapheme**: splitting it would
+/// break combining marks, variation selectors and ZWJ sequences.
+#[derive(Debug, Clone, PartialEq, Eq)]
+pub struct Gap {
+ /// The grapheme, exactly as it appeared.
+ pub text: String,
+ /// Where it sits in the text that was shaped, in bytes. The same basis
+ /// as `GlyphInfo::cluster`.
+ pub bytes: Range<usize>,
+ /// How many terminal columns it occupies. Taken from the row's own
+ /// width data, never re-derived: the fallback has to land on the grid
+ /// the terminal already decided on. In particular it is *not* the sum
+ /// of the stand-in glyphs' cells, which `shape_run` apportions by
+ /// advance and which does not add up to this when the advances are 0.
+ pub cells: u8,
+ /// Which presentation was asked for, so the caller can tell a text
+ /// glyph it may tint from an emoji that carries its own colour.
+ pub presentation: Option<Presentation>,
+ pub reason: GapReason,
+}
+
+/// What a shaped glyph is for.
+///
+/// This replaces "ask for the indices, then delete them": the set is
+/// computed once, the shared `WebShaped` is never mutated, and "exactly
+/// this group, no more and no fewer" becomes a property of the vector
+/// rather than of every caller's loop.
+#[derive(Debug, Clone, Copy, PartialEq, Eq)]
+pub enum GlyphRole {
+ /// Nothing to do with a gap. Draw it.
+ Own,
+ /// Where a gap's fallback glyph goes, indexing `gaps`.
+ GapLead(u16),
+ /// Another of the same gap's stand-ins. Drop it once the fallback is
+ /// drawn; keeping it draws the cell twice and advances the column
+ /// twice.
+ GapTail(u16),
+}
+
+/// `shape` plus what it could not do.
+#[derive(Debug, Clone)]
+pub struct WebShaped {
+ pub infos: Vec<GlyphInfo>,
+ /// Invariant: each of these has exactly one `GapLead` in `roles`.
+ pub gaps: Vec<Gap>,
+ /// Exactly `infos.len()` entries.
+ pub roles: Vec<GlyphRole>,
+}
+
+/// The gap whose byte range holds `at`, if any.
+///
+/// Gaps come from one forward walk over the graphemes, so they are in
+/// ascending order and do not overlap: the only candidate is the last one
+/// that starts at or before `at`.
+fn gap_at(gaps: &[Gap], at: usize) -> Option<usize> {
+ let g = match gaps.binary_search_by(|gap| gap.bytes.start.cmp(&at)) {
+ Ok(g) => g,
+ Err(0) => return None,
+ Err(pos) => pos - 1,
+ };
+ (at < gaps[g].bytes.end).then_some(g)
+}
+
+impl WebShaped {
+ fn new(infos: Vec<GlyphInfo>, gaps: Vec<Gap>) -> Self {
+ // A gap no glyph stands in for has no column to take over, so
+ // drawing a fallback for it would add one. `shape_web` folds
+ // graphemes onto shaped clusters before it gets here, so this is a
+ // backstop rather than the mechanism it once was -- what remains is
+ // a grapheme that produced no glyphs at all. Dropped here, so
+ // `gaps` and `roles` cannot disagree about how many there are.
+ let mut stood_in = vec![false; gaps.len()];
+ for info in &infos {
+ if let Some(g) = gap_at(&gaps, info.cluster as usize) {
+ stood_in[g] = true;
+ }
+ }
+ let gaps: Vec<Gap> = gaps
+ .into_iter()
+ .zip(&stood_in)
+ .filter(|(_, kept)| **kept)
+ .map(|(gap, _)| gap)
+ .collect();
+
+ debug_assert!(gaps.len() <= u16::MAX as usize, "more gaps than a role can index");
+ let mut roles = vec![GlyphRole::Own; infos.len()];
+ let mut has_lead = vec![false; gaps.len()];
+ for (i, info) in infos.iter().enumerate() {
+ if let Some(g) = gap_at(&gaps, info.cluster as usize) {
+ roles[i] = if std::mem::replace(&mut has_lead[g], true) {
+ GlyphRole::GapTail(g as u16)
+ } else {
+ GlyphRole::GapLead(g as u16)
+ };
+ }
+ }
+ Self { infos, gaps, roles }
+ }
+
+ /// The glyphs that stand in for a gap: every one whose cluster falls in
+ /// the gap's byte range.
+ ///
+ /// These are **not** all `.notdef`. rustybuzz maps a ZWJ to the face's
+ /// space glyph rather than deleting it, so a joined sequence puts a
+ /// glyph with a real id in here, and a base the face does cover puts
+ /// its own glyph in here too. Narrowing the removal to `glyph_pos == 0`
+ /// would leave those to be drawn on top of the fallback.
+ ///
+ /// Production code reads `roles`, which says the same thing without
+ /// scanning or allocating; this is the contract in readable form, and
+ /// the tests hold the two to each other.
+ pub fn glyphs_for(&self, gap: &Gap) -> Vec<usize> {
+ self.infos
+ .iter()
+ .enumerate()
+ .filter(|(_, info)| gap.bytes.contains(&(info.cluster as usize)))
+ .map(|(i, _)| i)
+ .collect()
+ }
+}
+
+impl FontSet {
+ /// Shape, and report the graphemes no bundled face can draw.
+ ///
+ /// The `FontShaper` trait reports missing characters as a flat
+ /// `Vec<char>`, which the desktop uses to log them. That is not enough
+ /// to draw a replacement: the grapheme's boundaries, its place in the
+ /// text and its column count are all gone by then. This is the same
+ /// walk, keeping them.
+ ///
+ /// `keeps_notdef` names the graphemes the caller draws itself and would
+ /// rather have the box for than a substitute. The browser passes
+ /// Braille that way -- it draws the dots -- and the filter is here
+ /// rather than applied to the result because a screenful of them would
+ /// otherwise allocate a `String` per cell. The desktop filters at the
+ /// same point, before it goes looking for a fallback font.
+ #[allow(clippy::too_many_arguments)]
+ pub fn shape_web(
&self,
text: &str,
size: f64,
dpi: u32,
- no_glyphs: &mut Vec<char>,
presentation: Option<Presentation>,
direction: Direction,
range: Option<Range<usize>>,
presentation_width: Option<&PresentationWidth>,
- ) -> Result<Vec<GlyphInfo>> {
+ keeps_notdef: &dyn Fn(&str) -> bool,
+ ) -> Result<WebShaped> {
let range = range.unwrap_or(0..text.len());
let px = Face::pixels(size, dpi);
let mut runs: Vec<(usize, Range<usize>)> = Vec::new();
+ // Every grapheme and the face that took it. Which of them are gaps
+ // cannot be decided yet: a face can take a cluster and still fail
+ // to draw it.
+ let mut candidates: Vec<(Range<usize>, Option<usize>)> = Vec::new();
let mut byte = range.start;
for grapheme in Graphemes::new(&text[range.clone()]) {
let end = byte + grapheme.len();
- let face = match self.face_for(grapheme, presentation) {
- Some(face) => face,
- None => {
- no_glyphs.extend(grapheme.chars().filter(|c| !is_joiner_or_selector(*c)));
- 0
- }
- };
+ let taken = self.face_for(grapheme, presentation);
+ candidates.push((byte..end, taken));
+ // A grapheme no face covers is still shaped with the base face:
+ // its `.notdef` holds the run's positions together, and `roles`
+ // says which glyphs a fallback replaces.
+ let face = taken.unwrap_or(0);
match runs.last_mut() {
Some((f, r)) if *f == face && r.end == byte => r.end = end,
_ => runs.push((face, byte..end)),
}
byte = end;
}
- let mut out = Vec::new();
+ let mut infos = Vec::new();
for (face, run) in runs {
- out.extend(self.shape_run(face, text, run, px, direction, presentation_width));
+ infos.extend(self.shape_run(face, text, run, px, direction, presentation_width));
+ }
+
+ // rustybuzz decides cluster boundaries by its own rules, and they
+ // are coarser than UAX#29 in several scripts: Burmese, Tai Tham and
+ // Cham all put a base and its vowel signs in one cluster where the
+ // walk above sees two or three graphemes.
+ //
+ // A grapheme inside a merged cluster has no glyph that starts at it.
+ // Judged on its own it would be measured against its neighbour's
+ // glyphs, and then dropped for having no stand-in -- taking its own
+ // glyphs with it, as `GapTail`s of the gap before it. The character
+ // would not fall back to a box; it would disappear. So fold them:
+ // one shaped cluster is one unit, whole.
+ let starts: HashSet<usize> = infos.iter().map(|g| g.cluster as usize).collect();
+ let mut units: Vec<(Range<usize>, Option<usize>, bool)> =
+ Vec::with_capacity(candidates.len());
+ for (bytes, taken) in candidates {
+ match units.last_mut() {
+ Some((range, face, folded)) if !starts.contains(&bytes.start) => {
+ range.end = bytes.end;
+ *folded = true;
+ // One uncovered part makes the whole cluster the
+ // platform's problem: they share glyphs, so there is no
+ // drawing one without the other.
+ if taken.is_none() {
+ *face = None;
+ }
+ }
+ _ => units.push((bytes, taken, false)),
+ }
}
- Ok(out)
+ // Every unit now begins at a cluster boundary and ends where the
+ // next one begins, so the glyphs whose cluster falls in its range
+ // are exactly its own -- which is what `judge` and `WebShaped::new`
+ // both rely on.
+
+ // Walk the glyphs in logical byte order alongside the units. A
+ // missing glyph must not make every covered unit scan the entire
+ // line. Sort references, leaving the drawing order in `infos`
+ // intact: RTL runs can have descending cluster offsets, and a
+ // change of face can start another run in either direction.
+ let mut by_cluster: Vec<_> = infos.iter().collect();
+ by_cluster.sort_unstable_by_key(|g| g.cluster);
+ let mut remaining = by_cluster.as_slice();
+ let mut gaps = Vec::new();
+ for (bytes, taken, folded) in &units {
+ let count = remaining
+ .iter()
+ .take_while(|g| (g.cluster as usize) < bytes.end)
+ .count();
+ let (mine, rest) = remaining.split_at(count);
+ remaining = rest;
+ let grapheme = &text[bytes.clone()];
+ if keeps_notdef(grapheme) {
+ continue;
+ }
+ let reason = match taken {
+ // Nobody covered it, so it was shaped with the base face
+ // and those glyphs are that face's `.notdef`. Not looking
+ // at them is what keeps a screenful of CJK linear.
+ None => Some(GapReason::Uncovered),
+ Some(idx) => {
+ self.judge(*idx, grapheme, mine, presentation, *folded)
+ }
+ };
+ let Some(reason) = reason else { continue };
+ gaps.push(Gap {
+ text: grapheme.to_string(),
+ bytes: bytes.clone(),
+ // The row's own width data. Never reconstructed from the
+ // shaped glyphs: `shape_run` splits a cluster's columns
+ // between them in proportion to their advances, and gives
+ // each one column when the advances are all zero, so their
+ // sum is not this number.
+ cells: presentation_width
+ .map(|pw| pw.num_cells(bytes.clone()))
+ .unwrap_or_else(|| unicode_column_width(grapheme, None) as u8),
+ presentation,
+ reason,
+ });
+ }
+ Ok(WebShaped::new(infos, gaps))
+ }
+
+ /// Why a grapheme a face *did* take still needs the platform's fonts,
+ /// if it does.
+ fn judge(
+ &self,
+ face_idx: usize,
+ grapheme: &str,
+ infos: &[&GlyphInfo],
+ presentation: Option<Presentation>,
+ folded: bool,
+ ) -> Option<GapReason> {
+ let joined = grapheme.contains('\u{200d}');
+ // The slice contains every cluster in this unit and no neighbour's
+ // glyphs, including when one Hangul grapheme spans several clusters.
+ if infos.iter().any(|g| g.glyph_pos == 0) {
+ return Some(GapReason::Notdef);
+ }
+ // The terminal asked for emoji and the face that answered has no
+ // colour tables. Bundling no emoji face at all is the usual case,
+ // so this is the rule that puts emoji on the screen.
+ if presentation == Some(Presentation::Emoji)
+ && self.faces[face_idx].presentation != Presentation::Emoji
+ {
+ return Some(GapReason::NoColor);
+ }
+ // A joined sequence the face has the pieces of but cannot join
+ // comes out as more than one drawn glyph.
+ //
+ // Counted by advance rather than by "is not .notdef": rustybuzz
+ // does not delete default ignorables, it maps them to the face's
+ // space glyph and zeroes the advance, so a ZWJ is a glyph with a
+ // real id sitting in this range. A variation selector never
+ // reaches here -- what it asks is a question about presentation,
+ // which the rule above already answered.
+ // Not asked of a folded unit. That already spans more than one
+ // grapheme, so several drawn glyphs is what it looks like when
+ // everything is fine, and the count would condemn it on sight.
+ if joined && !folded && infos.iter().filter(|g| g.x_advance.get() != 0.0).count() != 1 {
+ return Some(GapReason::Unjoined);
+ }
+ None
+ }
+}
+
+impl FontShaper for FontSet {
+ #[allow(clippy::too_many_arguments)]
+ fn shape(
+ &self,
+ text: &str,
+ size: f64,
+ dpi: u32,
+ no_glyphs: &mut Vec<char>,
+ presentation: Option<Presentation>,
+ direction: Direction,
+ range: Option<Range<usize>>,
+ presentation_width: Option<&PresentationWidth>,
+ ) -> Result<Vec<GlyphInfo>> {
+ // Nothing is drawn by this caller, so no grapheme keeps its box.
+ let shaped = self.shape_web(
+ text,
+ size,
+ dpi,
+ presentation,
+ direction,
+ range,
+ presentation_width,
+ &|_| false,
+ )?;
+ // The trait reports characters, not graphemes, and drops the ones
+ // that only join others. It now also reports graphemes a face took
+ // and could not draw, which it could not see before; the desktop
+ // uses this only to log, and has its own shaper besides.
+ no_glyphs.extend(
+ shaped
+ .gaps
+ .iter()
+ .flat_map(|gap| gap.text.chars())
+ .filter(|c| !is_joiner_or_selector(*c)),
+ );
+ Ok(shaped.infos)
}
fn metrics(&self, size: f64, dpi: u32) -> Result<FontMetrics> {
@@ -435,6 +799,318 @@ mod tests {
FontSet::new(vec![Face::new("JetBrains Mono", data, 0).unwrap()]).unwrap()
}
+ fn shape(set: &FontSet, text: &str, presentation: Option<Presentation>) -> WebShaped {
+ set.shape_web(
+ text,
+ 12.0,
+ 96,
+ presentation,
+ Direction::LeftToRight,
+ None,
+ None,
+ &|_| false,
+ )
+ .unwrap()
+ }
+
+ /// One glyph, placed by hand, for the tests that exercise `WebShaped`
+ /// rather than the shaper.
+ fn glyph_at(cluster: u32) -> GlyphInfo {
+ GlyphInfo::new(
+ "x",
+ GlyphInfoParts {
+ only_char: Some('x'),
+ is_space: false,
+ num_cells: 1,
+ cluster,
+ font_idx: 0,
+ glyph_pos: 1,
+ x_advance: PixelLength::new(1.0),
+ y_advance: PixelLength::new(0.0),
+ x_offset: PixelLength::new(0.0),
+ y_offset: PixelLength::new(0.0),
+ },
+ )
+ }
+
+ fn gap_over(bytes: Range<usize>) -> Gap {
+ Gap {
+ text: "?".to_string(),
+ bytes,
+ cells: 1,
+ presentation: None,
+ reason: GapReason::Uncovered,
+ }
+ }
+
+ /// The bundled faces cover no CJK, so these are the real thing rather
+ /// than a contrived gap.
+ #[test]
+ fn a_grapheme_no_face_covers_comes_back_whole() {
+ let set = jetbrains();
+ let shaped = shape(&set, "a\u{4e2d}b", None);
+ assert_eq!(shaped.gaps.len(), 1, "{:?}", shaped.gaps);
+ let gap = &shaped.gaps[0];
+ assert_eq!(gap.text, "\u{4e2d}");
+ assert_eq!(gap.bytes, 1..4, "the byte range of the grapheme itself");
+ assert_eq!(gap.cells, 2, "a wide character keeps its two columns");
+ assert_eq!(gap.reason, GapReason::Uncovered);
+ }
+
+ #[test]
+ fn a_zwj_sequence_is_one_gap_and_is_not_split() {
+ let set = jetbrains();
+ let family = "\u{1f468}\u{200d}\u{1f4bb}"; // man + ZWJ + laptop
+ let shaped = shape(&set, family, None);
+ assert_eq!(shaped.gaps.len(), 1, "the sequence must not be torn apart");
+ assert_eq!(shaped.gaps[0].text, family);
+ assert_eq!(shaped.gaps[0].bytes, 0..family.len());
+ }
+
+ #[test]
+ fn a_combining_mark_stays_with_its_base() {
+ let set = jetbrains();
+ // A base no face covers plus a mark: still one grapheme, so the
+ // fallback gets both or neither. The base has to be one the face
+ // cannot draw -- with a covered base there is no gap at all, and
+ // the assertion would run zero times.
+ let text = "\u{4e2d}\u{0301}";
+ assert_eq!(Graphemes::new(text).count(), 1, "premise: one grapheme");
+ let shaped = shape(&set, text, None);
+ assert_eq!(shaped.gaps.len(), 1, "{:?}", shaped.gaps);
+ assert_eq!(shaped.gaps[0].text, text, "a mark was separated from its base");
+ }
+
+ #[test]
+ fn the_glyphs_a_fallback_replaces_are_exactly_the_gap_s_own() {
+ let set = jetbrains();
+ let shaped = shape(&set, "ab\u{4e2d}de", None);
+ let gap = &shaped.gaps[0];
+ let idx = shaped.glyphs_for(gap);
+ assert_eq!(idx.len(), 1, "one .notdef stands in for one gap: {idx:?}");
+ assert_eq!(shaped.infos[idx[0]].glyph_pos, 0, "notdef from the base face");
+ // And nothing belonging to the covered text is caught up in it.
+ for (i, info) in shaped.infos.iter().enumerate() {
+ if !idx.contains(&i) {
+ assert!(!gap.bytes.contains(&(info.cluster as usize)));
+ }
+ }
+ }
+
+ #[test]
+ fn a_face_with_the_pieces_but_not_the_join_is_still_a_gap() {
+ let set = jetbrains();
+ let face = set.face(0).unwrap();
+ // The premise, asserted rather than assumed: the face has both
+ // pieces and not the joiner, which is exactly the hole `covers`
+ // cannot see.
+ assert!(face.has('\u{26a0}') && face.has('\u{2194}'), "premise: pieces present");
+ assert!(!face.has('\u{200d}'), "premise: no ZWJ glyph");
+ let text = "\u{26a0}\u{200d}\u{2194}";
+ assert_eq!(Graphemes::new(text).count(), 1, "premise: one grapheme");
+ let shaped = shape(&set, text, None);
+ assert_eq!(shaped.gaps.len(), 1, "{:?}", shaped.gaps);
+ assert_eq!(shaped.gaps[0].reason, GapReason::Unjoined);
+ }
+
+ #[test]
+ fn an_emoji_cluster_a_text_face_took_is_a_gap() {
+ let set = jetbrains();
+ assert!(set.face(0).unwrap().has('\u{26a1}'), "premise: the face has it");
+ let asked = shape(&set, "\u{26a1}", Some(Presentation::Emoji));
+ assert_eq!(asked.gaps.len(), 1, "{:?}", asked.gaps);
+ assert_eq!(asked.gaps[0].reason, GapReason::NoColor);
+ // The same character as text is drawn by the face, as it always was.
+ assert!(shape(&set, "\u{26a1}", Some(Presentation::Text)).gaps.is_empty());
+ }
+
+ #[test]
+ fn a_variation_selector_the_face_honours_is_not_a_gap() {
+ // U+26A0 U+FE0E asks for the text presentation the face already
+ // draws. Writing the joining rule as a bare glyph count would call
+ // this a gap: rustybuzz keeps the selector as a zero-advance space
+ // glyph, so the cluster has two glyphs and one of them is drawn.
+ let set = jetbrains();
+ let text = "\u{26a0}\u{fe0e}";
+ assert_eq!(Graphemes::new(text).count(), 1, "premise: one grapheme");
+ assert!(shape(&set, text, Some(Presentation::Text)).gaps.is_empty());
+ }
+
+ #[test]
+ fn an_uncovered_grapheme_is_uncovered_whatever_else_is_wrong_with_it() {
+ // Asked for as emoji, and no face has it at all. The reason has to
+ // be the one that says "nobody could draw this", not the one about
+ // colour: the caller keys its cache on the grapheme either way, but
+ // the reason is what a person reads when this goes wrong.
+ let set = jetbrains();
+ let shaped = shape(&set, "\u{4e2d}", Some(Presentation::Emoji));
+ assert_eq!(shaped.gaps.len(), 1);
+ assert_eq!(shaped.gaps[0].reason, GapReason::Uncovered);
+ }
+
+ #[test]
+ fn a_zero_width_non_joiner_does_not_send_a_covered_letter_to_the_canvas() {
+ // U+200C is in no bundled face. Before it was treated as a joiner,
+ // the whole grapheme was reported missing even though the face
+ // draws the `a` perfectly well.
+ let set = jetbrains();
+ assert!(!set.face(0).unwrap().has('\u{200c}'), "premise: no ZWNJ glyph");
+ assert!(shape(&set, "a\u{200c}", None).gaps.is_empty());
+ }
+
+ #[test]
+ fn roles_and_glyphs_for_agree() {
+ let set = jetbrains();
+ let shaped = shape(&set, "a\u{4e2d}b\u{d55c}c", None);
+ assert_eq!(shaped.roles.len(), shaped.infos.len());
+ assert!(shaped.gaps.len() >= 2, "{:?}", shaped.gaps);
+ for (g, gap) in shaped.gaps.iter().enumerate() {
+ let by_role: Vec<usize> = shaped
+ .roles
+ .iter()
+ .enumerate()
+ .filter(|(_, role)| {
+ matches!(role, GlyphRole::GapLead(n) | GlyphRole::GapTail(n) if *n as usize == g)
+ })
+ .map(|(i, _)| i)
+ .collect();
+ assert_eq!(by_role, shaped.glyphs_for(gap), "gap {g}");
+ }
+ }
+
+ #[test]
+ fn every_gap_has_exactly_one_lead() {
+ let set = jetbrains();
+ let shaped = shape(&set, "\u{4e2d}\u{6587}\u{d55c}", None);
+ assert!(!shaped.gaps.is_empty());
+ for g in 0..shaped.gaps.len() {
+ let leads = shaped
+ .roles
+ .iter()
+ .filter(|role| matches!(role, GlyphRole::GapLead(n) if *n as usize == g))
+ .count();
+ assert_eq!(leads, 1, "gap {g} has {leads} leads");
+ }
+ }
+
+ #[test]
+ fn a_gap_with_no_stand_in_is_dropped() {
+ // No glyph falls in the gap's byte range, so there is no column for
+ // a fallback to take over: drawing one would add a column. The gap
+ // has to be gone, not merely unreachable, or `gaps` and `roles`
+ // disagree about how many fallbacks there are.
+ let orphan = WebShaped::new(vec![], vec![gap_over(0..3)]);
+ assert!(orphan.gaps.is_empty());
+ let beside = WebShaped::new(vec![glyph_at(9)], vec![gap_over(0..3)]);
+ assert!(beside.gaps.is_empty());
+ assert_eq!(beside.roles, vec![GlyphRole::Own]);
+ // And one that does have a stand-in survives, so the test above is
+ // not passing for want of ever keeping anything.
+ let kept = WebShaped::new(vec![glyph_at(1)], vec![gap_over(0..3)]);
+ assert_eq!(kept.gaps.len(), 1);
+ assert_eq!(kept.roles, vec![GlyphRole::GapLead(0)]);
+ }
+
+ #[test]
+ fn nothing_a_face_cannot_draw_is_lost() {
+ // rustybuzz clusters a Burmese base with its vowel signs; UAX#29
+ // makes them separate graphemes. Before the two were folded onto
+ // cluster boundaries, the vowel signs were dropped for having no
+ // stand-in glyph and took their own glyphs with them -- they did not
+ // fall back to a box, they vanished.
+ let set = jetbrains();
+ for text in [
+ "\u{1005}\u{102c}\u{1038}", // Burmese, one HB cluster
+ "a\u{200d}\u{1f600}", // letter, ZWJ, emoji
+ "\u{4e2d}\u{6587}", // two clusters of their own
+ "\u{d55c}\u{ad6d}", // Hangul: several clusters each
+ ] {
+ let shaped = shape(&set, text, None);
+ let mut accounted = 0;
+ for gap in &shaped.gaps {
+ assert_eq!(gap.text, &text[gap.bytes.clone()], "{text:?}: {gap:?}");
+ accounted += gap.bytes.len();
+ }
+ assert_eq!(
+ accounted,
+ text.len(),
+ "{text:?}: {accounted} of {} bytes are in a gap; the rest would be drawn by \
+ nothing at all",
+ text.len()
+ );
+ }
+ }
+
+ #[test]
+ fn every_gap_owns_at_least_one_glyph_and_they_do_not_overlap() {
+ let set = jetbrains();
+ for text in ["a\u{4e2d}b\u{d55c}c", "\u{1005}\u{102c}\u{1038}", "e\u{301}\u{4e2d}\u{301}"] {
+ let shaped = shape(&set, text, None);
+ let mut end = 0;
+ for gap in &shaped.gaps {
+ assert!(gap.bytes.start >= end, "{text:?}: gaps overlap or run backwards");
+ end = gap.bytes.end;
+ assert!(!shaped.glyphs_for(gap).is_empty(), "{text:?}: {gap:?} has no stand-in");
+ }
+ }
+ }
+
+ #[test]
+ fn a_letter_the_face_draws_is_not_condemned_by_its_neighbour() {
+ // The `.notdef` belongs to the CJK character. Sweeping it into the
+ // letter's own judgement sent a perfectly drawable `a` to the canvas.
+ let set = jetbrains();
+ let shaped = shape(&set, "a\u{4e2d}", None);
+ assert_eq!(shaped.gaps.len(), 1, "{:?}", shaped.gaps);
+ assert_eq!(shaped.gaps[0].text, "\u{4e2d}");
+ }
+
+ #[test]
+ fn mixed_run_gap_judgments_preserve_direction_and_subrange() {
+ let set = jetbrains();
+ let joined = "\u{26a0}\u{200d}\u{2194}";
+ let middle = format!("{}中{}{}", "x".repeat(512), joined, "y".repeat(512));
+ // The missing characters outside the requested range must not be
+ // included, and range-relative offsets must not pick up neighbours.
+ let text = format!("韓{middle}文");
+ let start = "韓".len();
+ let end = start + middle.len();
+ for direction in [Direction::LeftToRight, Direction::RightToLeft] {
+ let shaped = set
+ .shape_web(
+ &text, 12.0, 96, None, direction, Some(start..end), None, &|_| false,
+ )
+ .unwrap();
+ assert_eq!(shaped.gaps.len(), 2, "{direction:?}: {:?}", shaped.gaps);
+ assert_eq!(shaped.gaps[0].text, "中");
+ assert_eq!(shaped.gaps[0].reason, GapReason::Uncovered);
+ assert_eq!(shaped.gaps[0].bytes, start + 512..start + 515);
+ assert_eq!(shaped.gaps[1].text, joined);
+ assert_eq!(shaped.gaps[1].reason, GapReason::Unjoined);
+ // Grouping for judgment must not reorder the actual glyphs.
+ assert!(shaped.infos.windows(2).all(|pair| match direction {
+ Direction::LeftToRight => pair[0].cluster <= pair[1].cluster,
+ Direction::RightToLeft => pair[0].cluster >= pair[1].cluster,
+ }));
+ }
+ }
+
+ #[test]
+ fn the_second_stand_in_of_a_gap_is_a_tail() {
+ let both = WebShaped::new(vec![glyph_at(0), glyph_at(2)], vec![gap_over(0..3)]);
+ assert_eq!(both.roles, vec![GlyphRole::GapLead(0), GlyphRole::GapTail(0)]);
+ }
+
+ #[test]
+ fn the_trait_still_reports_the_characters_it_always_did() {
+ let set = jetbrains();
+ let mut missing = vec![];
+ set.shape("a中\u{1f468}\u{200d}\u{1f4bb}", 12.0, 96, &mut missing, None, Direction::LeftToRight, None, None)
+ .unwrap();
+ // Joiners are still dropped, and the covered "a" is not reported.
+ assert_eq!(missing, vec!['中', '\u{1f468}', '\u{1f4bb}'], "{missing:?}");
+ }
+
#[test]
fn metrics_describe_a_monospace_cell() {
let set = jetbrains();