DesignCraft: An open-source page and layout design editor written by scratch in pure Rust
CJK 1.1: the character language reaches the shaper
aiongg2a516761ef17
diff --git a/ROADMAP.md b/ROADMAP.mdindex 805a7966c..6a9a7a9a9 100644--- a/ROADMAP.md+++ b/ROADMAP.md@@ -87,7 +87,7 @@ DesignCraft aims at full Adobe InDesign parity — and to be better: faster, ope- Edit ▸ Menus (`window.hideMenuItem`): hide menu items; menus with hidden items end in Show All Menu Items.- View ▸ Separations Preview (`view.separations`): one process plate as ink density, or areas over an ink limit in red (a GCR preview of the rendered view; spot plates and the flattener preview aren't there).- Separations Preview plates now come from each object's own colour (CMYK as authored; RGB and images through the colour settings, images via a cached 17³ table), so pure black shows only on the black plate.-- Language (Character panel ▸ Language, `type.char {language}`): text in other languages isn't hyphenated or spell-checked with the English dictionaries, and typographer's quotes follow the language (German „…“, French and Russian «…», Swiss, Spanish, Dutch, Nordic, CJK 「…」). Hyphenation dictionaries for other languages aren't bundled yet.+- Language (Character panel ▸ Language, `type.char {language}`): text in other languages isn't hyphenated or spell-checked with the English dictionaries, typographer's quotes follow the language (German „…“, French and Russian «…», Swiss, Spanish, Dutch, Nordic, Japanese and Traditional Chinese 「…」, Simplified Chinese “…”), and the font's localized forms (OpenType `locl`: Turkish i, Bulgarian Cyrillic, Japanese / Chinese / Korean glyph variants) follow it too. Hyphenation dictionaries for other languages aren't bundled yet.- Books (`book.new/open/add/remove/list/styleSource/paginate/syncStyles/exportPdf`, Book panel): `.dcbook` files, continuous page numbering, style sync from a style source, one merged PDF.- Track Changes (`changes.track/list/acceptAll/rejectAll`, Type ▸ Track Changes): typing is marked added (highlighted on screen), deletions are kept but hidden from layout and exports until accepted or rejected. Per-change accept/reject and authors aren't there yet.- Track Changes: per-change Accept / Reject (`changes.accept`, `changes.reject`) and a Track Changes panel listing every change.diff --git a/crates/compose/src/shape.rs b/crates/compose/src/shape.rsindex c39fb9861..6188f421b 100644--- a/crates/compose/src/shape.rs+++ b/crates/compose/src/shape.rs@@ -614,10 +614,10 @@ fn shape_run_raw(}type WordMap = HashMap<Box<str>, Arc<[ShapedGlyph]>>;-type WordKey = (u32, bool, String);+type WordKey = (u32, bool, String, Option<&'static str>);-/// Shaped words (font units) by (face, caps, features), then text. Lookups take read locks and-/// misses are shaped outside any lock, so parallel composition doesn't serialize here.+/// Shaped words (font units) by (face, caps, features, language), then text. Lookups take read+/// locks and misses are shaped outside any lock, so parallel composition doesn't serialize here.static WORD_CACHE: RwLock<Vec<(WordKey, Arc<RwLock<WordMap>>)>> = RwLock::new(Vec::new());const WORD_CACHE_MAX: usize = 50_000;@@ -640,12 +640,13 @@ fn word_map(key: WordKey) -> Arc<RwLock<WordMap>> {/// Shape `src` word by word through a cache (like a browser's word cache): the text is split/// after each U+0020 so repeated words are shaped once. Shaping does not cross word spaces in the/// scripts we lay out, and clusters stay byte offsets into `src`.-fn shape_cached(face: &FontFace, src: &str, feats: &[Feature], caps: bool, context: designcraft_fonts::ShapeContext<'_>) -> Vec<ShapedGlyph> {+fn shape_cached(face: &FontFace, src: &str, feats: &[Feature], lang: Option<&'static str>, caps: bool, around: (&str, &str)) -> Vec<ShapedGlyph> {let map = |c: char| if caps { c.to_uppercase().next().unwrap_or(c) } else { c };if src.len() < 2 || !src.contains(' ') || src.chars().any(designcraft_fonts::is_rtl) {- return designcraft_fonts::shape_with_context(face, src, feats, map, context);+ let (before, after) = around;+ return designcraft_fonts::shape_with_context(face, src, feats, map, designcraft_fonts::ShapeContext { before, after, language: lang });}- let words = word_map((face.id(), caps, format!("{feats:?}/{}", context.language)));+ let words = word_map((face.id(), caps, format!("{feats:?}"), lang));let pieces: Vec<&str> = src.split_inclusive(' ').collect();let mut found: Vec<Option<Arc<[ShapedGlyph]>>> = {let r = words.read().unwrap_or_else(|e| e.into_inner());@@ -659,7 +660,7 @@ fn shape_cached(face: &FontFace, src: &str, feats: &[Feature], caps: bool, contepieces[k],feats,map,- designcraft_fonts::ShapeContext { language: context.language, ..Default::default() },+ designcraft_fonts::ShapeContext { language: lang, ..Default::default() },).into();fresh.push((pieces[k].into(), g.clone()));@@ -780,12 +781,11 @@ fn shape_segment(feats.extend(["vert", "vrt2"].iter().filter_map(|t| designcraft_fonts::feature(t)));}let space = face.advance(face.glyph_for(' ')) * k * hs;- let context = designcraft_fonts::ShapeContext {- before: if replacement.is_none() { text.get(..range.start).unwrap_or("") } else { "" },- after: if replacement.is_none() { text.get(range.end..).unwrap_or("") } else { "" },- language: &p.language,- };- let shaped: Vec<ShapedGlyph> = shape_cached(face, src, &feats, caps, context);+ // The language picks the font's localized forms (`locl`); the text around the run lets+ // letters join across a change of style.+ let lang = designcraft_doc::language_tag(&p.language);+ let around = if replacement.is_none() { (text.get(..range.start).unwrap_or(""), text.get(range.end..).unwrap_or("")) } else { ("", "") };+ let shaped: Vec<ShapedGlyph> = shape_cached(face, src, &feats, lang, caps, around);let n = shaped.len();let fref = FaceRef::of(face);for (gi, sg) in shaped.iter().enumerate() {diff --git a/crates/compose/src/tests.rs b/crates/compose/src/tests.rsindex a168c1508..e68846b4b 100644--- a/crates/compose/src/tests.rs+++ b/crates/compose/src/tests.rs@@ -1428,3 +1428,43 @@ fn arabic_character_overrides_mirror_brackets_using_final_levels() {}}}++#[test]+fn language_picks_localized_forms_through_the_word_cache() {+ // Source Serif 4 has a Turkish `locl` form of `i`; the same words in English and Turkish+ // (both shaped word by word through the cache) keep their own forms.+ let text = "in in in in";+ let (mut d, sid, _) = doc_with(text, Rect::new(36.0, 36.0, 500.0, 200.0), ParaAttrs::default());+ d.story_mut(sid).unwrap().format_chars(6..text.len(), |f| f.over.language = Some("Turkish".into()));+ let cs = compose_story(&d, sid, &ComposeOptions::default());+ let gid_at = |byte: usize| all_lines(&cs).iter().flat_map(|l| l.glyphs.iter()).find(|g| g.byte == byte).map(|g| g.gid).unwrap();+ assert_eq!(gid_at(0), gid_at(3), "English i");+ assert_eq!(gid_at(6), gid_at(9), "Turkish i");+ assert_ne!(gid_at(0), gid_at(6), "Turkish i takes its localized form");+ // A single word is shaped directly, in its language too.+ let (mut d, sid, _) = doc_with("i", Rect::new(36.0, 36.0, 500.0, 200.0), ParaAttrs::default());+ d.story_mut(sid).unwrap().format_chars(0..1, |f| f.over.language = Some("Turkish".into()));+ let cs = compose_story(&d, sid, &ComposeOptions::default());+ assert_eq!(all_lines(&cs)[0].glyphs[0].gid, gid_at(6), "Turkish i, shaped on its own");+}++#[test]+fn chinese_and_japanese_take_their_localized_forms() {+ // Noto Sans CJK SC (craft-fonts) draws 直 differently for Japanese (`locl`).+ let Some(noto) = designcraft_fonts::CRAFT_FONTS.iter().find(|f| f.family == "Noto Sans CJK SC") else { return };+ FontDb::global().add_font(noto.bytes.to_vec());+ let gid = |language: &str| {+ let (mut d, sid, _) = doc_with("直", Rect::new(36.0, 36.0, 300.0, 100.0), ParaAttrs::default());+ d.story_mut(sid).unwrap().format_chars(0.."直".len(), |f| {+ f.over.font_family = Some(noto.family.into());+ f.over.language = Some(language.into());+ });+ let cs = compose_story(&d, sid, &ComposeOptions::default());+ let g = &all_lines(&cs)[0].glyphs[0];+ assert_eq!(g.face.family, noto.family);+ g.gid+ };+ let chinese = gid("Chinese: Simplified");+ assert_eq!(chinese, FontDb::global().face(noto.family, noto.style).glyph_for('直'));+ assert_ne!(gid("Japanese"), chinese, "Japanese 直");+}diff --git a/crates/doc/src/attrs.rs b/crates/doc/src/attrs.rsindex bd94ae323..c56752a5e 100644--- a/crates/doc/src/attrs.rs+++ b/crates/doc/src/attrs.rs@@ -190,6 +190,65 @@ impl Digits {}}+/// The BCP 47 tag of a language name as InDesign and IDML write it ("English: USA", "Chinese:+/// Simplified"), for the shaper's localized forms (`locl`). A variant we don't know falls back to+/// its language ("English: Australian" → `en`); an unknown language (or "[No Language]") has none.+pub fn language_tag(language: &str) -> Option<&'static str> {+ let l = language.trim().to_ascii_lowercase();+ let variant = match l.as_str() {+ "english: usa" => Some("en-US"),+ "english: uk" => Some("en-GB"),+ "english: canadian" => Some("en-CA"),+ "german: swiss 2006 reform" | "german: swiss" => Some("de-CH"),+ "french: canadian" => Some("fr-CA"),+ "portuguese: brazilian" => Some("pt-BR"),+ "norwegian: bokmål" | "norwegian: bokmal" => Some("nb"),+ "norwegian: nynorsk" => Some("nn"),+ "chinese: simplified" => Some("zh-Hans"),+ "chinese: traditional" => Some("zh-Hant"),+ _ => None,+ };+ if variant.is_some() {+ return variant;+ }+ Some(match l.split(':').next().unwrap_or("").trim() {+ "english" => "en",+ "german" => "de",+ "french" => "fr",+ "spanish" => "es",+ "catalan" => "ca",+ "italian" => "it",+ "portuguese" => "pt",+ "dutch" => "nl",+ "danish" => "da",+ "swedish" => "sv",+ "norwegian" => "no",+ "finnish" => "fi",+ "polish" => "pl",+ "czech" => "cs",+ "slovak" => "sk",+ "hungarian" => "hu",+ "romanian" => "ro",+ "croatian" => "hr",+ "lithuanian" => "lt",+ "greek" => "el",+ "russian" => "ru",+ "ukrainian" => "uk",+ "bulgarian" => "bg",+ "serbian" => "sr",+ "macedonian" => "mk",+ "turkish" => "tr",+ "japanese" => "ja",+ "korean" => "ko",+ "chinese" => "zh",+ "arabic" => "ar",+ "persian" | "farsi" => "fa",+ "urdu" => "ur",+ "hebrew" => "he",+ _ => return None,+ })+}+/// A tracked change on text.#[derive(Clone, Copy, Debug, Default, PartialEq, Eq, Serialize, Deserialize)]#[serde(rename_all = "camelCase")]@@ -701,6 +760,26 @@ mod tests {assert_eq!(Digits::Hindi.map('x', ""), 'x');}+ #[test]+ fn language_names_map_to_bcp47_tags() {+ assert_eq!(language_tag("English: USA"), Some("en-US"));+ assert_eq!(language_tag("English: Australian"), Some("en"));+ assert_eq!(language_tag("Turkish"), Some("tr"));+ assert_eq!(language_tag("Japanese"), Some("ja"));+ assert_eq!(language_tag("Korean"), Some("ko"));+ assert_eq!(language_tag("Chinese"), Some("zh"));+ assert_eq!(language_tag("Chinese: Simplified"), Some("zh-Hans"));+ assert_eq!(language_tag("Chinese: Traditional"), Some("zh-Hant"));+ assert_eq!(language_tag("Arabic"), Some("ar"));+ assert_eq!(language_tag("Persian"), Some("fa"));+ assert_eq!(language_tag("Farsi"), Some("fa"));+ assert_eq!(language_tag("Urdu"), Some("ur"));+ assert_eq!(language_tag("Hebrew"), Some("he"));+ assert_eq!(language_tag("[No Language]"), None);+ assert_eq!(language_tag("Klingon"), None);+ assert_eq!(language_tag(""), None);+ }+#[test]fn merge_and_resolve() {let mut a = CharAttrs { size: Some(10.0), ..Default::default() };diff --git a/crates/engine/src/cmd/text.rs b/crates/engine/src/cmd/text.rsindex b5204b45e..c1b10b9cf 100644--- a/crates/engine/src/cmd/text.rs+++ b/crates/engine/src/cmd/text.rs@@ -499,6 +499,8 @@ pub fn quote_marks(language: &str) -> [char; 4] {['\u{201E}', '\u{201D}', '\u{201A}', '\u{2019}']} else if l.starts_with("swedish") || l.starts_with("finnish") {['\u{201D}', '\u{201D}', '\u{2019}', '\u{2019}']+ } else if l.starts_with("chinese: simplified") {+ ['\u{201C}', '\u{201D}', '\u{2018}', '\u{2019}']} else if l.starts_with("japanese") || l.starts_with("chinese") {['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']} else if l.starts_with("danish") {@@ -1275,6 +1277,18 @@ mod language_tests {let st = s.execute("spelling.check", &json!({"story": sid})).unwrap();assert!(st.as_array().unwrap().is_empty(), "{st}");}++ #[test]+ fn cjk_languages_have_their_quotes_and_no_english_rules() {+ use super::quote_marks;+ assert_eq!(quote_marks("Chinese: Simplified"), ['\u{201C}', '\u{201D}', '\u{2018}', '\u{2019}']);+ assert_eq!(quote_marks("Chinese: Traditional"), ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']);+ assert_eq!(quote_marks("Chinese"), ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']);+ assert_eq!(quote_marks("Japanese"), ['\u{300C}', '\u{300D}', '\u{300E}', '\u{300F}']);+ for l in ["Japanese", "Korean", "Chinese", "Chinese: Simplified", "Chinese: Traditional"] {+ assert!(!designcraft_compose::is_english(l), "{l}: no English hyphenation or spelling");+ }+ }}#[cfg(test)]diff --git a/crates/fonts/src/lib.rs b/crates/fonts/src/lib.rsindex 72c19ec47..bda698706 100644--- a/crates/fonts/src/lib.rs+++ b/crates/fonts/src/lib.rs@@ -11,7 +11,7 @@ mod fontdb;pub use fontdb::{FALLBACK_FAMILY, FaceRef, FontDb, FontFace, base_style, bundled, system_font_dirs};pub use harfrust::Feature;-use harfrust::{Direction, ShapeOptions, Tag, UnicodeBuffer};+use harfrust::{Direction, Language, ShapeOptions, Tag, UnicodeBuffer};pub use kurbo::BezPath;use skrifa::MetadataProvider;use skrifa::instance::Size;@@ -113,23 +113,14 @@ fn direction_runs(text: &str) -> Vec<(std::ops::Range<usize>, bool, unicode_scriruns}+/// What surrounds a run being shaped: the text before and after it (so Arabic letters join across a+/// change of style) and its language, a BCP 47 tag such as `tr` or `zh-Hant` that picks the font's+/// localized forms (OpenType `locl`; `None` or a tag the shaper can't read: the font's defaults).#[derive(Clone, Copy, Default)]pub struct ShapeContext<'a> {pub before: &'a str,pub after: &'a str,- pub language: &'a str,-}--/// Normalize the supported application language names while also accepting BCP 47 tags.-pub fn language_tag(language: &str) -> &str {- match language.split(':').next().unwrap_or(language).trim() {- "Arabic" => "ar",- "Persian" | "Farsi" => "fa",- "Urdu" => "ur",- "Hebrew" => "he",- "English" => "en",- _ => language,- }+ pub language: Option<&'a str>,}/// Shape `text` with `face`. `chars` lets callers substitute characters (e.g. uppercase for All@@ -140,6 +131,7 @@ pub fn shape(face: &FontFace, text: &str, features: &[Feature], map: impl Fn(chashape_with_context(face, text, features, map, ShapeContext::default())}+/// [`shape`] `text` in its `context`: its language and the text around it.pub fn shape_with_context(face: &FontFace,text: &str,@@ -196,7 +188,7 @@ fn shape_dir(if let Ok(script) = script.short_name().parse() {buf.set_script(script);}- if let Ok(language) = language_tag(context.language).parse() {+ if let Some(language) = context.language.and_then(|t| t.parse::<Language>().ok()) {buf.set_language(language);}buf.set_flags(harfrust::BufferFlags::PRODUCE_SAFE_TO_INSERT_TATWEEL);@@ -254,10 +246,6 @@ mod tests {assert_eq!(runs.len(), 2);assert_eq!(runs[0], (0..5, true, Script::Hebrew));assert_eq!(runs[1], (5..text.len(), true, Script::Arabic));- assert_eq!(language_tag("Arabic"), "ar");- assert_eq!(language_tag("Farsi"), "fa");- assert_eq!(language_tag("Urdu"), "ur");- assert_eq!(language_tag("ar-SA"), "ar-SA");}#[test]@@ -297,6 +285,17 @@ mod tests {assert_eq!(off.len(), 6);}+ #[test]+ fn language_picks_localized_forms() {+ // Source Serif 4's `locl` has a Turkish `i`.+ let face = FontDb::global().face(DEFAULT_FAMILY, "Regular");+ let gid = |language| shape_with_context(&face, "i", &[], |c| c, ShapeContext { language, ..Default::default() })[0].gid;+ assert_eq!(gid(None), face.glyph_for('i'));+ assert_eq!(gid(Some("en-US")), face.glyph_for('i'));+ assert_ne!(gid(Some("tr")), face.glyph_for('i'));+ assert_eq!(gid(Some("not a tag")), face.glyph_for('i'));+ }+#[test]fn mapping_keeps_clusters() {let face = FontDb::global().face(DEFAULT_FAMILY, "Regular");diff --git a/crates/ui-egui/src/panels/properties.rs b/crates/ui-egui/src/panels/properties.rsindex bf3f0b3af..b819bbf83 100644--- a/crates/ui-egui/src/panels/properties.rs+++ b/crates/ui-egui/src/panels/properties.rs@@ -1206,6 +1206,8 @@ const LANGUAGES: &[&str] = &["Turkish","Japanese","Chinese",+ "Chinese: Simplified",+ "Chinese: Traditional","Korean","Arabic","Hebrew",diff --git a/docs/cjk.md b/docs/cjk.mdindex 534b36c90..24d40e0b0 100644--- a/docs/cjk.md+++ b/docs/cjk.md@@ -47,9 +47,8 @@ presets or tables are copied. Our default mojikumi and kinsoku tables are builtKnown bugs to fix first:- Korean breaks between every syllable (it should break at spaces by default).-- Only Arabic, Persian, Urdu, Hebrew and English language names reach the shaper as languages- it knows, so `locl` never applies for the others (Japanese, Chinese, Korean, Turkish…); Chinese- and Korean text get Japanese glyph forms and the Japanese fallback font.+- Fallback is language-blind: Chinese and Korean text without a covering font get the Japanese+ fallback font (and so Japanese glyph forms, whatever their language's `locl` asks for).- Vertical advances are the horizontal ones and upright glyphs turn about a guessed centre(`adv × 0.38`), so proportional and non-square glyphs sit wrong in vertical text.- A justified CJK line has no stretch except Single Word Justification.