DesignCraft: An open-source page and layout design editor written by scratch in pure Rust
Import: RTF and Tagged Text keep characters past U+FFFF
Jan Hallvard Larsenc63ca54227d7
diff --git a/crates/textimport/src/lib.rs b/crates/textimport/src/lib.rsindex 8dc50ea7e..22f79c513 100644--- a/crates/textimport/src/lib.rs+++ b/crates/textimport/src/lib.rs@@ -28,6 +28,33 @@ pub fn xlsx_records(bytes: &[u8], sheet: Option<&str>) -> Result<Vec<Vec<String>use designcraft_doc::{CharAttrs, ParaAttrs, Story};+/// Joins UTF-16 code units that arrive one at a time (RTF `\uN`, Tagged Text `<0xNNNN>`) into+/// characters. A character past U+FFFF comes as two units, a surrogate pair: the first half+/// waits for the second.+#[derive(Default)]+pub(crate) struct Utf16Units {+ high: Option<u32>,+}++impl Utf16Units {+ /// The character that `unit` is or completes. Half a pair without its partner is no character.+ pub(crate) fn push(&mut self, unit: u32) -> Option<char> {+ match (self.high.take(), unit) {+ (Some(high), 0xDC00..=0xDFFF) => char::from_u32(0x10000 + ((high - 0xD800) << 10) + (unit - 0xDC00)),+ (_, 0xD800..=0xDBFF) => {+ self.high = Some(unit);+ None+ }+ _ => char::from_u32(unit),+ }+ }++ /// Other text came in between: a waiting first half has no partner.+ pub(crate) fn reset(&mut self) {+ self.high = None;+ }+}+/// A paragraph or character style found in the source.#[derive(Clone, Debug, Default, PartialEq)]pub struct ImportedStyle {diff --git a/crates/textimport/src/rtf.rs b/crates/textimport/src/rtf.rsindex c02e5e551..9189b52d2 100644--- a/crates/textimport/src/rtf.rs+++ b/crates/textimport/src/rtf.rs@@ -70,6 +70,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {let mut st = Story::new(StoryId(0));let mut stack: Vec<State> = vec![State { uc: 1, ..Default::default() }];let mut pending = 0usize; // characters still to skip after \uN+ let mut units = crate::Utf16Units::default();let mut buf = String::new();let mut fmt = stack[0].format();let mut warnings = Vec::new();@@ -124,6 +125,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {if let Ok(b) = u8::from_str_radix(&hex, 16)&& !stack.last().is_some_and(|s| s.skip){+ units.reset();buf.push(cp1252(b));}continue;@@ -225,8 +227,8 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {}"u" => {if let Some(n) = param {- let cp = if n < 0 { (n + 65536) as u32 } else { n as u32 };- if let Some(ch) = char::from_u32(cp) {+ let unit = if n < 0 { (n + 65536) as u32 } else { n as u32 };+ if let Some(ch) = units.push(unit) {buf.push(ch);}pending = stack.last().map_or(1, |s| s.uc);@@ -243,6 +245,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {continue;}if !stack.last().is_some_and(|s| s.skip) {+ units.reset();buf.push(c);}}@@ -287,4 +290,19 @@ Caf\'e9 \u8364? price\tab tab\line next\parassert_eq!(st.format_after(0).over.size, Some(12.0));assert!(!st.text.contains("Times"), "font table skipped");}++ /// Characters past U+FFFF are written as two `\uN` words, a UTF-16 surrogate pair (the form+ /// our own exporter and Word write). Each half was converted alone and dropped.+ #[test]+ fn surrogate_pairs_become_one_character() {+ let i = import(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}").unwrap();+ assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}");+ // What the exporter writes comes back.+ let mut st = Story::new(StoryId(1));+ st.insert(0, "x \u{1F600} \u{20BB7}\u{7530} y");+ let doc = designcraft_doc::Document::new(&designcraft_doc::build::NewDocument::default());+ assert_eq!(import(crate::export::rtf(&doc, &st).as_bytes()).unwrap().story.text, st.text);+ // Half a pair is not a character: it is still left out.+ assert_eq!(import(br"{\rtf1 A\u-10179?B\u-8704?C}").unwrap().story.text, "ABC");+ }}diff --git a/crates/textimport/src/tagged.rs b/crates/textimport/src/tagged.rsindex ca78bc4b1..cfd08eea7 100644--- a/crates/textimport/src/tagged.rs+++ b/crates/textimport/src/tagged.rs@@ -110,12 +110,14 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {_ => runs.push((c.to_string(), fmt.clone())),}};+ let mut units = crate::Utf16Units::default();let mut i = 0;let mut skipped = 0;while i < chars.len() {let c = chars[i];match c {'\\' if i + 1 < chars.len() => {+ units.reset();push(&mut paras, chars[i + 1], &fmt);i += 2;}@@ -143,7 +145,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {match name {n if n.starts_with("0x") => {if let Ok(u) = u32::from_str_radix(&n[2..], 16)- && let Some(ch) = char::from_u32(u)+ && let Some(ch) = units.push(u){push(&mut paras, ch, &fmt);}@@ -193,6 +195,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {i += 1;}c => {+ units.reset();push(&mut paras, c, &fmt);i += 1;}@@ -226,6 +229,20 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {mod tests {use super::*;+ /// Characters past U+FFFF are written as two tags, a UTF-16 surrogate pair. Each half was+ /// converted alone and dropped.+ #[test]+ fn surrogate_pairs_become_one_character() {+ let i = import("<ASCII-WIN>\r\n<Version:7>\r\n<ParaStyle:Body>A<0xD83D><0xDE00>B <0xD842><0xDFB7><0x7530>".as_bytes()).unwrap();+ assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}");+ // What the exporter writes comes back.+ let doc = Document::new(&designcraft_doc::build::NewDocument::default());+ let back = import(tagged_text(&doc, &i.story).as_bytes()).unwrap();+ assert_eq!(back.story.text, i.story.text);+ // Half a pair is not a character: it is still left out.+ assert_eq!(import("<ASCII-WIN>\r\n<ParaStyle:Body>A<0xD83D>B<0xDE00>C".as_bytes()).unwrap().story.text, "ABC");+ }+#[test]fn tagged_text_round_trips() {let src = "<ASCII-WIN>\r\n<Version:7>\r\n<ParaStyle:Head>Caf<0x00E9> \\<1\\>\r\n<ParaStyle:Body>A <CharStyle:Em>word<CharStyle:> and <cSize:18.000000>big<cSize:> text";