Skip to content

DesignCraft: An open-source page and layout design editor written by scratch in pure Rust

Import: RTF and Tagged Text keep characters past U+FFFF

Jan Hallvard Larsenc63ca54227d7
diff --git a/crates/textimport/src/lib.rs b/crates/textimport/src/lib.rs
index 8dc50ea7e..22f79c513 100644
--- a/crates/textimport/src/lib.rs
+++ b/crates/textimport/src/lib.rs
@@ -28,6 +28,33 @@ pub fn xlsx_records(bytes: &[u8], sheet: Option<&str>) -> Result<Vec<Vec<String>
use designcraft_doc::{CharAttrs, ParaAttrs, Story};
+/// Joins UTF-16 code units that arrive one at a time (RTF `\uN`, Tagged Text `<0xNNNN>`) into
+/// characters. A character past U+FFFF comes as two units, a surrogate pair: the first half
+/// waits for the second.
+#[derive(Default)]
+pub(crate) struct Utf16Units {
+ high: Option<u32>,
+}
+
+impl Utf16Units {
+ /// The character that `unit` is or completes. Half a pair without its partner is no character.
+ pub(crate) fn push(&mut self, unit: u32) -> Option<char> {
+ match (self.high.take(), unit) {
+ (Some(high), 0xDC00..=0xDFFF) => char::from_u32(0x10000 + ((high - 0xD800) << 10) + (unit - 0xDC00)),
+ (_, 0xD800..=0xDBFF) => {
+ self.high = Some(unit);
+ None
+ }
+ _ => char::from_u32(unit),
+ }
+ }
+
+ /// Other text came in between: a waiting first half has no partner.
+ pub(crate) fn reset(&mut self) {
+ self.high = None;
+ }
+}
+
/// A paragraph or character style found in the source.
#[derive(Clone, Debug, Default, PartialEq)]
pub struct ImportedStyle {
diff --git a/crates/textimport/src/rtf.rs b/crates/textimport/src/rtf.rs
index c02e5e551..9189b52d2 100644
--- a/crates/textimport/src/rtf.rs
+++ b/crates/textimport/src/rtf.rs
@@ -70,6 +70,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
let mut st = Story::new(StoryId(0));
let mut stack: Vec<State> = vec![State { uc: 1, ..Default::default() }];
let mut pending = 0usize; // characters still to skip after \uN
+ let mut units = crate::Utf16Units::default();
let mut buf = String::new();
let mut fmt = stack[0].format();
let mut warnings = Vec::new();
@@ -124,6 +125,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
if let Ok(b) = u8::from_str_radix(&hex, 16)
&& !stack.last().is_some_and(|s| s.skip)
{
+ units.reset();
buf.push(cp1252(b));
}
continue;
@@ -225,8 +227,8 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
}
"u" => {
if let Some(n) = param {
- let cp = if n < 0 { (n + 65536) as u32 } else { n as u32 };
- if let Some(ch) = char::from_u32(cp) {
+ let unit = if n < 0 { (n + 65536) as u32 } else { n as u32 };
+ if let Some(ch) = units.push(unit) {
buf.push(ch);
}
pending = stack.last().map_or(1, |s| s.uc);
@@ -243,6 +245,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
continue;
}
if !stack.last().is_some_and(|s| s.skip) {
+ units.reset();
buf.push(c);
}
}
@@ -287,4 +290,19 @@ Caf\'e9 \u8364? price\tab tab\line next\par
assert_eq!(st.format_after(0).over.size, Some(12.0));
assert!(!st.text.contains("Times"), "font table skipped");
}
+
+ /// Characters past U+FFFF are written as two `\uN` words, a UTF-16 surrogate pair (the form
+ /// our own exporter and Word write). Each half was converted alone and dropped.
+ #[test]
+ fn surrogate_pairs_become_one_character() {
+ let i = import(br"{\rtf1\ansi\uc1 A\u-10179?\u-8704?B \u-10174?\u-8265?\u30000?}").unwrap();
+ assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}");
+ // What the exporter writes comes back.
+ let mut st = Story::new(StoryId(1));
+ st.insert(0, "x \u{1F600} \u{20BB7}\u{7530} y");
+ let doc = designcraft_doc::Document::new(&designcraft_doc::build::NewDocument::default());
+ assert_eq!(import(crate::export::rtf(&doc, &st).as_bytes()).unwrap().story.text, st.text);
+ // Half a pair is not a character: it is still left out.
+ assert_eq!(import(br"{\rtf1 A\u-10179?B\u-8704?C}").unwrap().story.text, "ABC");
+ }
}
diff --git a/crates/textimport/src/tagged.rs b/crates/textimport/src/tagged.rs
index ca78bc4b1..cfd08eea7 100644
--- a/crates/textimport/src/tagged.rs
+++ b/crates/textimport/src/tagged.rs
@@ -110,12 +110,14 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
_ => runs.push((c.to_string(), fmt.clone())),
}
};
+ let mut units = crate::Utf16Units::default();
let mut i = 0;
let mut skipped = 0;
while i < chars.len() {
let c = chars[i];
match c {
'\\' if i + 1 < chars.len() => {
+ units.reset();
push(&mut paras, chars[i + 1], &fmt);
i += 2;
}
@@ -143,7 +145,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
match name {
n if n.starts_with("0x") => {
if let Ok(u) = u32::from_str_radix(&n[2..], 16)
- && let Some(ch) = char::from_u32(u)
+ && let Some(ch) = units.push(u)
{
push(&mut paras, ch, &fmt);
}
@@ -193,6 +195,7 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
i += 1;
}
c => {
+ units.reset();
push(&mut paras, c, &fmt);
i += 1;
}
@@ -226,6 +229,20 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
mod tests {
use super::*;
+ /// Characters past U+FFFF are written as two tags, a UTF-16 surrogate pair. Each half was
+ /// converted alone and dropped.
+ #[test]
+ fn surrogate_pairs_become_one_character() {
+ let i = import("<ASCII-WIN>\r\n<Version:7>\r\n<ParaStyle:Body>A<0xD83D><0xDE00>B <0xD842><0xDFB7><0x7530>".as_bytes()).unwrap();
+ assert_eq!(i.story.text, "A\u{1F600}B \u{20BB7}\u{7530}");
+ // What the exporter writes comes back.
+ let doc = Document::new(&designcraft_doc::build::NewDocument::default());
+ let back = import(tagged_text(&doc, &i.story).as_bytes()).unwrap();
+ assert_eq!(back.story.text, i.story.text);
+ // Half a pair is not a character: it is still left out.
+ assert_eq!(import("<ASCII-WIN>\r\n<ParaStyle:Body>A<0xD83D>B<0xDE00>C".as_bytes()).unwrap().story.text, "ABC");
+ }
+
#[test]
fn tagged_text_round_trips() {
let src = "<ASCII-WIN>\r\n<Version:7>\r\n<ParaStyle:Head>Caf<0x00E9> \\<1\\>\r\n<ParaStyle:Body>A <CharStyle:Em>word<CharStyle:> and <cSize:18.000000>big<cSize:> text";