Skip to content

DesignCraft: An open-source page and layout design editor written by scratch in pure Rust

Decode UTF-16 plain text files by BOM on import

NetterSilver2ddcf6d2a120
diff --git a/crates/textimport/src/lib.rs b/crates/textimport/src/lib.rs
index ba9cb92e3..56eac1a00 100644
--- a/crates/textimport/src/lib.rs
+++ b/crates/textimport/src/lib.rs
@@ -114,7 +114,7 @@ pub fn import(name: &str, bytes: &[u8]) -> Result<Imported, ImportError> {
if l.ends_with(".doc") {
return Err(ImportError::Unsupported("legacy Word .doc (save it as .docx)".into()));
}
- Ok(plain(&String::from_utf8_lossy(bytes)))
+ Ok(plain(&tagged::decode(bytes)))
}
/// Plain text: line breaks become paragraphs (CRLF / CR normalised).
@@ -136,4 +136,14 @@ mod tests {
assert!(is_text_file("Report.DOCX") && !is_text_file("a.png"));
assert!(matches!(import("x.doc", b"\xD0\xCF"), Err(ImportError::Unsupported(_))));
}
+
+ #[test]
+ fn plain_text_utf16() {
+ let mut le = vec![0xFF, 0xFE];
+ le.extend("Hi there".encode_utf16().flat_map(u16::to_le_bytes));
+ assert_eq!(import("a.txt", &le).unwrap().story.text, "Hi there");
+ let mut be = vec![0xFE, 0xFF];
+ be.extend("Hi there".encode_utf16().flat_map(u16::to_be_bytes));
+ assert_eq!(import("a.txt", &be).unwrap().story.text, "Hi there");
+ }
}
diff --git a/crates/textimport/src/tagged.rs b/crates/textimport/src/tagged.rs
index 9bc84b766..18704cbc4 100644
--- a/crates/textimport/src/tagged.rs
+++ b/crates/textimport/src/tagged.rs
@@ -72,7 +72,7 @@ pub fn tagged_text(doc: &Document, story: &Story) -> String {
}
/// Text of a Tagged Text file (ASCII, UTF-8 or UTF-16 by byte-order mark).
-fn decode(bytes: &[u8]) -> String {
+pub(crate) fn decode(bytes: &[u8]) -> String {
if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) {
let le = bytes[0] == 0xFF;
let units: Vec<u16> = bytes[2..]