DesignCraft: An open-source page and layout design editor written by scratch in pure Rust
Decode UTF-16 plain text files by BOM on import
NetterSilver2ddcf6d2a120
diff --git a/crates/textimport/src/lib.rs b/crates/textimport/src/lib.rsindex ba9cb92e3..56eac1a00 100644--- a/crates/textimport/src/lib.rs+++ b/crates/textimport/src/lib.rs@@ -114,7 +114,7 @@ pub fn import(name: &str, bytes: &[u8]) -> Result<Imported, ImportError> {if l.ends_with(".doc") {return Err(ImportError::Unsupported("legacy Word .doc (save it as .docx)".into()));}- Ok(plain(&String::from_utf8_lossy(bytes)))+ Ok(plain(&tagged::decode(bytes)))}/// Plain text: line breaks become paragraphs (CRLF / CR normalised).@@ -136,4 +136,14 @@ mod tests {assert!(is_text_file("Report.DOCX") && !is_text_file("a.png"));assert!(matches!(import("x.doc", b"\xD0\xCF"), Err(ImportError::Unsupported(_))));}++ #[test]+ fn plain_text_utf16() {+ let mut le = vec![0xFF, 0xFE];+ le.extend("Hi there".encode_utf16().flat_map(u16::to_le_bytes));+ assert_eq!(import("a.txt", &le).unwrap().story.text, "Hi there");+ let mut be = vec![0xFE, 0xFF];+ be.extend("Hi there".encode_utf16().flat_map(u16::to_be_bytes));+ assert_eq!(import("a.txt", &be).unwrap().story.text, "Hi there");+ }}diff --git a/crates/textimport/src/tagged.rs b/crates/textimport/src/tagged.rsindex 9bc84b766..18704cbc4 100644--- a/crates/textimport/src/tagged.rs+++ b/crates/textimport/src/tagged.rs@@ -72,7 +72,7 @@ pub fn tagged_text(doc: &Document, story: &Story) -> String {}/// Text of a Tagged Text file (ASCII, UTF-8 or UTF-16 by byte-order mark).-fn decode(bytes: &[u8]) -> String {+pub(crate) fn decode(bytes: &[u8]) -> String {if bytes.starts_with(&[0xFF, 0xFE]) || bytes.starts_with(&[0xFE, 0xFF]) {let le = bytes[0] == 0xFF;let units: Vec<u16> = bytes[2..]