Skip to content

World's first open-source enterprise world model.

Keep XML references in imported Office text (#778)

Maya Kid4a96613bf431
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 8f92fe14..7be127b0 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -221,6 +221,24 @@ pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {
None => out.push_str(&s),
}
}
+ Ok(Event::CData(t)) if in_text => {
+ let s = t.xml_content(quick_xml::XmlVersion::Implicit1_0);
+ match cell.as_mut() {
+ Some(c) => c.0.push_str(&s),
+ None => out.push_str(&s),
+ }
+ }
+ // 字符引用是独立事件,丢掉它会把 R&amp;D 这样的正文变成 RD。
+ // 未识别的引用保留原样,不能因此让整篇导入失败。
+ Ok(Event::GeneralRef(e)) if in_text => {
+ let reference = format!("&{};", e.into_inner());
+ let s = quick_xml::escape::unescape(&reference)
+ .unwrap_or(std::borrow::Cow::Borrowed(&reference));
+ match cell.as_mut() {
+ Some(c) => c.0.push_str(&s),
+ None => out.push_str(&s),
+ }
+ }
Ok(Event::Eof) => break,
Err(e) => anyhow::bail!("XML parse error: {e}"),
_ => {}
@@ -374,6 +392,16 @@ fn extract_xml_text(xml: &str, text_tag: &str, para_tag: &str) -> anyhow::Result
Ok(Event::Text(t)) if in_text => {
out.push_str(&t.xml_content(quick_xml::XmlVersion::Implicit1_0));
}
+ Ok(Event::CData(t)) if in_text => {
+ out.push_str(&t.xml_content(quick_xml::XmlVersion::Implicit1_0));
+ }
+ Ok(Event::GeneralRef(e)) if in_text => {
+ let reference = format!("&{};", e.into_inner());
+ match quick_xml::escape::unescape(&reference) {
+ Ok(s) => out.push_str(&s),
+ Err(_) => out.push_str(&reference),
+ }
+ }
Ok(Event::Eof) => break,
Err(e) => anyhow::bail!("XML parse error: {e}"),
_ => {}
diff --git a/crates/utopia-ingest/tests/office_text_references.rs b/crates/utopia-ingest/tests/office_text_references.rs
new file mode 100644
index 00000000..93fb8e6b
--- /dev/null
+++ b/crates/utopia-ingest/tests/office_text_references.rs
@@ -0,0 +1,72 @@
+use std::io::{Cursor, Write};
+
+fn package(part: &str, xml: &str) -> Vec<u8> {
+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+ zip.start_file(part, zip::write::SimpleFileOptions::default())
+ .unwrap();
+ zip.write_all(xml.as_bytes()).unwrap();
+ zip.finish().unwrap().into_inner()
+}
+
+const ENCODED: &str =
+ "R&amp;D: 2 &lt; 3 &gt; 1; &quot;quote&quot; &apos;word&apos;; &#20013;&#x6587;; &amp;lt;";
+const DECODED: &str = "R&D: 2 < 3 > 1; \"quote\" 'word'; 中文; &lt;";
+
+#[test]
+fn a_docx_keeps_references_in_paragraphs_and_table_cells() {
+ let xml = format!(
+ r#"<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>
+ <w:p><w:r><w:t>{ENCODED}</w:t></w:r></w:p>
+ <w:tbl><w:tr><w:tc><w:p><w:r><w:t>Team</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Budget</w:t></w:r></w:p></w:tc></w:tr>
+ <w:tr><w:tc><w:p><w:r><w:t>R&amp;D</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>100</w:t></w:r></w:p></w:tc></w:tr></w:tbl>
+ </w:body></w:document>"#
+ );
+ let parsed = utopia_ingest::parse("report.docx", &package("word/document.xml", &xml)).unwrap();
+ assert!(parsed.text.starts_with(DECODED), "{}", parsed.text);
+ assert!(parsed.text.contains("| R&D | 100 |"), "{}", parsed.text);
+}
+
+#[test]
+fn a_pptx_keeps_references_in_slide_text() {
+ let xml = format!(
+ r#"<p:sld xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><p:cSld><p:spTree><p:sp><p:txBody><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>{ENCODED}</a:t></a:r></a:p></p:txBody></p:sp></p:spTree></p:cSld></p:sld>"#
+ );
+ let parsed =
+ utopia_ingest::parse("report.pptx", &package("ppt/slides/slide1.xml", &xml)).unwrap();
+ assert!(parsed.text.contains(DECODED), "{}", parsed.text);
+}
+
+#[test]
+fn office_cdata_is_preserved_literally() {
+ for (filename, part, xml) in [
+ ("report.docx", "word/document.xml", "<w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:body>outside&amp;<w:p><w:r><w:t>Before <![CDATA[R&D &amp;]]> after</w:t></w:r></w:p></w:body></w:document>"),
+ ("report.pptx", "ppt/slides/slide1.xml", "<a:p xmlns:a=\"http://schemas.openxmlformats.org/drawingml/2006/main\">outside&amp;<a:r><a:t>Before <![CDATA[R&D &amp;]]> after</a:t></a:r></a:p>"),
+ ] {
+ let parsed = utopia_ingest::parse(filename, &package(part, xml)).unwrap();
+ assert!(parsed.text.contains("Before R&D &amp; after"), "{}", parsed.text);
+ assert!(!parsed.text.contains("outside"), "{}", parsed.text);
+ }
+}
+
+#[test]
+fn unknown_office_references_do_not_abort_the_import() {
+ for (filename, part, xml) in [
+ (
+ "report.docx",
+ "word/document.xml",
+ r#"<!DOCTYPE w:document [<!ENTITY team "Research">]><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>Paris&nbsp;2024 and R&amp;D &team;</w:t></w:r></w:p></w:body></w:document>"#,
+ ),
+ (
+ "report.pptx",
+ "ppt/slides/slide1.xml",
+ r#"<!DOCTYPE a:p [<!ENTITY team "Research">]><a:p xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><a:r><a:t>Paris&nbsp;2024 and R&amp;D &team;</a:t></a:r></a:p>"#,
+ ),
+ ] {
+ let parsed = utopia_ingest::parse(filename, &package(part, xml)).unwrap();
+ assert!(
+ parsed.text.contains("Paris&nbsp;2024 and R&D &team;"),
+ "{}",
+ parsed.text
+ );
+ }
+}