World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Keep XML references in imported Office text (#778)
Maya Kid4a96613bf431
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex 8f92fe14..7be127b0 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -221,6 +221,24 @@ pub(crate) fn docx_xml_to_text(xml: &str) -> anyhow::Result<String> {None => out.push_str(&s),}}+ Ok(Event::CData(t)) if in_text => {+ let s = t.xml_content(quick_xml::XmlVersion::Implicit1_0);+ match cell.as_mut() {+ Some(c) => c.0.push_str(&s),+ None => out.push_str(&s),+ }+ }+ // 字符引用是独立事件,丢掉它会把 R&D 这样的正文变成 RD。+ // 未识别的引用保留原样,不能因此让整篇导入失败。+ Ok(Event::GeneralRef(e)) if in_text => {+ let reference = format!("&{};", e.into_inner());+ let s = quick_xml::escape::unescape(&reference)+ .unwrap_or(std::borrow::Cow::Borrowed(&reference));+ match cell.as_mut() {+ Some(c) => c.0.push_str(&s),+ None => out.push_str(&s),+ }+ }Ok(Event::Eof) => break,Err(e) => anyhow::bail!("XML parse error: {e}"),_ => {}@@ -374,6 +392,16 @@ fn extract_xml_text(xml: &str, text_tag: &str, para_tag: &str) -> anyhow::ResultOk(Event::Text(t)) if in_text => {out.push_str(&t.xml_content(quick_xml::XmlVersion::Implicit1_0));}+ Ok(Event::CData(t)) if in_text => {+ out.push_str(&t.xml_content(quick_xml::XmlVersion::Implicit1_0));+ }+ Ok(Event::GeneralRef(e)) if in_text => {+ let reference = format!("&{};", e.into_inner());+ match quick_xml::escape::unescape(&reference) {+ Ok(s) => out.push_str(&s),+ Err(_) => out.push_str(&reference),+ }+ }Ok(Event::Eof) => break,Err(e) => anyhow::bail!("XML parse error: {e}"),_ => {}diff --git a/crates/utopia-ingest/tests/office_text_references.rs b/crates/utopia-ingest/tests/office_text_references.rsnew file mode 100644index 00000000..93fb8e6b--- /dev/null+++ b/crates/utopia-ingest/tests/office_text_references.rs@@ -0,0 +1,72 @@+use std::io::{Cursor, Write};++fn package(part: &str, xml: &str) -> Vec<u8> {+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));+ zip.start_file(part, zip::write::SimpleFileOptions::default())+ .unwrap();+ zip.write_all(xml.as_bytes()).unwrap();+ zip.finish().unwrap().into_inner()+}++const ENCODED: &str =+ "R&D: 2 < 3 > 1; "quote" 'word'; 中文; &lt;";+const DECODED: &str = "R&D: 2 < 3 > 1; \"quote\" 'word'; 中文; <";++#[test]+fn a_docx_keeps_references_in_paragraphs_and_table_cells() {+ let xml = format!(+ r#"<w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body>+ <w:p><w:r><w:t>{ENCODED}</w:t></w:r></w:p>+ <w:tbl><w:tr><w:tc><w:p><w:r><w:t>Team</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>Budget</w:t></w:r></w:p></w:tc></w:tr>+ <w:tr><w:tc><w:p><w:r><w:t>R&D</w:t></w:r></w:p></w:tc><w:tc><w:p><w:r><w:t>100</w:t></w:r></w:p></w:tc></w:tr></w:tbl>+ </w:body></w:document>"#+ );+ let parsed = utopia_ingest::parse("report.docx", &package("word/document.xml", &xml)).unwrap();+ assert!(parsed.text.starts_with(DECODED), "{}", parsed.text);+ assert!(parsed.text.contains("| R&D | 100 |"), "{}", parsed.text);+}++#[test]+fn a_pptx_keeps_references_in_slide_text() {+ let xml = format!(+ r#"<p:sld xmlns:p="http://schemas.openxmlformats.org/presentationml/2006/main" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><p:cSld><p:spTree><p:sp><p:txBody><a:bodyPr/><a:lstStyle/><a:p><a:r><a:t>{ENCODED}</a:t></a:r></a:p></p:txBody></p:sp></p:spTree></p:cSld></p:sld>"#+ );+ let parsed =+ utopia_ingest::parse("report.pptx", &package("ppt/slides/slide1.xml", &xml)).unwrap();+ assert!(parsed.text.contains(DECODED), "{}", parsed.text);+}++#[test]+fn office_cdata_is_preserved_literally() {+ for (filename, part, xml) in [+ ("report.docx", "word/document.xml", "<w:document xmlns:w=\"http://schemas.openxmlformats.org/wordprocessingml/2006/main\"><w:body>outside&<w:p><w:r><w:t>Before <![CDATA[R&D &]]> after</w:t></w:r></w:p></w:body></w:document>"),+ ("report.pptx", "ppt/slides/slide1.xml", "<a:p xmlns:a=\"http://schemas.openxmlformats.org/drawingml/2006/main\">outside&<a:r><a:t>Before <![CDATA[R&D &]]> after</a:t></a:r></a:p>"),+ ] {+ let parsed = utopia_ingest::parse(filename, &package(part, xml)).unwrap();+ assert!(parsed.text.contains("Before R&D & after"), "{}", parsed.text);+ assert!(!parsed.text.contains("outside"), "{}", parsed.text);+ }+}++#[test]+fn unknown_office_references_do_not_abort_the_import() {+ for (filename, part, xml) in [+ (+ "report.docx",+ "word/document.xml",+ r#"<!DOCTYPE w:document [<!ENTITY team "Research">]><w:document xmlns:w="http://schemas.openxmlformats.org/wordprocessingml/2006/main"><w:body><w:p><w:r><w:t>Paris 2024 and R&D &team;</w:t></w:r></w:p></w:body></w:document>"#,+ ),+ (+ "report.pptx",+ "ppt/slides/slide1.xml",+ r#"<!DOCTYPE a:p [<!ENTITY team "Research">]><a:p xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><a:r><a:t>Paris 2024 and R&D &team;</a:t></a:r></a:p>"#,+ ),+ ] {+ let parsed = utopia_ingest::parse(filename, &package(part, xml)).unwrap();+ assert!(+ parsed.text.contains("Paris 2024 and R&D &team;"),+ "{}",+ parsed.text+ );+ }+}