DesignCraft: An open-source page and layout design editor written by scratch in pure Rust
Import: a shared string and a footnote are not copied per reference (#1584)
Hamza96b498ca7086
diff --git a/crates/textimport/src/docx.rs b/crates/textimport/src/docx.rsindex 94085ca69..5f1624267 100644--- a/crates/textimport/src/docx.rs+++ b/crates/textimport/src/docx.rs@@ -236,7 +236,17 @@ struct Ctx {para_names: HashMap<String, String>,char_names: HashMap<String, String>,footnotes: HashMap<String, El>,+ /// Footnote ids already imported, and how much note text has been built. A reference clones+ /// its note and walks it again, so one 1 MB footnote referenced 100k times meant ~100 GB+ /// (#1584). Word gives each note its own id and references it once; a second reference to the+ /// same id gets the mark without the text.+ notes_done: std::collections::HashSet<String>,+ note_bytes: usize,warnings: Vec<String>,+ /// What has already been said, so saying it again is a lookup rather than a scan of+ /// everything said so far. A warning that names a footnote id is one per id, and the+ /// document chooses how many ids it has (noted in review).+ warned: std::collections::HashSet<String>,next_table: u64,numbering: Numbering,/// A paragraph style's list (`w:numPr` numId and ilvl, through `w:basedOn`), by style id.@@ -249,7 +259,47 @@ struct Ctx {default_para: String,}+/// A note's tables may hold tables of their own. The document model stops nesting at 64, so+/// counting no deeper than that reaches every cell a note can actually have.+const MAX_NOTE_DEPTH: usize = 64;++/// The text `s` holds, including its tables' cell text. A table's cells each keep their text in+/// a story of their own rather than in `s.text`, which holds only the anchor, so counting+/// `s.text` scored a table of any size as a couple of bytes (noted in review).+fn note_bytes(s: &Story, depth: usize) -> usize {+ let mut n = s.text.len();+ if depth == 0 {+ return n;+ }+ for t in s.tables.values() {+ for c in &t.cells {+ n = n.saturating_add(note_bytes(&c.text, depth - 1));+ }+ }+ n+}++/// Warnings one import reports. A document that earns more than this has earned the point.+const MAX_WARNINGS: usize = 100;+impl Ctx {+ /// Report `message` once. Beyond [`MAX_WARNINGS`] distinct messages nothing further is kept,+ /// not even in the set, so neither the report nor the memory behind it grows with the+ /// document -- a footnote warning names an id, and a file can hold as many ids as it likes.+ fn warn(&mut self, message: String) {+ if self.warnings.len() > MAX_WARNINGS {+ return;+ }+ if !self.warned.insert(message.clone()) {+ return;+ }+ if self.warnings.len() == MAX_WARNINGS {+ self.warnings.push(format!("The Word file has more than {MAX_WARNINGS} kinds of problem: the rest are not listed."));+ } else {+ self.warnings.push(message);+ }+ }+fn para_style(&self, ppr: Option<&El>) -> String {ppr.and_then(|p| p.val("pStyle")).and_then(|id| self.para_names.get(&id).cloned()).unwrap_or_else(|| self.default_para.clone())}@@ -412,11 +462,39 @@ impl Ctx {"noBreakHyphen" => "\u{2011}".into(),"softHyphen" => "\u{AD}".into(),"footnoteReference" if allow_notes => {+ const MAX_NOTE_BYTES: usize = 8 << 20;+ let id = e.attr("id").unwrap_or_default().to_string();+ // The note's text is built once per id, and only while the budget lasts. A+ // repeat reference, or one past the budget, still gets its mark and its+ // number -- the reference is part of the text -- with an empty note (#1584).+ //+ // The id is recorded only once its text has actually been kept, further down:+ // recording it here meant a first reference arriving after the budget was+ // spent counted as "done" with an empty note, and the next reference to it+ // then claimed the text was kept at the first one (noted in review).+ let again = !id.is_empty() && self.notes_done.contains(&id);+ let spent = self.note_bytes >= MAX_NOTE_BYTES;+ if again || spent {+ let len = st.len();+ if st.insert_note(len, "", ParaFormat::default()).is_some() {+ let message = if again {+ format!("Footnote {id} is referenced more than once in the Word file: the text is kept at its first reference.")+ } else {+ format!("The Word file's footnotes are longer than {} MB: the rest were imported empty.", MAX_NOTE_BYTES / (1 << 20))+ };+ self.warn(message);+ }+ continue;+ }if let Some(note) = e.attr("id").and_then(|id| self.footnotes.get(id).cloned()) {let mut ns = Story::new(StoryId(0));let counter = std::mem::take(&mut self.model_counter);self.blocks(¬e, &mut ns, false);self.model_counter = counter;+ self.note_bytes = self.note_bytes.saturating_add(note_bytes(&ns, MAX_NOTE_DEPTH));+ if !id.is_empty() {+ self.notes_done.insert(id.clone());+ }// Word puts the reference mark in the note text: we generate the number.let len = st.len();match st.insert_note(len, "", ParaFormat::default()) {@@ -425,15 +503,13 @@ impl Ctx {n.text = ns;}}- None => self.warnings.push("A footnote in the Word file could not be imported.".into()),+ None => self.warn("A footnote in the Word file could not be imported.".into()),}}continue;}"drawing" | "pict" | "object" => {- if !self.warnings.iter().any(|w| w.starts_with("Images")) {- self.warnings.push("Images and drawings in the Word file were not imported (place them separately).".into());- }+ self.warn("Images and drawings in the Word file were not imported (place them separately).".into());continue;}_ => continue,@@ -485,7 +561,10 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {para_names: HashMap::new(),char_names: HashMap::new(),footnotes: HashMap::new(),+ notes_done: std::collections::HashSet::new(),+ note_bytes: 0,warnings: vec![],+ warned: std::collections::HashSet::new(),next_table: 0,numbering: Numbering::default(),style_lists: HashMap::new(),diff --git a/crates/textimport/src/docx/tests.rs b/crates/textimport/src/docx/tests.rsindex c70ca851f..7f1e425cf 100644--- a/crates/textimport/src/docx/tests.rs+++ b/crates/textimport/src/docx/tests.rs@@ -89,6 +89,124 @@ fn a_measure_or_size_a_file_cannot_mean_is_bounded() {assert_eq!(size, None, "a size the engine cannot mean is dropped, so the style's size stands");}+/// A footnote reference clones its note and walks it again, so one large footnote referenced+/// many times multiplied: the audit's case is a 1 MB note referenced 100k times, around+/// 100 GB, which aborts rather than fails (#1584). The text is kept at the first reference;+/// later ones get the mark and an empty note, and the report says so.+#[test]+fn a_footnote_referenced_many_times_is_imported_once() {+ let note_text = "x".repeat(2_000);+ let footnotes = format!(r#"<w:footnotes {W}><w:footnote w:id="2"><w:p><w:r><w:t>{note_text}</w:t></w:r></w:p></w:footnote></w:footnotes>"#);+ // One note, referenced 50 times.+ let refs = r#"<w:r><w:footnoteReference w:id="2"/></w:r>"#.repeat(50);+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);+ let i = import(&docx(&doc, "", &footnotes)).unwrap();++ // Every reference is still in the text: the marks are the document's content.+ assert_eq!(i.story.notes.len(), 50, "all 50 references are kept");+ let with_text = i.story.notes.iter().filter(|n| !n.text.text.is_empty()).count();+ assert_eq!(with_text, 1, "only the first reference carries the note's text");+ let total: usize = i.story.notes.iter().map(|n| n.text.text.len()).sum();+ assert!(total <= note_text.len() + 64, "the note text is stored once, not 50 times: {total} bytes");+ assert!(i.warnings.iter().any(|w| w.contains("referenced more than once")), "the report says what it did: {:?}", i.warnings);++ // A document whose notes each have their own id is unaffected.+ let many = (2..6).map(|id| format!(r#"<w:footnote w:id="{id}"><w:p><w:r><w:t>note {id}</w:t></w:r></w:p></w:footnote>"#)).collect::<String>();+ let footnotes = format!(r#"<w:footnotes {W}>{many}</w:footnotes>"#);+ let refs = (2..6).map(|id| format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#)).collect::<String>();+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);+ let i = import(&docx(&doc, "", &footnotes)).unwrap();+ assert_eq!(i.story.notes.len(), 4);+ assert!(i.story.notes.iter().all(|n| n.text.text.contains("note")), "each distinct note keeps its text");+ assert!(!i.warnings.iter().any(|w| w.contains("referenced more than once")), "{:?}", i.warnings);+}++/// A footnote's table keeps its cell text in stories of its own, so the budget counted a+/// table of any size as the couple of bytes of its anchor and the cap did not hold (noted in+/// review of #1584). Eighteen notes of ~600 KB of cell text each are 10.8 MB of table text+/// against an 8 MB budget.+#[test]+fn a_footnote_table_counts_toward_the_note_budget() {+ let big = "x".repeat(600_000);+ let cell = format!(r#"<w:tbl><w:tr><w:tc><w:p><w:r><w:t>{big}</w:t></w:r></w:p></w:tc></w:tr></w:tbl>"#);+ let notes: String = (2..20).map(|id| format!(r#"<w:footnote w:id="{id}">{cell}</w:footnote>"#)).collect();+ let footnotes = format!(r#"<w:footnotes {W}>{notes}</w:footnotes>"#);+ let refs: String = (2..20).map(|id| format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#)).collect();+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);+ let i = import(&docx(&doc, "", &footnotes)).unwrap();++ // Every reference is still a mark in the text.+ assert_eq!(i.story.notes.len(), 18, "all 18 references are kept");+ // What was retained is the budget plus at most the one note that was already being built+ // when it ran out -- not all 10.8 MB.+ let held: usize = i.story.notes.iter().map(|n| note_bytes(&n.text, MAX_NOTE_DEPTH)).sum();+ assert!(held <= (8 << 20) + big.len() + 1024, "the cell text is inside the budget: {held} bytes");+ assert!(held > big.len(), "and the notes that fit were imported in full: {held} bytes");+ assert!(i.warnings.iter().any(|w| w.contains("longer than 8 MB")), "the budget warning is reported: {:?}", i.warnings);+}++/// The report is one line per distinct footnote id, and the file chooses how many ids it+/// has: 400 ids referenced twice each meant 400 warnings, every one of them pushed after a+/// `Vec::contains` scan of all the earlier ones (noted in review of #1584). Both the report+/// and the set behind it stop at the cap.+#[test]+fn the_report_does_not_grow_with_the_document() {+ let ids = 2..402;+ // Small notes, so the byte budget is not what stops this -- the ids are.+ let notes: String = ids.clone().map(|id| format!(r#"<w:footnote w:id="{id}"><w:p><w:r><w:t>note {id}</w:t></w:r></w:p></w:footnote>"#)).collect();+ let footnotes = format!(r#"<w:footnotes {W}>{notes}</w:footnotes>"#);+ let refs: String = ids+ .clone()+ .flat_map(|id| {+ let one = format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#);+ [one.clone(), one]+ })+ .collect();+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);+ let i = import(&docx(&doc, "", &footnotes)).unwrap();++ // The document is imported in full: every reference is a mark in the text.+ assert_eq!(i.story.notes.len(), 800, "all 800 references are kept");+ let duplicates = i.warnings.iter().filter(|w| w.contains("referenced more than once")).count();+ assert!(duplicates >= 1, "the repeats are reported: {:?}", i.warnings);+ assert!(i.warnings.len() <= MAX_WARNINGS + 1, "400 duplicated ids do not make 400 warnings: {} warnings", i.warnings.len());+ // And the last line says the list was cut, rather than ending without explanation.+ assert!(i.warnings.iter().any(|w| w.contains("the rest are not listed")), "{:?}", i.warnings);+}++/// When the budget runs out before a note's *first* reference, that reference gets an empty+/// note -- and must not be recorded as the one that kept the text, or a later reference to the+/// same id would be told the text is at a reference that has none (noted in review of #1584).+#[test]+fn a_note_whose_first_reference_is_past_the_budget_is_not_recorded_as_kept() {+ // Notes totalling more than the 8 MB budget, so the later ids are never built.+ let big = "x".repeat(600_000);+ let notes: String = (2..20).map(|id| format!(r#"<w:footnote w:id="{id}"><w:p><w:r><w:t>{big}</w:t></w:r></w:p></w:footnote>"#)).collect();+ let footnotes = format!(r#"<w:footnotes {W}>{notes}</w:footnotes>"#);+ // Each id referenced twice, in order.+ let refs: String = (2..20)+ .flat_map(|id| {+ let one = format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#);+ [one.clone(), one]+ })+ .collect();+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);+ let i = import(&docx(&doc, "", &footnotes)).unwrap();++ // Every reference is kept, and the budget is respected.+ assert_eq!(i.story.notes.len(), 36, "all 36 references are in the text");+ let total: usize = i.story.notes.iter().map(|n| n.text.text.len()).sum();+ assert!(total <= (8 << 20) + big.len(), "the note text stays inside the budget: {total} bytes");++ // The "referenced more than once" warning is only about ids whose text was actually kept.+ // A note that was never built cannot have kept its text at a first reference.+ let kept: std::collections::HashSet<&str> = i.story.notes.iter().filter(|n| !n.text.text.is_empty()).map(|n| n.text.text.as_str()).collect();+ if i.warnings.iter().any(|w| w.contains("referenced more than once")) {+ assert!(!kept.is_empty(), "that warning implies some text was kept: {:?}", i.warnings);+ }+ assert!(i.warnings.iter().any(|w| w.contains("longer than 8 MB")), "the budget warning is the one this document earns: {:?}", i.warnings);+}+#[test]fn imports_styles_runs_footnotes_and_tables() {let styles = format!(diff --git a/crates/textimport/src/xlsx.rs b/crates/textimport/src/xlsx.rsindex a26665ecc..7f129cce1 100644--- a/crates/textimport/src/xlsx.rs+++ b/crates/textimport/src/xlsx.rs@@ -212,19 +212,41 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {.transpose()?.map(|sst| sst.els().filter(|e| e.name == "si").map(string_item).collect()).unwrap_or_default();+ // A `t="s"` cell holds an index into the shared strings and each one is materialised as its+ // own `String`, so one large shared string referenced by many cells multiplies: a 1 MB string+ // and 3M references is terabytes, allocated before the grid cap below ever applies, and an+ // allocation that large aborts the process rather than failing (#1584). Collection carries a+ // byte budget and a cell count of its own.+ const MAX_TEXT_BYTES: usize = 64 << 20;+ const MAX_COLLECTED_CELLS: usize = 400_000;let mut cells: Vec<((usize, usize), String)> = Vec::new();+ let mut text_bytes = 0usize;+ let mut cut = false;if let Some(data) = sheet.child("sheetData") {- for (ri, row) in data.els().filter(|e| e.name == "row").enumerate() {+ 'rows: for (ri, row) in data.els().filter(|e| e.name == "row").enumerate() {for (ci, c) in row.els().filter(|e| e.name == "c").enumerate() {let at = c.attr("r").and_then(cell_ref).unwrap_or((ri, ci));let v = value(c, &shared);- if !v.is_empty() {- cells.push((at, v));+ if v.is_empty() {+ continue;+ }+ if cells.len() >= MAX_COLLECTED_CELLS || text_bytes.saturating_add(v.len()) > MAX_TEXT_BYTES {+ cut = true;+ break 'rows;}+ text_bytes = text_bytes.saturating_add(v.len());+ cells.push((at, v));}}}let mut warnings = Vec::new();+ if cut {+ warnings.push(format!(+ "the worksheet holds more than {} cells or {} MB of text: the rest was not read",+ MAX_COLLECTED_CELLS,+ MAX_TEXT_BYTES / (1 << 20)+ ));+ }if cells.is_empty() {warnings.push("the worksheet is empty".into());}@@ -293,6 +315,50 @@ mod tests {buf.into_inner()}+ /// A `t="s"` cell materialises its own copy of the shared string, so one big string with many+ /// references multiplies: the audit's case is a 1 MB string and 3M cells, terabytes, built+ /// before the grid cap applies (#1584). Collection now carries its own byte budget.+ #[test]+ fn a_shared_string_referenced_many_times_does_not_multiply() {+ // A 64 KB shared string referenced 2,000 times would be 128 MB of cell text; the budget+ // is 64 MB, so collection stops and says so.+ let big = "y".repeat(64 * 1024);+ let cells: String = (1..=2_000).map(|r| format!(r#"<row r="{r}"><c r="A{r}" t="s"><v>0</v></c></row>"#)).collect();+ let mut buf = std::io::Cursor::new(Vec::new());+ {+ let mut z = zip::ZipWriter::new(&mut buf);+ let o = zip::write::SimpleFileOptions::default();+ for (name, body) in [+ (+ "xl/workbook.xml",+ r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Big" sheetId="1" r:id="rId1"/></sheets></workbook>"#.to_string(),+ ),+ (+ "xl/_rels/workbook.xml.rels",+ r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="x" Target="worksheets/sheet1.xml"/></Relationships>"#.to_string(),+ ),+ ("xl/sharedStrings.xml", format!("<sst><si><t>{big}</t></si></sst>")),+ ("xl/worksheets/sheet1.xml", format!("<worksheet><sheetData>{cells}</sheetData></worksheet>")),+ ] {+ z.start_file(name, o).unwrap();+ z.write_all(body.as_bytes()).unwrap();+ }+ z.finish().unwrap();+ }+ let imp = import(&buf.into_inner()).unwrap();+ let table = imp.story.tables.values().next().expect("a table");+ let text: usize = (0..table.nrows())+ .flat_map(|r| (0..table.ncols()).map(move |c| (r, c)))+ .filter_map(|(r, c)| table.cell(r, c).map(|cell| cell.text.text.len()))+ .sum();+ assert!(text <= 64 << 20, "the table holds at most the budget: {text} bytes");+ assert!(imp.warnings.iter().any(|w| w.contains("was not read")), "the report says the sheet was cut: {:?}", imp.warnings);++ // An ordinary workbook is unaffected and reports nothing of the sort.+ let plain = import(&workbook()).unwrap();+ assert!(!plain.warnings.iter().any(|w| w.contains("was not read")), "{:?}", plain.warnings);+ }+#[test]fn first_sheet_used_range_becomes_a_table() {let imp = import(&workbook()).unwrap();