Skip to content

DesignCraft: An open-source page and layout design editor written by scratch in pure Rust

Import: a shared string and a footnote are not copied per reference (#1584)

Hamza96b498ca7086
diff --git a/crates/textimport/src/docx.rs b/crates/textimport/src/docx.rs
index 94085ca69..5f1624267 100644
--- a/crates/textimport/src/docx.rs
+++ b/crates/textimport/src/docx.rs
@@ -236,7 +236,17 @@ struct Ctx {
para_names: HashMap<String, String>,
char_names: HashMap<String, String>,
footnotes: HashMap<String, El>,
+ /// Footnote ids already imported, and how much note text has been built. A reference clones
+ /// its note and walks it again, so one 1 MB footnote referenced 100k times meant ~100 GB
+ /// (#1584). Word gives each note its own id and references it once; a second reference to the
+ /// same id gets the mark without the text.
+ notes_done: std::collections::HashSet<String>,
+ note_bytes: usize,
warnings: Vec<String>,
+ /// What has already been said, so saying it again is a lookup rather than a scan of
+ /// everything said so far. A warning that names a footnote id is one per id, and the
+ /// document chooses how many ids it has (noted in review).
+ warned: std::collections::HashSet<String>,
next_table: u64,
numbering: Numbering,
/// A paragraph style's list (`w:numPr` numId and ilvl, through `w:basedOn`), by style id.
@@ -249,7 +259,47 @@ struct Ctx {
default_para: String,
}
+/// A note's tables may hold tables of their own. The document model stops nesting at 64, so
+/// counting no deeper than that reaches every cell a note can actually have.
+const MAX_NOTE_DEPTH: usize = 64;
+
+/// The text `s` holds, including its tables' cell text. A table's cells each keep their text in
+/// a story of their own rather than in `s.text`, which holds only the anchor, so counting
+/// `s.text` scored a table of any size as a couple of bytes (noted in review).
+fn note_bytes(s: &Story, depth: usize) -> usize {
+ let mut n = s.text.len();
+ if depth == 0 {
+ return n;
+ }
+ for t in s.tables.values() {
+ for c in &t.cells {
+ n = n.saturating_add(note_bytes(&c.text, depth - 1));
+ }
+ }
+ n
+}
+
+/// Warnings one import reports. A document that earns more than this has earned the point.
+const MAX_WARNINGS: usize = 100;
+
impl Ctx {
+ /// Report `message` once. Beyond [`MAX_WARNINGS`] distinct messages nothing further is kept,
+ /// not even in the set, so neither the report nor the memory behind it grows with the
+ /// document -- a footnote warning names an id, and a file can hold as many ids as it likes.
+ fn warn(&mut self, message: String) {
+ if self.warnings.len() > MAX_WARNINGS {
+ return;
+ }
+ if !self.warned.insert(message.clone()) {
+ return;
+ }
+ if self.warnings.len() == MAX_WARNINGS {
+ self.warnings.push(format!("The Word file has more than {MAX_WARNINGS} kinds of problem: the rest are not listed."));
+ } else {
+ self.warnings.push(message);
+ }
+ }
+
fn para_style(&self, ppr: Option<&El>) -> String {
ppr.and_then(|p| p.val("pStyle")).and_then(|id| self.para_names.get(&id).cloned()).unwrap_or_else(|| self.default_para.clone())
}
@@ -412,11 +462,39 @@ impl Ctx {
"noBreakHyphen" => "\u{2011}".into(),
"softHyphen" => "\u{AD}".into(),
"footnoteReference" if allow_notes => {
+ const MAX_NOTE_BYTES: usize = 8 << 20;
+ let id = e.attr("id").unwrap_or_default().to_string();
+ // The note's text is built once per id, and only while the budget lasts. A
+ // repeat reference, or one past the budget, still gets its mark and its
+ // number -- the reference is part of the text -- with an empty note (#1584).
+ //
+ // The id is recorded only once its text has actually been kept, further down:
+ // recording it here meant a first reference arriving after the budget was
+ // spent counted as "done" with an empty note, and the next reference to it
+ // then claimed the text was kept at the first one (noted in review).
+ let again = !id.is_empty() && self.notes_done.contains(&id);
+ let spent = self.note_bytes >= MAX_NOTE_BYTES;
+ if again || spent {
+ let len = st.len();
+ if st.insert_note(len, "", ParaFormat::default()).is_some() {
+ let message = if again {
+ format!("Footnote {id} is referenced more than once in the Word file: the text is kept at its first reference.")
+ } else {
+ format!("The Word file's footnotes are longer than {} MB: the rest were imported empty.", MAX_NOTE_BYTES / (1 << 20))
+ };
+ self.warn(message);
+ }
+ continue;
+ }
if let Some(note) = e.attr("id").and_then(|id| self.footnotes.get(id).cloned()) {
let mut ns = Story::new(StoryId(0));
let counter = std::mem::take(&mut self.model_counter);
self.blocks(&note, &mut ns, false);
self.model_counter = counter;
+ self.note_bytes = self.note_bytes.saturating_add(note_bytes(&ns, MAX_NOTE_DEPTH));
+ if !id.is_empty() {
+ self.notes_done.insert(id.clone());
+ }
// Word puts the reference mark in the note text: we generate the number.
let len = st.len();
match st.insert_note(len, "", ParaFormat::default()) {
@@ -425,15 +503,13 @@ impl Ctx {
n.text = ns;
}
}
- None => self.warnings.push("A footnote in the Word file could not be imported.".into()),
+ None => self.warn("A footnote in the Word file could not be imported.".into()),
}
}
continue;
}
"drawing" | "pict" | "object" => {
- if !self.warnings.iter().any(|w| w.starts_with("Images")) {
- self.warnings.push("Images and drawings in the Word file were not imported (place them separately).".into());
- }
+ self.warn("Images and drawings in the Word file were not imported (place them separately).".into());
continue;
}
_ => continue,
@@ -485,7 +561,10 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
para_names: HashMap::new(),
char_names: HashMap::new(),
footnotes: HashMap::new(),
+ notes_done: std::collections::HashSet::new(),
+ note_bytes: 0,
warnings: vec![],
+ warned: std::collections::HashSet::new(),
next_table: 0,
numbering: Numbering::default(),
style_lists: HashMap::new(),
diff --git a/crates/textimport/src/docx/tests.rs b/crates/textimport/src/docx/tests.rs
index c70ca851f..7f1e425cf 100644
--- a/crates/textimport/src/docx/tests.rs
+++ b/crates/textimport/src/docx/tests.rs
@@ -89,6 +89,124 @@ fn a_measure_or_size_a_file_cannot_mean_is_bounded() {
assert_eq!(size, None, "a size the engine cannot mean is dropped, so the style's size stands");
}
+/// A footnote reference clones its note and walks it again, so one large footnote referenced
+/// many times multiplied: the audit's case is a 1 MB note referenced 100k times, around
+/// 100 GB, which aborts rather than fails (#1584). The text is kept at the first reference;
+/// later ones get the mark and an empty note, and the report says so.
+#[test]
+fn a_footnote_referenced_many_times_is_imported_once() {
+ let note_text = "x".repeat(2_000);
+ let footnotes = format!(r#"<w:footnotes {W}><w:footnote w:id="2"><w:p><w:r><w:t>{note_text}</w:t></w:r></w:p></w:footnote></w:footnotes>"#);
+ // One note, referenced 50 times.
+ let refs = r#"<w:r><w:footnoteReference w:id="2"/></w:r>"#.repeat(50);
+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);
+ let i = import(&docx(&doc, "", &footnotes)).unwrap();
+
+ // Every reference is still in the text: the marks are the document's content.
+ assert_eq!(i.story.notes.len(), 50, "all 50 references are kept");
+ let with_text = i.story.notes.iter().filter(|n| !n.text.text.is_empty()).count();
+ assert_eq!(with_text, 1, "only the first reference carries the note's text");
+ let total: usize = i.story.notes.iter().map(|n| n.text.text.len()).sum();
+ assert!(total <= note_text.len() + 64, "the note text is stored once, not 50 times: {total} bytes");
+ assert!(i.warnings.iter().any(|w| w.contains("referenced more than once")), "the report says what it did: {:?}", i.warnings);
+
+ // A document whose notes each have their own id is unaffected.
+ let many = (2..6).map(|id| format!(r#"<w:footnote w:id="{id}"><w:p><w:r><w:t>note {id}</w:t></w:r></w:p></w:footnote>"#)).collect::<String>();
+ let footnotes = format!(r#"<w:footnotes {W}>{many}</w:footnotes>"#);
+ let refs = (2..6).map(|id| format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#)).collect::<String>();
+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);
+ let i = import(&docx(&doc, "", &footnotes)).unwrap();
+ assert_eq!(i.story.notes.len(), 4);
+ assert!(i.story.notes.iter().all(|n| n.text.text.contains("note")), "each distinct note keeps its text");
+ assert!(!i.warnings.iter().any(|w| w.contains("referenced more than once")), "{:?}", i.warnings);
+}
+
+/// A footnote's table keeps its cell text in stories of its own, so the budget counted a
+/// table of any size as the couple of bytes of its anchor and the cap did not hold (noted in
+/// review of #1584). Eighteen notes of ~600 KB of cell text each are 10.8 MB of table text
+/// against an 8 MB budget.
+#[test]
+fn a_footnote_table_counts_toward_the_note_budget() {
+ let big = "x".repeat(600_000);
+ let cell = format!(r#"<w:tbl><w:tr><w:tc><w:p><w:r><w:t>{big}</w:t></w:r></w:p></w:tc></w:tr></w:tbl>"#);
+ let notes: String = (2..20).map(|id| format!(r#"<w:footnote w:id="{id}">{cell}</w:footnote>"#)).collect();
+ let footnotes = format!(r#"<w:footnotes {W}>{notes}</w:footnotes>"#);
+ let refs: String = (2..20).map(|id| format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#)).collect();
+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);
+ let i = import(&docx(&doc, "", &footnotes)).unwrap();
+
+ // Every reference is still a mark in the text.
+ assert_eq!(i.story.notes.len(), 18, "all 18 references are kept");
+ // What was retained is the budget plus at most the one note that was already being built
+ // when it ran out -- not all 10.8 MB.
+ let held: usize = i.story.notes.iter().map(|n| note_bytes(&n.text, MAX_NOTE_DEPTH)).sum();
+ assert!(held <= (8 << 20) + big.len() + 1024, "the cell text is inside the budget: {held} bytes");
+ assert!(held > big.len(), "and the notes that fit were imported in full: {held} bytes");
+ assert!(i.warnings.iter().any(|w| w.contains("longer than 8 MB")), "the budget warning is reported: {:?}", i.warnings);
+}
+
+/// The report is one line per distinct footnote id, and the file chooses how many ids it
+/// has: 400 ids referenced twice each meant 400 warnings, every one of them pushed after a
+/// `Vec::contains` scan of all the earlier ones (noted in review of #1584). Both the report
+/// and the set behind it stop at the cap.
+#[test]
+fn the_report_does_not_grow_with_the_document() {
+ let ids = 2..402;
+ // Small notes, so the byte budget is not what stops this -- the ids are.
+ let notes: String = ids.clone().map(|id| format!(r#"<w:footnote w:id="{id}"><w:p><w:r><w:t>note {id}</w:t></w:r></w:p></w:footnote>"#)).collect();
+ let footnotes = format!(r#"<w:footnotes {W}>{notes}</w:footnotes>"#);
+ let refs: String = ids
+ .clone()
+ .flat_map(|id| {
+ let one = format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#);
+ [one.clone(), one]
+ })
+ .collect();
+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);
+ let i = import(&docx(&doc, "", &footnotes)).unwrap();
+
+ // The document is imported in full: every reference is a mark in the text.
+ assert_eq!(i.story.notes.len(), 800, "all 800 references are kept");
+ let duplicates = i.warnings.iter().filter(|w| w.contains("referenced more than once")).count();
+ assert!(duplicates >= 1, "the repeats are reported: {:?}", i.warnings);
+ assert!(i.warnings.len() <= MAX_WARNINGS + 1, "400 duplicated ids do not make 400 warnings: {} warnings", i.warnings.len());
+ // And the last line says the list was cut, rather than ending without explanation.
+ assert!(i.warnings.iter().any(|w| w.contains("the rest are not listed")), "{:?}", i.warnings);
+}
+
+/// When the budget runs out before a note's *first* reference, that reference gets an empty
+/// note -- and must not be recorded as the one that kept the text, or a later reference to the
+/// same id would be told the text is at a reference that has none (noted in review of #1584).
+#[test]
+fn a_note_whose_first_reference_is_past_the_budget_is_not_recorded_as_kept() {
+ // Notes totalling more than the 8 MB budget, so the later ids are never built.
+ let big = "x".repeat(600_000);
+ let notes: String = (2..20).map(|id| format!(r#"<w:footnote w:id="{id}"><w:p><w:r><w:t>{big}</w:t></w:r></w:p></w:footnote>"#)).collect();
+ let footnotes = format!(r#"<w:footnotes {W}>{notes}</w:footnotes>"#);
+ // Each id referenced twice, in order.
+ let refs: String = (2..20)
+ .flat_map(|id| {
+ let one = format!(r#"<w:r><w:footnoteReference w:id="{id}"/></w:r>"#);
+ [one.clone(), one]
+ })
+ .collect();
+ let doc = format!(r#"<w:document {W}><w:body><w:p>{refs}</w:p></w:body></w:document>"#);
+ let i = import(&docx(&doc, "", &footnotes)).unwrap();
+
+ // Every reference is kept, and the budget is respected.
+ assert_eq!(i.story.notes.len(), 36, "all 36 references are in the text");
+ let total: usize = i.story.notes.iter().map(|n| n.text.text.len()).sum();
+ assert!(total <= (8 << 20) + big.len(), "the note text stays inside the budget: {total} bytes");
+
+ // The "referenced more than once" warning is only about ids whose text was actually kept.
+ // A note that was never built cannot have kept its text at a first reference.
+ let kept: std::collections::HashSet<&str> = i.story.notes.iter().filter(|n| !n.text.text.is_empty()).map(|n| n.text.text.as_str()).collect();
+ if i.warnings.iter().any(|w| w.contains("referenced more than once")) {
+ assert!(!kept.is_empty(), "that warning implies some text was kept: {:?}", i.warnings);
+ }
+ assert!(i.warnings.iter().any(|w| w.contains("longer than 8 MB")), "the budget warning is the one this document earns: {:?}", i.warnings);
+}
+
#[test]
fn imports_styles_runs_footnotes_and_tables() {
let styles = format!(
diff --git a/crates/textimport/src/xlsx.rs b/crates/textimport/src/xlsx.rs
index a26665ecc..7f129cce1 100644
--- a/crates/textimport/src/xlsx.rs
+++ b/crates/textimport/src/xlsx.rs
@@ -212,19 +212,41 @@ pub fn import(bytes: &[u8]) -> Result<Imported, ImportError> {
.transpose()?
.map(|sst| sst.els().filter(|e| e.name == "si").map(string_item).collect())
.unwrap_or_default();
+ // A `t="s"` cell holds an index into the shared strings and each one is materialised as its
+ // own `String`, so one large shared string referenced by many cells multiplies: a 1 MB string
+ // and 3M references is terabytes, allocated before the grid cap below ever applies, and an
+ // allocation that large aborts the process rather than failing (#1584). Collection carries a
+ // byte budget and a cell count of its own.
+ const MAX_TEXT_BYTES: usize = 64 << 20;
+ const MAX_COLLECTED_CELLS: usize = 400_000;
let mut cells: Vec<((usize, usize), String)> = Vec::new();
+ let mut text_bytes = 0usize;
+ let mut cut = false;
if let Some(data) = sheet.child("sheetData") {
- for (ri, row) in data.els().filter(|e| e.name == "row").enumerate() {
+ 'rows: for (ri, row) in data.els().filter(|e| e.name == "row").enumerate() {
for (ci, c) in row.els().filter(|e| e.name == "c").enumerate() {
let at = c.attr("r").and_then(cell_ref).unwrap_or((ri, ci));
let v = value(c, &shared);
- if !v.is_empty() {
- cells.push((at, v));
+ if v.is_empty() {
+ continue;
+ }
+ if cells.len() >= MAX_COLLECTED_CELLS || text_bytes.saturating_add(v.len()) > MAX_TEXT_BYTES {
+ cut = true;
+ break 'rows;
}
+ text_bytes = text_bytes.saturating_add(v.len());
+ cells.push((at, v));
}
}
}
let mut warnings = Vec::new();
+ if cut {
+ warnings.push(format!(
+ "the worksheet holds more than {} cells or {} MB of text: the rest was not read",
+ MAX_COLLECTED_CELLS,
+ MAX_TEXT_BYTES / (1 << 20)
+ ));
+ }
if cells.is_empty() {
warnings.push("the worksheet is empty".into());
}
@@ -293,6 +315,50 @@ mod tests {
buf.into_inner()
}
+ /// A `t="s"` cell materialises its own copy of the shared string, so one big string with many
+ /// references multiplies: the audit's case is a 1 MB string and 3M cells, terabytes, built
+ /// before the grid cap applies (#1584). Collection now carries its own byte budget.
+ #[test]
+ fn a_shared_string_referenced_many_times_does_not_multiply() {
+ // A 64 KB shared string referenced 2,000 times would be 128 MB of cell text; the budget
+ // is 64 MB, so collection stops and says so.
+ let big = "y".repeat(64 * 1024);
+ let cells: String = (1..=2_000).map(|r| format!(r#"<row r="{r}"><c r="A{r}" t="s"><v>0</v></c></row>"#)).collect();
+ let mut buf = std::io::Cursor::new(Vec::new());
+ {
+ let mut z = zip::ZipWriter::new(&mut buf);
+ let o = zip::write::SimpleFileOptions::default();
+ for (name, body) in [
+ (
+ "xl/workbook.xml",
+ r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Big" sheetId="1" r:id="rId1"/></sheets></workbook>"#.to_string(),
+ ),
+ (
+ "xl/_rels/workbook.xml.rels",
+ r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="x" Target="worksheets/sheet1.xml"/></Relationships>"#.to_string(),
+ ),
+ ("xl/sharedStrings.xml", format!("<sst><si><t>{big}</t></si></sst>")),
+ ("xl/worksheets/sheet1.xml", format!("<worksheet><sheetData>{cells}</sheetData></worksheet>")),
+ ] {
+ z.start_file(name, o).unwrap();
+ z.write_all(body.as_bytes()).unwrap();
+ }
+ z.finish().unwrap();
+ }
+ let imp = import(&buf.into_inner()).unwrap();
+ let table = imp.story.tables.values().next().expect("a table");
+ let text: usize = (0..table.nrows())
+ .flat_map(|r| (0..table.ncols()).map(move |c| (r, c)))
+ .filter_map(|(r, c)| table.cell(r, c).map(|cell| cell.text.text.len()))
+ .sum();
+ assert!(text <= 64 << 20, "the table holds at most the budget: {text} bytes");
+ assert!(imp.warnings.iter().any(|w| w.contains("was not read")), "the report says the sheet was cut: {:?}", imp.warnings);
+
+ // An ordinary workbook is unaffected and reports nothing of the sort.
+ let plain = import(&workbook()).unwrap();
+ assert!(!plain.warnings.iter().any(|w| w.contains("was not read")), "{:?}", plain.warnings);
+ }
+
#[test]
fn first_sheet_used_range_becomes_a_table() {
let imp = import(&workbook()).unwrap();