Skip to content

World's first open-source enterprise world model.

Preserve ordinary XLSX imports while bounding sparse grid storage (#1255)

Maya Kide5eba4ba6a14
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 686b59b3..a80d11ce 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -7,6 +7,11 @@ use std::io::{Cursor, Read, Write};
use std::process::Command;
const SPREADSHEET_ROW_LIMIT: usize = 2_000;
+// Each retained cell passes through a 40-byte GridRow entry and two 48-byte renderer
+// Cells, besides source values, strings and Vec spare capacity (~370 B/cell in a
+// full dense import). Bound actual records/segments, not their empty column spans:
+// a 2,000 x 150 sheet uses 300,000 entries and fits without losing any rows.
+const XLSX_SEGMENT_LIMIT: usize = 1_000_000;
const CSV_RECORD_LIMIT: usize = 10_000;
/// 文本解码:chardetng 探测编码(覆盖 GBK/GB18030/BIG5 等中文常见编码)。
@@ -773,53 +778,200 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<(String, Vec<crate::ParseWarn
let mut out = String::new();
let mut warnings = Vec::new();
for sheet_name in workbook.sheet_names() {
- let Ok(range) = workbook.worksheet_range(&sheet_name) else {
- continue;
+ let (text, rows_read, rows_total) = match &mut workbook {
+ calamine::Sheets::Xlsx(book) => {
+ let Ok(sheet) = xlsx_sheet(book, &sheet_name, XLSX_SEGMENT_LIMIT) else {
+ continue;
+ };
+ sheet
+ }
+ _ => {
+ let Ok(range) = workbook.worksheet_range(&sheet_name) else {
+ continue;
+ };
+ let merges = match &mut workbook {
+ calamine::Sheets::Xls(book) => book
+ .merge_cells_by_sheet_name(&sheet_name)
+ .unwrap_or_default(),
+ _ => Vec::new(),
+ };
+ let (rows, headers) = spreadsheet_grid(&range, &merges);
+ let text = crate::table::render_grid_with_headers(&rows, headers)
+ .map(|text| format!("{text}\n"))
+ .unwrap_or_else(|| {
+ let mut text = String::new();
+ for row in &rows {
+ let cells: Vec<&str> = row.iter().map(|(s, _, _)| s.as_str()).collect();
+ text.push_str(&cells.join("\t"));
+ text.push('\n');
+ }
+ text
+ });
+ (
+ text,
+ range.height().min(SPREADSHEET_ROW_LIMIT),
+ range.height(),
+ )
+ }
};
- if range.is_empty() {
+ if rows_read == 0 {
continue;
}
- // xlsx/xls 把 merge 单独给出;xlsb/ods 的 calamine reader 没有这一步,按无 merge 读。
- let merges = match &mut workbook {
- calamine::Sheets::Xlsx(book) => book
- .merge_cells_by_sheet_name(&sheet_name)
- .unwrap_or_default(),
- calamine::Sheets::Xls(book) => book
- .merge_cells_by_sheet_name(&sheet_name)
- .unwrap_or_default(),
- _ => Vec::new(),
- };
out.push_str(&format!("\n# Sheet: {sheet_name}\n\n"));
- let rows_total = range.height();
- if rows_total > SPREADSHEET_ROW_LIMIT {
- let rows_read = SPREADSHEET_ROW_LIMIT;
+ if rows_total > rows_read {
warnings.push(crate::ParseWarning {
kind: crate::ParseWarning::SPREADSHEET_ROWS_TRUNCATED,
detail: serde_json::json!({
- "sheet": sheet_name.clone(),
+ "sheet": sheet_name,
"rows_read": rows_read,
"rows_total": rows_total,
"rows_omitted": rows_total - rows_read,
}),
});
}
- // 每张表按网格渲染成带列头的 Markdown 表;一格字都没有的表退回制表符分隔
- let (rows, headers) = spreadsheet_grid(&range, &merges);
- match crate::table::render_grid_with_headers(&rows, headers) {
- Some(md) => {
- out.push_str(&md);
- out.push('\n');
+ out.push_str(&text);
+ }
+ Ok((out, warnings))
+}
+
+fn xlsx_sheet(
+ book: &mut calamine::Xlsx<Cursor<Vec<u8>>>,
+ sheet: &str,
+ budget: usize,
+) -> anyhow::Result<(String, usize, usize)> {
+ let mut start_row = u32::MAX;
+ let mut end_row = 0;
+ let mut row_counts = vec![0usize; SPREADSHEET_ROW_LIMIT];
+ let mut reader = book.worksheet_cells_reader(sheet)?;
+ // Keep only the first 2,000 physical rows' counts, even if XML cells arrive
+ // out of order. The full row bounds survive so truncation remains honest.
+ while let Some(cell) = reader.next_cell()? {
+ if cell.get_value() == &calamine::DataRef::Empty {
+ continue;
+ }
+ let (row, col) = cell.get_position();
+ anyhow::ensure!(
+ row < 1_048_576 && col < 16_384,
+ "Cell exceeds the Excel worksheet limits"
+ );
+ if row < start_row {
+ let shift = (start_row - row).min(SPREADSHEET_ROW_LIMIT as u32) as usize;
+ row_counts.rotate_right(shift);
+ row_counts[..shift].fill(0);
+ start_row = row;
+ }
+ end_row = end_row.max(row);
+ let offset = (row - start_row) as usize;
+ if let Some(count) = row_counts.get_mut(offset) {
+ // Repeated coordinates overwrite values below, so even malformed
+ // XML cannot retain more than one physical row's worth of cells.
+ *count = (*count + 1).min(16_384);
+ }
+ }
+ drop(reader);
+ if start_row == u32::MAX {
+ return Ok((String::new(), 0, 0));
+ }
+ let rows_total = (end_row - start_row + 1) as usize;
+ let mut height = rows_total.min(SPREADSHEET_ROW_LIMIT);
+ let mut count = 0;
+ for (row, &cells) in row_counts[..height].iter().enumerate() {
+ if cells > budget - count {
+ height = row;
+ break;
+ }
+ count += cells;
+ }
+ // Counts bound this allocation before any source values are retained. A
+ // distant note adds one entry, not thousands of empty Data/String objects.
+ let mut values = vec![std::collections::BTreeMap::<u32, String>::new(); height];
+ let mut reader = book.worksheet_cells_reader(sheet)?;
+ while let Some(cell) = reader.next_cell()? {
+ let (row, col) = cell.get_position();
+ let offset = row.saturating_sub(start_row) as usize;
+ if row >= start_row && offset < height && cell.get_value() != &calamine::DataRef::Empty {
+ let text = spreadsheet_cell_text(Some(&calamine::Data::from(cell.get_value().clone())));
+ values[offset].insert(col, text);
+ }
+ }
+ drop(reader);
+ let merges = book.merge_cells_by_sheet_name(sheet).unwrap_or_default();
+ loop {
+ let start_col = values
+ .iter()
+ .filter_map(|row| row.keys().next())
+ .min()
+ .copied()
+ .unwrap();
+ let end_col = values
+ .iter()
+ .filter_map(|row| row.keys().next_back())
+ .max()
+ .copied()
+ .unwrap();
+ let width = (end_col - start_col + 1) as usize;
+ let text_at = |row: usize, col: usize| {
+ values[row]
+ .get(&(start_col + col as u32))
+ .cloned()
+ .unwrap_or_default()
+ };
+ let regions = spreadsheet_regions(
+ (start_row, start_col),
+ (start_row + height as u32 - 1, end_col),
+ &merges,
+ text_at,
+ );
+ let mut owner = vec![usize::MAX; width];
+ let mut rows = Vec::with_capacity(height);
+ let mut segments = 0;
+ for row in 0..height {
+ let cells = spreadsheet_row(
+ row,
+ width,
+ &regions,
+ &mut owner,
+ |col| text_at(row, col),
+ true,
+ );
+ if cells.len() > budget - segments {
+ break;
}
+ segments += cells.len();
+ rows.push(cells);
+ }
+ if rows.len() != height {
+ height = rows.len();
+ values.truncate(height);
+ // Dropped rows cannot leave a distant column in the kept window's
+ // geometry. Reproject merges too, retaining their physical spans.
+ continue;
+ }
+ let headers = spreadsheet_headers(&regions, &rows);
+ let text = match crate::table::render_grid_with_headers(&rows, headers) {
+ Some(text) => format!("{text}\n"),
None => {
- for line in &rows {
- let cells: Vec<&str> = line.iter().map(|(s, _, _)| s.as_str()).collect();
- out.push_str(&cells.join("\t"));
- out.push('\n');
+ // Preserve the old TSV fallback exactly, but materialize only
+ // one dense row at a time rather than the whole rectangle.
+ let mut text = String::new();
+ for row in 0..height {
+ let cells = spreadsheet_row(
+ row,
+ width,
+ &regions,
+ &mut owner,
+ |col| text_at(row, col),
+ false,
+ );
+ let cells: Vec<&str> = cells.iter().map(|(s, _, _)| s.as_str()).collect();
+ text.push_str(&cells.join("\t"));
+ text.push('\n');
}
+ text
}
- }
+ };
+ return Ok((text, height, rows_total));
}
- Ok((out, warnings))
}
#[derive(Clone)]
@@ -843,9 +995,36 @@ fn spreadsheet_grid(
if height == 0 || width == 0 {
return (Vec::new(), None);
}
- let end_row = start_row + height as u32 - 1;
- let end_col = start_col + width as u32 - 1;
+ let text_at = |row, col| spreadsheet_cell_text(range.get((row, col)));
+ let regions = spreadsheet_regions(
+ (start_row, start_col),
+ (start_row + height as u32 - 1, start_col + width as u32 - 1),
+ merges,
+ text_at,
+ );
+ let mut owner = vec![usize::MAX; width];
+ let rows: Vec<_> = (0..height)
+ .map(|row| {
+ spreadsheet_row(
+ row,
+ width,
+ &regions,
+ &mut owner,
+ |col| text_at(row, col),
+ false,
+ )
+ })
+ .collect();
+ let headers = spreadsheet_headers(&regions, &rows);
+ (rows, headers)
+}
+fn spreadsheet_regions(
+ (start_row, start_col): (u32, u32),
+ (end_row, end_col): (u32, u32),
+ merges: &[calamine::Dimensions],
+ text_at: impl Fn(usize, usize) -> String,
+) -> Vec<GridMerge> {
let mut regions = Vec::new();
for merge in merges {
if merge.start.0 > merge.end.0
@@ -868,30 +1047,40 @@ fn spreadsheet_grid(
regions.push(GridMerge {
start,
end,
- text: spreadsheet_cell_text(range.get(start)),
+ text: text_at(start.0, start.1),
});
}
+ regions
+}
- let mut owner = vec![usize::MAX; height * width];
+fn spreadsheet_row(
+ row: usize,
+ width: usize,
+ regions: &[GridMerge],
+ owner: &mut [usize],
+ text_at: impl Fn(usize) -> String,
+ compact_empty: bool,
+) -> GridRow {
+ // One owner row keeps the old last-merge-wins projection and anchor jumps,
+ // without allocating height * width indices. XLSX bounds width at XFD.
+ owner.fill(usize::MAX);
for (index, region) in regions.iter().enumerate() {
- for row in region.start.0..=region.end.0 {
- for col in region.start.1..=region.end.1 {
- owner[row * width + col] = index;
- }
+ if region.start.0 <= row && row <= region.end.0 {
+ owner[region.start.1..=region.end.1].fill(index);
}
}
-
- let mut rows: Vec<GridRow> = Vec::with_capacity(height);
- for row in 0..height {
- let mut cells = GridRow::new();
- let mut col = 0usize;
- while col < width {
- let index = owner[row * width + col];
- if index == usize::MAX {
- cells.push((spreadsheet_cell_text(range.get((row, col))), 1, 0));
- col += 1;
- continue;
+ let mut cells: GridRow = Vec::new();
+ let mut col = 0;
+ while col < width {
+ let index = owner[col];
+ if index == usize::MAX {
+ let text = text_at(col);
+ match (compact_empty && text.is_empty(), cells.last_mut()) {
+ (true, Some((previous, span, _))) if previous.is_empty() => *span += 1,
+ _ => cells.push((text, 1, 0)),
}
+ col += 1;
+ } else {
let region = &regions[index];
if col == region.start.1 {
cells.push((region.text.clone(), region.end.1 - region.start.1 + 1, 0));
@@ -900,11 +1089,8 @@ fn spreadsheet_grid(
col += 1;
}
}
- rows.push(cells);
}
-
- let headers = spreadsheet_headers(&regions, &rows);
- (rows, headers)
+ cells
}
fn spreadsheet_cell_text(cell: Option<&calamine::Data>) -> String {
@@ -1205,6 +1391,95 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result<String> {
mod tests {
use super::*;
+ fn budget_book(sheet_data: &str, merges: &str) -> calamine::Xlsx<Cursor<Vec<u8>>> {
+ let worksheet = format!(
+ r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData>{merges}</worksheet>"#
+ );
+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+ for (name, xml) in [
+ (
+ "[Content_Types].xml",
+ r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>"#,
+ ),
+ (
+ "_rels/.rels",
+ r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#,
+ ),
+ (
+ "xl/workbook.xml",
+ r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Budget" sheetId="1" r:id="rId1"/></sheets></workbook>"#,
+ ),
+ (
+ "xl/_rels/workbook.xml.rels",
+ r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#,
+ ),
+ ("xl/worksheets/sheet1.xml", &worksheet),
+ ] {
+ zip.start_file(name, zip::write::SimpleFileOptions::default())
+ .unwrap();
+ zip.write_all(xml.as_bytes()).unwrap();
+ }
+ calamine::open_workbook_from_rs(Cursor::new(zip.finish().unwrap().into_inner())).unwrap()
+ }
+
+ #[test]
+ fn a_segment_budget_keeps_only_complete_rows() {
+ // Four source cells fit; two rows of three emitted segments do not.
+ let rows = r#"<row r="1"><c r="A1" t="inlineStr"><is><t>left</t></is></c><c r="C1" t="inlineStr"><is><t>right</t></is></c></row><row r="2"><c r="A2"><v>1</v></c><c r="C2"><v>2</v></c></row>"#;
+ let (_, rows_read, rows_total) =
+ xlsx_sheet(&mut budget_book(rows, ""), "Budget", 5).unwrap();
+ assert_eq!((rows_read, rows_total), (1, 2));
+ }
+
+ #[test]
+ fn dropping_a_row_reprojects_the_retained_title_and_merged_span() {
+ let title = r#"<row r="1"><c r="B1" t="inlineStr"><is><t>Title</t></is></c></row>"#;
+ let rest = r#"<row r="2"><c r="A2" t="inlineStr"><is><t xml:space="preserve"> </t></is></c><c r="C2" t="inlineStr"><is><t xml:space="preserve"> </t></is></c></row><row r="3"><c r="Z3" t="inlineStr"><is><t>note</t></is></c></row>"#;
+ let merges = r#"<mergeCells count="1"><mergeCell ref="B1:C1"/></mergeCells>"#;
+ // Four source cells fit, but the first two rows use 3 + 4 segments.
+ let (text, rows_read, rows_total) = xlsx_sheet(
+ &mut budget_book(&format!("{title}{rest}"), merges),
+ "Budget",
+ 5,
+ )
+ .unwrap();
+ let (expected, _, _) = xlsx_sheet(&mut budget_book(title, merges), "Budget", 5).unwrap();
+ assert_eq!((rows_read, rows_total), (1, 3));
+ assert!(text.contains("Title"));
+ assert_eq!(text, expected);
+ }
+
+ #[test]
+ fn empty_runs_keep_the_physical_cell_and_merge_coordinates() {
+ let regions = [GridMerge {
+ start: (0, 2),
+ end: (0, 4),
+ text: "Merged".into(),
+ }];
+ let row = spreadsheet_row(
+ 0,
+ 10,
+ &regions,
+ &mut [usize::MAX; 10],
+ |col| match col {
+ 0 => "Item".into(),
+ 7 => "3".into(),
+ _ => String::new(),
+ },
+ true,
+ );
+ let expected = [
+ ("Item", 1),
+ ("", 1),
+ ("Merged", 3),
+ ("", 2),
+ ("3", 1),
+ ("", 2),
+ ]
+ .map(|(text, span)| (text.to_owned(), span, 0));
+ assert_eq!(row, expected);
+ }
+
const DOC: &str = r#"<?xml version="1.0"?><w:document xmlns:w="x"><w:body>
<w:p><w:r><w:t>Segment results</w:t></w:r></w:p>
<w:tbl><w:tr><w:tc><w:p><w:r><w:t>Segment</w:t></w:r></w:p></w:tc><w:tc><w:tcPr><w:gridSpan w:val="2"/></w:tcPr><w:p><w:r><w:t>Revenue</w:t></w:r></w:p></w:tc></w:tr>
diff --git a/crates/utopia-ingest/tests/spreadsheet_truncation.rs b/crates/utopia-ingest/tests/spreadsheet_truncation.rs
index e4f8b7cd..d4df2fd2 100644
--- a/crates/utopia-ingest/tests/spreadsheet_truncation.rs
+++ b/crates/utopia-ingest/tests/spreadsheet_truncation.rs
@@ -4,7 +4,7 @@
use std::fmt::Write as _;
use std::io::{Cursor, Write};
-fn workbook(data_rows: usize) -> Vec<u8> {
+fn ledger_rows(data_rows: usize) -> String {
let mut sheet_data = String::from(
r#"<row r="1"><c r="A1" t="inlineStr"><is><t>id</t></is></c><c r="B1" t="inlineStr"><is><t>name</t></is></c></row>"#,
);
@@ -14,17 +14,69 @@ fn workbook(data_rows: usize) -> Vec<u8> {
r#"<row r="{row}"><c r="A{row}"><v>{id}</v></c><c r="B{row}" t="inlineStr"><is><t>customer-{id}</t></is></c></row>"#
));
}
- let worksheet = format!(
- r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#
+ sheet_data
+}
+
+fn workbook(data_rows: usize) -> Vec<u8> {
+ workbook_xml(&ledger_rows(data_rows))
+}
+
+fn workbook_xml(sheet_data: &str) -> Vec<u8> {
+ workbook_sheets(&[("Ledger", sheet_data)])
+}
+
+fn workbook_sheets(sheets: &[(&str, &str)]) -> Vec<u8> {
+ workbook_sheets_with_chart(sheets, false, "")
+}
+
+fn workbook_sheets_with_chart(
+ sheets: &[(&str, &str)],
+ chart: bool,
+ worksheet_suffix: &str,
+) -> Vec<u8> {
+ let mut content_types = String::from(
+ r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>"#,
);
+ let mut sheet_entries = String::new();
+ let mut sheet_rels = String::new();
+ let mut parts = Vec::new();
+ for (i, (name, sheet_data)) in sheets.iter().enumerate() {
+ let id = i + 1;
+ write!(content_types, r#"<Override PartName="/xl/worksheets/sheet{id}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>"#).unwrap();
+ write!(
+ sheet_entries,
+ r#"<sheet name="{name}" sheetId="{id}" r:id="rId{id}"/>"#
+ )
+ .unwrap();
+ write!(sheet_rels, r#"<Relationship Id="rId{id}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet{id}.xml"/>"#).unwrap();
+ parts.push((format!("xl/worksheets/sheet{id}.xml"), format!(r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData>{worksheet_suffix}</worksheet>"#)));
+ }
+ if chart {
+ let id = sheets.len() + 1;
+ write!(content_types, r#"<Override PartName="/xl/chartsheets/sheet{id}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml"/><Override PartName="/xl/drawings/drawing1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawing+xml"/><Override PartName="/xl/charts/chart1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawingml.chart+xml"/>"#).unwrap();
+ write!(
+ sheet_entries,
+ r#"<sheet name="Chart" sheetId="{id}" r:id="rId{id}"/>"#
+ )
+ .unwrap();
+ write!(sheet_rels, r#"<Relationship Id="rId{id}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/chartsheet" Target="chartsheets/sheet{id}.xml"/>"#).unwrap();
+ parts.extend([
+ (format!("xl/chartsheets/sheet{id}.xml"), r#"<chartsheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheetViews><sheetView workbookViewId="0"/></sheetViews><drawing r:id="rId1"/></chartsheet>"#.into()),
+ (format!("xl/chartsheets/_rels/sheet{id}.xml.rels"), r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/drawing" Target="../drawings/drawing1.xml"/></Relationships>"#.into()),
+ ("xl/drawings/drawing1.xml".into(), r#"<xdr:wsDr xmlns:xdr="http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><xdr:absoluteAnchor><xdr:pos x="0" y="0"/><xdr:ext cx="3600000" cy="2400000"/><xdr:graphicFrame macro=""><xdr:nvGraphicFramePr><xdr:cNvPr id="1" name="Chart 1"/><xdr:cNvGraphicFramePr/></xdr:nvGraphicFramePr><xdr:xfrm><a:off x="0" y="0"/><a:ext cx="0" cy="0"/></xdr:xfrm><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/chart"><c:chart xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" r:id="rId1"/></a:graphicData></a:graphic></xdr:graphicFrame><xdr:clientData/></xdr:absoluteAnchor></xdr:wsDr>"#.into()),
+ ("xl/drawings/_rels/drawing1.xml.rels".into(), r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/chart" Target="../charts/chart1.xml"/></Relationships>"#.into()),
+ ("xl/charts/chart1.xml".into(), r#"<c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart"><c:chart><c:plotArea><c:layout/><c:pieChart><c:varyColors val="1"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:cat><c:strLit><c:ptCount val="1"/><c:pt idx="0"><c:v>Item</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="1"/><c:pt idx="0"><c:v>1</c:v></c:pt></c:numLit></c:val></c:ser></c:pieChart></c:plotArea><c:plotVisOnly val="1"/></c:chart></c:chartSpace>"#.into()),
+ ]);
+ }
+ content_types.push_str("</Types>");
+ parts.extend([
+ ("[Content_Types].xml".into(), content_types),
+ ("_rels/.rels".into(), r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.into()),
+ ("xl/workbook.xml".into(), format!(r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>{sheet_entries}</sheets></workbook>"#)),
+ ("xl/_rels/workbook.xml.rels".into(), format!(r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">{sheet_rels}</Relationships>"#)),
+ ]);
let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
- for (part, xml) in [
- ("[Content_Types].xml", r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>"#.to_string()),
- ("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),
- ("xl/workbook.xml", r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#.to_string()),
- ("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#.to_string()),
- ("xl/worksheets/sheet1.xml", worksheet),
- ] {
+ for (part, xml) in parts {
zip.start_file(part, zip::write::SimpleFileOptions::default())
.unwrap();
zip.write_all(xml.as_bytes()).unwrap();
@@ -32,6 +84,23 @@ fn workbook(data_rows: usize) -> Vec<u8> {
zip.finish().unwrap().into_inner()
}
+fn words(cell: &str, text: &str) -> String {
+ format!(r#"<c r="{cell}" t="inlineStr"><is><t>{text}</t></is></c>"#)
+}
+
+fn column(mut col: usize) -> String {
+ let mut letters = Vec::new();
+ loop {
+ letters.push(b'A' + (col % 26) as u8);
+ if col < 26 {
+ break;
+ }
+ col = col / 26 - 1;
+ }
+ letters.reverse();
+ String::from_utf8(letters).unwrap()
+}
+
fn csv(records: usize) -> String {
let mut csv = String::from("id,name\n");
for id in 1..=records {
@@ -78,6 +147,126 @@ fn a_spreadsheet_at_the_row_cap_has_no_warning() {
assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
}
+#[test]
+fn a_distant_cell_past_the_row_window_does_not_shrink_the_retained_ledger() {
+ let mut rows = ledger_rows(1_999);
+ rows.push_str(&format!(
+ r#"<row r="1000000">{}</row>"#,
+ words("P1000000", "omitted-distant-note")
+ ));
+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();
+ assert!(parsed.text.contains("| 1999 | customer-1999 |"));
+ assert!(!parsed.text.contains("omitted-distant-note"));
+ assert_eq!(parsed.warnings.len(), 1);
+ assert_eq!(parsed.warnings[0].kind, "spreadsheet.rows_truncated");
+ assert_eq!(
+ parsed.warnings[0].detail,
+ serde_json::json!({
+ "sheet": "Ledger", "rows_read": 2000,
+ "rows_total": 1000000, "rows_omitted": 998000,
+ })
+ );
+}
+
+#[test]
+fn a_header_note_beyond_empty_columns_does_not_truncate_a_ledger() {
+ let rows = ledger_rows(1_500);
+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();
+ assert!(expected.text.contains("| 1500 | customer-1500 |"));
+ for reference in ["AZZ1", "XFD1"] {
+ let with_note = rows.replacen(
+ "</row>",
+ &format!("{}</row>", words(reference, "header-only-note")),
+ 1,
+ );
+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&with_note)).unwrap();
+ assert!(
+ parsed.text == expected.text,
+ "note at {reference} changed the ledger"
+ );
+ assert!(
+ parsed.warnings.is_empty(),
+ "{reference}: {:?}",
+ parsed.warnings
+ );
+ }
+}
+
+#[test]
+fn an_ordinary_dense_sheet_retains_two_thousand_rows_and_one_hundred_fifty_columns() {
+ let columns: Vec<_> = (0..150).map(column).collect();
+ let mut rows = String::from(r#"<row r="1">"#);
+ for (col, name) in columns.iter().enumerate() {
+ rows.push_str(&words(&format!("{name}1"), &format!("column-{col}")));
+ }
+ rows.push_str("</row>");
+ for row in 2..=2_000 {
+ write!(
+ rows,
+ r#"<row r="{row}">{}"#,
+ words(&format!("A{row}"), &format!("customer-{}", row - 1))
+ )
+ .unwrap();
+ for (col, name) in columns.iter().enumerate().skip(1) {
+ write!(
+ rows,
+ r#"<c r="{name}{row}"><v>{}</v></c>"#,
+ row * 1_000 + col
+ )
+ .unwrap();
+ }
+ rows.push_str("</row>");
+ }
+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();
+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
+ let last = parsed
+ .text
+ .lines()
+ .find(|line| line.starts_with("| customer-1999 |"))
+ .expect("the final row is retained");
+ assert!(last.ends_with("| 2000149 |"), "the last column is retained");
+ assert_eq!(last.split('|').count(), 152, "all 150 columns are present");
+}
+
+#[test]
+fn a_broken_sheet_does_not_prevent_reading_the_other_sheets() {
+ let good = ledger_rows(1);
+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&good)).unwrap();
+ let broken = r#"<row r="1"><c r="A1" t="n"><v>not-a-number</v></c></row>"#;
+ let bytes = workbook_sheets(&[("Ledger", &good), ("Broken", broken)]);
+ let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap();
+ assert_eq!(parsed.text, expected.text);
+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
+}
+
+#[test]
+fn unreadable_merges_do_not_prevent_reading_the_worksheet_cells() {
+ use calamine::Reader as _;
+
+ let rows = ledger_rows(1);
+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();
+ let merges = r#"<mergeCells count="1"><mergeCell ref="not-a-range"/></mergeCells>"#;
+ let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], false, merges);
+ let mut book: calamine::Xlsx<_> =
+ calamine::open_workbook_from_rs(Cursor::new(bytes.clone())).unwrap();
+ assert_eq!(book.worksheet_range("Ledger").unwrap().height(), 2);
+ assert!(book.merge_cells_by_sheet_name("Ledger").is_err());
+
+ let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap();
+ assert_eq!(parsed.text, expected.text);
+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
+}
+
+#[test]
+fn a_chartsheet_does_not_prevent_reading_the_worksheets() {
+ let rows = ledger_rows(1);
+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();
+ let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], true, "");
+ let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap();
+ assert_eq!(parsed.text, expected.text);
+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);
+}
+
#[test]
fn a_csv_past_the_record_cap_reports_the_records_it_left_out() {
let parsed = utopia_ingest::parse("customers.csv", csv(10_000).as_bytes()).unwrap();