World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Preserve ordinary XLSX imports while bounding sparse grid storage (#1255)
Maya Kide5eba4ba6a14
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex 686b59b3..a80d11ce 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -7,6 +7,11 @@ use std::io::{Cursor, Read, Write};use std::process::Command;const SPREADSHEET_ROW_LIMIT: usize = 2_000;+// Each retained cell passes through a 40-byte GridRow entry and two 48-byte renderer+// Cells, besides source values, strings and Vec spare capacity (~370 B/cell in a+// full dense import). Bound actual records/segments, not their empty column spans:+// a 2,000 x 150 sheet uses 300,000 entries and fits without losing any rows.+const XLSX_SEGMENT_LIMIT: usize = 1_000_000;const CSV_RECORD_LIMIT: usize = 10_000;/// 文本解码:chardetng 探测编码(覆盖 GBK/GB18030/BIG5 等中文常见编码)。@@ -773,53 +778,200 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<(String, Vec<crate::ParseWarnlet mut out = String::new();let mut warnings = Vec::new();for sheet_name in workbook.sheet_names() {- let Ok(range) = workbook.worksheet_range(&sheet_name) else {- continue;+ let (text, rows_read, rows_total) = match &mut workbook {+ calamine::Sheets::Xlsx(book) => {+ let Ok(sheet) = xlsx_sheet(book, &sheet_name, XLSX_SEGMENT_LIMIT) else {+ continue;+ };+ sheet+ }+ _ => {+ let Ok(range) = workbook.worksheet_range(&sheet_name) else {+ continue;+ };+ let merges = match &mut workbook {+ calamine::Sheets::Xls(book) => book+ .merge_cells_by_sheet_name(&sheet_name)+ .unwrap_or_default(),+ _ => Vec::new(),+ };+ let (rows, headers) = spreadsheet_grid(&range, &merges);+ let text = crate::table::render_grid_with_headers(&rows, headers)+ .map(|text| format!("{text}\n"))+ .unwrap_or_else(|| {+ let mut text = String::new();+ for row in &rows {+ let cells: Vec<&str> = row.iter().map(|(s, _, _)| s.as_str()).collect();+ text.push_str(&cells.join("\t"));+ text.push('\n');+ }+ text+ });+ (+ text,+ range.height().min(SPREADSHEET_ROW_LIMIT),+ range.height(),+ )+ }};- if range.is_empty() {+ if rows_read == 0 {continue;}- // xlsx/xls 把 merge 单独给出;xlsb/ods 的 calamine reader 没有这一步,按无 merge 读。- let merges = match &mut workbook {- calamine::Sheets::Xlsx(book) => book- .merge_cells_by_sheet_name(&sheet_name)- .unwrap_or_default(),- calamine::Sheets::Xls(book) => book- .merge_cells_by_sheet_name(&sheet_name)- .unwrap_or_default(),- _ => Vec::new(),- };out.push_str(&format!("\n# Sheet: {sheet_name}\n\n"));- let rows_total = range.height();- if rows_total > SPREADSHEET_ROW_LIMIT {- let rows_read = SPREADSHEET_ROW_LIMIT;+ if rows_total > rows_read {warnings.push(crate::ParseWarning {kind: crate::ParseWarning::SPREADSHEET_ROWS_TRUNCATED,detail: serde_json::json!({- "sheet": sheet_name.clone(),+ "sheet": sheet_name,"rows_read": rows_read,"rows_total": rows_total,"rows_omitted": rows_total - rows_read,}),});}- // 每张表按网格渲染成带列头的 Markdown 表;一格字都没有的表退回制表符分隔- let (rows, headers) = spreadsheet_grid(&range, &merges);- match crate::table::render_grid_with_headers(&rows, headers) {- Some(md) => {- out.push_str(&md);- out.push('\n');+ out.push_str(&text);+ }+ Ok((out, warnings))+}++fn xlsx_sheet(+ book: &mut calamine::Xlsx<Cursor<Vec<u8>>>,+ sheet: &str,+ budget: usize,+) -> anyhow::Result<(String, usize, usize)> {+ let mut start_row = u32::MAX;+ let mut end_row = 0;+ let mut row_counts = vec![0usize; SPREADSHEET_ROW_LIMIT];+ let mut reader = book.worksheet_cells_reader(sheet)?;+ // Keep only the first 2,000 physical rows' counts, even if XML cells arrive+ // out of order. The full row bounds survive so truncation remains honest.+ while let Some(cell) = reader.next_cell()? {+ if cell.get_value() == &calamine::DataRef::Empty {+ continue;+ }+ let (row, col) = cell.get_position();+ anyhow::ensure!(+ row < 1_048_576 && col < 16_384,+ "Cell exceeds the Excel worksheet limits"+ );+ if row < start_row {+ let shift = (start_row - row).min(SPREADSHEET_ROW_LIMIT as u32) as usize;+ row_counts.rotate_right(shift);+ row_counts[..shift].fill(0);+ start_row = row;+ }+ end_row = end_row.max(row);+ let offset = (row - start_row) as usize;+ if let Some(count) = row_counts.get_mut(offset) {+ // Repeated coordinates overwrite values below, so even malformed+ // XML cannot retain more than one physical row's worth of cells.+ *count = (*count + 1).min(16_384);+ }+ }+ drop(reader);+ if start_row == u32::MAX {+ return Ok((String::new(), 0, 0));+ }+ let rows_total = (end_row - start_row + 1) as usize;+ let mut height = rows_total.min(SPREADSHEET_ROW_LIMIT);+ let mut count = 0;+ for (row, &cells) in row_counts[..height].iter().enumerate() {+ if cells > budget - count {+ height = row;+ break;+ }+ count += cells;+ }+ // Counts bound this allocation before any source values are retained. A+ // distant note adds one entry, not thousands of empty Data/String objects.+ let mut values = vec![std::collections::BTreeMap::<u32, String>::new(); height];+ let mut reader = book.worksheet_cells_reader(sheet)?;+ while let Some(cell) = reader.next_cell()? {+ let (row, col) = cell.get_position();+ let offset = row.saturating_sub(start_row) as usize;+ if row >= start_row && offset < height && cell.get_value() != &calamine::DataRef::Empty {+ let text = spreadsheet_cell_text(Some(&calamine::Data::from(cell.get_value().clone())));+ values[offset].insert(col, text);+ }+ }+ drop(reader);+ let merges = book.merge_cells_by_sheet_name(sheet).unwrap_or_default();+ loop {+ let start_col = values+ .iter()+ .filter_map(|row| row.keys().next())+ .min()+ .copied()+ .unwrap();+ let end_col = values+ .iter()+ .filter_map(|row| row.keys().next_back())+ .max()+ .copied()+ .unwrap();+ let width = (end_col - start_col + 1) as usize;+ let text_at = |row: usize, col: usize| {+ values[row]+ .get(&(start_col + col as u32))+ .cloned()+ .unwrap_or_default()+ };+ let regions = spreadsheet_regions(+ (start_row, start_col),+ (start_row + height as u32 - 1, end_col),+ &merges,+ text_at,+ );+ let mut owner = vec![usize::MAX; width];+ let mut rows = Vec::with_capacity(height);+ let mut segments = 0;+ for row in 0..height {+ let cells = spreadsheet_row(+ row,+ width,+ ®ions,+ &mut owner,+ |col| text_at(row, col),+ true,+ );+ if cells.len() > budget - segments {+ break;}+ segments += cells.len();+ rows.push(cells);+ }+ if rows.len() != height {+ height = rows.len();+ values.truncate(height);+ // Dropped rows cannot leave a distant column in the kept window's+ // geometry. Reproject merges too, retaining their physical spans.+ continue;+ }+ let headers = spreadsheet_headers(®ions, &rows);+ let text = match crate::table::render_grid_with_headers(&rows, headers) {+ Some(text) => format!("{text}\n"),None => {- for line in &rows {- let cells: Vec<&str> = line.iter().map(|(s, _, _)| s.as_str()).collect();- out.push_str(&cells.join("\t"));- out.push('\n');+ // Preserve the old TSV fallback exactly, but materialize only+ // one dense row at a time rather than the whole rectangle.+ let mut text = String::new();+ for row in 0..height {+ let cells = spreadsheet_row(+ row,+ width,+ ®ions,+ &mut owner,+ |col| text_at(row, col),+ false,+ );+ let cells: Vec<&str> = cells.iter().map(|(s, _, _)| s.as_str()).collect();+ text.push_str(&cells.join("\t"));+ text.push('\n');}+ text}- }+ };+ return Ok((text, height, rows_total));}- Ok((out, warnings))}#[derive(Clone)]@@ -843,9 +995,36 @@ fn spreadsheet_grid(if height == 0 || width == 0 {return (Vec::new(), None);}- let end_row = start_row + height as u32 - 1;- let end_col = start_col + width as u32 - 1;+ let text_at = |row, col| spreadsheet_cell_text(range.get((row, col)));+ let regions = spreadsheet_regions(+ (start_row, start_col),+ (start_row + height as u32 - 1, start_col + width as u32 - 1),+ merges,+ text_at,+ );+ let mut owner = vec![usize::MAX; width];+ let rows: Vec<_> = (0..height)+ .map(|row| {+ spreadsheet_row(+ row,+ width,+ ®ions,+ &mut owner,+ |col| text_at(row, col),+ false,+ )+ })+ .collect();+ let headers = spreadsheet_headers(®ions, &rows);+ (rows, headers)+}+fn spreadsheet_regions(+ (start_row, start_col): (u32, u32),+ (end_row, end_col): (u32, u32),+ merges: &[calamine::Dimensions],+ text_at: impl Fn(usize, usize) -> String,+) -> Vec<GridMerge> {let mut regions = Vec::new();for merge in merges {if merge.start.0 > merge.end.0@@ -868,30 +1047,40 @@ fn spreadsheet_grid(regions.push(GridMerge {start,end,- text: spreadsheet_cell_text(range.get(start)),+ text: text_at(start.0, start.1),});}+ regions+}- let mut owner = vec![usize::MAX; height * width];+fn spreadsheet_row(+ row: usize,+ width: usize,+ regions: &[GridMerge],+ owner: &mut [usize],+ text_at: impl Fn(usize) -> String,+ compact_empty: bool,+) -> GridRow {+ // One owner row keeps the old last-merge-wins projection and anchor jumps,+ // without allocating height * width indices. XLSX bounds width at XFD.+ owner.fill(usize::MAX);for (index, region) in regions.iter().enumerate() {- for row in region.start.0..=region.end.0 {- for col in region.start.1..=region.end.1 {- owner[row * width + col] = index;- }+ if region.start.0 <= row && row <= region.end.0 {+ owner[region.start.1..=region.end.1].fill(index);}}-- let mut rows: Vec<GridRow> = Vec::with_capacity(height);- for row in 0..height {- let mut cells = GridRow::new();- let mut col = 0usize;- while col < width {- let index = owner[row * width + col];- if index == usize::MAX {- cells.push((spreadsheet_cell_text(range.get((row, col))), 1, 0));- col += 1;- continue;+ let mut cells: GridRow = Vec::new();+ let mut col = 0;+ while col < width {+ let index = owner[col];+ if index == usize::MAX {+ let text = text_at(col);+ match (compact_empty && text.is_empty(), cells.last_mut()) {+ (true, Some((previous, span, _))) if previous.is_empty() => *span += 1,+ _ => cells.push((text, 1, 0)),}+ col += 1;+ } else {let region = ®ions[index];if col == region.start.1 {cells.push((region.text.clone(), region.end.1 - region.start.1 + 1, 0));@@ -900,11 +1089,8 @@ fn spreadsheet_grid(col += 1;}}- rows.push(cells);}-- let headers = spreadsheet_headers(®ions, &rows);- (rows, headers)+ cells}fn spreadsheet_cell_text(cell: Option<&calamine::Data>) -> String {@@ -1205,6 +1391,95 @@ fn pptx_xml_to_text(xml: &str) -> anyhow::Result<String> {mod tests {use super::*;+ fn budget_book(sheet_data: &str, merges: &str) -> calamine::Xlsx<Cursor<Vec<u8>>> {+ let worksheet = format!(+ r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData>{merges}</worksheet>"#+ );+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));+ for (name, xml) in [+ (+ "[Content_Types].xml",+ r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/></Types>"#,+ ),+ (+ "_rels/.rels",+ r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#,+ ),+ (+ "xl/workbook.xml",+ r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Budget" sheetId="1" r:id="rId1"/></sheets></workbook>"#,+ ),+ (+ "xl/_rels/workbook.xml.rels",+ r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#,+ ),+ ("xl/worksheets/sheet1.xml", &worksheet),+ ] {+ zip.start_file(name, zip::write::SimpleFileOptions::default())+ .unwrap();+ zip.write_all(xml.as_bytes()).unwrap();+ }+ calamine::open_workbook_from_rs(Cursor::new(zip.finish().unwrap().into_inner())).unwrap()+ }++ #[test]+ fn a_segment_budget_keeps_only_complete_rows() {+ // Four source cells fit; two rows of three emitted segments do not.+ let rows = r#"<row r="1"><c r="A1" t="inlineStr"><is><t>left</t></is></c><c r="C1" t="inlineStr"><is><t>right</t></is></c></row><row r="2"><c r="A2"><v>1</v></c><c r="C2"><v>2</v></c></row>"#;+ let (_, rows_read, rows_total) =+ xlsx_sheet(&mut budget_book(rows, ""), "Budget", 5).unwrap();+ assert_eq!((rows_read, rows_total), (1, 2));+ }++ #[test]+ fn dropping_a_row_reprojects_the_retained_title_and_merged_span() {+ let title = r#"<row r="1"><c r="B1" t="inlineStr"><is><t>Title</t></is></c></row>"#;+ let rest = r#"<row r="2"><c r="A2" t="inlineStr"><is><t xml:space="preserve"> </t></is></c><c r="C2" t="inlineStr"><is><t xml:space="preserve"> </t></is></c></row><row r="3"><c r="Z3" t="inlineStr"><is><t>note</t></is></c></row>"#;+ let merges = r#"<mergeCells count="1"><mergeCell ref="B1:C1"/></mergeCells>"#;+ // Four source cells fit, but the first two rows use 3 + 4 segments.+ let (text, rows_read, rows_total) = xlsx_sheet(+ &mut budget_book(&format!("{title}{rest}"), merges),+ "Budget",+ 5,+ )+ .unwrap();+ let (expected, _, _) = xlsx_sheet(&mut budget_book(title, merges), "Budget", 5).unwrap();+ assert_eq!((rows_read, rows_total), (1, 3));+ assert!(text.contains("Title"));+ assert_eq!(text, expected);+ }++ #[test]+ fn empty_runs_keep_the_physical_cell_and_merge_coordinates() {+ let regions = [GridMerge {+ start: (0, 2),+ end: (0, 4),+ text: "Merged".into(),+ }];+ let row = spreadsheet_row(+ 0,+ 10,+ ®ions,+ &mut [usize::MAX; 10],+ |col| match col {+ 0 => "Item".into(),+ 7 => "3".into(),+ _ => String::new(),+ },+ true,+ );+ let expected = [+ ("Item", 1),+ ("", 1),+ ("Merged", 3),+ ("", 2),+ ("3", 1),+ ("", 2),+ ]+ .map(|(text, span)| (text.to_owned(), span, 0));+ assert_eq!(row, expected);+ }+const DOC: &str = r#"<?xml version="1.0"?><w:document xmlns:w="x"><w:body><w:p><w:r><w:t>Segment results</w:t></w:r></w:p><w:tbl><w:tr><w:tc><w:p><w:r><w:t>Segment</w:t></w:r></w:p></w:tc><w:tc><w:tcPr><w:gridSpan w:val="2"/></w:tcPr><w:p><w:r><w:t>Revenue</w:t></w:r></w:p></w:tc></w:tr>diff --git a/crates/utopia-ingest/tests/spreadsheet_truncation.rs b/crates/utopia-ingest/tests/spreadsheet_truncation.rsindex e4f8b7cd..d4df2fd2 100644--- a/crates/utopia-ingest/tests/spreadsheet_truncation.rs+++ b/crates/utopia-ingest/tests/spreadsheet_truncation.rs@@ -4,7 +4,7 @@use std::fmt::Write as _;use std::io::{Cursor, Write};-fn workbook(data_rows: usize) -> Vec<u8> {+fn ledger_rows(data_rows: usize) -> String {let mut sheet_data = String::from(r#"<row r="1"><c r="A1" t="inlineStr"><is><t>id</t></is></c><c r="B1" t="inlineStr"><is><t>name</t></is></c></row>"#,);@@ -14,17 +14,69 @@ fn workbook(data_rows: usize) -> Vec<u8> {r#"<row r="{row}"><c r="A{row}"><v>{id}</v></c><c r="B{row}" t="inlineStr"><is><t>customer-{id}</t></is></c></row>"#));}- let worksheet = format!(- r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#+ sheet_data+}++fn workbook(data_rows: usize) -> Vec<u8> {+ workbook_xml(&ledger_rows(data_rows))+}++fn workbook_xml(sheet_data: &str) -> Vec<u8> {+ workbook_sheets(&[("Ledger", sheet_data)])+}++fn workbook_sheets(sheets: &[(&str, &str)]) -> Vec<u8> {+ workbook_sheets_with_chart(sheets, false, "")+}++fn workbook_sheets_with_chart(+ sheets: &[(&str, &str)],+ chart: bool,+ worksheet_suffix: &str,+) -> Vec<u8> {+ let mut content_types = String::from(+ r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/>"#,);+ let mut sheet_entries = String::new();+ let mut sheet_rels = String::new();+ let mut parts = Vec::new();+ for (i, (name, sheet_data)) in sheets.iter().enumerate() {+ let id = i + 1;+ write!(content_types, r#"<Override PartName="/xl/worksheets/sheet{id}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/>"#).unwrap();+ write!(+ sheet_entries,+ r#"<sheet name="{name}" sheetId="{id}" r:id="rId{id}"/>"#+ )+ .unwrap();+ write!(sheet_rels, r#"<Relationship Id="rId{id}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet{id}.xml"/>"#).unwrap();+ parts.push((format!("xl/worksheets/sheet{id}.xml"), format!(r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData>{worksheet_suffix}</worksheet>"#)));+ }+ if chart {+ let id = sheets.len() + 1;+ write!(content_types, r#"<Override PartName="/xl/chartsheets/sheet{id}.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.chartsheet+xml"/><Override PartName="/xl/drawings/drawing1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawing+xml"/><Override PartName="/xl/charts/chart1.xml" ContentType="application/vnd.openxmlformats-officedocument.drawingml.chart+xml"/>"#).unwrap();+ write!(+ sheet_entries,+ r#"<sheet name="Chart" sheetId="{id}" r:id="rId{id}"/>"#+ )+ .unwrap();+ write!(sheet_rels, r#"<Relationship Id="rId{id}" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/chartsheet" Target="chartsheets/sheet{id}.xml"/>"#).unwrap();+ parts.extend([+ (format!("xl/chartsheets/sheet{id}.xml"), r#"<chartsheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheetViews><sheetView workbookViewId="0"/></sheetViews><drawing r:id="rId1"/></chartsheet>"#.into()),+ (format!("xl/chartsheets/_rels/sheet{id}.xml.rels"), r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/drawing" Target="../drawings/drawing1.xml"/></Relationships>"#.into()),+ ("xl/drawings/drawing1.xml".into(), r#"<xdr:wsDr xmlns:xdr="http://schemas.openxmlformats.org/drawingml/2006/spreadsheetDrawing" xmlns:a="http://schemas.openxmlformats.org/drawingml/2006/main"><xdr:absoluteAnchor><xdr:pos x="0" y="0"/><xdr:ext cx="3600000" cy="2400000"/><xdr:graphicFrame macro=""><xdr:nvGraphicFramePr><xdr:cNvPr id="1" name="Chart 1"/><xdr:cNvGraphicFramePr/></xdr:nvGraphicFramePr><xdr:xfrm><a:off x="0" y="0"/><a:ext cx="0" cy="0"/></xdr:xfrm><a:graphic><a:graphicData uri="http://schemas.openxmlformats.org/drawingml/2006/chart"><c:chart xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships" r:id="rId1"/></a:graphicData></a:graphic></xdr:graphicFrame><xdr:clientData/></xdr:absoluteAnchor></xdr:wsDr>"#.into()),+ ("xl/drawings/_rels/drawing1.xml.rels".into(), r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/chart" Target="../charts/chart1.xml"/></Relationships>"#.into()),+ ("xl/charts/chart1.xml".into(), r#"<c:chartSpace xmlns:c="http://schemas.openxmlformats.org/drawingml/2006/chart"><c:chart><c:plotArea><c:layout/><c:pieChart><c:varyColors val="1"/><c:ser><c:idx val="0"/><c:order val="0"/><c:tx><c:v>Series 1</c:v></c:tx><c:cat><c:strLit><c:ptCount val="1"/><c:pt idx="0"><c:v>Item</c:v></c:pt></c:strLit></c:cat><c:val><c:numLit><c:formatCode>General</c:formatCode><c:ptCount val="1"/><c:pt idx="0"><c:v>1</c:v></c:pt></c:numLit></c:val></c:ser></c:pieChart></c:plotArea><c:plotVisOnly val="1"/></c:chart></c:chartSpace>"#.into()),+ ]);+ }+ content_types.push_str("</Types>");+ parts.extend([+ ("[Content_Types].xml".into(), content_types),+ ("_rels/.rels".into(), r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.into()),+ ("xl/workbook.xml".into(), format!(r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets>{sheet_entries}</sheets></workbook>"#)),+ ("xl/_rels/workbook.xml.rels".into(), format!(r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships">{sheet_rels}</Relationships>"#)),+ ]);let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));- for (part, xml) in [- ("[Content_Types].xml", r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/></Types>"#.to_string()),- ("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),- ("xl/workbook.xml", r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships"><sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#.to_string()),- ("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/></Relationships>"#.to_string()),- ("xl/worksheets/sheet1.xml", worksheet),- ] {+ for (part, xml) in parts {zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap();zip.write_all(xml.as_bytes()).unwrap();@@ -32,6 +84,23 @@ fn workbook(data_rows: usize) -> Vec<u8> {zip.finish().unwrap().into_inner()}+fn words(cell: &str, text: &str) -> String {+ format!(r#"<c r="{cell}" t="inlineStr"><is><t>{text}</t></is></c>"#)+}++fn column(mut col: usize) -> String {+ let mut letters = Vec::new();+ loop {+ letters.push(b'A' + (col % 26) as u8);+ if col < 26 {+ break;+ }+ col = col / 26 - 1;+ }+ letters.reverse();+ String::from_utf8(letters).unwrap()+}+fn csv(records: usize) -> String {let mut csv = String::from("id,name\n");for id in 1..=records {@@ -78,6 +147,126 @@ fn a_spreadsheet_at_the_row_cap_has_no_warning() {assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);}+#[test]+fn a_distant_cell_past_the_row_window_does_not_shrink_the_retained_ledger() {+ let mut rows = ledger_rows(1_999);+ rows.push_str(&format!(+ r#"<row r="1000000">{}</row>"#,+ words("P1000000", "omitted-distant-note")+ ));+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();+ assert!(parsed.text.contains("| 1999 | customer-1999 |"));+ assert!(!parsed.text.contains("omitted-distant-note"));+ assert_eq!(parsed.warnings.len(), 1);+ assert_eq!(parsed.warnings[0].kind, "spreadsheet.rows_truncated");+ assert_eq!(+ parsed.warnings[0].detail,+ serde_json::json!({+ "sheet": "Ledger", "rows_read": 2000,+ "rows_total": 1000000, "rows_omitted": 998000,+ })+ );+}++#[test]+fn a_header_note_beyond_empty_columns_does_not_truncate_a_ledger() {+ let rows = ledger_rows(1_500);+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();+ assert!(expected.text.contains("| 1500 | customer-1500 |"));+ for reference in ["AZZ1", "XFD1"] {+ let with_note = rows.replacen(+ "</row>",+ &format!("{}</row>", words(reference, "header-only-note")),+ 1,+ );+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&with_note)).unwrap();+ assert!(+ parsed.text == expected.text,+ "note at {reference} changed the ledger"+ );+ assert!(+ parsed.warnings.is_empty(),+ "{reference}: {:?}",+ parsed.warnings+ );+ }+}++#[test]+fn an_ordinary_dense_sheet_retains_two_thousand_rows_and_one_hundred_fifty_columns() {+ let columns: Vec<_> = (0..150).map(column).collect();+ let mut rows = String::from(r#"<row r="1">"#);+ for (col, name) in columns.iter().enumerate() {+ rows.push_str(&words(&format!("{name}1"), &format!("column-{col}")));+ }+ rows.push_str("</row>");+ for row in 2..=2_000 {+ write!(+ rows,+ r#"<row r="{row}">{}"#,+ words(&format!("A{row}"), &format!("customer-{}", row - 1))+ )+ .unwrap();+ for (col, name) in columns.iter().enumerate().skip(1) {+ write!(+ rows,+ r#"<c r="{name}{row}"><v>{}</v></c>"#,+ row * 1_000 + col+ )+ .unwrap();+ }+ rows.push_str("</row>");+ }+ let parsed = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);+ let last = parsed+ .text+ .lines()+ .find(|line| line.starts_with("| customer-1999 |"))+ .expect("the final row is retained");+ assert!(last.ends_with("| 2000149 |"), "the last column is retained");+ assert_eq!(last.split('|').count(), 152, "all 150 columns are present");+}++#[test]+fn a_broken_sheet_does_not_prevent_reading_the_other_sheets() {+ let good = ledger_rows(1);+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&good)).unwrap();+ let broken = r#"<row r="1"><c r="A1" t="n"><v>not-a-number</v></c></row>"#;+ let bytes = workbook_sheets(&[("Ledger", &good), ("Broken", broken)]);+ let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap();+ assert_eq!(parsed.text, expected.text);+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);+}++#[test]+fn unreadable_merges_do_not_prevent_reading_the_worksheet_cells() {+ use calamine::Reader as _;++ let rows = ledger_rows(1);+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();+ let merges = r#"<mergeCells count="1"><mergeCell ref="not-a-range"/></mergeCells>"#;+ let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], false, merges);+ let mut book: calamine::Xlsx<_> =+ calamine::open_workbook_from_rs(Cursor::new(bytes.clone())).unwrap();+ assert_eq!(book.worksheet_range("Ledger").unwrap().height(), 2);+ assert!(book.merge_cells_by_sheet_name("Ledger").is_err());++ let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap();+ assert_eq!(parsed.text, expected.text);+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);+}++#[test]+fn a_chartsheet_does_not_prevent_reading_the_worksheets() {+ let rows = ledger_rows(1);+ let expected = utopia_ingest::parse("ledger.xlsx", &workbook_xml(&rows)).unwrap();+ let bytes = workbook_sheets_with_chart(&[("Ledger", &rows)], true, "");+ let parsed = utopia_ingest::parse("ledger.xlsx", &bytes).unwrap();+ assert_eq!(parsed.text, expected.text);+ assert!(parsed.warnings.is_empty(), "{:?}", parsed.warnings);+}+#[test]fn a_csv_past_the_record_cap_reports_the_records_it_left_out() {let parsed = utopia_ingest::parse("customers.csv", csv(10_000).as_bytes()).unwrap();