World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A date in a spreadsheet is read as the date it shows, not as Excel's day count (#1017)
Maya Kid13bbb06c83e2
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex 62b33720..7e537a9f 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -561,6 +561,7 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {.map(|c| {let text = match c {Data::Empty => String::new(),+ Data::DateTime(d) => excel_date(d),other => other.to_string(),};(text, 1, 0)@@ -586,6 +587,49 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {Ok(out)}+/// 日期格按它显示的样子写。+///+/// calamine 按格子挂的数字格式认出了日期,交来的却还是 Excel 存的那个数:从 1899-12-30+/// 起的天数(1904 纪年的工作簿从 1904-01-01 起),小数部分是一天里的时刻。`Data` 的+/// Display 原样写这个数,2024-01-15 就成了 45306——文档里没有这个日期了,时间抽取看不见+/// 它,模型只当它是个量;同一天在 1904 纪年的工作簿里还是另一个数(43844)。+///+/// 写成 ISO:整天只写日期,带时刻的加上时刻,整数部分是 0 的只写时刻(`h:mm` 一类格式),+/// 累计时长(`[h]:mm:ss`)写累计的时分秒。显示用的格式 calamine 不交出来,所以只显示年月+/// 的格子也写到日。出了 Excel 日历的数照原样写+fn excel_date(d: &calamine::ExcelDateTime) -> String {+ let value = d.as_f64();+ if d.is_duration() {+ let seconds = (value.abs() * 86_400.0).round() as u64;+ let sign = if value < 0.0 { "-" } else { "" };+ return format!(+ "{sign}{}:{:02}:{:02}",+ seconds / 3600,+ seconds / 60 % 60,+ seconds % 60+ );+ }+ // Excel 的日历止于 9999-12-31(序数 2958465);负数它自己也只显示 ####+ if !(0.0..2_958_466.0).contains(&value) {+ return value.to_string();+ }+ let (year, month, day, hour, minute, second, _) = d.to_ymd_hms_milli();+ let time = if second == 0 {+ format!("{hour:02}:{minute:02}")+ } else {+ format!("{hour:02}:{minute:02}:{second:02}")+ };+ if value < 1.0 {+ return time;+ }+ let date = format!("{year:04}-{month:02}-{day:02}");+ if (hour, minute, second) == (0, 0, 0) {+ date+ } else {+ format!("{date} {time}")+ }+}+/// Decode before conversion so legacy HTML encodings remain supported.pub fn html(bytes: &[u8]) -> anyhow::Result<String> {Ok(crate::html::page_to_markdown(&plain_text(bytes), None)?)diff --git a/crates/utopia-ingest/tests/spreadsheet_dates.rs b/crates/utopia-ingest/tests/spreadsheet_dates.rsnew file mode 100644index 00000000..ec7cd2ae--- /dev/null+++ b/crates/utopia-ingest/tests/spreadsheet_dates.rs@@ -0,0 +1,143 @@+//! 电子表格里的日期格读成它显示的日期,不是 Excel 的序数。+//!+//! 照 Excel 写出来的样子拼一份 xlsx:日期存成从 1899-12-30 起的天数(`<v>45306</v>`),+//! 小数部分是一天里的时刻;它是日期,只因为 styles.xml 里挂的数字格式是日期格式——内置的+//! 14(短日期)、22(日期加时刻)、20(时刻)、46(累计时长),和中文 Excel 自定义的+//! `yyyy"年"m"月"d"日"`。++use std::io::{Cursor, Write};++// cellXfs by index: 0 General, 1 short date (14), 2 a Chinese custom date (176),+// 3 date and time (22), 4 time of day (20), 5 elapsed time (46), 6 "#,##0.00" (4).+const STYLES: &str = r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><numFmts count="1"><numFmt numFmtId="176" formatCode="yyyy"年"m"月"d"日""/></numFmts><cellXfs count="7"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/><xf numFmtId="14" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="176" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="22" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="20" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="46" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="4" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/></cellXfs></styleSheet>"#;++fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {+ let sheet_data: String = rows+ .iter()+ .enumerate()+ .map(|(i, cells)| format!(r#"<row r="{}">{}</row>"#, i + 1, cells.concat()))+ .collect();+ let workbook_pr = if date1904 {+ r#"<workbookPr date1904="1"/>"#+ } else {+ "<workbookPr/>"+ };+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));+ for (part, xml) in [+ ("[Content_Types].xml", r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/></Types>"#.to_string()),+ ("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),+ ("xl/workbook.xml", format!(r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">{workbook_pr}<sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#)),+ ("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#.to_string()),+ ("xl/styles.xml", STYLES.to_string()),+ ("xl/worksheets/sheet1.xml", format!(r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#)),+ ] {+ zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap();+ zip.write_all(xml.as_bytes()).unwrap();+ }+ zip.finish().unwrap().into_inner()+}++fn read(rows: &[Vec<String>], date1904: bool) -> String {+ utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904))+ .unwrap()+ .text+}++fn words(cell: &str, text: &str) -> String {+ format!(r#"<c r="{cell}" t="inlineStr"><is><t>{text}</t></is></c>"#)+}++fn number(cell: &str, style: u8, value: &str) -> String {+ format!(r#"<c r="{cell}" s="{style}"><v>{value}</v></c>"#)+}++fn header(row: u32) -> Vec<String> {+ vec![+ words(&format!("A{row}"), "Item"),+ words(&format!("B{row}"), "When"),+ words(&format!("C{row}"), "Amount"),+ ]+}++#[test]+fn a_date_cell_reads_as_the_date_it_shows() {+ let text = read(+ &[+ header(1),+ vec![+ words("A2", "Short date"),+ number("B2", 1, "45306"),+ number("C2", 6, "1200"),+ ],+ vec![+ words("A3", "Chinese date"),+ number("B3", 2, "45307"),+ number("C3", 6, "800"),+ ],+ // 没挂日期格式的数就是数,哪怕它碰巧落在日期的范围里+ vec![+ words("A4", "Plain number"),+ number("B4", 0, "45306"),+ number("C4", 6, "5"),+ ],+ ],+ false,+ );+ assert!(+ text.contains("| Short date | 2024-01-15 | 1200 |"),+ "{text}"+ );+ assert!(+ text.contains("| Chinese date | 2024-01-16 | 800 |"),+ "{text}"+ );+ assert!(text.contains("| Plain number | 45306 | 5 |"), "{text}");+}++#[test]+fn a_time_of_day_and_an_elapsed_time_read_as_clock_time() {+ let text = read(+ &[+ header(1),+ vec![+ words("A2", "Opened"),+ number("B2", 3, "45306.395833333336"),+ number("C2", 6, "1"),+ ],+ vec![+ words("A3", "Daily call"),+ number("B3", 4, "0.39583333333333331"),+ number("C3", 6, "2"),+ ],+ vec![+ words("A4", "Downtime"),+ number("B4", 5, "1.5104166666666667"),+ number("C4", 6, "3"),+ ],+ ],+ false,+ );+ assert!(text.contains("| Opened | 2024-01-15 09:30 | 1 |"), "{text}");+ assert!(text.contains("| Daily call | 09:30 | 2 |"), "{text}");+ assert!(text.contains("| Downtime | 36:15:00 | 3 |"), "{text}");+}++#[test]+fn a_workbook_counting_from_1904_reads_the_same_date() {+ // Excel for Mac 从前默认 1904 纪年:同一天存成少 1462 的数+ let text = read(+ &[+ header(1),+ vec![+ words("A2", "Short date"),+ number("B2", 1, "43844"),+ number("C2", 6, "1200"),+ ],+ ],+ true,+ );+ assert!(+ text.contains("| Short date | 2024-01-15 | 1200 |"),+ "{text}"+ );+}diff --git a/docs/design/sources.md b/docs/design/sources.mdindex 00f478be..856d3ad9 100644--- a/docs/design/sources.md+++ b/docs/design/sources.md@@ -54,8 +54,9 @@ path with the first row promoted to header. Measured on the NVDA earnings releasbecame 32, none of them a table without its header. DOCX tables, spreadsheets and CSV files are laid on the same grid by their parsers (#750): a DOCXcell keeps its column span and the indent of its first paragraph, a sheet's or a CSV file's firstrow with two or more cells is its header even when the headings are years, and a table with no-figure in it is read as a header row followed by records; the text layer of a PDF carries no table-structure.+figure in it is read as a header row followed by records; a sheet's date cell is written as the+date it shows (ISO, with its time of day, or an elapsed time as hours:minutes:seconds), not as+Excel's day count; the text layer of a PDF carries no table structure.**Origin and anchor.** `chunks.origin` (stated, ocr, transcribed, described), `origin_model`, and an`anchor` whose shape is checked per origin (page and box; start, end and speakers; page and image or