Skip to content

World's first open-source enterprise world model.

A date in a spreadsheet is read as the date it shows, not as Excel's day count (#1017)

Maya Kid13bbb06c83e2
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rs
index 62b33720..7e537a9f 100644
--- a/crates/utopia-ingest/src/parsers.rs
+++ b/crates/utopia-ingest/src/parsers.rs
@@ -561,6 +561,7 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {
.map(|c| {
let text = match c {
Data::Empty => String::new(),
+ Data::DateTime(d) => excel_date(d),
other => other.to_string(),
};
(text, 1, 0)
@@ -586,6 +587,49 @@ pub fn spreadsheet(bytes: &[u8]) -> anyhow::Result<String> {
Ok(out)
}
+/// 日期格按它显示的样子写。
+///
+/// calamine 按格子挂的数字格式认出了日期,交来的却还是 Excel 存的那个数:从 1899-12-30
+/// 起的天数(1904 纪年的工作簿从 1904-01-01 起),小数部分是一天里的时刻。`Data` 的
+/// Display 原样写这个数,2024-01-15 就成了 45306——文档里没有这个日期了,时间抽取看不见
+/// 它,模型只当它是个量;同一天在 1904 纪年的工作簿里还是另一个数(43844)。
+///
+/// 写成 ISO:整天只写日期,带时刻的加上时刻,整数部分是 0 的只写时刻(`h:mm` 一类格式),
+/// 累计时长(`[h]:mm:ss`)写累计的时分秒。显示用的格式 calamine 不交出来,所以只显示年月
+/// 的格子也写到日。出了 Excel 日历的数照原样写
+fn excel_date(d: &calamine::ExcelDateTime) -> String {
+ let value = d.as_f64();
+ if d.is_duration() {
+ let seconds = (value.abs() * 86_400.0).round() as u64;
+ let sign = if value < 0.0 { "-" } else { "" };
+ return format!(
+ "{sign}{}:{:02}:{:02}",
+ seconds / 3600,
+ seconds / 60 % 60,
+ seconds % 60
+ );
+ }
+ // Excel 的日历止于 9999-12-31(序数 2958465);负数它自己也只显示 ####
+ if !(0.0..2_958_466.0).contains(&value) {
+ return value.to_string();
+ }
+ let (year, month, day, hour, minute, second, _) = d.to_ymd_hms_milli();
+ let time = if second == 0 {
+ format!("{hour:02}:{minute:02}")
+ } else {
+ format!("{hour:02}:{minute:02}:{second:02}")
+ };
+ if value < 1.0 {
+ return time;
+ }
+ let date = format!("{year:04}-{month:02}-{day:02}");
+ if (hour, minute, second) == (0, 0, 0) {
+ date
+ } else {
+ format!("{date} {time}")
+ }
+}
+
/// Decode before conversion so legacy HTML encodings remain supported.
pub fn html(bytes: &[u8]) -> anyhow::Result<String> {
Ok(crate::html::page_to_markdown(&plain_text(bytes), None)?)
diff --git a/crates/utopia-ingest/tests/spreadsheet_dates.rs b/crates/utopia-ingest/tests/spreadsheet_dates.rs
new file mode 100644
index 00000000..ec7cd2ae
--- /dev/null
+++ b/crates/utopia-ingest/tests/spreadsheet_dates.rs
@@ -0,0 +1,143 @@
+//! 电子表格里的日期格读成它显示的日期,不是 Excel 的序数。
+//!
+//! 照 Excel 写出来的样子拼一份 xlsx:日期存成从 1899-12-30 起的天数(`<v>45306</v>`),
+//! 小数部分是一天里的时刻;它是日期,只因为 styles.xml 里挂的数字格式是日期格式——内置的
+//! 14(短日期)、22(日期加时刻)、20(时刻)、46(累计时长),和中文 Excel 自定义的
+//! `yyyy"年"m"月"d"日"`。
+
+use std::io::{Cursor, Write};
+
+// cellXfs by index: 0 General, 1 short date (14), 2 a Chinese custom date (176),
+// 3 date and time (22), 4 time of day (20), 5 elapsed time (46), 6 "#,##0.00" (4).
+const STYLES: &str = r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><numFmts count="1"><numFmt numFmtId="176" formatCode="yyyy&quot;年&quot;m&quot;月&quot;d&quot;日&quot;"/></numFmts><cellXfs count="7"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/><xf numFmtId="14" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="176" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="22" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="20" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="46" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="4" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/></cellXfs></styleSheet>"#;
+
+fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {
+ let sheet_data: String = rows
+ .iter()
+ .enumerate()
+ .map(|(i, cells)| format!(r#"<row r="{}">{}</row>"#, i + 1, cells.concat()))
+ .collect();
+ let workbook_pr = if date1904 {
+ r#"<workbookPr date1904="1"/>"#
+ } else {
+ "<workbookPr/>"
+ };
+ let mut zip = zip::ZipWriter::new(Cursor::new(Vec::new()));
+ for (part, xml) in [
+ ("[Content_Types].xml", r#"<Types xmlns="http://schemas.openxmlformats.org/package/2006/content-types"><Default Extension="rels" ContentType="application/vnd.openxmlformats-package.relationships+xml"/><Default Extension="xml" ContentType="application/xml"/><Override PartName="/xl/workbook.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.sheet.main+xml"/><Override PartName="/xl/worksheets/sheet1.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.worksheet+xml"/><Override PartName="/xl/styles.xml" ContentType="application/vnd.openxmlformats-officedocument.spreadsheetml.styles+xml"/></Types>"#.to_string()),
+ ("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),
+ ("xl/workbook.xml", format!(r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">{workbook_pr}<sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#)),
+ ("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#.to_string()),
+ ("xl/styles.xml", STYLES.to_string()),
+ ("xl/worksheets/sheet1.xml", format!(r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#)),
+ ] {
+ zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap();
+ zip.write_all(xml.as_bytes()).unwrap();
+ }
+ zip.finish().unwrap().into_inner()
+}
+
+fn read(rows: &[Vec<String>], date1904: bool) -> String {
+ utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904))
+ .unwrap()
+ .text
+}
+
+fn words(cell: &str, text: &str) -> String {
+ format!(r#"<c r="{cell}" t="inlineStr"><is><t>{text}</t></is></c>"#)
+}
+
+fn number(cell: &str, style: u8, value: &str) -> String {
+ format!(r#"<c r="{cell}" s="{style}"><v>{value}</v></c>"#)
+}
+
+fn header(row: u32) -> Vec<String> {
+ vec![
+ words(&format!("A{row}"), "Item"),
+ words(&format!("B{row}"), "When"),
+ words(&format!("C{row}"), "Amount"),
+ ]
+}
+
+#[test]
+fn a_date_cell_reads_as_the_date_it_shows() {
+ let text = read(
+ &[
+ header(1),
+ vec![
+ words("A2", "Short date"),
+ number("B2", 1, "45306"),
+ number("C2", 6, "1200"),
+ ],
+ vec![
+ words("A3", "Chinese date"),
+ number("B3", 2, "45307"),
+ number("C3", 6, "800"),
+ ],
+ // 没挂日期格式的数就是数,哪怕它碰巧落在日期的范围里
+ vec![
+ words("A4", "Plain number"),
+ number("B4", 0, "45306"),
+ number("C4", 6, "5"),
+ ],
+ ],
+ false,
+ );
+ assert!(
+ text.contains("| Short date | 2024-01-15 | 1200 |"),
+ "{text}"
+ );
+ assert!(
+ text.contains("| Chinese date | 2024-01-16 | 800 |"),
+ "{text}"
+ );
+ assert!(text.contains("| Plain number | 45306 | 5 |"), "{text}");
+}
+
+#[test]
+fn a_time_of_day_and_an_elapsed_time_read_as_clock_time() {
+ let text = read(
+ &[
+ header(1),
+ vec![
+ words("A2", "Opened"),
+ number("B2", 3, "45306.395833333336"),
+ number("C2", 6, "1"),
+ ],
+ vec![
+ words("A3", "Daily call"),
+ number("B3", 4, "0.39583333333333331"),
+ number("C3", 6, "2"),
+ ],
+ vec![
+ words("A4", "Downtime"),
+ number("B4", 5, "1.5104166666666667"),
+ number("C4", 6, "3"),
+ ],
+ ],
+ false,
+ );
+ assert!(text.contains("| Opened | 2024-01-15 09:30 | 1 |"), "{text}");
+ assert!(text.contains("| Daily call | 09:30 | 2 |"), "{text}");
+ assert!(text.contains("| Downtime | 36:15:00 | 3 |"), "{text}");
+}
+
+#[test]
+fn a_workbook_counting_from_1904_reads_the_same_date() {
+ // Excel for Mac 从前默认 1904 纪年:同一天存成少 1462 的数
+ let text = read(
+ &[
+ header(1),
+ vec![
+ words("A2", "Short date"),
+ number("B2", 1, "43844"),
+ number("C2", 6, "1200"),
+ ],
+ ],
+ true,
+ );
+ assert!(
+ text.contains("| Short date | 2024-01-15 | 1200 |"),
+ "{text}"
+ );
+}
diff --git a/docs/design/sources.md b/docs/design/sources.md
index 00f478be..856d3ad9 100644
--- a/docs/design/sources.md
+++ b/docs/design/sources.md
@@ -54,8 +54,9 @@ path with the first row promoted to header. Measured on the NVDA earnings releas
became 32, none of them a table without its header. DOCX tables, spreadsheets and CSV files are laid on the same grid by their parsers (#750): a DOCX
cell keeps its column span and the indent of its first paragraph, a sheet's or a CSV file's first
row with two or more cells is its header even when the headings are years, and a table with no
-figure in it is read as a header row followed by records; the text layer of a PDF carries no table
-structure.
+figure in it is read as a header row followed by records; a sheet's date cell is written as the
+date it shows (ISO, with its time of day, or an elapsed time as hours:minutes:seconds), not as
+Excel's day count; the text layer of a PDF carries no table structure.
**Origin and anchor.** `chunks.origin` (stated, ocr, transcribed, described), `origin_model`, and an
`anchor` whose shape is checked per origin (page and box; start, end and speakers; page and image or