Skip to content

World's first open-source enterprise world model.

Recognize East Asian Excel dates with calamine 0.37.0 (#1312)

Daniel2bcc763140de
diff --git a/Cargo.lock b/Cargo.lock
index f7a8637e..123cae83 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -524,9 +524,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"
[[package]]
name = "calamine"
-version = "0.36.1"
+version = "0.37.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3"
+checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb"
dependencies = [
"atoi_simd",
"byteorder",
@@ -534,9 +534,9 @@ dependencies = [
"encoding_rs",
"fast-float2",
"log",
- "quick-xml 0.41.0",
+ "quick-xml 0.42.0",
"serde",
- "zip 8.6.0",
+ "zip 9.0.5",
]
[[package]]
@@ -3423,6 +3423,7 @@ version = "0.42.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"
dependencies = [
+ "encoding_rs",
"memchr",
]
@@ -6293,14 +6294,15 @@ dependencies = [
[[package]]
name = "zip"
-version = "8.6.0"
+version = "9.0.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"
+checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"
dependencies = [
"crc32fast",
"flate2",
"indexmap",
"memchr",
+ "nix",
"typed-path",
"zopfli",
]
diff --git a/Cargo.toml b/Cargo.toml
index 5b65de56..fc845702 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -66,7 +66,7 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] }
tiktoken-rs = "0.12"
# 句子边界按 Unicode 标准(UAX #29)切,不自己写标点规则
pdf-extract = "0.12"
-calamine = "0.36"
+calamine = "0.37"
pgvector = { version = "0.4", features = ["sqlx"] }
# `gzip` 不是可选项:一次嵌入回 64×1024 个浮点数,写成 JSON 文本每个数字占
# 21 字节而信息量只有 4 字节,一次响应 1.33 MB。实测同一条链路上不压缩 62 秒、
diff --git a/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md
new file mode 100644
index 00000000..16a30bcb
--- /dev/null
+++ b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md
@@ -0,0 +1,23 @@
+The MIT License (MIT)
+
+Copyright (c) 2016 Johann Tuffe
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
diff --git a/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls
new file mode 100644
index 00000000..6c6decfa
Binary files /dev/null and b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls differ
diff --git a/crates/utopia-ingest/tests/spreadsheet_dates.rs b/crates/utopia-ingest/tests/spreadsheet_dates.rs
index ec7cd2ae..ef23c576 100644
--- a/crates/utopia-ingest/tests/spreadsheet_dates.rs
+++ b/crates/utopia-ingest/tests/spreadsheet_dates.rs
@@ -11,7 +11,7 @@ use std::io::{Cursor, Write};
// 3 date and time (22), 4 time of day (20), 5 elapsed time (46), 6 "#,##0.00" (4).
const STYLES: &str = r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><numFmts count="1"><numFmt numFmtId="176" formatCode="yyyy&quot;年&quot;m&quot;月&quot;d&quot;日&quot;"/></numFmts><cellXfs count="7"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/><xf numFmtId="14" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="176" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="22" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="20" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="46" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="4" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/></cellXfs></styleSheet>"#;
-fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {
+fn workbook(rows: &[Vec<String>], date1904: bool, styles: &str) -> Vec<u8> {
let sheet_data: String = rows
.iter()
.enumerate()
@@ -28,7 +28,7 @@ fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {
("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),
("xl/workbook.xml", format!(r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">{workbook_pr}<sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#)),
("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#.to_string()),
- ("xl/styles.xml", STYLES.to_string()),
+ ("xl/styles.xml", styles.to_string()),
("xl/worksheets/sheet1.xml", format!(r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#)),
] {
zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap();
@@ -38,7 +38,7 @@ fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {
}
fn read(rows: &[Vec<String>], date1904: bool) -> String {
- utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904))
+ utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904, STYLES))
.unwrap()
.text
}
@@ -141,3 +141,94 @@ fn a_workbook_counting_from_1904_reads_the_same_date() {
"{text}"
);
}
+
+fn read_format(format_id: u16, value: &str, date1904: bool, format_code: Option<&str>) -> String {
+ let custom = format_code.map_or_else(String::new, |code| {
+ let code = quick_xml::escape::escape(code);
+ format!(
+ r#"<numFmts count="1"><numFmt numFmtId="{format_id}" formatCode="{code}"/></numFmts>"#
+ )
+ });
+ let styles = format!(
+ r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">{custom}<cellXfs count="1"><xf numFmtId="{format_id}" applyNumberFormat="1"/></cellXfs></styleSheet>"#
+ );
+ let rows = [
+ header(1),
+ vec![
+ words("A2", "Value"),
+ number("B2", 0, value),
+ words("C2", "ok"),
+ ],
+ ];
+ utopia_ingest::parse("ledger.xlsx", &workbook(&rows, date1904, &styles))
+ .unwrap()
+ .text
+}
+
+#[test]
+fn east_asian_builtin_dates_need_no_custom_format_definition() {
+ for format_id in [27, 31, 50, 58] {
+ for (date1904, serial) in [(false, "45306"), (true, "43844")] {
+ let text = read_format(format_id, serial, date1904, None);
+ assert!(
+ text.contains("| Value | 2024-01-15 | ok |"),
+ "format {format_id}: {text}"
+ );
+ }
+ }
+}
+
+#[test]
+fn east_asian_fractional_clock_times_and_explicit_formats_are_preserved() {
+ // calamine does not retain clock-only metadata; a serial >= 1 still gains a date.
+ for format_id in [32, 33] {
+ let text = read_format(format_id, "0.5", false, None);
+ assert!(
+ text.contains("| Value | 12:00 | ok |"),
+ "format {format_id}: {text}"
+ );
+ }
+ for (format_id, code, serial, expected) in [
+ (27, "0.00", "45306", "45306"),
+ (32, "yyyy-mm-dd", "45306", "2024-01-15"),
+ (27, "[h]:mm:ss", "1.5104166666666667", "36:15:00"),
+ ] {
+ let text = read_format(format_id, serial, false, Some(code));
+ assert!(
+ text.contains(&format!("| Value | {expected} | ok |")),
+ "{text}"
+ );
+ }
+}
+
+#[test]
+fn east_asian_builtin_formats_are_normalized_in_xls() {
+ // Fixture from calamine PR #734, commit d93c5a463399a5554d287ebda0ed1d2a43377ec2,
+ // tests/date_east_asian.xls (MIT; license alongside). Rows 0..19 are IDs 27..36/50..58;
+ // each row stores 45306, 0.5, 45306.123456789 with no explicit format definition.
+ let text = utopia_ingest::parse(
+ "ledger.xls",
+ include_bytes!("fixtures/calamine/date_east_asian.xls"),
+ )
+ .unwrap()
+ .text;
+ let rows: Vec<_> = text
+ .lines()
+ .filter(|line| line.starts_with("| ") && !line.starts_with("| ---"))
+ .collect();
+ assert_eq!(rows.len(), 28, "{text}");
+ for row in [0, 4, 10, 18] {
+ // formats 27, 31, 50, 58
+ assert!(rows[row].starts_with("| 2024-01-15 |"), "row {row}: {text}");
+ }
+ for row in [5, 6] {
+ // formats 32, 33
+ assert_eq!(
+ rows[row].split('|').nth(2).unwrap().trim(),
+ "12:00",
+ "{text}"
+ );
+ }
+ assert!(rows[19].starts_with("| 45306 | 0.5 |"), "{text}"); // General
+ assert!(rows[24].starts_with("| 2024-01-15 |"), "{text}"); // Chinese custom date
+}