World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
Recognize East Asian Excel dates with calamine 0.37.0 (#1312)
Daniel2bcc763140de
diff --git a/Cargo.lock b/Cargo.lockindex f7a8637e..123cae83 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -524,9 +524,9 @@ checksum = "fc652a48c352aef3ea3aed32080501cf3ef6ed5da78602a020c991775b0aff04"[[package]]name = "calamine"-version = "0.36.1"+version = "0.37.0"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "5fa68281b1a76b54a62156474adb06bb380a67e07dd60656e3217152b42183f3"+checksum = "c9ca341b94bc4b29e08970b1a03914fad2dfe177e2ecb67fcd6de9046004aafb"dependencies = ["atoi_simd","byteorder",@@ -534,9 +534,9 @@ dependencies = ["encoding_rs","fast-float2","log",- "quick-xml 0.41.0",+ "quick-xml 0.42.0","serde",- "zip 8.6.0",+ "zip 9.0.5",][[package]]@@ -3423,6 +3423,7 @@ version = "0.42.0"source = "registry+https://github.com/rust-lang/crates.io-index"checksum = "41b1177fdf999d2321d3fb46ff47159d9c1fb9ad66a4879f8c50a0b504615e9b"dependencies = [+ "encoding_rs","memchr",]@@ -6293,14 +6294,15 @@ dependencies = [[[package]]name = "zip"-version = "8.6.0"+version = "9.0.5"source = "registry+https://github.com/rust-lang/crates.io-index"-checksum = "2d04a6b5381502aa6087c94c669499eb1602eb9c5e8198e534de571f7154809b"+checksum = "6e04976da629d3b04204ba8105b396a193af8ad2583b2c79bfb3ab8af580a11e"dependencies = ["crc32fast","flate2","indexmap","memchr",+ "nix","typed-path","zopfli",]diff --git a/Cargo.toml b/Cargo.tomlindex 5b65de56..fc845702 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -66,7 +66,7 @@ text-splitter = { version = "0.32", features = ["tiktoken-rs"] }tiktoken-rs = "0.12"# 句子边界按 Unicode 标准(UAX #29)切,不自己写标点规则pdf-extract = "0.12"-calamine = "0.36"+calamine = "0.37"pgvector = { version = "0.4", features = ["sqlx"] }# `gzip` 不是可选项:一次嵌入回 64×1024 个浮点数,写成 JSON 文本每个数字占# 21 字节而信息量只有 4 字节,一次响应 1.33 MB。实测同一条链路上不压缩 62 秒、diff --git a/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.mdnew file mode 100644index 00000000..16a30bcb--- /dev/null+++ b/crates/utopia-ingest/tests/fixtures/calamine/LICENSE-MIT.md@@ -0,0 +1,23 @@+The MIT License (MIT)++Copyright (c) 2016 Johann Tuffe++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:+++The above copyright notice and this permission notice shall be included in+all copies or substantial portions of the Software.+++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN+THE SOFTWARE.diff --git a/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xlsnew file mode 100644index 00000000..6c6decfaBinary files /dev/null and b/crates/utopia-ingest/tests/fixtures/calamine/date_east_asian.xls differdiff --git a/crates/utopia-ingest/tests/spreadsheet_dates.rs b/crates/utopia-ingest/tests/spreadsheet_dates.rsindex ec7cd2ae..ef23c576 100644--- a/crates/utopia-ingest/tests/spreadsheet_dates.rs+++ b/crates/utopia-ingest/tests/spreadsheet_dates.rs@@ -11,7 +11,7 @@ use std::io::{Cursor, Write};// 3 date and time (22), 4 time of day (20), 5 elapsed time (46), 6 "#,##0.00" (4).const STYLES: &str = r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><numFmts count="1"><numFmt numFmtId="176" formatCode="yyyy"年"m"月"d"日""/></numFmts><cellXfs count="7"><xf numFmtId="0" fontId="0" fillId="0" borderId="0" xfId="0"/><xf numFmtId="14" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="176" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="22" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="20" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="46" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/><xf numFmtId="4" fontId="0" fillId="0" borderId="0" xfId="0" applyNumberFormat="1"/></cellXfs></styleSheet>"#;-fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {+fn workbook(rows: &[Vec<String>], date1904: bool, styles: &str) -> Vec<u8> {let sheet_data: String = rows.iter().enumerate()@@ -28,7 +28,7 @@ fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {("_rels/.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/officeDocument" Target="xl/workbook.xml"/></Relationships>"#.to_string()),("xl/workbook.xml", format!(r#"<workbook xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main" xmlns:r="http://schemas.openxmlformats.org/officeDocument/2006/relationships">{workbook_pr}<sheets><sheet name="Ledger" sheetId="1" r:id="rId1"/></sheets></workbook>"#)),("xl/_rels/workbook.xml.rels", r#"<Relationships xmlns="http://schemas.openxmlformats.org/package/2006/relationships"><Relationship Id="rId1" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/worksheet" Target="worksheets/sheet1.xml"/><Relationship Id="rId2" Type="http://schemas.openxmlformats.org/officeDocument/2006/relationships/styles" Target="styles.xml"/></Relationships>"#.to_string()),- ("xl/styles.xml", STYLES.to_string()),+ ("xl/styles.xml", styles.to_string()),("xl/worksheets/sheet1.xml", format!(r#"<worksheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main"><sheetData>{sheet_data}</sheetData></worksheet>"#)),] {zip.start_file(part, zip::write::SimpleFileOptions::default()).unwrap();@@ -38,7 +38,7 @@ fn workbook(rows: &[Vec<String>], date1904: bool) -> Vec<u8> {}fn read(rows: &[Vec<String>], date1904: bool) -> String {- utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904))+ utopia_ingest::parse("ledger.xlsx", &workbook(rows, date1904, STYLES)).unwrap().text}@@ -141,3 +141,94 @@ fn a_workbook_counting_from_1904_reads_the_same_date() {"{text}");}++fn read_format(format_id: u16, value: &str, date1904: bool, format_code: Option<&str>) -> String {+ let custom = format_code.map_or_else(String::new, |code| {+ let code = quick_xml::escape::escape(code);+ format!(+ r#"<numFmts count="1"><numFmt numFmtId="{format_id}" formatCode="{code}"/></numFmts>"#+ )+ });+ let styles = format!(+ r#"<styleSheet xmlns="http://schemas.openxmlformats.org/spreadsheetml/2006/main">{custom}<cellXfs count="1"><xf numFmtId="{format_id}" applyNumberFormat="1"/></cellXfs></styleSheet>"#+ );+ let rows = [+ header(1),+ vec![+ words("A2", "Value"),+ number("B2", 0, value),+ words("C2", "ok"),+ ],+ ];+ utopia_ingest::parse("ledger.xlsx", &workbook(&rows, date1904, &styles))+ .unwrap()+ .text+}++#[test]+fn east_asian_builtin_dates_need_no_custom_format_definition() {+ for format_id in [27, 31, 50, 58] {+ for (date1904, serial) in [(false, "45306"), (true, "43844")] {+ let text = read_format(format_id, serial, date1904, None);+ assert!(+ text.contains("| Value | 2024-01-15 | ok |"),+ "format {format_id}: {text}"+ );+ }+ }+}++#[test]+fn east_asian_fractional_clock_times_and_explicit_formats_are_preserved() {+ // calamine does not retain clock-only metadata; a serial >= 1 still gains a date.+ for format_id in [32, 33] {+ let text = read_format(format_id, "0.5", false, None);+ assert!(+ text.contains("| Value | 12:00 | ok |"),+ "format {format_id}: {text}"+ );+ }+ for (format_id, code, serial, expected) in [+ (27, "0.00", "45306", "45306"),+ (32, "yyyy-mm-dd", "45306", "2024-01-15"),+ (27, "[h]:mm:ss", "1.5104166666666667", "36:15:00"),+ ] {+ let text = read_format(format_id, serial, false, Some(code));+ assert!(+ text.contains(&format!("| Value | {expected} | ok |")),+ "{text}"+ );+ }+}++#[test]+fn east_asian_builtin_formats_are_normalized_in_xls() {+ // Fixture from calamine PR #734, commit d93c5a463399a5554d287ebda0ed1d2a43377ec2,+ // tests/date_east_asian.xls (MIT; license alongside). Rows 0..19 are IDs 27..36/50..58;+ // each row stores 45306, 0.5, 45306.123456789 with no explicit format definition.+ let text = utopia_ingest::parse(+ "ledger.xls",+ include_bytes!("fixtures/calamine/date_east_asian.xls"),+ )+ .unwrap()+ .text;+ let rows: Vec<_> = text+ .lines()+ .filter(|line| line.starts_with("| ") && !line.starts_with("| ---"))+ .collect();+ assert_eq!(rows.len(), 28, "{text}");+ for row in [0, 4, 10, 18] {+ // formats 27, 31, 50, 58+ assert!(rows[row].starts_with("| 2024-01-15 |"), "row {row}: {text}");+ }+ for row in [5, 6] {+ // formats 32, 33+ assert_eq!(+ rows[row].split('|').nth(2).unwrap().trim(),+ "12:00",+ "{text}"+ );+ }+ assert!(rows[19].starts_with("| 45306 | 0.5 |"), "{text}"); // General+ assert!(rows[24].starts_with("| 2024-01-15 |"), "{text}"); // Chinese custom date+}