World's first open-source enterprise world model.
Rust agent-memorybitemporalgraphragknowledge-baseknowledge-graphllmontologypgvectorpostgresqlragrustself-hostedsemantic-searchtemporal-knowledge-graphworld-model
A delimited record keeps its empty fields and its own row (#1035)
Maya Kid9e00b4eacff6
diff --git a/crates/utopia-ingest/src/parsers.rs b/crates/utopia-ingest/src/parsers.rsindex 5da4462e..d715634e 100644--- a/crates/utopia-ingest/src/parsers.rs+++ b/crates/utopia-ingest/src/parsers.rs@@ -995,13 +995,13 @@ pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<(String, Vec<crate::P.flexible(true).has_headers(false).from_reader(decoded.as_bytes());- let mut rows: Vec<GridRow> = Vec::new();+ let mut rows: Vec<Vec<String>> = Vec::new();let mut records_total = 0usize;for record in reader.records() {let record = record?;records_total += 1;if rows.len() < CSV_RECORD_LIMIT {- rows.push(record.iter().map(|s| (s.to_string(), 1, 0)).collect());+ rows.push(record.iter().map(str::to_string).collect());}}let mut warnings = Vec::new();@@ -1017,16 +1017,11 @@ pub fn csv_text(bytes: &[u8], tsv: bool) -> anyhow::Result<(String, Vec<crate::P});}// 第一条记录是列头(csv 的惯例);渲染不出表时退回竖线分隔的行- let text = match crate::table::render_grid(&rows, true) {+ let text = match crate::table::render_records(&rows) {Some(md) => md + "\n",None => {rows.iter()- .map(|r| {- r.iter()- .map(|(s, _, _)| s.as_str())- .collect::<Vec<_>>()- .join(" | ")- })+ .map(|r| r.iter().map(String::as_str).collect::<Vec<_>>().join(" | ")).collect::<Vec<_>>().join("\n")+ "\n"@@ -1308,7 +1303,7 @@ mod tests {assert!(text.contains("| 2 | Ada | 9 |"), "{text}");}- /// 一格数字都没有的 csv 也是表:标签列是最左那格+ /// 一格数字都没有的 csv 也是表,字段仍按原列分开#[test]fn a_csv_of_words_is_still_a_table() {let text = csv_text(b"name,role\nAda,engineer\nGrace,admiral\n", false)@@ -1319,4 +1314,112 @@ mod tests {"{text}");}++ #[test]+ fn a_missing_email_does_not_attach_the_name_to_the_next_record() {+ let text = csv_text(b"name,email\nAlice,\nBob,[email protected]\n", false)+ .unwrap()+ .0;+ assert_eq!(+ text,+ "| name | email |\n| --- | --- |\n| Alice | |\n| Bob | [email protected] |\n"+ );+ }++ #[test]+ fn sparse_delimited_records_keep_their_rows_and_columns() {+ let csv = "name,email,city\nAlice,,\n,,\n,[email protected],\n,,Sydney\nCarol,,Melbourne\nDave,[email protected],\nEve,,\n,,\n";+ let expected = "| name | email | city |\n| --- | --- | --- |\n| Alice | | |\n| | [email protected] | |\n| | | Sydney |\n| Carol | | Melbourne |\n| Dave | [email protected] | |\n| Eve | | |\n";+ for tsv in [false, true] {+ let input = if tsv {+ csv.replace(',', "\t")+ } else {+ csv.to_string()+ };+ assert_eq!(csv_text(input.as_bytes(), tsv).unwrap().0, expected);+ }+ }++ #[test]+ fn a_single_text_column_keeps_the_header_and_each_record() {+ for tsv in [false, true] {+ assert_eq!(+ csv_text(b"name\nAlice\nBob\n", tsv).unwrap().0,+ "| name |\n| --- |\n| Alice |\n| Bob |\n"+ );+ }+ }++ #[test]+ fn delimited_fields_keep_their_columns_whether_text_or_numbers() {+ for (csv, expected) in [+ (+ "name,role,team\nAda,engineer,platform\nGrace,admiral,navy\n",+ "| name | role | team |\n| --- | --- | --- |\n| Ada | engineer | platform |\n| Grace | admiral | navy |\n",+ ),+ (+ "name,role,score\nAda,engineer,9\nGrace,admiral,8\n",+ "| name | role | score |\n| --- | --- | --- |\n| Ada | engineer | 9 |\n| Grace | admiral | 8 |\n",+ ),+ ] {+ for tsv in [false, true] {+ let input = if tsv {+ csv.replace(',', "\t")+ } else {+ csv.to_string()+ };+ assert_eq!(csv_text(input.as_bytes(), tsv).unwrap().0, expected);+ }+ }+ }++ #[test]+ fn a_delimited_header_can_have_empty_columns_and_records_can_vary_in_width() {+ let csv = "name,,email\nAlice\nBob,,[email protected],extra\n,,[email protected]\n";+ let expected = "| name | | email | |\n| --- | --- | --- | --- |\n| Alice | | | |\n| Bob | | [email protected] | extra |\n| | | [email protected] | |\n";+ for tsv in [false, true] {+ let input = if tsv {+ csv.replace(',', "\t")+ } else {+ csv.to_string()+ };+ assert_eq!(csv_text(input.as_bytes(), tsv).unwrap().0, expected);+ }+ for (csv, expected) in [+ (+ "name,\nAlice,[email protected]\n",+ "| name | |\n| --- | --- |\n| Alice | [email protected] |\n",+ ),+ (+ ",email\n,[email protected]\n",+ "| | email |\n| --- | --- |\n| | [email protected] |\n",+ ),+ ] {+ for tsv in [false, true] {+ let input = if tsv {+ csv.replace(',', "\t")+ } else {+ csv.to_string()+ };+ assert_eq!(csv_text(input.as_bytes(), tsv).unwrap().0, expected);+ }+ }+ }++ #[test]+ fn symbols_and_quoted_text_stay_in_their_delimited_fields() {+ let expected = "| kind | symbol | value |\n| --- | --- | --- |\n| price, retail | $ | 10 |\n| rate | % | 20 |\n| pipe \\| and line | hello | 30 |\n";+ for (input, tsv) in [+ (+ "kind,symbol,value\n\"price, retail\",$,10\nrate,%,20\n\"pipe | and\nline\",hello,30\n",+ false,+ ),+ (+ "kind\tsymbol\tvalue\n\"price, retail\"\t$\t10\nrate\t%\t20\n\"pipe | and\nline\"\thello\t30\n",+ true,+ ),+ ] {+ assert_eq!(csv_text(input.as_bytes(), tsv).unwrap().0, expected);+ }+ }}diff --git a/crates/utopia-ingest/src/table.rs b/crates/utopia-ingest/src/table.rsindex 2ab1b9cd..8893c703 100644--- a/crates/utopia-ingest/src/table.rs+++ b/crates/utopia-ingest/src/table.rs@@ -15,9 +15,9 @@//!//! 只处理最外层且不含内层表的表;套着的表留给 htmd 原路。//!-//! docx、电子表格和 csv 的表不经过 DOM:解析器把格子收成网格交给 [`render_grid`],之后的-//! 分类与渲染和 HTML 表一模一样。电子表格和 csv 的第一排(至少两格有字的那排)按惯例是-//! 列头,哪怕列头是年份这种数字。+//! docx 和电子表格的表不经过 DOM:解析器把格子收成网格交给 [`render_grid`],之后的+//! 分类与渲染和 HTML 表一模一样。电子表格的第一排(至少两格有字的那排)按惯例是+//! 列头,哪怕列头是年份这种数字。csv/tsv 则保留记录与字段边界,交给 [`render_records`]。use dom_query::{Document, Selection};@@ -312,6 +312,36 @@ fn escape(text: &str) -> String {text.replace('|', "\\|")}+/// csv/tsv 的空字段只是缺失值,不能把该记录当成小节,或把独立字段并成财报的标签列。+pub(crate) fn render_records(rows: &[Vec<String>]) -> Option<String> {+ let rows: Vec<Vec<String>> = rows+ .iter()+ .map(|r| r.iter().map(|s| clean(s)).collect())+ .collect();+ let header = rows.first()?;+ let width = rows.iter().map(Vec::len).max().unwrap_or(0);+ if width == 0 {+ return None;+ }+ let render = |row: &[String]| {+ let cells: Vec<String> = row+ .iter()+ .map(|s| escape(s))+ .chain(std::iter::repeat(String::new()))+ .take(width)+ .collect();+ format!("| {} |", cells.join(" | "))+ };+ let mut lines = vec![render(header), format!("|{}", " --- |".repeat(width))];+ lines.extend(+ rows.iter()+ .skip(1)+ .filter(|r| r.iter().any(|s| !s.is_empty()))+ .map(|r| render(r)),+ );+ Some(lines.join("\n"))+}+/// 缩进深度:先看标签格前面有几根空列(Workiva 报表用空格子缩进),再看左内边距/// (另一些用 padding-left)。两者合成一个数,列在前、内边距在后fn depth_of(cell: &Cell) -> u32 {@@ -325,7 +355,7 @@ fn render_table(tbl: &Selection<'_>, inherited: &[String]) -> Option<(String, Ve}/// 别的解析器用的入口:一行是若干 (文字, 跨几列, 左内边距 pt)。`first_is_header` 为真时,-/// 第一排至少两格有字的行按列头算(电子表格、csv 的惯例),其余行照常分类。+/// 第一排至少两格有字的行按列头算(电子表格的惯例),其余行照常分类。pub(crate) fn render_grid(rows: &[Vec<(String, usize, u32)>],first_is_header: bool,diff --git a/docs/design/sources.md b/docs/design/sources.mdindex 3454c199..bbc34da1 100644--- a/docs/design/sources.md+++ b/docs/design/sources.md@@ -57,10 +57,12 @@ does what it already did: the caption and the header travel with every piece, anfive short paragraphs before a table is its caption, so a statement's chunk names the report, theunit and the period. Nested tables and the tables of other parsers still take the converter'spath with the first row promoted to header. Measured on the NVDA earnings release: 52 chunks-became 32, none of them a table without its header. DOCX tables, spreadsheets and CSV files are laid on the same grid by their parsers (#750): a DOCX-cell keeps its column span and the indent of its first paragraph, a sheet's or a CSV file's first+became 32, none of them a table without its header. DOCX tables and spreadsheets are laid on the same grid by their parsers (#750): a DOCX+cell keeps its column span and the indent of its first paragraph, a sheet's firstrow with two or more cells is its header even when the headings are years, and a table with no-figure in it is read as a header row followed by records; a sheet's date cell is written as the+figure in it is read as a header row followed by records; a CSV or TSV file is read as records+instead: its first record is the header, every field keeps its own column, and an empty field is+a missing value, not a section or a label (#1035); a sheet's date cell is written as thedate it shows (ISO, with its time of day, or an elapsed time as hours:minutes:seconds), not asExcel's day count; the text layer of a PDF carries no table structure.