Skip to content

DesignCraft: An open-source page and layout design editor written by scratch in pure Rust

PARITY-DATA-SOURCE: honor quoted delimiters and record boundaries

sanyamsingh0017abcbc745f33
diff --git a/crates/engine/src/cmd/datamerge/parse.rs b/crates/engine/src/cmd/datamerge/parse.rs
index e65493dfc..6ab10f3d9 100644
--- a/crates/engine/src/cmd/datamerge/parse.rs
+++ b/crates/engine/src/cmd/datamerge/parse.rs
@@ -367,10 +367,16 @@ fn sniff(name: &str, text: &str) -> Delimiter {
if ext == "csv" {
return Delimiter::Comma;
}
- let header = first_physical_line(text);
- if header.contains('\t') {
+ let tab = header_delimiter_candidate(text, '\t');
+ let semicolon = header_delimiter_candidate(text, ';');
+ let comma = header_delimiter_candidate(text, ',');
+ // Prefer formats with coherent quoted-field boundaries. If none separates fields,
+ // retain the legacy permissive choice; this is detection, not record validation.
+ let prefer_coherent = [&tab, &semicolon, &comma].iter().any(|c| c.found && c.coherent);
+ let eligible = |c: &HeaderDelimiterCandidate| c.found && (!prefer_coherent || c.coherent);
+ if eligible(&tab) {
Delimiter::Tab
- } else if header.contains(';') && !header.contains(',') {
+ } else if eligible(&semicolon) && !eligible(&comma) {
Delimiter::Semicolon
} else {
Delimiter::Comma
@@ -381,8 +387,66 @@ pub fn extension(name: &str) -> String {
Path::new(name).extension().and_then(|e| e.to_str()).unwrap_or("").to_ascii_lowercase()
}
-fn first_physical_line(text: &str) -> &str {
- text.split('\n').next().unwrap_or("").trim_end_matches('\r')
+struct HeaderDelimiterCandidate {
+ found: bool,
+ coherent: bool,
+}
+
+/// A candidate separator outside quotes in its first nonempty logical record.
+/// Only that candidate starts a new field. Quotes in an unquoted field stay literal;
+/// Non-whitespace text after a quoted field closes lowers coherence, not validity.
+fn header_delimiter_candidate(text: &str, delimiter: char) -> HeaderDelimiterCandidate {
+ let mut found = false;
+ let mut coherent = true;
+ let mut quoted = false;
+ let mut after_quote = false;
+ let mut field_start = true;
+ let mut nonempty = false;
+ let mut chars = text.chars().peekable();
+ while let Some(c) = chars.next() {
+ if quoted {
+ if c == '"' {
+ if chars.peek() == Some(&'"') {
+ chars.next();
+ nonempty = true;
+ } else {
+ quoted = false;
+ after_quote = true;
+ }
+ } else if !c.is_whitespace() {
+ nonempty = true;
+ }
+ continue;
+ }
+ match c {
+ '"' if field_start => {
+ quoted = true;
+ field_start = false;
+ }
+ separator if separator == delimiter => {
+ found = true;
+ field_start = true;
+ after_quote = false;
+ }
+ '\r' | '\n' => {
+ if nonempty {
+ break;
+ }
+ found = false;
+ coherent = true;
+ field_start = true;
+ after_quote = false;
+ }
+ _ => {
+ // Whitespace may follow a closing quote, but does not start a new field
+ // or allow a later non-whitespace suffix to regain coherence.
+ coherent &= !after_quote || c.is_whitespace();
+ nonempty |= !c.is_whitespace();
+ field_start = false;
+ }
+ }
+ }
+ HeaderDelimiterCandidate { found, coherent: coherent && !quoted }
}
/// Header cell: trim, drop one leading apostrophe, then `@` is image and `#` is qr.
@@ -452,6 +516,7 @@ fn parse_delimited(s: &str, delim: char) -> Result<Vec<RawRow>, String> {
let mut row: Vec<String> = Vec::new();
let mut field = String::new();
let mut quoted = false;
+ let mut field_start = true;
let mut line = 1usize;
let mut row_line = 1usize;
let mut data_rows = 0usize;
@@ -501,21 +566,25 @@ fn parse_delimited(s: &str, delim: char) -> Result<Vec<RawRow>, String> {
};
while let Some(c) = chars.next() {
- if c == '\n' {
+ if c == '\n' || (c == '\r' && chars.peek() != Some(&'\n')) {
line = line.saturating_add(1);
}
if quoted {
match c {
'"' if chars.peek() == Some(&'"') => {
- field.push('"');
+ if !discarding {
+ field.push('"');
+ }
chars.next();
}
'"' => quoted = false,
- _ => field.push(c),
+ _ if !discarding => field.push(c),
+ _ => {}
}
continue;
}
if c == delim {
+ field_start = true;
if discarding {
field.clear();
continue;
@@ -531,14 +600,25 @@ fn parse_delimited(s: &str, delim: char) -> Result<Vec<RawRow>, String> {
continue;
}
match c {
- '"' if field.is_empty() && !discarding => quoted = true,
- '\r' => {}
- '\n' => {
+ '"' if field_start => {
+ quoted = true;
+ field_start = false;
+ }
+ '\r' | '\n' => {
+ if c == '\r' && chars.peek() == Some(&'\n') {
+ chars.next();
+ line = line.saturating_add(1);
+ }
finish(&mut row, &mut field, &mut rows, &mut data_rows, &mut header_width, row_line, &mut discarding)?;
row_line = line;
+ field_start = true;
+ }
+ _ => {
+ field_start = false;
+ if !discarding {
+ field.push(c);
+ }
}
- _ if discarding => {}
- _ => field.push(c),
}
}
if !field.is_empty() || !row.is_empty() || discarding {
diff --git a/crates/engine/src/cmd/datamerge/tests.rs b/crates/engine/src/cmd/datamerge/tests.rs
index e4636d109..f4c2004e4 100644
--- a/crates/engine/src/cmd/datamerge/tests.rs
+++ b/crates/engine/src/cmd/datamerge/tests.rs
@@ -85,6 +85,282 @@ fn csv_parsing() {
assert_eq!(quoted[1][0], "Ada\nGrace");
}
+#[test]
+fn data_source_sniffs_quoted_logical_headers() {
+ for (body, delimiter, names) in [
+ ("\"Last, First\";City\nAda;London\n", Delimiter::Semicolon, ["Last, First", "City"]),
+ ("\"Name\tLabel\",City\nAda,London\n", Delimiter::Comma, ["Name\tLabel", "City"]),
+ ("\"Name\nLabel\";City\nAda;London\n", Delimiter::Semicolon, ["Name\nLabel", "City"]),
+ ("\r\n \r\nName;City\r\nAda;London\r\n", Delimiter::Semicolon, ["Name", "City"]),
+ ("\"Name \"\"label,\"\"\";City\nAda;London\n", Delimiter::Semicolon, ["Name \"label,\"", "City"]),
+ ] {
+ let mut s = Session::new();
+ new_doc(&mut s, 1);
+ let result = exec(&mut s, "data.source.select", json!({"bytes": crate::cmd::base64_encode(body.as_bytes()), "name": "people.txt"}));
+ assert_eq!(result["records"], 1, "{body:?}");
+ let source = &s.documents()[0].doc.data_merge.sources[0];
+ assert_eq!(source.delimiter, delimiter, "{body:?}");
+ assert_eq!(source.fields.iter().map(|f| f.name.as_str()).collect::<Vec<_>>(), names, "{body:?}");
+ assert_eq!(source.rows, vec![vec!["Ada", "London"]], "{body:?}");
+ }
+}
+
+#[test]
+fn data_source_sniff_quote_state_belongs_to_the_candidate_delimiter() {
+ for ending in ["\n", "\r\n", "\r"] {
+ for (body, delimiter, names) in [
+ (format!("Name,\"preferred\tCity\"{ending}Ada\tLondon{ending}"), Delimiter::Tab, ["Name,\"preferred", "City\""]),
+ (format!("\"Last, First\";City{ending}Ada;London{ending}"), Delimiter::Semicolon, ["Last, First", "City"]),
+ (format!("\"Name\tLabel\",City{ending}Ada,London{ending}"), Delimiter::Comma, ["Name\tLabel", "City"]),
+ (format!("\"\";\"\"{ending} \t{ending}\"Last, First\";City{ending}Ada;London{ending}"), Delimiter::Semicolon, ["Last, First", "City"]),
+ ] {
+ let automatic = parse::table_from_bytes(body.as_bytes(), "people.txt", None).unwrap();
+ let explicit = parse::table_from_bytes(body.as_bytes(), "people.txt", Some(delimiter)).unwrap();
+ let fields: Vec<_> = names.into_iter().map(|name| DataField { name: name.into(), kind: DataFieldKind::Text }).collect();
+ assert_eq!(automatic.delimiter, delimiter, "{body:?}");
+ assert_eq!(automatic.fields, fields, "{body:?}");
+ assert_eq!(automatic.rows, vec![vec!["Ada", "London"]], "{body:?}");
+ assert!(automatic.warnings.is_empty(), "{body:?}: {:?}", automatic.warnings);
+ assert_eq!(automatic, explicit, "{body:?}");
+ }
+ }
+}
+
+#[test]
+fn data_source_sniff_prefers_coherent_quoted_field_boundaries() {
+ for ending in ["\n", "\r\n", "\r"] {
+ for (header, first, second, delimiter, names, rows, warnings) in [
+ (
+ "\"Name\";\"Last, First\"",
+ "\"Ada\";\"Lovelace, Countess\"",
+ "\"Grace\";\"Hopper, Admiral\"",
+ Delimiter::Semicolon,
+ vec!["Name", "Last, First"],
+ vec![vec!["Ada", "Lovelace, Countess"], vec!["Grace", "Hopper, Admiral"]],
+ vec![],
+ ),
+ (
+ "\"Name\",\"Region\tCity\"",
+ "\"Ada\",\"London\"",
+ "\"Grace\",\"Paris\"",
+ Delimiter::Comma,
+ vec!["Name", "Region\tCity"],
+ vec![vec!["Ada", "London"], vec!["Grace", "Paris"]],
+ vec![],
+ ),
+ (
+ "\"Name\";\"City\";\"Last, \"\"First\"\"\";\"Title\"",
+ "\"Ada\";\"London\";\"Lovelace, \"\"Countess\"\"\";\"Analyst\"",
+ "\"Grace\";\"Paris\";\"Hopper, \"\"Admiral\"\"\";\"Engineer\";\"ignored\"",
+ Delimiter::Semicolon,
+ vec!["Name", "City", "Last, \"First\"", "Title"],
+ vec![vec!["Ada", "London", "Lovelace, \"Countess\"", "Analyst"], vec!["Grace", "Paris", "Hopper, \"Admiral\"", "Engineer"]],
+ vec!["Row 3 has cells past the header. Those cells were ignored."],
+ ),
+ (
+ "\"Name\",\"Role\",\"Region\t\"\"City\"\"\",\"Title\"",
+ "\"Ada\",\"Analyst\",\"London\t\"\"West End\"\"\",\"Countess\"",
+ "\"Grace\",\"Engineer\",\"Paris\t\"\"Centre\"\"\",\"Admiral\",\"ignored\"",
+ Delimiter::Comma,
+ vec!["Name", "Role", "Region\t\"City\"", "Title"],
+ vec![vec!["Ada", "Analyst", "London\t\"West End\"", "Countess"], vec!["Grace", "Engineer", "Paris\t\"Centre\"", "Admiral"]],
+ vec!["Row 3 has cells past the header. Those cells were ignored."],
+ ),
+ ] {
+ let body = format!("{header}{ending}{first}{ending}{second}{ending}");
+ let automatic = parse::table_from_bytes(body.as_bytes(), "people.txt", None).unwrap();
+ let explicit = parse::table_from_bytes(body.as_bytes(), "people.txt", Some(delimiter)).unwrap();
+ let fields: Vec<_> = names.into_iter().map(|name| DataField { name: name.into(), kind: DataFieldKind::Text }).collect();
+ assert_eq!(automatic.delimiter, delimiter, "{body:?}");
+ assert_eq!(automatic.fields, fields, "{body:?}");
+ assert_eq!(automatic.rows, rows, "{body:?}");
+ assert_eq!(automatic.warnings, warnings, "{body:?}");
+ assert_eq!(automatic, explicit, "{body:?}");
+ }
+ }
+}
+
+#[test]
+fn data_source_sniff_retains_priority_and_permissive_fallback() {
+ for ending in ["\n", "\r\n", "\r"] {
+ for (header, row, delimiter, names, cells) in [
+ // Equally coherent candidates retain Tab, then Comma, then Semicolon priority.
+ ("Name,Title\tCity", "Ada,Analyst\tLondon\tignored", Delimiter::Tab, ["Name,Title", "City"], ["Ada,Analyst", "London"]),
+ ("Name;Title,City", "Ada;Analyst,London,ignored", Delimiter::Comma, ["Name;Title", "City"], ["Ada;Analyst", "London"]),
+ // No coherent candidate separates fields here. Keep parsing the trailing text
+ // after a quoted field exactly as before, using the same delimiter priority.
+ (
+ "\"Name\"suffix,Title\tCity",
+ "\"Ada\"suffix,Analyst\tLondon\tignored",
+ Delimiter::Tab,
+ ["Namesuffix,Title", "City"],
+ ["Adasuffix,Analyst", "London"],
+ ),
+ (
+ "\"Name\"suffix;Title,City",
+ "\"Ada\"suffix;Analyst,London,ignored",
+ Delimiter::Comma,
+ ["Namesuffix;Title", "City"],
+ ["Adasuffix;Analyst", "London"],
+ ),
+ ("\"Name\"suffix;City", "\"Ada\"suffix;London;ignored", Delimiter::Semicolon, ["Namesuffix", "City"], ["Adasuffix", "London"]),
+ ] {
+ let body = format!("{header}{ending}{row}{ending}");
+ let automatic = parse::table_from_bytes(body.as_bytes(), "people.txt", None).unwrap();
+ let explicit = parse::table_from_bytes(body.as_bytes(), "people.txt", Some(delimiter)).unwrap();
+ let fields: Vec<_> = names.into_iter().map(|name| DataField { name: name.into(), kind: DataFieldKind::Text }).collect();
+ assert_eq!(automatic.delimiter, delimiter, "{body:?}");
+ assert_eq!(automatic.fields, fields, "{body:?}");
+ assert_eq!(automatic.rows, vec![cells.to_vec()], "{body:?}");
+ assert_eq!(automatic.warnings, vec!["Row 2 has cells past the header. Those cells were ignored."], "{body:?}");
+ assert_eq!(automatic, explicit, "{body:?}");
+ }
+ }
+}
+
+#[test]
+fn data_source_sniff_allows_post_quote_whitespace() {
+ for ending in ["\n", "\r\n", "\r"] {
+ for space in [" ", "\u{00A0}", "\u{2003}"] {
+ for (header, separator, delimiter, names) in [
+ (format!("\"Name\"{space};\"Last, First\""), ';', Delimiter::Semicolon, ["Name", "Last, First"]),
+ (format!("\"Name\"{space},\"Region\tCity\"{space}"), ',', Delimiter::Comma, ["Name", "Region\tCity"]),
+ (format!("\"Name\"{space}\t\"Last, First; City\"{space}"), '\t', Delimiter::Tab, ["Name", "Last, First; City"]),
+ ] {
+ let body = format!(
+ "{header}{ending}\"Ada\"{space}{separator}\"London\"{space}{ending}\"Grace\"{space}{separator}\"Paris\"{space}{separator}ignored{ending}"
+ );
+ let automatic = parse::table_from_bytes(body.as_bytes(), "people.txt", None).unwrap();
+ let explicit = parse::table_from_bytes(body.as_bytes(), "people.txt", Some(delimiter)).unwrap();
+ let fields: Vec<_> = names.into_iter().map(|name| DataField { name: name.into(), kind: DataFieldKind::Text }).collect();
+ assert_eq!(automatic.delimiter, delimiter, "{body:?}");
+ assert_eq!(automatic.fields, fields, "{body:?}");
+ // Header classification trims; record values must keep their whitespace.
+ assert_eq!(
+ automatic.rows,
+ vec![vec![format!("Ada{space}"), format!("London{space}")], vec![format!("Grace{space}"), format!("Paris{space}")]],
+ "{body:?}"
+ );
+ assert_eq!(automatic.warnings, vec!["Row 3 has cells past the header. Those cells were ignored."], "{body:?}");
+ assert_eq!(automatic, explicit, "{body:?}");
+ }
+ }
+ }
+}
+
+#[test]
+fn data_source_sniff_post_quote_whitespace_keeps_closed_field_state() {
+ for ending in ["\n", "\r\n", "\r"] {
+ for space in [" ", "\u{00A0}", "\u{2003}"] {
+ for suffix in ["suffix", "\"suffix\""] {
+ let body = format!("\"Name\"{space}{suffix};\"Last, First\"{ending}Ada,London,ignored{ending}");
+ // Both separated formats have incompatible post-quote suffixes. Keep
+ // the permissive Comma fallback, including literal quotes in the name.
+ let automatic = parse::table_from_bytes(body.as_bytes(), "people.txt", None).unwrap();
+ let explicit = parse::table_from_bytes(body.as_bytes(), "people.txt", Some(Delimiter::Comma)).unwrap();
+ assert_eq!(automatic.delimiter, Delimiter::Comma, "{body:?}");
+ assert_eq!(
+ automatic.fields,
+ vec![
+ DataField { name: format!("Name{space}{suffix};\"Last"), kind: DataFieldKind::Text },
+ DataField { name: "First\"".into(), kind: DataFieldKind::Text },
+ ],
+ "{body:?}"
+ );
+ assert_eq!(automatic.rows, vec![vec!["Ada", "London"]], "{body:?}");
+ assert_eq!(automatic.warnings, vec!["Row 2 has cells past the header. Those cells were ignored."], "{body:?}");
+ assert_eq!(automatic, explicit, "{body:?}");
+ }
+ }
+ }
+}
+
+#[test]
+fn data_source_literal_quote_header_merges_the_detected_column() {
+ let mut s = Session::new();
+ new_doc(&mut s, 1);
+ text_frame(&mut s, [36.0, 36.0, 240.0, 80.0], "", None);
+ let body = "Name,\"preferred\tCity\"\nAda\tLondon\nGrace\tParis\n";
+ let selected = exec(&mut s, "data.source.select", json!({"bytes": crate::cmd::base64_encode(body.as_bytes()), "name": "people.txt"}));
+ assert_eq!(selected["records"], 2);
+ let source = &s.documents()[0].doc.data_merge.sources[0];
+ assert_eq!(source.delimiter, Delimiter::Tab);
+ assert_eq!(source.rows, vec![vec!["Ada", "London"], vec!["Grace", "Paris"]]);
+ exec(&mut s, "data.placeholder.add", json!({"field": "City\""}));
+ let merged = exec(&mut s, "data.merge", json!({}));
+ assert_eq!(merged["records"], 2);
+ assert_eq!(merged["pages"], 2);
+ assert_eq!(texts(&s), vec!["London", "Paris"]);
+}
+
+#[test]
+fn data_source_preserves_quoted_newlines_and_counts_record_lines() {
+ for ending in ["\n", "\r\n", "\r"] {
+ let body = format!("Name,City{ending}\"Ada{ending}Lovelace\",London{ending}Grace,Paris,extra{ending}");
+ let mut s = Session::new();
+ new_doc(&mut s, 1);
+ let result = select_csv(&mut s, &body);
+ assert_eq!(result["records"], 2, "{ending:?}");
+ let source = &s.documents()[0].doc.data_merge.sources[0];
+ assert_eq!(source.rows, vec![vec![format!("Ada{ending}Lovelace"), "London".into()], vec!["Grace".into(), "Paris".into()]]);
+ assert_eq!(source.warnings.len(), 1);
+ assert!(source.warnings[0].contains("Row 4"), "{:?}", source.warnings);
+ }
+}
+
+#[test]
+fn data_source_quoted_header_and_cr_records_merge_independently() {
+ let mut s = Session::new();
+ new_doc(&mut s, 1);
+ text_frame(&mut s, [36.0, 36.0, 240.0, 80.0], "", None);
+ let body = "\u{FEFF}\"Last, First\";City\rAda;London\rGrace;Paris\r";
+ exec(&mut s, "data.source.select", json!({"bytes": crate::cmd::base64_encode(body.as_bytes()), "name": "people.txt"}));
+ exec(&mut s, "data.placeholder.add", json!({"field": "Last, First"}));
+ let template = s.documents()[0].doc.clone();
+ let result = exec(&mut s, "data.merge", json!({}));
+ assert_eq!(result["records"], 2);
+ assert_eq!(result["pages"], 2);
+ assert_eq!(texts(&s), vec!["Ada", "Grace"]);
+ assert_eq!(serde_json::to_value(&s.documents()[0].doc).unwrap(), serde_json::to_value(&template).unwrap());
+}
+
+#[test]
+fn data_source_discarded_quoted_fields_keep_records_and_warning_lines() {
+ for ending in ["\n", "\r\n", "\r"] {
+ for (discarded, following_line) in [(format!("\"Address \"\"line 1\"\"{ending}Address line 2\""), 4), ("literal\"quote".into(), 3)] {
+ let mut s = Session::new();
+ new_doc(&mut s, 1);
+ text_frame(&mut s, [36.0, 36.0, 240.0, 80.0], "", None);
+ let body = format!("Name{ending}Ada,,,{discarded}{ending}Grace,extra{ending}");
+ let selected = select_csv(&mut s, &body);
+ assert_eq!(selected["records"], 2, "{body:?}");
+ let source = &s.documents()[0].doc.data_merge.sources[0];
+ assert_eq!(source.rows, vec![vec!["Ada"], vec!["Grace"]], "{body:?}");
+ assert_eq!(source.warnings.len(), 2, "{:?}", source.warnings);
+ assert!(source.warnings[0].contains("Row 2"), "{:?}", source.warnings);
+ assert!(source.warnings[1].contains(&format!("Row {following_line}")), "{:?}", source.warnings);
+ exec(&mut s, "data.placeholder.add", json!({"field": "Name"}));
+ let merged = exec(&mut s, "data.merge", json!({}));
+ assert_eq!(merged["records"], 2);
+ assert_eq!(merged["pages"], 2);
+ assert_eq!(texts(&s), vec!["Ada", "Grace"]);
+ }
+ }
+}
+
+#[test]
+fn data_source_delimiter_override_and_known_extensions_win() {
+ let body = "Name;City\nAda;London\n";
+ let csv = parse::table_from_bytes(body.as_bytes(), "people.csv", None).unwrap();
+ assert_eq!(csv.delimiter, Delimiter::Comma);
+ assert_eq!(csv.rows, vec![vec!["Ada;London"]]);
+ let explicit = parse::table_from_bytes(body.as_bytes(), "people.csv", Some(Delimiter::Semicolon)).unwrap();
+ assert_eq!(explicit.rows, vec![vec!["Ada", "London"]]);
+ let tab = parse::table_from_bytes(b"Name,Title\tCity\nAda,Analyst\tLondon", "people.tsv", None).unwrap();
+ assert_eq!(tab.delimiter, Delimiter::Tab);
+ assert_eq!(tab.rows, vec![vec!["Ada,Analyst", "London"]]);
+}
+
#[test]
fn text_sniff_padding_skips_and_warnings() {
let dir = scratch("sniff");