Skip to content

armgit / bento

Public Imported from GitHub
0

Bento, the office suite that fits in a file

dash M0: CSV import that refuses to guess (#230)

nyblnet4c75f4ac90fd
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 9e2c4aa3..ee3361d0 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -214,6 +214,14 @@ jobs:
# checked apply → undo → identical, because that failure is silent.
run: node scripts/test-dash-store.ts
+ - name: bento/dash import rig
+ # Import is where a data tool does its damage, because every mistake is
+ # SILENT: the file opens, the grid fills, the totals compute, and the
+ # numbers are wrong. Two guards matter most and are checked from both
+ # sides — an undecidable date order is REFUSED rather than guessed, and
+ # the decimal convention is decided per COLUMN from the whole column.
+ run: node scripts/test-dash-import.ts
+
- name: Built-in layout geometry rig
# The built-ins are drawn against 1600x900 while the model default is
# 1280x720, so they used to hang 200px off the right edge of a default
diff --git a/dash/src/import.ts b/dash/src/import.ts
new file mode 100644
index 00000000..619b49a6
--- /dev/null
+++ b/dash/src/import.ts
@@ -0,0 +1,391 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 The Bento authors
+// CSV / delimited import: parse, infer, encode.
+//
+// This is M1's whole demo — "the CSV someone emailed you, cleaned and typed and
+// totalled and sent back as one file anyone can open" — and it is also where a
+// data tool does its most damage, because every mistake here is SILENT and
+// arrives wearing the shape of an answer.
+//
+// Three of them, in the order they cost:
+//
+// 1. THE DATE AMBIGUITY. 03/04/2026 is 3 April or 4 March and the file does
+// not say. A column where every day is <= 12 is genuinely undecidable, and
+// picking one silently is how a quarter's numbers move by a month. This
+// module REFUSES to decide: it reports `ambiguous` and imports as text
+// until someone says. Guessing is the industry norm and it is wrong.
+// 2. THE DECIMAL COMMA. "1.234" is one thousand two hundred and thirty four in
+// Germany and one-point-two-three-four in Britain. Inference decides per
+// COLUMN, from the whole column, never per cell — a cell-by-cell guess
+// turns one column into two conventions.
+// 3. VALUES THAT WILL NOT COERCE. A column typed `number` with 4% junk in it
+// must say so. `failed` is recorded on the column, never hidden, because a
+// column that quietly dropped 4% of its rows is a wrong answer wearing a
+// right answer's clothes.
+//
+// Nothing here touches the DOM: it is pure data, so it is provable in CI.
+
+import type { Column, ColumnData, ColumnType, TableSheet } from './model.ts'
+
+// --- Delimited parsing -----------------------------------------------------
+
+export interface ParsedDelimited {
+ rows: string[][]
+ delimiter: string
+ /** rows whose field count differed from the header's */
+ ragged: number
+}
+
+/**
+ * Pick the delimiter by counting candidates in the header line.
+ *
+ * Semicolon matters as much as comma: it is what Excel writes on any machine
+ * whose locale uses a decimal comma, so "CSV" from half of Europe is not
+ * comma-separated at all.
+ */
+function sniffDelimiter(head: string): string {
+ const counts = [',', ';', '\t', '|'].map((d) => ({
+ d,
+ // count only OUTSIDE quotes, or a quoted "Smith, John" votes for comma
+ n: countUnquoted(head, d),
+ }))
+ counts.sort((a, b) => b.n - a.n)
+ return counts[0].n > 0 ? counts[0].d : ','
+}
+
+function countUnquoted(line: string, ch: string): number {
+ let n = 0
+ let inQ = false
+ for (let i = 0; i < line.length; i++) {
+ const c = line[i]
+ if (c === '"') { inQ = !inQ; continue }
+ if (!inQ && c === ch) n++
+ }
+ return n
+}
+
+/**
+ * RFC 4180 with the escapes real files actually contain: CRLF or LF, a UTF-8
+ * BOM, doubled quotes inside quoted fields, and newlines inside quoted fields.
+ */
+export function parseDelimited(text: string, delimiter?: string): ParsedDelimited {
+ let s = text
+ if (s.charCodeAt(0) === 0xFEFF) s = s.slice(1) // BOM: invisible, and it would
+ // otherwise become part of the first column's NAME
+ const firstBreak = s.search(/\r?\n/)
+ const head = firstBreak < 0 ? s : s.slice(0, firstBreak)
+ const d = delimiter ?? sniffDelimiter(head)
+
+ const rows: string[][] = []
+ let row: string[] = []
+ let field = ''
+ let inQ = false
+ let i = 0
+ const push = () => { row.push(field); field = '' }
+ const endRow = () => { push(); rows.push(row); row = [] }
+
+ while (i < s.length) {
+ const c = s[i]
+ if (inQ) {
+ if (c === '"') {
+ if (s[i + 1] === '"') { field += '"'; i += 2; continue } // doubled = literal
+ inQ = false; i++; continue
+ }
+ field += c; i++; continue
+ }
+ if (c === '"' && field === '') { inQ = true; i++; continue }
+ if (c === d) { push(); i++; continue }
+ if (c === '\r') { i++; continue }
+ if (c === '\n') { endRow(); i++; continue }
+ field += c; i++
+ }
+ if (field !== '' || row.length) endRow()
+ // a trailing newline produces one empty final row; that is not a row of data
+ if (rows.length && rows[rows.length - 1].length === 1 && rows[rows.length - 1][0] === '') rows.pop()
+
+ const width = rows[0]?.length ?? 0
+ const ragged = rows.reduce((n, r) => n + (r.length === width ? 0 : 1), 0)
+ return { rows, delimiter: d, ragged }
+}
+
+// --- Inference -------------------------------------------------------------
+
+export interface Inference {
+ type: ColumnType
+ /** the convention this column was read with — kept so a re-import is reproducible */
+ parsed?: string
+ /** values that would not coerce to `type` */
+ failed: number
+ /**
+ * Set when the data cannot decide and a guess would be a silent wrong answer.
+ * The column imports as TEXT and the caller must ask. The only honest state.
+ */
+ ambiguous?: { kind: 'date-order'; detail: string }
+}
+
+const BOOLS = new Set(['true', 'false', 'yes', 'no', 'y', 'n', '1', '0', 't', 'f'])
+const blank = (s: string) => s.trim() === ''
+
+/**
+ * Read a number under ONE convention. `point` is the decimal separator; the
+ * other of . and , is then the grouping separator and must appear only in
+ * groups of three.
+ */
+/**
+ * The digit shape each convention allows, checked in full rather than by
+ * stripping characters until something parses.
+ *
+ * Stripping was the first implementation and it was wrong in a way worth
+ * recording: removing `-` to handle negatives also removed the separators in
+ * `2026-01-15`, which then read as the NUMBER 20260115. A date column silently
+ * became a number column, and every value in it was eight digits of nonsense.
+ */
+const NUM_DOT = /^(?:\d{1,3}(?:,\d{3})+|\d+)(?:\.\d+)?$/
+const NUM_COMMA = /^(?:\d{1,3}(?:\.\d{3})+|\d+)(?:,\d+)?$/
+
+function readNumber(raw: string, point: '.' | ','): number | null {
+ let s = raw.trim()
+ if (s === '') return null
+ // accounting negatives
+ let neg = false
+ if (/^\(.*\)$/.test(s)) { neg = true; s = s.slice(1, -1).trim() }
+ // currency and spacing, either end
+ s = s.replace(/^[\s$£€¥]+/, '').replace(/[\s$£€¥]+$/, '')
+ // a trailing percent is part of the NOTATION, not the number
+ if (s.endsWith('%')) s = s.slice(0, -1).trim()
+ if (s.startsWith('-')) { neg = true; s = s.slice(1) }
+ else if (s.startsWith('+')) s = s.slice(1)
+ if (!(point === '.' ? NUM_DOT : NUM_COMMA).test(s)) return null
+ const group = point === '.' ? ',' : '.'
+ s = s.split(group).join('')
+ if (point === ',') s = s.replace(',', '.')
+ const n = Number(s)
+ if (!Number.isFinite(n)) return null
+ return neg ? -n : n
+}
+
+const DATE_ISO = /^(\d{4})-(\d{2})-(\d{2})([T ].*)?$/
+const DATE_SLASH = /^(\d{1,2})[/.-](\d{1,2})[/.-](\d{2,4})$/
+
+/**
+ * Infer one column's type from ALL of its values.
+ *
+ * Per column, never per cell: a cell-by-cell decision reads half a column as
+ * British and half as German and the total is then wrong in a way nobody can
+ * see.
+ */
+export function inferColumn(values: string[]): Inference {
+ const present = values.filter((v) => !blank(v))
+ if (!present.length) return { type: 'text', failed: 0 }
+
+ // --- bool
+ if (present.every((v) => BOOLS.has(v.trim().toLowerCase()))) {
+ // all-numeric 0/1 is a number until something says otherwise
+ if (!present.every((v) => v.trim() === '0' || v.trim() === '1')) {
+ return { type: 'bool', failed: 0 }
+ }
+ }
+
+ // --- number, under each convention; the one that reads MORE wins
+ const dot = present.reduce((n, v) => n + (readNumber(v, '.') === null ? 0 : 1), 0)
+ const comma = present.reduce((n, v) => n + (readNumber(v, ',') === null ? 0 : 1), 0)
+ const best = Math.max(dot, comma)
+ if (best / present.length >= 0.9) {
+ const point: '.' | ',' = dot >= comma ? '.' : ','
+ const pct = present.filter((v) => v.includes('%')).length / present.length
+ const cur = present.filter((v) => /[$£€¥]/.test(v)).length / present.length
+ const type: ColumnType = pct > 0.5 ? 'percent' : cur > 0.5 ? 'money' : 'number'
+ return { type, parsed: point === '.' ? 'dot' : 'comma', failed: present.length - best }
+ }
+
+ // --- date
+ const iso = present.filter((v) => DATE_ISO.test(v.trim())).length
+ if (iso / present.length >= 0.9) {
+ return { type: 'date', parsed: 'iso', failed: present.length - iso }
+ }
+ const slash = present.map((v) => v.trim().match(DATE_SLASH)).filter(Boolean) as RegExpMatchArray[]
+ if (slash.length / present.length >= 0.9) {
+ const a = slash.map((m) => Number(m[1]))
+ const b = slash.map((m) => Number(m[2]))
+ const aIsDay = a.some((n) => n > 12)
+ const bIsDay = b.some((n) => n > 12)
+ if (aIsDay && !bIsDay) return { type: 'date', parsed: 'dmy', failed: present.length - slash.length }
+ if (bIsDay && !aIsDay) return { type: 'date', parsed: 'mdy', failed: present.length - slash.length }
+ if (aIsDay && bIsDay) {
+ // both positions exceed 12 somewhere — the column is not one convention
+ return { type: 'text', failed: 0, ambiguous: { kind: 'date-order', detail: 'both positions exceed 12, so no single day/month order fits the whole column' } }
+ }
+ // NEITHER exceeds 12: undecidable from the data. This is the one that
+ // matters, and the honest answer is to refuse.
+ return {
+ type: 'text',
+ failed: 0,
+ ambiguous: {
+ kind: 'date-order',
+ detail: 'every value fits both DD/MM and MM/DD — the file does not say which, and guessing moves dates by up to eleven months',
+ },
+ }
+ }
+
+ return { type: 'text', failed: 0 }
+}
+
+// --- Encoding --------------------------------------------------------------
+
+/**
+ * Dictionary-encode when the column repeats itself enough to pay for the
+ * dictionary. Measured on a 12-column fact table: 4.8 bytes/cell dictionary
+ * columnar against 8.1 plain columnar. A near-unique column (an id, a
+ * free-text note) pays the dictionary and gets nothing, so it stays raw.
+ */
+const DICT_RATIO = 0.5
+const DICT_MAX = 50_000
+
+export function encodeColumn(values: unknown[], type: ColumnType): ColumnData {
+ const raw = values as Array<number | string | boolean | null>
+ if (type !== 'text' && type !== 'enum') return { enc: 'raw', v: raw }
+ const uniq = new Set(values.filter((v) => v != null).map(String))
+ if (uniq.size > DICT_MAX || uniq.size > values.length * DICT_RATIO) {
+ return { enc: 'raw', v: raw }
+ }
+ const dict = [...uniq]
+ const at = new Map(dict.map((s, i) => [s, i]))
+ return { enc: 'dict', dict, idx: values.map((v) => (v == null ? null : at.get(String(v)) ?? null)) }
+}
+
+// --- Building a sheet ------------------------------------------------------
+
+export interface ImportFinding {
+ code: 'ragged-rows' | 'coerce-failed' | 'date-ambiguous' | 'duplicate-header' | 'empty-header'
+ column?: string
+ message: string
+}
+
+export interface ImportResult {
+ sheet: TableSheet
+ findings: ImportFinding[]
+ delimiter: string
+}
+
+const slug = (s: string, i: number): string =>
+ (s.trim().toLowerCase().replace(/[^a-z0-9]+/g, '-').replace(/^-|-$/g, '') || `col-${i + 1}`).slice(0, 40)
+
+/** Coerce one raw string under the convention inference chose. */
+export function coerce(raw: string, inf: Inference): unknown {
+ if (blank(raw)) return null
+ const s = raw.trim()
+ switch (inf.type) {
+ case 'number': case 'money': case 'percent': {
+ const n = readNumber(s, inf.parsed === 'comma' ? ',' : '.')
+ if (n === null) return null
+ return inf.type === 'percent' && s.includes('%') ? n / 100 : n
+ }
+ case 'bool': {
+ const t = s.toLowerCase()
+ return t === 'true' || t === 'yes' || t === 'y' || t === 't' || t === '1'
+ }
+ case 'date': {
+ if (inf.parsed === 'iso') return s.slice(0, 10)
+ const m = s.match(DATE_SLASH)
+ if (!m) return null
+ const [, p1, p2, p3] = m
+ const day = inf.parsed === 'dmy' ? p1 : p2
+ const mon = inf.parsed === 'dmy' ? p2 : p1
+ const yr = p3.length === 2 ? `20${p3}` : p3
+ return `${yr}-${mon.padStart(2, '0')}-${day.padStart(2, '0')}`
+ }
+ default:
+ return raw
+ }
+}
+
+/**
+ * Build a table sheet from delimited text.
+ *
+ * `header` defaults to true because a file someone emailed you almost always
+ * has one; when it is false the columns are named by position and nothing is
+ * silently eaten.
+ */
+export function importDelimited(
+ text: string,
+ opts: { name?: string; sheetId?: string; header?: boolean; delimiter?: string; source?: string; at?: string } = {},
+): ImportResult {
+ const { rows, delimiter, ragged } = parseDelimited(text, opts.delimiter)
+ const findings: ImportFinding[] = []
+ const header = opts.header !== false
+ const width = rows.reduce((w, r) => Math.max(w, r.length), 0)
+
+ const names = header && rows.length
+ ? Array.from({ length: width }, (_, i) => rows[0][i] ?? '')
+ : Array.from({ length: width }, (_, i) => `Column ${i + 1}`)
+ const body = header ? rows.slice(1) : rows
+
+ if (ragged) {
+ findings.push({
+ code: 'ragged-rows',
+ message: `${ragged} row${ragged === 1 ? '' : 's'} had a different number of fields than the header; short rows are padded with blanks and nothing is dropped`,
+ })
+ }
+
+ const ids = new Set<string>()
+ const columns: Column[] = []
+ const data: Record<string, ColumnData> = {}
+
+ names.forEach((rawName, i) => {
+ let id = slug(rawName, i)
+ if (ids.has(id)) {
+ let n = 2
+ while (ids.has(`${id}-${n}`)) n++
+ findings.push({ code: 'duplicate-header', column: rawName, message: `two columns are called "${rawName}"; the second is kept separately` })
+ id = `${id}-${n}`
+ }
+ ids.add(id)
+ if (header && blank(rawName)) {
+ findings.push({ code: 'empty-header', column: id, message: `column ${i + 1} has no name in the header row` })
+ }
+
+ const raw = body.map((r) => r[i] ?? '')
+ const inf = inferColumn(raw)
+ if (inf.ambiguous) {
+ findings.push({
+ code: 'date-ambiguous',
+ column: rawName || id,
+ message: `"${rawName || id}" looks like dates, but ${inf.ambiguous.detail}. Imported as text — set the column type to choose.`,
+ })
+ }
+ if (inf.failed) {
+ findings.push({
+ code: 'coerce-failed',
+ column: rawName || id,
+ message: `${inf.failed} value${inf.failed === 1 ? '' : 's'} in "${rawName || id}" could not be read as ${inf.type} and are blank; the originals are in the import step`,
+ })
+ }
+
+ const values = raw.map((v) => coerce(v, inf))
+ columns.push({
+ id,
+ name: header && !blank(rawName) ? rawName.trim() : `Column ${i + 1}`,
+ type: inf.type,
+ ...(inf.parsed ? { parsed: inf.parsed } : {}),
+ ...(inf.failed ? { failed: inf.failed } : {}),
+ })
+ data[id] = encodeColumn(values, inf.type)
+ })
+
+ const sheet: TableSheet = {
+ id: opts.sheetId ?? 'sheet-1',
+ name: opts.name ?? 'Sheet 1',
+ kind: 'table',
+ rids: body.length ? [[1, body.length]] : [],
+ columns,
+ data,
+ // steps[0] is ALWAYS the provenance record and is never re-run
+ steps: [{
+ op: 'import',
+ from: opts.source ?? 'pasted',
+ at: opts.at ?? '',
+ rows: body.length,
+ }],
+ }
+ return { sheet, findings, delimiter }
+}
diff --git a/scripts/test-dash-import.ts b/scripts/test-dash-import.ts
new file mode 100644
index 00000000..eba76d98
--- /dev/null
+++ b/scripts/test-dash-import.ts
@@ -0,0 +1,177 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 The Bento authors
+// bento/dash import rig.
+//
+// node scripts/test-dash-import.ts (Node ≥ 23.6 strips types natively)
+//
+// WHAT THIS PROVES. Import is where a data tool does its damage, because every
+// mistake here is SILENT: the file opens, the grid fills, the totals compute,
+// and the numbers are wrong. Three failures matter more than the rest, and each
+// is checked from both sides — the right answer, AND that the plausible wrong
+// answer is refused.
+//
+// 1. DATE ORDER. 03/04/2026 is 3 April or 4 March. A column where every day
+// is <= 12 cannot be decided from the data, and the industry norm is to
+// guess by locale. Guessing moves dates by up to eleven months, silently.
+// This rig asserts the import REFUSES rather than guesses.
+// 2. DECIMAL COMMA. "1.234" is 1234 in Germany and 1.234 in Britain. The
+// decision must be per COLUMN and from the whole column.
+// 3. NOTHING IS DROPPED. Ragged rows pad, they do not truncate; values that
+// will not coerce are counted on the column, not hidden.
+
+import { parseDelimited, inferColumn, encodeColumn, coerce, importDelimited } from '../dash/src/import.ts'
+import { parseDoc } from '../dash/src/model.ts'
+import { readCell } from '../dash/src/store.ts'
+
+let failures = 0
+let checks = 0
+function ok(cond: boolean, msg: string) {
+ checks++
+ if (!cond) { failures++; console.log(` FAIL ${msg}`) }
+ else console.log(` ok ${msg}`)
+}
+
+// ------------------------------------------------------------------ parsing
+{
+ const p = parseDelimited('a,b\n1,2\n')
+ ok(p.rows.length === 2 && p.rows[1][1] === '2', 'a plain CSV parses')
+ ok(parseDelimited('a,b\r\n1,2\r\n').rows.length === 2, 'CRLF parses the same')
+ ok(parseDelimited('a,b\n1,2').rows[0][0] === 'a',
+ 'a UTF-8 BOM is stripped — otherwise it becomes part of the first column NAME')
+ ok(parseDelimited('a,b\n"x,y",2').rows[1][0] === 'x,y', 'a quoted delimiter stays in the field')
+ ok(parseDelimited('a\n"say ""hi"""').rows[1][0] === 'say "hi"', 'doubled quotes are one literal quote')
+ ok(parseDelimited('a,b\n"two\nlines",2').rows.length === 2,
+ 'a newline inside quotes does not end the row')
+ ok(parseDelimited('a,b\n1,2\n\n').rows.length === 2, 'a trailing newline is not a row of data')
+}
+
+// --------------------------------------------------------------- delimiters
+ok(parseDelimited('a;b;c\n1;2;3').delimiter === ';',
+ 'semicolon is detected — it is what Excel writes on a decimal-comma machine')
+ok(parseDelimited('a\tb\n1\t2').delimiter === '\t', 'tab is detected')
+ok(parseDelimited('name,note\n"Smith, John",x').delimiter === ',',
+ 'a comma inside quotes does not get a vote')
+ok(parseDelimited('"a;b",c\n1,2').delimiter === ',',
+ 'nor does a semicolon inside quotes')
+
+// ------------------------------------------------------------ THE BIG ONE
+{
+ // every day <= 12: genuinely undecidable
+ const inf = inferColumn(['01/02/2026', '03/04/2026', '05/06/2026'])
+ ok(inf.ambiguous?.kind === 'date-order',
+ 'a date column where every value fits BOTH DD/MM and MM/DD is reported ambiguous')
+ ok(inf.type === 'text',
+ 'and it imports as TEXT — refusing to guess, because guessing moves dates by months')
+
+ // one value settles it
+ ok(inferColumn(['13/02/2026', '03/04/2026']).parsed === 'dmy',
+ 'a day > 12 in the FIRST position settles the column as DD/MM')
+ ok(inferColumn(['02/13/2026', '04/03/2026']).parsed === 'mdy',
+ 'a day > 12 in the SECOND position settles it as MM/DD')
+ ok(inferColumn(['2026-02-13', '2026-04-03']).parsed === 'iso', 'ISO dates are unambiguous')
+
+ // and the whole-file path reports it where a user will see it
+ const r = importDelimited('when,what\n01/02/2026,a\n03/04/2026,b')
+ ok(r.findings.some((f) => f.code === 'date-ambiguous'),
+ 'the ambiguity reaches the caller as a finding, not just a type')
+ ok(r.sheet.columns[0].type === 'text', 'and the column really is text')
+}
+
+// ------------------------------------------------------------ decimal comma
+{
+ ok(inferColumn(['1.234', '5.678']).parsed === 'dot', 'dot-decimal is read as dot')
+ const de = inferColumn(['1.234,56', '2.000,00'])
+ ok(de.type === 'number' && de.parsed === 'comma', 'German grouping+decimal is read as comma')
+ ok(coerce('1.234,56', de) === 1234.56, 'and coerces to the right number')
+ ok(coerce('1,234.56', inferColumn(['1,234.56'])) === 1234.56, 'as does the British convention')
+ // The decision is per column, from the whole column. The discriminating case
+ // is a value that is VALID UNDER BOTH conventions and means different things:
+ // "1.234" is 1234 in a German column and 1.234 in a British one. A per-cell
+ // implementation reads it under whichever convention it tries first, so one
+ // column ends up holding both — and the total is then wrong by a factor of a
+ // thousand, in a way nothing on screen reveals.
+ const german = inferColumn(['1.234,56', '2.345,67', '9.876,54', '1.234'])
+ ok(german.parsed === 'comma', 'a German column is identified from the whole column')
+ ok(coerce('1.234', german) === 1234,
+ 'and an ambiguous value inside it reads as GERMAN (1234), not as 1.234')
+ const british = inferColumn(['1,234.56', '2,345.67', '1,234'])
+ ok(british.parsed === 'dot' && coerce('1,234', british) === 1234,
+ 'the same string in a British column also reads as 1234, by the other route')
+}
+
+// ------------------------------------------------------------------- types
+ok(inferColumn(['true', 'false', 'yes']).type === 'bool', 'booleans are detected')
+ok(inferColumn(['0', '1', '1']).type === 'number',
+ 'a 0/1 column stays a number — it is far more often a count than a flag')
+ok(inferColumn(['$1,200', '$3,400']).type === 'money', 'currency is money')
+ok(inferColumn(['12%', '4%']).type === 'percent', 'percentages are percent')
+ok(coerce('12%', inferColumn(['12%', '4%'])) === 0.12, 'and a percent coerces to its fraction')
+ok(inferColumn(['(1,200)', '(3,400)']).type === 'number', 'accounting parentheses are numbers')
+ok(coerce('(1,200)', inferColumn(['(1,200)'])) === -1200, 'and they are NEGATIVE')
+ok(inferColumn(['North', 'South']).type === 'text', 'words are text')
+ok(inferColumn(['', ' ']).type === 'text', 'an entirely blank column is text, not a crash')
+
+// ---------------------------------------------------------- nothing dropped
+{
+ const r = importDelimited('a,b,c\n1,2,3\n4,5\n6,7,8,9')
+ ok(r.findings.some((f) => f.code === 'ragged-rows'), 'ragged rows are reported')
+ ok(r.sheet.rids[0][1] === 3, 'and all three rows are kept — short rows pad, they do not vanish')
+}
+{
+ const r = importDelimited('n\n1\n2\nnot-a-number\n4\n5\n6\n7\n8\n9\n10')
+ const col = r.sheet.columns[0]
+ ok(col.type === 'number' && col.failed === 1,
+ 'one junk value in a number column is COUNTED on the column, not hidden')
+ ok(r.findings.some((f) => f.code === 'coerce-failed'), 'and reported to the caller')
+}
+{
+ const r = importDelimited('a,a\n1,2')
+ ok(r.findings.some((f) => f.code === 'duplicate-header'), 'a duplicate header is reported')
+ ok(r.sheet.columns[0].id !== r.sheet.columns[1].id, 'and the two columns stay separate')
+}
+{
+ const r = importDelimited('a,,c\n1,2,3')
+ ok(r.findings.some((f) => f.code === 'empty-header'), 'an unnamed column is reported')
+ ok(r.sheet.columns.length === 3, 'and kept')
+}
+
+// ---------------------------------------------------------------- encoding
+{
+ const repetitive = Array.from({ length: 1000 }, (_, i) => (i % 3 ? 'North' : 'South'))
+ const enc = encodeColumn(repetitive, 'text')
+ ok(enc.enc === 'dict' && enc.dict.length === 2,
+ 'a repetitive text column is dictionary-encoded — the 4.8 bytes/cell that makes the format fit')
+ const unique = Array.from({ length: 1000 }, (_, i) => `id-${i}`)
+ ok(encodeColumn(unique, 'text').enc === 'raw',
+ 'a near-unique column stays raw — a dictionary would cost and buy nothing')
+ ok(encodeColumn([1, 2, 3], 'number').enc === 'raw', 'numbers are never dictionary-encoded')
+ // the encoding must survive a read
+ ok(readCell(enc, 0) === 'South' && readCell(enc, 1) === 'North',
+ 'and the dictionary round-trips through readCell')
+}
+
+// ------------------------------------------------------- the whole pipeline
+{
+ const csv = 'Region,Amount,When\nNorth,"1,200",2026-01-15\nSouth,"3,400",2026-02-20\nNorth,"900",2026-03-05'
+ const r = importDelimited(csv, { name: 'Q1', sheetId: 'sh1', source: 'q1.csv', at: '2026-08-03T00:00:00Z' })
+ ok(r.sheet.columns.map((c) => c.type).join() === 'text,number,date',
+ 'a realistic file types as text/number/date')
+ ok(r.findings.length === 0, 'and produces no findings — a clean file imports quietly')
+ ok(readCell(r.sheet.data.amount, 1) === 3400, 'quoted thousands parse')
+ ok(r.sheet.steps[0].op === 'import' && (r.sheet.steps[0] as any).from === 'q1.csv',
+ 'steps[0] is the provenance record')
+ ok((r.sheet.steps[0] as any).rows === 3, 'and it records how many rows arrived')
+
+ // the imported sheet must be a legal document
+ const doc = {
+ format: 'bento/dash', version: 1, policy: 'bento-dash-1',
+ docId: 'd', title: 'Q1', sheets: [r.sheet],
+ }
+ const parsed = parseDoc(JSON.stringify(doc))
+ ok(parsed.ok === true && parsed.repairs.length === 0,
+ 'and the result parses as a bento/dash document with nothing to repair')
+}
+
+console.log(`\n${checks - failures}/${checks} checks passed`)
+if (failures) process.exit(1)