Skip to content

armgit / bento

Public Imported from GitHub
0

Bento, the office suite that fits in a file

dash M0: the format, and a rig that refuses to lose data (#228)

nyblnet81274372ce07
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
index 11f80ab9..71713f1f 100644
--- a/.github/workflows/ci.yml
+++ b/.github/workflows/ci.yml
@@ -199,6 +199,14 @@ jobs:
slides/node_modules/.bin/esbuild scripts/test-validate.ts --bundle --platform=node --format=esm --outfile="$RUNNER_TEMP/test-validate.mjs"
node "$RUNNER_TEMP/test-validate.mjs"
+ - name: bento/dash model rig
+ # The format is permanent — additive forever, no server to migrate a
+ # single file. Two failures are unrecoverable once files exist: dropping
+ # a field this build does not recognise, and letting anything but an
+ # ABSENT block reach the starter document (which then overwrites live
+ # data on the first save). Both are asserted from both sides.
+ run: node scripts/test-dash-model.ts
+
- name: Built-in layout geometry rig
# The built-ins are drawn against 1600x900 while the model default is
# 1280x720, so they used to hang 200px off the right edge of a default
diff --git a/dash/src/model.ts b/dash/src/model.ts
new file mode 100644
index 00000000..cbe501c3
--- /dev/null
+++ b/dash/src/model.ts
@@ -0,0 +1,559 @@
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 The Bento authors
+// The bento/dash document model. This JSON is what lives inside the #bento-doc
+// block — the format IS the product (docs/PLATFORM.md §3).
+//
+// ONE FILE = ONE WORKBOOK: sheets of typed columns, stored COLUMNAR, with an
+// ordered transform chain that says how the numbers were arrived at. `dash`
+// contracts DAta SHeets (docs/DECISIONS.md, 2026-08-02) — the data model and
+// the grid, not "dashboard".
+//
+// ADDITIVE FOREVER (PLATFORM §3): unknown top-level fields, unknown per-node
+// fields and unknown transform OPS all survive parse → serialize. There is no
+// server; a break here is permanent, in every file already on someone's disk.
+//
+// The encoding below is the single most expensive decision in the app and it
+// cannot be migrated. Measured on a 12-column fact table: array-of-objects
+// 15.9 bytes/cell, array-of-arrays 8.3, plain columnar 8.1, columnar +
+// dictionary-encoded strings 4.8. Base64'd typed arrays are WORSE than JSON
+// numbers (8 binary bytes become 10.67 base64 chars), and the only thing that
+// beats dictionary-columnar is an opaque deflate blob that forfeits PLATFORM §7
+// and reships the whole column on every edit under collab.
+
+export const FORMAT = 'bento/dash'
+export const FORMAT_VERSION = 1
+
+/**
+ * Rules that could ever CHANGE are pinned as VALUES, never inferred from
+ * `version` — the SYNC_V lesson: discard what you cannot read rather than
+ * misread it. The OPEN half of the union is load-bearing: an unrecognised
+ * policy must PARSE, so the file opens and round-trips byte-exact with
+ * canonicalization and id repair disabled.
+ *
+ * Copied from spaces (spaces/src/model.ts:26), which reached it independently.
+ */
+export type Policy = 'bento-dash-1' | (string & {})
+
+/**
+ * Transform ops this build knows. `Step.op` is `string`, NOT this union — an
+ * unknown op must survive a round-trip, so the type system must not be able to
+ * express "only these". Same trick as spaces' KnownBlockType.
+ */
+export type KnownOp =
+ | 'import' | 'bind' | 'type' | 'filter' | 'derive' | 'sort'
+ | 'group' | 'join' | 'union' | 'pivot' | 'unpivot' | 'limit' | 'patch'
+
+export type ColumnType = 'text' | 'number' | 'money' | 'percent' | 'date' | 'bool' | 'enum'
+
+/**
+ * A column.
+ *
+ * PROPERTIES ARE FLAT, not nested under a `props` object, and that is a
+ * correctness decision rather than a style one. Under collaboration each
+ * (node, key) pair is one last-writer-wins register. With a nested `props` the
+ * whole object is ONE register: measured in spaces, one author ticking a field
+ * while another sets a second field loses one of the two edits SILENTLY.
+ * Slides' elements are flat for the same reason, spaces' blocks are flat for
+ * the same reason, and the shape is in every file ever saved — so it is free
+ * now and unrecoverable later.
+ */
+export interface Column {
+ /** Identity. `name` is display and position is neither — this is what
+ * formulas, overrides, tiles and CRDT ops all key off, so renaming a column
+ * or dragging it never breaks a reference. Kills the `#REF!` class and the
+ * shifted-VLOOKUP class outright. */
+ id: string
+ name: string
+ type: ColumnType
+ /** "$#,##0" — DOCUMENT data, because the author chose it. The decimal
+ * separator and grouping glyph are viewer chrome and live nowhere in here
+ * (PLATFORM §8 does not draw this line yet; see the open questions). */
+ format?: string
+ /** "USD". Makes unit-mismatch checkable rather than merely regrettable. */
+ unit?: string
+ /** Money computes in scaled integers; this is the scale. */
+ scale?: number
+ role?: 'key' | 'fk' | 'label'
+ /** ONE expression for the WHOLE column — the largest structural improvement
+ * over Excel available here. One stored string, one dependency-graph node,
+ * one vectorised pass: a 100k-row model with 12 formula columns has a
+ * 12-node graph, not a 1.2M-node one. */
+ formula?: string
+ /** The date/number format detected at import, kept so a re-import is
+ * reproducible and a mis-detection is visible rather than mysterious. */
+ parsed?: string
+ /** Values that would not coerce to `type`. Recorded, never hidden — a column
+ * that silently dropped 4% of its rows is a wrong answer wearing a right
+ * answer's clothes. */
+ failed?: number
+ w?: number
+ [extra: string]: unknown
+}
+
+/**
+ * Column storage. `dict` is the default and the reason the format fits.
+ *
+ * `pack` is an explicit, author-chosen ARCHIVE encoding for a frozen column:
+ * 1.8 bytes/cell, but it forfeits the readable-JSON promise of PLATFORM §7 and
+ * reships the whole column on every edit under collab. Never the default,
+ * never chosen by the app on the author's behalf.
+ */
+export type ColumnData =
+ | { enc: 'raw'; v: Array<number | string | boolean | null> }
+ | { enc: 'dict'; dict: string[]; idx: Array<number | null> }
+ | { enc: 'pack'; b64: string; n: number }
+
+/**
+ * The sparse overlay: only what DIFFERS from the column, keyed `<colId>:<rid>`.
+ * Homogeneous data costs nothing; the ragged 1% costs ~40 bytes each.
+ */
+export interface CellOverride {
+ /** A hand correction to imported data. */
+ v?: unknown
+ was?: unknown
+ /** A per-cell formula, for the cases a column expression cannot express. */
+ f?: string
+ /** Volatiles (`TODAY()`): the moment this value was committed. */
+ froze?: string
+ /**
+ * The dataset hash this override was made against.
+ *
+ * Row keys deliberately survive a re-import, so WITHOUT this an attributed
+ * human correction silently re-applies to fresh data — and every replica
+ * converges, on a wrong number. A non-matching override is SUSPENDED (kept,
+ * not applied, flagged) and raises `stale-override`. Unaddable after the
+ * first release, so it ships in v1.
+ */
+ againstHash?: string
+ by?: string
+ at?: string
+ why?: string
+ note?: string
+ color?: string
+ bg?: string
+ bold?: boolean
+ align?: string
+ [extra: string]: unknown
+}
+
+export interface PatchEdit {
+ col: string
+ v: unknown
+ was?: unknown
+ why?: string
+}
+
+/**
+ * One transform. `steps[0]` is ALWAYS the provenance record and is never re-run
+ * by recalculation; everything after it is re-executable by anyone who opens the
+ * file. That seam is exactly what `explain()` reports as its two tiers —
+ * `attested` upstream of the dataset hash, `reexecutable` from it forward.
+ *
+ * The `bind` op is RESERVED and unimplemented, and ships as a TYPE in v1 because
+ * a discriminant cannot be retrofitted. It carries a query NAME, parameters and
+ * a timestamp — never query text, never an endpoint, never a broker id. Those
+ * are SHELL configuration (docs/DECISIONS.md, 2026-07-28: "otherwise every deck
+ * anyone opens is an exfiltration tool against the database").
+ */
+export type Step =
+ | { op: 'import'; from: string; at: string; rows?: number; by?: string; note?: string }
+ | {
+ op: 'bind'; query: string; params?: Record<string, unknown>; at: string
+ rows?: number; attest?: { by?: string; sig?: string; note?: string }
+ }
+ | { op: 'type'; col: string; as: ColumnType; fmt?: string; failed?: number }
+ | { op: 'filter'; where: string }
+ | { op: 'derive'; col: string; name: string; expr: string; type?: ColumnType; unit?: string }
+ | { op: 'sort'; by: string; dir: 'asc' | 'desc' }
+ | { op: 'group'; by: string[]; agg: Array<{ fn: string; of?: string; as: string }> }
+ | { op: 'join'; with: string; on: [string, string]; card: 'one' | 'many'; fields: string[] }
+ /** Hand corrections to a DERIVED sheet, keyed by a business key rather than a
+ * row position, so a re-import cannot re-apply one to the wrong row. A patch
+ * whose key vanished is a `stale-patch` finding, not a disappearance. */
+ | { op: 'patch'; key: string[]; edits: PatchEdit[] }
+ /** Unknown ops survive parse → serialize and mark their descendants
+ * `unresolved`. Views then show last known values with a badge, NEVER zero —
+ * an empty bar chart reads as ZERO, which is a number, which is a lie. */
+ | { op: string; [k: string]: unknown }
+
+export interface TableSheet {
+ id: string
+ name: string
+ kind: 'table'
+ /**
+ * Row ids: minted at insert, never reused, run-length encoded as
+ * `[start, count]`. `[[1, 4200]]` is 20 bytes for 4200 rows.
+ *
+ * Rows stay POSITIONAL — in a spreadsheet, order is data — but every row has
+ * a stable key, which is what overrides, comments, patch edits and CRDT ops
+ * attach to.
+ */
+ rids: Array<[number, number]>
+ columns: Column[]
+ /** colId → encoded column. */
+ data: Record<string, ColumnData>
+ /** Keyed `<colId>:<rid>`. */
+ cells?: Record<string, CellOverride>
+ /** The Excel *table* totals row — correct because it cannot fall out of range
+ * when someone appends, which the `=SUM(B2:B41)` idiom cannot promise. */
+ totals?: Record<string, 'sum' | 'avg' | 'count' | 'min' | 'max' | { f: string }>
+ steps: Step[]
+ /** Derived sheet: the source sheet id. */
+ from?: string
+ [extra: string]: unknown
+}
+
+/**
+ * The classic sparse A1 map — the right tool for an invoice or a scratch pad,
+ * the wrong one for 100k rows. In the format from commit one, because an escape
+ * hatch added later is permanently second-class (PLATFORM §3).
+ */
+export interface CanvasSheet {
+ id: string
+ name: string
+ kind: 'canvas'
+ cells: Record<string, CanvasCell>
+ cols?: Record<string, number>
+ rows?: Record<string, number>
+ [extra: string]: unknown
+}
+
+export interface CanvasCell {
+ v?: unknown
+ f?: string
+ format?: string
+ note?: string
+ color?: string
+ bg?: string
+ bold?: boolean
+ align?: string
+ [extra: string]: unknown
+}
+
+export type Sheet = TableSheet | CanvasSheet
+
+/**
+ * A named measure. Measured at 7,289 bytes for a real six-entity model — 0.13%
+ * of a 5.6 MB dataset — so it is carried because it is nearly free, and because
+ * grain-mismatch and join-fanout are undetectable without it.
+ */
+export interface Measure {
+ name: string
+ expr: string
+ /** Sheet id — what ONE ROW of the source means. */
+ grain: string
+ additive: boolean | 'semi'
+ unit?: string
+ format?: string
+ /** "Agreed with finance 2026-Q1. Ex-intercompany." The sentence that makes a
+ * number arguable rather than merely present. */
+ desc?: string
+ owner?: string
+ since?: string
+ [extra: string]: unknown
+}
+
+export interface Tile {
+ kind: 'chart' | 'kpi' | 'slice' | 'text' | 'gl'
+ x: number
+ y: number
+ w: number
+ h: number
+ /** A tile NAMES a sheet and columns. Series arrays are DERIVED at render and
+ * never stored — slides' `syncLinkedChart` promoted from a special case to
+ * the rule. A chart then cannot disagree with its data, cannot go stale, and
+ * does not double the file. */
+ bind?: {
+ sheet: string; x?: string; by?: string; series?: string[]
+ measure?: string; agg?: string
+ }
+ option?: Record<string, unknown>
+ /** REQUIRED on `kind:'gl'`: what to draw where WebGL is unavailable. */
+ fallback?: 'heatmap' | 'scatter2d' | 'table'
+ [extra: string]: unknown
+}
+
+export interface View {
+ id: string
+ name: string
+ w: number
+ h: number
+ tiles: Tile[]
+ [extra: string]: unknown
+}
+
+export interface Theme {
+ background: string
+ color: string
+ accent: string
+ fontFamily: string
+ headingFamily?: string
+ dir?: 'ltr' | 'rtl'
+ [extra: string]: unknown
+}
+
+export interface DocMeta {
+ author?: string
+ company?: string
+ subject?: string
+ event?: string
+ keywords?: string
+ [extra: string]: unknown
+}
+
+export interface DashDoc {
+ format: typeof FORMAT
+ version: number
+ policy?: Policy
+ /** Minted once at creation/load and NEVER regenerated (PLATFORM §3). */
+ docId: string
+ title: string
+ modified?: string
+ meta?: DocMeta
+ sheets: Sheet[]
+ measures?: Record<string, Measure>
+ names?: Record<string, { v: number | string; note?: string }>
+ views?: View[]
+ theme?: Theme
+ /**
+ * Source files kept as provenance, and images in views. NEVER column data:
+ * the CRDT offloads asset strings over 64 KB and an unresolvable reference
+ * renders as ''. An absent image reads as absent; an absent column of numbers
+ * reads as data.
+ */
+ assets?: Record<string, string>
+ collab?: unknown
+ blobs?: Record<string, { key: string; mime: string; size: number }>
+ readonly?: boolean
+ template?: boolean
+ [extra: string]: unknown
+}
+
+// --- Budgets. App constants, never kernel — nothing in kernel/src reads an
+// app's budgets, and these are dash's alone.
+//
+// There is NO row cap. Rows are the wrong unit: 1M × 12 is 65 MB but 1M × 2 is
+// ~10 MB and comfortable, so a row limit refuses workable files and admits
+// unworkable ones (docs/DECISIONS.md, 2026-08-02, "dash budgets BYTES with
+// consent, not rows with a refusal"). The budget is bytes, it scales with
+// whether this browser can write in place, and past it the user is told what
+// will break — never refused.
+
+/** Where the grid starts warning. A target, not a cap. */
+export const ROW_WARN = 250_000
+/** Document JSON, where saving is still imperceptible with an in-place write. */
+export const DOC_BUDGET_FSA = 25 * 1024 * 1024
+/**
+ * Without the File System Access API — Safari, Firefox, every iOS browser —
+ * every save downloads the WHOLE file to ~/Downloads, so the practical ceiling
+ * is far lower. `canWriteInPlace()` (kernel/src/save.ts) reports which we are in.
+ */
+export const DOC_BUDGET_DOWNLOAD = 8 * 1024 * 1024
+/** A canvas sheet past this should have been a table sheet. */
+export const CANVAS_CELL_MAX = 50_000
+
+export const docBudget = (canWriteInPlace: boolean): number =>
+ canWriteInPlace ? DOC_BUDGET_FSA : DOC_BUDGET_DOWNLOAD
+
+// --- Parsing -------------------------------------------------------------
+
+export interface Repair {
+ what: 'sheet-id' | 'column-id' | 'duplicate-id' | 'missing-id'
+ id: string
+ became: string
+}
+
+/**
+ * The result of reading a `#bento-doc` block.
+ *
+ * NEVER null-then-starter. `parseDoc` returning null let the caller fall back to
+ * the starter document, which means opening a slides file — or a workbook with
+ * one hand-edited typo — gave you an EMPTY workbook over live data, and the
+ * first ⌘S wrote it to disk. The ONLY path to the starter is an absent or empty
+ * block.
+ *
+ * Shape adopted from the shipped spaces implementation (spaces/src/model.ts:156)
+ * rather than dash's design doc, which made `frozen` a third discriminant and so
+ * forced every consumer into a three-way switch for a flag only two of them read.
+ *
+ * `unreadable` is dash's addition, and it exists because the kernel cannot tell
+ * the two empty cases apart: `readEmbeddedDoc` (kernel/src/save.ts:42) returns
+ * `text || null`, collapsing "no #bento-doc element" and "element present with
+ * an empty text node" into one answer. Measured: past the parser's limit the
+ * block IS present with exactly one zero-length text node and no throw — so
+ * treating that as 'empty' boots the starter over live data, which is the exact
+ * loss this type exists to prevent. The boot site decides it, because only the
+ * boot site can still see the DOM.
+ */
+export type ParseResult =
+ | { ok: true; doc: DashDoc; repairs: Repair[]; frozen?: 'policy' | 'version' }
+ | { ok: false; err: 'empty' }
+ | { ok: false; err: 'unreadable'; detail: string }
+ | { ok: false; err: 'json' | 'format' | 'shape'; detail: string; found?: string }
+
+let idSeq = 0
+/** Deterministic within a parse, so two readers of one file repair identically. */
+const mintId = (prefix: string, seed: string): string =>
+ `${prefix}-${hash32(`${seed}:${idSeq++}`)}`
+
+/**
+ * A stable 32-bit hash of the bytes. NOT Math.random (two readers of one file
+ * would diverge) and NOT docId (which `template: true` re-mints on every open,
+ * so a docId-derived id gives two readers DIFFERENT ids — the exact failure
+ * repair exists to prevent). Copied in spirit from spaces' id repair.
+ */
+function hash32(s: string): string {
+ let h = 2166136261
+ for (let i = 0; i < s.length; i++) {
+ h ^= s.charCodeAt(i)
+ h = Math.imul(h, 16777619)
+ }
+ return (h >>> 0).toString(36)
+}
+
+const isObj = (v: unknown): v is Record<string, unknown> =>
+ typeof v === 'object' && v !== null && !Array.isArray(v)
+
+/**
+ * Read a `#bento-doc` body into a document, or say precisely why not.
+ *
+ * Never throws, never returns null, never silently discards a field it did not
+ * recognise — every level spreads the source object so unknown keys survive.
+ */
+export function parseDoc(json: string): ParseResult {
+ if (!json || !json.trim()) return { ok: false, err: 'empty' }
+
+ let raw: unknown
+ try {
+ raw = JSON.parse(json)
+ } catch (e) {
+ return { ok: false, err: 'json', detail: e instanceof Error ? e.message : String(e) }
+ }
+ if (!isObj(raw)) {
+ return { ok: false, err: 'shape', detail: 'the document block is not a JSON object' }
+ }
+ if (raw.format !== FORMAT) {
+ const found = typeof raw.format === 'string' ? raw.format : undefined
+ return {
+ ok: false,
+ err: 'format',
+ detail: found
+ ? `this document declares ${found}, which needs its own app`
+ : 'this document does not declare a bento format',
+ found,
+ }
+ }
+ if (!Array.isArray(raw.sheets)) {
+ return { ok: false, err: 'shape', detail: 'a bento/dash document needs a "sheets" array' }
+ }
+
+ const version = typeof raw.version === 'number' ? raw.version : FORMAT_VERSION
+ const policy = typeof raw.policy === 'string' ? raw.policy : undefined
+ // Version wins if both: a future version implies rules we cannot name.
+ const frozen: 'policy' | 'version' | undefined =
+ version > FORMAT_VERSION ? 'version'
+ : policy !== undefined && policy !== 'bento-dash-1' ? 'policy'
+ : undefined
+
+ const repairs: Repair[] = []
+ idSeq = 0
+
+ /**
+ * Claim an id within one namespace, minting a replacement if it is missing or
+ * already taken.
+ *
+ * Frozen documents are NEVER rewritten, not even a duplicate: the file
+ * declares rules this build does not have, so it round-trips byte-exact.
+ */
+ const claimIn = (seen: Set<string>, id: unknown, prefix: string): string => {
+ const s = typeof id === 'string' ? id : ''
+ if (frozen) return s
+ if (s && !seen.has(s)) { seen.add(s); return s }
+ const became = mintId(prefix, s || prefix)
+ repairs.push({ what: s ? 'duplicate-id' : 'missing-id', id: s, became })
+ seen.add(became)
+ return became
+ }
+
+ // Sheet ids are document-scoped. COLUMN ids are SHEET-scoped: two sheets may
+ // both call a column `c1`, and that is ordinary, not a collision. A shared
+ // namespace would rename the second one — and renaming a column orphans it,
+ // because `data`, `cells` and `totals` are all keyed by the id it used to
+ // have. Measured before this was fixed: the column kept its new id, `data`
+ // kept the old key, and the column rendered EMPTY. Silent loss, of exactly
+ // the kind this whole type exists to prevent.
+ const sheetIds = new Set<string>()
+
+ const sheets: Sheet[] = (raw.sheets as unknown[]).map((s) => {
+ if (!isObj(s)) {
+ return {
+ id: claimIn(sheetIds, '', 'sh'), name: '', kind: 'table',
+ rids: [], columns: [], data: {}, steps: [],
+ } as unknown as Sheet
+ }
+ const id = claimIn(sheetIds, s.id, 'sh')
+ if (s.kind === 'canvas') {
+ return { ...s, id, kind: 'canvas', cells: isObj(s.cells) ? s.cells : {} } as unknown as CanvasSheet
+ }
+
+ const colIds = new Set<string>()
+ const renamed = new Map<string, string>()
+ const columns = Array.isArray(s.columns)
+ ? (s.columns as unknown[]).map((c) => {
+ if (!isObj(c)) return c as unknown as Column
+ const was = typeof c.id === 'string' ? c.id : ''
+ const now = claimIn(colIds, c.id, 'col')
+ if (was && now !== was) renamed.set(was, now)
+ return { ...c, id: now } as unknown as Column
+ })
+ : []
+
+ /** A repaired column id takes its data with it, or the repair is the loss. */
+ const remap = <T,>(rec: unknown, keyOf: (k: string) => string): Record<string, T> => {
+ const src = isObj(rec) ? rec : {}
+ if (!renamed.size) return src as Record<string, T>
+ const out: Record<string, T> = {}
+ for (const [k, v] of Object.entries(src)) out[keyOf(k)] = v as T
+ return out
+ }
+
+ return {
+ ...s,
+ id,
+ kind: 'table',
+ rids: Array.isArray(s.rids) ? s.rids : [],
+ columns,
+ data: remap<ColumnData>(s.data, (k) => renamed.get(k) ?? k),
+ // `cells` is keyed "<colId>:<rid>" — only the column half moves.
+ ...(isObj(s.cells)
+ ? { cells: remap<CellOverride>(s.cells, (k) => {
+ const i = k.indexOf(':')
+ if (i < 0) return k
+ const col = k.slice(0, i)
+ return `${renamed.get(col) ?? col}:${k.slice(i + 1)}`
+ }) }
+ : {}),
+ ...(isObj(s.totals)
+ ? { totals: remap<NonNullable<TableSheet['totals']>[string]>(s.totals, (k) => renamed.get(k) ?? k) }
+ : {}),
+ steps: Array.isArray(s.steps) ? s.steps : [],
+ } as unknown as TableSheet
+ })
+
+ const doc = { ...raw, format: FORMAT, version, sheets } as unknown as DashDoc
+ return frozen ? { ok: true, doc, repairs, frozen } : { ok: true, doc, repairs }
+}
+
+/**
+ * Bytes this document would occupy in the file. `#bento-doc` is plaintext
+ * forever (PLATFORM §2), so file size IS JSON size — there is no compression
+ * between the model and the disk, which is why the budget can be checked here.
+ */
+export const docBytes = (doc: DashDoc): number => JSON.stringify(doc).length
+
+/** Total rows across every table sheet, from the run-length encoded ids. */
+export const rowCount = (doc: DashDoc): number =>
+ doc.sheets.reduce(
+ (n, s) => n + (s.kind === 'table' ? s.rids.reduce((m, [, c]) => m + c, 0) : 0),
+ 0,
+ )
diff --git a/scripts/test-dash-model.ts b/scripts/test-dash-model.ts
new file mode 100644
index 00000000..00da8ad1
--- /dev/null
+++ b/scripts/test-dash-model.ts
@@ -0,0 +1,215 @@
+#!/usr/bin/env node
+// SPDX-License-Identifier: MIT
+// Copyright (c) 2026 The Bento authors
+// bento/dash document model rig.
+//
+// node scripts/test-dash-model.ts (Node ≥ 23.6 strips types natively)
+//
+// WHAT THIS PROVES. The format is permanent — additive forever, with no server
+// to migrate a single file (PLATFORM §3) — so the two ways it can be got wrong
+// are both unrecoverable once files exist:
+//
+// 1. LOSS ON READ. A field, a per-node key or a transform op this build does
+// not recognise must survive parse → serialize. An older build meeting a
+// future op keeps it; it does not quietly drop it and then save the
+// result over the user's file.
+// 2. THE STARTER OVER LIVE DATA. `parseDoc` returning null let spaces' caller
+// fall back to the starter, so opening a slides file gave you an empty
+// document over real content and the first ⌘S made it permanent. Only an
+// ABSENT or EMPTY block may reach the starter. Everything else refuses and
+// says what it found.
+//
+// Both are checked from both sides: the rig asserts the good case AND that a
+// naive implementation (reconstruct-the-known-fields; return null on failure)
+// would fail these very assertions.
+
+import {
+ FORMAT, FORMAT_VERSION, parseDoc, docBytes, rowCount, docBudget,
+ DOC_BUDGET_FSA, DOC_BUDGET_DOWNLOAD,
+ type DashDoc,
+} from '../dash/src/model.ts'
+
+let failures = 0
+let checks = 0
+function ok(cond: boolean, msg: string) {
+ checks++
+ if (!cond) { failures++; console.log(` FAIL ${msg}`) }
+ else console.log(` ok ${msg}`)
+}
+
+const minimal = () => ({
+ format: FORMAT,
+ version: FORMAT_VERSION,
+ policy: 'bento-dash-1',
+ docId: 'doc-test',
+ title: 'test',
+ sheets: [{
+ id: 'sh1', name: 'Sheet 1', kind: 'table',
+ rids: [[1, 3]],
+ columns: [{ id: 'c1', name: 'Region', type: 'text' }],
+ data: { c1: { enc: 'dict', dict: ['North', 'South'], idx: [0, 1, 0] } },
+ steps: [{ op: 'import', from: 'q3.csv', at: '2026-08-03T00:00:00Z', rows: 3 }],
+ }],
+})
+
+// ---------------------------------------------------------------- the states
+ok(parseDoc('').ok === false && (parseDoc('') as any).err === 'empty',
+ 'an empty block is err:"empty" — the ONLY state that may boot the starter')
+ok((parseDoc(' ') as any).err === 'empty', 'whitespace is empty too')
+ok((parseDoc('{oops') as any).err === 'json', 'malformed JSON is err:"json", not empty')
+ok((parseDoc('[]') as any).err === 'shape', 'a JSON array is err:"shape", not empty')
+ok((parseDoc('{"format":"bento/slides","slides":[]}') as any).err === 'format',
+ 'a slides document is err:"format", not empty')
+ok((parseDoc('{"format":"bento/slides","slides":[]}') as any).found === 'bento/slides',
+ 'the format failure reports what it actually found, so the gate can name it')
+ok((parseDoc(JSON.stringify({ format: FORMAT, version: 1, docId: 'd', title: 't' })) as any).err === 'shape',
+ 'a bento/dash document with no sheets array is err:"shape"')
+ok(parseDoc(JSON.stringify(minimal())).ok === true, 'a well-formed document parses')
+
+// THE POINT OF THE TYPE: every failure is distinguishable from 'empty'.
+const failures_ = ['{oops', '[]', '{"format":"bento/slides"}', '{"format":"bento/dash","version":1}']
+ok(failures_.every((j) => { const r = parseDoc(j) as any; return r.ok === false && r.err !== 'empty' }),
+ 'NO malformed input is ever reported as empty — the starter is unreachable from a parse failure')
+
+// The tag the kernel cannot produce: present-but-unreadable is not empty.
+// (kernel readEmbeddedDoc collapses both to null; the boot site must distinguish.)
+type PR = ReturnType<typeof parseDoc>
+const unreadable: PR = { ok: false, err: 'unreadable', detail: 'block present, body unreadable' }
+ok(unreadable.ok === false && unreadable.err === 'unreadable',
+ 'the ParseResult type can express "present but unreadable" as distinct from "empty"')
+
+// ------------------------------------------------------------- additivity
+const exotic = {
+ ...minimal(),
+ futureTopLevel: { anything: [1, 2, 3] },
+ sheets: [{
+ ...minimal().sheets[0],
+ futureSheetKey: 'kept',
+ columns: [{ id: 'c1', name: 'Region', type: 'text', futureColumnKey: 42 }],
+ steps: [
+ { op: 'import', from: 'q3.csv', at: '2026-08-03T00:00:00Z' },
+ { op: 'kanban-pivot', mystery: true, nested: { deep: ['x'] } },
+ ],
+ }],
+}
+const parsedExotic = parseDoc(JSON.stringify(exotic))
+ok(parsedExotic.ok === true, 'a document carrying unknown fields and an unknown op still parses')
+if (parsedExotic.ok) {
+ const d = parsedExotic.doc as any
+ ok(d.futureTopLevel?.anything?.length === 3, 'an unknown TOP-LEVEL field survives')
+ ok(d.sheets[0].futureSheetKey === 'kept', 'an unknown SHEET field survives')
+ ok(d.sheets[0].columns[0].futureColumnKey === 42, 'an unknown COLUMN field survives')
+ ok(d.sheets[0].steps[1].op === 'kanban-pivot' && d.sheets[0].steps[1].nested.deep[0] === 'x',
+ 'an unknown transform OP survives whole, including nested values')
+ // the assertion a reconstruct-the-known-fields parser fails
+ ok(JSON.stringify(d) === JSON.stringify({ ...exotic, format: FORMAT, version: FORMAT_VERSION }),
+ 'parse → serialize is byte-identical for a document with nothing to repair')
+}
+
+// ------------------------------------------------------------------ frozen
+const futureVersion = parseDoc(JSON.stringify({ ...minimal(), version: FORMAT_VERSION + 1 }))
+ok(futureVersion.ok === true && futureVersion.frozen === 'version',
+ 'a future version OPENS, frozen — it does not refuse')
+const futurePolicy = parseDoc(JSON.stringify({ ...minimal(), policy: 'bento-dash-2' }))
+ok(futurePolicy.ok === true && futurePolicy.frozen === 'policy', 'an unknown policy opens frozen')
+const both = parseDoc(JSON.stringify({ ...minimal(), version: 9, policy: 'bento-dash-9' }))
+ok(both.ok === true && both.frozen === 'version', 'version wins when both are unknown')
+
+// A frozen document is never rewritten, not even a duplicate id: the file
+// declares rules this build does not have, so it round-trips byte-exact.
+const dupFrozen = parseDoc(JSON.stringify({
+ ...minimal(), version: 9,
+ sheets: [{ ...minimal().sheets[0], id: 'same' }, { ...minimal().sheets[0], id: 'same' }],
+}))
+ok(dupFrozen.ok === true && dupFrozen.repairs.length === 0,
+ 'a FROZEN document repairs nothing, even a duplicate id')
+
+// -------------------------------------------------------------- id repair
+const dup = JSON.stringify({
+ ...minimal(),
+ sheets: [{ ...minimal().sheets[0], id: 'same' }, { ...minimal().sheets[0], id: 'same' }],
+})
+const r1 = parseDoc(dup)
+const r2 = parseDoc(dup)
+ok(r1.ok === true && r1.repairs.length === 1, 'a duplicate sheet id is repaired, and reported')
+ok(r1.ok === true && r2.ok === true
+ && JSON.stringify(r1.repairs) === JSON.stringify(r2.repairs),
+ 'repair is DETERMINISTIC — two readers of one file produce the same ids')
+if (r1.ok && r2.ok) {
+ ok((r1.doc.sheets[1] as any).id === (r2.doc.sheets[1] as any).id,
+ 'and therefore two readers agree on what the repaired sheet is called')
+ ok((r1.doc.sheets[0] as any).id === 'same', 'the FIRST claimant keeps the id')
+}
+const missing = parseDoc(JSON.stringify({
+ ...minimal(), sheets: [{ ...minimal().sheets[0], id: undefined }],
+}))
+ok(missing.ok === true && missing.repairs[0]?.what === 'missing-id',
+ 'a missing id is repaired and reported as missing, not as a duplicate')
+
+// ------------------------------------------- column ids are SHEET-scoped
+// Two sheets may both call a column `c1`. Sharing a namespace would rename the
+// second, and a renamed column orphans its own data — measured: the column kept
+// the new id, `data` kept the old key, and the column rendered EMPTY.
+const twoSheets = (id: string) => ({
+ id, name: 'S', kind: 'table', rids: [[1, 2]],
+ columns: [{ id: 'c1', name: 'Region', type: 'text' }],
+ data: { c1: { enc: 'raw', v: ['North', 'South'] } },
+ cells: { 'c1:1': { note: 'checked' } },
+ totals: { c1: 'count' },
+ steps: [],
+})
+const scoped = parseDoc(JSON.stringify({
+ ...minimal(), sheets: [twoSheets('a'), twoSheets('b')],
+}))
+ok(scoped.ok === true && scoped.repairs.length === 0,
+ 'the same column id in two different sheets is NOT a collision')
+if (scoped.ok) {
+ const s2 = scoped.doc.sheets[1] as any
+ ok(s2.columns[0].id === 'c1' && Object.keys(s2.data)[0] === 'c1',
+ 'and neither sheet is rewritten, so no column is orphaned')
+}
+
+// When a column id genuinely IS duplicated within one sheet, the repair must
+// carry the data with it or the repair is itself the loss.
+const dupCol = parseDoc(JSON.stringify({
+ ...minimal(),
+ sheets: [{
+ id: 'sh1', name: 'S', kind: 'table', rids: [[1, 2]],
+ columns: [
+ { id: 'c1', name: 'A', type: 'text' },
+ { id: 'c1', name: 'B', type: 'number' },
+ ],
+ data: { c1: { enc: 'raw', v: [1, 2] } },
+ cells: { 'c1:1': { note: 'kept' } },
+ totals: { c1: 'sum' },
+ steps: [],
+ }],
+}))
+ok(dupCol.ok === true && dupCol.repairs.length === 1,
+ 'a duplicate column id WITHIN one sheet is repaired once')
+if (dupCol.ok) {
+ const sh = dupCol.doc.sheets[0] as any
+ const ids = sh.columns.map((c: any) => c.id)
+ ok(new Set(ids).size === 2, 'the two columns end up with distinct ids')
+ ok(Object.keys(sh.data).every((k: string) => ids.includes(k)),
+ 'every data key still names a column that exists — no orphaned column data')
+ ok(Object.keys(sh.cells).every((k: string) => ids.includes(k.split(':')[0])),
+ 'cell overrides follow the rename ("<colId>:<rid>" — only the column half moves)')
+ ok(Object.keys(sh.totals).every((k: string) => ids.includes(k)),
+ 'the totals row follows the rename')
+}
+
+// --------------------------------------------------------------- budgets
+const doc = (parseDoc(JSON.stringify(minimal())) as any).doc as DashDoc
+ok(rowCount(doc) === 3, 'rowCount reads the run-length encoded ids')
+ok(docBytes(doc) === JSON.stringify(doc).length,
+ 'docBytes is the serialized length — #bento-doc is plaintext, so file size IS JSON size')
+ok(docBudget(true) === DOC_BUDGET_FSA && docBudget(false) === DOC_BUDGET_DOWNLOAD,
+ 'the budget scales with whether this browser can write in place')
+ok(DOC_BUDGET_DOWNLOAD < DOC_BUDGET_FSA,
+ 'and the download-only budget is the SMALLER one — every save rewrites the whole file')
+ok(!Object.keys({ FORMAT, FORMAT_VERSION }).includes('ROW_MAX'),
+ 'there is no row cap: rows are the wrong unit (1M x 12 is 65 MB; 1M x 2 is ~10 MB)')
+
+console.log(`\n${checks - failures}/${checks} checks passed`)
+if (failures) process.exit(1)