Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
feat(databases): support non-nullable columns (#7708)
Wolf Mermelstein8aef269a0830
diff --git a/.sqlx/query-3b505e6d80b018da1a1ade0143f3ce5ab67125262bea35d4078e850471726dd8.json b/.sqlx/query-3b505e6d80b018da1a1ade0143f3ce5ab67125262bea35d4078e850471726dd8.jsonnew file mode 100644index 0000000000..1830129e85--- /dev/null+++ b/.sqlx/query-3b505e6d80b018da1a1ade0143f3ce5ab67125262bea35d4078e850471726dd8.json@@ -0,0 +1,24 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT id FROM database_rows WHERE table_id = $1\n AND ($2::uuid[] IS NULL OR id = ANY($2))\n AND ($3::uuid IS NULL OR id > $3) ORDER BY id LIMIT 1000",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "UuidArray",+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "3b505e6d80b018da1a1ade0143f3ce5ab67125262bea35d4078e850471726dd8"+}diff --git a/.sqlx/query-12a6a163774f963d047a613a09cf72849e24304152b02469de21e623d5ca1b6f.json b/.sqlx/query-85b5defacd5cd3080925a196cc2346b61cf1e2480baefc962ce3bc7922a7f972.jsonsimilarity index 56%rename from .sqlx/query-12a6a163774f963d047a613a09cf72849e24304152b02469de21e623d5ca1b6f.jsonrename to .sqlx/query-85b5defacd5cd3080925a196cc2346b61cf1e2480baefc962ce3bc7922a7f972.jsonindex a41f3ffc07..46d54c50a6 100644--- a/.sqlx/query-12a6a163774f963d047a613a09cf72849e24304152b02469de21e623d5ca1b6f.json+++ b/.sqlx/query-85b5defacd5cd3080925a196cc2346b61cf1e2480baefc962ce3bc7922a7f972.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n INSERT INTO database_columns (id, table_id, property_definition_id, position, config, infer_type)\n VALUES ($1, $2, $3, $4, $5, $6)\n ",+ "query": "\n INSERT INTO database_columns (id, table_id, property_definition_id, position, config, infer_type, nullable)\n VALUES ($1, $2, $3, $4, $5, $6, $7)\n ","describe": {"columns": [],"parameters": {@@ -10,10 +10,11 @@"Uuid","Text","Jsonb",+ "Bool","Bool"]},"nullable": []},- "hash": "12a6a163774f963d047a613a09cf72849e24304152b02469de21e623d5ca1b6f"+ "hash": "85b5defacd5cd3080925a196cc2346b61cf1e2480baefc962ce3bc7922a7f972"}diff --git a/.sqlx/query-8e8e07d0c17b8fadf9066957557860fbcc8f29ff07db355a0f09b4e096f25f57.json b/.sqlx/query-a920e0df460f481ee4ae05606fa7e50c404dc5e2a90beaeca08c531b639a216d.jsonsimilarity index 67%rename from .sqlx/query-8e8e07d0c17b8fadf9066957557860fbcc8f29ff07db355a0f09b4e096f25f57.jsonrename to .sqlx/query-a920e0df460f481ee4ae05606fa7e50c404dc5e2a90beaeca08c531b639a216d.jsonindex 1e84d4d3df..32fe760cc9 100644--- a/.sqlx/query-8e8e07d0c17b8fadf9066957557860fbcc8f29ff07db355a0f09b4e096f25f57.json+++ b/.sqlx/query-a920e0df460f481ee4ae05606fa7e50c404dc5e2a90beaeca08c531b639a216d.json@@ -1,6 +1,6 @@{"db_name": "PostgreSQL",- "query": "\n SELECT id, table_id, property_definition_id, position, config, display_name, infer_type,\n ARRAY(SELECT capability FROM database_column_protections p WHERE p.column_id = database_columns.id ORDER BY capability) AS \"protections!\"\n FROM database_columns\n WHERE table_id = ANY($1)\n ORDER BY table_id, position\n ",+ "query": "\n SELECT id, table_id, property_definition_id, position, config, display_name, infer_type, nullable,\n ARRAY(SELECT capability FROM database_column_protections p WHERE p.column_id = database_columns.id ORDER BY capability) AS \"protections!\"\n FROM database_columns\n WHERE table_id = ANY($1)\n ORDER BY table_id, position\n ","describe": {"columns": [{@@ -40,6 +40,11 @@},{"ordinal": 7,+ "name": "nullable",+ "type_info": "Bool"+ },+ {+ "ordinal": 8,"name": "protections!","type_info": "TextArray"}@@ -57,8 +62,9 @@true,true,false,+ false,null]},- "hash": "8e8e07d0c17b8fadf9066957557860fbcc8f29ff07db355a0f09b4e096f25f57"+ "hash": "a920e0df460f481ee4ae05606fa7e50c404dc5e2a90beaeca08c531b639a216d"}diff --git a/.sqlx/query-da9c5cdb1954ef50046075785d4558b7d5b0da80aa7e75c56a5f8c1fe3de3fa0.json b/.sqlx/query-da9c5cdb1954ef50046075785d4558b7d5b0da80aa7e75c56a5f8c1fe3de3fa0.jsonnew file mode 100644index 0000000000..74362d1d56--- /dev/null+++ b/.sqlx/query-da9c5cdb1954ef50046075785d4558b7d5b0da80aa7e75c56a5f8c1fe3de3fa0.json@@ -0,0 +1,28 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT id, property_definition_id FROM database_columns WHERE table_id = $1 AND NOT nullable ORDER BY id",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "property_definition_id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "da9c5cdb1954ef50046075785d4558b7d5b0da80aa7e75c56a5f8c1fe3de3fa0"+}diff --git a/apps/web/src/lib/service-clients/service-storage/generated/schemas/column.ts b/apps/web/src/lib/service-clients/service-storage/generated/schemas/column.tsindex 168b4b085d..55885af521 100644--- a/apps/web/src/lib/service-clients/service-storage/generated/schemas/column.ts+++ b/apps/web/src/lib/service-clients/service-storage/generated/schemas/column.ts@@ -23,6 +23,8 @@ the definition's name. */id: string;/** Whether the first nonempty value may settle this new text column's type. */infer_type: boolean;+ /** Whether a row may omit this cell; empty collections also count as absent. */+ nullable?: boolean;/** Fractional index for column ordering. */position: string;/** The bound property definition. */diff --git a/apps/web/src/lib/service-clients/service-storage/generated/zod.ts b/apps/web/src/lib/service-clients/service-storage/generated/zod.tsindex 2ba3b2d8d5..fbc0ff882a 100644--- a/apps/web/src/lib/service-clients/service-storage/generated/zod.ts+++ b/apps/web/src/lib/service-clients/service-storage/generated/zod.ts@@ -4830,6 +4830,12 @@ export const getDatabaseResponse = zod.describe("Whether the first nonempty value may settle this new text column's type."),+ nullable: zod+ .boolean()+ .optional()+ .describe(+ 'Whether a row may omit this cell; empty collections also count as absent.'+ ),position: zod.string().describe('Fractional index for column ordering.'),@@ -9244,6 +9250,12 @@ export const inferDatabaseColumnTypeResponse = zod.describe("Whether the first nonempty value may settle this new text column's type."),+ nullable: zod+ .boolean()+ .optional()+ .describe(+ 'Whether a row may omit this cell; empty collections also count as absent.'+ ),position: zod.string().describe('Fractional index for column ordering.'),diff --git a/apps/web/src/lib/service-clients/service-storage/openapi.json b/apps/web/src/lib/service-clients/service-storage/openapi.jsonindex 9f566c6864..6b7343de18 100644--- a/apps/web/src/lib/service-clients/service-storage/openapi.json+++ b/apps/web/src/lib/service-clients/service-storage/openapi.json@@ -21081,6 +21081,10 @@"type": "boolean","description": "Whether the first nonempty value may settle this new text column's type."},+ "nullable": {+ "type": "boolean",+ "description": "Whether a row may omit this cell; empty collections also count as absent."+ },"position": {"type": "string","description": "Fractional index for column ordering."diff --git a/crates/databases/README.md b/crates/databases/README.mdindex 4ec3211c35..89ab5e0933 100644--- a/crates/databases/README.md+++ b/crates/databases/README.md@@ -147,3 +147,19 @@ The suite covers permission scoping, typed round trips, relations, safe casts,rollback, stale/concurrent writes, sharing, typed views and card moves, andretry-safe import/starter provisioning. SQLx tests create isolated databasesusing the repository migrator.++### Required columns++A trusted host can create a column with `Column.nullable = false` through+`DatabaseStorage::apply_storage_writes`. Existing columns and ordinary user-created+columns default to nullable. This does not add a UI toggle for requiredness.++The planner rejects omitted required cells on insert and explicit clears on edit.+Both write ports also check affected required cells in the final transaction state+under table locks, including schema changes and undo. A batch can add a required+column and populate existing rows atomically. Empty selections, entity-reference+lists and link lists are absent; empty text, zero and false are present. Undo+restores the column setting alongside its cells.++Enforcement belongs to the Rust database engine. Direct SQL and writes through+unrelated property APIs bypass it; the migration only adds the nullable flag.diff --git a/crates/databases/src/domain/catalog.rs b/crates/databases/src/domain/catalog.rsindex 763d015e8f..d76bf6dd2d 100644--- a/crates/databases/src/domain/catalog.rs+++ b/crates/databases/src/domain/catalog.rs@@ -309,6 +309,7 @@ pub fn schema_image(entries: &[TableEntry]) -> crate::domain::journal::SchemaImaoptions.sort_by_key(|option| option.display_order);image.columns.push(ColumnImage {infer_type: column.column.infer_type,+ nullable: column.column.nullable,id: column.column.id,table: entry.table.id,name: column.name().to_owned(),diff --git a/crates/databases/src/domain/journal.rs b/crates/databases/src/domain/journal.rsindex a8c5af04b5..894cfd631a 100644--- a/crates/databases/src/domain/journal.rs+++ b/crates/databases/src/domain/journal.rs@@ -132,6 +132,9 @@ pub struct RestoredColumn {pub kind: Option<ColumnKind>,/// Whether the first value should still infer a type.pub infer_type: bool,+ /// Whether this column permits absent values.+ #[serde(default = "super::models::column_nullable_default")]+ pub nullable: bool,}/// A row a delete removed: its id and place.@@ -369,6 +372,8 @@ pub struct TableImage {pub struct ColumnImage {/// Whether a first value still determines the column's type.pub infer_type: bool,+ /// Whether this column permits absent values.+ pub nullable: bool,/// The column.pub id: ColumnId,/// Its table.diff --git a/crates/databases/src/domain/journal/invert.rs b/crates/databases/src/domain/journal/invert.rsindex 79c9affce0..32ea3a93f7 100644--- a/crates/databases/src/domain/journal/invert.rs+++ b/crates/databases/src/domain/journal/invert.rs@@ -247,6 +247,7 @@ impl Inverter<'_> {super::RestoredColumn {kind: image.kind,infer_type: image.infer_type,+ nullable: image.nullable,},);self.push(DatabaseOp::Column {diff --git a/crates/databases/src/domain/journal/test.rs b/crates/databases/src/domain/journal/test.rsindex b0beac6db7..efe2a23535 100644--- a/crates/databases/src/domain/journal/test.rs+++ b/crates/databases/src/domain/journal/test.rs@@ -113,6 +113,7 @@ fn schema() -> SchemaImage {],columns: vec![ColumnImage {+ nullable: true,id: NAME,infer_type: false,table: GUESTS,@@ -123,6 +124,7 @@ fn schema() -> SchemaImage {options: Vec::new(),},ColumnImage {+ nullable: true,id: RSVP,infer_type: false,table: GUESTS,@@ -149,6 +151,7 @@ fn schema() -> SchemaImage {],},ColumnImage {+ nullable: true,id: PLUS_ONES,infer_type: false,table: GUESTS,@@ -409,6 +412,7 @@ fn a_column_create_is_undone_by_deleting_the_column() {let write = Write::CreateColumn {column: Column {protections: vec![],+ nullable: true,id: seats,table_id: GUESTS,property_definition_id: Uuid::from_u128(0xe9),@@ -645,6 +649,7 @@ fn a_type_change_is_undone_by_the_old_type_with_the_old_definition_and_cells() {replacement: ColumnReplacement {column: Column {protections: vec![],+ nullable: true,id: PLUS_ONES,table_id: GUESTS,property_definition_id: PLUS_ONES_DEFINITION,@@ -1221,6 +1226,7 @@ fn what_a_batch_creates_and_removes_again_needs_no_inverse() {Write::CreateColumn {column: Column {protections: vec![],+ nullable: true,id: seats,table_id: GUESTS,property_definition_id: Uuid::from_u128(0xe9),diff --git a/crates/databases/src/domain/models.rs b/crates/databases/src/domain/models.rsindex ad473317da..ebef0c56be 100644--- a/crates/databases/src/domain/models.rs+++ b/crates/databases/src/domain/models.rs@@ -119,6 +119,27 @@ pub struct Column {/// Schema operations reserved by a feature; ordinary edits cannot clear them.#[serde(default)]pub protections: Vec<ColumnProtection>,+ /// Whether a row may omit this cell; empty collections also count as absent.+ #[serde(default = "column_nullable_default")]+ pub nullable: bool,+}++/// Whether a stored cell satisfies a required column. Empty collections represent+/// no selection; scalar values, including empty text, zero and false, are present.+pub(crate) fn cell_has_value(value: &PropertyValue) -> bool {+ match value {+ PropertyValue::SelectOption(values) => !values.is_empty(),+ PropertyValue::EntityRef(values) => !values.is_empty(),+ PropertyValue::Link(values) => !values.is_empty(),+ PropertyValue::Bool(_)+ | PropertyValue::Num(_)+ | PropertyValue::Str(_)+ | PropertyValue::Date(_) => true,+ }+}++pub(crate) fn column_nullable_default() -> bool {+ true}impl Column {@@ -628,6 +649,15 @@ impl Writes {/// What applying [`Writes`] did. Anything but `Applied` wrote nothing.#[derive(Debug, Clone, PartialEq)]pub enum WritesOutcome {+ /// The final batch would leave a required cell empty; nothing committed.+ MissingRequiredCell {+ /// The write that affected the row or its schema.+ write: usize,+ /// The required column.+ column: ColumnId,+ /// The row missing its value.+ row: RowId,+ },/// A row the undo would remove has a surviving incoming relation.RowInUse,/// An option the batch must leave unused is selected by an entity.diff --git a/crates/databases/src/domain/ports.rs b/crates/databases/src/domain/ports.rsindex a793a6968d..b959d7f263 100644--- a/crates/databases/src/domain/ports.rs+++ b/crates/databases/src/domain/ports.rs@@ -227,7 +227,8 @@ pub trait CellStore: Send + Sync + 'static {/// live and at its expected version, each new definition created before/// any column binds it, each updated or deleted row checked to belong to/// its table, each related row to its target table, each changed option- /// or column to its definition, and each changed table's version bumped+ /// or column to its definition, all affected required cells checked in the final+ /// batch state, and each changed table's version bumped/// once. Anything but [`WritesOutcome::Applied`] wrote nothing.fn apply_writes(&self,diff --git a/crates/databases/src/domain/service/ops.rs b/crates/databases/src/domain/service/ops.rsindex 1c64422ad6..99ab05011f 100644--- a/crates/databases/src/domain/service/ops.rs+++ b/crates/databases/src/domain/service/ops.rs@@ -522,6 +522,12 @@ fn applied(table_versions,changes,}),+ WritesOutcome::MissingRequiredCell { write, column, row } => Err(refuse(+ write,+ row_index(&ops[write], row),+ Some(column),+ "the required column needs a value",+ )),WritesOutcome::SchemaMoved(_) => Err(DatabaseError::VersionConflict),WritesOutcome::TableNotFound(_) => Err(DatabaseError::NotFound),WritesOutcome::VersionConflict(_) | WritesOutcome::TablesChanged { .. } => {diff --git a/crates/databases/src/domain/service/ops/cells.rs b/crates/databases/src/domain/service/ops/cells.rsindex 9da310ddca..c52999bab6 100644--- a/crates/databases/src/domain/service/ops/cells.rs+++ b/crates/databases/src/domain/service/ops/cells.rs@@ -40,6 +40,13 @@ impl Planner {.find(|column| column.column.id == cell.column).ok_or_else(|| place.refuse("no such column in this table"))?;let value = self.value(place, column, &cell.value)?;+ if !column.column.nullable+ && !value+ .as_ref()+ .is_some_and(crate::domain::models::cell_has_value)+ {+ return Err(place.refuse(format!("\"{}\" requires a value", column.name())));+ }stored.push((column.definition.definition.id, value));}Ok(stored)diff --git a/crates/databases/src/domain/service/ops/columns.rs b/crates/databases/src/domain/service/ops/columns.rsindex 17f53717df..34e7ef9d45 100644--- a/crates/databases/src/domain/service/ops/columns.rs+++ b/crates/databases/src/domain/service/ops/columns.rs@@ -120,6 +120,11 @@ impl Planner {};let column = Column {protections: vec![],+ nullable: self+ .restoration+ .columns+ .get(&id)+ .is_none_or(|column| column.nullable),id,table_id: entry.table.id,property_definition_id: definition.definition.id,diff --git a/crates/databases/src/domain/service/ops/rows.rs b/crates/databases/src/domain/service/ops/rows.rsindex c54422cdc1..11cb581944 100644--- a/crates/databases/src/domain/service/ops/rows.rs+++ b/crates/databases/src/domain/service/ops/rows.rs@@ -4,7 +4,7 @@use models_databases::views::LaneKey;use models_databases::{RowChanges, RowsChange};-use super::{MAX_WRITTEN_ROWS, Planner, refuse};+use super::{MAX_WRITTEN_ROWS, Place, Planner, refuse};use crate::domain::catalog::TableEntry;use crate::domain::journal::cell_value;use crate::domain::models::{DatabaseError, Write};@@ -42,6 +42,18 @@ impl Planner {.iter().enumerate().map(|(row, cells)| {+ for column in &entry.columns {+ if !column.column.nullable+ && !cells.iter().any(|cell| cell.column == column.column.id)+ {+ return Err(Place {+ op: index,+ row: Some(row),+ column: column.column.id,+ }+ .refuse(format!("\"{}\" requires a value", column.name())));+ }+ }let cells = self.cells(entry, index, Some(row), cells)?;Ok(cells.into_iter()diff --git a/crates/databases/src/domain/service/ops/views.rs b/crates/databases/src/domain/service/ops/views.rsindex 77cfbb57c3..a3f9faa4a9 100644--- a/crates/databases/src/domain/service/ops/views.rs+++ b/crates/databases/src/domain/service/ops/views.rs@@ -278,6 +278,14 @@ impl Planner {column,&lane.cell(),)?;+ if !column.column.nullable && cell.is_none() {+ return Err(refuse(+ index,+ None,+ Some(group_by),+ "the required column needs a value",+ ));+ }let definition = column.definition.definition.id;let state = self.boardsdiff --git a/crates/databases/src/domain/service/test/fakes.rs b/crates/databases/src/domain/service/test/fakes.rsindex 2662a8bdf8..7b34eb95d7 100644--- a/crates/databases/src/domain/service/test/fakes.rs+++ b/crates/databases/src/domain/service/test/fakes.rs@@ -117,6 +117,7 @@ impl DatabasesRepo for FakeRepo {world.databases.push(database.clone());world.columns.push(Column {protections: vec![],+ nullable: true,id: ColumnId::new(),table_id: table.id,property_definition_id: title.definition.id,diff --git a/crates/databases/src/domain/service/test/row_writes.rs b/crates/databases/src/domain/service/test/row_writes.rsindex b88390ce92..78b997c1a6 100644--- a/crates/databases/src/domain/service/test/row_writes.rs+++ b/crates/databases/src/domain/service/test/row_writes.rs@@ -229,3 +229,70 @@ async fn grants_scope_writes_per_database() {assert_eq!(w.rows[&rooms].len(), 1);assert_eq!(w.rows[&seeded.table_id].len(), 1);}++#[tokio::test]+async fn required_columns_reject_missing_and_cleared_values_but_accept_empty_text() {+ let seeded = seeded().await;+ seeded+ .world+ .lock()+ .unwrap()+ .columns+ .iter_mut()+ .find(|column| column.id == seeded.name_column.id)+ .unwrap()+ .nullable = false;+ for change in [+ RowsChange::Insert { rows: vec![vec![]] },+ RowsChange::Insert {+ rows: vec![vec![CellWrite {+ column: seeded.name_column.id,+ value: CellValue::Clear,+ }]],+ },+ RowsChange::Update {+ changes: RowChanges::Uniform {+ rows: vec![seeded.row_id],+ cells: vec![CellWrite {+ column: seeded.name_column.id,+ value: CellValue::Clear,+ }],+ },+ },+ ] {+ assert!(matches!(+ seeded+ .service+ .apply_ops(+ edit(seeded.database_id),+ viewer(OWNER),+ vec![DatabaseOp::Rows {+ table: seeded.table_id,+ change+ }]+ .into()+ )+ .await,+ Err(DatabaseError::InvalidOp(_))+ ));+ }+ assert_eq!(seeded.world.lock().unwrap().write_batches, 0);+ seeded+ .service+ .apply_ops(+ edit(seeded.database_id),+ viewer(OWNER),+ vec![DatabaseOp::Rows {+ table: seeded.table_id,+ change: RowsChange::Insert {+ rows: vec![vec![CellWrite {+ column: seeded.name_column.id,+ value: CellValue::Text(String::new()),+ }]],+ },+ }]+ .into(),+ )+ .await+ .unwrap();+}diff --git a/crates/databases/src/inbound/toolset/test.rs b/crates/databases/src/inbound/toolset/test.rsindex 72e2a2fefb..ecdd54b0ff 100644--- a/crates/databases/src/inbound/toolset/test.rs+++ b/crates/databases/src/inbound/toolset/test.rs@@ -93,6 +93,7 @@ fn status_column() -> ColumnDetail {ColumnDetail {column: Column {protections: vec![],+ nullable: true,infer_type: false,display_name: None,id: COLUMN_ID,diff --git a/crates/databases/src/outbound/pg_cell_store.rs b/crates/databases/src/outbound/pg_cell_store.rsindex 5bb9e17c92..b2f40b1b90 100644--- a/crates/databases/src/outbound/pg_cell_store.rs+++ b/crates/databases/src/outbound/pg_cell_store.rs@@ -3,6 +3,7 @@//! options.mod core;+mod required;mod transfer;use std::collections::HashMap;@@ -347,6 +348,13 @@ where}}+ if let Some(missing) = self+ .check_required_cells(&mut transaction, writes, &inserted)+ .await?+ {+ return Ok(missing);+ }+let mut grouped: Vec<(TableId, Vec<RowId>)> = Vec::new();for (table, row) in &writes.related_rows {match grouped.iter_mut().find(|(target, _)| target == table) {diff --git a/crates/databases/src/outbound/pg_cell_store/required.rs b/crates/databases/src/outbound/pg_cell_store/required.rsnew file mode 100644index 0000000000..be9b9d8812--- /dev/null+++ b/crates/databases/src/outbound/pg_cell_store/required.rs@@ -0,0 +1,110 @@+//! Recheck required cells against the final transaction state while table locks+//! are held. Properties supplies values through its transactional port.++use std::collections::{BTreeMap, BTreeSet, HashSet};++use super::*;+use crate::domain::models::{ColumnId, cell_has_value};++impl<Properties> PgCellStore<Properties>+where+ Properties: DatabaseCellWriter<Transaction = Transaction<'static, Postgres>>,+{+ pub(super) async fn check_required_cells(+ &self,+ transaction: &mut Transaction<'static, Postgres>,+ writes: &Writes,+ inserted: &[Vec<RowId>],+ ) -> Result<Option<WritesOutcome>, PgCellStoreError> {+ // A schema change checks the whole table. Ordinary writes check only+ // their affected rows; optional tables never load any cell values.+ let mut scopes: BTreeMap<TableId, (usize, Option<BTreeSet<RowId>>)> = BTreeMap::new();+ for (index, (write, inserted)) in writes.writes.iter().zip(inserted).enumerate() {+ let (table, rows) = match write {+ Write::CreateColumn { column, .. } => (column.table_id, None),+ Write::ReplaceColumn { table_id, .. } => (*table_id, None),+ Write::InsertRows { table_id, .. } => {+ (*table_id, Some(inserted.iter().copied().collect()))+ }+ Write::UpdateRows { table_id, rows } => {+ (*table_id, Some(rows.iter().map(|(row, _)| *row).collect()))+ }+ Write::MoveCard { table_id, row, .. } => (*table_id, Some(BTreeSet::from([*row]))),+ _ => continue,+ };+ scopes+ .entry(table)+ .and_modify(|(first, scope)| {+ if let Some(existing) = scope {+ match &rows {+ Some(rows) => existing.extend(rows),+ None => {+ *scope = None;+ *first = index;+ }+ }+ }+ })+ .or_insert((index, rows));+ }+ for (table, (write, rows)) in scopes {+ let columns = sqlx::query!(+ "SELECT id, property_definition_id FROM database_columns WHERE table_id = $1 AND NOT nullable ORDER BY id",+ table.into_uuid()+ ).fetch_all(&mut **transaction).await?;+ if columns.is_empty() {+ continue;+ }+ let definitions: Vec<_> = columns+ .iter()+ .map(|column| column.property_definition_id)+ .collect();+ let row_ids: Option<Vec<_>> =+ rows.map(|rows| rows.into_iter().map(RowId::into_uuid).collect());+ let mut after: Option<uuid::Uuid> = None;+ loop {+ let page = sqlx::query_scalar!(+ "SELECT id FROM database_rows WHERE table_id = $1+ AND ($2::uuid[] IS NULL OR id = ANY($2))+ AND ($3::uuid IS NULL OR id > $3) ORDER BY id LIMIT 1000",+ table.into_uuid(),+ row_ids.as_deref(),+ after+ )+ .fetch_all(&mut **transaction)+ .await?;+ if page.is_empty() {+ break;+ }+ let entities: Vec<_> = page.iter().map(ToString::to_string).collect();+ let present: HashSet<_> = self+ .properties+ .entity_values_in(+ transaction,+ EntityType::DatabaseRow,+ &entities,+ Some(&definitions),+ )+ .await+ .map_err(cells_error)?+ .into_iter()+ .filter(|(_, _, value)| cell_has_value(value))+ .map(|(row, definition, _)| (row, definition))+ .collect();+ for (row, entity) in page.iter().zip(&entities) {+ for column in &columns {+ if !present.contains(&(entity.clone(), column.property_definition_id)) {+ return Ok(Some(WritesOutcome::MissingRequiredCell {+ write,+ column: ColumnId::from_uuid(column.id),+ row: RowId::from_uuid(*row),+ }));+ }+ }+ }+ after = page.last().copied();+ }+ }+ Ok(None)+ }+}diff --git a/crates/databases/src/outbound/pg_databases_repo.rs b/crates/databases/src/outbound/pg_databases_repo.rsindex 7bb670a8ea..85b4891baf 100644--- a/crates/databases/src/outbound/pg_databases_repo.rs+++ b/crates/databases/src/outbound/pg_databases_repo.rs@@ -524,7 +524,7 @@ whereasync fn columns_for_tables(&self, table_ids: &[TableId]) -> Result<Vec<Column>, Self::Error> {let rows = sqlx::query!(r#"- SELECT id, table_id, property_definition_id, position, config, display_name, infer_type,+ SELECT id, table_id, property_definition_id, position, config, display_name, infer_type, nullable,ARRAY(SELECT capability FROM database_column_protections p WHERE p.column_id = database_columns.id ORDER BY capability) AS "protections!"FROM database_columnsWHERE table_id = ANY($1)@@ -552,6 +552,7 @@ whereconfig: row.config.map(serde_json::from_value).transpose()?,display_name: row.display_name,infer_type: row.infer_type,+ nullable: row.nullable,})}).collect()diff --git a/crates/databases/src/outbound/pg_databases_repo/schema.rs b/crates/databases/src/outbound/pg_databases_repo/schema.rsindex 6dfc810560..6accf628aa 100644--- a/crates/databases/src/outbound/pg_databases_repo/schema.rs+++ b/crates/databases/src/outbound/pg_databases_repo/schema.rs@@ -243,8 +243,8 @@ pub(crate) async fn insert_column(.transpose()?;let inserted = sqlx::query!(r#"- INSERT INTO database_columns (id, table_id, property_definition_id, position, config, infer_type)- VALUES ($1, $2, $3, $4, $5, $6)+ INSERT INTO database_columns (id, table_id, property_definition_id, position, config, infer_type, nullable)+ VALUES ($1, $2, $3, $4, $5, $6, $7)"#,column.id.into_uuid(),column.table_id.into_uuid(),@@ -252,6 +252,7 @@ pub(crate) async fn insert_column(column.position.as_str(),config,column.infer_type,+ column.nullable,).execute(connection).await;diff --git a/crates/databases/src/outbound/pg_databases_repo/test.rs b/crates/databases/src/outbound/pg_databases_repo/test.rsindex 1bc1d5de96..47fbd8aacc 100644--- a/crates/databases/src/outbound/pg_databases_repo/test.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test.rs@@ -23,6 +23,7 @@ mod journal;mod rename_column;#[cfg(feature = "gateway")]mod reorder_tables;+mod required;mod saved_queries;#[cfg(feature = "gateway")]mod schema_ops;@@ -167,6 +168,7 @@ fn bind(table_id: TableId, definition_id: Uuid, position: &str) -> Write {Write::CreateColumn {column: Column {protections: vec![],+ nullable: true,id: ColumnId::new(),table_id,property_definition_id: definition_id,diff --git a/crates/databases/src/outbound/pg_databases_repo/test/journal.rs b/crates/databases/src/outbound/pg_databases_repo/test/journal.rsindex 122170a7e0..b8f9b6ac73 100644--- a/crates/databases/src/outbound/pg_databases_repo/test/journal.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test/journal.rs@@ -456,7 +456,7 @@ async fn every_committed_batch_of_the_guest_list_is_journaled_with_its_inverse(pjson!({"formatVersion": 1,"incomplete": false,- "restoredColumns": {wedding.plus_ones.to_string(): {"kind": {"type": "number"}, "infer_type": false}},+ "restoredColumns": {wedding.plus_ones.to_string(): {"kind": {"type": "number"}, "infer_type": false, "nullable": true}},"ops": [{"kind": "column",diff --git a/crates/databases/src/outbound/pg_databases_repo/test/required.rs b/crates/databases/src/outbound/pg_databases_repo/test/required.rsnew file mode 100644index 0000000000..30fe5d132d--- /dev/null+++ b/crates/databases/src/outbound/pg_databases_repo/test/required.rs@@ -0,0 +1,200 @@+//! Required cells are enforced by both Rust write entry points, without SQL triggers.++use super::*;+use crate::domain::ports::DatabaseStorage;+use models_properties::service::property_value::PropertyValue;++fn batch(database_id: DatabaseId, writes: Vec<Write>) -> Writes {+ Writes {+ database_id,+ created_by: user(),+ writes,+ related_rows: vec![],+ expected_versions: vec![],+ journal: crate::domain::journal::JournalPlan::default(),+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn required_cells_are_checked_for_app_and_core_writes(pool: PgPool) {+ let (repo, table, _) = fixture(&pool).await;+ let definition = insert_definition(&pool, "Required").await;+ let position =+ models_databases::position::key_between(Some(&"80".parse().unwrap()), None).unwrap();+ let mut create = bind(table.id, definition, position.as_str());+ let Write::CreateColumn { column, .. } = &mut create else {+ unreachable!()+ };+ column.nullable = false;+ let column_id = column.id;+ commit(&pool, table.database_id, vec![create]).await;+ assert!(+ !repo+ .columns_for_tables(&[table.id])+ .await+ .unwrap()+ .iter()+ .find(|c| c.id == column_id)+ .unwrap()+ .nullable+ );+ let store = PgCellStore::new(pool.clone(), PropertiesPgRepo::new(pool.clone()));+ for core in [false, true] {+ let missing = batch(+ table.database_id,+ vec![Write::InsertRows {+ table_id: table.id,+ rows: vec![vec![]],+ restored: vec![],+ }],+ );+ let outcome = if core {+ store.apply_storage_writes(&missing).await+ } else {+ store.apply_writes(&missing).await+ }+ .unwrap();+ assert!(+ matches!(outcome, WritesOutcome::MissingRequiredCell { column, .. } if column == column_id)+ );+ }+ assert_eq!(+ sqlx::query_scalar!(+ "SELECT count(*) FROM database_rows WHERE table_id = $1",+ table.id.into_uuid()+ )+ .fetch_one(&pool)+ .await+ .unwrap(),+ Some(0)+ );+ let inserted = store+ .apply_storage_writes(&batch(+ table.database_id,+ vec![Write::InsertRows {+ table_id: table.id,+ rows: vec![vec![(definition, PropertyValue::Str(String::new()))]],+ restored: vec![],+ }],+ ))+ .await+ .unwrap();+ let WritesOutcome::Applied {+ inserted,+ table_versions,+ ..+ } = inserted+ else {+ panic!("{inserted:?}")+ };+ let row = inserted[0][0];+ for value in [+ None,+ Some(PropertyValue::SelectOption(vec![])),+ Some(PropertyValue::EntityRef(vec![])),+ Some(PropertyValue::Link(vec![])),+ ] {+ let outcome = store+ .apply_storage_writes(&batch(+ table.database_id,+ vec![Write::UpdateRows {+ table_id: table.id,+ rows: vec![(row, vec![(definition, value)])],+ }],+ ))+ .await+ .unwrap();+ assert_eq!(+ outcome,+ WritesOutcome::MissingRequiredCell {+ write: 0,+ column: column_id,+ row+ }+ );+ }+ assert_eq!(+ repo.table_versions(&[table.id]).await.unwrap(),+ table_versions+ );+ assert_eq!(+ store.cells(&[row]).await.unwrap()[&row][&definition],+ PropertyValue::Str(String::new())+ );+ // A batch may temporarily clear a value if its final state is valid.+ let outcome = store+ .apply_storage_writes(&batch(+ table.database_id,+ vec![+ Write::UpdateRows {+ table_id: table.id,+ rows: vec![(row, vec![(definition, None)])],+ },+ Write::UpdateRows {+ table_id: table.id,+ rows: vec![(+ row,+ vec![(definition, Some(PropertyValue::Str("Restored".into())))],+ )],+ },+ ],+ ))+ .await+ .unwrap();+ assert!(matches!(outcome, WritesOutcome::Applied { .. }));+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn adding_a_required_column_requires_values_for_existing_rows_in_the_same_batch(+ pool: PgPool,+) {+ let (repo, table, _) = fixture(&pool).await;+ let row = repo.insert_rows(table.id, USER, 1).await.unwrap().unwrap()[0].id;+ let definition = insert_definition(&pool, "Required").await;+ let position =+ models_databases::position::key_between(Some(&"80".parse().unwrap()), None).unwrap();+ let mut create = bind(table.id, definition, position.as_str());+ let Write::CreateColumn { column, .. } = &mut create else {+ unreachable!()+ };+ column.nullable = false;+ let column_id = column.id;+ let store = PgCellStore::new(pool.clone(), PropertiesPgRepo::new(pool.clone()));+ let outcome = store+ .apply_storage_writes(&batch(table.database_id, vec![create.clone()]))+ .await+ .unwrap();+ assert_eq!(+ outcome,+ WritesOutcome::MissingRequiredCell {+ write: 0,+ column: column_id,+ row+ }+ );+ assert!(+ !repo+ .columns_for_tables(&[table.id])+ .await+ .unwrap()+ .iter()+ .any(|c| c.id == column_id)+ );+ let outcome = store+ .apply_storage_writes(&batch(+ table.database_id,+ vec![+ create,+ Write::UpdateRows {+ table_id: table.id,+ rows: vec![(+ row,+ vec![(definition, Some(PropertyValue::Str("Present".into())))],+ )],+ },+ ],+ ))+ .await+ .unwrap();+ assert!(matches!(outcome, WritesOutcome::Applied { .. }));+}diff --git a/crates/databases/src/outbound/pg_databases_repo/test/undo/structure.rs b/crates/databases/src/outbound/pg_databases_repo/test/undo/structure.rsindex 07e1a7211c..c840b9007f 100644--- a/crates/databases/src/outbound/pg_databases_repo/test/undo/structure.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test/undo/structure.rs@@ -555,3 +555,56 @@ async fn deleting_a_lane_with_card_positions_does_not_claim_complete_restoration});}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn undo_restores_required_column_metadata_and_its_cells(pool: PgPool) {+ let wedding = wedding(&pool).await;+ change_as(+ &pool,+ &wedding,+ WOLF,+ vec![DatabaseOp::Rows {+ table: wedding.table_id,+ change: RowsChange::Insert {+ rows: vec![vec![CellWrite {+ column: wedding.name,+ value: CellValue::Text("Sam".into()),+ }]],+ },+ }],+ )+ .await;+ sqlx::query!(+ "UPDATE database_columns SET nullable = false WHERE id = $1",+ wedding.name.into_uuid()+ )+ .execute(&pool)+ .await+ .unwrap();+ let change = change_as(+ &pool,+ &wedding,+ WOLF,+ vec![DatabaseOp::Column {+ table: wedding.table_id,+ column: wedding.name,+ change: ColumnChange::Delete,+ }],+ )+ .await;+ assert!(matches!(+ undo_as(&pool, &wedding, WOLF, change).await,+ UndoOutcome::Reverted { .. }+ ));+ assert!(+ !sqlx::query_scalar!(+ "SELECT nullable FROM database_columns WHERE id = $1",+ wedding.name.into_uuid()+ )+ .fetch_one(&pool)+ .await+ .unwrap()+ );+ let cells: i64 = sqlx::query_scalar!(r#"SELECT count(*) AS "count!" FROM entity_properties p JOIN database_columns c ON c.property_definition_id = p.property_definition_id WHERE c.id = $1 AND p.entity_type = 'DATABASE_ROW'"#, wedding.name.into_uuid()).fetch_one(&pool).await.unwrap();+ assert_eq!(cells, 1);+}diff --git a/crates/databases_sql/src/catalog/test.rs b/crates/databases_sql/src/catalog/test.rsindex 1ffd993d15..58451f4b7a 100644--- a/crates/databases_sql/src/catalog/test.rs+++ b/crates/databases_sql/src/catalog/test.rs@@ -45,6 +45,7 @@ fn details_become_the_schema_the_engine_builds_its_catalog_from() {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: ColumnId::from_uuid(Uuid::from_u128(0xb001)),table_id: DEALS,property_definition_id: Uuid::from_u128(0xc001),@@ -75,6 +76,7 @@ fn details_become_the_schema_the_engine_builds_its_catalog_from() {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: ColumnId::from_uuid(Uuid::from_u128(0xb002)),table_id: DEALS,property_definition_id: Uuid::from_u128(0xc002),@@ -124,6 +126,7 @@ fn details_become_the_schema_the_engine_builds_its_catalog_from() {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: ColumnId::from_uuid(Uuid::from_u128(0xb003)),table_id: DEALS,property_definition_id: Uuid::from_u128(0xc003),@@ -154,6 +157,7 @@ fn details_become_the_schema_the_engine_builds_its_catalog_from() {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: ColumnId::from_uuid(Uuid::from_u128(0xb004)),table_id: DEALS,property_definition_id: Uuid::from_u128(0xc004),@@ -301,6 +305,7 @@ fn a_shared_relation_definition_relates_each_table_to_its_own_target() {columns: vec![ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: column,table_id: id,property_definition_id: LINKED,diff --git a/crates/databases_sql/src/service/test.rs b/crates/databases_sql/src/service/test.rsindex 0295f6ceeb..59f7bfb396 100644--- a/crates/databases_sql/src/service/test.rs+++ b/crates/databases_sql/src/service/test.rs@@ -94,6 +94,7 @@ fn world() -> Shared {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: NAME_COLUMN,table_id: GUESTS,property_definition_id: NAME,@@ -124,6 +125,7 @@ fn world() -> Shared {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: STATUS_COLUMN,table_id: GUESTS,property_definition_id: STATUS,@@ -173,6 +175,7 @@ fn world() -> Shared {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: HALL_COLUMN,table_id: GUESTS,property_definition_id: HALL,@@ -206,6 +209,7 @@ fn world() -> Shared {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: CONTACT_COLUMN,table_id: GUESTS,property_definition_id: CONTACT,@@ -258,6 +262,7 @@ fn world() -> Shared {columns: vec![ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: HALL_NAME_COLUMN,table_id: HALLS,property_definition_id: HALL_NAME,@@ -309,6 +314,7 @@ fn world() -> Shared {columns: vec![ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: PLAN_NAME_COLUMN,table_id: PLANS,property_definition_id: PLAN_NAME,diff --git a/crates/databases_sql/src/toolset/test.rs b/crates/databases_sql/src/toolset/test.rsindex 4fa5d93dbe..dec6382c76 100644--- a/crates/databases_sql/src/toolset/test.rs+++ b/crates/databases_sql/src/toolset/test.rs@@ -66,6 +66,7 @@ fn world() -> Shared {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: ColumnId::from_uuid(Uuid::from_u128(0xb001)),table_id: GUESTS,property_definition_id: NAME,@@ -96,6 +97,7 @@ fn world() -> Shared {ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: STATUS_COLUMN,table_id: GUESTS,property_definition_id: STATUS,diff --git a/crates/forms/src/domain/service/test/fakes/databases.rs b/crates/forms/src/domain/service/test/fakes/databases.rsindex 665b0dc524..d8d283ab67 100644--- a/crates/forms/src/domain/service/test/fakes/databases.rs+++ b/crates/forms/src/domain/service/test/fakes/databases.rs@@ -136,6 +136,7 @@ fn column_detail(ColumnDetail {column: Column {protections: vec![],+ nullable: true,id: column.id,table_id: table.id,property_definition_id: definition_id,diff --git a/crates/macro_db_client/migrations/20261007153246_database_column_nullable.sql b/crates/macro_db_client/migrations/20261007153246_database_column_nullable.sqlnew file mode 100644index 0000000000..13a0fd3058--- /dev/null+++ b/crates/macro_db_client/migrations/20261007153246_database_column_nullable.sql@@ -0,0 +1,2 @@+-- Required values are enforced by the Rust database engine. Existing columns remain optional.+ALTER TABLE database_columns ADD COLUMN nullable BOOLEAN NOT NULL DEFAULT true;diff --git a/packages/sdk/generated/storage/types.gen.ts b/packages/sdk/generated/storage/types.gen.tsindex 22d3aca434..55291f52be 100644--- a/packages/sdk/generated/storage/types.gen.ts+++ b/packages/sdk/generated/storage/types.gen.ts@@ -2894,6 +2894,10 @@ export type Column = {* Whether the first nonempty value may settle this new text column's type.*/infer_type: boolean;+ /**+ * Whether a row may omit this cell; empty collections also count as absent.+ */+ nullable?: boolean;/*** Fractional index for column ordering.*/diff --git a/packages/sdk/specs/storage.json b/packages/sdk/specs/storage.jsonindex 9f566c6864..6b7343de18 100644--- a/packages/sdk/specs/storage.json+++ b/packages/sdk/specs/storage.json@@ -21081,6 +21081,10 @@"type": "boolean","description": "Whether the first nonempty value may settle this new text column's type."},+ "nullable": {+ "type": "boolean",+ "description": "Whether a row may omit this cell; empty collections also count as absent."+ },"position": {"type": "string","description": "Fractional index for column ordering."