Macro is a unified workspace for teams: email, chat, docs, tasks, agents, calls, and CRM — @-linked together with shared AI memory.
TypeScript agentaiai-agentsall-in-onecrmcrm-systememaillinearmcpmessagingnotesnotionofficerustslackslack-alternativestartupstartup-toolstypescriptworkspace
perf(databases): page large tables on the server and store formula values (#7539)
Wolf Mermelstein156af352fd77
diff --git a/.github/workspace-dep-closures.json b/.github/workspace-dep-closures.jsonindex 63b3cf434e..5ba00ff896 100644--- a/.github/workspace-dep-closures.json+++ b/.github/workspace-dep-closures.json@@ -4074,6 +4074,7 @@"crates/database_env_vars","crates/database_sql","crates/databases",+ "crates/databases_sql","crates/document_sub_type","crates/documents","crates/docx_engine",@@ -4110,6 +4111,7 @@"crates/graphql_channel","crates/graphql_common","crates/graphql_crm",+ "crates/graphql_databases","crates/graphql_email","crates/graphql_entity_mutation","crates/graphql_favorite",@@ -5248,6 +5250,7 @@"crates/graphql_channel","crates/graphql_common","crates/graphql_crm",+ "crates/graphql_databases","crates/graphql_email","crates/graphql_entity_mutation","crates/graphql_favorite",@@ -7851,6 +7854,156 @@"crates/unfurl","crates/workspace-hack"],+ "graphql_databases": [+ "crates/activity",+ "crates/agent",+ "crates/agent_changes",+ "crates/agent_runtime_protocol",+ "crates/agent_session",+ "crates/ai_billing",+ "crates/ai_engine",+ "crates/ai_toolset",+ "crates/ai_usage",+ "crates/analytics_client",+ "crates/attachment",+ "crates/authentication_service_client",+ "crates/bot_id",+ "crates/bot_token",+ "crates/bots",+ "crates/broadcast",+ "crates/calendar_events",+ "crates/call",+ "crates/channel_sender",+ "crates/channels",+ "crates/chat",+ "crates/comms_db_client",+ "crates/connection",+ "crates/connection_gateway_client",+ "crates/connection_gateway_models",+ "crates/contacts",+ "crates/cowlike",+ "crates/crm",+ "crates/database_env_vars",+ "crates/database_sql",+ "crates/databases",+ "crates/databases_sql",+ "crates/document_sub_type",+ "crates/documents",+ "crates/docx_engine",+ "crates/dynamodb_client",+ "crates/email",+ "crates/email_api_client",+ "crates/email_db_client",+ "crates/email_utils",+ "crates/embedding",+ "crates/entity_access",+ "crates/entity_access_db_utils",+ "crates/entity_access_management",+ "crates/entity_mutation",+ "crates/entity_registry/entity_registry",+ "crates/entity_registry/entity_registry_db_utils",+ "crates/entity_registry/shared_entity_registry",+ "crates/favorites",+ "crates/fig_engine",+ "crates/filter_ast",+ "crates/folds/agent_fold",+ "crates/foreign_entity",+ "crates/frecency",+ "crates/genai_telemetry",+ "crates/generic_email_domains",+ "crates/git_patch",+ "crates/github_pull_requests",+ "crates/gmail_client",+ "crates/google_token",+ "crates/graphql_common",+ "crates/graphql_databases",+ "crates/graphql_email",+ "crates/graphql_permission",+ "crates/graphql_soup",+ "crates/graphql_soup_filter_input",+ "crates/harness_id",+ "crates/harness_token",+ "crates/initiative",+ "crates/invite_email",+ "crates/ip_extractor",+ "crates/item_filter_index",+ "crates/item_filters",+ "crates/jev",+ "crates/kafka_util",+ "crates/last_online_tracker",+ "crates/lexical_client",+ "crates/macro_auth",+ "crates/macro_authorization",+ "crates/macro_aws_config",+ "crates/macro_db_client",+ "crates/macro_entrypoint",+ "crates/macro_env",+ "crates/macro_env_var",+ "crates/macro_event_broker",+ "crates/macro_event_topics",+ "crates/macro_service_urls",+ "crates/macro_sha_count_client",+ "crates/macro_sync_service_jwt",+ "crates/macro_tower_layers",+ "crates/macro_user_id",+ "crates/macro_uuid",+ "crates/maybe_send",+ "crates/mention_utils",+ "crates/messages",+ "crates/model",+ "crates/model-entity",+ "crates/model-error-response",+ "crates/model_file_type",+ "crates/model_notifications",+ "crates/model_owner",+ "crates/model_user",+ "crates/models_ai_projection",+ "crates/models_bulk_upload",+ "crates/models_databases",+ "crates/models_dcs",+ "crates/models_email",+ "crates/models_entity_access_management",+ "crates/models_grouping",+ "crates/models_opensearch",+ "crates/models_pagination",+ "crates/models_permissions",+ "crates/models_properties",+ "crates/models_sfs",+ "crates/models_soup",+ "crates/models_team",+ "crates/non_empty",+ "crates/notification",+ "crates/notification_state",+ "crates/pptx_engine",+ "crates/predicate_index",+ "crates/projects",+ "crates/prompt",+ "crates/properties",+ "crates/psd_engine",+ "crates/rate_limit",+ "crates/readonly_pool",+ "crates/remote_env_var",+ "crates/roles_and_permissions",+ "crates/s3_key",+ "crates/secretsmanager_client",+ "crates/share_permission_db_utils",+ "crates/soup",+ "crates/soup_filter_projection",+ "crates/soup_realtime",+ "crates/sqs_client",+ "crates/sqs_worker",+ "crates/static_file_service_client",+ "crates/sync_service",+ "crates/sync_service_client",+ "crates/system_properties",+ "crates/system_skills",+ "crates/task_dedup",+ "crates/teams",+ "crates/trigger_context",+ "crates/unfurl",+ "crates/user_quota",+ "crates/workspace-hack"+ ],"graphql_email": ["crates/activity","crates/agent",diff --git a/.sqlx/query-0460937ee4ee99d89d628fd32e584e3cf855f0fc4f4603cf0fb79b5c03ec1eaa.json b/.sqlx/query-0460937ee4ee99d89d628fd32e584e3cf855f0fc4f4603cf0fb79b5c03ec1eaa.jsonnew file mode 100644index 0000000000..2a679d35ea--- /dev/null+++ b/.sqlx/query-0460937ee4ee99d89d628fd32e584e3cf855f0fc4f4603cf0fb79b5c03ec1eaa.json@@ -0,0 +1,26 @@+{+ "db_name": "PostgreSQL",+ "query": "\n SELECT DISTINCT t.database_id, t.id\n FROM database_columns c\n JOIN database_tables t ON t.id = c.table_id\n JOIN database_entities d ON d.database_id = t.database_id\n WHERE c.config ->> 'kind' = 'derived' AND d.trashed_at IS NULL\n ORDER BY t.database_id, t.id\n ",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "database_id",+ "type_info": "Uuid"+ },+ {+ "ordinal": 1,+ "name": "id",+ "type_info": "Uuid"+ }+ ],+ "parameters": {+ "Left": []+ },+ "nullable": [+ false,+ false+ ]+ },+ "hash": "0460937ee4ee99d89d628fd32e584e3cf855f0fc4f4603cf0fb79b5c03ec1eaa"+}diff --git a/.sqlx/query-176c9fedb95c7dfd1ff80a97c293a0edc718b72efc243e769a82eefb6b98cd6b.json b/.sqlx/query-176c9fedb95c7dfd1ff80a97c293a0edc718b72efc243e769a82eefb6b98cd6b.jsonnew file mode 100644index 0000000000..c903505166--- /dev/null+++ b/.sqlx/query-176c9fedb95c7dfd1ff80a97c293a0edc718b72efc243e769a82eefb6b98cd6b.json@@ -0,0 +1,22 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT version FROM database_tables WHERE id = $1",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "version",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "176c9fedb95c7dfd1ff80a97c293a0edc718b72efc243e769a82eefb6b98cd6b"+}diff --git a/.sqlx/query-3065089c6dc3baefa2adf9e0bcee8e4dc3bd289db89ec09c0e12a256e0b47ab3.json b/.sqlx/query-3065089c6dc3baefa2adf9e0bcee8e4dc3bd289db89ec09c0e12a256e0b47ab3.jsonnew file mode 100644index 0000000000..6d62f22c88--- /dev/null+++ b/.sqlx/query-3065089c6dc3baefa2adf9e0bcee8e4dc3bd289db89ec09c0e12a256e0b47ab3.json@@ -0,0 +1,39 @@+{+ "db_name": "PostgreSQL",+ "query": "\n INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values)\n SELECT cell.id, cell.entity_id, $3, $4, cell.value\n FROM UNNEST($1::uuid[], $2::text[], $5::jsonb[]) AS cell(id, entity_id, value)\n WHERE NOT EXISTS (\n SELECT 1 FROM entity_properties stored\n WHERE stored.entity_id = cell.entity_id\n AND stored.entity_type = $3\n AND stored.property_definition_id = $4\n )\n ON CONFLICT (entity_id, entity_type, property_definition_id)\n DO UPDATE SET values = EXCLUDED.values, updated_at = NOW()\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "UuidArray",+ "TextArray",+ {+ "Custom": {+ "name": "property_entity_type",+ "kind": {+ "Enum": [+ "CHANNEL",+ "CHAT",+ "DOCUMENT",+ "PROJECT",+ "THREAD",+ "USER",+ "COMPANY",+ "TASK",+ "CALL_RECORD",+ "CALENDAR_EVENT",+ "INITIATIVE",+ "CONTACT",+ "DATABASE_ROW"+ ]+ }+ }+ },+ "Uuid",+ "JsonbArray"+ ]+ },+ "nullable": []+ },+ "hash": "3065089c6dc3baefa2adf9e0bcee8e4dc3bd289db89ec09c0e12a256e0b47ab3"+}diff --git a/.sqlx/query-4ad669e09da64a15d35a89cca4ac1576d244732e8868b4220ba5d87a022746de.json b/.sqlx/query-4ad669e09da64a15d35a89cca4ac1576d244732e8868b4220ba5d87a022746de.jsonnew file mode 100644index 0000000000..09c5670c13--- /dev/null+++ b/.sqlx/query-4ad669e09da64a15d35a89cca4ac1576d244732e8868b4220ba5d87a022746de.json@@ -0,0 +1,23 @@+{+ "db_name": "PostgreSQL",+ "query": "SELECT version FROM database_tables WHERE id = $1 AND database_id = $2",+ "describe": {+ "columns": [+ {+ "ordinal": 0,+ "name": "version",+ "type_info": "Int8"+ }+ ],+ "parameters": {+ "Left": [+ "Uuid",+ "Uuid"+ ]+ },+ "nullable": [+ false+ ]+ },+ "hash": "4ad669e09da64a15d35a89cca4ac1576d244732e8868b4220ba5d87a022746de"+}diff --git a/.sqlx/query-4af6a0999eb209c36f581a632a63b15bae8de290de2c2d1e81ef6620ce6211ea.json b/.sqlx/query-4af6a0999eb209c36f581a632a63b15bae8de290de2c2d1e81ef6620ce6211ea.jsonnew file mode 100644index 0000000000..35352c14c1--- /dev/null+++ b/.sqlx/query-4af6a0999eb209c36f581a632a63b15bae8de290de2c2d1e81ef6620ce6211ea.json@@ -0,0 +1,38 @@+{+ "db_name": "PostgreSQL",+ "query": "\n UPDATE entity_properties\n SET values = cell.value, updated_at = NOW()\n FROM UNNEST($1::text[], $2::jsonb[]) AS cell(entity_id, value)\n WHERE entity_properties.entity_type = $3\n AND entity_properties.property_definition_id = $4\n AND entity_properties.entity_id = cell.entity_id\n AND entity_properties.values IS DISTINCT FROM cell.value\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "TextArray",+ "JsonbArray",+ {+ "Custom": {+ "name": "property_entity_type",+ "kind": {+ "Enum": [+ "CHANNEL",+ "CHAT",+ "DOCUMENT",+ "PROJECT",+ "THREAD",+ "USER",+ "COMPANY",+ "TASK",+ "CALL_RECORD",+ "CALENDAR_EVENT",+ "INITIATIVE",+ "CONTACT",+ "DATABASE_ROW"+ ]+ }+ }+ },+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "4af6a0999eb209c36f581a632a63b15bae8de290de2c2d1e81ef6620ce6211ea"+}diff --git a/.sqlx/query-536900a16f8e0e3b41ae2b5e50b32be256a56180d59389694215738d971b0d56.json b/.sqlx/query-536900a16f8e0e3b41ae2b5e50b32be256a56180d59389694215738d971b0d56.jsonnew file mode 100644index 0000000000..92a8752d5a--- /dev/null+++ b/.sqlx/query-536900a16f8e0e3b41ae2b5e50b32be256a56180d59389694215738d971b0d56.json@@ -0,0 +1,12 @@+{+ "db_name": "PostgreSQL",+ "query": "SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": []+ },+ "nullable": []+ },+ "hash": "536900a16f8e0e3b41ae2b5e50b32be256a56180d59389694215738d971b0d56"+}diff --git a/.sqlx/query-6bbe4107bbc963ae8b3a3311d92df487cb814c690e2faa30b31e813e484b141b.json b/.sqlx/query-6bbe4107bbc963ae8b3a3311d92df487cb814c690e2faa30b31e813e484b141b.jsonnew file mode 100644index 0000000000..196826f953--- /dev/null+++ b/.sqlx/query-6bbe4107bbc963ae8b3a3311d92df487cb814c690e2faa30b31e813e484b141b.json@@ -0,0 +1,14 @@+{+ "db_name": "PostgreSQL",+ "query": "UPDATE database_rows SET created_at = '2026-01-03T00:00:00Z' WHERE table_id = $1",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid"+ ]+ },+ "nullable": []+ },+ "hash": "6bbe4107bbc963ae8b3a3311d92df487cb814c690e2faa30b31e813e484b141b"+}diff --git a/.sqlx/query-74b572764c780d49b3d9de07bd8e94f443260781a6ab05ae56ce42e8a6a61752.json b/.sqlx/query-74b572764c780d49b3d9de07bd8e94f443260781a6ab05ae56ce42e8a6a61752.jsonnew file mode 100644index 0000000000..c333b8fc78--- /dev/null+++ b/.sqlx/query-74b572764c780d49b3d9de07bd8e94f443260781a6ab05ae56ce42e8a6a61752.json@@ -0,0 +1,17 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values) VALUES ($1, $2, 'DATABASE_ROW', $3, $4)",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Text",+ "Uuid",+ "Jsonb"+ ]+ },+ "nullable": []+ },+ "hash": "74b572764c780d49b3d9de07bd8e94f443260781a6ab05ae56ce42e8a6a61752"+}diff --git a/.sqlx/query-82313749ee4c8c51262f78ab5bfb04c52e9c8f161c7fde4eb53ca93a2d756f6b.json b/.sqlx/query-82313749ee4c8c51262f78ab5bfb04c52e9c8f161c7fde4eb53ca93a2d756f6b.jsonnew file mode 100644index 0000000000..70e9b72166--- /dev/null+++ b/.sqlx/query-82313749ee4c8c51262f78ab5bfb04c52e9c8f161c7fde4eb53ca93a2d756f6b.json@@ -0,0 +1,17 @@+{+ "db_name": "PostgreSQL",+ "query": "INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values) VALUES ($1, $2, 'DATABASE_ROW', $3, $4) ON CONFLICT (entity_id, entity_type, property_definition_id) DO UPDATE SET values = EXCLUDED.values",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ "Uuid",+ "Text",+ "Uuid",+ "Jsonb"+ ]+ },+ "nullable": []+ },+ "hash": "82313749ee4c8c51262f78ab5bfb04c52e9c8f161c7fde4eb53ca93a2d756f6b"+}diff --git a/.sqlx/query-9edfb6f5bb5a129187f8aa6fc91e024b45e5ba328e59a55b8fcb62d3fdaa88db.json b/.sqlx/query-9edfb6f5bb5a129187f8aa6fc91e024b45e5ba328e59a55b8fcb62d3fdaa88db.jsonnew file mode 100644index 0000000000..ac30790e49--- /dev/null+++ b/.sqlx/query-9edfb6f5bb5a129187f8aa6fc91e024b45e5ba328e59a55b8fcb62d3fdaa88db.json@@ -0,0 +1,37 @@+{+ "db_name": "PostgreSQL",+ "query": "\n DELETE FROM entity_properties\n WHERE entity_type = $1 AND property_definition_id = $2 AND entity_id = ANY($3)\n ",+ "describe": {+ "columns": [],+ "parameters": {+ "Left": [+ {+ "Custom": {+ "name": "property_entity_type",+ "kind": {+ "Enum": [+ "CHANNEL",+ "CHAT",+ "DOCUMENT",+ "PROJECT",+ "THREAD",+ "USER",+ "COMPANY",+ "TASK",+ "CALL_RECORD",+ "CALENDAR_EVENT",+ "INITIATIVE",+ "CONTACT",+ "DATABASE_ROW"+ ]+ }+ }+ },+ "Uuid",+ "TextArray"+ ]+ },+ "nullable": []+ },+ "hash": "9edfb6f5bb5a129187f8aa6fc91e024b45e5ba328e59a55b8fcb62d3fdaa88db"+}diff --git a/Cargo.lock b/Cargo.lockindex b5dd35fdcb..1bbd65e609 100644--- a/Cargo.lock+++ b/Cargo.lock@@ -1768,6 +1768,7 @@ dependencies = ["asynk-strim","base64 0.22.1","bytes",+ "chrono","fast_chemail","fnv","futures-channel",@@ -3721,7 +3722,7 @@ dependencies = [[[package]]name = "cache-wasm"-version = "0.6.53"+version = "0.6.54"dependencies = ["async-lock","cache-core",@@ -4779,6 +4780,7 @@ dependencies = ["chrono","crm","databases",+ "databases_sql","email","entity_access","entity_mutation",@@ -4788,6 +4790,7 @@ dependencies = ["graphql_channel","graphql_common","graphql_crm",+ "graphql_databases","graphql_email","graphql_entity_mutation","graphql_favorite",@@ -4804,6 +4807,7 @@ dependencies = ["model_notifications","model_owner","model_user",+ "models_databases","models_pagination","models_permissions","models_properties",@@ -5966,13 +5970,16 @@ dependencies = ["axum","bot_id","chrono",+ "clap","connection_gateway_client",+ "database_env_vars","database_sql","entity_access","entity_access_db_utils","entity_mutation","macro_authorization","macro_db_migrator",+ "macro_entrypoint","macro_event_broker","macro_event_topics","macro_user_id",@@ -6010,6 +6017,7 @@ dependencies = ["anyhow","async-trait","axum",+ "base64 0.22.1","bot_id","chrono","contacts",@@ -6032,6 +6040,7 @@ dependencies = ["schemars 1.2.1","serde","serde_json",+ "sha2 0.10.9","soup","thiserror 2.0.18","tokio",@@ -6567,6 +6576,7 @@ dependencies = ["gmail_client","graphql_calendar","graphql_crm",+ "graphql_databases","graphql_initiative","graphql_scheduled_action","graphql_soup",@@ -9154,6 +9164,27 @@ dependencies = ["workspace-hack",]+[[package]]+name = "graphql_databases"+version = "0.1.0"+dependencies = [+ "async-graphql",+ "chrono",+ "databases",+ "databases_sql",+ "entity_access",+ "futures",+ "graphql_common",+ "graphql_soup",+ "macro_user_id",+ "model-entity",+ "models_databases",+ "rootcause",+ "tokio",+ "tracing",+ "workspace-hack",+]+[[package]]name = "graphql_email"version = "0.1.0"diff --git a/Cargo.toml b/Cargo.tomlindex f14974db70..2c8f7a0aaa 100644--- a/Cargo.toml+++ b/Cargo.toml@@ -71,6 +71,7 @@ members = ["crates/graphql_calendar","crates/graphql_channel","crates/graphql_common",+ "crates/graphql_databases","crates/graphql_email","crates/graphql_entity_mutation","crates/graphql_favorite",diff --git a/PERF_REPORT.md b/PERF_REPORT.mdnew file mode 100644index 0000000000..8cebfa2762--- /dev/null+++ b/PERF_REPORT.md@@ -0,0 +1,399 @@+# Database performance audit — 2026-10-05–08++At 100,000 stored records, the table opens in **1.73 seconds cold and 1.66 seconds+warm**, compared with 33.82 seconds cold and a Chrome crash on warm reload before+server pagination and virtualization. All 18 final opening samples across three+views were under 3 seconds. These are local measurements, not a production SLA.++The initial view now loads 500 records and mounts about 25 visible rows. Filtering+and ordering cover the entire table on the server before pagination. Opening does+not wait for all 100,000 records to download. The Soup and engine page limits remain+500; boards, tables containing formula columns, and arbitrary SQL retain the+existing engine reader and its 20,000-row cap.++- Profiling PR: <https://github.com/macro-inc/macro/pull/7496>+- Optimization PR: <https://github.com/macro-inc/macro/pull/7539>+- Current measured code: `c517578d32` (optimized frontend bundle, development Rust backend).+- Original optimized measurements: `8bec61c252`, retained below for attribution.+- Local app: a database on an isolated local stack (`just run_local --instance <name>`), opened at `/app/database/<id>`.+- Synthetic login through the local email-code flow.++## Method and fixture++The original isolated `database-profiling-audit` stack uses its own Postgres on+port 21000 and HTTPS proxy on 21009. The current revalidation uses the separate+`database-profiling-final` stack on 26000/26009. Tests used a private Playwright Chrome+153.0.8010.52, a 1600×1000 viewport, and a synthetic account. The shared Chrome and+the other worktree's stack were not used. Existing databases and volumes were kept.+The host exposes 144 logical Intel Xeon 6985P-C cores and about 283 GiB RAM; it is+shared with other development work. Both frontends are optimized Vite bundles+configured for the local backend. The current bundle is staged in Caddy. Its+build explicitly enables databases and browser telemetry, exporting to the local+`/i/otlp/v1/traces` endpoint. No Vite development server is used for final timings.++The fixture contains 100,000 Records, 20 Accounts, and nine columns: Name, Amount,+Bucket, Stage, Active, Due, Website, Notes and Account. Values include text, numbers,+selects, booleans, dates, links and record references. Records were grown through+1k, 5k, 10k, 25k, 50k and 100k stored entries. The selective views are Bucket 99+(1,000 matches) and Stage New plus the first Account (1,667 matches).++The committed [measurement harness](apps/web/scripts/database-profile/measure.mjs)+records navigation through the first populated grid and two animation frames.+This is a rendering opportunity, not proof of a physical paint. Polling adds up to+100 ms uncertainty. Each cold sample starts with a fresh browser context containing+login state; its warm sample reloads the same context. “Cold” does not mean a cold+Postgres buffer cache or operating-system filesystem cache. Final results use three+repetitions per view, with CPU profiling disabled and browser tracing enabled.+A TypeScript check overlapped the beginning of the original optimized run.+The current final runs started after seeding, builds and recovery of the local+collaboration gateway finished. The final browser runs used an isolated network+namespace so unrelated host Docker network changes could not interrupt requests.+The integrated app's default service-worker behavior was retained.++The new fixture was also grown through all six sizes. Its 1k–25k opening checks+were preflight runs with browser tracing disabled and are excluded from the final+comparison table. At 50k, a traced cold/warm pair measured 1.976/1.216 seconds.+The 100k results below contain three traced cold/warm pairs for each view.++Raw results, traces, SQL plans, profiles and logs stayed outside the repository,+with the credentials. The harness writes them to the `--output` directory it is given. See the+[profiling guide](apps/web/scripts/database-profile/README.md) for reproduction,+span definitions and safe Postgres statistics collection. Measurements use deltas+from `pg_stat_statements`; shared statistics were not reset.++## Opening results++Current code, 100,000 stored records, first 500 matching records loaded. Values are+seconds, **median [minimum–maximum]**, three samples in each column.++| View | Cold | Warm |+| --- | --- | --- |+| All records | **1.731 [1.665–1.801]** | **1.662 [1.628–1.910]** |+| Bucket 99 | **1.956 [1.869–2.024]** | **2.337 [1.872–2.890]** |+| Stage + Account | **1.936 [1.743–1.962]** | **2.294 [1.922–2.338]** |++Source: `entity-final/main-integrated-100000-{all,filtered,indexed}/results.json`.+All 18 samples mounted exactly 25 rows. This revalidation includes main through+`b9f4217a52`. Warm navigation is slower and more variable than the previous build;+these latest results replace the earlier numbers as the current performance claim.++The previous integration (`9a4bd03d65`) measured:++| View | Cold | Warm |+| --- | --- | --- |+| All records | **1.975 [1.952–1.988]** | **1.163 [1.079–1.200]** |+| Bucket 99 | **2.093 [2.084–2.160]** | **1.245 [1.161–1.302]** |+| Stage + Account | **1.991 [1.986–2.177]** | **1.140 [1.133–1.153]** |++Source: `entity-final/entity-healthy-100000-{all,filtered,indexed}/results.json`.+That was a fresh-fixture revalidation after the upstream entity refactor and local+service recovery. Upstream startup changes, environment and serving differences+prevent attributing the differences between these runs to one change.++The original optimized build (`8bec61c252`, October 5) measured:++| View | Cold | Warm |+| --- | --- | --- |+| All records | **2.308 [2.300–2.331]** | **1.287 [1.278–1.343]** |+| Bucket 99 | **2.475 [2.435–2.541]** | **1.462 [1.425–1.480]** |+| Stage + Account | **2.367 [2.280–2.592]** | **1.295 [1.246–1.326]** |++Source: `paging-release-100000-{all,filtered,indexed}/results.json`.++The scale audit exposed the previous materialization and mounting cost:++| Stored records | Initial rows loaded before paging | Cold opening | Warm opening |+| --- | ---: | ---: | ---: |+| 1,000 | 1,000 | 4.458 s | 3.498 s |+| 5,000 | 5,000 | 11.047 s | 11.132 s |+| 10,000 | 10,000 | 17.544 s | 17.557 s |+| 25,000 | 20,000 | 36.245 s | Chrome crashed |+| 50,000 | 20,000 | 36.512 s | Chrome crashed |+| 100,000 | 20,000 | 37.370 s | Chrome crashed |+| 100,000, improved Soup SQL | 20,000 | 33.823 s | Chrome crashed |++The first three rows are three-sample medians. Runs that crashed have one completed+cold sample; they are not medians. These are successive audit stages, not one+unchanged binary at every size. Sources: `baseline-1000`,+`healthy-baseline-5000`, `text-10000`, `text-{25000,50000,100000}-all`,+and `final-100000-all`.++At 100k, the previous numeric filtered view took 15.058 s cold / 23.323 s warm;+the indexed cohort took 6.050 s / 4.139 s. The warm numeric regression was real:+the capped engine replayed many pages after cache eviction. Increasing retention+alone did not solve scaling. Virtualization alone stopped the huge DOM/crash but+still waited for capped reads. Server pagination removed that opening work.++An intermediate paged build measured 2.718 s / 1.643 s for all records and+3.330 s / 2.278 s for the indexed cohort. The final property-index and cache-opening+changes account for the last improvement. These results are preserved in+`paging-controlled-100000-*`.++## Attribution through the stack++Current median-cold traces confirm browser-to-Postgres propagation and the+expected 500-row read. Timings below are milliseconds and include child spans.++| View | Trace | Ordered-ID SQL | GraphQL including serialization | Serialization |+| --- | --- | ---: | ---: | ---: |+| All records | `f9c8cfc1c6cae251f581162d1cfa6b8c` | 5.53 | 194.54 | 28.61 |+| Bucket 99 | `11e311c4aac6c25ab9968c44e1e37aaf` | 204.52 | 382.98 | 29.88 |+| Stage + Account | `c6636043be69f92e34cdb16d037d4c90` | 72.72 | 252.09 | 29.37 |++Connection acquisition was about 0.4 ms for the all-records sample. Browser row+decoding took 8.9 ms, publication took 34.8 ms, and the database query span took+398.8 ms. App startup and other browser work account for much of the remaining+1.73-second navigation time.++The slowest warm sample (Bucket 99, 2.890 seconds) has trace+`ef52acf0b33cf2498c5ad7b8af8cf51c`. Its database query span took 997.1 ms, including+203.8 ms of ID selection, 403.0 ms of GraphQL execution, 4.6 ms of browser decoding+and 25.7 ms of publication. The HTTP request began about 471 ms after the query+span began; earlier navigation also contains a roughly one-second gap between+metadata and block requests. These timings identify work outside SQL as a remaining+target, but do not establish the cause of the startup/request scheduling delays.++The following before/after attribution uses the original optimized run so it+remains paired with the SQL plans and profiles collected during those iterations.+Representative trace IDs below are query traces, not unrelated bootstrap requests.+Timings are inclusive; parent and child durations must not be summed.++| Scenario | Before trace | Final trace |+| --- | --- | --- |+| All records | `2d56848faeb84bc8b515e7911651fae0` | `fd5f1f124d1148ba1b0e1acc01a5cec4` |+| Numeric filter | `769c5ec8fa461febfe3a314adbc05a7a` | `f06fdfb13b5d1126c69da6811055200c` |+| Stage + Account | `ae483a5126ab90582c51a44b6e684f1f` | `5144646b50b90f276e91b83f5b358bb6` |++The all-records final trace belongs to the median cold sample. Its ordered-ID SQL+took 5.08 ms; GraphQL execution including serialization took 188.54 ms, of which+serialization was 28.79 ms. Browser cell decoding took 44.6 ms and synchronous+publication 48.9 ms. The database query span lasted 475.9 ms; navigation and app+bootstrap account for much of the remaining opening time.++The numeric filter's final median-cold trace spent 206.77 ms selecting IDs and+426.98 ms executing GraphQL. The indexed cohort's representative cold trace spent+234.68 ms selecting IDs; representative warm trace+`42eeca1e4f3214d0adff8807222fde29` spent 77.33 ms. A standalone EXPLAIN comparison+reduced that cohort's selection from 796.83 ms to 163.81 ms by exposing positive+JSONB membership to the existing GIN index and preserving indexable joins in both+directions. The typed predicate still determines scalar, list and empty semantics.++Before pagination, the all-records trace spent about 11.57 seconds fetching,+0.39 seconds folding, and 16.67 seconds publishing 20,000 rows. The old browser+reached about 3.47 GB JavaScript heap and 2.08 million DOM nodes. Earlier Soup+EXPLAINs also exposed avoidable scans: pruning impossible project branches and+preserving UUID joins reduced an individual 501-row SQL page from 115.73 ms to+roughly 3–5 ms. Faster individual pages could not eliminate the cost of reading+and mounting forty pages on opening.++Instrumentation covers browser query/decode/publication/frame, cache requests and+queueing, GraphQL execution/serialization, authorization/catalog reads, property+hydration, connection acquisition and PostgreSQL selection. The collector lost its+Docker network attachment after a restart; it was reattached before the final+release measurements. Missing spans from the earlier intermediate run are not+treated as zero-cost work.++## Interaction and deep-read checks++Current-code browser interaction results, milliseconds. Table switches and edits+are three-sample medians; the picker, insert and view switch are single samples.+Edits measure the visible optimistic result separately from the HTTP response.++| Action | All records | Stage + Account |+| --- | ---: | ---: |+| Cached table switch | 430.7 | 514.4 |+| Visible edit | 19.5 | 17.4 |+| Edit HTTP response | 57.7 | 50.2 |+| First picker open | 66.3 | 68.9 |+| Insert HTTP response | 146.1 | 109.7 |+| Switch to numeric filtered view | 901.5 | — |+| Scroll frame interval p50 / p95 | 16.7 / 66.6 | 16.7 / 66.7 |++The numeric view-switch check waits for Record 099001, proving the new filter's+membership is visible. The earlier 133.1 ms check waited only for 500 rows; because+both views have that count, it did not establish completion of the view change.+Inserted records opened with the exact submitted name, including the saved draft+that remains editable outside a filtered view. The harness follows the accessible+record-opening control or the creation notice; a retained draft's UI identity can+differ from its saved server ID.++Scroll uses sixty 800-pixel wheel events, about 16 ms apart. Maximum observed+frame intervals were 83.4 ms (all) and 100.0 ms (indexed), so this is not a claim of+smooth 60 fps. Both runs had no page errors. Temporary edits and inserted records+were restored/deleted; their cleanup journals are empty. Sources:+`entity-final/interactions-main-all` and+`entity-final/interactions-main-indexed`.++After the machine restart, LocalStack's temporary resources were missing and the+collaboration gateway had exited. Writes committed quickly but waited 3–4 seconds+for failed gateway notifications. Idempotent LocalStack provisioning and restarting+the existing gateway restored 50–53 ms median write acknowledgement without a code+change or database reset. All final interaction measurements above use the+recovered services.++A separate stress run on the current code requested all 200 pages and reached+100,000 loaded records in **87.45 seconds**, with **26 mounted rows**, **3,854 DOM+nodes**, and **303.46 MB JavaScript heap after GC**. The last record opened, no page+errors occurred, and switching to Accounts took **99.0 ms**. Heap immediately after+that switch was **302.29 MB**. Source: `entity-final/deep-main`. The fixture+ended with exactly 100,000 Records and 20 Accounts.++The original optimized deep test also requested all 200 pages, collecting garbage+at milestones. It reached 100,000 rows in **89.35 seconds**, with **26 mounted rows**,+**3,701 DOM nodes**, **308.48 MB JavaScript heap after GC**, and no page errors.+The last record opened successfully. Switching to Accounts afterward took+**82.8 ms**, compared with **10,130 ms** before the cache-opening fix. Heap just+after that switch remained **308.26 MB**; the test does not establish eventual+reclamation of the retained data/cache work. Source: `paging-deep-final`.++Earlier append code repeatedly decoded accumulated pages and took 226.69 seconds+to traverse the same 100k fixture. Reusing unchanged decoded cells brought this+to about 90 seconds. These stress timings include GC pauses and are distinct+from the opening measurements above.++## Changes and correctness++The GraphQL field `user.databaseViewRows(databaseId, input)` accepts a typed view+query and cursor. The existing view compiler validates filters and sorts against+the authorized catalog. PostgreSQL evaluates them globally, applies stable ordering+with row-ID ties, and returns at most 500 rows plus continuation information.+Cursors bind the query, catalog and table version; concurrent changes restart the+read instead of mixing versions. The response uses canonical Soup row entities.++The hexagonal boundary was checked: typed access receipts and domain services own+authorization and use-case policy; the GraphQL adapter maps transport types; the+database repository owns SQL and transactions. Generic property mutations now+reject database-row targets so edits cannot bypass database versions and the change+journal. Supported row writes continue through database operations.++| Commit | Change and reason |+| --- | --- |+| `9e8e5ef8b4` | Browser query, fetch/fold, comparison, publication and frame spans. |+| `a822d25b79` | Repeatable local scale harness and profiling instructions. |+| `f2a37be54c` | Separate connection wait, SQL/decode and response serialization. |+| `df6f0713e4` | Restrict profiling capture to relevant backend database requests. |+| `757f813c9b` | Preserve unchanged grid rows and shared cell setup across edits. |+| `ef2d6e603a` | Retain complete capped reads in the bounded page cache. |+| `d2d4e28e7c` | Prune impossible Soup branches; index row cursors and UUID joins. |+| `0e40f2b68a` | Cancel superseded reads between pages while allowing accepted writes to finish. |+| `36ec95efdf` | Authorized, globally filtered and ordered server pages; typed GraphQL and cursor/version checks. |+| `2d288e3991` | Paged table source, virtual rows, focus/edit preservation and incremental cell decoding. |+| `aeba0c1a0f` | Use property membership indexes without changing canonical filter semantics. |+| `151003d902` | Start network reads while the cache is busy; retain offline fallback using the real client's stream semantics. |+| `8bec61c252` | Regenerate dependency unification and closure metadata required by CI. |+| `73a084633c` | Integrate current main and regenerate the closure after the upstream crate move. |+| `0025b45ee9` | Preserve cancellation classification during engine opening; build and remove indexes concurrently with safe interrupted-build retries. |+| `9a4bd03d65` | Integrate the upstream storage/entity split while retaining paged reads, stable virtual rows and optional schema actions. |+| `c517578d32` | Integrate main through the AI/PSD editors and regenerate dependency closures; rebuild and remeasure the combined app. |++API traversal verified all 100,000 unique IDs in manual order, both endpoint+numeric cohorts (Bucket 0 and Bucket 99, exactly 1,000 each), the 1,667-row indexed+cohort, and the globally largest 1,000 Amount values in descending order+(1099.99 through 1090.00). Tests cover ties, nulls, scalar/list differences,+case mapping, wildcard escaping, nanosecond dates, stale cursors, late responses,+disposal and offline behavior. Real URQL plus the normalized cache exchange is used+for the new cache/network tests; mocking only the page function concealed an+operation-deduplication problem during review, which was fixed before publication.++Validation: 590 frontend tests across 83 files; TypeScript; `just check`; generated+dependency verification; production frontend build and local DSS build; five QC+reviews. Affected Rust suites passed: `database_sql` (136), `databases_sql` (35),+`databases` with Postgres/view features (222), `properties` (273, four ignored),+`soup` (258, two ignored), `cache-core` (96 unit tests plus integration suites),+`graphql_databases` (1), `complete_graph` (59), and DSS binary tests (17). Tests ran+from the repository root with SQLX_OFFLINE unset. SQLx metadata and GraphQL schema+were generated using the repository tools.++For CI integration, the branch was merged with `main` at `55f06de203` and the+dependency generator updated the closure entry for the relocated `agent_fold`+crate. The database optimization code was unchanged. The 590 frontend tests,+TypeScript and schema checks, scoped `just check`, generated dependency check,+and the `database_sql`, `databases_sql`, `databases`, `graphql_databases`,+`complete_graph` and DSS suites passed again. These checks used a separate test+database; the measured 100k fixture and running bundle were preserved.++Review follow-up verified the index migrations against separate 100k-row fixtures.+The original build blocked another insert until its 400 ms lock timeout; the+concurrent build allowed the insert while waiting for an older transaction.+Cancelling a build left an invalid index, and retry correctly failed instead of+silently accepting it. Successful migration runs remain no-ops on rerun through+the SQLx ledger. Each concurrent operation has its own nontransactional migration+because PostgreSQL treats a multi-statement SQLx batch as an implicit transaction.+Both final cursor indexes were valid. Evidence: `review-index-locks.json`.++The engine-open cancellation regression failed before the fix and passed after+it. All 12 driver tests, seven paged-reader tests, 222 database tests, two migration+tests, TypeScript, `just check`, and five follow-up QC reviews passed.++The subsequent integration with `main` at `4f83c028f2` introduced the upstream+`database_entities` boundary. On head `9a4bd03d65`, 604 frontend tests across 84+files, all five standalone browser tests, TypeScript, `just check`, generated+dependency verification and five integration reviews passed. Backend revalidation+passed 226 database tests, 35 database-query service tests, one GraphQL adapter+test, 59 complete-graph tests, 312 Soup tests with the `all` feature (four ignored),+and 273 property tests (four ignored). All code and test CI checks passed on that+head; the optional preview deployment was cancelled to keep runtime work local.++The upstream entity-split migration discards existing database content. It was+applied only to a new test database and the fresh `database-profiling-final` stack;+the original 100k fixture was preserved. The hexagonal boundary was checked:+GraphQL forwards authenticated, typed requests to the domain service; the domain+owns authorized catalog access and cursor/query policy; the Postgres adapter owns+SQL and transaction mechanics. Core storage without an app entity stays outside+app discovery and canonical Soup hydration.++The final integration includes `main` at `b9f4217a52`. GitHub's combined-tree CI+found two missing transitive dependency entries after main added the AI and PSD+engines; the dependency generator repaired them. Revalidation passed 614 frontend+tests, seven standalone browser tests, TypeScript, 139 database-engine tests,+228 database tests, 35 query-service tests, one GraphQL adapter test, 60 complete-+graph tests, 17 DSS tests and five QC reviews. The standalone browser ran in the+isolated network namespace after host network changes interrupted module downloads.+TypeScript passed with a 16 GiB Node heap after exhausting the default 4 GiB limit.+The unrelated agent-fold WASM suite also passed both tests locally after a cold+build hook timed out in the earlier CI run. The optimized frontend, all eight WASM+modules and DSS were rebuilt before the current measurements. API traversal and+the browser interaction/deep-read checks above passed again on `c517578d32`.++### October 8 main integration++Main through `9f2a13df42` is integrated, including the shared database UI, sticky+summaries and formula columns. Local revalidation passed 822 frontend tests in+117 files, nine browser tests, TypeScript and schema checks. Rust revalidation+passed 244 database tests, 39 database-query service tests, one GraphQL adapter+test, 68 complete-graph tests and 29 DSS tests with SQLX_OFFLINE unset.++The integration preserves editor focus when virtualized rows cross the sticky+header/footer and when adding or removing formulas switches readers. Formula+tables retain the existing engine reader; a delayed or failed handoff keeps the+displayed rows and their partial-result label until its replacement arrives.+These are functional checks of the integration, not new opening benchmarks.+The performance measurements and the local demo above remain on `c517578d32`.++## Remaining limits++- This achieves the requested opening target on the local table fixture. Broader+ production datasets, hardware, network latency and concurrent load still need+ production measurements. App bootstrap is now a larger share than SQL for the+ unfiltered first page.+- Loading every page remains proportional to the rows requested. Very deep reads+ accumulate decoded rows and background cache persistence work. Virtualization+ bounds mounted DOM, not the complete in-memory dataset.+- Refreshing after a write currently re-reads the number of pages already loaded+ to preserve global membership/order. After loading all 100k, this is expensive.+ Sustained concurrent writes can exhaust the automatic retry and show a refresh+ error. Resetting to the first page after every edit would lose the user's place.+ A bounded page window and targeted invalidation with correct cursor/version+ handling are the next changes for sustained deep editing.+- Boards retain the existing capped reader. They need pagination designed around+ lanes and card order; arbitrary SQL also retains its current engine limit.+ After integration with formula columns on October 8, tables containing formulas+ use that engine too, so their computed values, filters and sorts remain correct.+ The opening measurements above cover tables without formula columns.+- Aggressive scrolling still has long frames. Cell mounting/paint and cache+ backpressure remain worth profiling separately from opening latency.++The repository automatically published frontend previews when the PRs were opened.+Both previews were removed using the repository cleanup workflow. Runtime testing+and the link above use only the isolated local stack; the PR was not merged and+no backend release was deployed.diff --git a/apps/web/scripts/database-profile/README.md b/apps/web/scripts/database-profile/README.mdindex 3392aa6e08..40bb04c813 100644--- a/apps/web/scripts/database-profile/README.md+++ b/apps/web/scripts/database-profile/README.md@@ -5,6 +5,20 @@ Run the isolated stack with browser telemetry enabled, followingthroughout the audit; `run_local` startup and its quit command rebuild/removeinstance data. Use the `r` hotkey to rebuild services without replacing data.+For an optimized headless frontend, explicitly set `VITE_ENABLE_DATABASES=true`,+`VITE_ENABLE_BROWSER_OTEL=true` and+`VITE_OTEL_EXPORTER_URL=/i/otlp/v1/traces` when building. The static build does not+inherit the interactive Vite server's tracing configuration. Before recording+comparisons, verify that a database request carries `traceparent` and its browser+and backend spans can be retrieved from the local collector.++After a machine restart, follow the existing-instance recovery steps in the local+running guide. Restore missing LocalStack resources with+`cargo x localstack-provision --instance <name> --port-base <base>`, then start services+that exited while their dependencies were unavailable. Check the collaboration+gateway as well as Postgres and DSS: a stopped gateway can add seconds of DNS or+connection retries to otherwise fast database writes.+```shnix develop --command just run_local --no-doppler --instance database-profiling-audit --port-base 21000```@@ -22,7 +36,7 @@ From `apps/web`, measure without changing data:bun scripts/database-profile/measure.mjs \--url https://localhost:21009/app/database/UUID \--state /tmp/database-profile-auth.json \- --table Records --rows 1000 --entries 1000 \+ --table Records --rows 500 --entries 1000 \--output /tmp/database-profile-1000 --repeats 3 \--chrome /path/to/google-chrome```@@ -31,10 +45,12 @@ Use the actual proxy URL printed by `run_local`. The harness accepts loopbackURLs only, opens its own browser, and refuses to overwrite an output directory.Each repetition starts with a fresh context containing only the saved loginstate, then measures a warm reload in that same context. `--rows` is the number-of saved rows expected in the selected view; the blank insertion row is excluded.-`--entries` is the fixture's total stored row count. The engine currently caps-reads at 20,000 rows, so a 100k database does **not** imply 100k rendered rows.-Confirm the trace's `database_sql.truncated` attribute when interpreting results.+of saved rows expected to be loaded initially; the blank insertion row is excluded.+`--entries` is the fixture's total stored row count. Table views load a first+page of at most 500 globally filtered and ordered records. Use `--rows 500+--entries 100000` for a full 100k table. The harness distinguishes loaded rows+from mounted rows in the virtual grid. Boards and arbitrary SQL retain the+engine's 20,000-row cap; inspect `database_sql.truncated` for those reads.The JSON results include first complete grid and subsequent frame opportunity,long tasks, CDP script/layout/task counters, heap size, HTTP timing/bytes, and@@ -62,6 +78,9 @@ latency. Exported profiles may contain application source URLs and fixture IDs.| `database_sql.compare` | Equality comparison of the new catalog/result with the prior answer || `database_sql.publish` | Signal updates and synchronous downstream reactive work || `database_sql.frame` | Time after a changed result is published until a rendering opportunity; cancelled/hidden/timeout outcomes are not ready frames |+| `database.view_rows.read` | Authorized catalog, view compilation, cursor validation and ordered identity read |+| `database.view_rows.pool.acquire` / `.select` | Primary connection acquisition and filtered/ordered PostgreSQL page selection |+| `database.view.load_more` | Continuation fetch and publishing the extended table result || `soup.pool.acquire` | Connection acquisition, with pool size and idle count at entry || `soup.sql.fetch_decode` | Query construction, SQL round trip, row decoding and flat-row conversion, excluding connection acquisition || `soup.rows.convert` | Grouped-row conversion after the SQL read |diff --git a/apps/web/scripts/database-profile/measure.mjs b/apps/web/scripts/database-profile/measure.mjsindex 165d048899..5e83d5a18d 100644--- a/apps/web/scripts/database-profile/measure.mjs+++ b/apps/web/scripts/database-profile/measure.mjs@@ -119,10 +119,12 @@ try {await session.send('Profiler.start');}const started = performance.now();- await page.goto(values.url, {+ const navigation = {waitUntil: 'domcontentloaded',timeout: 180_000,- });+ };+ if (temperature === 'cold') await page.goto(values.url, navigation);+ else await page.reload(navigation);if (values.table)await page.locator('[aria-label="Database tables"]')@@ -130,12 +132,18 @@ try {.click();await page.waitForFunction((count) => {+ if (+ [...document.querySelectorAll('[role="alert"]')].some((alert) =>+ alert.textContent.includes('This database could not be displayed')+ )+ )+ throw new Error('The database grid crashed while opening.');const grid = document.querySelector('[role="grid"]');return (grid &&- grid.querySelectorAll(+ Number(grid.getAttribute('data-grid-saved-rows') ?? grid.querySelectorAll('[data-grid-row-id]:not([data-grid-row-id^="draft:"])'- ).length >= count+ ).length) === count);},expectedRows,@@ -150,7 +158,10 @@ try {);const grid = document.querySelector('[role="grid"]');return {- rows: grid.querySelectorAll(+ rows: Number(grid.getAttribute('data-grid-saved-rows') ?? grid.querySelectorAll(+ '[data-grid-row-id]:not([data-grid-row-id^="draft:"])'+ ).length),+ mountedRows: grid.querySelectorAll('[data-grid-row-id]:not([data-grid-row-id^="draft:"])').length,cells: grid.querySelectorAll('[role="gridcell"]').length,@@ -173,6 +184,12 @@ try {}// Capture refresh requests too, separately from the first rendering opportunity.await page.waitForTimeout(1500);+ if (!(await page.locator('[role="grid"]').count())) {+ await page.screenshot({+ path: `${output}/${repeat}-${temperature}-failure.png`,+ });+ throw new Error('The database grid disappeared during refresh.');+ }page.off('requestfinished', finished);await Promise.all(pending);const result = {diff --git a/apps/web/src/components/view-shell/MobileFilterDrawer.tsx b/apps/web/src/components/view-shell/MobileFilterDrawer.tsxindex 61077bf0ac..1e23556fe7 100644--- a/apps/web/src/components/view-shell/MobileFilterDrawer.tsx+++ b/apps/web/src/components/view-shell/MobileFilterDrawer.tsx@@ -235,8 +235,14 @@ function Option(props: {<MobileDrawer.Itemrole={props.selectionMode === 'single' ? 'radio' : 'checkbox'}aria-checked={props.checked}+ // Neighbouring checked boxes join through their own attribute: a sibling+ // rule keyed on `role` restyles every element with a role inserted+ // anywhere, such as each row a scrolling database grid mounts.+ data-checked-checkbox={+ props.selectionMode !== 'single' && props.checked ? '' : undefined+ }class={cn(- '[&[role=checkbox][aria-checked=true]:has(+[role=checkbox][aria-checked=true])]:rounded-b-none [&[role=checkbox][aria-checked=true]+[role=checkbox][aria-checked=true]]:rounded-t-none',+ '[&[data-checked-checkbox]:has(+[data-checked-checkbox])]:rounded-b-none [&[data-checked-checkbox]+[data-checked-checkbox]]:rounded-t-none',props.class)}onClick={() => props.onChange(!props.checked)}diff --git a/apps/web/src/features/agents-view/agents-view.css b/apps/web/src/features/agents-view/agents-view.cssindex 661f322e92..875b7ce9b9 100644--- a/apps/web/src/features/agents-view/agents-view.css+++ b/apps/web/src/features/agents-view/agents-view.css@@ -805,11 +805,11 @@overflow: hidden;}- :where(.agents-view, .agents-view-portal) .scard > * + * {+ :where(.agents-view, .agents-view-portal) .scard > :not(:first-child) {border-top: var(--hair) solid var(--edge-muted);}- :where(.agents-view, .agents-view-portal) .runtimes > #runtimeList > * + * {+ :where(.agents-view, .agents-view-portal) .runtimes > #runtimeList > :not(:first-child) {border-top: var(--hair) solid var(--edge-muted);}@@ -1361,8 +1361,8 @@background: var(--surface-1);}- :where(.agents-view, .agents-view-portal) .picker .results > * + *,- :where(.agents-view, .agents-view-portal) .picker .picked > * + * {+ :where(.agents-view, .agents-view-portal) .picker .results > :not(:first-child),+ :where(.agents-view, .agents-view-portal) .picker .picked > :not(:first-child) {border-top: var(--hair) solid var(--edge-muted);}@@ -1753,7 +1753,7 @@scrollbar-color: var(--surface-4) transparent;}- :where(.agents-view, .agents-view-portal) .plist > * + * {+ :where(.agents-view, .agents-view-portal) .plist > :not(:first-child) {border-top: var(--hair) solid var(--edge-muted);}diff --git a/apps/web/src/features/block-database/browser-test/table.browser.e2e.ts b/apps/web/src/features/block-database/browser-test/table.browser.e2e.tsindex 9eb43d245d..dc39d7af5a 100644--- a/apps/web/src/features/block-database/browser-test/table.browser.e2e.ts+++ b/apps/web/src/features/block-database/browser-test/table.browser.e2e.ts@@ -67,6 +67,61 @@ test('resizes from the rendered width and saves once on release', async ({.toBe(Math.round(before.width + 70));});+test('reveals virtualized editors between the sticky header and summary', async ({+ page,+}) => {+ await page+ .getByRole('button', { name: 'Load 1000 records', exact: true })+ .click();+ await page+ .getByRole('button', { name: 'Edit last record', exact: true })+ .click();+ const input = page.getByRole('textbox', { name: 'Edit Name', exact: true });+ await expect(input).toBeFocused();+ await expect(input).toHaveValue('Record 999');+ await expect+ .poll(() =>+ input.evaluate((element) => {+ const rect = element.getBoundingClientRect();+ const summary = document+ .querySelector('[data-grid-summary]')!+ .getBoundingClientRect();+ return (+ rect.bottom <= summary.top &&+ document.elementFromPoint(+ rect.x + rect.width / 2,+ rect.y + rect.height / 2+ ) === element+ );+ })+ )+ .toBe(true);+ await input.press('Escape');+ await page+ .getByRole('button', { name: 'Edit first record', exact: true })+ .click();+ await expect(input).toBeFocused();+ await expect(input).toHaveValue('Record 0');+ await expect+ .poll(() =>+ input.evaluate((element) => {+ const rect = element.getBoundingClientRect();+ const header = document+ .querySelector('[role="columnheader"]')!+ .getBoundingClientRect();+ return (+ rect.top >= header.bottom &&+ document.elementFromPoint(+ rect.x + rect.width / 2,+ rect.y + rect.height / 2+ ) === element+ );+ })+ )+ .toBe(true);+ expect(await page.locator('[data-grid-row-id]').count()).toBeLessThan(80);+});+test('keeps an unsafe integer draft instead of saving a rounded value', async ({page,}) => {diff --git a/apps/web/src/features/block-database/browser-test/table.tsx b/apps/web/src/features/block-database/browser-test/table.tsxindex 5d89f1a217..a7de453a9c 100644--- a/apps/web/src/features/block-database/browser-test/table.tsx+++ b/apps/web/src/features/block-database/browser-test/table.tsx@@ -2,6 +2,7 @@ import '@app/index.css';import type { SortKey } from '@core/database-sql/generated/types';import { createSignal } from 'solid-js';import { render } from 'solid-js/web';+import type { DatabaseTableControls } from '../../database/components/database-table';import { GridCell } from '../../database/components/grid-cell';import type { DatabaseViewColumn } from '../../database/core/database-view';import { moveBeside } from '../../database/core/move-beside';@@ -11,6 +12,7 @@ import { DatabaseTableView } from '../../database/views/database-table-view';// Real grid and editors; records and saved layout are local to this browser fixture.function TableFixture() {+ let controls: DatabaseTableControls | undefined;const [columns, setColumns] = createSignal<DatabaseViewColumn[]>([{id: 'name',@@ -96,8 +98,34 @@ function TableFixture() {<output aria-label="Resize saves">{resizeSaves()}</output><output aria-label="Writes">{writes()}</output><output aria-label="Sort">{JSON.stringify(sort())}</output>+ <button+ onClick={() =>+ setRows(+ Array.from({ length: 1000 }, (_, index) => ({+ rowId: `row-${index}`,+ cells: {+ name: `Record ${index}`,+ notes: '',+ status: 'Open',+ amount: index,+ },+ }))+ )+ }+ >+ Load 1000 records+ </button>+ <button onClick={() => controls?.editCell('row-999', 'name')}>+ Edit last record+ </button>+ <button onClick={() => controls?.editCell('row-0', 'name')}>+ Edit first record+ </button></div><DatabaseTableView+ controlsRef={(value) => {+ controls = value;+ }}name="Tasks"rows={rows()}columns={columns()}diff --git a/apps/web/src/features/block-database/component/DatabaseGrid.tsx b/apps/web/src/features/block-database/component/DatabaseGrid.tsxindex 5b379a7973..ef0c38a3bb 100644--- a/apps/web/src/features/block-database/component/DatabaseGrid.tsx+++ b/apps/web/src/features/block-database/component/DatabaseGrid.tsx@@ -163,6 +163,8 @@ function RelatedDatabaseSync(props: {}function TableAdapter(props: DatabaseGridProps & { tableId: string }) {+ const canEdit = createMemo(() => props.canEdit);+const queryClient = useQueryClient();const openDocumentMention = createDatabaseDocumentNavigation();// Keep the final schema for this table available to already-queued writes@@ -331,15 +333,15 @@ function TableAdapter(props: DatabaseGridProps & { tableId: string }) {<StaticMarkdownContext><ColumnUsageContext.Provider value={formsAsking}><OptionEditingContext.Provider- value={props.canEdit ? optionEditing : undefined}+ value={canEdit() ? optionEditing : undefined}><DatabaseProviderapi={createMacroDatabaseApi(databaseId, queryClient)}tableId={props.tableId}view={props.view}capabilities={{- editRows: props.canEdit,- editColumns: props.canEdit,+ editRows: canEdit(),+ editColumns: canEdit(),}}data={{ table: () => editorTable(table()), rows: source }}inferNewColumnsdiff --git a/apps/web/src/features/block-database/queries/table-rows.test.tsx b/apps/web/src/features/block-database/queries/table-rows.test.tsxindex c3aeaf724d..f3bbb2c94d 100644--- a/apps/web/src/features/block-database/queries/table-rows.test.tsx+++ b/apps/web/src/features/block-database/queries/table-rows.test.tsx@@ -8,7 +8,11 @@ import type {} from '@core/database-sql/generated/types';import type { CacheHost } from '@graphql-cache/host/types';import { queryClient } from '@queries/client';-import type { DatabaseSqlQueryCapabilities } from '@queries/database-sql/create-database-sql-query';+import {+ createDatabaseSqlQuery,+ type DatabaseSqlQueryCapabilities,+ type DatabaseSqlStatement,+} from '@queries/database-sql/create-database-sql-query';import {applyDatabaseOps,applyDatabaseTableVersions,@@ -298,6 +302,8 @@ function setup(addOption?: DatabaseRowsSource['addOption'];onSource?: (source: DatabaseRowsSource) => void;view?: Accessor<DatabaseView>;+ table?: Accessor<DatabaseDetail['tables'][number]>;+ viewQuery?: Parameters<typeof createDatabaseRowsSource>[0]['viewQuery'];onTableChanged?: (listener: (version: number) => void) => void;changes?: Parameters<typeof createDatabaseRowsSource>[0]['changes'];} = {}@@ -314,10 +320,14 @@ function setup(source = createDatabaseRowsSource({databaseId: 'db',// Deliberately retain old props: retries must use the refreshed cache.- table: () => initialDetail.tables[0],+ table: options.table ?? (() => initialDetail.tables[0]),view: options.view ?? (() => allGuests),applyOps,read: options.read ?? engine().read,+ viewQuery:+ options.viewQuery ??+ ((statement) =>+ createDatabaseSqlQuery(statement, options.read ?? engine().read)),changes: options.changes,onTableChanged:options.onTableChanged ??@@ -361,6 +371,96 @@ afterEach(() => {});describe('database view reads', () => {+ it('reads formula tables through server pages, their values stored like any cell', async () => {+ const schema = detail();+ const column = schema.tables[0].columns[0];+ const input = structuredClone(column);+ input.column.id = 'amount';+ input.column.property_definition_id = 'amount-definition';+ input.sql_name = '"Amount"';+ input.definition.definition.id = 'amount-definition';+ input.definition.definition.display_name = 'Amount';+ input.definition.definition.data_type = 'NUMBER';+ schema.tables[0].columns.push(input);+ column.column.config = {+ kind: 'derived',+ formula: { kind: 'column', column: 'amount' },+ };+ column.definition.definition.data_type = 'NUMBER';+ column.writable = false;+ const reader = engine(() => ({+ ...guests(),+ columns: [+ { name: 'Name', column: 'definition', kind: 'number' },+ { name: 'Amount', column: 'amount-definition', kind: 'number' },+ ],+ rows: [+ [+ { type: 'number', value: 42 },+ { type: 'number', value: 42 },+ ],+ ],+ }));+ let pagedStatement: Accessor<DatabaseSqlStatement | undefined> | undefined;+ const { source } = setup(schema, () => okAsync([]), {+ read: reader.read,+ view: () => ({+ ...allGuests,+ query: {+ filter: null,+ sort: [{ column: 'name', direction: 'descending' }],+ },+ }),+ viewQuery: (statement) => {+ pagedStatement = statement;+ return createDatabaseSqlQuery(statement, reader.read);+ },+ });+ await waitFor(() =>+ expect(source.snapshot()?.rows).toEqual([+ { rowId: 'record', cells: { name: 42, amount: 42 } },+ ])+ );+ expect(pagedStatement?.()?.view?.query).toEqual({+ filter: null,+ sort: [{ column: 'name', direction: 'descending' }],+ });+ });++ it('keeps every board row available for manual lane ordering beyond page one', async () => {+ const complete = {+ ...guests(),+ rowIds: Array.from({ length: 501 }, (_, index) => `row-${index}`),+ rows: Array.from({ length: 501 }, (_, index) => [+ { type: 'text' as const, value: `Guest ${index}` },+ ]),+ };+ const reader = engine(() => complete);+ let pagedStatement: Accessor<DatabaseSqlStatement | undefined> | undefined;+ const { source } = setup(detail(), () => okAsync([]), {+ read: reader.read,+ view: () => ({+ ...allGuests,+ layout: {+ kind: 'board',+ groupBy: 'status',+ lanes: [],+ cardFields: [],+ title: 'name',+ hideEmptyLanes: false,+ },+ }),+ viewQuery: (statement) => {+ pagedStatement = statement;+ return createDatabaseSqlQuery(statement, reader.read);+ },+ });+ await waitFor(() => expect(source.snapshot()?.rows).toHaveLength(501));+ expect(source.snapshot()?.rows.at(-1)?.rowId).toBe('row-500');+ expect(pagedStatement?.()).toBeUndefined();+ expect(source.pagination?.hasMore() ?? false).toBe(false);+ });+it('runs the view in the engine and keeps the previous rows while a changed one loads', async () => {const [view, setView] = createSignal(allGuests);let finishSearch!: (outcome: Outcome) => void;@@ -713,39 +813,40 @@ describe('a column type change', () => {table,view: () => allGuests,applyOps: vi.fn<ApplyOps>(),- read: {- client: () => client,- cacheHost: () => undefined,- people: async () => [],- catalog: async (schema) => ({- tables: [- {- id: 'guests-table',- databaseId: 'db',- database: 'Personal',- name: 'Guests',- source: 'database',- columns: schema.databases[0].tables[0].columns.map(- (column) => ({- id: column.definition,- placement: column.id,- name: column.name,- kind:- column.definition === 'number-definition'- ? { kind: 'number' }- : { kind: 'text' },- })- ),- },- ],+ viewQuery: (statement) =>+ createDatabaseSqlQuery(statement, {+ client: () => client,+ cacheHost: () => undefined,+ people: async () => [],+ catalog: async (schema) => ({+ tables: [+ {+ id: 'guests-table',+ databaseId: 'db',+ database: 'Personal',+ name: 'Guests',+ source: 'database',+ columns: schema.databases[0].tables[0].columns.map(+ (column) => ({+ id: column.definition,+ placement: column.id,+ name: column.name,+ kind:+ column.definition === 'number-definition'+ ? { kind: 'number' }+ : { kind: 'text' },+ })+ ),+ },+ ],+ }),+ openView: async (catalog) =>+ answering(+ catalog.tables[0].columns[0].id === 'number-definition'+ ? await numbers+ : guests()+ ),}),- openView: async (catalog) =>- answering(- catalog.tables[0].columns[0].id === 'number-definition'- ? await numbers- : guests()- ),- },onTableChanged: () => {},onCommitted: () => {},applyVersions: () => {},diff --git a/apps/web/src/features/block-database/queries/table-rows.ts b/apps/web/src/features/block-database/queries/table-rows.tsindex 0ab9fce5d5..f81b524d7c 100644--- a/apps/web/src/features/block-database/queries/table-rows.ts+++ b/apps/web/src/features/block-database/queries/table-rows.ts@@ -48,12 +48,9 @@ import {inferDatabaseNumber,} from '../../database/core/column-inference';import type { DatabaseCellValue } from '../../database/core/database-view';-import { gridRows } from '../../database/core/grid-cells';+import { type GridRowsRead, gridRows } from '../../database/core/grid-cells';import { writeDatabaseRow } from '../../database/core/row-write';-import type {- DatabaseRow,- DatabaseRowMutation,-} from '../../database/core/table';+import type { DatabaseRowMutation } from '../../database/core/table';import type {DatabaseCellFailure,DatabaseReadFailure,@@ -65,6 +62,7 @@ import {refreshChangedRows,type TableChangesCapabilities,} from './table-changes';+import { createDatabaseViewQuery, type DatabaseViewQuery } from './view-rows';/** A stale table or column name, which a refreshed schema may resolve. */function isStaleSchema(@@ -72,6 +70,7 @@ function isStaleSchema(): boolean {return (failure.kind === 'engine' ||+ ('staleSchema' in failure && failure.staleSchema === true) ||(failure.kind === 'ops' && failure.error.code === 'INVALID_OP'));}@@ -102,6 +101,10 @@ export function createDatabaseRowsSource(props: {applyOps: (ops: DatabaseOp[]) => ResultAsync<OpResult[], DatabaseOpsError>;/** Where the engine reads rows from; the app's GraphQL client by default. */read?: DatabaseSqlQueryCapabilities;+ /** The live view reader; production pages its global server order. */+ viewQuery?: (+ statement: Accessor<DatabaseSqlStatement | undefined>+ ) => DatabaseViewQuery;/** Calls back with the version of each change the gateway reports for this table. */onTableChanged: (listener: (version: number) => void) => void;/** Calls back with the version each batch this viewer commits, such as an added or renamed column, moves this table to. */@@ -191,7 +194,50 @@ export function createDatabaseRowsSource(props: {{ equals: sameDatabaseSqlStatement });const viewStatement = tableStatement(() => ({ view: props.view() }));- const rowsQuery = createDatabaseSqlQuery(viewStatement, props.read);+ // Board lanes are arranged from every row, so boards read through the engine.+ const engineView = () => props.view().layout.kind === 'board';+ const pagedQuery = (props.viewQuery ?? createDatabaseViewQuery)(() =>+ engineView() ? undefined : viewStatement()+ );+ const engineQuery = createDatabaseSqlQuery(+ () => (engineView() ? viewStatement() : undefined),+ props.read+ );+ const activeQuery = () => (engineView() ? engineQuery : pagedQuery);+ const currentRead = () => {+ const query = activeQuery();+ const outcome = query.outcome();+ const catalog = query.catalog();+ if (!outcome || !catalog) return undefined;+ const pagination = engineView() ? undefined : pagedQuery.pagination;+ return {+ outcome,+ catalog,+ hasMore: pagination?.hasMore() ?? false,+ version: pagination?.version(),+ };+ };+ // Switching a view between table and board switches readers; keep the last read meanwhile.+ const completed = createMemo<ReturnType<typeof currentRead>>((previous) =>+ viewStatement() ? (currentRead() ?? previous) : undefined+ );+ // Accepted writes may refresh a reader after its reactive owner is disposed.+ const shownRead = () => currentRead() ?? completed();+ const rowsQuery: DatabaseViewQuery = {+ outcome: () => shownRead()?.outcome,+ catalog: () => shownRead()?.catalog,+ loading: () => activeQuery().loading(),+ error: () => activeQuery().error(),+ cached: () => activeQuery().cached(),+ refresh: (reason) => activeQuery().refresh(reason),+ answerFromCache: () => activeQuery().answerFromCache(),+ pagination: pagedQuery.pagination && {+ hasMore: () => shownRead()?.hasMore ?? false,+ loading: () => pagedQuery.pagination!.loading(),+ version: () => shownRead()?.version,+ loadMore: () => pagedQuery.pagination!.loadMore(),+ },+ };// A type change gives a column a new definition. Until the read of it// lands, the column keeps the definition its shown cells were read with.const shownDetails = createMemo<ColumnDetail[]>((held) => {@@ -238,24 +284,40 @@ export function createDatabaseRowsSource(props: {// The newest version this writer's row writes made; it reads them back itself.let writtenVersion = 0;- function rowsOf(query: DatabaseSqlQuery): DatabaseRow[] | undefined {- const outcome = query.outcome();- const catalog = query.catalog();- if (!outcome || !catalog) return undefined;- return gridRows(outcome, catalog, shownDetails());+ function convertedRows(query: DatabaseSqlQuery) {+ let previous: GridRowsRead | undefined;+ return () => {+ const outcome = query.outcome();+ const catalog = query.catalog();+ if (!outcome || !catalog) return undefined;+ const columns = shownDetails();+ if (+ previous?.outcome === outcome &&+ previous.catalog === catalog &&+ previous.columns === columns+ )+ return previous.rows;+ previous = {+ outcome,+ catalog,+ columns,+ rows: gridRows(outcome, catalog, columns, previous),+ };+ return previous.rows;+ };}+ const viewRows = convertedRows(rowsQuery);+ const heldRows = convertedRows(retainedQuery);const retainedRows = () => {const ids = retainedRowIds();if (!ids.length) return [];- return (rowsOf(retainedQuery) ?? []).filter((row) =>- ids.includes(row.rowId)- );+ return (heldRows() ?? []).filter((row) => ids.includes(row.rowId));};const snapshot = () => {- const rows = rowsOf(rowsQuery);+ const rows = viewRows();if (!rows) return undefined;return {- version: readVersion(),+ version: rowsQuery.pagination?.version() ?? readVersion(),rows,retained: retainedRows(),};@@ -607,6 +669,7 @@ export function createDatabaseRowsSource(props: {}return {+ pagination: rowsQuery.pagination,columns,snapshot,read: () => {diff --git a/apps/web/src/features/block-database/queries/view-rows.test.ts b/apps/web/src/features/block-database/queries/view-rows.test.tsnew file mode 100644index 0000000000..70684bda8a--- /dev/null+++ b/apps/web/src/features/block-database/queries/view-rows.test.ts@@ -0,0 +1,243 @@+import type { Catalog, DatabaseView } from '@core/database-sql/generated/types';+import type {+ DatabaseViewPage,+ DatabaseViewPageFailure,+ DatabaseViewPageRequest,+} from '@service-storage/database-view-rows';+import { waitFor } from '@solidjs/testing-library';+import { errAsync, okAsync, ResultAsync } from 'neverthrow';+import { createRoot, createSignal } from 'solid-js';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { createDatabaseViewQuery } from './view-rows';++const view: DatabaseView = {+ id: 'view',+ databaseId: 'database',+ tableId: 'table',+ name: 'All records',+ position: '80',+ createdAt: '2026-01-01T00:00:00Z',+ updatedAt: '2026-01-01T00:00:00Z',+ query: { filter: null, sort: [] },+ layout: { kind: 'table', columns: [] },+};+const catalog: Catalog = {+ tables: [+ {+ id: 'table',+ databaseId: 'database',+ database: 'Database',+ name: 'Records',+ source: 'database',+ columns: [+ {+ id: 'name',+ placement: 'name-column',+ name: 'Name',+ kind: { kind: 'text' },+ },+ ],+ },+ ],+};++function page(+ ids: string[],+ nextCursor: string | null = null,+ version = 1+): DatabaseViewPage {+ return {+ nextCursor,+ version,+ items: ids.map((id) => ({+ __typename: 'GraphqlSoupDatabaseRow',+ id,+ tableId: 'table',+ position: id,+ ownerId: 'macro|[email protected]',+ createdAt: view.createdAt,+ updatedAt: view.updatedAt,+ cacheProjection: null,+ notifications: [],+ properties: [+ {+ id: `${id}-name`,+ propertyDefinitionId: 'name',+ value: { __typename: 'GraphqlStringPropertyValue', stringValue: id },+ },+ ],+ })),+ };+}++const disposals: (() => void)[] = [];+afterEach(() => disposals.splice(0).forEach((dispose) => dispose()));++function harness(+ read: (+ request: DatabaseViewPageRequest+ ) => ResultAsync<DatabaseViewPage, DatabaseViewPageFailure>+) {+ return createRoot((dispose) => {+ disposals.push(dispose);+ const [current, setCurrent] = createSignal(view);+ const readPage = vi.fn((request: DatabaseViewPageRequest) => read(request));+ const query = createDatabaseViewQuery(+ () => ({ schema: { databases: [], platform: [] }, view: current() }),+ { readPage, catalog: async () => catalog }+ );+ return { query, readPage, setCurrent, dispose };+ });+}++const uncached = () =>+ errAsync<DatabaseViewPage, DatabaseViewPageFailure>({+ kind: 'fetch',+ message: 'Not cached',+ });++describe('server view pages', () => {+ it('keeps loaded pages when only the view name or column layout changes', async () => {+ const { query, readPage, setCurrent } = harness((request) =>+ request.requestPolicy === 'cache-only'+ ? uncached()+ : okAsync(page(['a'], 'next'))+ );+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['a']));+ setCurrent({+ ...view,+ name: 'Renamed view',+ updatedAt: '2026-02-01T00:00:00Z',+ layout: {+ kind: 'table',+ columns: [{ column: 'name-column', width: 300 }],+ },+ });+ await Promise.resolve();+ expect(readPage).toHaveBeenCalledTimes(1);+ expect(query.pagination?.hasMore()).toBe(true);+ });+ it('opens with one page and fetches the continuation only on demand', async () => {+ const { query, readPage } = harness((request) =>+ request.requestPolicy === 'cache-only'+ ? uncached()+ : okAsync(+ request.cursor ? page(['c', 'd']) : page(['a', 'b'], 'after-b')+ )+ );+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['a', 'b']));+ expect(+ readPage.mock.calls.map(([request]) => [+ request.requestPolicy,+ request.cursor,+ ])+ ).toEqual([['cache-and-network', undefined]]);+ expect(query.pagination?.hasMore()).toBe(true);+ const firstCells = query.outcome()?.rows[0];+ await query.pagination?.loadMore();+ expect(query.outcome()?.rowIds).toEqual(['a', 'b', 'c', 'd']);+ expect(query.outcome()?.rows[0]).toBe(firstCells);+ expect(query.outcome()?.rows).toEqual(+ ['a', 'b', 'c', 'd'].map((value) => [{ type: 'text', value }])+ );+ expect(query.pagination?.hasMore()).toBe(false);+ await query.pagination?.loadMore();+ expect(readPage).toHaveBeenCalledTimes(2);+ });++ it('shows the cached first page while the current server page is pending', async () => {+ let resolve!: (page: DatabaseViewPage) => void;+ const pending = new Promise<DatabaseViewPage>((done) => {+ resolve = done;+ });+ const { query } = harness((request) => {+ request.onCachedPage?.(page(['cached'], 'cached-next'));+ return ResultAsync.fromSafePromise(pending);+ });+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['cached']));+ expect(query.loading()).toBe(true);+ resolve(page(['fresh']));+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['fresh']));+ expect(query.loading()).toBe(false);+ });++ it('keeps a delayed cache hit when the network fails first', async () => {+ let resolve!: (page: DatabaseViewPage) => void;+ const pending = new Promise<DatabaseViewPage>((done) => {+ resolve = done;+ });+ const { query, readPage } = harness((request) =>+ request.requestPolicy === 'cache-only'+ ? ResultAsync.fromSafePromise(pending)+ : errAsync({ kind: 'fetch', message: 'Offline' })+ );+ await waitFor(() => expect(readPage).toHaveBeenCalledTimes(2));+ resolve(page(['cached']));+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['cached']));+ expect(query.error()).toEqual({ kind: 'fetch', message: 'Offline' });+ expect(query.loading()).toBe(false);+ });++ it('restarts a stale continuation without appending rows from two versions', async () => {+ let calls = 0;+ const { query } = harness((request) => {+ if (request.requestPolicy === 'cache-only') return uncached();+ calls += 1;+ if (calls === 1) return okAsync(page(['old'], 'old-next'));+ if (calls === 2)+ return errAsync({ kind: 'fetch', message: 'Changed', stale: true });+ return okAsync(+ request.cursor+ ? page(['new-b'], null, 2)+ : page(['new-a'], 'new-next', 2)+ );+ });+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['old']));+ await query.pagination?.loadMore();+ expect(query.outcome()?.rowIds).toEqual(['new-a', 'new-b']);+ expect(query.pagination?.version()).toBe(2);+ expect(query.pagination?.hasMore()).toBe(false);+ expect(query.error()).toBeUndefined();+ });++ it('ignores an old view response that arrives after a newer filter', async () => {+ let resolve!: (page: DatabaseViewPage) => void;+ const pending = new Promise<DatabaseViewPage>((done) => {+ resolve = done;+ });+ const { query, readPage, setCurrent } = harness((request) =>+ request.requestPolicy === 'cache-only'+ ? uncached()+ : request.query.sort?.length+ ? okAsync(page(['sorted']))+ : ResultAsync.fromSafePromise(pending)+ );+ await waitFor(() => expect(readPage).toHaveBeenCalledTimes(1));+ setCurrent({+ ...view,+ query: {+ filter: null,+ sort: [{ column: 'name-column', direction: 'descending' }],+ },+ });+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['sorted']));+ resolve(page(['late']));+ await Promise.resolve();+ expect(query.outcome()?.rowIds).toEqual(['sorted']);+ });++ it('still completes an explicit read after its owner closes, for an accepted write', async () => {+ let value = 'before';+ const { query, dispose } = harness((request) =>+ request.requestPolicy === 'cache-only'+ ? uncached()+ : okAsync(page([value]))+ );+ await waitFor(() => expect(query.outcome()?.rowIds).toEqual(['before']));+ dispose();+ value = 'saved';+ const result = await query.refresh('after-write');+ expect(result.isOk() && result.value.landed).toBe(true);+ expect(query.outcome()?.rowIds).toEqual(['saved']);+ });+});diff --git a/apps/web/src/features/block-database/queries/view-rows.ts b/apps/web/src/features/block-database/queries/view-rows.tsnew file mode 100644index 0000000000..030519959f--- /dev/null+++ b/apps/web/src/features/block-database/queries/view-rows.ts@@ -0,0 +1,331 @@+import {+ type DatabaseSqlFailure,+ engineFailure,+} from '@core/database-sql/driver';+import type {+ Catalog,+ ColumnKind,+ Outcome,+ OutcomeKind,+} from '@core/database-sql/generated/types';+import {+ profileDatabasePhase,+ traceDatabaseFrame,+} from '@core/database-sql/profile';+import type { DatabaseSqlReadReason } from '@core/database-sql/trace';+import { buildDatabaseSqlCatalog } from '@core/database-sql/wasm-module';+import { Telemetry } from '@macro-inc/observability';+import type {+ DatabaseSqlQuery,+ DatabaseSqlRun,+ DatabaseSqlStatement,+} from '@queries/database-sql/create-database-sql-query';+import { rowCells } from '@queries/database-sql/graphql-source';+import {+ type DatabaseViewPage,+ type DatabaseViewPageRequest,+ readDatabaseViewPage,+} from '@service-storage/database-view-rows';+import { err, ok, okAsync, type Result, ResultAsync } from 'neverthrow';+import {+ type Accessor,+ batch,+ createEffect,+ createMemo,+ createSignal,+ on,+ onCleanup,+ untrack,+} from 'solid-js';+import { match } from 'ts-pattern';+import type { DatabaseRowsPagination } from '../../database/context/table-source';++export type DatabaseViewQuery = DatabaseSqlQuery & {+ pagination?: DatabaseRowsPagination;+};+type Capabilities = {+ readPage: typeof readDatabaseViewPage;+ catalog: typeof buildDatabaseSqlCatalog;+};++function outcomeKind(kind: ColumnKind): OutcomeKind {+ return (+ match(kind)+ .returnType<OutcomeKind>()+ .with({ kind: 'text' }, { kind: 'link' }, () => 'text')+ .with({ kind: 'number' }, () => 'number')+ .with({ kind: 'boolean' }, () => 'boolean')+ .with({ kind: 'date' }, () => 'date')+ .with({ kind: 'dateRange' }, () => 'dateRange')+ .with({ kind: 'select' }, () => 'select')+ .with({ kind: 'entity' }, () => 'entity')+ // An engine kind this build doesn't know still shows its cells as text.+ .exhaustive(() => 'text')+ );+}++/** View order and membership are authoritative server facts; cells retain their shared cache identity. */+export function createDatabaseViewQuery(+ statement: Accessor<DatabaseSqlStatement | undefined>,+ capabilities: Capabilities = {+ readPage: readDatabaseViewPage,+ catalog: buildDatabaseSqlCatalog,+ }+): DatabaseViewQuery {+ const readStatement = createMemo(statement, undefined, {+ equals: (left, right) => {+ const key = (value: DatabaseSqlStatement | undefined) =>+ value?.view &&+ JSON.stringify({+ schema: value.schema,+ scope: value.scope,+ table: value.view.tableId,+ query: value.view.query,+ });+ return key(left) === key(right);+ },+ });+ const [outcome, setOutcome] = createSignal<Outcome>();+ const [catalog, setCatalog] = createSignal<Catalog>();+ const [error, setError] = createSignal<DatabaseSqlFailure>();+ const [loading, setLoading] = createSignal(false);+ const [loadingMore, setLoadingMore] = createSignal(false);+ const [cursor, setCursor] = createSignal<string>();+ const [version, setVersion] = createSignal<number>();+ let pages: DatabaseViewPage[] = [];+ let generation = 0;+ let active: AbortController | undefined;+ let cancelFrame: (() => void) | undefined;++ function request(+ current: DatabaseSqlStatement,+ signal: AbortSignal,+ next?: string+ ): DatabaseViewPageRequest | undefined {+ return current.view+ ? {+ databaseId: current.view.databaseId,+ tableId: current.view.tableId,+ query: current.view.query,+ cursor: next,+ requestPolicy: 'network-only',+ signal,+ }+ : undefined;+ }++ function publish(+ built: Catalog,+ tableId: string,+ next: DatabaseViewPage[],+ span: ReturnType<typeof Telemetry.span>,+ previous?: Outcome+ ) {+ const table = built.tables.find((table) => table.id === tableId);+ if (!table) throw new Error('The view table is missing from its catalog.');+ const items = (previous ? next.slice(-1) : next).flatMap(+ (page) => page.items+ );+ const decoded = profileDatabasePhase(+ span.span('database_sql.rows.decode'),+ () =>+ items.map((item) => {+ const cells = rowCells(item);+ return table.columns.map((column) => cells[column.id] ?? null);+ })+ );+ const answer: Outcome = {+ columns:+ previous?.columns ??+ table.columns.map((column) => ({+ name: column.name,+ column: column.id,+ kind: outcomeKind(column.kind),+ })),+ rows: [...(previous?.rows ?? []), ...decoded],+ rowIds: [...(previous?.rowIds ?? []), ...items.map((item) => item.id)],+ readTables: [tableId],+ truncated: false,+ insertedRowIds: [],+ changesApplied: 0,+ };+ span.setAttr('database_sql.row_count', answer.rows.length);+ profileDatabasePhase(span.span('database_sql.publish'), () =>+ batch(() => {+ pages = next;+ setCatalog(built);+ setOutcome(answer);+ setCursor(next.at(-1)?.nextCursor ?? undefined);+ setVersion(next[0]?.version);+ setError(undefined);+ })+ );+ cancelFrame?.();+ cancelFrame = traceDatabaseFrame(span.span('database_sql.frame'));+ }++ function refresh(+ reason: DatabaseSqlReadReason = 'refresh',+ cached = false,+ count = Math.max(1, pages.length)+ ): ResultAsync<DatabaseSqlRun, DatabaseSqlFailure> {+ const current = untrack(readStatement);+ if (!current?.view) return okAsync({ landed: false });+ const run = ++generation;+ active?.abort();+ const controller = new AbortController();+ active = controller;+ const input = request(current, controller.signal);+ if (!input) return okAsync({ landed: false });+ setLoading(true);+ setLoadingMore(false);+ const span = Telemetry.span('database_sql.query');+ span.setAttr('database_sql.execution', 'server-view');+ span.setAttr('database_sql.read_reason', reason);+ span.setAttr('database_sql.table_id', input.tableId);+ const headers: Record<string, string> = {};+ span.injectTraceHeaders(headers);+ input.headers = headers;+ const read = async (): Promise<+ Result<DatabaseSqlRun, DatabaseSqlFailure>+ > => {+ const built = await capabilities.catalog(current.schema, current.scope);+ if (run !== generation) return ok({ landed: false });+ // A concurrent write may invalidate an in-progress multi-page refresh once.+ for (let attempt = 0; attempt < 2; attempt += 1) {+ const fresh: DatabaseViewPage[] = [];+ let next: string | undefined;+ for (let index = 0; index < count; index += 1) {+ const initial = cached && attempt === 0 && index === 0;+ const page = await capabilities.readPage({+ ...input,+ cursor: next,+ requestPolicy: initial ? 'cache-and-network' : 'network-only',+ onCachedPage: initial+ ? (page) => {+ if (run === generation)+ publish(built, input.tableId, [page], span);+ }+ : undefined,+ });+ if (run !== generation) return ok({ landed: false });+ if (page.isErr()) {+ if (page.error.stale && attempt === 0) break;+ // A fast offline failure may arrive before the worker's cache hit.+ if (initial) {+ const saved = await capabilities.readPage({+ ...input,+ requestPolicy: 'cache-only',+ });+ if (run !== generation) return ok({ landed: false });+ if (saved.isOk())+ publish(built, input.tableId, [saved.value], span);+ }+ return err(page.error);+ }+ fresh.push(page.value);+ next = page.value.nextCursor ?? undefined;+ if (!next || index === count - 1) {+ publish(built, input.tableId, fresh, span);+ return ok({ landed: true });+ }+ }+ }+ return err({+ kind: 'fetch',+ message: 'The table is changing. Refresh to read its latest rows.',+ });+ };+ return ResultAsync.fromPromise(read(), engineFailure)+ .andThen((result) => result)+ .orTee((failure) => {+ if (run === generation) setError(failure);+ })+ .andTee(() => {+ if (run === generation) setLoading(false);+ span.end();+ })+ .orTee(() => {+ if (run === generation) setLoading(false);+ span.end();+ });+ }++ function loadMore(): ResultAsync<DatabaseSqlRun, DatabaseSqlFailure> {+ const current = untrack(readStatement);+ const built = untrack(catalog);+ const next = untrack(cursor);+ if (!current || !built || !next || loading() || loadingMore() || !active)+ return okAsync({ landed: false });+ const run = generation;+ const input = request(current, active.signal, next);+ if (!input) return okAsync({ landed: false });+ setLoadingMore(true);+ const span = Telemetry.span('database.view.load_more');+ const headers: Record<string, string> = {};+ span.injectTraceHeaders(headers);+ return capabilities+ .readPage({ ...input, headers })+ .andThen((page): Result<DatabaseSqlRun, DatabaseSqlFailure> => {+ if (run !== generation) return ok({ landed: false });+ publish(built, input.tableId, [...pages, page], span, untrack(outcome));+ return ok({ landed: true });+ })+ .orElse((failure) => {+ if (run !== generation) return okAsync({ landed: false });+ if ('stale' in failure && failure.stale)+ return refresh('refresh', false, pages.length + 1);+ setError(failure);+ return new ResultAsync(+ Promise.resolve(err<DatabaseSqlRun, DatabaseSqlFailure>(failure))+ );+ })+ .andTee(() => {+ if (run === generation) setLoadingMore(false);+ span.end();+ })+ .orTee(() => {+ if (run === generation) setLoadingMore(false);+ span.end();+ });+ }++ createEffect(+ on(readStatement, (current, previous) => {+ pages = [];+ setCursor(undefined);+ setVersion(undefined);+ setError(undefined);+ if (!current) {+ generation += 1;+ active?.abort();+ setOutcome(undefined);+ setCatalog(undefined);+ setLoading(false);+ return;+ }+ void refresh(previous ? 'statement-change' : 'initial', true, 1);+ })+ );+ onCleanup(() => {+ generation += 1;+ active?.abort();+ cancelFrame?.();+ });+ return {+ outcome,+ catalog,+ error,+ loading,+ refresh,+ // A changed row may enter or leave the global page; only the server can decide membership.+ cached: () => false,+ answerFromCache: () => refresh('cache-reconcile'),+ pagination: {+ hasMore: () => !!cursor(),+ loading: loadingMore,+ version,+ loadMore,+ },+ };+}diff --git a/apps/web/src/features/block-pr/component/PrLinks.tsx b/apps/web/src/features/block-pr/component/PrLinks.tsxindex 85c9657f13..9b8d9a7954 100644--- a/apps/web/src/features/block-pr/component/PrLinks.tsx+++ b/apps/web/src/features/block-pr/component/PrLinks.tsx@@ -298,7 +298,7 @@ function ChannelOriginPills(props: {return (<Show when={props.origins.length > 0}><span- class="group/channels flex min-w-0 shrink items-center [&>*+*]:-ml-2 hover:[&>*+*]:ml-0.5 focus-within:[&>*+*]:ml-0.5 [&>*]:relative [&>*]:transition-[margin] [&>*:hover]:z-10"+ class="group/channels flex min-w-0 shrink items-center [&>:not(:first-child)]:-ml-2 hover:[&>:not(:first-child)]:ml-0.5 focus-within:[&>:not(:first-child)]:ml-0.5 [&>*]:relative [&>*]:transition-[margin] [&>*:hover]:z-10"aria-label={`Started from ${props.origins.length} channel ${props.origins.length === 1 ? 'message' : 'messages'}`}diff --git a/apps/web/src/features/database-query/core/query.test.ts b/apps/web/src/features/database-query/core/query.test.tsindex 1fc8a99816..3a7b43e328 100644--- a/apps/web/src/features/database-query/core/query.test.ts+++ b/apps/web/src/features/database-query/core/query.test.ts@@ -78,6 +78,11 @@ describe('database questions', () => {});describe('queryErrorMessage', () => {+ it('explains a cancelled read without presenting it as an engine failure', () => {+ expect(queryErrorMessage({ kind: 'cancelled' }, false)).toBe(+ 'This request was cancelled. Try asking again.'+ );+ });it('explains that questions only read when a write is refused', () => {expect(queryErrorMessage({ kind: 'read-only' }, false)).toBe('Questions only read your data. Ask a question about it above.'diff --git a/apps/web/src/features/database-query/core/query.ts b/apps/web/src/features/database-query/core/query.tsindex 58aa9d074d..ee5009d367 100644--- a/apps/web/src/features/database-query/core/query.ts+++ b/apps/web/src/features/database-query/core/query.ts@@ -238,6 +238,10 @@ export function queryErrorMessage(showSql ? message : engineErrorMessage(error)).with({ kind: 'crash' }, () => `${UNCOMPUTED}. Try asking again.`)+ .with(+ { kind: 'cancelled' },+ () => 'This request was cancelled. Try asking again.'+ ).with({ kind: 'fetch' }, () => OFFLINE).with({ kind: 'read-only' }, () =>showSqldiff --git a/apps/web/src/features/database/components/database-table-summary.tsx b/apps/web/src/features/database/components/database-table-summary.tsxindex 4eff76ee2e..26f5074e18 100644--- a/apps/web/src/features/database/components/database-table-summary.tsx+++ b/apps/web/src/features/database/components/database-table-summary.tsx@@ -28,13 +28,17 @@ const format = new Intl.NumberFormat(undefined, { maximumFractionDigits: 6 });export function DatabaseTableSummary(props: {columns: DatabaseViewColumn[];rows: DatabaseRow[];+ hasMoreRows?: boolean;template: string;addColumn: boolean;}) {return (<divrole="row"- aria-label="Table summary"+ data-grid-summary+ aria-label={+ props.hasMoreRows ? 'Loaded records summary' : 'Table summary'+ }class="sticky bottom-0 z-2 grid h-10 shrink-0 border-t border-edge-muted bg-panel text-xs"style={{ 'grid-template-columns': props.template }}>@@ -47,7 +51,10 @@ export function DatabaseTableSummary(props: {<span class="tabular-nums text-ink">{format.format(props.rows.length)}</span>- <span>{props.rows.length === 1 ? 'record' : 'records'} in view</span>+ <span>+ {props.rows.length === 1 ? 'record' : 'records'}{' '}+ {props.hasMoreRows ? 'loaded' : 'in view'}+ </span></div><Key each={props.columns.slice(1)} by="id">{(column) => <SummaryCell column={column()} rows={props.rows} />}diff --git a/apps/web/src/features/database/components/database-table.test.tsx b/apps/web/src/features/database/components/database-table.test.tsxindex 6eb870455b..a0c9658ede 100644--- a/apps/web/src/features/database/components/database-table.test.tsx+++ b/apps/web/src/features/database/components/database-table.test.tsx@@ -8,13 +8,14 @@ import {} from '@solidjs/testing-library';import userEvent from '@testing-library/user-event';import { okAsync } from 'neverthrow';-import { createSignal } from 'solid-js';+import { createMemo, createSignal } from 'solid-js';import { afterEach, beforeEach, describe, expect, it, vi } from 'vitest';import type { DatabaseViewColumn } from '../core/database-view';import type { DatabaseRow } from '../core/table';+import { GridResizeObserver } from '../tests/grid-resize-observer';import { DatabaseTableView as DatabaseTable } from '../views/database-table-view';import type { DatabaseTableControls } from './database-table';-import { GridCell } from './grid-cell';+import { GridCell, GridCellPreview } from './grid-cell';// JSDOM has no intersection observer to drive the date selector's focus.vi.mock('@property/utils/focus', () => ({@@ -53,13 +54,17 @@ const rows: DatabaseRow[] = [{ rowId: 'one', cells: { name: 'First', notes: 'First note' } },{ rowId: 'two', cells: { name: 'Second', notes: 'Second note' } },];-function setup(canEdit = true) {+function setup(+ canEdit = true,+ ready?: (controls: DatabaseTableControls) => void+) {const onAddColumnMount = vi.fn();function AddColumn() {onAddColumnMount();return <button>Add column</button>;}const onOpen = vi.fn();+ const onLoadMore = vi.fn();const onClearCells = vi.fn(async () => true);const onCellFocus = vi.fn();const onDuplicate = vi.fn(async () => true);@@ -75,12 +80,15 @@ function setup(canEdit = true) {const [columns, setColumns] = createSignal([name, readonly, notes]);const [records, setRecords] = createSignal(rows);const [unsavedRow, setUnsavedRow] = createSignal<string>();+ const [hasMoreRows, setHasMoreRows] = createSignal(false);let controls!: DatabaseTableControls;render(() => (<DatabaseTablename="Tasks"rows={records()}isUnsavedRow={(rowId) => rowId === unsavedRow()}+ hasMoreRows={hasMoreRows()}+ onLoadMore={onLoadMore}columns={columns()}titleColumnId="name"sort={[]}@@ -90,6 +98,7 @@ function setup(canEdit = true) {addColumn={<AddColumn />}controlsRef={(tableControls) => {controls = tableControls;+ ready?.(tableControls);}}getRowTitle={(row) => String(row.cells.name)}onOpen={onOpen}@@ -110,11 +119,19 @@ function setup(canEdit = true) {{...options}/>)}+ renderCellPreview={(row, column) => (+ <GridCellPreview+ column={column()}+ value={row().cells[column().id] ?? null}+ canEdit={canEdit}+ />+ )}/>));return {onAddColumnMount,onOpen,+ onLoadMore,onClearCells,onCellFocus,onDuplicate,@@ -126,6 +143,7 @@ function setup(canEdit = true) {controls,setRecords,setUnsavedRow,+ setHasMoreRows,};}async function selectMenu(name: string) {@@ -136,6 +154,12 @@ async function selectMenu(name: string) {let menuStyles: HTMLStyleElement;beforeEach(() => {+ vi.stubGlobal('ResizeObserver', GridResizeObserver);+ vi.spyOn(HTMLElement.prototype, 'offsetParent', 'get').mockImplementation(+ function (this: HTMLElement) {+ return this.parentElement;+ }+ );vi.spyOn(window, 'scrollTo').mockImplementation(() => {});Element.prototype.scrollIntoView = vi.fn();// JSDOM reports an empty animation name; presence expects CSS's default none.@@ -148,9 +172,155 @@ afterEach(() => {cleanup();menuStyles.remove();vi.restoreAllMocks();+ vi.unstubAllGlobals();});describe('spreadsheet interactions', () => {+ it('scopes summary totals to loaded records until the final page', () => {+ const { setHasMoreRows } = setup();+ setHasMoreRows(true);+ expect(+ screen.getByRole('row', { name: 'Loaded records summary' }).textContent+ ).toContain('2records loaded');+ setHasMoreRows(false);+ expect(+ screen.getByRole('row', { name: 'Table summary' }).textContent+ ).toContain('2records in view');+ });++ it('shows placeholder rows past the loaded records while pages remain', () => {+ const { setHasMoreRows } = setup();+ expect(document.querySelectorAll('[data-grid-pending-row]').length).toBe(0);+ setHasMoreRows(true);+ expect(+ document.querySelectorAll('[data-grid-pending-row]').length+ ).toBeGreaterThan(0);+ expect(document.querySelectorAll('[data-grid-row-id]').length).toBe(2);+ setHasMoreRows(false);+ expect(document.querySelectorAll('[data-grid-pending-row]').length).toBe(0);+ });++ it('requests the next page half a page before the last loaded record', async () => {+ const { setRecords, setHasMoreRows, onLoadMore } = setup();+ setRecords(+ Array.from({ length: 1000 }, (_, index) => ({+ rowId: `record-${index}`,+ cells: { name: `Record ${index}` },+ }))+ );+ setHasMoreRows(true);+ const grid = screen.getByRole('grid').parentElement!;+ grid.scrollTop = 700 * 43;+ fireEvent.scroll(grid);+ await new Promise(requestAnimationFrame);+ expect(onLoadMore).not.toHaveBeenCalled();+ // 760 of 1000: 240 records before the end.+ grid.scrollTop = 760 * 43;+ fireEvent.scroll(grid);+ await waitFor(() => expect(onLoadMore).toHaveBeenCalled());+ });++ it('shows previews for rows mounted mid-scroll until scrolling settles', async () => {+ const { setRecords } = setup();+ setRecords(+ Array.from({ length: 1000 }, (_, index) => ({+ rowId: `record-${index}`,+ cells: { name: `Record ${index}` },+ }))+ );+ const grid = screen.getByRole('grid').parentElement!;+ grid.scrollTop = 500 * 43;+ fireEvent.scroll(grid);+ await new Promise(requestAnimationFrame);+ const row = document.querySelector('[data-grid-row-id="record-502"]')!;+ expect(row.textContent).toContain('Record 502');+ expect(+ screen.queryByRole('button', { name: 'Name: Record 502. Click to edit' })+ ).toBeNull();+ await waitFor(+ () =>+ expect(+ screen.getByRole('button', {+ name: 'Name: Record 502. Click to edit',+ })+ ).toBeTruthy(),+ { timeout: 3000 }+ );+ });++ it('opens an edit requested synchronously when the table becomes ready', async () => {+ setup(true, (controls) => controls.editCell('one', 'name'));+ await waitFor(() => expect(screen.getByDisplayValue('First')).toBeTruthy());+ });++ it('mounts a bounded window and opens an editor outside that window', async () => {+ const { setRecords, controls } = setup();+ setRecords(+ Array.from({ length: 1000 }, (_, index) => ({+ rowId: `record-${index}`,+ cells: { name: `Record ${index}` },+ }))+ );+ expect(screen.getByRole('grid').getAttribute('aria-rowcount')).toBe('1002');+ expect(document.querySelectorAll('[data-grid-row-id]').length).toBeLessThan(+ 80+ );+ expect(+ screen.queryByRole('button', { name: 'Name: Record 999. Click to edit' })+ ).toBeNull();+ controls.editCell('record-999', 'name');+ await waitFor(() =>+ expect(screen.getByDisplayValue('Record 999')).toBeTruthy()+ );+ expect(document.querySelectorAll('[data-grid-row-id]').length).toBeLessThan(+ 80+ );+ });++ it('only reads changed rows while keeping column changes reactive', () => {+ const [records, setRecords] = createSignal(rows);+ const [columns, setColumns] = createSignal([name, notes]);+ const reads: string[] = [];+ render(() => (+ <DatabaseTable+ name="Tasks"+ rows={records()}+ columns={columns()}+ sort={[]}+ widths={{}}+ canEdit={false}+ pending={false}+ addColumn={null}+ getRowTitle={(row) => String(row.cells.name)}+ onOpen={() => {}}+ onSort={() => {}}+ renderCell={(row, column) => {+ const value = createMemo(() => {+ const text = `${row().rowId}:${column().name}:${row().cells[column().id]}`;+ reads.push(text);+ return text;+ });+ return <span>{value()}</span>;+ }}+ />+ ));+ reads.length = 0;+ setRecords([+ { rowId: 'one', cells: { name: 'Changed', notes: 'First note' } },+ rows[1],+ ]);+ expect(reads).toEqual(['one:Name:Changed', 'one:Notes:First note']);+ expect(screen.getByText('two:Name:Second')).toBeTruthy();++ reads.length = 0;+ setColumns([name, { ...notes, name: 'Details' }]);+ expect(reads.toSorted()).toEqual([+ 'one:Details:First note',+ 'two:Details:Second note',+ ]);+ expect(screen.getByText('one:Details:First note')).toBeTruthy();+ });+it('creates the add-column control once across grid layout and row updates', () => {const fixture = setup();expect(fixture.onAddColumnMount).toHaveBeenCalledOnce();diff --git a/apps/web/src/features/database/components/database-table.tsx b/apps/web/src/features/database/components/database-table.tsxindex 31eb4a6333..a2501a005b 100644--- a/apps/web/src/features/database/components/database-table.tsx+++ b/apps/web/src/features/database/components/database-table.tsx@@ -1,3 +1,4 @@+import { debounce } from '@core/util/debounce';import { fireAndForget } from '@core/util/fireAndForget';import ArrowSquareOutIcon from '@icon/system/regular/arrow-square-out.svg';import CopyIcon from '@icon/system/regular/copy.svg';@@ -5,6 +6,13 @@ import PencilIcon from '@icon/system/regular/pencil-simple.svg';import TrashIcon from '@icon/system/regular/trash.svg';import { ContextMenu } from '@kobalte/core/context-menu';import { Key } from '@solid-primitives/keyed';+import { createElementSize } from '@solid-primitives/resize-observer';+import {+ createVirtualizer,+ defaultRangeExtractor,+ measureElement,+ type Range,+} from '@tanstack/solid-virtual';import { DragDropProvider, DragOverlay } from '@thisbeyond/solid-dnd';import { getHashedPaletteColor } from '@ui/utils/palette';import {@@ -18,6 +26,7 @@ import {on,onCleanup,Show,+ untrack,} from 'solid-js';import { createHorizontalReorder } from '../../../components/drag-drop/create-horizontal-reorder';import { createReorderItem } from '../../../components/drag-drop/create-reorder';@@ -86,6 +95,27 @@ const EDITOR_FIELDS ='input:not([type="checkbox"]), textarea, [contenteditable="true"]';/** Portaled editors keep the cell current while they hold focus. */const PORTALED_EDITORS = '[role="dialog"], [role="listbox"]';+/** A row's height until one is measured. */+const ROW_HEIGHT = 47;+// Half a 500-record page: the next page is requested well before the last row.+const PREFETCH_ROW_COUNT = 250;+/**+ * Rows mounted beyond the viewport in the scroll direction, and behind it.+ * Generous behind too: reversing a fast scroll must find rows already there.+ */+const ROWS_AHEAD = 60;+const ROWS_BEHIND = 40;+/** Row lines extend this far past the mounted rows, moving in steps this size. */+const ROW_LINES_STEP = 100;+/**+ * Main-thread time per frame spent turning row previews into live rows. Large+ * enough to finish the rows in view in a frame or two: each frame of upgrades+ * repaints the viewport and delays the tiles prepainted around it.+ */+const UPGRADE_BUDGET_MS = 50;+/** Rows turn live only once scrolling has paused this long. */+const UPGRADE_DELAY_MS = 400;+const PENDING_ROW_COUNT = 100;export type DatabaseTableProps = {model: DatabaseTableModel;@@ -93,6 +123,8 @@ export type DatabaseTableProps = {titleColumnId?: string;isUnsavedRow?: (rowId: string) => boolean;onRowFocus?: (rowId: string | undefined) => void;+ onLoadMore?: () => void;+ hasMoreRows?: boolean;onCellFocus?: (cell: DatabaseCellFocus | undefined) => void;remoteUsers?: DatabaseCellPresence[];/** Scrolled into view and briefly tinted, e.g. the target of a relation. */@@ -108,6 +140,11 @@ export type DatabaseTableProps = {column: Accessor<DatabaseViewColumn>,options?: GridCellEditorOptions) => JSX.Element;+ /** What a cell shows while the grid scrolls, before `renderCell` mounts. */+ renderCellPreview?: (+ row: Accessor<DatabaseRow>,+ column: Accessor<DatabaseViewColumn>+ ) => JSX.Element;getRowTitle: (row: DatabaseRow) => string;onOpen: (rowId: string) => void;onDuplicate?: (rowId: string) => Promise<boolean>;@@ -140,6 +177,7 @@ export type DatabaseTableProps = {export function DatabaseTable(props: DatabaseTableProps) {const columns = () => props.model.visibleColumns();+ const rows = () => props.model.rows();const addColumn = children(() =>props.canEdit ? props.addColumn : undefined);@@ -147,14 +185,60 @@ export function DatabaseTable(props: DatabaseTableProps) {addColumn.toArray().some((child) => child != null && typeof child !== 'boolean');- const rows = () => props.model.table.getRowModel().rows;const summaryRows = createMemo(() =>- rows()- .filter((row) => !props.isUnsavedRow?.(row.id))- .map((row) => row.original)+ rows().filter((row) => !props.isUnsavedRow?.(row.rowId)));let scrollContainer!: HTMLDivElement;let gridElement!: HTMLDivElement;+ const [header, setHeader] = createSignal<HTMLDivElement>();+ const headerSize = createElementSize(header);+ const [focusedRow, setFocusedRow] = createSignal<string>();+ let pendingFocus: { rowId: string; column: number } | undefined;+ const cellElement = (rowId: string, column: number) => {+ const index = selection.rowIndex().get(rowId);+ return gridElement?.querySelector<HTMLElement>(+ `[data-grid-row="${index}"][data-grid-column="${column}"]`+ );+ };+ const revealRow = (rowId: string) => {+ setFocusedRow(rowId);+ const index = selection.rowIndex().get(rowId);+ const item =+ index === undefined ? undefined : virtualizer.measurementsCache[index];+ // Before the grid mounts there is nothing to scroll.+ if (!item || !gridElement) return;+ // Rows start below the sticky header, which also covers the top of the viewport.+ const headerHeight = headerSize.height ?? 40;+ const summaryHeight =+ gridElement.querySelector<HTMLElement>('[data-grid-summary]')+ ?.offsetHeight ?? 0;+ if (item.start < scrollContainer.scrollTop + headerHeight)+ virtualizer.scrollToOffset(item.start - headerHeight);+ else if (+ item.end >+ scrollContainer.scrollTop + scrollContainer.clientHeight - summaryHeight+ )+ virtualizer.scrollToOffset(+ item.end + summaryHeight - scrollContainer.clientHeight+ );+ };+ const focusRequestedCell = () => {+ if (!pendingFocus) return;+ const { rowId, column } = pendingFocus;+ previewRows.get(rowId)?.upgrade();+ const cell = cellElement(rowId, column);+ if (!cell) return;+ pendingFocus = undefined;+ const columnId = columns()[column - 1]?.id;+ const editor = columnId ? control(rowId, columnId) : undefined;+ if (editor) {+ editor.focus();+ return;+ }+ (+ cell.querySelector<HTMLElement>('button, input, [tabindex]') ?? cell+ ).focus();+ };const columnReorder = createHorizontalReorder({order: () => columns().map((column) => column.id),getViewport: () => scrollContainer,@@ -188,22 +272,17 @@ export function DatabaseTable(props: DatabaseTableProps) {controls.get(rowId)?.get(columnId);const editRequestedCell = () => {if (!pendingEdit || !props.canEdit) return;+ previewRows.get(pendingEdit.rowId)?.upgrade();const editor = control(pendingEdit.rowId, pendingEdit.columnId);if (!editor) return;+ const column =+ columns().findIndex(({ id }) => id === pendingEdit?.columnId) + 1;+ const cell = cellElement(pendingEdit.rowId, column);+ if (!cell) return;const { selectAll } = pendingEdit;pendingEdit = undefined;editor.edit(undefined, { selectAll });};- props.controlsRef?.({- editCell: (rowId, columnId) => {- pendingEdit = { rowId, columnId };- editRequestedCell();- },- renameColumn: (columnId) => {- requestedHeader = columnId;- renameRequestedHeader();- },- });const register = (rowId: string,columnId: string,@@ -222,11 +301,12 @@ export function DatabaseTable(props: DatabaseTableProps) {}row.set(columnId, editor);editRequestedCell();+ queueMicrotask(focusRequestedCell);};const navigate = (rowId: string, columnId: string, direction: 1 | -1) => {if (!props.canEdit) return false;const editableColumns = columns().filter(canEditCell);- const rowIndex = rows().findIndex((row) => row.id === rowId);+ const rowIndex = rows().findIndex((row) => row.rowId === rowId);const columnIndex = editableColumns.findIndex((column) => column.id === columnId);@@ -238,17 +318,19 @@ export function DatabaseTable(props: DatabaseTableProps) {return false;const row = rows()[Math.floor(nextIndex / editableColumns.length)];const column = editableColumns[nextIndex % editableColumns.length];- pendingEdit = { rowId: row.id, columnId: column.id };+ pendingEdit = { rowId: row.rowId, columnId: column.id };+ revealRow(row.rowId);editRequestedCell();return true;};const navigateRow = (rowId: string, columnId: string, direction: 1 | -1) => {if (!props.canEdit) return false;const column = columns().find((candidate) => candidate.id === columnId);- const rowIndex = rows().findIndex((row) => row.id === rowId);+ const rowIndex = rows().findIndex((row) => row.rowId === rowId);const row = rows()[rowIndex + direction];if (rowIndex < 0 || !row || !column || !canEditCell(column)) return false;- pendingEdit = { rowId: row.id, columnId, selectAll: false };+ pendingEdit = { rowId: row.rowId, columnId, selectAll: false };+ revealRow(row.rowId);editRequestedCell();return true;};@@ -281,7 +363,7 @@ export function DatabaseTable(props: DatabaseTableProps) {};};const selection = createCellSelection({- rows: () => rows().map((row) => row.id),+ rows: () => rows().map((row) => row.rowId),columns: () => columns().map((column) => column.id),cellAt: (target) => {if (!(target instanceof Element) || !gridElement?.contains(target))@@ -308,6 +390,212 @@ export function DatabaseTable(props: DatabaseTableProps) {? (rows, columns) => props.onClearCells!(rows, columns): undefined,});+ const rowIds = createMemo(() => [...selection.rowIndex().keys()]);+ // One menu for every row: right-click (or a long press) picks its row and cell.+ const [contextTarget, setContextTarget] = createSignal<{+ rowId: string;+ columnId?: string;+ }>();+ const contextRowId = () => contextTarget()?.rowId ?? '';+ const keptRows = createMemo(() =>+ [+ ...new Set([focusedRow(), contextTarget()?.rowId, props.highlightRowId]),+ ].flatMap((rowId) => {+ const index =+ rowId === undefined ? undefined : selection.rowIndex().get(rowId);+ return index === undefined ? [] : [index];+ })+ );+ // Placeholder rows past the loaded ones, so a fast scroll lands on rows that+ // fill in rather than on empty space while the next page loads.+ const pendingRows = () => (props.hasMoreRows ? PENDING_ROW_COUNT : 0);+ const [rowHeight, setRowHeight] = createSignal(ROW_HEIGHT);+ // Rows mounted mid-scroll, or out of view, render cell previews. Once+ // scrolling settles, the ones in view turn live, nearest the middle first and+ // within a per-frame budget, so a pause between flicks doesn't stall the next+ // one. Rows out of view stay previews: they look the same, and repainting+ // them would discard the tiles the compositor shows on the next flick.+ const [scrolling, setScrolling] = createSignal(false);+ const previewRows = new Map<+ string,+ { index: () => number; upgrade: () => void }+ >();+ let upgradeFrame: number | undefined;+ const inView = (index: number) => {+ const range = virtualizer.range;+ return !range || (index >= range.startIndex && index <= range.endIndex);+ };+ const upgradePreviews = () => {+ upgradeFrame = undefined;+ if (scrolling()) return;+ const range = virtualizer.range;+ const middle = range ? (range.startIndex + range.endIndex) / 2 : 0;+ const visible = [...previewRows.values()]+ .filter((row) => inView(row.index()))+ .sort(+ (a, b) => Math.abs(a.index() - middle) - Math.abs(b.index() - middle)+ );+ const deadline = performance.now() + UPGRADE_BUDGET_MS;+ for (const row of visible) {+ if (performance.now() > deadline) {+ upgradeFrame = requestAnimationFrame(upgradePreviews);+ return;+ }+ row.upgrade();+ }+ };+ // A repainted row discards its tiles, so upgrading as a flick starts would+ // leave the compositor nothing to show for a frame.+ const scheduleUpgrades = debounce(() => {+ if (!scrolling() && upgradeFrame === undefined)+ upgradeFrame = requestAnimationFrame(upgradePreviews);+ }, UPGRADE_DELAY_MS);+ const changeScrolling = (isScrolling: boolean) => {+ setScrolling(isScrolling);+ if (isScrolling) scheduleUpgrades.clear();+ else scheduleUpgrades();+ };+ /** Rows in view while idle, and rows a focus or edit is waiting for. */+ const mountsLive = (rowId: string, index: number) =>+ (!scrolling() && inView(index)) ||+ pendingFocus?.rowId === rowId ||+ pendingEdit?.rowId === rowId;+ onCleanup(() => {+ if (upgradeFrame !== undefined) cancelAnimationFrame(upgradeFrame);+ });+ let rowHeightMeasured = false;+ const virtualizer = createVirtualizer<HTMLDivElement, HTMLDivElement>({+ get count() {+ return rowIds().length + pendingRows();+ },+ get getItemKey() {+ const ids = rowIds();+ return (index: number) => ids[index] ?? index;+ },+ getScrollElement: () => scrollContainer,+ get scrollMargin() {+ return headerSize.height ?? 40;+ },+ // Unmeasured rows take the first measured row's height, so rows mount+ // where they stay instead of shifting every row below them as they load.+ get estimateSize() {+ const height = rowHeight();+ return () => height;+ },+ // Only the ResizeObserver measures: a synchronous read per new row forces+ // a layout on every scroll frame. A grid without layout (a hidden pane)+ // measures 0; keep the estimate so every row doesn't mount at once.+ measureElement: (element, entry, instance) => {+ const size = entry+ ? measureElement(element, entry, instance)+ : instance.itemSizeCache.get(+ instance.options.getItemKey(instance.indexFromElement(element))+ );+ if (entry && size && !rowHeightMeasured) {+ rowHeightMeasured = true;+ // Rows mounted before this were cached at the old estimate; correcting+ // them one by one would move every row below each, many times over.+ if (size !== rowHeight()) {+ setRowHeight(size);+ queueMicrotask(() => instance.measure());+ }+ }+ return size || rowHeight();+ },+ // The compositor scrolls while the main thread mounts rows, so mount well+ // ahead in the scroll direction and a good way behind.+ overscan: 0,+ onChange: (instance) => changeScrolling(instance.isScrolling),+ get rangeExtractor() {+ const kept = keptRows();+ return (range: Range) => {+ const backward = virtualizer.scrollDirection === 'backward';+ const window = defaultRangeExtractor({+ ...range,+ startIndex: Math.max(+ 0,+ range.startIndex - (backward ? ROWS_AHEAD : ROWS_BEHIND)+ ),+ endIndex: Math.min(+ range.count - 1,+ range.endIndex + (backward ? ROWS_BEHIND : ROWS_AHEAD)+ ),+ });+ return [...new Set([...window, ...kept])].sort((a, b) => a - b);+ };+ },+ });+ // The adapter reconciles items by index; copies let rows key by record.+ const virtualItems = () =>+ virtualizer.getVirtualItems().map((item) => ({ ...item }));+ // Empty row lines around the mounted rows, anchored on one of them, so a+ // frame that outruns mounting shows grid rows instead of bare panel. Steps+ // keep it from repainting every frame; one as tall as the table made+ // Firefox drop a quarter of its scroll frames.+ const rowLines = createMemo(+ () => {+ const items = virtualizer.getVirtualItems();+ const first = items[0];+ const last = items.at(-1);+ if (!first || !last) return undefined;+ const height = rowHeight();+ const start = Math.max(+ 0,+ Math.floor(first.index / ROW_LINES_STEP - 1) * ROW_LINES_STEP+ );+ const end = Math.min(+ virtualizer.options.count,+ Math.ceil(last.index / ROW_LINES_STEP + 1) * ROW_LINES_STEP+ );+ return {+ top:+ first.start -+ virtualizer.options.scrollMargin -+ (first.index - start) * height,+ height: (end - start) * height,+ rowHeight: height,+ };+ },+ undefined,+ {+ equals: (a, b) =>+ a?.top === b?.top &&+ a?.height === b?.height &&+ a?.rowHeight === b?.rowHeight,+ }+ );+ createEffect(() => {+ // Reruns as the mounted rows change; the range is the rows in view.+ const lastMounted = virtualizer.getVirtualItems().at(-1);+ const lastInView = virtualizer.range?.endIndex;+ if (+ props.hasMoreRows &&+ lastMounted &&+ lastInView !== undefined &&+ lastInView >= rowIds().length - PREFETCH_ROW_COUNT &&+ !props.isUnsavedRow?.(focusedRow() ?? '')+ )+ untrack(() => props.onLoadMore?.());+ });+ props.controlsRef?.({+ editCell: (rowId, columnId) => {+ pendingEdit = { rowId, columnId };+ revealRow(rowId);+ editRequestedCell();+ },+ renameColumn: (columnId) => {+ requestedHeader = columnId;+ renameRequestedHeader();+ },+ });+ createEffect(+ on(+ () => props.highlightRowId,+ (rowId) => {+ if (rowId) revealRow(rowId);+ }+ )+ );const remoteRanges = createMemo(() =>(props.remoteUsers ?? []).flatMap((user) => {if (!user.rowId || !user.columnId) return [];@@ -380,7 +668,7 @@ export function DatabaseTable(props: DatabaseTableProps) {if (!row || !column || !canEditCell(column)) return;event.preventDefault();event.stopPropagation();- fireAndForget(props.onClearCells([row.id], [column.id]), 'database');+ fireAndForget(props.onClearCells([row.rowId], [column.id]), 'database');return;}if (@@ -415,32 +703,33 @@ export function DatabaseTable(props: DatabaseTableProps) {);event.preventDefault();event.stopPropagation();- if (!next) return;const row = rows()[rowIndex];const column = columns()[columnIndex - 1];- const nextControl = row && column ? control(row.id, column.id) : undefined;- // Stepping down onto the new-record row starts typing there, like a spreadsheet.- if (- nextControl &&+ if (!row || columnIndex < 0 || columnIndex > columns().length) return;+ const editDraft =delta[0] === 1 &&props.canEdit &&column &&canEditCell(column) &&- props.isUnsavedRow?.(row.id)- )- nextControl.edit();+ props.isUnsavedRow?.(row.rowId);+ if (!next) {+ if (editDraft) pendingEdit = { rowId: row.rowId, columnId: column.id };+ else pendingFocus = { rowId: row.rowId, column: columnIndex };+ revealRow(row.rowId);+ editRequestedCell();+ queueMicrotask(focusRequestedCell);+ return;+ }+ const nextControl =+ row && column ? control(row.rowId, column.id) : undefined;+ // Stepping down onto the new-record row starts typing there, like a spreadsheet.+ if (nextControl && editDraft) nextControl.edit();else if (nextControl) nextControl.focus();else(next.querySelector<HTMLElement>('button, input, [tabindex]') ?? next).focus();}- // One menu for every row: right-click (or a long press) picks its row and cell.- const [contextTarget, setContextTarget] = createSignal<{- rowId: string;- columnId?: string;- }>();- const contextRowId = () => contextTarget()?.rowId ?? '';const rowButtons = new Map<string, HTMLButtonElement>();let afterClose: (() => void) | undefined;const deferAction = (action: () => void) => {@@ -511,9 +800,10 @@ export function DatabaseTable(props: DatabaseTableProps) {tabIndex={-1}aria-multiselectable="true"aria-label={props.name}- aria-rowcount={rows().length + 2}+ aria-rowcount={props.hasMoreRows ? -1 : rows().length + 2}aria-colcount={columns().length + 1 + Number(hasAddColumn())}data-grid+ data-grid-saved-rows={summaryRows().length}class="relative flex min-h-full min-w-fit flex-col"onKeyDown={moveFocus}onFocusIn={(event) => {@@ -522,7 +812,10 @@ export function DatabaseTable(props: DatabaseTableProps) {? event.target.closest<HTMLElement>('[data-grid-row-id]')?.dataset.gridRowId: undefined;- if (rowId) props.onRowFocus?.(rowId);+ if (rowId) {+ setFocusedRow(rowId);+ props.onRowFocus?.(rowId);+ }if (!selection.range()) announceCell(cellAt(event.target));}}onFocusOut={(event) => {@@ -547,6 +840,7 @@ export function DatabaseTable(props: DatabaseTableProps) {}}><div+ ref={setHeader}role="row"aria-rowindex={1}class="sticky top-0 z-1 grid min-h-10 border-b border-edge-muted bg-panel"@@ -568,10 +862,11 @@ export function DatabaseTable(props: DatabaseTableProps) {const row = rows()[0];if (!row || !canEditCell(column())) return;pendingEdit = {- rowId: row.id,+ rowId: row.rowId,columnId: column().id,selectAll: false,};+ revealRow(row.rowId);editRequestedCell();}}registerRename={(rename) => {@@ -642,165 +937,281 @@ export function DatabaseTable(props: DatabaseTableProps) {captureContext(event.target);}}>- {/* Preserve editor identity when TanStack refreshes its row models. */}- <Key each={rows()} by="id">- {(tableRow, index) => {- const row = () => tableRow().original;- const highlighted = () =>- props.highlightRowId === row().rowId;- return (+ {/* Keep unchanged records and their editors mounted across reads. */}+ {/* A fast scroll can outrun mounting rows; the compositor then+ shows the column lines and the row lines below. */}+ <div+ class="relative grid shrink-0 bg-panel"+ style={{+ height: `${virtualizer.getTotalSize()}px`,+ 'grid-template-columns': template(),+ }}+ >+ <div+ aria-hidden="true"+ class="border-r border-edge-muted/40"+ />+ <Show when={rowLines()}>+ {(lines) => (<div- ref={(element: HTMLElement) => {- createEffect(- on(highlighted, (isHighlighted) => {- if (!isHighlighted) return;- element.scrollIntoView({ block: 'nearest' });- const first = columns()[0];- if (first)- control(row().rowId, first.id)?.focus();- })- );+ aria-hidden="true"+ class="pointer-events-none absolute inset-x-0 top-0"+ style={{+ transform: `translateY(${Math.max(0, lines().top)}px)`,+ height: `${lines().height}px`,+ 'background-image':+ 'linear-gradient(to bottom, transparent calc(100% - 1px), color-mix(in srgb, var(--color-edge-muted) 60%, transparent) 0)',+ 'background-size': `100% ${lines().rowHeight}px`,}}- role="row"- data-grid-row-id={row().rowId}- data-highlighted={highlighted() ? '' : undefined}- aria-rowindex={index() + 2}- // Off-screen rows skip style, layout and paint; they stay- // in the DOM, the accessibility tree and find-in-page.- class="group grid min-h-10 border-b border-edge-muted/60 [contain-intrinsic-size:auto_41px] [content-visibility:auto] hover:bg-hover/50"- classList={{ 'bg-accent/15': highlighted() }}- style={{ 'grid-template-columns': template() }}- >+ />+ )}+ </Show>+ <For each={columns()}>+ {() => (+ <div+ aria-hidden="true"+ class="border-r border-edge-muted/40"+ />+ )}+ </For>+ <Key each={virtualItems()} by={(item) => item.key}>+ {(item) => {+ const rowId = item().key;+ // Items are fresh copies on every scroll update; memos keep+ // a row's bindings from rerunning unless its place changed.+ const index = createMemo(() => item().index);+ const offset = createMemo(+ () => item().start - virtualizer.options.scrollMargin+ );+ const position = () => ({+ transform: `translateY(${offset()}px)`,+ });+ if (typeof rowId !== 'string')+ return (+ <div+ aria-hidden="true"+ data-grid-pending-row+ class="absolute inset-x-0 top-0 grid border-b border-edge-muted/60"+ style={{+ ...position(),+ height: `${rowHeight()}px`,+ 'grid-template-columns': template(),+ }}+ >+ <div class="sticky left-0 z-1 border-r border-edge-muted/40 bg-panel" />+ <For each={columns()}>+ {() => (+ <div class="flex items-center border-r border-edge-muted/40 px-2">+ <span class="h-2 w-1/2 rounded-full bg-hover" />+ </div>+ )}+ </For>+ </div>+ );+ const initial = rows()[index()];+ const row = createMemo(+ () =>+ rows()[selection.rowIndex().get(rowId)!] ?? initial+ );+ const highlighted = () =>+ props.highlightRowId === row().rowId;+ const [live, setLive] = createSignal(+ !props.renderCellPreview || mountsLive(rowId, index())+ );+ if (!live()) {+ previewRows.set(rowId, {+ index,+ upgrade: () => {+ previewRows.delete(rowId);+ setLive(true);+ },+ });+ onCleanup(() => previewRows.delete(rowId));+ scheduleUpgrades();+ }+ return (<div- role="gridcell"- aria-colindex={1}- tabindex={-1}- // Opaque so cells scrolled beneath it stay hidden; the- // overlay repeats the row's hover and highlight tint.- class="sticky left-0 z-1 flex items-center justify-center border-r border-edge-muted/40 bg-panel outline-none before:pointer-events-none before:absolute before:inset-0 group-hover:before:bg-hover/50 group-data-highlighted:before:bg-accent/15 focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ink/50"- data-grid-cell- data-grid-row={index()}- data-grid-column={0}+ ref={(element: HTMLDivElement) => {+ // Observed once `data-index` is set; sizes are cached by record.+ queueMicrotask(() =>+ virtualizer.measureElement(element)+ );+ createEffect(+ on(highlighted, (isHighlighted) => {+ if (!isHighlighted) return;+ element.scrollIntoView({ block: 'nearest' });+ const first = columns()[0];+ if (first)+ control(row().rowId, first.id)?.focus();+ })+ );+ }}+ role="row"+ data-index={index()}+ data-grid-row-id={row().rowId}+ data-highlighted={highlighted() ? '' : undefined}+ aria-rowindex={index() + 2}+ class="group absolute inset-x-0 top-0 grid min-h-10 border-b border-edge-muted/60 hover:bg-hover/50"+ classList={{ 'bg-accent/15': highlighted() }}+ style={{+ ...position(),+ // A preview keeps the live row's height, so rows+ // below don't move when it turns live.+ height: live() ? undefined : `${rowHeight()}px`,+ 'grid-template-columns': template(),+ }}+ // A touch may start a swipe; turning the row live then+ // would repaint it as the scroll begins. Taps click.+ onPointerDown={(event) => {+ if (event.pointerType !== 'touch')+ previewRows.get(rowId)?.upgrade();+ }}+ onClick={() => previewRows.get(rowId)?.upgrade()}>- <Show- when={!props.isUnsavedRow?.(row().rowId)}- fallback={- <span class="text-[10px] tabular-nums text-ink-placeholder">- {index() + 1}- </span>- }+ <div+ role="gridcell"+ aria-colindex={1}+ tabindex={-1}+ // Opaque so cells scrolled beneath it stay hidden; the+ // overlay repeats the row's hover and highlight tint.+ class="sticky left-0 z-1 flex items-center justify-center border-r border-edge-muted/40 bg-panel outline-none before:pointer-events-none before:absolute before:inset-0 group-hover:before:bg-hover/50 group-data-highlighted:before:bg-accent/15 focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ink/50"+ data-grid-cell+ data-grid-row={index()}+ data-grid-column={0}>- <button- ref={(button) => {- rowButtons.set(row().rowId, button);- onCleanup(() => {- if (rowButtons.get(row().rowId) === button)- rowButtons.delete(row().rowId);- });- }}- type="button"- class="relative grid size-7 place-items-center rounded text-[10px] tabular-nums text-ink-placeholder outline-none hover:bg-hover focus-visible:ring-2 focus-visible:ring-ink/50"- aria-label={`Open ${props.getRowTitle(row())}`}- title="Open record"- onClick={() => props.onOpen(row().rowId)}+ <Show+ when={!props.isUnsavedRow?.(row().rowId)}+ fallback={+ <span class="text-[10px] tabular-nums text-ink-placeholder">+ {index() + 1}+ </span>+ }>- <span class="group-hover:opacity-0 group-focus-within:opacity-0">- {index() + 1}- </span>- <ArrowSquareOutIcon class="absolute size-3.5 opacity-0 group-hover:opacity-100 group-focus-within:opacity-100" />- </button>- </Show>- </div>- <Key each={tableRow().getVisibleCells()} by="id">- {(cell, columnIndex) => {- const column = () => cell().column.columnDef.meta!;- const presence = () =>- presenceAt(row().rowId, column().id);- return (- <div- role="gridcell"- aria-selected={selection.contains(- index(),- columnIndex()- )}- aria-colindex={columnIndex() + 2}- tabindex={-1}- class="relative grid min-w-0 border-r border-edge-muted/40 px-0.5 py-0.5 outline-none focus-within:ring-1 focus-within:ring-inset focus-within:ring-ink/30 focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ink/30"- classList={{- 'bg-accent/15 ring-1 ring-inset ring-accent/50':- selection.contains(index(), columnIndex()),+ <button+ ref={(button) => {+ rowButtons.set(rowId, button);+ queueMicrotask(focusRequestedCell);+ onCleanup(() => {+ if (rowButtons.get(rowId) === button)+ rowButtons.delete(rowId);+ });}}- data-grid-cell- data-grid-row={index()}- data-grid-column={columnIndex() + 1}- data-remote-users={- presence().length- ? presence()- .map(({ user }) => user.userId)- .join(' ')- : undefined- }- style={presenceOutline(- presence()[0]?.user,- presence()[0]- ? {- top:- index() === presence()[0].bounds.top,- bottom:- index() ===- presence()[0].bounds.bottom,- left:- columnIndex() ===- presence()[0].bounds.left,- right:- columnIndex() ===- presence()[0].bounds.right,- }- : undefined- )}+ type="button"+ class="relative grid size-7 place-items-center rounded text-[10px] tabular-nums text-ink-placeholder outline-none hover:bg-hover focus-visible:ring-2 focus-visible:ring-ink/50"+ aria-label={`Open ${props.getRowTitle(row())}`}+ title="Open record"+ onClick={() => props.onOpen(row().rowId)}>- {props.renderCell(row, column, {- onReady: (editor) =>- register(row().rowId, column().id, editor),- onNavigate: (direction) =>- navigate(- row().rowId,- column().id,- direction- ),- onNavigateRow: (direction) =>- navigateRow(- row().rowId,- column().id,- direction- ),- })}- <Show when={presence().length}>- <span class="pointer-events-none absolute top-0 left-0 z-1 flex gap-px">- <For- each={presence().filter(- ({ bounds }) =>- bounds.top === index() &&- bounds.left === columnIndex()- )}- >- {({ user }) => (- <PresenceTag user={user} below />- )}- </For>- </span>- </Show>- </div>- );- }}- </Key>- <Show when={hasAddColumn()}>- <div role="gridcell" />- </Show>- </div>- );- }}- </Key>+ <span class="group-hover:opacity-0 group-focus-within:opacity-0">+ {index() + 1}+ </span>+ <ArrowSquareOutIcon class="absolute size-3.5 opacity-0 group-hover:opacity-100 group-focus-within:opacity-100" />+ </button>+ </Show>+ </div>+ <Key each={columns()} by="id">+ {(column, columnIndex) => {+ const columnId = column().id;+ const presence = () =>+ presenceAt(row().rowId, column().id);+ return (+ <div+ role="gridcell"+ aria-selected={selection.contains(+ index(),+ columnIndex()+ )}+ aria-colindex={columnIndex() + 2}+ tabindex={-1}+ class="relative grid min-w-0 border-r border-edge-muted/40 px-0.5 py-0.5 outline-none focus-within:ring-1 focus-within:ring-inset focus-within:ring-ink/30 focus-visible:ring-1 focus-visible:ring-inset focus-visible:ring-ink/30"+ classList={{+ 'bg-accent/15 ring-1 ring-inset ring-accent/50':+ selection.contains(+ index(),+ columnIndex()+ ),+ }}+ data-grid-cell+ data-grid-row={index()}+ data-grid-column={columnIndex() + 1}+ data-remote-users={+ presence().length+ ? presence()+ .map(({ user }) => user.userId)+ .join(' ')+ : undefined+ }+ style={presenceOutline(+ presence()[0]?.user,+ presence()[0]+ ? {+ top:+ index() ===+ presence()[0].bounds.top,+ bottom:+ index() ===+ presence()[0].bounds.bottom,+ left:+ columnIndex() ===+ presence()[0].bounds.left,+ right:+ columnIndex() ===+ presence()[0].bounds.right,+ }+ : undefined+ )}+ >+ <Show+ when={live()}+ fallback={props.renderCellPreview?.(+ row,+ column+ )}+ >+ {props.renderCell(row, column, {+ onReady: (editor) =>+ register(rowId, columnId, editor),+ onNavigate: (direction) =>+ navigate(+ row().rowId,+ column().id,+ direction+ ),+ onNavigateRow: (direction) =>+ navigateRow(+ row().rowId,+ column().id,+ direction+ ),+ })}+ </Show>+ <Show when={presence().length}>+ <span class="pointer-events-none absolute top-0 left-0 z-1 flex gap-px">+ <For+ each={presence().filter(+ ({ bounds }) =>+ bounds.top === index() &&+ bounds.left === columnIndex()+ )}+ >+ {({ user }) => (+ <PresenceTag user={user} below />+ )}+ </For>+ </span>+ </Show>+ </div>+ );+ }}+ </Key>+ <Show when={hasAddColumn()}>+ <div role="gridcell" />+ </Show>+ </div>+ );+ }}+ </Key>+ </div></div></ContextMenu.Trigger><ContextMenu.Portal>@@ -931,6 +1342,7 @@ export function DatabaseTable(props: DatabaseTableProps) {<DatabaseTableSummarycolumns={columns()}rows={summaryRows()}+ hasMoreRows={props.hasMoreRows}template={template()}addColumn={hasAddColumn()}/>diff --git a/apps/web/src/features/database/components/database-toolbar.test.tsx b/apps/web/src/features/database/components/database-toolbar.test.tsxindex 6e8f7909be..ed39e811fe 100644--- a/apps/web/src/features/database/components/database-toolbar.test.tsx+++ b/apps/web/src/features/database/components/database-toolbar.test.tsx@@ -887,7 +887,8 @@ describe('database toolbar view controls', () => {expect(saving.textContent).toBe('Saving…');});- it('has no New button on a table view', () => {+ it('creates a record from the table toolbar without scrolling through loaded rows', () => {+ const createRecord = vi.fn();render(() => (<DatabaseToolbarcolumns={[]}@@ -911,11 +912,12 @@ describe('database toolbar view controls', () => {onShowViewAs={vi.fn()}onDeleteView={vi.fn(() => okAsync(undefined))}onReorderViews={vi.fn()}- onCreateRecord={vi.fn()}+ onCreateRecord={createRecord}canCreateRecord/>));- expect(screen.queryByRole('button', { name: 'New record' })).toBeNull();+ fireEvent.click(screen.getByRole('button', { name: 'New record' }));+ expect(createRecord).toHaveBeenCalledTimes(1);});});diff --git a/apps/web/src/features/database/components/database-toolbar.tsx b/apps/web/src/features/database/components/database-toolbar.tsxindex 593af535e5..0f41d936da 100644--- a/apps/web/src/features/database/components/database-toolbar.tsx+++ b/apps/web/src/features/database/components/database-toolbar.tsx@@ -91,9 +91,6 @@ export function DatabaseToolbar(props: DatabaseToolbarProps) {let viewRail: HTMLDivElement | undefined;/** Viewers change only what All records shows them; stored views are everyone's. */const canChangeView = () => !props.selectedViewId || props.canEdit;- const layout = () => props.view.layout;- /** Boards and timelines add records from the toolbar; a table has its blank row. */- const addsRecords = () => layout().kind !== 'table';const viewRename = createInlineRename({name: (view: DatabaseView) => view.name,rename: (view, name) => props.onRenameView(view, name),@@ -167,7 +164,9 @@ export function DatabaseToolbar(props: DatabaseToolbarProps) {canChangeView={canChangeView()}search={props.search}actions={props.actions}- onCreateRecord={addsRecords() ? props.onCreateRecord : undefined}+ // Every layout adds records here: a paged table's blank row only+ // follows its last page.+ onCreateRecord={props.onCreateRecord}canCreateRecord={props.canCreateRecord}creating={props.creating}selection={diff --git a/apps/web/src/features/database/components/grid-cell.tsx b/apps/web/src/features/database/components/grid-cell.tsxindex 1d6f4d8e2e..5ff3bb6b69 100644--- a/apps/web/src/features/database/components/grid-cell.tsx+++ b/apps/web/src/features/database/components/grid-cell.tsx@@ -1,10 +1,11 @@import { TimeRangeEditor } from '@app/components/date-time-range/time-range-editor';import { fireAndForget } from '@core/util/fireAndForget';import CaretDownIcon from '@icon/system/regular/caret-down.svg';+import CheckIcon from '@icon/system/regular/check.svg';import { Popover } from '@kobalte/core/popover';import { Tooltip } from '@kobalte/core/tooltip';import { PropertyDateSelector } from '@property/editors/selectors/PropertyDateSelector';-import { Checkbox } from '@ui/components/Checkbox';+import { Checkbox, InlineCheckbox } from '@ui/components/Checkbox';import { Dropdown } from '@ui/components/Dropdown';import {createMemo,@@ -536,12 +537,106 @@ function BooleanCell(props: {if (!isComposingKey(event)) navigateOnTab(event, props.onNavigate);}}>- <Checkbox.Control />+ {/* A CSS check, not `Checkbox.Indicator`: its presence reads computed+ styles as each row mounts, forcing a style recalculation per checked+ row while scrolling. */}+ <Checkbox.Control class="group">+ <CheckIcon class="hidden size-3 group-data-checked:block" />+ </Checkbox.Control><Checkbox.Label class="sr-only">{props.column.name}</Checkbox.Label></Checkbox>);}+/**+ * What a cell shows, without its editors, focus handling or popups: rows+ * mounted mid-scroll render this until scrolling settles, because a full+ * `GridCell` costs more than a fast scroll leaves time for. Matches the+ * display each `GridCell` kind renders.+ */+export function GridCellPreview(props: {+ column: DatabaseViewColumn;+ value: DatabaseCellValue;+ canEdit: boolean;+}) {+ const muted = () => !(props.canEdit && canEditCell(props.column));+ const kind = () =>+ match({+ boolean:+ props.column.dataType === 'BOOLEAN' && !props.column.isMultiSelect,+ option: isOptionColumn(props.column),+ // A mention's label is read per record, which is the cost being saved.+ mention:+ props.column.dataType === 'ENTITY' &&+ !props.column.isMultiSelect &&+ !props.column.relation,+ date:+ (props.column.dataType === 'DATE' ||+ props.column.dataType === 'DATE_RANGE') &&+ !props.column.isMultiSelect,+ })+ .with({ boolean: true }, () => 'boolean' as const)+ .with({ option: true }, () => 'option' as const)+ .with({ mention: true }, () => 'mention' as const)+ .with({ date: true }, () => 'date' as const)+ .with(+ { boolean: false, option: false, mention: false, date: false },+ () => 'text' as const+ )+ .exhaustive();+ return (+ <div class="relative grid min-w-0" aria-hidden="true">+ <Switch>+ <Match when={kind() === 'boolean'}>+ <span class="flex min-h-9 items-center px-3">+ <InlineCheckbox checked={Boolean(props.value)} size="md" />+ </span>+ </Match>+ <Match when={kind() === 'option'}>+ <span class="flex min-h-9 min-w-0 items-center px-2.5 py-1">+ <span class="flex min-w-0 flex-wrap gap-1">+ <For+ each={databaseCellValues(props.value, props.column)+ .filter((value) => value !== null)+ .map(String)}+ >+ {(value) => <SelectPill label={value} column={props.column} />}+ </For>+ </span>+ </span>+ </Match>+ <Match when={kind() === 'mention'}>+ <span class="flex min-h-9 items-center px-2.5">+ <Show when={props.value !== null}>+ <span class="h-2 w-1/2 rounded-full bg-hover" />+ </Show>+ </span>+ </Match>+ <Match when={kind() === 'date'}>+ <span class="flex min-h-9 min-w-0 items-center px-2.5 py-1.5 text-[13px] leading-5 whitespace-nowrap text-ink-muted">+ <span class="truncate">+ {formatCellValue(props.column, props.value)}+ </span>+ </span>+ </Match>+ <Match when={kind() === 'text'}>+ <span+ class="flex min-h-9 min-w-0 items-center px-2.5 py-1.5 text-[13px] leading-5"+ classList={{+ 'text-ink-muted': muted(),+ 'font-medium': props.column.dataType === 'STRING',+ }}+ >+ <span class="truncate">+ {formatCellValue(props.column, props.value)}+ </span>+ </span>+ </Match>+ </Switch>+ </div>+ );+}+type MentionPreview = {mention: DatabaseMention;originalValue: DatabaseCellValue;diff --git a/apps/web/src/features/database/context/table-source.ts b/apps/web/src/features/database/context/table-source.tsindex f4a771c179..70af123b05 100644--- a/apps/web/src/features/database/context/table-source.ts+++ b/apps/web/src/features/database/context/table-source.ts@@ -25,8 +25,16 @@ export type DatabaseWriteResult = {version: number | undefined;};+export type DatabaseRowsPagination = {+ hasMore: Accessor<boolean>;+ loading: Accessor<boolean>;+ version: Accessor<number | undefined>;+ loadMore(): ResultAsync<unknown, DatabaseReadFailure>;+};+/** Rows are undefined before the first successful read; background errors may coexist with them. */export type DatabaseRowsSource = {+ pagination?: DatabaseRowsPagination;columns: Accessor<DatabaseViewColumn[]>;snapshot: Accessor<DatabaseRowsSnapshot | undefined>;/** The view's last read as the engine answered it. */diff --git a/apps/web/src/features/database/core/grid-cells.test.ts b/apps/web/src/features/database/core/grid-cells.test.tsindex 56bd278ca6..10d8cdb5d7 100644--- a/apps/web/src/features/database/core/grid-cells.test.ts+++ b/apps/web/src/features/database/core/grid-cells.test.ts@@ -2,6 +2,42 @@ import { describe, expect, it } from 'vitest';import { gridRows, keepUnchangedRows, UNAVAILABLE_OPTION } from './grid-cells';describe('engine cells as grid values', () => {+ it('reuses appended row vectors but recomputes after a cell or schema change', () => {+ const catalog: Parameters<typeof gridRows>[1] = { tables: [] };+ const columns: Parameters<typeof gridRows>[2] = [];+ const outcome: Parameters<typeof gridRows>[0] = {+ columns: [],+ rows: [[]],+ rowIds: ['first'],+ readTables: ['table'],+ truncated: false,+ insertedRowIds: [],+ changesApplied: 0,+ };+ const rows = gridRows(outcome, catalog, columns);+ const previous = { outcome, catalog, columns, rows };+ const appended = {+ ...outcome,+ rows: [...outcome.rows, []],+ rowIds: ['first', 'second'],+ };+ const next = gridRows(appended, catalog, columns, previous);+ expect(next).toEqual([+ { rowId: 'first', cells: {} },+ { rowId: 'second', cells: {} },+ ]);+ expect(next[0]).toBe(rows[0]);+ expect(+ gridRows({ ...outcome, rows: [[]] }, catalog, columns, previous)[0]+ ).not.toBe(rows[0]);+ expect(gridRows(outcome, { ...catalog }, columns, previous)[0]).not.toBe(+ rows[0]+ );+ expect(gridRows(outcome, catalog, [...columns], previous)[0]).not.toBe(+ rows[0]+ );+ });+it('shows an option the catalog lacks instead of dropping it', () => {expect(gridRows(@@ -168,6 +204,27 @@ describe('engine date ranges as grid values', () => {});describe('a new read of rows the grid already shows', () => {+ it('keeps the displayed array when every row and its position is unchanged', () => {+ const previous = [+ { rowId: 'ada', cells: { name: 'Ada', guests: 2 } },+ { rowId: 'grace', cells: { name: 'Grace', guests: 1 } },+ ];+ expect(+ keepUnchangedRows(previous, [+ { rowId: 'ada', cells: { name: 'Ada', guests: 2 } },+ { rowId: 'grace', cells: { name: 'Grace', guests: 1 } },+ ])+ ).toBe(previous);++ const reordered = keepUnchangedRows(previous, [+ { rowId: 'grace', cells: { name: 'Grace', guests: 1 } },+ { rowId: 'ada', cells: { name: 'Ada', guests: 2 } },+ ]);+ expect(reordered).not.toBe(previous);+ expect(reordered).toEqual([previous[1], previous[0]]);+ expect(keepUnchangedRows(previous, [])).toEqual([]);+ });+it('hands back the shown row for each row whose cells read the same', () => {const ada = { rowId: 'ada', cells: { name: 'Ada', guests: 2 } };const grace = { rowId: 'grace', cells: { name: 'Grace', guests: 1 } };diff --git a/apps/web/src/features/database/core/grid-cells.ts b/apps/web/src/features/database/core/grid-cells.tsindex d3decd21c1..465d4080b2 100644--- a/apps/web/src/features/database/core/grid-cells.ts+++ b/apps/web/src/features/database/core/grid-cells.ts@@ -57,11 +57,26 @@ function gridValue(}/** A row-shaped outcome's rows, with a cell for each of the table's columns. */+export type GridRowsRead = {+ outcome: Outcome;+ catalog: Catalog;+ columns: readonly ColumnDetail[];+ rows: DatabaseRow[];+};++/** Convert new cells, reusing unchanged vectors only under the same schema. */export function gridRows(outcome: Outcome,catalog: Catalog,- columns: readonly ColumnDetail[]+ columns: readonly ColumnDetail[],+ previous?: GridRowsRead): DatabaseRow[] {+ const reusable =+ previous?.catalog === catalog &&+ previous.columns === columns &&+ previous.outcome.columns === outcome.columns+ ? previous+ : undefined;const catalogColumns = new Map(catalog.tables.flatMap((table) =>table.columns.map((column) => [column.id, column] as const)@@ -77,27 +92,36 @@ export function gridRows(kind: catalogColumns.get(definition)?.kind,};});- return outcome.rowIds.map((rowId, rowIndex) => ({- rowId,- cells: Object.fromEntries(- placed.map(({ id, index, kind }) => [- id,- gridValue(outcome.rows[rowIndex]?.[index] ?? null, kind),- ])- ),- }));+ return outcome.rowIds.map((rowId, rowIndex) =>+ reusable?.outcome.rowIds[rowIndex] === rowId &&+ reusable.outcome.rows[rowIndex] === outcome.rows[rowIndex]+ ? reusable.rows[rowIndex]+ : {+ rowId,+ cells: Object.fromEntries(+ placed.map(({ id, index, kind }) => [+ id,+ gridValue(outcome.rows[rowIndex]?.[index] ?? null, kind),+ ])+ ),+ }+ );}-/** `next`, with each row whose cells read exactly as in `previous` kept as that row. */+/** Reuse unchanged rows, and the array when their order also stays the same. */export function keepUnchangedRows(- previous: readonly DatabaseRow[],+ previous: DatabaseRow[],next: DatabaseRow[]): DatabaseRow[] {const shown = new Map(previous.map((row) => [row.rowId, row]));- return next.map((row) => {+ const kept = next.map((row) => {const before = shown.get(row.rowId);return before && sameCells(before.cells, row.cells) ? before : row;});+ return kept.length === previous.length &&+ kept.every((row, index) => row === previous[index])+ ? previous+ : kept;}function sameCells(diff --git a/apps/web/src/features/database/core/write-failure.ts b/apps/web/src/features/database/core/write-failure.tsindex 5734646ea0..004bbec9f8 100644--- a/apps/web/src/features/database/core/write-failure.ts+++ b/apps/web/src/features/database/core/write-failure.ts@@ -186,7 +186,11 @@ export function databaseReadMessage(failure: DatabaseReadFailure): string {{ kind: 'fetch' },() => 'The rows could not be loaded. Check your connection.')- .with({ kind: 'read-only' }, () => 'Try refreshing the table.')+ .with(+ { kind: 'read-only' },+ { kind: 'cancelled' },+ () => 'Try refreshing the table.'+ ).with({ kind: 'table-unavailable' },() => 'This table is no longer available.'diff --git a/apps/web/src/features/database/primitives/cell-selection.test.ts b/apps/web/src/features/database/primitives/cell-selection.test.tsnew file mode 100644index 0000000000..74415209d9--- /dev/null+++ b/apps/web/src/features/database/primitives/cell-selection.test.ts@@ -0,0 +1,45 @@+import { createRoot, createSignal } from 'solid-js';+import { describe, expect, it } from 'vitest';+import { createCellSelection } from './cell-selection';++describe('cell positions', () => {+ it('keeps positions stable for value changes and updates them when rows or columns move', () => {+ createRoot((dispose) => {+ const [rows, setRows] = createSignal([+ { id: 'one', name: 'First' },+ { id: 'two', name: 'Second' },+ ]);+ const [columns, setColumns] = createSignal(['name', 'notes']);+ const selection = createCellSelection({+ rows: () => rows().map((row) => row.id),+ columns,+ cellAt: () => undefined,+ onChange: () => {},+ focus: () => {},+ });+ const rowPositions = selection.rowIndex();+ const columnPositions = selection.columnIndex();+ setRows([+ { id: 'one', name: 'Changed' },+ { id: 'two', name: 'Second' },+ ]);+ setColumns(['name', 'notes']);+ expect(selection.rowIndex()).toBe(rowPositions);+ expect(selection.columnIndex()).toBe(columnPositions);++ setRows([+ { id: 'two', name: 'Second' },+ { id: 'three', name: 'Third' },+ { id: 'one', name: 'Changed' },+ ]);+ setColumns(['notes']);+ expect([...selection.rowIndex()]).toEqual([+ ['two', 0],+ ['three', 1],+ ['one', 2],+ ]);+ expect([...selection.columnIndex()]).toEqual([['notes', 0]]);+ dispose();+ });+ });+});diff --git a/apps/web/src/features/database/primitives/cell-selection.ts b/apps/web/src/features/database/primitives/cell-selection.tsindex 68bb98bbe0..33602c777e 100644--- a/apps/web/src/features/database/primitives/cell-selection.ts+++ b/apps/web/src/features/database/primitives/cell-selection.ts@@ -1,3 +1,4 @@+import { arrayEquals } from '@core/util/compareUtils';import { fireAndForget } from '@core/util/fireAndForget';import { createMemo, createSignal, onCleanup } from 'solid-js';@@ -34,11 +35,13 @@ export function createCellSelection(options: {onClear?: (rows: string[], columns: string[]) => Promise<boolean>;}) {const [range, setRange] = createSignal<CellRange>();+ const rowIds = createMemo(options.rows, [], { equals: arrayEquals });+ const columnIds = createMemo(options.columns, [], { equals: arrayEquals });const rowIndex = createMemo(- () => new Map(options.rows().map((id, index) => [id, index]))+ () => new Map(rowIds().map((id, index) => [id, index])));const columnIndex = createMemo(- () => new Map(options.columns().map((id, index) => [id, index]))+ () => new Map(columnIds().map((id, index) => [id, index])));const bounds = createMemo(() => {const selected = range();diff --git a/apps/web/src/features/database/primitives/held-grid-rows.test.ts b/apps/web/src/features/database/primitives/held-grid-rows.test.tsindex 661d7dab75..259e9ab6bb 100644--- a/apps/web/src/features/database/primitives/held-grid-rows.test.ts+++ b/apps/web/src/features/database/primitives/held-grid-rows.test.ts@@ -4,6 +4,33 @@ import type { DatabaseRow } from '../core/table';import { createHeldGridRows } from './held-grid-rows';describe('held grid rows', () => {+ it('keeps the displayed rows stable when focus moves between visible records', () => {+ createRoot((dispose) => {+ const [records, setRecords] = createSignal<DatabaseRow[]>([+ { rowId: 'first', cells: { title: 'First' } },+ { rowId: 'last', cells: { title: 'Last' } },+ ]);+ const grid = createHeldGridRows({+ rows: () => [...records(), { rowId: 'draft', cells: { title: null } }],+ knownRows: records,+ });+ const displayed = grid.rows();+ grid.setEditingRowId('first');+ expect(grid.rows()).toBe(displayed);+ grid.setEditingRowId('last');+ expect(grid.rows()).toBe(displayed);+ grid.setEditingRowId(undefined);+ expect(grid.rows()).toBe(displayed);++ setRecords([{ rowId: 'first', cells: { title: 'Changed' } }]);+ expect(grid.rows()).toEqual([+ { rowId: 'first', cells: { title: 'Changed' } },+ { rowId: 'draft', cells: { title: null } },+ ]);+ dispose();+ });+ });+it('keeps the row being typed in at its place after it leaves the view, until the edit ends', () => {createRoot((dispose) => {const [rows, setRows] = createSignal<DatabaseRow[]>([diff --git a/apps/web/src/features/database/primitives/held-grid-rows.ts b/apps/web/src/features/database/primitives/held-grid-rows.tsindex 4feb9e8854..7bef0d2b4d 100644--- a/apps/web/src/features/database/primitives/held-grid-rows.ts+++ b/apps/web/src/features/database/primitives/held-grid-rows.ts@@ -12,8 +12,9 @@ export function createHeldGridRows(options: {knownRows: Accessor<DatabaseRow[]>;}) {const [editingRowId, setEditingRowId] = createSignal<string>();+ const currentRows = createMemo(options.rows);const rows = createMemo<DatabaseRow[]>((shownBefore) => {- const current = options.rows();+ const current = currentRows();const held = editingRowId();const index = shownBefore.findIndex((row) => row.rowId === held);if (diff --git a/apps/web/src/features/database/primitives/record-actions.ts b/apps/web/src/features/database/primitives/record-actions.tsindex 4229ff9f38..f5d05218be 100644--- a/apps/web/src/features/database/primitives/record-actions.ts+++ b/apps/web/src/features/database/primitives/record-actions.ts@@ -23,12 +23,14 @@ import type { createTableController } from './table-controller';/** How long a revealed row stays tinted. */const HIGHLIGHT_MS = 1_600;-/** A saved record the view's filters leave out. */+/** A saved record outside the currently loaded rows. */type HiddenSavedRecord = {rowId: string;created: boolean;/** The record was created, but the view has no column to type its first value in. */noEditableColumns?: boolean;+ /** More pages remain, so absence does not prove a filter exclusion. */+ unloaded?: boolean;};/** A new record: its first values, and whether to start editing it once saved. */@@ -135,10 +137,14 @@ export function createRecordActions(options: {}if (mutation.kind !== 'delete' &&- options.constrained() &&+ (options.constrained() || options.source.pagination?.hasMore()) &&!rows().some((row) => row.rowId === rowId)) {- setHiddenSavedRecord({ rowId, created: mutation.kind === 'create' });+ setHiddenSavedRecord({+ rowId,+ created: mutation.kind === 'create',+ unloaded: options.source.pagination?.hasMore(),+ });} else if (hiddenSavedRecord()?.rowId === rowId) {setHiddenSavedRecord(undefined);}@@ -211,6 +217,7 @@ export function createRecordActions(options: {options.editCell(rowId, field.id);else if (!field)setHiddenSavedRecord({ rowId, created: true, noEditableColumns: true });+ else open(rowId);}async function duplicateRow(rowId: string) {const actualId = actualRowId(rowId);diff --git a/apps/web/src/features/database/primitives/table-model.ts b/apps/web/src/features/database/primitives/table-model.tsindex 76f0df1e4e..ca37a7359b 100644--- a/apps/web/src/features/database/primitives/table-model.ts+++ b/apps/web/src/features/database/primitives/table-model.ts@@ -37,7 +37,7 @@ export type DatabaseTableModelOptions = {onResizeColumn?: (columnId: string, width: number) => void;};-/** TanStack owns the display model; SQL owns which records match and their order. */+/** TanStack owns columns; SQL owns which records match and their order. */export function createDatabaseTableModel(options: DatabaseTableModelOptions) {const columns = createMemo<ColumnDef<typeof features, DatabaseRow>[]>(() =>options.columns.map((column) => ({@@ -146,6 +146,7 @@ export function createDatabaseTableModel(options: DatabaseTableModelOptions) {}return {table,+ rows: () => options.rows,visibleColumns,width,resize,diff --git a/apps/web/src/features/database/tests/grid-resize-observer.ts b/apps/web/src/features/database/tests/grid-resize-observer.tsnew file mode 100644index 0000000000..0983917162--- /dev/null+++ b/apps/web/src/features/database/tests/grid-resize-observer.ts@@ -0,0 +1,22 @@+/** Supplies the row and viewport measurements JSDOM does not implement. */+export class GridResizeObserver implements ResizeObserver {+ constructor(private readonly callback: ResizeObserverCallback) {}+ observe(target: Element) {+ const height = target.querySelector('[role="grid"]') ? 400 : 43;+ const size = { inlineSize: 800, blockSize: height };+ this.callback(+ [+ {+ target,+ contentRect: new DOMRect(0, 0, 800, height),+ borderBoxSize: [size],+ contentBoxSize: [size],+ devicePixelContentBoxSize: [size],+ },+ ],+ this+ );+ }+ unobserve() {}+ disconnect() {}+}diff --git a/apps/web/src/features/database/views/database-records-view.test.tsx b/apps/web/src/features/database/views/database-records-view.test.tsxindex 1b32a7a2f0..7602f48b58 100644--- a/apps/web/src/features/database/views/database-records-view.test.tsx+++ b/apps/web/src/features/database/views/database-records-view.test.tsx@@ -26,11 +26,13 @@ import type {} from '../../../lib/core/database-sql/generated/types';import type { DatabaseWriteResult } from '../context/table-source';import type { DatabaseViewColumn } from '../core/database-view';+import type { DatabaseRow } from '../core/table';import type { ViewChange } from '../core/view-state';import { allRecordsView } from '../core/views';import type { DatabaseWriteFailure } from '../core/write-failure';import { boardLane } from '../tests/board-layout';import { createFakeRowsSource, titleContains } from '../tests/fake-rows-source';+import { GridResizeObserver } from '../tests/grid-resize-observer';import type { BoardPositions } from './database-board-view';import {type DatabaseRecordsActions,@@ -221,12 +223,10 @@ beforeEach(() => {vi.spyOn(window, 'scrollTo').mockImplementation(() => {});// JSDOM has no layout, so a highlighted row cannot scroll itself into view.Element.prototype.scrollIntoView = vi.fn();- vi.stubGlobal(- 'ResizeObserver',- class {- observe() {}- unobserve() {}- disconnect() {}+ vi.stubGlobal('ResizeObserver', GridResizeObserver);+ vi.spyOn(HTMLElement.prototype, 'offsetParent', 'get').mockImplementation(+ function (this: HTMLElement) {+ return this.parentElement;});});@@ -1086,6 +1086,84 @@ describe('database table view', () => {);});+ it.each([false, true])(+ 'opens a duplicate beyond the loaded page without claiming it fails filters (filtered=%s)',+ async (filtered) => {+ const view = filtered+ ? {+ ...allRecords,+ query: {+ filter: {+ conjunction: 'and' as const,+ conditions: [+ {+ kind: 'condition' as const,+ column: 'title',+ test: {+ kind: 'text' as const,+ operator: 'contains' as const,+ value: 'Plan',+ },+ },+ ],+ },+ sort: [],+ },+ }+ : allRecords;+ const fixture = createFakeRowsSource({+ columns,+ table: {+ version: 1,+ rows: Array.from({ length: 501 }, (_, index) => ({+ rowId: `row-${index}`,+ cells: { title: `Plan ${index}`, status: 'To do' },+ })),+ },+ view,+ });+ fixture.setAnswer(() => (rows: DatabaseRow[]) => rows.slice(0, 500));+ fixture.source.pagination = {+ hasMore: () => true,+ loading: () => false,+ version: () => 1,+ loadMore: () => okAsync(undefined),+ };+ fixture.persistWrites();+ render(() => (+ <DatabaseRecordsView+ name="Projects"+ source={fixture.source}+ canEdit+ view={view}+ stored={false}+ addColumn={() => null}+ boardPositions={unplacedCards}+ />+ ));+ fireEvent.contextMenu(+ screen.getByRole('button', { name: 'Open Plan 0' })+ );+ const duplicate = await screen.findByRole('menuitem', {+ name: 'Duplicate',+ });+ duplicate.focus();+ fireEvent.keyDown(duplicate, { key: 'Enter' });+ const dialog = await screen.findByRole('dialog');+ expect(+ within(dialog).getByRole('heading', { name: 'Plan 0' })+ ).toBeTruthy();+ expect(screen.queryByText(/doesn’t match your filters/)).toBeNull();+ expect(screen.queryByText('Record created outside this view')).toBeNull();+ expect(fixture.source.write).toHaveBeenCalledTimes(1);+ expect(+ fixture.source+ .snapshot()+ ?.retained.some((row) => row.rowId === 'created')+ ).toBe(true);+ }+ );+it('confirms a context-menu deletion and retries the same record without leaving a stale failure', async () => {const fixture = createFakeRowsSource({columns,diff --git a/apps/web/src/features/database/views/database-records-view.tsx b/apps/web/src/features/database/views/database-records-view.tsxindex 9c0fb76aac..7372dccdda 100644--- a/apps/web/src/features/database/views/database-records-view.tsx+++ b/apps/web/src/features/database/views/database-records-view.tsx@@ -31,7 +31,11 @@ import {SchemaErrorNotice,} from '../components/database-table-notices';import { filterConditionCount } from '../components/database-view-filters';-import { GridCell, type GridCellProps } from '../components/grid-cell';+import {+ GridCell,+ GridCellPreview,+ type GridCellProps,+} from '../components/grid-cell';import { RecordPanel } from '../components/record-panel';import type { DatabaseRowsSource } from '../context/table-source';import { toCellTimeRange } from '../core/cell-date-range';@@ -466,15 +470,20 @@ export function DatabaseRecordsView(props: DatabaseRecordsViewProps) {<HiddenRecordNoticetitle={records.hiddenSavedRecord()?.created- ? records.hiddenSavedRecord()?.noEditableColumns+ ? records.hiddenSavedRecord()?.noEditableColumns ||+ records.hiddenSavedRecord()?.unloaded? 'Record created': 'Record created outside this view'- : 'Record saved outside this view'+ : records.hiddenSavedRecord()?.unloaded+ ? 'Record saved'+ : 'Record saved outside this view'}message={records.hiddenSavedRecord()?.noEditableColumns? 'This view has no editable columns. Open the record to see its details.'- : `“${rowTitle(row(), columns())}” doesn’t match your filters.`+ : records.hiddenSavedRecord()?.unloaded+ ? 'Open the record to view or edit it.'+ : `“${rowTitle(row(), columns())}” doesn’t match your filters.`}onOpen={() => records.open(row().rowId)}onDismiss={records.dismissHiddenRecord}@@ -571,6 +580,11 @@ export function DatabaseRecordsView(props: DatabaseRecordsViewProps) {</Match><Match when={layoutKind() === 'table'}><DatabaseTableView+ hasMoreRows={props.source.pagination?.hasMore()}+ onLoadMore={() => {+ if (props.source.pagination?.hasMore())+ void props.source.pagination.loadMore();+ }}onClearCells={props.canEdit? async (rowIds, columnIds) => {@@ -631,6 +645,13 @@ export function DatabaseRecordsView(props: DatabaseRecordsViewProps) {focusColumn)}renderCell={renderCell}+ renderCellPreview={(row, column) => (+ <GridCellPreview+ column={column()}+ value={rowValue(row(), column().id)}+ canEdit={props.canEdit}+ />+ )}controlsRef={tableReady}titleColumnId={titleColumn(columns())?.id}getRowTitle={(row) => rowTitle(row, columns())}@@ -693,7 +714,9 @@ export function DatabaseRecordsView(props: DatabaseRecordsViewProps) {canEdit={props.canEdit}pending={controller.pending()}outsideViewReason={- records.selectedPosition() < 0 && filtered()+ records.selectedPosition() < 0 &&+ filtered() &&+ !props.source.pagination?.hasMore()? `This record doesn’t match your filters. You can ${props.canEdit ? 'keep editing' : 'view'} it here.`: undefined}diff --git a/apps/web/src/features/email-compose/queued-send-status.tsx b/apps/web/src/features/email-compose/queued-send-status.tsxindex ec8bf230c4..01e68424df 100644--- a/apps/web/src/features/email-compose/queued-send-status.tsx+++ b/apps/web/src/features/email-compose/queued-send-status.tsx@@ -104,9 +104,13 @@ export function QueuedSendStatus(props: {}>{/* Full-frame phone views leave the top inset to their content. This- panel sits above that content, so it takes the inset in its place. */}+ panel sits above that content, so it takes the inset in its place.+ The sibling rule names `html` outright: `touch:[&~*]` puts a complex+ selector inside `:where()`, which Chrome treats as a universal+ sibling rule, so every element inserted anywhere in the app would+ restyle its parent's whole subtree. */}<div- class="shrink-0 border-b border-edge-muted p-3 touch:pt-[calc(var(--mobile-content-inset-top,0px)+0.75rem)] touch:[&~*]:[--mobile-content-inset-top:0px]"+ class="shrink-0 border-b border-edge-muted p-3 touch:pt-[calc(var(--mobile-content-inset-top,0px)+0.75rem)] [html[data-touch-device=true]_&~*]:[--mobile-content-inset-top:0px]"aria-label="Queued email sends"><For each={visible()}>diff --git a/apps/web/src/features/entity/composed/list-entity/compacting-pills.tsx b/apps/web/src/features/entity/composed/list-entity/compacting-pills.tsxindex 777370b3e5..606b57f2d9 100644--- a/apps/web/src/features/entity/composed/list-entity/compacting-pills.tsx+++ b/apps/web/src/features/entity/composed/list-entity/compacting-pills.tsx@@ -25,7 +25,7 @@ export function CompactingPills(props: {'group/pills flex min-w-0 items-center gap-1 overflow-hidden','data-compact:gap-0 data-compact:[&_[data-pill-text]]:hidden',// Pill groups render as `contents`, so their pills stack here too.- 'data-compact:[&>*+*]:-ml-1.5 data-compact:[&>.contents:not(:first-child)>*]:-ml-1.5 data-compact:[&>.contents>*+*]:-ml-1.5',+ 'data-compact:[&>:not(:first-child)]:-ml-1.5 data-compact:[&>.contents:not(:first-child)>*]:-ml-1.5 data-compact:[&>.contents>:not(:first-child)]:-ml-1.5','[&>*]:relative [&>.contents>*]:relative [&_*:hover]:z-10',props.class)}diff --git a/apps/web/src/features/settings/connections/integrations-view.tsx b/apps/web/src/features/settings/connections/integrations-view.tsxindex 8b974f4e3f..1592dab5bd 100644--- a/apps/web/src/features/settings/connections/integrations-view.tsx+++ b/apps/web/src/features/settings/connections/integrations-view.tsx@@ -322,7 +322,7 @@ function IntegrationsOverview(<IntegrationsListHeader /><divrole="rowgroup"- class="flex flex-col [&>[role=row]+[role=row]]:border-t [&>[role=row]+[role=row]]:border-ink/5"+ class="flex flex-col [&>[role=row]:not(:first-child)]:border-t [&>[role=row]:not(:first-child)]:border-ink/5"><For each={customLeftovers()}>{(leftover) => <LeftoverRow leftover={leftover} />}diff --git a/apps/web/src/lib/core/database-sql/driver.test.ts b/apps/web/src/lib/core/database-sql/driver.test.tsindex 8c3f483cdb..b016901fdb 100644--- a/apps/web/src/lib/core/database-sql/driver.test.ts+++ b/apps/web/src/lib/core/database-sql/driver.test.ts@@ -1,10 +1,53 @@-import { errAsync, okAsync } from 'neverthrow';+import { errAsync, okAsync, ResultAsync } from 'neverthrow';import { describe, expect, it, vi } from 'vitest';import { checkReadStatement, runDatabaseSql } from './driver';import type { Bin, GqlQuery, Page } from './generated/types';import { readTranscript, replay } from './tests/transcript';describe('runDatabaseSql', () => {+ it('reports cancellation when opening the engine fails after an abort', async () => {+ const pending = Promise.withResolvers<never>();+ const open = vi.fn(() => pending.promise);+ const page = vi.fn();+ const bins = vi.fn();+ const controller = new AbortController();+ const reading = runDatabaseSql({ tables: [] }, 'SELECT 1', {+ source: { page, bins },+ open,+ signal: controller.signal,+ });+ await vi.waitFor(() => expect(open).toHaveBeenCalledTimes(1));+ controller.abort();+ pending.reject(new Error('Engine module could not load'));++ expect((await reading)._unsafeUnwrapErr()).toEqual({ kind: 'cancelled' });+ expect(page).not.toHaveBeenCalled();+ expect(bins).not.toHaveBeenCalled();+ });++ it('stops a superseded read after its in-flight page and frees the engine', async () => {+ const paging = readTranscript('paging');+ const pending = Promise.withResolvers<Page>();+ const page = vi.fn(() => ResultAsync.fromSafePromise(pending.promise));+ const free = vi.fn();+ const open = replay(paging);+ const controller = new AbortController();+ const reading = runDatabaseSql(paging.catalog, paging.sql, {+ source: { page, bins: vi.fn() },+ signal: controller.signal,+ open: async (catalog, sql) => ({ ...(await open(catalog, sql)), free }),+ });+ await vi.waitFor(() => expect(page).toHaveBeenCalledTimes(1));+ controller.abort();+ const first = paging.exchanges[0];+ if (!('page' in first)) throw new Error('recorded bins');+ pending.resolve(first.page);++ expect((await reading)._unsafeUnwrapErr()).toEqual({ kind: 'cancelled' });+ expect(page).toHaveBeenCalledTimes(1);+ expect(free).toHaveBeenCalledTimes(1);+ });+it('asks the source for each page the engine wants, following the cursor', async () => {const paging = readTranscript('paging');const page = vi.fn(diff --git a/apps/web/src/lib/core/database-sql/driver.ts b/apps/web/src/lib/core/database-sql/driver.tsindex 7c167985c3..6d1af6d81e 100644--- a/apps/web/src/lib/core/database-sql/driver.ts+++ b/apps/web/src/lib/core/database-sql/driver.ts@@ -33,6 +33,8 @@ export type DatabaseSqlFailure =| { kind: 'crash'; message: string }/** The row source could not read what the engine asked for. */| { kind: 'fetch'; message: string }+ /** The caller replaced or disposed this read. */+ | { kind: 'cancelled' }/** The statement writes, and only reads run here. */| { kind: 'read-only' };@@ -145,10 +147,13 @@ function nextStep(async function steps(query: DatabaseSqlQuery,source: RowSource,- trace: DatabaseSqlRunTrace+ trace: DatabaseSqlRunTrace,+ signal?: AbortSignal): Promise<Result<Outcome, DatabaseSqlFailure>> {+ if (signal?.aborted) return err({ kind: 'cancelled' });let step = trace.start(() => feed(() => query.start()));while (step.isOk()) {+ if (signal?.aborted) return err({ kind: 'cancelled' });const current = step.value;if (current.step === 'done') {const { step: _done, ...outcome } = current;@@ -156,18 +161,28 @@ async function steps(}step = await nextStep(query, current, source, trace);}- return err(step.error);+ return err(signal?.aborted ? { kind: 'cancelled' } : step.error);}function drive(opened: () => Promise<DatabaseSqlQuery>,source: RowSource,- trace: DatabaseSqlRunTrace+ trace: DatabaseSqlRunTrace,+ signal?: AbortSignal): ResultAsync<Outcome, DatabaseSqlFailure> {- return ResultAsync.fromPromise(trace.open(opened), engineFailure).andThen(- (query) =>- new ResultAsync(steps(query, source, trace).finally(() => query.free()))- );+ if (signal?.aborted) return errAsync({ kind: 'cancelled' });+ return ResultAsync.fromPromise(trace.open(opened), (thrown) =>+ signal?.aborted ? ({ kind: 'cancelled' } as const) : engineFailure(thrown)+ ).andThen((query) => {+ const read = async () => {+ try {+ return await steps(query, source, trace, signal);+ } finally {+ query.free();+ }+ };+ return new ResultAsync(read());+ });}/** Run a read-only statement to its outcome; a write is refused. */@@ -178,12 +193,18 @@ export function runDatabaseSql(source,open = openDatabaseSqlQuery,context,- }: { source: RowSource; open?: OpenEngine; context?: DatabaseSqlReadContext }+ signal,+ }: {+ source: RowSource;+ open?: OpenEngine;+ context?: DatabaseSqlReadContext;+ signal?: AbortSignal;+ }): ResultAsync<Outcome, DatabaseSqlFailure> {return traceDatabaseSqlRun({ kind: 'sql', sql },catalog,- (trace) => drive(() => open(catalog, sql), source, trace),+ (trace) => drive(() => open(catalog, sql), source, trace, signal),context);}@@ -196,12 +217,18 @@ export function runDatabaseView(source,open = openDatabaseViewQuery,context,- }: { source: RowSource; open?: OpenView; context?: DatabaseSqlReadContext }+ signal,+ }: {+ source: RowSource;+ open?: OpenView;+ context?: DatabaseSqlReadContext;+ signal?: AbortSignal;+ }): ResultAsync<Outcome, DatabaseSqlFailure> {return traceDatabaseSqlRun({ kind: 'view', view },catalog,- (trace) => drive(() => open(catalog, view), source, trace),+ (trace) => drive(() => open(catalog, view), source, trace, signal),context);}diff --git a/apps/web/src/lib/core/database-sql/trace.test.ts b/apps/web/src/lib/core/database-sql/trace.test.tsindex d32f65fd4c..732f8c8d89 100644--- a/apps/web/src/lib/core/database-sql/trace.test.ts+++ b/apps/web/src/lib/core/database-sql/trace.test.ts@@ -54,6 +54,19 @@ afterEach(() => {vi.restoreAllMocks();});+it('ends a cancelled read without reporting a query failure', async () => {+ await traceDatabaseSqlRun(+ { kind: 'sql', sql: 'SELECT 1' },+ { tables: [] },+ () => errAsync({ kind: 'cancelled' }),+ { reason: 'statement-change', requestPolicy: 'network-only' }+ );+ expect(recorded.roots[0].attributes['database_sql.cancelled']).toBe(true);+ expect(recorded.roots[0].attributes.error).toBeUndefined();+ expect(recorded.roots[0].ended).toBe(true);+ expect(recorded.warnings).not.toHaveBeenCalled();+});+const catalog: Catalog = {tables: [{diff --git a/apps/web/src/lib/core/database-sql/trace.ts b/apps/web/src/lib/core/database-sql/trace.tsindex a51f27ca85..3c3f2d5609 100644--- a/apps/web/src/lib/core/database-sql/trace.ts+++ b/apps/web/src/lib/core/database-sql/trace.ts@@ -347,6 +347,11 @@ export function traceDatabaseSqlRun<);}).orTee((failure) => {+ if (failure.kind === 'cancelled') {+ runSpan.setAttr('database_sql.cancelled', true);+ finish({ error: 'cancelled' }, failure);+ return;+ }const engineError = failure.error;const detail = {...(engineErrordiff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.tsindex f94ded65e6..c3eb573ef8 100644--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.test.ts@@ -1034,6 +1034,19 @@ describe('normalizedCacheExchange', () => {}});+ it('publishes a network result it was asked not to persist without writing it', async () => {+ const { ops, results } = harness(host, () => ({+ data: { user: { emailThread: null } },+ }));+ const op = makeOp(902, 'network-only');+ ops.next(+ makeOperation('query', op, { ...op.context, persistResult: false })+ );+ await tick();+ expect(host.cacheActions).toEqual([]);+ expect(results.at(-1)?.data).toEqual({ user: { emailThread: null } });+ });+it('evicts inferred missing records after writing the network response', async () => {const deletedRecordKeys = vi.fn(() => ['GraphqlSoupEmailThread:gone']);const { ops, results } = harness(diff --git a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.tsindex 4f2e2002a9..01a061983a 100644--- a/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts+++ b/apps/web/src/lib/graphql-cache/exchange/normalized-cache-exchange.ts@@ -121,6 +121,14 @@ const AFFECTED_READ_CONTEXT_KEY = 'normalizedCacheAffectedRead';const REPLACEMENT_REGISTRATION_ONLY_CONTEXT_KEY ='normalizedCacheReplacementRegistrationOnly';/** Marks a query as network-to-cache hydration with a projected result. */++declare module '@urql/core' {+ interface OperationContext {+ /** Publish the network result without storing it in the cache. */+ persistResult?: false;+ }+}+export const HYDRATE_ONLY_CONTEXT_KEY = 'normalizedCacheHydrateOnly';/** Retains the client-annotated document while the transport uses a stripped copy. */const HYDRATION_DOCUMENT_CONTEXT_KEY = 'normalizedCacheHydrationDocument';@@ -1505,7 +1513,10 @@ export function normalizedCacheExchange(// Publication, not persistence, supersedes an older cache snapshot.// A failed network request still permits a slower offline cache hit.if (result.data != null && !result.error) state.cacheReadVersion += 1;- const persistence = persistQueryResult(result, state, version);+ const persistence =+ op.context.persistResult === false+ ? Promise.resolve(undefined)+ : persistQueryResult(result, state, version);return withResultMetadata(result, {source: 'live-network',persistence,diff --git a/apps/web/src/lib/queries/database-sql/create-database-sql-query.test.ts b/apps/web/src/lib/queries/database-sql/create-database-sql-query.test.tsindex 87bdbddfa1..2de1cfc22a 100644--- a/apps/web/src/lib/queries/database-sql/create-database-sql-query.test.ts+++ b/apps/web/src/lib/queries/database-sql/create-database-sql-query.test.ts@@ -138,6 +138,34 @@ afterEach(() => {});describe('createDatabaseSqlQuery', () => {+ it('does not open an engine for a pending catalog after its owner is disposed', async () => {+ const pending = Promise.withResolvers<Catalog>();+ const build = vi.fn(() => pending.promise);+ const open = vi.fn(names);+ const query = createRoot((cleanup) => {+ dispose = cleanup;+ return createDatabaseSqlQuery(+ () => ({ schema, sql: 'SELECT name FROM crm.deals' }),+ {+ client: () =>+ createClient({ url: 'http://test.invalid', exchanges: [] }),+ cacheHost: () => undefined,+ people: async () => [],+ catalog: build,+ open,+ }+ );+ });+ await vi.waitFor(() => expect(build).toHaveBeenCalledTimes(1));+ dispose?.();+ pending.resolve(catalog);+ await new Promise((resolve) => setTimeout(resolve, 0));++ expect(open).not.toHaveBeenCalled();+ expect(query.outcome()).toBeUndefined();+ expect(build).toHaveBeenCalledTimes(1);+ });+it.each([true, false])('answers before the network only when cached rows exist (%s)',async (cacheHit) => {@@ -681,8 +709,8 @@ describe('a refresh another run replaced', () => {);const refreshed = query.refresh();- setSql("SELECT name FROM crm.deals WHERE name = 'Acme'");await vi.waitFor(() => expect(answerRefresh).toBeDefined());+ setSql("SELECT name FROM crm.deals WHERE name = 'Acme'");answerRefresh?.();expect(await refreshed).toEqual(ok({ landed: false }));diff --git a/apps/web/src/lib/queries/database-sql/create-database-sql-query.ts b/apps/web/src/lib/queries/database-sql/create-database-sql-query.tsindex 70689f258e..ad3a0e593f 100644--- a/apps/web/src/lib/queries/database-sql/create-database-sql-query.ts+++ b/apps/web/src/lib/queries/database-sql/create-database-sql-query.ts@@ -112,17 +112,20 @@ function runStatement(statement: DatabaseSqlStatement,source: RowSource,capabilities: Pick<DatabaseSqlQueryCapabilities, 'open' | 'openView'>,- context: DatabaseSqlReadContext+ context: DatabaseSqlReadContext,+ signal?: AbortSignal): ResultAsync<Outcome, DatabaseSqlFailure> {if (statement.view)return runDatabaseView(catalog, statement.view, {source,context,+ signal,...(capabilities.openView ? { open: capabilities.openView } : {}),});return runDatabaseSql(catalog, statement.sql, {source,context,+ signal,...(capabilities.open ? { open: capabilities.open } : {}),});}@@ -173,6 +176,7 @@ export function createDatabaseSqlQuery(const [error, setError] = createSignal<DatabaseSqlFailure>();const [loading, setLoading] = createSignal(false);let latest = 0;+ let activeRead: AbortController | undefined;let cancelFrame: (() => void) | undefined;// The cache may not hold what an in-flight network read will bring, so a// cache change waits for it instead of answering from older rows.@@ -189,6 +193,9 @@ export function createDatabaseSqlQuery(options: { reportFailure?: boolean; keepLoading?: boolean } = {}): ResultAsync<DatabaseSqlRun, DatabaseSqlFailure> => {const generation = ++latest;+ activeRead?.abort();+ const controller = new AbortController();+ activeRead = controller;const querySpan = Telemetry.span('database_sql.query');querySpan.setAttr('database_sql.read_reason', reason);querySpan.setAttr('database_sql.request_policy', requestPolicy);@@ -223,7 +230,8 @@ export function createDatabaseSqlQuery(scope: current.scope,requestPolicy,reportFailure: options.reportFailure,- }+ },+ controller.signal)).map((answer): DatabaseSqlRun => {@@ -280,6 +288,7 @@ export function createDatabaseSqlQuery();return result;} finally {+ if (activeRead === controller) activeRead = undefined;querySpan.end();}};@@ -325,6 +334,7 @@ export function createDatabaseSqlQuery(cancelFrame?.();baselines = new Map();latest += 1;+ activeRead?.abort();setError(undefined);setLoading(false);if (!current) {@@ -363,6 +373,7 @@ export function createDatabaseSqlQuery(onCleanup(() => {cancelFrame?.();latest += 1;+ activeRead?.abort();});return {diff --git a/apps/web/src/lib/queries/database-sql/graphql-source.ts b/apps/web/src/lib/queries/database-sql/graphql-source.tsindex d57f693333..1f284f7e54 100644--- a/apps/web/src/lib/queries/database-sql/graphql-source.ts+++ b/apps/web/src/lib/queries/database-sql/graphql-source.ts@@ -476,7 +476,7 @@ function tableRows(}/** A row's cells by property definition; an empty property is no cell. */-function rowCells(item: DatabaseRowItem): Record<string, Cell> {+export function rowCells(item: DatabaseRowItem): Record<string, Cell> {const cells: Record<string, Cell> = {};for (const property of item.properties) {const value = cell(property.value);diff --git a/apps/web/src/lib/service-clients/service-storage/database-view-rows.test.ts b/apps/web/src/lib/service-clients/service-storage/database-view-rows.test.tsnew file mode 100644index 0000000000..92c96985d3--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/database-view-rows.test.ts@@ -0,0 +1,221 @@+import { normalizedCacheExchange } from '@graphql-cache/exchange/normalized-cache-exchange';+import { createNoopCacheHost } from '@graphql-cache/host/noop-host';+import type { ReadResult } from '@graphql-cache/protocol';+import {+ CombinedError,+ createClient,+ type Exchange,+ type Operation,+ type OperationResult,+} from '@urql/core';+import { afterEach, describe, expect, it, vi } from 'vitest';+import { filter, fromPromise, map, mergeMap, pipe, takeUntil } from 'wonka';+import {+ type DatabaseViewPage,+ type DatabaseViewPageRequest,+ readDatabaseViewPage,+} from './database-view-rows';+import type { DatabaseViewRowsQuery } from './graphql/generated/graphql';++const platform = vi.hoisted(() => ({ native: false }));+vi.mock('@core/util/platform', async (actual) => ({+ ...(await actual<typeof import('@core/util/platform')>()),+ isTauri: () => platform.native,+}));+afterEach(() => {+ platform.native = false;+});++function deferred<T>() {+ let resolve!: (value: T) => void;+ const promise = new Promise<T>((done) => {+ resolve = done;+ });+ return { promise, resolve };+}++const page = (version: number): DatabaseViewPage => ({+ items: [],+ nextCursor: null,+ version,+});+const data = (version: number): DatabaseViewRowsQuery => ({+ user: { id: 'viewer', databaseViewRows: page(version) },+});++function harness() {+ const cached = deferred<ReadResult>();+ const network =+ deferred<Pick<OperationResult<DatabaseViewRowsQuery>, 'data' | 'error'>>();+ const host = {+ ...createNoopCacheHost(),+ disabled: false,+ readQuery: vi.fn(() => cached.promise),+ };+ const requests: Operation[] = [];+ const exchange: Exchange = () => (operations) =>+ pipe(+ operations,+ filter((operation) => operation.kind === 'query'),+ mergeMap((operation) => {+ requests.push(operation);+ return pipe(+ fromPromise(network.promise),+ map((result) => ({+ ...result,+ operation,+ stale: false,+ hasNext: false,+ })),+ takeUntil(+ pipe(+ operations,+ filter(+ (next) => next.kind === 'teardown' && next.key === operation.key+ )+ )+ )+ );+ })+ );+ const client = createClient({+ url: 'http://database.test/graphql',+ exchanges: [normalizedCacheExchange(host), exchange],+ });+ const onCachedPage = vi.fn();+ const input: DatabaseViewPageRequest = {+ databaseId: 'database',+ tableId: 'table',+ query: { filter: null, sort: [] },+ requestPolicy: 'cache-and-network',+ onCachedPage,+ };+ return { cached, network, host, requests, client, input, onCachedPage };+}++describe('database view pages with the real GraphQL client', () => {+ it('fetches one server page after a cache miss', async () => {+ const test = harness();+ const read = readDatabaseViewPage(test.input, test.client);+ test.cached.resolve({ kind: 'miss' });+ await vi.waitFor(() => expect(test.requests).toHaveLength(1));+ test.network.resolve({ data: data(2) });+ const result = await read;+ expect(result.isOk() && result.value).toEqual(page(2));+ expect(test.onCachedPage).not.toHaveBeenCalled();+ });++ it('sends a date range filter as the typed dateRange test', async () => {+ const test = harness();+ const read = readDatabaseViewPage(+ {+ ...test.input,+ query: {+ filter: {+ conjunction: 'and',+ conditions: [+ {+ kind: 'condition',+ column: 'stay',+ test: {+ kind: 'dateRange',+ operator: 'contains',+ value: '2026-01-04T00:00:00Z',+ },+ },+ ],+ },+ sort: [],+ },+ },+ test.client+ );+ test.cached.resolve({ kind: 'miss' });+ await vi.waitFor(() => expect(test.requests).toHaveLength(1));+ expect(test.requests[0].variables).toMatchObject({+ input: {+ query: {+ filter: {+ conjunction: 'AND',+ conditions: [+ {+ condition: {+ column: 'stay',+ test: {+ dateRange: {+ operator: 'CONTAINS',+ value: '2026-01-04T00:00:00Z',+ },+ },+ },+ },+ ],+ },+ },+ },+ });+ test.network.resolve({ data: data(2) });+ await read;+ });++ it('keeps view pages out of a native app cache', async () => {+ platform.native = true;+ const test = harness();+ const read = readDatabaseViewPage(test.input, test.client);+ await vi.waitFor(() => expect(test.requests).toHaveLength(1));+ expect(test.requests[0].context.requestPolicy).toBe('network-only');+ expect(test.requests[0].context.persistResult).toBe(false);+ test.network.resolve({ data: data(2) });+ const result = await read;+ expect(result.isOk() && result.value).toEqual(page(2));+ expect(test.host.readQuery).not.toHaveBeenCalled();+ expect(test.onCachedPage).not.toHaveBeenCalled();+ });++ it('publishes cached rows while awaiting the single revalidation', async () => {+ const test = harness();+ const read = readDatabaseViewPage(test.input, test.client);+ test.cached.resolve({ kind: 'hit', data: data(1) });+ await vi.waitFor(() =>+ expect(test.onCachedPage).toHaveBeenCalledWith(page(1))+ );+ expect(test.requests).toHaveLength(1);+ test.network.resolve({ data: data(2) });+ const result = await read;+ expect(result.isOk() && result.value).toEqual(page(2));+ });++ it('does not wait for a busy cache or publish its late stale rows', async () => {+ const test = harness();+ const read = readDatabaseViewPage(test.input, test.client);+ await vi.waitFor(() => expect(test.requests).toHaveLength(1));+ test.network.resolve({ data: data(2) });+ const result = await read;+ expect(result.isOk() && result.value).toEqual(page(2));+ test.cached.resolve({ kind: 'hit', data: data(1) });+ await Promise.resolve();+ expect(test.onCachedPage).not.toHaveBeenCalled();+ });++ it('can recover a delayed cached page after an early network error', async () => {+ const test = harness();+ const read = readDatabaseViewPage(test.input, test.client);+ await vi.waitFor(() => expect(test.requests).toHaveLength(1));+ test.network.resolve({+ error: new CombinedError({ networkError: new Error('Offline') }),+ });+ const failed = await read;+ expect(failed.isErr() && failed.error.message).toContain('Offline');+ const saved = readDatabaseViewPage(+ { ...test.input, requestPolicy: 'cache-only' },+ test.client+ );+ await vi.waitFor(() =>+ expect(test.host.readQuery).toHaveBeenCalledTimes(2)+ );+ test.cached.resolve({ kind: 'hit', data: data(1) });+ const result = await saved;+ expect(result.isOk() && result.value).toEqual(page(1));+ expect(test.requests).toHaveLength(1);+ });+});diff --git a/apps/web/src/lib/service-clients/service-storage/database-view-rows.ts b/apps/web/src/lib/service-clients/service-storage/database-view-rows.tsnew file mode 100644index 0000000000..d2b9178635--- /dev/null+++ b/apps/web/src/lib/service-clients/service-storage/database-view-rows.ts@@ -0,0 +1,238 @@+import type { ViewQuery } from '@core/database-sql/generated/types';+import { isTauri } from '@core/util/platform';+import type { Client, RequestPolicy } from '@urql/core';+import { err, errAsync, ok, type Result, ResultAsync } from 'neverthrow';+import { match } from 'ts-pattern';+import { filter, pipe, take, tap, toPromise } from 'wonka';+import {+ type DatabaseViewFilterGroup,+ type DatabaseViewFilterTest,+ type DatabaseViewQueryInput,+ DatabaseViewRowsDocument,+ type DatabaseViewRowsQuery,+} from './graphql/generated/graphql';+import { getGraphqlSoupClient } from './graphql-soup';++export type DatabaseViewPage =+ DatabaseViewRowsQuery['user']['databaseViewRows'];+export type DatabaseViewPageFailure = {+ kind: 'fetch';+ message: string;+ stale?: boolean;+ staleSchema?: boolean;+};+export type DatabaseViewPageRequest = {+ databaseId: string;+ tableId: string;+ query: ViewQuery;+ cursor?: string;+ requestPolicy: RequestPolicy;+ signal?: AbortSignal;+ headers?: Record<string, string>;+ /** A usable cached page while a cache-and-network request revalidates. */+ onCachedPage?: (page: DatabaseViewPage) => void;+};++function filterTest(+ test: Extract<+ NonNullable<ViewQuery['filter']>['conditions'][number],+ { kind: 'condition' }+ >['test']+): DatabaseViewFilterTest | undefined {+ return (+ match(test)+ .returnType<DatabaseViewFilterTest | undefined>()+ .with({ kind: 'presence' }, ({ operator }) => ({+ presence: {+ operator: operator === 'isEmpty' ? 'IS_EMPTY' : 'IS_NOT_EMPTY',+ },+ }))+ .with({ kind: 'text' }, ({ operator, value }) => ({+ text: {+ value,+ operator: (+ {+ is: 'IS',+ isNot: 'IS_NOT',+ contains: 'CONTAINS',+ doesNotContain: 'DOES_NOT_CONTAIN',+ startsWith: 'STARTS_WITH',+ endsWith: 'ENDS_WITH',+ } as const+ )[operator],+ },+ }))+ .with({ kind: 'number' }, ({ operator, value }) => ({+ number: {+ value,+ operator: (+ {+ is: 'IS',+ isNot: 'IS_NOT',+ greaterThan: 'GREATER_THAN',+ greaterThanOrEqual: 'GREATER_THAN_OR_EQUAL',+ lessThan: 'LESS_THAN',+ lessThanOrEqual: 'LESS_THAN_OR_EQUAL',+ } as const+ )[operator],+ },+ }))+ .with({ kind: 'date' }, ({ operator, value }) => ({+ date: {+ value,+ operator: (+ {+ before: 'BEFORE',+ after: 'AFTER',+ onOrBefore: 'ON_OR_BEFORE',+ onOrAfter: 'ON_OR_AFTER',+ } as const+ )[operator],+ },+ }))+ .with({ kind: 'dateRange' }, ({ operator, value }) => ({+ dateRange: {+ value,+ operator: (+ {+ contains: 'CONTAINS',+ doesNotContain: 'DOES_NOT_CONTAIN',+ endsBefore: 'ENDS_BEFORE',+ endsOnOrAfter: 'ENDS_ON_OR_AFTER',+ startsAfter: 'STARTS_AFTER',+ startsOnOrBefore: 'STARTS_ON_OR_BEFORE',+ } as const+ )[operator],+ },+ }))+ .with({ kind: 'checkbox' }, ({ checked }) => ({ checkbox: { checked } }))+ .with({ kind: 'options' }, ({ operator, options }) => ({+ options: {+ options,+ operator: (+ {+ isAnyOf: 'IS_ANY_OF',+ isNoneOf: 'IS_NONE_OF',+ hasAny: 'HAS_ANY',+ hasAll: 'HAS_ALL',+ hasNone: 'HAS_NONE',+ } as const+ )[operator],+ },+ }))+ .with({ kind: 'entities' }, ({ operator, entities }) => ({+ entities: {+ entities,+ operator: (+ {+ isAnyOf: 'IS_ANY_OF',+ isNoneOf: 'IS_NONE_OF',+ hasAny: 'HAS_ANY',+ hasAll: 'HAS_ALL',+ hasNone: 'HAS_NONE',+ } as const+ )[operator],+ },+ }))+ // A test this build can't send fails the read instead of filtering wrongly.+ .exhaustive(() => undefined)+ );+}++function filterGroup(+ group: NonNullable<ViewQuery['filter']>+): DatabaseViewFilterGroup | undefined {+ const conditions: DatabaseViewFilterGroup['conditions'] = [];+ for (const node of group.conditions) {+ if (node.kind === 'group') {+ const nested = filterGroup(node);+ if (!nested) return undefined;+ conditions.push({ group: nested });+ } else {+ const test = filterTest(node.test);+ if (!test) return undefined;+ conditions.push({ condition: { column: node.column, test } });+ }+ }+ return {+ conjunction: group.conjunction === 'and' ? 'AND' : 'OR',+ conditions,+ };+}++/** Transport conversion only; the server checks the query against the authorized schema. */+export function databaseViewQueryInput(+ query: ViewQuery+): Result<DatabaseViewQueryInput, DatabaseViewPageFailure> {+ const filter = query.filter ? filterGroup(query.filter) : null;+ if (filter === undefined)+ return err({+ kind: 'fetch',+ message: 'This view uses a filter this version of the app cannot send.',+ });+ return ok({+ filter,+ sort: (query.sort ?? []).map(({ column, direction }) => ({+ column,+ direction: direction === 'ascending' ? 'ASCENDING' : 'DESCENDING',+ })),+ });+}++export function readDatabaseViewPage(+ request: DatabaseViewPageRequest,+ client: Client = getGraphqlSoupClient()+): ResultAsync<DatabaseViewPage, DatabaseViewPageFailure> {+ const query = databaseViewQueryInput(request.query);+ if (query.isErr()) return errAsync(query.error);+ // Installed native apps run an older compiled cache that neither bounds these+ // pages nor skips them at startup, so the native cache never stores them.+ const native = isTauri();+ return ResultAsync.fromPromise(+ pipe(+ client.query(+ DatabaseViewRowsDocument,+ {+ databaseId: request.databaseId,+ input: {+ tableId: request.tableId,+ query: query.value,+ cursor: request.cursor,+ limit: 500,+ },+ },+ {+ requestPolicy: native ? 'network-only' : request.requestPolicy,+ persistResult: native ? false : undefined,+ fetchOptions: { signal: request.signal, headers: request.headers },+ }+ ),+ tap((result) => {+ if (result.stale && result.data && !result.error)+ request.onCachedPage?.(result.data.user.databaseViewRows);+ }),+ filter((result) => !result.stale && !result.hasNext),+ take(1),+ toPromise+ ),+ (error): DatabaseViewPageFailure => ({+ kind: 'fetch',+ message: error instanceof Error ? error.message : String(error),+ })+ ).andThen(+ ({ data, error }): Result<DatabaseViewPage, DatabaseViewPageFailure> => {+ if (error || !data)+ return err({+ kind: 'fetch',+ message: error?.message ?? 'This view is not cached.',+ stale: error?.graphQLErrors.some(+ (error) => error.extensions.code === 'DATABASE_VIEW_STALE'+ ),+ staleSchema: error?.graphQLErrors.some(+ (error) => error.extensions.code === 'DATABASE_VIEW_INVALID'+ ),+ });+ return ok(data.user.databaseViewRows);+ }+ );+}diff --git a/apps/web/src/lib/service-clients/service-storage/graphql/database-rows.graphql b/apps/web/src/lib/service-clients/service-storage/graphql/database-rows.graphqlindex 5aac006d98..142ad35f5b 100644--- a/apps/web/src/lib/service-clients/service-storage/graphql/database-rows.graphql+++ b/apps/web/src/lib/service-clients/service-storage/graphql/database-rows.graphql@@ -33,3 +33,15 @@ fragment DatabaseRowFields on GraphqlSoupEntity {}}}++# Global view membership and ordering are evaluated before this page is selected.+query DatabaseViewRows($databaseId: ID!, $input: DatabaseViewRowsInput!) {+ user {+ id+ databaseViewRows(databaseId: $databaseId, input: $input) {+ items { ...DatabaseRowFields }+ nextCursor+ version+ }+ }+}diff --git a/apps/web/src/lib/service-clients/service-storage/graphql/generated/graphql.ts b/apps/web/src/lib/service-clients/service-storage/graphql/generated/graphql.tsindex 0abd85ba32..5b97920485 100644--- a/apps/web/src/lib/service-clients/service-storage/graphql/generated/graphql.ts+++ b/apps/web/src/lib/service-clients/service-storage/graphql/generated/graphql.ts@@ -233,6 +233,226 @@ export type CreateSavedViewInput = {sharedTeamId?: string | number | null | undefined;};+/** Match the checked or unchecked state of a cell. */+export type DatabaseViewCheckboxTest = {+ /** Whether the cell must be checked. */+ checked: boolean;+};++/** How the conditions within a filter group combine. */+export type DatabaseViewConjunction =+ /** Require every condition. */+ | 'AND'+ /** Require at least one condition. */+ | 'OR';++/** An ordering comparison between instants. */+export type DatabaseViewDateOperator =+ /** The instant is after the comparison instant. */+ | 'AFTER'+ /** The instant is before the comparison instant. */+ | 'BEFORE'+ /** The instant is at or after the comparison instant. */+ | 'ON_OR_AFTER'+ /** The instant is at or before the comparison instant. */+ | 'ON_OR_BEFORE';++/** How a date range relates to a day. */+export type DatabaseViewDateRangeOperator =+ /** The range overlaps the day. */+ | 'CONTAINS'+ /** The range misses the day, or is empty. */+ | 'DOES_NOT_CONTAIN'+ /** The range ends before the day. */+ | 'ENDS_BEFORE'+ /** The range ends on or after the day. */+ | 'ENDS_ON_OR_AFTER'+ /** The range starts after the day. */+ | 'STARTS_AFTER'+ /** The range starts on or before the day. */+ | 'STARTS_ON_OR_BEFORE';++/** Compare a date range cell with the day starting at a UTC instant. */+export type DatabaseViewDateRangeTest = {+ /** The comparison to apply. */+ operator: DatabaseViewDateRangeOperator;+ /** The start of the day compared against. */+ value: unknown;+};++/** Compare a date cell with a UTC instant. */+export type DatabaseViewDateTest = {+ /** The comparison to apply. */+ operator: DatabaseViewDateOperator;+ /** The value to compare the cell against. */+ value: unknown;+};++/** Compare referenced entity identities with a set of entities. */+export type DatabaseViewEntitiesTest = {+ /** Referenced entity identities. */+ entities: Array<string>;+ /** The comparison to apply. */+ operator: DatabaseViewSetOperator;+};++/** A typed predicate on one column placement. */+export type DatabaseViewFilterCondition = {+ /** Column placement identity within the selected table. */+ column: string | number;+ /** The comparison appropriate for this column. */+ test: DatabaseViewFilterTest;+};++/** A conjunction of column conditions or nested groups. */+export type DatabaseViewFilterGroup = {+ /** Column tests or nested groups. */+ conditions: Array<DatabaseViewFilterNode>;+ /** Require every condition or at least one condition. */+ conjunction: DatabaseViewConjunction;+};++/** Exactly one column condition or nested filter group. */+export type DatabaseViewFilterNode =+ { /** A predicate on one column. */+ condition: DatabaseViewFilterCondition; group?: never; }+ | { condition?: never; /** A nested group of predicates. */+ group: DatabaseViewFilterGroup; };++/** Exactly one test compatible with the selected column type. */+export type DatabaseViewFilterTest =+ { /** Test a checkbox cell. */+ checkbox: DatabaseViewCheckboxTest; date?: never; dateRange?: never; entities?: never; number?: never; options?: never; presence?: never; text?: never; }+ | { checkbox?: never; /** Test a date cell. */+ date: DatabaseViewDateTest; dateRange?: never; entities?: never; number?: never; options?: never; presence?: never; text?: never; }+ | { checkbox?: never; date?: never; /** Test a date range cell against a day. */+ dateRange: DatabaseViewDateRangeTest; entities?: never; number?: never; options?: never; presence?: never; text?: never; }+ | { checkbox?: never; date?: never; dateRange?: never; /** Test referenced entity identities. */+ entities: DatabaseViewEntitiesTest; number?: never; options?: never; presence?: never; text?: never; }+ | { checkbox?: never; date?: never; dateRange?: never; entities?: never; /** Test a numeric cell. */+ number: DatabaseViewNumberTest; options?: never; presence?: never; text?: never; }+ | { checkbox?: never; date?: never; dateRange?: never; entities?: never; number?: never; /** Test selected option identities. */+ options: DatabaseViewOptionsTest; presence?: never; text?: never; }+ | { checkbox?: never; date?: never; dateRange?: never; entities?: never; number?: never; options?: never; /** Test cell emptiness. */+ presence: DatabaseViewPresenceTest; text?: never; }+ | { checkbox?: never; date?: never; dateRange?: never; entities?: never; number?: never; options?: never; presence?: never; /** Test a text or link cell. */+ text: DatabaseViewTextTest; };++/** A numeric equality or ordering comparison. */+export type DatabaseViewNumberOperator =+ /** The number is greater than the comparison value. */+ | 'GREATER_THAN'+ /** The number is at least the comparison value. */+ | 'GREATER_THAN_OR_EQUAL'+ /** The value equals the comparison value. */+ | 'IS'+ /** The value differs from the comparison value. */+ | 'IS_NOT'+ /** The number is less than the comparison value. */+ | 'LESS_THAN'+ /** The number is at most the comparison value. */+ | 'LESS_THAN_OR_EQUAL';++/** Compare a numeric cell with a finite number. */+export type DatabaseViewNumberTest = {+ /** The comparison to apply. */+ operator: DatabaseViewNumberOperator;+ /** The value to compare the cell against. */+ value: number;+};++/** Compare selected option identities with a set of options. */+export type DatabaseViewOptionsTest = {+ /** The comparison to apply. */+ operator: DatabaseViewSetOperator;+ /** Option identities from the selected column. */+ options: Array<string | number>;+};++/** The requested cell-presence condition. */+export type DatabaseViewPresenceOperator =+ /** The cell has no value. */+ | 'IS_EMPTY'+ /** The cell has a value. */+ | 'IS_NOT_EMPTY';++/** Test whether the cell has a value. */+export type DatabaseViewPresenceTest = {+ /** The comparison to apply. */+ operator: DatabaseViewPresenceOperator;+};++/** Filtering and ordering applied to the entire table before pagination. */+export type DatabaseViewQueryInput = {+ /** Optional filter; omission includes every row. */+ filter?: DatabaseViewFilterGroup | null | undefined;+ /** Ordering keys in priority order; omission uses manual row order. */+ sort?: Array<DatabaseViewSort>;+};++/** A table view and its optional continuation, with a bounded page size. */+export type DatabaseViewRowsInput = {+ /** The preceding page’s opaque cursor; omit for the first page. */+ cursor?: string | null | undefined;+ /** Page size from 1 through 500; defaults to 500. */+ limit?: number | null | undefined;+ /** The complete filtering and ordering definition. */+ query: DatabaseViewQueryInput;+ /** The table within the authorized database. */+ tableId: string | number;+};++/** A membership condition for single- or multi-valued cells. */+export type DatabaseViewSetOperator =+ /** The cell contains every supplied identity. */+ | 'HAS_ALL'+ /** The cell contains at least one supplied identity. */+ | 'HAS_ANY'+ /** The cell contains no supplied identity. */+ | 'HAS_NONE'+ /** The single-valued cell matches one supplied identity. */+ | 'IS_ANY_OF'+ /** The single-valued cell matches no supplied identity. */+ | 'IS_NONE_OF';++/** One ordering key; earlier keys take precedence. */+export type DatabaseViewSort = {+ /** Column placement identity within the selected table. */+ column: string | number;+ /** Ascending or descending value order. */+ direction: DatabaseViewSortDirection;+};++/** The direction of a view ordering key. */+export type DatabaseViewSortDirection =+ /** Order values from lowest to highest. */+ | 'ASCENDING'+ /** Order values from highest to lowest. */+ | 'DESCENDING';++/** A text equality or substring comparison. */+export type DatabaseViewTextOperator =+ /** The text contains the comparison string. */+ | 'CONTAINS'+ /** The text does not contain the comparison string. */+ | 'DOES_NOT_CONTAIN'+ /** The text ends with the comparison string. */+ | 'ENDS_WITH'+ /** The value equals the comparison value. */+ | 'IS'+ /** The value differs from the comparison value. */+ | 'IS_NOT'+ /** The text starts with the comparison string. */+ | 'STARTS_WITH';++/** Compare a text or link cell with a string. */+export type DatabaseViewTextTest = {+ /** The comparison to apply. */+ operator: DatabaseViewTextOperator;+ /** The value to compare the cell against. */+ value: string;+};+/** Delete an event, one occurrence, or an occurrence onward. */export type DeleteCalendarEventInput = {/**@@ -4605,6 +4825,23 @@ export type DatabaseRowFieldsFragment =| DatabaseRowFields_GraphqlSoupReminder_Fragment;+export type DatabaseViewRowsQueryVariables = Exact<{+ databaseId: string | number;+ input: DatabaseViewRowsInput;+}>;+++export type DatabaseViewRowsQuery = { user: { id: string, databaseViewRows: { nextCursor: string | null, version: number, items: Array<{ __typename: 'GraphqlSoupDatabaseRow', tableId: string, position: string, ownerId: string, createdAt: string, updatedAt: string, cacheProjection: string | null, id: string, properties: Array<{ id: string, propertyDefinitionId: string, value:+ | { __typename: 'GraphqlBooleanPropertyValue', boolValue: boolean }+ | { __typename: 'GraphqlDatePropertyValue', dateValue: string }+ | { __typename: 'GraphqlDateRangePropertyValue', rangeStart: string, rangeEnd: string }+ | { __typename: 'GraphqlEntityReferencePropertyValue', references: Array<{ entityId: string, entityType: GraphqlPropertyEntityType, specificMessageId: string | null }> }+ | { __typename: 'GraphqlLinkPropertyValue', urls: Array<string> }+ | { __typename: 'GraphqlNumberPropertyValue', numberValue: number }+ | { __typename: 'GraphqlSelectOptionPropertyValue', optionIds: Array<string> }+ | { __typename: 'GraphqlStringPropertyValue', stringValue: string }+ | null }>, notifications: Array<{ __typename: 'GraphqlNotification', id: string, eventType: string, entityType: GraphqlSoupEntityType, targetEntityType: GraphqlEntityType, entityId: string, sent: boolean, state: NotificationState, createdAt: string, viewedAt: string | null, updatedAt: string, senderId: string | null }> }> } } };+export type DeleteEntityPropertyMutationVariables = Exact<{entityType: GraphqlPropertyTargetEntityType;entityId: string;@@ -21988,6 +22225,7 @@ export const CrmRecordPipelineEntriesDocument = {"kind":"Document","definitions"export const RenameDatabaseDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"RenameDatabase"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"displayName"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"renameEntities"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"inputs"},"value":{"kind":"ListValue","values":[{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"entity"},"value":{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"type"},"value":{"kind":"EnumValue","value":"DATABASE"}},{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}]}},{"kind":"ObjectField","name":{"kind":"Name","value":"displayName"},"value":{"kind":"Variable","name":{"kind":"Name","value":"displayName"}}}]}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"results"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"DatabaseEntityMutationResultFields"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DatabaseEntityMutationResultFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlEntityMutationResult"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlMutationError"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"errorCode"}},{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]} as unknown as DocumentNode<RenameDatabaseMutation, RenameDatabaseMutationVariables>;export const TrashDatabaseDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"TrashDatabase"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"id"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"trashEntities"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"entities"},"value":{"kind":"ListValue","values":[{"kind":"ObjectValue","fields":[{"kind":"ObjectField","name":{"kind":"Name","value":"type"},"value":{"kind":"EnumValue","value":"DATABASE"}},{"kind":"ObjectField","name":{"kind":"Name","value":"id"},"value":{"kind":"Variable","name":{"kind":"Name","value":"id"}}}]}]}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"results"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"DatabaseEntityMutationResultFields"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DatabaseEntityMutationResultFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlEntityMutationResult"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlMutationError"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"errorCode"}},{"kind":"Field","name":{"kind":"Name","value":"message"}}]}}]}}]} as unknown as DocumentNode<TrashDatabaseMutation, TrashDatabaseMutationVariables>;export const DatabaseRowsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"DatabaseRows"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SoupInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"user"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"soup"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"items"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"DatabaseRowFields"}}]}},{"kind":"Field","name":{"kind":"Name","value":"nextCursor"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"SoupNotificationStateFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlNotification"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"eventType"}},{"kind":"Field","name":{"kind":"Name","value":"entityType"}},{"kind":"Field","name":{"kind":"Name","value":"targetEntityType"}},{"kind":"Field","name":{"kind":"Name","value":"entityId"}},{"kind":"Field","name":{"kind":"Name","value":"sent"}},{"kind":"Field","name":{"kind":"Name","value":"state"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"viewedAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"senderId"}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"SoupPropertyValueFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlBooleanPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"boolValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlNumberPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"numberValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlStringPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"stringValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlDatePropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"dateValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlDateRangePropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"rangeStart"},"name":{"kind":"Name","value":"start"}},{"kind":"Field","alias":{"kind":"Name","value":"rangeEnd"},"name":{"kind":"Name","value":"end"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlSelectOptionPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"optionIds"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlEntityReferencePropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"references"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"entityId"}},{"kind":"Field","name":{"kind":"Name","value":"entityType"}},{"kind":"Field","name":{"kind":"Name","value":"specificMessageId"}}]}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlLinkPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"urls"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DatabaseRowFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlSoupEntity"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"cacheProjection"},"directives":[{"kind":"Directive","name":{"kind":"Name","value":"cacheOnly"}}]},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"notifications"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"SoupNotificationStateFields"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlSoupDatabaseRow"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"tableId"}},{"kind":"Field","name":{"kind":"Name","value":"position"}},{"kind":"Field","name":{"kind":"Name","value":"ownerId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"properties"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"propertyDefinitionId"}},{"kind":"Field","name":{"kind":"Name","value":"value"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"SoupPropertyValueFields"}}]}}]}}]}}]}}]} as unknown as DocumentNode<DatabaseRowsQuery, DatabaseRowsQueryVariables>;+export const DatabaseViewRowsDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"query","name":{"kind":"Name","value":"DatabaseViewRows"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"databaseId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"DatabaseViewRowsInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"user"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"databaseViewRows"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"databaseId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"databaseId"}}},{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"items"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"DatabaseRowFields"}}]}},{"kind":"Field","name":{"kind":"Name","value":"nextCursor"}},{"kind":"Field","name":{"kind":"Name","value":"version"}}]}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"SoupNotificationStateFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlNotification"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"eventType"}},{"kind":"Field","name":{"kind":"Name","value":"entityType"}},{"kind":"Field","name":{"kind":"Name","value":"targetEntityType"}},{"kind":"Field","name":{"kind":"Name","value":"entityId"}},{"kind":"Field","name":{"kind":"Name","value":"sent"}},{"kind":"Field","name":{"kind":"Name","value":"state"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"viewedAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"senderId"}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"SoupPropertyValueFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlBooleanPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"boolValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlNumberPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"numberValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlStringPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"stringValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlDatePropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"dateValue"},"name":{"kind":"Name","value":"value"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlDateRangePropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","alias":{"kind":"Name","value":"rangeStart"},"name":{"kind":"Name","value":"start"}},{"kind":"Field","alias":{"kind":"Name","value":"rangeEnd"},"name":{"kind":"Name","value":"end"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlSelectOptionPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"optionIds"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlEntityReferencePropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"references"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"entityId"}},{"kind":"Field","name":{"kind":"Name","value":"entityType"}},{"kind":"Field","name":{"kind":"Name","value":"specificMessageId"}}]}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlLinkPropertyValue"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"urls"}}]}}]}},{"kind":"FragmentDefinition","name":{"kind":"Name","value":"DatabaseRowFields"},"typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlSoupEntity"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"cacheProjection"},"directives":[{"kind":"Directive","name":{"kind":"Name","value":"cacheOnly"}}]},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"notifications"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"SoupNotificationStateFields"}}]}},{"kind":"InlineFragment","typeCondition":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlSoupDatabaseRow"}},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"tableId"}},{"kind":"Field","name":{"kind":"Name","value":"position"}},{"kind":"Field","name":{"kind":"Name","value":"ownerId"}},{"kind":"Field","name":{"kind":"Name","value":"createdAt"}},{"kind":"Field","name":{"kind":"Name","value":"updatedAt"}},{"kind":"Field","name":{"kind":"Name","value":"properties"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"propertyDefinitionId"}},{"kind":"Field","name":{"kind":"Name","value":"value"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"FragmentSpread","name":{"kind":"Name","value":"SoupPropertyValueFields"}}]}}]}}]}}]}}]} as unknown as DocumentNode<DatabaseViewRowsQuery, DatabaseViewRowsQueryVariables>;export const DeleteEntityPropertyDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"DeleteEntityProperty"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"entityType"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"GraphqlPropertyTargetEntityType"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"entityId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"String"}}}},{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"entityPropertyId"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"ID"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"deleteEntityProperty"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"entityType"},"value":{"kind":"Variable","name":{"kind":"Name","value":"entityType"}}},{"kind":"Argument","name":{"kind":"Name","value":"entityId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"entityId"}}},{"kind":"Argument","name":{"kind":"Name","value":"entityPropertyId"},"value":{"kind":"Variable","name":{"kind":"Name","value":"entityPropertyId"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"graphqlTypeName"}},{"kind":"Field","name":{"kind":"Name","value":"entityId"}}]}}]}}]} as unknown as DocumentNode<DeleteEntityPropertyMutation, DeleteEntityPropertyMutationVariables>;export const SetEmailThreadArchivedDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"SetEmailThreadArchived"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SetEmailThreadArchivedInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"setEmailThreadArchived"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"inboxVisible"}}]}}]}}]} as unknown as DocumentNode<SetEmailThreadArchivedMutation, SetEmailThreadArchivedMutationVariables>;export const SetEmailThreadsArchivedDocument = {"kind":"Document","definitions":[{"kind":"OperationDefinition","operation":"mutation","name":{"kind":"Name","value":"SetEmailThreadsArchived"},"variableDefinitions":[{"kind":"VariableDefinition","variable":{"kind":"Variable","name":{"kind":"Name","value":"input"}},"type":{"kind":"NonNullType","type":{"kind":"NamedType","name":{"kind":"Name","value":"SetEmailThreadsArchivedInput"}}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"setEmailThreadsArchived"},"arguments":[{"kind":"Argument","name":{"kind":"Name","value":"input"},"value":{"kind":"Variable","name":{"kind":"Name","value":"input"}}}],"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"threads"},"selectionSet":{"kind":"SelectionSet","selections":[{"kind":"Field","name":{"kind":"Name","value":"__typename"}},{"kind":"Field","name":{"kind":"Name","value":"id"}},{"kind":"Field","name":{"kind":"Name","value":"inboxVisible"}}]}},{"kind":"Field","name":{"kind":"Name","value":"failedThreadIds"}}]}}]}}]} as unknown as DocumentNode<SetEmailThreadsArchivedMutation, SetEmailThreadsArchivedMutationVariables>;diff --git a/apps/web/tauri/src-tauri/gen/apple/NotificationServiceExtension/Info.plist b/apps/web/tauri/src-tauri/gen/apple/NotificationServiceExtension/Info.plistindex d97d52d351..570fb888b4 100644--- a/apps/web/tauri/src-tauri/gen/apple/NotificationServiceExtension/Info.plist+++ b/apps/web/tauri/src-tauri/gen/apple/NotificationServiceExtension/Info.plist@@ -17,9 +17,9 @@<key>CFBundlePackageType</key><string>XPC!</string><key>CFBundleShortVersionString</key>- <string>2.0.15</string>+ <string>2.0.16</string><key>CFBundleVersion</key>- <string>194</string>+ <string>195</string><key>NSExtension</key><dict><key>NSExtensionPointIdentifier</key>diff --git a/apps/web/tauri/src-tauri/gen/apple/ShareExtension/Info.plist b/apps/web/tauri/src-tauri/gen/apple/ShareExtension/Info.plistindex cb91796768..4ce7de0ec0 100644--- a/apps/web/tauri/src-tauri/gen/apple/ShareExtension/Info.plist+++ b/apps/web/tauri/src-tauri/gen/apple/ShareExtension/Info.plist@@ -17,9 +17,9 @@<key>CFBundlePackageType</key><string>XPC!</string><key>CFBundleShortVersionString</key>- <string>2.0.15</string>+ <string>2.0.16</string><key>CFBundleVersion</key>- <string>194</string>+ <string>195</string><key>LSApplicationQueriesSchemes</key><array><string>macro</string>diff --git a/apps/web/tauri/src-tauri/gen/apple/app_iOS/Info.plist b/apps/web/tauri/src-tauri/gen/apple/app_iOS/Info.plistindex b78dc39329..ab12ad3640 100644--- a/apps/web/tauri/src-tauri/gen/apple/app_iOS/Info.plist+++ b/apps/web/tauri/src-tauri/gen/apple/app_iOS/Info.plist@@ -15,7 +15,7 @@<key>CFBundlePackageType</key><string>APPL</string><key>CFBundleShortVersionString</key>- <string>2.0.15</string>+ <string>2.0.16</string><key>CFBundleURLTypes</key><array><dict>@@ -28,7 +28,7 @@</dict></array><key>CFBundleVersion</key>- <string>194</string>+ <string>195</string><key>ITSAppUsesNonExemptEncryption</key><false/><key>LSApplicationQueriesSchemes</key>diff --git a/apps/web/tauri/src-tauri/gen/apple/project.yml b/apps/web/tauri/src-tauri/gen/apple/project.ymlindex d2bce3f603..f708822e44 100644--- a/apps/web/tauri/src-tauri/gen/apple/project.yml+++ b/apps/web/tauri/src-tauri/gen/apple/project.yml@@ -58,8 +58,8 @@ targets:- UIInterfaceOrientationPortraitUpsideDown- UIInterfaceOrientationLandscapeLeft- UIInterfaceOrientationLandscapeRight- CFBundleShortVersionString: 2.0.15- CFBundleVersion: "194"+ CFBundleShortVersionString: 2.0.16+ CFBundleVersion: "195"# Exempt encryption only (HTTPS / OS crypto): answers App Store# Connect's export-compliance questionnaire for every build. Keep in# sync with src-tauri/Info.ios.plist.@@ -145,8 +145,8 @@ targets:path: ShareExtension/Info.plistproperties:CFBundleDisplayName: Share to Macro- CFBundleShortVersionString: 2.0.15- CFBundleVersion: "194"+ CFBundleShortVersionString: 2.0.16+ CFBundleVersion: "195"LSApplicationQueriesSchemes:- macroNSExtension:@@ -180,8 +180,8 @@ targets:path: NotificationServiceExtension/Info.plistproperties:CFBundleDisplayName: NotificationServiceExtension- CFBundleShortVersionString: 2.0.15- CFBundleVersion: "194"+ CFBundleShortVersionString: 2.0.16+ CFBundleVersion: "195"NSExtension:NSExtensionPointIdentifier: com.apple.usernotifications.serviceNSExtensionPrincipalClass: $(PRODUCT_MODULE_NAME).NotificationServicediff --git a/apps/web/tauri/src-tauri/tauri.android.conf.json b/apps/web/tauri/src-tauri/tauri.android.conf.jsonindex 77c89ee1a8..a20753eea3 100644--- a/apps/web/tauri/src-tauri/tauri.android.conf.json+++ b/apps/web/tauri/src-tauri/tauri.android.conf.json@@ -2,7 +2,7 @@"version": "2.5.0","bundle": {"android": {- "versionCode": 20+ "versionCode": 21}},"$schema": "https://schema.tauri.app/config/2",diff --git a/apps/web/tauri/src-tauri/tauri.ios.conf.json b/apps/web/tauri/src-tauri/tauri.ios.conf.jsonindex 7da9452478..732d7d8ac4 100644--- a/apps/web/tauri/src-tauri/tauri.ios.conf.json+++ b/apps/web/tauri/src-tauri/tauri.ios.conf.json@@ -1,9 +1,9 @@{"productName": "macro",- "version": "2.0.15",+ "version": "2.0.16","bundle": {"iOS": {- "bundleVersion": "194"+ "bundleVersion": "195"}}}diff --git a/crates/client/README.md b/crates/client/README.mdindex cb1e5e3996..6658aa51c0 100644--- a/crates/client/README.md+++ b/crates/client/README.md@@ -109,12 +109,16 @@ pages), identity, email links and filter/search projections, but not the viewer'`soup(...)` / `groupSoup(...)` page wrappers. `@cacheOnly` still controls thereturned cursor projection; it is not an entity-eviction directive.-Foreground Soup snapshots share a per-viewer budget of 64 pages and 512 KiB+Foreground Soup snapshots share a per-viewer budget of 64 pages and 2 MiB(encoded fields plus retention metadata). Most recently written pages win; readsdo not persist recency or cause a COMMIT. Individually oversized snapshots arenot retained. Evicted snapshots miss and require a network refetch; their entityrecords and pending offline mutations remain available.+The byte budget accommodates a capped 20,000-row database read at 500 rows per+page, including filter-bearing cursors. A smaller budget evicted the beginning+of a table while loading its end, forcing warm reads back onto the network.+On the first compatible Turso open after this upgrade, a metadata-versionedtransaction compacts only `GraphqlUser` records via the type/id index. It prefersinitial pages when legacy data has no recency ordering. It preserves the storagediff --git a/crates/client/cache-core/src/page_retention.rs b/crates/client/cache-core/src/page_retention.rsindex 87df784806..308069d2f5 100644--- a/crates/client/cache-core/src/page_retention.rs+++ b/crates/client/cache-core/src/page_retention.rs@@ -6,17 +6,21 @@ use crate::normalize::RecordUpdates;use crate::value::{CacheValue, Record};use std::collections::BTreeSet;-/// Maximum combined flat/grouped Soup snapshots retained on one viewer.+/// Maximum combined Soup and database view snapshots retained on one viewer.pub const MAX_SOUP_PAGES: usize = 64;-/// Encoded budget for page fields and their write-recency metadata.-pub const MAX_SOUP_PAGE_BYTES: usize = 512 * 1024;+/// Encoded budget for page fields and their write-recency metadata. Fits a+/// 20,000-row database read, including 500-row pages and filter-bearing cursors.+pub const MAX_SOUP_PAGE_BYTES: usize = 2 * 1024 * 1024;/// Viewer type owning the disposable query entry points.pub const SOUP_PAGE_OWNER: &str = "GraphqlUser";pub(crate) const PAGE_ORDER_FIELD: &str = "__cache_soup_page_order_v1";const ORDER_OVERHEAD_BYTES: usize = PAGE_ORDER_FIELD.len() + 32;fn is_page_field(key: &str) -> bool {- matches!(key.split('(').next(), Some("soup" | "groupSoup"))+ matches!(+ key.split('(').next(),+ Some("soup" | "groupSoup" | "databaseViewRows")+ )}/// Remove only the viewer's page wrappers from hydration updates. Normalized@@ -69,7 +73,11 @@ pub(crate) fn retain_soup_pages(record: &mut Record, updated: &[String]) -> bool})).chain(legacy()- .filter(|key| key.contains("\"initial\":"))+ .filter(|key| {+ key.contains("\"initial\":")+ || (key.starts_with("databaseViewRows(")+ && (!key.contains("\"cursor\":") || key.contains("\"cursor\":null")))+ }).map(String::as_str),).chain(legacy().map(String::as_str));diff --git a/crates/client/cache-core/src/page_retention/test.rs b/crates/client/cache-core/src/page_retention/test.rsindex 847843f005..fe26316203 100644--- a/crates/client/cache-core/src/page_retention/test.rs+++ b/crates/client/cache-core/src/page_retention/test.rs@@ -72,7 +72,7 @@ fn combined_flat_and_grouped_pages_obey_encoded_byte_budget() {for n in 0..MAX_SOUP_PAGES {record.fields.insert(format!("groupSoup({n})"),- CacheValue::String("x".repeat(32 * 1024)),+ CacheValue::String("x".repeat(64 * 1024)),);}record.fields.insert(key(0), page());@@ -150,3 +150,30 @@ fn hydration_drops_only_viewer_pages_not_children_or_other_relations() {child);}++#[test]+fn database_view_pages_share_retention_and_hydration_policy() {+ let initial = r#"databaseViewRows({"databaseId":"db","input":{"tableId":"table","query":{"sort":[]},"limit":500}})"#;+ let mut record = viewer();+ for cursor in 0..MAX_SOUP_PAGES * 2 {+ let mut update = viewer();+ update.fields.insert(format!(r#"databaseViewRows({{"databaseId":"db","input":{{"tableId":"table","query":{{"sort":[]}},"cursor":"{cursor:04}","limit":500}}}})"#), page());+ record.merge(update);+ }+ let mut update = viewer();+ update.fields.insert(initial.into(), page());+ record.merge(update);+ assert_eq!(+ record+ .fields+ .keys()+ .filter(|key| key.starts_with("databaseViewRows("))+ .count(),+ 64+ );+ assert!(record.fields.contains_key(initial));+ let viewer_key = EntityKey("GraphqlUser:viewer".into());+ let mut updates = [(viewer_key.clone(), record)].into();+ omit_hydration_pages(&mut updates);+ assert_eq!(updates[&viewer_key], viewer());+}diff --git a/crates/client/cache-core/tests/page_retention.rs b/crates/client/cache-core/tests/page_retention.rsindex 6b292ad83b..7e4313c772 100644--- a/crates/client/cache-core/tests/page_retention.rs+++ b/crates/client/cache-core/tests/page_retention.rs@@ -130,3 +130,62 @@ fn foreground_page_eviction_keeps_entities_and_makes_evicted_queries_miss() {);});}++#[test]+fn capped_database_read_remains_available_after_reopen() {+ block_on(async {+ let mut engine = Engine::new(InMemoryStorage::new());+ let query = "query Rows($input: SoupInput!) { user { id soup(input: $input) { items { __typename id } nextCursor } } }";+ // Forty full 500-row pages plus the terminal page. Real Soup cursors+ // also carry the filter, so include that overhead in the fixture.+ let page_variables = |page: usize| {+ json!({"input":{"continuation":{"cursor":format!("{page}:{}", "x".repeat(2048))}}})+ .as_object()+ .unwrap()+ .clone()+ };+ for page in 0..=40 {+ let items: Vec<Value> = if page == 40 {+ Vec::new()+ } else {+ (0..500)+ .map(|row| {+ json!({+ "__typename":"GraphqlSoupDatabaseRow",+ "id":format!("00000000-0000-7000-8000-{:012}", page * 500 + row)+ })+ })+ .collect()+ };+ let data = json!({"user":{"id":"viewer","soup":{+ "items":items,+ "nextCursor":if page == 40 { None } else { Some(format!("{}:{}", page + 1, "x".repeat(2048))) }+ }}});+ engine+ .write_query(+ None,+ query,+ Some("Rows"),+ &page_variables(page),+ &data,+ None,+ )+ .await+ .unwrap();+ }+ let mut reopened = Engine::new(engine.into_storage());+ for page in [0, 19, 39, 40] {+ let result = reopened+ .read_query(None, query, Some("Rows"), &page_variables(page))+ .await+ .unwrap();+ let ReadResult::Hit { data, .. } = result else {+ panic!("database page {page} was evicted");+ };+ assert_eq!(+ data["user"]["soup"]["items"].as_array().unwrap().len(),+ if page == 40 { 0 } else { 500 }+ );+ }+ });+}diff --git a/crates/client/cache-wasm/Cargo.toml b/crates/client/cache-wasm/Cargo.tomlindex fad64ce060..310691c96a 100644--- a/crates/client/cache-wasm/Cargo.toml+++ b/crates/client/cache-wasm/Cargo.toml@@ -2,8 +2,8 @@edition = "2024"name = "cache-wasm"publish = false-# Rebuild with spam-free All Mail projections and per-tab cache membership.-version = "0.6.53"+# Rebuild with paged database view reads retained in the cache.+version = "0.6.54"[features]default = []diff --git a/crates/complete_graph/Cargo.toml b/crates/complete_graph/Cargo.tomlindex 1242ad7c91..d28c8d5cf9 100644--- a/crates/complete_graph/Cargo.toml+++ b/crates/complete_graph/Cargo.toml@@ -33,6 +33,7 @@ graphql_email = { path = "../graphql_email" }graphql_entity_mutation = { path = "../graphql_entity_mutation" }graphql_favorite = { path = "../graphql_favorite" }graphql_initiative = { path = "../graphql_initiative" }+graphql_databases = { path = "../graphql_databases" }graphql_notification = { path = "../graphql_notification" }graphql_permission = { path = "../graphql_permission" }graphql_properties = { path = "../graphql_properties" }@@ -64,6 +65,8 @@ saved_views = { path = "../saved_views" }bot_id = { path = "../bot_id" }crm = { path = "../crm", default-features = false, features = ["ports"] }databases = { path = "../databases", default-features = false, features = ["ports"] }+databases_sql = { path = "../databases_sql" }+models_databases = { path = "../models_databases" }activity = { path = "../activity", default-features = false, features = ["realtime"] }work_feed = { path = "../work_feed" }agent_runtime_protocol = { path = "../agent_runtime_protocol" }diff --git a/crates/complete_graph/src/schema.rs b/crates/complete_graph/src/schema.rsindex b49bbf27dd..cfc2e6a332 100644--- a/crates/complete_graph/src/schema.rs+++ b/crates/complete_graph/src/schema.rs@@ -727,6 +727,18 @@ whereasync_graphql::ID(self.user_id.to_string())}+ /// A page filtered and ordered across the entire authorized table.+ async fn database_view_rows(+ &self,+ ctx: &Context<'_>,+ database_id: ID,+ input: graphql_databases::DatabaseViewRowsInput,+ ) -> async_graphql::Result<+ graphql_databases::DatabaseViewRowsPage<SoupEdges<NR, PR, ER, FR, AR, AcR>>,+ > {+ graphql_databases::resolve_database_view_rows(ctx, database_id, input).await+ }+/// One initiative accessible to the authenticated viewer.async fn initiative(&self,diff --git a/crates/complete_graph/src/schema/test.rs b/crates/complete_graph/src/schema/test.rsindex 6f5c38e1a0..1736ccf433 100644--- a/crates/complete_graph/src/schema/test.rs+++ b/crates/complete_graph/src/schema/test.rs@@ -49,6 +49,7 @@ mod calendar;mod crm_record;mod database_activity;mod database_row;+mod database_view_rows;mod email_archive;mod form_activity;mod initiative;diff --git a/crates/complete_graph/src/schema/test/database_view_rows.rs b/crates/complete_graph/src/schema/test/database_view_rows.rsnew file mode 100644index 0000000000..6e3c729ce9--- /dev/null+++ b/crates/complete_graph/src/schema/test/database_view_rows.rs@@ -0,0 +1,147 @@+use databases_sql::view_rows::{ViewPageError, ViewRowsPage, ViewRowsRequest, ViewRowsService};+use models_databases::{+ RowId, TableVersion,+ views::{DateRangeOperator, FilterNode, FilterTest, NumberOperator, SortDirection},+};+use models_soup::database_row::SoupDatabaseRow;++use super::*;++struct ViewRows(Arc<AtomicUsize>);++impl ViewRowsService for ViewRows {+ async fn page(+ &self,+ receipt: EntityAccessReceipt<ViewAccessLevel>,+ request: ViewRowsRequest,+ ) -> Result<ViewRowsPage, ViewPageError> {+ self.0.fetch_add(1, Ordering::SeqCst);+ assert_eq!(+ receipt.entity().entity_id,+ database_activity::VIEWABLE_DATABASE_ID+ );+ assert_eq!(request.limit, 500);+ assert_eq!(request.query.sort[0].direction, SortDirection::Descending);+ let FilterNode::Condition(condition) =+ &request.query.filter.as_ref().unwrap().conditions[0]+ else {+ panic!("expected a condition")+ };+ assert_eq!(+ condition.test,+ FilterTest::Number {+ operator: NumberOperator::GreaterThan,+ value: 25.0+ }+ );+ let FilterNode::Condition(condition) =+ &request.query.filter.as_ref().unwrap().conditions[1]+ else {+ panic!("expected a condition")+ };+ assert_eq!(+ condition.test,+ FilterTest::DateRange {+ operator: DateRangeOperator::Contains,+ value: "2026-01-04T00:00:00Z".parse().unwrap()+ }+ );+ Ok(ViewRowsPage {+ rows: vec![+ RowId::from_uuid(Uuid::from_u128(2)),+ RowId::from_uuid(Uuid::from_u128(1)),+ ],+ next_cursor: Some("next-page".into()),+ version: TableVersion(7),+ })+ }+}++fn query(database: &str) -> String {+ format!(+ r#"{{ user {{ id databaseViewRows(databaseId: "{database}", input: {{tableId: "00000000-0000-0000-0000-000000000003", query: {{filter: {{conjunction: AND, conditions: [{{condition: {{column: "00000000-0000-0000-0000-000000000004", test: {{number: {{operator: GREATER_THAN, value: 25}}}}}}}}, {{condition: {{column: "00000000-0000-0000-0000-000000000005", test: {{dateRange: {{operator: CONTAINS, value: "2026-01-04T00:00:00Z"}}}}}}}}]}}, sort: [{{column: "00000000-0000-0000-0000-000000000004", direction: DESCENDING}}]}}}}) {{ items {{ __typename id tableId }} nextCursor version }} }} }}"#+ )+}++async fn execute(+ harness: &TestHarness,+ database: &str,+ calls: Arc<AtomicUsize>,+) -> async_graphql::Response {+ let context = graphql_databases::DatabaseRowsGraphqlContext::new(+ ViewRows(calls),+ harness.state.entity_access.clone(),+ graphql_soup::soup_item_loader(+ harness.soup_service.clone(),+ Arc::new(harness.email_service.clone()),+ harness.state.entity_access.as_ref().clone(),+ ),+ );+ harness+ .schema+ .execute(+ harness+ .request(&query(database), authenticated_parts())+ .data(MacroUserIdStr::parse_from_str(VALID_USER_ID).unwrap())+ .data(context),+ )+ .await+}++#[tokio::test]+async fn view_pages_return_shared_row_entities_in_the_service_order() {+ let harness = harness();+ harness.soup_service.set_raw_response(+ [1, 2]+ .into_iter()+ .map(|id| {+ SoupItem::DatabaseRow(SoupDatabaseRow {+ id: Uuid::from_u128(id),+ table_id: Uuid::from_u128(3),+ database_id: database_activity::VIEWABLE_DATABASE_ID.parse().unwrap(),+ position: id.to_string(),+ owner_id: Owner::from_principal_str(VALID_USER_ID).unwrap(),+ created_by: None,+ created_at: "2026-01-01T00:00:00Z".parse().unwrap(),+ updated_at: "2026-01-01T00:00:00Z".parse().unwrap(),+ extra: (),+ })+ })+ .collect(),+ );+ let calls = Arc::new(AtomicUsize::new(0));+ let response = execute(+ &harness,+ database_activity::VIEWABLE_DATABASE_ID,+ calls.clone(),+ )+ .await;+ assert!(response.errors.is_empty(), "{:?}", response.errors);+ assert_eq!(+ response.data,+ async_graphql::value!({ "user": { "id": VALID_USER_ID, "databaseViewRows": {+ "items": [+ { "__typename": "GraphqlSoupDatabaseRow", "id": "00000000-0000-0000-0000-000000000002", "tableId": "00000000-0000-0000-0000-000000000003" },+ { "__typename": "GraphqlSoupDatabaseRow", "id": "00000000-0000-0000-0000-000000000001", "tableId": "00000000-0000-0000-0000-000000000003" }+ ], "nextCursor": "next-page", "version": 7+ } } })+ );+ assert_eq!(calls.load(Ordering::SeqCst), 1);+ assert_eq!(harness.raw_soup_calls.load(Ordering::SeqCst), 1);+}++#[tokio::test]+async fn a_denied_database_never_reaches_the_view_reader() {+ let harness = harness();+ let calls = Arc::new(AtomicUsize::new(0));+ let response = execute(+ &harness,+ "0199a000-0000-7000-8000-00000000d1d1",+ calls.clone(),+ )+ .await;+ assert_eq!(response.errors.len(), 1);+ assert_eq!(response.errors[0].message, "not found");+ assert_eq!(calls.load(Ordering::SeqCst), 0);+ assert_eq!(harness.raw_soup_calls.load(Ordering::SeqCst), 0);+}diff --git a/crates/database_sql/src/fold.rs b/crates/database_sql/src/fold.rsindex 637163c496..31e8119fc6 100644--- a/crates/database_sql/src/fold.rs+++ b/crates/database_sql/src/fold.rs@@ -19,14 +19,13 @@ use std::collections::{HashMap, HashSet};use chrono::{DateTime, Utc};use models_databases::position::Position;-use models_databases::{DateRange, Formula, OptionId, RowId};+use models_databases::{DateRange, OptionId, RowId};use serde::{Deserialize, Serialize};use specta::Type;use uuid::Uuid;use crate::catalog::Catalog;-use crate::formula;-use crate::resolve::{AggregateFunction, OrderKey, SelectItem, binding, column_key};+use crate::resolve::{AggregateFunction, OrderKey, SelectItem};use crate::run::RunError;use crate::split::{Plan, Shape};@@ -129,43 +128,11 @@ pub type Table = Vec<Vec<Option<Cell>>>;pub fn fold_relations(catalog: &Catalog,plan: &Plan,- mut fetched: Vec<Vec<Row>>,+ fetched: Vec<Vec<Row>>,) -> (Table, Vec<RowId>) {- for (index, rows) in fetched.iter_mut().enumerate() {- derive(catalog, plan, index, rows);- }fold_joined(catalog, plan, join::join(plan, fetched))}-/// Give the rows of relation `index` the cells of its table's derived-/// columns the plan reads, computed from their other cells.-fn derive(catalog: &Catalog, plan: &Plan, index: usize, rows: &mut [Row]) {- let Some(table) = plan.relations.get(index).and_then(|relation| {- catalog- .tables- .iter()- .find(|table| table.id == relation.relation.table)- }) else {- return;- };- let key = |definition: Uuid| column_key(index, definition);- let derived: Vec<(Uuid, &Formula)> = formula::derived(table)- .map(|(column, formula)| (key(column.id), formula))- .filter(|(derived_key, _)| binding(&plan.bindings, *derived_key).is_some())- .collect();- if derived.is_empty() {- return;- }- for row in rows {- for (derived_key, formula) in &derived {- match formula::evaluate(table, formula, &row.cells, key) {- Some(cell) => row.cells.insert(*derived_key, cell),- None => row.cells.remove(derived_key),- };- }- }-}-/// Finish a plan over rows that are already joined (or come from one/// relation).#[cfg(test)]diff --git a/crates/database_sql/src/lib.rs b/crates/database_sql/src/lib.rsindex 3ae7f79d84..86a471a28a 100644--- a/crates/database_sql/src/lib.rs+++ b/crates/database_sql/src/lib.rs@@ -44,4 +44,4 @@ pub use split::{GqlQuery, JoinPlan, KeyHint, MAX_KEY_HINT_VALUES, Plan, Propf, PropfLiteral, PropfValue,RelationPlan, Shape, split,};-pub use view::{Board, BoardLane, board, compile_view};+pub use view::{Board, BoardLane, board, compile_table_query, compile_view};diff --git a/crates/database_sql/src/run/test.rs b/crates/database_sql/src/run/test.rsindex e433d3c0ef..66fb687c95 100644--- a/crates/database_sql/src/run/test.rs+++ b/crates/database_sql/src/run/test.rs@@ -901,8 +901,24 @@ const DOUBLED: Uuid = Uuid::from_u128(0x0d);const FOLLOW_UP: Uuid = Uuid::from_u128(0x0e);#[test]-fn derived_columns_are_computed_filtered_and_sorted_in_the_fold() {- let source = source(deals());+fn derived_columns_read_their_stored_cells_like_any_other() {+ // The writer stores formula cells; the fold reads them as it reads any+ // cell, and never computes them.+ let mut rows = deals();+ for row in &mut rows {+ let stored = match row.id {+ ACME => vec![(DOUBLED, Cell::Number(24000.0))],+ GLOBEX => vec![(DOUBLED, Cell::Number(6000.0))],+ HOOLI => vec![(+ FOLLOW_UP,+ Cell::Date(Utc.with_ymd_and_hms(2026, 9, 22, 0, 0, 0).unwrap()),+ )],+ INITECH => vec![(DOUBLED, Cell::Number(14000.0))],+ _ => vec![],+ };+ row.cells.extend(stored);+ }+ let source = source(rows);let sink = FakeSink::new();let outcome = pollster::block_on(run(diff --git a/crates/database_sql/src/view.rs b/crates/database_sql/src/view.rsindex 91ed364d8c..b741e2813d 100644--- a/crates/database_sql/src/view.rs+++ b/crates/database_sql/src/view.rs@@ -11,7 +11,7 @@ use models_databases::views::{DatabaseView, SchemaColumn, ViewProblem, check};use crate::catalog::{Catalog, Column, ColumnKind, Table};pub use board::{Board, BoardLane, board};-pub use compile::compile_view;+pub use compile::{compile_table_query, compile_view};/// The view's table, once the view checks out against it.fn checked_table<'catalog>(diff --git a/crates/database_sql/src/view/compile.rs b/crates/database_sql/src/view/compile.rsindex 15e059d97b..26769a7483 100644--- a/crates/database_sql/src/view/compile.rs+++ b/crates/database_sql/src/view/compile.rs@@ -3,10 +3,11 @@#[cfg(test)]mod test;+use models_databases::TableId;use models_databases::views::{Conjunction, DatabaseView, DateOperator, DateRangeOperator, FilterCondition, FilterGroup,FilterNode, FilterTest, NumberOperator, PresenceOperator, SetOperator, SortDirection,- TextOperator, ViewProblem,+ TextOperator, ViewLayout, ViewProblem, ViewQuery,};use crate::catalog::{Catalog, Column, Table};@@ -26,11 +27,35 @@ pub fn compile_view(view: &DatabaseView, catalog: &Catalog) -> Result<SelectQuercompile_checked(view, checked_table(view, catalog)?)}+/// Compile the filter and ordering of a table read, independent of its presentation.+pub fn compile_table_query(+ table_id: TableId,+ query: &ViewQuery,+ catalog: &Catalog,+) -> Result<SelectQuery, ViewProblem> {+ let table = catalog+ .tables+ .iter()+ .find(|table| table.id == table_id)+ .ok_or(ViewProblem::UnknownTable { table: table_id })?;+ let columns = table+ .columns+ .iter()+ .map(super::schema_column)+ .collect::<Vec<_>>();+ models_databases::views::check(query, &ViewLayout::Table { columns: vec![] }, &columns)?;+ compile_query_checked(query, table)+}+/// [`compile_view`] once the view has checked out against `table`.pub(super) fn compile_checked(view: &DatabaseView,table: &Table,) -> Result<SelectQuery, ViewProblem> {+ compile_query_checked(&view.query, table)+}++fn compile_query_checked(query: &ViewQuery, table: &Table) -> Result<SelectQuery, ViewProblem> {let position = row_position_key(table.id);let mut bindings: Vec<Binding> = table.columns@@ -46,8 +71,7 @@ pub(super) fn compile_checked(relation: 0,column: None,});- let order_by = view- .query+ let order_by = query.sort.iter().map(|key| {@@ -64,7 +88,7 @@ pub(super) fn compile_checked(direction: Direction::Ascending,}))).collect::<Result<_, ViewProblem>>()?;- let where_ = match &view.query.filter {+ let where_ = match &query.filter {Some(group) => group_filter(table, group)?,None => None,};diff --git a/crates/database_sql/src/view/compile/test.rs b/crates/database_sql/src/view/compile/test.rsindex d3907deb27..28294bea0d 100644--- a/crates/database_sql/src/view/compile/test.rs+++ b/crates/database_sql/src/view/compile/test.rs@@ -196,7 +196,11 @@ fn a_nested_view_is_the_select_its_sql_resolves_to() {],};- assert_eq!(compile_view(&view, &issues_catalog()), Ok(expected));+ assert_eq!(compile_view(&view, &issues_catalog()), Ok(expected.clone()));+ assert_eq!(+ compile_table_query(ISSUES, &view.query, &issues_catalog()),+ Ok(expected)+ );}#[test]diff --git a/crates/databases/Cargo.toml b/crates/databases/Cargo.tomlindex 336a42e14e..5c12c383fc 100644--- a/crates/databases/Cargo.toml+++ b/crates/databases/Cargo.toml@@ -4,6 +4,11 @@ name = "databases"publish = falseversion = "0.1.0"+[[bin]]+name = "backfill_formula_cells"+path = "src/bin/backfill_formula_cells.rs"+required-features = ["backfill_binary"]+[features]default = []# AI tools. A driving adapter like `inbound`, but for the agent loop rather@@ -16,6 +21,15 @@ ai_tools = ["models_databases/schema","ports",]+# The one-off backfill of formula columns' stored cells.+backfill_binary = [+ "dep:clap",+ "dep:database_env_vars",+ "dep:macro_entrypoint",+ "postgres",+ "tokio/macros",+ "tokio/rt-multi-thread",+]# Axum HTTP router: ops, schema ops, and saved queries.inbound = ["dep:axum",@@ -33,6 +47,8 @@ gateway = ["dep:anyhow", "dep:connection_gateway_client", "outbound"]# Driven adapters share the port contracts.outbound = ["ports"]ports = ["dep:entity_access", "dep:tokio", "dep:tracing"]+# Ordered, filtered reads of a database view.+view_rows = ["ports"]# Postgres persistence for databases/tables/columns/rows, and cells through# the properties adapter.postgres = [@@ -51,13 +67,16 @@ ai_toolset = { path = "../ai_toolset", optional = true }anyhow = { workspace = true, optional = true }async-trait = { workspace = true, optional = true }axum = { workspace = true, optional = true }+clap = { workspace = true, optional = true }bot_id = { path = "../bot_id" }chrono = { workspace = true, features = ["serde"] }connection_gateway_client = { path = "../connection_gateway_client", optional = true }+database_env_vars = { path = "../database_env_vars", optional = true }entity_access = { path = "../entity_access", optional = true, default-features = false, features = ["ports"] }entity_access_db_utils = { path = "../entity_access_db_utils", optional = true }entity_mutation = { path = "../entity_mutation", optional = true }macro_authorization = { path = "../macro_authorization", optional = true, default-features = false, features = ["axum"] }+macro_entrypoint = { path = "../macro_entrypoint", optional = true }macro_event_broker = { path = "../macro_event_broker", default-features = false }macro_event_topics = { path = "../macro_event_topics" }macro_user_id = { path = "../macro_user_id" }diff --git a/crates/databases/src/backfill.rs b/crates/databases/src/backfill.rsnew file mode 100644index 0000000000..847afa55be--- /dev/null+++ b/crates/databases/src/backfill.rs@@ -0,0 +1,113 @@+//! Store the cells of formula columns created before their values were+//! stored: one batch per table through the cell store, so each table is+//! derived under its own lock and only bumps its version when a cell+//! changed. Running it again changes nothing.++use anyhow::Context as _;+use macro_user_id::cowlike::CowLike;+use macro_user_id::user_id::MacroUserIdStr;+use properties::outbound::properties_pg_repo::PropertiesPgRepo;+use sqlx::PgPool;++use crate::domain::catalog::{ColumnEntry, StorageTable};+use crate::domain::derivation::Derivation;+use crate::domain::journal::JournalPlan;+use crate::domain::models::{DatabaseId, TableId, Writes, WritesOutcome};+use crate::domain::ports::{CellStore, ColumnDefinitionStore, DatabasesRepo};+use crate::outbound::pg_cell_store::PgCellStore;+use crate::outbound::pg_databases_repo::PgDatabasesRepo;+use crate::outbound::pg_definition_store::PgDefinitionStore;++/// What backfilling one table did.+#[derive(Debug, Clone, PartialEq, Eq)]+pub struct TableBackfill {+ /// The table's rows.+ pub rows: usize,+ /// Whether a cell changed, and so the table's version moved.+ pub changed: bool,+}++/// Every table of a live database with a formula column, by database.+pub async fn tables_with_formulas(pool: &PgPool) -> anyhow::Result<Vec<(DatabaseId, TableId)>> {+ Ok(+ PgDatabasesRepo::new(pool.clone(), PropertiesPgRepo::new(pool.clone()))+ .tables_with_derived_columns()+ .await?,+ )+}++/// Derive every formula column of one table on all its rows, as the+/// database's owner.+pub async fn backfill_table(+ pool: &PgPool,+ database_id: DatabaseId,+ table_id: TableId,+) -> anyhow::Result<TableBackfill> {+ let properties = PropertiesPgRepo::new(pool.clone());+ let repository = PgDatabasesRepo::new(pool.clone(), properties.clone());+ let (database, tables) = repository+ .get_database(database_id)+ .await?+ .context("the database is gone")?;+ let table = tables+ .into_iter()+ .find(|table| table.id == table_id)+ .context("the table is gone")?;+ let columns = repository.columns_for_tables(&[table_id]).await?;+ let definitions = PgDefinitionStore::new(pool.clone(), properties.clone())+ .definitions(+ &columns+ .iter()+ .map(|column| column.property_definition_id)+ .collect::<Vec<_>>(),+ )+ .await?;+ let columns = columns+ .into_iter()+ .map(|column| {+ let definition = definitions+ .iter()+ .find(|definition| definition.definition.id == column.property_definition_id)+ .cloned()+ .with_context(|| format!("column {} has no definition", column.id))?;+ Ok(ColumnEntry {+ column,+ definition,+ writable: false,+ })+ })+ .collect::<anyhow::Result<Vec<_>>>()?;+ let rows = repository.row_refs(table_id).await?.len();+ let entry = StorageTable {+ table,+ columns,+ views: Vec::new(),+ };+ let Some(derivation) = Derivation::every(&entry) else {+ return Ok(TableBackfill {+ rows,+ changed: false,+ });+ };+ let owner = MacroUserIdStr::parse_from_str(&database.owner_id)+ .context("the database's owner is not a user id")?+ .into_owned();+ let outcome = PgCellStore::new(pool.clone(), properties)+ .apply_writes(&Writes {+ database_id,+ created_by: owner,+ writes: Vec::new(),+ related_rows: Vec::new(),+ expected_versions: Vec::new(),+ journal: JournalPlan::default(),+ derivations: vec![derivation],+ })+ .await?;+ match outcome {+ WritesOutcome::Applied { table_versions, .. } => Ok(TableBackfill {+ rows,+ changed: table_versions.contains_key(&table_id),+ }),+ refused => anyhow::bail!("the cell store refused the batch: {refused:?}"),+ }+}diff --git a/crates/databases/src/bin/backfill_formula_cells.rs b/crates/databases/src/bin/backfill_formula_cells.rsnew file mode 100644index 0000000000..5735c09aaf--- /dev/null+++ b/crates/databases/src/bin/backfill_formula_cells.rs@@ -0,0 +1,93 @@+//! Store the cells of every formula column created before formula values+//! were stored. Each table is one batch under its own lock; a table whose+//! cells are already right is left at its version, so a second run changes+//! nothing.++use std::time::Instant;++use anyhow::Context as _;+use clap::Parser;+use database_env_vars::DatabaseUrl;+use databases::backfill::{backfill_table, tables_with_formulas};+use macro_entrypoint::MacroEntrypoint;+use sqlx::postgres::PgPoolOptions;++#[derive(Parser)]+#[command(about = "Store the cells of formula columns created before they were stored")]+struct Arguments {+ /// List the tables with a formula column, and change nothing.+ #[arg(long)]+ dry_run: bool,+}++#[derive(Default)]+struct Progress {+ changed: usize,+ unchanged: usize,+ failed: usize,+}++#[tokio::main]+async fn main() -> anyhow::Result<()> {+ MacroEntrypoint::default().init();+ let arguments = Arguments::parse();++ let database_url = DatabaseUrl::new().context("DATABASE_URL must be provided")?;+ let pool = PgPoolOptions::new()+ .min_connections(1)+ .max_connections(2)+ .connect(database_url.as_ref())+ .await+ .context("could not connect to db")?;++ let tables = tables_with_formulas(&pool).await?;+ println!("tables with a formula column: {}", tables.len());+ if arguments.dry_run {+ for (database_id, table_id) in &tables {+ println!("database={database_id} table={table_id}");+ }+ return Ok(());+ }++ let mut total = Progress::default();+ for (database_id, table_id) in tables {+ let started = Instant::now();+ match backfill_table(&pool, database_id, table_id).await {+ Ok(backfill) => {+ if backfill.changed {+ total.changed += 1;+ } else {+ total.unchanged += 1;+ }+ println!(+ "database={database_id} table={table_id} rows={} changed={} ms={}",+ backfill.rows,+ backfill.changed,+ started.elapsed().as_millis()+ );+ }+ Err(error) => {+ total.failed += 1;+ tracing::error!(+ error=?error,+ %database_id,+ %table_id,+ "failed to backfill the table's formula cells"+ );+ println!(+ "database={database_id} table={table_id} failed ms={}",+ started.elapsed().as_millis()+ );+ }+ }+ }++ println!(+ "total changed={} unchanged={} failed={}",+ total.changed, total.unchanged, total.failed+ );+ if total.failed > 0 {+ anyhow::bail!("the formula cell backfill completed with failures");+ }+ Ok(())+}diff --git a/crates/databases/src/domain.rs b/crates/databases/src/domain.rsindex 0ea8069a18..ae20eb045e 100644--- a/crates/databases/src/domain.rs+++ b/crates/databases/src/domain.rs@@ -3,9 +3,12 @@pub mod activity;pub mod catalog;+pub mod derivation;pub mod events;pub mod journal;pub mod models;+#[cfg(feature = "view_rows")]+pub mod view_rows;#[cfg(feature = "ports")]pub mod storage;diff --git a/crates/databases/src/domain/derivation.rs b/crates/databases/src/domain/derivation.rsnew file mode 100644index 0000000000..63f14e5464--- /dev/null+++ b/crates/databases/src/domain/derivation.rs@@ -0,0 +1,170 @@+//! Derived columns' stored cells. A formula column's definition holds each+//! row's computed value, written by the one writer whenever a formula or a+//! cell it reads changes, so every reader sees a plain number or date+//! column. No op writes them, and the journal records none: undoing an+//! input derives them again.++#[cfg(test)]+mod test;++use std::collections::HashMap;++use database_sql::catalog::{Column, Table};+use database_sql::fold::Cell;+use database_sql::formula;+use models_databases::{ColumnId, Formula};+use models_properties::service::property_value::PropertyValue;+use uuid::Uuid;++use crate::domain::catalog::{self, StorageTable};+use crate::domain::models::{PropertyDefinitionId, TableId};++/// How deep derived columns may nest, as the formula engine guards it.+const MAX_DEPTH: usize = 32;++/// Some derived columns of one table, with the table as their formulas+/// read it: what a write stores in their cells.+#[derive(Debug, Clone, PartialEq)]+pub struct Derivation {+ table: Table,+ derived: Vec<(PropertyDefinitionId, Formula)>,+}++impl Derivation {+ /// The derived columns of `table` that `changed` picks, or that read a+ /// column it picks, directly or through other derived columns; `None`+ /// when there are none.+ pub fn reading(table: &Table, changed: impl Fn(&Column) -> bool) -> Option<Self> {+ let derived: Vec<(PropertyDefinitionId, Formula)> = formula::derived(table)+ .filter(|(column, formula)| changed(column) || reads(table, formula, &changed, 0))+ .map(|(column, formula)| (column.id, formula.clone()))+ .collect();+ (!derived.is_empty()).then(|| Self {+ table: table.clone(),+ derived,+ })+ }++ /// Every derived column of `entry`; `None` when it has none.+ pub fn every(entry: &StorageTable) -> Option<Self> {+ Self::reading(&catalog::engine_table(entry, ""), |_| true)+ }++ /// The derived columns of `entry` that cells of the `written`+ /// definitions feed.+ pub fn fed_by(entry: &StorageTable, written: &[PropertyDefinitionId]) -> Option<Self> {+ if !entry+ .columns+ .iter()+ .any(|column| column.column.formula().is_some())+ {+ return None;+ }+ Self::reading(&catalog::engine_table(entry, ""), |column| {+ written.contains(&column.id)+ })+ }++ /// The table.+ pub fn table_id(&self) -> TableId {+ self.table.id+ }++ /// The definitions of the stored columns the formulas read, through+ /// nested formulas too.+ pub fn inputs(&self) -> Vec<PropertyDefinitionId> {+ let mut inputs = Vec::new();+ for (_, formula) in &self.derived {+ stored_inputs(&self.table, formula, &mut inputs, 0);+ }+ inputs+ }++ /// The definitions whose cells it computes.+ pub fn outputs(&self) -> Vec<PropertyDefinitionId> {+ self.derived+ .iter()+ .map(|(definition, _)| *definition)+ .collect()+ }++ /// A row's derived cells, from its stored cells by definition; `None`+ /// empties a cell.+ pub fn cells(+ &self,+ row: &HashMap<PropertyDefinitionId, PropertyValue>,+ ) -> Vec<(PropertyDefinitionId, Option<PropertyValue>)> {+ let cells: HashMap<Uuid, Cell> = row+ .iter()+ .filter_map(|(definition, value)| {+ let cell = match value {+ PropertyValue::Num(number) => Cell::Number(*number),+ PropertyValue::Date(date) => Cell::Date(*date),+ _ => return None,+ };+ Some((*definition, cell))+ })+ .collect();+ self.derived+ .iter()+ .map(|(definition, formula)| {+ let value = match formula::evaluate(&self.table, formula, &cells, |id| id) {+ Some(Cell::Number(number)) => Some(PropertyValue::Num(number)),+ Some(Cell::Date(date)) => Some(PropertyValue::Date(date)),+ _ => None,+ };+ (*definition, value)+ })+ .collect()+ }+}++fn placement(table: &Table, column: ColumnId) -> Option<&Column> {+ table+ .columns+ .iter()+ .find(|candidate| candidate.placement == column)+}++/// Whether `formula` reads a column `changed` picks, directly or through+/// the derived columns it reads.+fn reads(+ table: &Table,+ formula: &Formula,+ changed: &impl Fn(&Column) -> bool,+ depth: usize,+) -> bool {+ depth <= MAX_DEPTH+ && formula.columns().into_iter().any(|column| {+ placement(table, column).is_some_and(|found| {+ changed(found)+ || found+ .formula+ .as_ref()+ .is_some_and(|inner| reads(table, inner, changed, depth + 1))+ })+ })+}++/// Collect the definitions of the stored columns `formula` reads, through+/// the derived columns it reads, each once.+fn stored_inputs(+ table: &Table,+ formula: &Formula,+ inputs: &mut Vec<PropertyDefinitionId>,+ depth: usize,+) {+ if depth > MAX_DEPTH {+ return;+ }+ for column in formula.columns() {+ let Some(found) = placement(table, column) else {+ continue;+ };+ match &found.formula {+ Some(inner) => stored_inputs(table, inner, inputs, depth + 1),+ None if !inputs.contains(&found.id) => inputs.push(found.id),+ None => {}+ }+ }+}diff --git a/crates/databases/src/domain/derivation/test.rs b/crates/databases/src/domain/derivation/test.rsnew file mode 100644index 0000000000..58337ee107--- /dev/null+++ b/crates/databases/src/domain/derivation/test.rs@@ -0,0 +1,214 @@+use std::collections::HashMap;++use database_sql::catalog::{Column, ColumnKind, Table, TableSource};+use models_databases::{ColumnId, DatabaseId, Formula, Operator, TableId};+use models_properties::service::property_value::PropertyValue;+use uuid::Uuid;++use super::Derivation;++const PRICE: Uuid = Uuid::from_u128(1);+const QTY: Uuid = Uuid::from_u128(2);+const DUE: Uuid = Uuid::from_u128(3);+const NAME: Uuid = Uuid::from_u128(4);+const TOTAL: Uuid = Uuid::from_u128(5);+const DOUBLE: Uuid = Uuid::from_u128(6);+const RATIO: Uuid = Uuid::from_u128(7);+const FOLLOW_UP: Uuid = Uuid::from_u128(8);++/// Each column's definition is its placement's id shifted by 100, so a+/// test that mixed the two up would fail.+fn placement(definition: Uuid) -> ColumnId {+ ColumnId::from_uuid(Uuid::from_u128(definition.as_u128() + 100))+}++fn read(definition: Uuid) -> Box<Formula> {+ Box::new(Formula::Column {+ column: placement(definition),+ })+}++/// Price, Qty, Due and Name, and four formulas over them: Total = Price *+/// Qty, Double = Total * 2, Ratio = Price / Qty and Follow-up = Due + Qty.+fn orders(price_kind: ColumnKind) -> Table {+ let column = |id: Uuid, name: &str, kind: ColumnKind, formula: Option<Formula>| Column {+ id,+ placement: placement(id),+ name: name.into(),+ kind,+ formula,+ };+ Table {+ id: TableId::from_uuid(Uuid::from_u128(50)),+ database_id: DatabaseId::from_uuid(Uuid::from_u128(51)),+ database: "Shop".into(),+ name: "Orders".into(),+ source: TableSource::Database,+ columns: vec![+ column(PRICE, "Price", price_kind, None),+ column(QTY, "Qty", ColumnKind::Number, None),+ column(DUE, "Due", ColumnKind::Date, None),+ column(NAME, "Name", ColumnKind::Text, None),+ column(+ TOTAL,+ "Total",+ ColumnKind::Number,+ Some(Formula::Binary {+ operator: Operator::Multiply,+ left: read(PRICE),+ right: read(QTY),+ }),+ ),+ column(+ DOUBLE,+ "Double",+ ColumnKind::Number,+ Some(Formula::Binary {+ operator: Operator::Multiply,+ left: read(TOTAL),+ right: Box::new(Formula::Number { value: 2.0 }),+ }),+ ),+ column(+ RATIO,+ "Ratio",+ ColumnKind::Number,+ Some(Formula::Binary {+ operator: Operator::Divide,+ left: read(PRICE),+ right: read(QTY),+ }),+ ),+ column(+ FOLLOW_UP,+ "Follow-up",+ ColumnKind::Date,+ Some(Formula::Binary {+ operator: Operator::Add,+ left: read(DUE),+ right: read(QTY),+ }),+ ),+ ],+ }+}++fn date(text: &str) -> PropertyValue {+ PropertyValue::Date(text.parse().unwrap())+}++#[test]+fn a_change_reaches_the_formulas_reading_it_directly_or_through_another() {+ let table = orders(ColumnKind::Number);++ let price = Derivation::reading(&table, |column| column.id == PRICE).unwrap();+ assert_eq!(price.table_id(), table.id);+ assert_eq!(price.outputs(), vec![TOTAL, DOUBLE, RATIO]);++ let due = Derivation::reading(&table, |column| column.id == DUE).unwrap();+ assert_eq!(due.outputs(), vec![FOLLOW_UP]);++ let total = Derivation::reading(&table, |column| column.id == TOTAL).unwrap();+ assert_eq!(total.outputs(), vec![TOTAL, DOUBLE]);++ let every = Derivation::reading(&table, |_| true).unwrap();+ assert_eq!(every.outputs(), vec![TOTAL, DOUBLE, RATIO, FOLLOW_UP]);++ assert_eq!(+ Derivation::reading(&table, |column| column.id == NAME),+ None+ );+}++#[test]+fn inputs_are_the_stored_columns_read_through_nested_formulas() {+ let table = orders(ColumnKind::Number);++ let double = Derivation::reading(&table, |column| column.id == DOUBLE).unwrap();+ assert_eq!(double.outputs(), vec![DOUBLE]);+ assert_eq!(double.inputs(), vec![PRICE, QTY]);++ let every = Derivation::reading(&table, |_| true).unwrap();+ assert_eq!(every.inputs(), vec![PRICE, QTY, DUE]);+}++#[test]+fn a_row_with_every_input_computes_numbers_and_dates() {+ let every = Derivation::reading(&orders(ColumnKind::Number), |_| true).unwrap();++ let cells = every.cells(&HashMap::from([+ (PRICE, PropertyValue::Num(10.0)),+ (QTY, PropertyValue::Num(4.0)),+ (DUE, date("2026-10-01T00:00:00Z")),+ (NAME, PropertyValue::Str("Lamp".into())),+ ]));++ assert_eq!(+ cells,+ vec![+ (TOTAL, Some(PropertyValue::Num(40.0))),+ (DOUBLE, Some(PropertyValue::Num(80.0))),+ (RATIO, Some(PropertyValue::Num(2.5))),+ (FOLLOW_UP, Some(date("2026-10-05T00:00:00Z"))),+ ]+ );+}++#[test]+fn an_empty_number_reads_as_zero_until_every_input_is_empty() {+ let every = Derivation::reading(&orders(ColumnKind::Number), |_| true).unwrap();++ assert_eq!(+ every.cells(&HashMap::from([(QTY, PropertyValue::Num(3.0))])),+ vec![+ (TOTAL, Some(PropertyValue::Num(0.0))),+ (DOUBLE, Some(PropertyValue::Num(0.0))),+ (RATIO, Some(PropertyValue::Num(0.0))),+ // Due is empty, and an empty date empties the sum.+ (FOLLOW_UP, None),+ ]+ );+ assert_eq!(+ every.cells(&HashMap::new()),+ vec![+ (TOTAL, None),+ (DOUBLE, None),+ (RATIO, None),+ (FOLLOW_UP, None),+ ]+ );+}++#[test]+fn dividing_by_zero_is_empty() {+ let every = Derivation::reading(&orders(ColumnKind::Number), |_| true).unwrap();++ assert_eq!(+ every.cells(&HashMap::from([(PRICE, PropertyValue::Num(5.0))])),+ vec![+ (TOTAL, Some(PropertyValue::Num(0.0))),+ (DOUBLE, Some(PropertyValue::Num(0.0))),+ (RATIO, None),+ (FOLLOW_UP, None),+ ]+ );+}++#[test]+fn an_input_that_is_no_longer_a_number_empties_what_reads_it() {+ let every = Derivation::reading(&orders(ColumnKind::Text), |_| true).unwrap();++ assert_eq!(+ every.cells(&HashMap::from([+ (PRICE, PropertyValue::Str("10".into())),+ (QTY, PropertyValue::Num(4.0)),+ (DUE, date("2026-10-01T00:00:00Z")),+ ])),+ vec![+ (TOTAL, None),+ (DOUBLE, None),+ (RATIO, None),+ (FOLLOW_UP, Some(date("2026-10-05T00:00:00Z"))),+ ]+ );+}diff --git a/crates/databases/src/domain/journal.rs b/crates/databases/src/domain/journal.rsindex 5660bee4c8..27c7f85501 100644--- a/crates/databases/src/domain/journal.rs+++ b/crates/databases/src/domain/journal.rs@@ -495,7 +495,8 @@ pub fn cell_value(value: &PropertyValue) -> Option<CellValue> {}/// A row's stored cells, by definition, as the before-image keeps them: by-/// the column of its table each definition is bound to.+/// the column of its table each definition is bound to. Derived columns'+/// cells are left out: they follow the others, and no op writes them.pub fn row_cells(schema: &SchemaImage,table: TableId,@@ -504,7 +505,9 @@ pub fn row_cells(cells.iter().filter_map(|(definition, value)| {- let column = schema.column_for(table, *definition)?;+ let column = schema+ .column_for(table, *definition)+ .filter(|column| column.formula.is_none())?;Some((column.id, cell_value(value)?))}).collect()diff --git a/crates/databases/src/domain/journal/entries.rs b/crates/databases/src/domain/journal/entries.rsindex adecdf556c..446610fb31 100644--- a/crates/databases/src/domain/journal/entries.rs+++ b/crates/databases/src/domain/journal/entries.rs@@ -15,7 +15,16 @@ use super::{use crate::domain::models::{PropertyDefinitionId, Write, Writes};/// What the adapter reads under the batch's locks, before it writes.+/// Derived columns' cells are not read: they follow the others, so an undo+/// derives them again rather than writing them back.pub fn reads(writes: &Writes) -> Reads {+ let derived = |column: &ColumnId| {+ writes+ .journal+ .schema+ .column(*column)+ .is_some_and(|image| image.formula.is_some())+ };let mut reads = Reads::default();for (op, write) in writes.journal.ops.iter().zip(&writes.writes) {match write {@@ -48,12 +57,12 @@ pub fn reads(writes: &Writes) -> Reads {column_id,definition_id,..- } => reads.columns.push((*table_id, *column_id, *definition_id)),+ } if !derived(column_id) => reads.columns.push((*table_id, *column_id, *definition_id)),Write::ReplaceColumn {table_id,replacement,..- } => reads.columns.push((+ } if !derived(&replacement.column.id) => reads.columns.push((*table_id,replacement.column.id,replacement.column.property_definition_id,diff --git a/crates/databases/src/domain/journal/test.rs b/crates/databases/src/domain/journal/test.rsindex 902c09f6a5..2b91a84de9 100644--- a/crates/databases/src/domain/journal/test.rs+++ b/crates/databases/src/domain/journal/test.rs@@ -210,6 +210,7 @@ fn a_row_insert_is_undone_by_deleting_the_rows_it_inserted() {},};let write = Write::InsertRows {+ derivation: None,table_id: GUESTS,rows: vec![vec![(NAME_DEFINITION, PropertyValue::Str("Maria".into()))]],restored: Vec::new(),@@ -261,6 +262,7 @@ fn a_cell_update_is_undone_by_writing_back_the_old_value_or_clearing_an_empty_ce},};let write = Write::UpdateRows {+ derivation: None,table_id: GUESTS,rows: vec![(@@ -1241,6 +1243,7 @@ fn what_a_batch_creates_and_removes_again_needs_no_inverse() {definition: None,},Write::InsertRows {+ derivation: None,table_id: GUESTS,rows: vec![Vec::new()],restored: Vec::new(),diff --git a/crates/databases/src/domain/models.rs b/crates/databases/src/domain/models.rsindex 985fe54514..553f1435e6 100644--- a/crates/databases/src/domain/models.rs+++ b/crates/databases/src/domain/models.rs@@ -24,6 +24,7 @@ pub use models_databases::{ChangeId, ColumnId, DatabaseId, OptionId, QueryId, RowId, TableId, TableVersion, TakenId,};+use crate::domain::derivation::Derivation;use crate::domain::journal::{JournalPlan, RestoredRow};/// Identifier of a `models_properties` property definition bound as a column.@@ -478,6 +479,8 @@ pub enum Write {/// when the insert puts back rows a removal took; empty to mint/// new ones after the table's last row.restored: Vec<RestoredRow>,+ /// The derived columns whose cells the rows' cells feed.+ derivation: Option<Derivation>,},/// Set (or, with `None`, clear) cells of existing rows of the table.UpdateRows {@@ -485,6 +488,9 @@ pub enum Write {table_id: TableId,/// Each row with its cells.rows: Vec<(RowId, CellChanges)>,+ /// The derived columns whose cells the written cells feed, derived+ /// again on these rows.+ derivation: Option<Derivation>,},/// Remove rows of the table with their cells.DeleteRows {@@ -643,6 +649,9 @@ pub struct Writes {pub expected_versions: Vec<(TableId, TableVersion)>,/// What the batch's journal entries are built from.pub journal: JournalPlan,+ /// The derived columns its schema changes affect, derived again on+ /// every row of their tables once its writes are applied.+ pub derivations: Vec<Derivation>,}impl Writes {diff --git a/crates/databases/src/domain/service/ops.rs b/crates/databases/src/domain/service/ops.rsindex 1b818913bb..6317893b8d 100644--- a/crates/databases/src/domain/service/ops.rs+++ b/crates/databases/src/domain/service/ops.rs@@ -21,6 +21,7 @@ use models_properties::service::property_definition_with_options::PropertyDefiniuse super::*;use crate::domain::catalog::{ColumnEntry, StorageTable};+use crate::domain::derivation::Derivation;use crate::domain::journal::{JournalPlan, Restoration};use crate::domain::models::{AppliedOps, CommittedChange};use crate::domain::models::{@@ -187,6 +188,7 @@ wherecreated_tables: HashSet::new(),changed_options: HashSet::new(),restoration: restoration.clone(),+ rederived: Vec::new(),};let writes = ops.iter()@@ -215,6 +217,7 @@ where),acting_bot: viewer.acting_bot,},+ derivations: planner.derivations(),};Ok((writes, planner))}@@ -927,6 +930,9 @@ pub(super) struct Planner {changed_options: HashSet<PropertyDefinitionId>,/// What the batch's row inserts put back and its type changes bind back.restoration: Restoration,+ /// The columns whose formula, type or creation changes what the+ /// derived columns reading them hold on every row.+ rederived: Vec<(TableId, ColumnId)>,}/// Where in the batch a cell is: its op, the row's index within the op@@ -945,6 +951,25 @@ impl Place {}impl Planner {+ /// The derived columns of each table, as the batch leaves it, that read+ /// a column the batch retyped or that the batch created or gave a new+ /// formula.+ fn derivations(&self) -> Vec<Derivation> {+ self.entries+ .iter()+ .filter(|entry| {+ self.rederived+ .iter()+ .any(|(table, _)| *table == entry.table.id)+ })+ .filter_map(|entry| {+ Derivation::reading(&catalog::engine_table(entry, ""), |column| {+ self.rederived.contains(&(entry.table.id, column.placement))+ })+ })+ .collect()+ }+pub(super) fn finish(&self,entries: &[StorageTable],diff --git a/crates/databases/src/domain/service/ops/columns.rs b/crates/databases/src/domain/service/ops/columns.rsindex 491a86cb55..42280c13f5 100644--- a/crates/databases/src/domain/service/ops/columns.rs+++ b/crates/databases/src/domain/service/ops/columns.rs@@ -141,6 +141,9 @@ impl Planner {display_name: None,infer_type,};+ if column.formula().is_some() {+ self.rederived.push((entry.table.id, id));+ }let writable = grant_writes(self.grant);if let Some(entry) = self.entry_mut(entry.table.id) {entry.columns.push(ColumnEntry {@@ -506,6 +509,7 @@ impl Planner {cells: converted.into_iter().collect(),},);+ self.rederived.push((entry.table.id, column_id));Ok(Write::ReplaceColumn {table_id: entry.table.id,read_version,@@ -551,6 +555,7 @@ impl Planner {stored.definition = definition;}self.found.cells.remove(&place.column);+ self.rederived.push((entry.table.id, place.column));Ok(Write::ReplaceColumn {table_id: entry.table.id,read_version: Some(entry.table.version),@@ -613,6 +618,7 @@ impl Planner {planned.definition = stored;}}+ self.rederived.push((entry.table.id, place.column));Ok(Write::ReplaceColumn {table_id: entry.table.id,read_version,diff --git a/crates/databases/src/domain/service/ops/rows.rs b/crates/databases/src/domain/service/ops/rows.rsindex 0f0ad387fc..e58f051d51 100644--- a/crates/databases/src/domain/service/ops/rows.rs+++ b/crates/databases/src/domain/service/ops/rows.rs@@ -6,8 +6,9 @@ use models_databases::{RowChanges, RowsChange};use super::{MAX_WRITTEN_ROWS, Place, Planner, refuse};use crate::domain::catalog::StorageTable;+use crate::domain::derivation::Derivation;use crate::domain::journal::cell_value;-use crate::domain::models::{DatabaseError, Write};+use crate::domain::models::{CellChanges, DatabaseError, PropertyDefinitionId, RowId, Write};impl Planner {pub(super) fn rows_write(@@ -84,34 +85,40 @@ impl Planner {}}}+ let written: Vec<PropertyDefinitionId> = rows+ .iter()+ .flatten()+ .map(|(definition, _)| *definition)+ .collect();Ok(Write::InsertRows {table_id: table,rows,restored,+ derivation: Derivation::fed_by(entry, &written),})}RowsChange::Update {changes: RowChanges::Uniform { rows, cells },} => {let cells = self.cells(entry, index, None, cells)?;- Ok(Write::UpdateRows {- table_id: table,- rows: rows.iter().map(|row| (*row, cells.clone())).collect(),- })+ Ok(update_rows(+ entry,+ rows.iter().map(|row| (*row, cells.clone())).collect(),+ ))}RowsChange::Update {changes: RowChanges::PerRow { rows },- } => Ok(Write::UpdateRows {- table_id: table,- rows: rows+ } => {+ let rows = rows.iter().enumerate().map(|(row, change)| {let cells = self.cells(entry, index, Some(row), &change.cells)?;Ok((change.row, cells))})- .collect::<Result<_, DatabaseError>>()?,- }),+ .collect::<Result<_, DatabaseError>>()?;+ Ok(update_rows(entry, rows))+ }RowsChange::Delete { rows } => {for (row, id) in rows.iter().enumerate() {if rows[..row].contains(id) {@@ -134,3 +141,16 @@ impl Planner {}}}++/// Set cells of existing rows, deriving again the formulas they feed.+fn update_rows(entry: &StorageTable, rows: Vec<(RowId, CellChanges)>) -> Write {+ let written: Vec<PropertyDefinitionId> = rows+ .iter()+ .flat_map(|(_, cells)| cells.iter().map(|(definition, _)| *definition))+ .collect();+ Write::UpdateRows {+ table_id: entry.table.id,+ derivation: Derivation::fed_by(entry, &written),+ rows,+ }+}diff --git a/crates/databases/src/domain/service/test/derived_columns.rs b/crates/databases/src/domain/service/test/derived_columns.rsindex 75804a884d..9921b4eabd 100644--- a/crates/databases/src/domain/service/test/derived_columns.rs+++ b/crates/databases/src/domain/service/test/derived_columns.rs@@ -336,3 +336,89 @@ async fn a_formula_cannot_read_its_own_column() {"Seats can't be used here: its value depends on this column.");}++#[tokio::test]+async fn a_formula_cell_is_derived_on_creation_on_a_new_formula_and_on_each_input_write() {+ let seeded = seeded().await;+ let seats = ColumnId::new();+ let seats_cell = |seeded: &Seeded| {+ let world = seeded.world.lock().unwrap();+ let definition = world+ .columns+ .iter()+ .find(|column| column.id == seats)+ .unwrap()+ .property_definition_id;+ world.cells[&seeded.row_id].get(&definition).cloned()+ };+ apply(+ &seeded,+ vec![DatabaseOp::Column {+ table: seeded.table_id,+ column: seats,+ change: ColumnChange::Create {+ definition: NewColumn::Derived {+ name: "Seats".into(),+ formula: plus_ones_times(&seeded, 2.0),+ },+ after: None,+ },+ }],+ )+ .await+ .unwrap();+ // Sam brings 2 plus ones.+ assert_eq!(seats_cell(&seeded), Some(PropertyValue::Num(4.0)));++ let formula_set = seeded+ .service+ .apply_ops_with_changes(+ edit(seeded.database_id),+ viewer(OWNER),+ vec![DatabaseOp::Column {+ table: seeded.table_id,+ column: seats,+ change: ColumnChange::SetFormula {+ formula: plus_ones_times(&seeded, 3.0),+ },+ }]+ .into(),+ )+ .await+ .unwrap();+ assert_eq!(seats_cell(&seeded), Some(PropertyValue::Num(6.0)));++ apply(+ &seeded,+ vec![DatabaseOp::Rows {+ table: seeded.table_id,+ change: RowsChange::Update {+ changes: RowChanges::Uniform {+ rows: vec![seeded.row_id],+ cells: vec![CellWrite {+ column: seeded.plus_ones_column.id,+ value: CellValue::Number(5.0),+ }],+ },+ },+ }],+ )+ .await+ .unwrap();+ assert_eq!(seats_cell(&seeded), Some(PropertyValue::Num(15.0)));++ let outcome = seeded+ .service+ .undo_change(+ edit(seeded.database_id),+ viewer(OWNER),+ formula_set.changes[0].change,+ )+ .await+ .unwrap();+ assert!(+ matches!(outcome, UndoOutcome::Reverted { .. }),+ "{outcome:?}"+ );+ assert_eq!(seats_cell(&seeded), Some(PropertyValue::Num(10.0)));+}diff --git a/crates/databases/src/domain/service/test/fakes/apply_writes.rs b/crates/databases/src/domain/service/test/fakes/apply_writes.rsindex c337077c44..5eeccde452 100644--- a/crates/databases/src/domain/service/test/fakes/apply_writes.rs+++ b/crates/databases/src/domain/service/test/fakes/apply_writes.rs@@ -1,6 +1,7 @@//! The fake cell store's write batch, applied straight to the world.use super::*;+use crate::domain::derivation::Derivation;use crate::domain::journal::{Before, RowImage, cell_value, row_cells};use crate::domain::models::{ChangeId, CommittedChange};use models_databases::views::LaneKey;@@ -19,6 +20,19 @@ pub(super) fn settle(world: &mut World, table_id: TableId, definitions: Vec<Propworld.settled.push((table_id, definitions));}+/// Store a derivation's cells on rows, from their other cells.+fn derive(world: &mut World, derivation: &Derivation, rows: &[RowId]) {+ for row in rows {+ let stored = world.cells.entry(*row).or_default();+ for (definition, value) in derivation.cells(stored) {+ match value {+ Some(value) => stored.insert(definition, value),+ None => stored.remove(&definition),+ };+ }+ }+}+/// The fake cell store's batch, applied straight to the world; the caller/// rolls the world back unless everything applied. An option for a missing/// definition fails the batch, as the foreign key does in Postgres.@@ -355,6 +369,7 @@ pub(super) fn apply_in_world(table_id,rows,restored,+ derivation,} => {let mut ids = Vec::new();for (row, cells) in rows.iter().enumerate() {@@ -386,9 +401,16 @@ pub(super) fn apply_in_world(let valued: Vec<_> = cells.iter().map(|(definition, _)| *definition).collect();settle(world, *table_id, valued);}+ if let Some(derivation) = derivation {+ derive(world, derivation, &ids);+ }inserted.push(ids);}- Write::UpdateRows { table_id, rows } => {+ Write::UpdateRows {+ table_id,+ rows,+ derivation,+ } => {for (row, cells) in rows {let owned = world.rows@@ -418,6 +440,10 @@ pub(super) fn apply_in_world(.collect();settle(world, *table_id, valued);}+ if let Some(derivation) = derivation {+ let named: Vec<RowId> = rows.iter().map(|(row, _)| *row).collect();+ derive(world, derivation, &named);+ }inserted.push(Vec::new());}Write::DeleteRows {@@ -614,6 +640,16 @@ pub(super) fn apply_in_world(}}}+ for derivation in &writes.derivations {+ let rows: Vec<RowId> = world+ .rows+ .get(&derivation.table_id())+ .into_iter()+ .flatten()+ .map(|row| row.id)+ .collect();+ derive(world, derivation, &rows);+ }for (table_id, row) in &writes.related_rows {if !world.rows@@ -637,6 +673,12 @@ pub(super) fn apply_in_world(.flat_map(|write| write.versioned_tables().iter().copied()).chain(created.iter().copied()).chain(related)+ .chain(+ writes+ .derivations+ .iter()+ .map(|derivation| derivation.table_id()),+ ).filter(|table| !deleted.contains(table)).collect();changed.sort();diff --git a/crates/databases/src/domain/view_rows.rs b/crates/databases/src/domain/view_rows.rsnew file mode 100644index 0000000000..cf257c6559--- /dev/null+++ b/crates/databases/src/domain/view_rows.rs@@ -0,0 +1,34 @@+//! The ordered read model for a validated view of one authorized table.++use std::future::Future;++use database_sql::{catalog::Table, resolve::SelectQuery};+use models_databases::{RowId, TableVersion};++/// A view read failed before a page could be returned.+#[derive(Debug, thiserror::Error)]+pub enum ViewRowsError {+ /// The caller must restart pagination against the current table.+ #[error("the table changed; refresh the view before continuing")]+ Stale,+ /// The compiled view is not a supported single-table read.+ #[error("invalid database view query")]+ InvalidQuery,+ /// Persistence failed.+ #[error("could not read database rows")]+ Infrastructure(rootcause::Report),+}++/// Read row identities after applying all of a view's filters and ordering.+/// The adapter checks the table version and reads the rows in one snapshot.+pub trait ViewRowsRepository: Send + Sync + 'static {+ /// At most `limit` matching row identities, strictly after `after`.+ fn page(+ &self,+ table: &Table,+ query: &SelectQuery,+ version: TableVersion,+ after: Option<RowId>,+ limit: u16,+ ) -> impl Future<Output = Result<Vec<RowId>, ViewRowsError>> + Send;+}diff --git a/crates/databases/src/lib.rs b/crates/databases/src/lib.rsindex e343b3550f..7c7dcb03bb 100644--- a/crates/databases/src/lib.rs+++ b/crates/databases/src/lib.rs@@ -12,3 +12,7 @@ pub mod outbound;#[cfg(feature = "postgres")]pub mod wiring;++/// The one-off backfill of formula columns' stored cells.+#[cfg(feature = "postgres")]+pub mod backfill;diff --git a/crates/databases/src/outbound.rs b/crates/databases/src/outbound.rsindex 345e89c65a..de3e62b070 100644--- a/crates/databases/src/outbound.rs+++ b/crates/databases/src/outbound.rs@@ -6,6 +6,9 @@ pub mod entity_access_directory;#[cfg(feature = "postgres")]pub mod pg_databases_repo;+#[cfg(all(feature = "postgres", feature = "view_rows"))]+pub mod pg_view_rows;+#[cfg(feature = "postgres")]pub mod pg_definition_store;diff --git a/crates/databases/src/outbound/pg_cell_store.rs b/crates/databases/src/outbound/pg_cell_store.rsindex acfbf8b861..06fb7e46a4 100644--- a/crates/databases/src/outbound/pg_cell_store.rs+++ b/crates/databases/src/outbound/pg_cell_store.rs@@ -3,6 +3,7 @@//! options.mod core;+mod derive;mod provisioning;mod required;mod transfer;@@ -257,6 +258,12 @@ where.flat_map(|write| write.versioned_tables().iter().copied()).chain(deleted.iter().copied()).chain(writes.expected_versions.iter().map(|(table, _)| *table))+ .chain(+ writes+ .derivations+ .iter()+ .map(|derivation| derivation.table_id()),+ ).filter(|table| !created.contains(table)).collect();required.sort();@@ -349,6 +356,16 @@ where}}+ // A table whose derived cells all held their values already keeps+ // its version, unless another write changed it.+ let mut rederived = Vec::new();+ for derivation in &writes.derivations {+ let rows = journal::table_rows(&mut *transaction, derivation.table_id()).await?;+ if self.derive(&mut transaction, derivation, &rows).await? {+ rederived.push(derivation.table_id());+ }+ }+if let Some(missing) = self.check_required_cells(&mut transaction, writes, &inserted).await?@@ -376,6 +393,7 @@ where.filter(|write| write.changes()).flat_map(|write| write.versioned_tables().iter().copied()).chain(created.iter().copied())+ .chain(rederived).chain(related.iter()@@ -589,6 +607,7 @@ wheretable_id,rows,restored,+ derivation,} => {let minted = if restored.is_empty() {rows::append_rows(@@ -635,11 +654,17 @@ where}}rows::settle_inference(&mut **transaction, *table_id, &valued).await?;- return Ok(Applied::Rows(- minted.into_iter().map(|row| row.id).collect(),- ));+ let minted: Vec<RowId> = minted.into_iter().map(|row| row.id).collect();+ if let Some(derivation) = derivation {+ self.derive(transaction, derivation, &minted).await?;+ }+ return Ok(Applied::Rows(minted));}- Write::UpdateRows { table_id, rows } => {+ Write::UpdateRows {+ table_id,+ rows,+ derivation,+ } => {let named: Vec<RowId> = rows.iter().map(|(row, _)| *row).collect();let owned = rows::lock_rows(&mut **transaction, *table_id, &named).await?;if let Some(row) = named.iter().find(|row| !owned.contains(row)) {@@ -666,6 +691,9 @@ where}}rows::settle_inference(&mut **transaction, *table_id, &valued).await?;+ if let Some(derivation) = derivation {+ self.derive(transaction, derivation, &named).await?;+ }}Write::DeleteRows {table_id,diff --git a/crates/databases/src/outbound/pg_cell_store/derive.rs b/crates/databases/src/outbound/pg_cell_store/derive.rsnew file mode 100644index 0000000000..ace65948c1--- /dev/null+++ b/crates/databases/src/outbound/pg_cell_store/derive.rs@@ -0,0 +1,62 @@+//! Store derived columns' cells: read the rows' inputs inside the batch's+//! transaction, let the domain compute each row's values, and write every+//! derived column in one statement.++use super::*;+use crate::domain::derivation::Derivation;++impl<Properties> PgCellStore<Properties>+where+ Properties: DatabaseCellWriter<Transaction = Transaction<'static, Postgres>>,+{+ /// Store `derivation`'s cells on `rows` from the cells the batch has+ /// left them, answering whether a stored cell changed.+ ///+ /// Every batch writing a table holds its version row `FOR UPDATE` until+ /// it commits, and an update holds its rows' locks too, so the inputs+ /// read here cannot change before the derived values are stored: two+ /// people editing different inputs of one row run one after the other,+ /// and the later reads the earlier's value.+ pub(super) async fn derive(+ &self,+ transaction: &mut Transaction<'static, Postgres>,+ derivation: &Derivation,+ rows: &[RowId],+ ) -> Result<bool, PgCellStoreError> {+ if rows.is_empty() {+ return Ok(false);+ }+ let entity_ids = journal::entity_ids(rows);+ let stored = self+ .properties+ .entity_values_in(+ transaction,+ EntityType::DatabaseRow,+ &entity_ids,+ Some(&derivation.inputs()),+ )+ .await+ .map_err(cells_error)?;+ let mut inputs = journal::by_row(stored);+ let mut columns: HashMap<PropertyDefinitionId, Vec<(String, Option<PropertyValue>)>> =+ HashMap::new();+ for (row, entity_id) in rows.iter().zip(entity_ids) {+ let cells = inputs.remove(row).unwrap_or_default();+ for (definition, value) in derivation.cells(&cells) {+ columns+ .entry(definition)+ .or_default()+ .push((entity_id.clone(), value));+ }+ }+ let mut changed = 0;+ for (definition, values) in columns {+ changed += self+ .properties+ .set_entity_values_in(transaction, EntityType::DatabaseRow, definition, &values)+ .await+ .map_err(cells_error)?;+ }+ Ok(changed > 0)+ }+}diff --git a/crates/databases/src/outbound/pg_cell_store/required.rs b/crates/databases/src/outbound/pg_cell_store/required.rsindex be9b9d8812..373c094e78 100644--- a/crates/databases/src/outbound/pg_cell_store/required.rs+++ b/crates/databases/src/outbound/pg_cell_store/required.rs@@ -26,7 +26,7 @@ whereWrite::InsertRows { table_id, .. } => {(*table_id, Some(inserted.iter().copied().collect()))}- Write::UpdateRows { table_id, rows } => {+ Write::UpdateRows { table_id, rows, .. } => {(*table_id, Some(rows.iter().map(|(row, _)| *row).collect()))}Write::MoveCard { table_id, row, .. } => (*table_id, Some(BTreeSet::from([*row]))),diff --git a/crates/databases/src/outbound/pg_databases_repo.rs b/crates/databases/src/outbound/pg_databases_repo.rsindex 1fe87a4ccb..c9da7301d0 100644--- a/crates/databases/src/outbound/pg_databases_repo.rs+++ b/crates/databases/src/outbound/pg_databases_repo.rs@@ -232,6 +232,34 @@ impl<Properties> PgDatabasesRepo<Properties> {pub fn new(pool: PgPool, properties: Properties) -> Self {Self { pool, properties }}++ /// Every table of a live database with a derived column, ordered by+ /// database and table.+ pub async fn tables_with_derived_columns(+ &self,+ ) -> Result<Vec<(DatabaseId, TableId)>, PgDatabasesRepoError> {+ let rows = sqlx::query!(+ r#"+ SELECT DISTINCT t.database_id, t.id+ FROM database_columns c+ JOIN database_tables t ON t.id = c.table_id+ JOIN database_entities d ON d.database_id = t.database_id+ WHERE c.config ->> 'kind' = 'derived' AND d.trashed_at IS NULL+ ORDER BY t.database_id, t.id+ "#+ )+ .fetch_all(&self.pool)+ .await?;+ Ok(rows+ .into_iter()+ .map(|row| {+ (+ DatabaseId::from_uuid(row.database_id),+ TableId::from_uuid(row.id),+ )+ })+ .collect())+ }}impl<Properties> DatabasesRepo for PgDatabasesRepo<Properties>diff --git a/crates/databases/src/outbound/pg_databases_repo/test.rs b/crates/databases/src/outbound/pg_databases_repo/test.rsindex 4a66548d55..2c8f7be7dc 100644--- a/crates/databases/src/outbound/pg_databases_repo/test.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test.rs@@ -16,6 +16,8 @@ mod apply_ops;mod cell_triggers;#[cfg(feature = "gateway")]mod changes;+#[cfg(feature = "gateway")]+mod derived_cells;mod entity_split;#[cfg(feature = "gateway")]mod journal;@@ -35,6 +37,8 @@ mod team_share;mod transfer;#[cfg(feature = "gateway")]mod undo;+#[cfg(feature = "view_rows")]+mod view_paging;#[cfg(feature = "gateway")]mod views;@@ -150,6 +154,7 @@ async fn commit() -> HashMap<TableId, TableVersion> {let outcome = PgCellStore::new(pool.clone(), PropertiesPgRepo::new(pool.clone())).apply_writes(&Writes {+ derivations: Vec::new(),database_id,created_by: user(),writes,diff --git a/crates/databases/src/outbound/pg_databases_repo/test/derived_cells.rs b/crates/databases/src/outbound/pg_databases_repo/test/derived_cells.rsnew file mode 100644index 0000000000..996da715a2--- /dev/null+++ b/crates/databases/src/outbound/pg_databases_repo/test/derived_cells.rs@@ -0,0 +1,739 @@+//! Formula columns over Postgres: the writer stores each row's computed+//! value in the formula's definition, so readers see an ordinary number or+//! date column, and keeps it current as inputs and formulas change.++use models_databases::{+ CellValue, CellWrite, ColumnChange, ColumnKind, DatabaseOp, Formula, NewColumn, OpResult,+ Operator, RowChange, RowChanges, RowsChange, RowsResult,+};+use models_properties::service::property_value::PropertyValue;++use super::apply_ops::{Guests, cells, edit, guests, service, version, viewer};+use super::journal::{WOLF, edit_as, insert_guests, person, wedding};+use super::*;+use crate::domain::journal::UndoOutcome;+use crate::domain::models::{DatabaseError, OpRefusal};+use crate::domain::ports::{DatabasesRepo, DatabasesService};++/// The guest list with Price and Qty, Total = Price * Qty and Double =+/// Total * 2.+struct Orders {+ guests: Guests,+ price: ColumnId,+ qty: ColumnId,+ total: ColumnId,+ double: ColumnId,+}++fn product(left: ColumnId, right: ColumnId) -> Formula {+ Formula::Binary {+ operator: Operator::Multiply,+ left: Box::new(Formula::Column { column: left }),+ right: Box::new(Formula::Column { column: right }),+ }+}++fn twice(column: ColumnId) -> Formula {+ Formula::Binary {+ operator: Operator::Multiply,+ left: Box::new(Formula::Column { column }),+ right: Box::new(Formula::Number { value: 2.0 }),+ }+}++fn number_column(column: ColumnId, name: &str, guests: &Guests) -> DatabaseOp {+ DatabaseOp::Column {+ table: guests.table_id,+ column,+ change: ColumnChange::Create {+ definition: NewColumn::New {+ name: name.into(),+ kind: ColumnKind::Number,+ options: vec![],+ infer_type: false,+ },+ after: None,+ },+ }+}++fn formula_column(column: ColumnId, name: &str, formula: Formula, guests: &Guests) -> DatabaseOp {+ DatabaseOp::Column {+ table: guests.table_id,+ column,+ change: ColumnChange::Create {+ definition: NewColumn::Derived {+ name: name.into(),+ formula,+ },+ after: None,+ },+ }+}++async fn orders(pool: &PgPool) -> Orders {+ let guests = guests(pool).await;+ let [price, qty, total, double] = [(); 4].map(|_| ColumnId::new());+ apply(+ pool,+ &guests,+ vec![+ number_column(price, "Price", &guests),+ number_column(qty, "Qty", &guests),+ formula_column(total, "Total", product(price, qty), &guests),+ formula_column(double, "Double", twice(total), &guests),+ ],+ )+ .await+ .unwrap();+ Orders {+ guests,+ price,+ qty,+ total,+ double,+ }+}++async fn apply(+ pool: &PgPool,+ guests: &Guests,+ ops: Vec<DatabaseOp>,+) -> Result<Vec<OpResult>, DatabaseError> {+ service(pool)+ .apply_ops(edit(guests.database_id), viewer(), ops.into())+ .await+}++async fn insert(pool: &PgPool, guests: &Guests, rows: Vec<Vec<CellWrite>>) -> Vec<RowId> {+ let results = apply(+ pool,+ guests,+ vec![DatabaseOp::Rows {+ table: guests.table_id,+ change: RowsChange::Insert { rows },+ }],+ )+ .await+ .unwrap();+ let [+ OpResult::Rows {+ change: RowsResult::Inserted { rows },+ ..+ },+ ] = results.as_slice()+ else {+ panic!("expected one insert, got {results:?}");+ };+ rows.clone()+}++fn number(column: ColumnId, value: f64) -> CellWrite {+ CellWrite {+ column,+ value: CellValue::Number(value),+ }+}++/// What a column's definition stores on each row, in the order given.+async fn stored(+ pool: &PgPool,+ table: TableId,+ column: ColumnId,+ rows: &[RowId],+) -> Vec<Option<f64>> {+ let definition = PgDatabasesRepo::new(pool.clone(), PropertiesPgRepo::new(pool.clone()))+ .columns_for_tables(&[table])+ .await+ .unwrap()+ .into_iter()+ .find(|placement| placement.id == column)+ .expect("the column should be stored")+ .property_definition_id;+ let values = cells(pool).column_cells(rows, definition).await.unwrap();+ rows.iter()+ .map(|row| match values.get(row) {+ Some(PropertyValue::Num(value)) => Some(*value),+ None => None,+ Some(other) => panic!("expected a number, got {other:?}"),+ })+ .collect()+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn inserted_rows_store_their_formula_cells(pool: PgPool) {+ let orders = orders(&pool).await;++ let rows = insert(+ &pool,+ &orders.guests,+ vec![+ vec![number(orders.price, 10.0), number(orders.qty, 3.0)],+ vec![number(orders.price, 5.0)],+ vec![CellWrite {+ column: orders.guests.name,+ value: CellValue::Text("Nothing priced".into()),+ }],+ ],+ )+ .await;++ let table = orders.guests.table_id;+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(30.0), Some(0.0), None]+ );+ assert_eq!(+ stored(&pool, table, orders.double, &rows).await,+ vec![Some(60.0), Some(0.0), None]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn updating_an_input_updates_the_formula_cells_reading_it(pool: PgPool) {+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![+ vec![number(orders.price, 10.0), number(orders.qty, 3.0)],+ vec![number(orders.price, 5.0)],+ ],+ )+ .await;+ let before = version(&pool, orders.guests.table_id).await;++ apply(+ &pool,+ &orders.guests,+ vec![DatabaseOp::Rows {+ table: orders.guests.table_id,+ change: RowsChange::Update {+ changes: RowChanges::PerRow {+ rows: vec![+ RowChange {+ row: rows[0],+ cells: vec![CellWrite {+ column: orders.price,+ value: CellValue::Clear,+ }],+ },+ RowChange {+ row: rows[1],+ cells: vec![number(orders.qty, 4.0)],+ },+ ],+ },+ },+ }],+ )+ .await+ .unwrap();++ let table = orders.guests.table_id;+ // Row 0 keeps Qty 3, and an empty Price reads as 0.+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(0.0), Some(20.0)]+ );+ assert_eq!(+ stored(&pool, table, orders.double, &rows).await,+ vec![Some(0.0), Some(40.0)]+ );+ assert_eq!(+ version(&pool, table).await,+ crate::domain::models::TableVersion(before.0 + 1)+ );++ apply(+ &pool,+ &orders.guests,+ vec![DatabaseOp::Rows {+ table,+ change: RowsChange::Update {+ changes: RowChanges::Uniform {+ rows: vec![rows[0]],+ cells: vec![CellWrite {+ column: orders.qty,+ value: CellValue::Clear,+ }],+ },+ },+ }],+ )+ .await+ .unwrap();+ // Every input of row 0 is empty now, so its formulas are too.+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![None, Some(20.0)]+ );+ assert_eq!(+ stored(&pool, table, orders.double, &rows).await,+ vec![None, Some(40.0)]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_formula_column_added_to_a_table_with_rows_computes_every_row(pool: PgPool) {+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![+ vec![number(orders.price, 10.0), number(orders.qty, 3.0)],+ vec![number(orders.price, 5.0)],+ vec![],+ ],+ )+ .await;+ let quadruple = ColumnId::new();++ apply(+ &pool,+ &orders.guests,+ vec![formula_column(+ quadruple,+ "Quadruple",+ twice(orders.double),+ &orders.guests,+ )],+ )+ .await+ .unwrap();++ assert_eq!(+ stored(&pool, orders.guests.table_id, quadruple, &rows).await,+ vec![Some(120.0), Some(0.0), None]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn changing_a_formula_recomputes_it_and_the_formulas_reading_it(pool: PgPool) {+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![+ vec![number(orders.price, 10.0), number(orders.qty, 3.0)],+ vec![number(orders.price, 5.0)],+ ],+ )+ .await;++ apply(+ &pool,+ &orders.guests,+ vec![DatabaseOp::Column {+ table: orders.guests.table_id,+ column: orders.total,+ change: ColumnChange::SetFormula {+ formula: Formula::Binary {+ operator: Operator::Add,+ left: Box::new(Formula::Column {+ column: orders.price,+ }),+ right: Box::new(Formula::Column { column: orders.qty }),+ },+ },+ }],+ )+ .await+ .unwrap();++ let table = orders.guests.table_id;+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(13.0), Some(5.0)]+ );+ assert_eq!(+ stored(&pool, table, orders.double, &rows).await,+ vec![Some(26.0), Some(10.0)]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn an_input_retyped_away_from_numbers_empties_the_formulas_reading_it(pool: PgPool) {+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![vec![number(orders.price, 10.0), number(orders.qty, 3.0)]],+ )+ .await;+ let table = orders.guests.table_id;+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(30.0)]+ );++ // An input cannot be deleted while a formula reads it.+ let deleted = apply(+ &pool,+ &orders.guests,+ vec![DatabaseOp::Column {+ table: orders.guests.table_id,+ column: orders.qty,+ change: ColumnChange::Delete,+ }],+ )+ .await;+ assert!(+ matches!(+ &deleted,+ Err(DatabaseError::InvalidOp(OpRefusal { reason, .. }))+ if reason == "Total's formula uses Qty; change that formula first"+ ),+ "{deleted:?}"+ );++ apply(+ &pool,+ &orders.guests,+ vec![DatabaseOp::Column {+ table: orders.guests.table_id,+ column: orders.qty,+ change: ColumnChange::ChangeType {+ to: ColumnKind::Text,+ },+ }],+ )+ .await+ .unwrap();++ assert_eq!(stored(&pool, table, orders.total, &rows).await, vec![None]);+ assert_eq!(stored(&pool, table, orders.double, &rows).await, vec![None]);+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn formula_cells_are_not_writable(pool: PgPool) {+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![vec![number(orders.price, 10.0), number(orders.qty, 3.0)]],+ )+ .await;++ let written = apply(+ &pool,+ &orders.guests,+ vec![DatabaseOp::Rows {+ table: orders.guests.table_id,+ change: RowsChange::Update {+ changes: RowChanges::Uniform {+ rows: rows.clone(),+ cells: vec![number(orders.total, 99.0)],+ },+ },+ }],+ )+ .await;++ assert!(+ matches!(+ &written,+ Err(DatabaseError::InvalidOp(OpRefusal { reason, .. }))+ if reason == "\"Total\" is computed by its formula; write the columns it uses instead"+ ),+ "{written:?}"+ );+ assert_eq!(+ stored(&pool, orders.guests.table_id, orders.total, &rows).await,+ vec![Some(30.0)]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn undoing_an_input_edit_or_a_removal_derives_the_formula_again(pool: PgPool) {+ let wedding = wedding(&pool).await;+ let (maria, omar) = insert_guests(&pool, &wedding).await;+ let seats = ColumnId::new();+ let service = service(&pool);+ let change = |ops: Vec<DatabaseOp>| {+ let service = &service;+ let wedding = &wedding;+ async move {+ service+ .apply_ops_with_changes(+ edit_as(WOLF, wedding.database_id),+ person(WOLF),+ ops.into(),+ )+ .await+ .unwrap()+ .changes[0]+ .change+ }+ };+ change(vec![DatabaseOp::Column {+ table: wedding.table_id,+ column: seats,+ change: ColumnChange::Create {+ definition: NewColumn::Derived {+ name: "Seats".into(),+ formula: Formula::Binary {+ operator: Operator::Add,+ left: Box::new(Formula::Column {+ column: wedding.plus_ones,+ }),+ right: Box::new(Formula::Number { value: 1.0 }),+ },+ },+ after: None,+ },+ }])+ .await;+ let rows = [maria, omar];+ // Omar has no plus-ones, his only input, so his seats are empty.+ assert_eq!(+ stored(&pool, wedding.table_id, seats, &rows).await,+ vec![Some(2.0), None]+ );++ let edited = change(vec![DatabaseOp::Rows {+ table: wedding.table_id,+ change: RowsChange::Update {+ changes: RowChanges::Uniform {+ rows: vec![maria],+ cells: vec![number(wedding.plus_ones, 3.0)],+ },+ },+ }])+ .await;+ assert_eq!(+ stored(&pool, wedding.table_id, seats, &rows).await,+ vec![Some(4.0), None]+ );+ let undone = service+ .undo_change(edit_as(WOLF, wedding.database_id), person(WOLF), edited)+ .await+ .unwrap();+ assert!(matches!(undone, UndoOutcome::Reverted { .. }), "{undone:?}");+ assert_eq!(+ stored(&pool, wedding.table_id, seats, &rows).await,+ vec![Some(2.0), None]+ );++ let column_removed = change(vec![DatabaseOp::Column {+ table: wedding.table_id,+ column: seats,+ change: ColumnChange::Delete,+ }])+ .await;+ let undone = service+ .undo_change(+ edit_as(WOLF, wedding.database_id),+ person(WOLF),+ column_removed,+ )+ .await+ .unwrap();+ assert!(matches!(undone, UndoOutcome::Reverted { .. }), "{undone:?}");+ assert_eq!(+ stored(&pool, wedding.table_id, seats, &rows).await,+ vec![Some(2.0), None]+ );++ let removed = change(vec![DatabaseOp::Rows {+ table: wedding.table_id,+ change: RowsChange::Delete { rows: vec![maria] },+ }])+ .await;+ let undone = service+ .undo_change(edit_as(WOLF, wedding.database_id), person(WOLF), removed)+ .await+ .unwrap();+ assert!(matches!(undone, UndoOutcome::Reverted { .. }), "{undone:?}");+ assert_eq!(+ stored(&pool, wedding.table_id, seats, &rows).await,+ vec![Some(2.0), None]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn a_batch_of_only_derivations_backfills_every_row_and_bumps_the_table(pool: PgPool) {+ use database_sql::catalog::{Column as QueryColumn, ColumnKind as QueryKind, TableSource};++ use crate::domain::derivation::Derivation;+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![+ vec![number(orders.price, 10.0), number(orders.qty, 3.0)],+ vec![number(orders.price, 5.0)],+ ],+ )+ .await;+ let table = orders.guests.table_id;+ let definitions = PgDatabasesRepo::new(pool.clone(), PropertiesPgRepo::new(pool.clone()))+ .columns_for_tables(&[table])+ .await+ .unwrap();+ let definition = |column: ColumnId| {+ definitions+ .iter()+ .find(|placement| placement.id == column)+ .unwrap()+ .property_definition_id+ };+ // Cells stored before formulas were materialized: inputs only.+ sqlx::query!(+ "DELETE FROM entity_properties WHERE property_definition_id = ANY($1)",+ &[definition(orders.total), definition(orders.double)],+ )+ .execute(&pool)+ .await+ .unwrap();+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![None, None]+ );+ let column = |placement: ColumnId, name: &str, formula: Option<Formula>| QueryColumn {+ id: definition(placement),+ placement,+ name: name.into(),+ kind: QueryKind::Number,+ formula,+ };+ let query_table = database_sql::catalog::Table {+ id: table,+ database_id: orders.guests.database_id,+ database: "Offsite".into(),+ name: "Table 1".into(),+ source: TableSource::Database,+ columns: vec![+ column(orders.price, "Price", None),+ column(orders.qty, "Qty", None),+ column(+ orders.total,+ "Total",+ Some(product(orders.price, orders.qty)),+ ),+ column(orders.double, "Double", Some(twice(orders.total))),+ ],+ };+ let before = version(&pool, table).await;++ let outcome = cells(&pool)+ .apply_writes(&Writes {+ database_id: orders.guests.database_id,+ created_by: viewer().user_id,+ writes: vec![],+ related_rows: vec![],+ expected_versions: vec![],+ journal: crate::domain::journal::JournalPlan::default(),+ derivations: Derivation::reading(&query_table, |_| true)+ .into_iter()+ .collect(),+ })+ .await+ .unwrap();++ assert!(+ matches!(outcome, WritesOutcome::Applied { .. }),+ "{outcome:?}"+ );+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(30.0), Some(0.0)]+ );+ assert_eq!(+ stored(&pool, table, orders.double, &rows).await,+ vec![Some(60.0), Some(0.0)]+ );+ assert_eq!(+ version(&pool, table).await,+ crate::domain::models::TableVersion(before.0 + 1)+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn the_backfill_stores_missing_formula_cells_and_a_second_run_changes_nothing(pool: PgPool) {+ use crate::backfill::{TableBackfill, backfill_table, tables_with_formulas};+ let orders = orders(&pool).await;+ let rows = insert(+ &pool,+ &orders.guests,+ vec![+ vec![number(orders.price, 10.0), number(orders.qty, 3.0)],+ vec![number(orders.price, 5.0)],+ vec![],+ ],+ )+ .await;+ let table = orders.guests.table_id;+ let columns = PgDatabasesRepo::new(pool.clone(), PropertiesPgRepo::new(pool.clone()))+ .columns_for_tables(&[table])+ .await+ .unwrap();+ let definition = |column: ColumnId| {+ columns+ .iter()+ .find(|placement| placement.id == column)+ .unwrap()+ .property_definition_id+ };+ // Cells stored before formulas were materialized: inputs only.+ sqlx::query!(+ "DELETE FROM entity_properties WHERE property_definition_id = ANY($1)",+ &[definition(orders.total), definition(orders.double)],+ )+ .execute(&pool)+ .await+ .unwrap();+ // A second database, whose table holds no formula, is not listed.+ service(&pool)+ .create_database(crate::domain::models::CreateDatabase {+ name: "Plain".into(),+ owner_id: viewer().user_id,+ acting_bot: None,+ })+ .await+ .unwrap();++ assert_eq!(+ tables_with_formulas(&pool).await.unwrap(),+ vec![(orders.guests.database_id, table)]+ );++ let before = version(&pool, table).await;+ assert_eq!(+ backfill_table(&pool, orders.guests.database_id, table)+ .await+ .unwrap(),+ TableBackfill {+ rows: 3,+ changed: true+ }+ );+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(30.0), Some(0.0), None]+ );+ assert_eq!(+ stored(&pool, table, orders.double, &rows).await,+ vec![Some(60.0), Some(0.0), None]+ );+ let backfilled = version(&pool, table).await;+ assert_eq!(+ backfilled,+ crate::domain::models::TableVersion(before.0 + 1)+ );++ assert_eq!(+ backfill_table(&pool, orders.guests.database_id, table)+ .await+ .unwrap(),+ TableBackfill {+ rows: 3,+ changed: false+ }+ );+ assert_eq!(version(&pool, table).await, backfilled);+ assert_eq!(+ stored(&pool, table, orders.total, &rows).await,+ vec![Some(30.0), Some(0.0), None]+ );+}diff --git a/crates/databases/src/outbound/pg_databases_repo/test/entity_split.rs b/crates/databases/src/outbound/pg_databases_repo/test/entity_split.rsindex 502f9938d5..d7b460deba 100644--- a/crates/databases/src/outbound/pg_databases_repo/test/entity_split.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test/entity_split.rs@@ -21,6 +21,7 @@ const WEBHOOKS_DOWN: &str = include_str!(concat!(fn batch(database_id: DatabaseId, writes: Vec<Write>) -> Writes {Writes {+ derivations: Vec::new(),database_id,created_by: user(),writes,@@ -72,6 +73,7 @@ async fn core_storage_supports_cells_and_versions_without_an_app_entity(pool: Pg}),},Write::InsertRows {+ derivation: None,table_id: table,rows: vec![vec![(definition, PropertyValue::Str("Acme".into()))]],restored: Vec::new(),@@ -100,6 +102,7 @@ async fn core_storage_supports_cells_and_versions_without_an_app_entity(pool: Pglet mut stale = batch(database,vec![Write::InsertRows {+ derivation: None,table_id: table,rows: vec![Vec::new()],restored: Vec::new(),diff --git a/crates/databases/src/outbound/pg_databases_repo/test/required.rs b/crates/databases/src/outbound/pg_databases_repo/test/required.rsindex 30fe5d132d..08e87eef93 100644--- a/crates/databases/src/outbound/pg_databases_repo/test/required.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test/required.rs@@ -6,6 +6,7 @@ use models_properties::service::property_value::PropertyValue;fn batch(database_id: DatabaseId, writes: Vec<Write>) -> Writes {Writes {+ derivations: Vec::new(),database_id,created_by: user(),writes,@@ -43,6 +44,7 @@ async fn required_cells_are_checked_for_app_and_core_writes(pool: PgPool) {let missing = batch(table.database_id,vec![Write::InsertRows {+ derivation: None,table_id: table.id,rows: vec![vec![]],restored: vec![],@@ -72,6 +74,7 @@ async fn required_cells_are_checked_for_app_and_core_writes(pool: PgPool) {.apply_storage_writes(&batch(table.database_id,vec![Write::InsertRows {+ derivation: None,table_id: table.id,rows: vec![vec![(definition, PropertyValue::Str(String::new()))]],restored: vec![],@@ -98,6 +101,7 @@ async fn required_cells_are_checked_for_app_and_core_writes(pool: PgPool) {.apply_storage_writes(&batch(table.database_id,vec![Write::UpdateRows {+ derivation: None,table_id: table.id,rows: vec![(row, vec![(definition, value)])],}],@@ -127,10 +131,12 @@ async fn required_cells_are_checked_for_app_and_core_writes(pool: PgPool) {table.database_id,vec![Write::UpdateRows {+ derivation: None,table_id: table.id,rows: vec![(row, vec![(definition, None)])],},Write::UpdateRows {+ derivation: None,table_id: table.id,rows: vec![(row,@@ -186,6 +192,7 @@ async fn adding_a_required_column_requires_values_for_existing_rows_in_the_same_vec![create,Write::UpdateRows {+ derivation: None,table_id: table.id,rows: vec![(row,diff --git a/crates/databases/src/outbound/pg_databases_repo/test/schema_ops.rs b/crates/databases/src/outbound/pg_databases_repo/test/schema_ops.rsindex 9a5bd68264..037cc79da3 100644--- a/crates/databases/src/outbound/pg_databases_repo/test/schema_ops.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test/schema_ops.rs@@ -25,6 +25,7 @@ async fn protected_schema_ops_are_refused_and_stale_writes_are_rechecked(pool: Plet guests = guests(&pool).await;// A write planned before the protection was registered must also be refused.let planned = Writes {+ derivations: Vec::new(),database_id: guests.database_id,created_by: viewer().user_id,writes: vec![Write::DeleteColumn {@@ -597,6 +598,7 @@ async fn a_stale_expected_version_conflicts_and_writes_nothing(pool: PgPool) {let outcome = cells(&pool).apply_writes(&Writes {+ derivations: Vec::new(),database_id: guests.database_id,created_by: viewer().user_id,writes: vec![diff --git a/crates/databases/src/outbound/pg_databases_repo/test/tables.rs b/crates/databases/src/outbound/pg_databases_repo/test/tables.rsindex efb2d1017e..ab8f09fbed 100644--- a/crates/databases/src/outbound/pg_databases_repo/test/tables.rs+++ b/crates/databases/src/outbound/pg_databases_repo/test/tables.rs@@ -92,6 +92,7 @@ async fn a_deleted_database_is_not_found_rather_than_a_name_conflict_or_storage_] {let outcome = store.apply_writes(&Writes {+ derivations: Vec::new(),database_id: table.database_id,created_by: user(),writes: vec![write],diff --git a/crates/databases/src/outbound/pg_databases_repo/test/view_paging.rs b/crates/databases/src/outbound/pg_databases_repo/test/view_paging.rsnew file mode 100644index 0000000000..5366acbe1d--- /dev/null+++ b/crates/databases/src/outbound/pg_databases_repo/test/view_paging.rs@@ -0,0 +1,969 @@+use database_sql::catalog::{Catalog, Column, ColumnKind, Table as QueryTable, TableSource};+use models_databases::views::{+ Conjunction, FilterCondition, FilterGroup, FilterNode, FilterTest, NumberOperator,+ SortDirection, SortKey, ViewQuery,+};+use models_properties::service::property_value::PropertyValue;++use super::*;+use crate::domain::view_rows::{ViewRowsError, ViewRowsRepository};+use crate::outbound::pg_view_rows::PgViewRows;++async fn page_fixture(pool: &PgPool) -> (QueryTable, Vec<RowId>) {+ let (repo, table, definition) = fixture(pool).await;+ let rows = repo.insert_rows(table.id, USER, 6).await.unwrap().unwrap();+ // Creation order is deliberately unrelated to the requested number order.+ for (row, value) in+ rows.iter()+ .zip([Some(40.0), Some(10.0), None, Some(20.0), Some(10.0), None])+ {+ if let Some(value) = value {+ sqlx::query!(+ "INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values) VALUES ($1, $2, 'DATABASE_ROW', $3, $4)",+ macro_uuid::generate_uuid_v7(), row.id.to_string(), definition,+ serde_json::to_value(PropertyValue::Num(value)).unwrap(),+ ).execute(pool).await.unwrap();+ }+ }+ (+ QueryTable {+ id: table.id,+ database_id: table.database_id,+ database: "Summer Offsite".into(),+ name: "Guests".into(),+ source: TableSource::Database,+ columns: vec![Column {+ id: definition,+ placement: ColumnId::from_uuid(definition),+ name: "Amount".into(),+ kind: ColumnKind::Number,+ formula: None,+ }],+ },+ rows.into_iter().map(|row| row.id).collect(),+ )+}++async fn pages(pool: &PgPool, table: &QueryTable, view: &ViewQuery) -> Vec<RowId> {+ let version = sqlx::query_scalar!(+ "SELECT version FROM database_tables WHERE id = $1",+ table.id.into_uuid()+ )+ .fetch_one(pool)+ .await+ .unwrap();+ let query = database_sql::compile_table_query(+ table.id,+ view,+ &Catalog {+ tables: vec![table.clone()],+ scope: None,+ },+ )+ .unwrap();+ let reader = PgViewRows::new(pool.clone());+ let mut all = Vec::new();+ loop {+ let page = reader+ .page(table, &query, TableVersion(version), all.last().copied(), 2)+ .await+ .unwrap();+ if page.is_empty() {+ break;+ }+ all.extend(page);+ assert!(all.len() <= 6, "a cursor must advance");+ }+ all+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn filters_and_sorting_precede_every_page_with_stable_ties_and_nulls_last(pool: PgPool) {+ let (table, rows) = page_fixture(&pool).await;+ assert_eq!(pages(&pool, &table, &ViewQuery::default()).await, rows);+ let mut view = ViewQuery {+ filter: None,+ sort: vec![SortKey {+ column: table.columns[0].placement,+ direction: SortDirection::Ascending,+ }],+ };+ assert_eq!(+ pages(&pool, &table, &view).await,+ vec![rows[1], rows[4], rows[3], rows[0], rows[2], rows[5]]+ );+ view.sort[0].direction = SortDirection::Descending;+ assert_eq!(+ pages(&pool, &table, &view).await,+ vec![rows[0], rows[3], rows[1], rows[4], rows[2], rows[5]]+ );+ view.filter = Some(FilterGroup {+ conjunction: Conjunction::And,+ conditions: vec![FilterNode::Condition(FilterCondition {+ column: table.columns[0].placement,+ test: FilterTest::Number {+ operator: NumberOperator::LessThanOrEqual,+ value: 20.0,+ },+ })],+ });+ assert_eq!(+ pages(&pool, &table, &view).await,+ vec![rows[3], rows[1], rows[4]]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn changed_versions_cannot_continue_an_old_page(pool: PgPool) {+ let (table, _) = page_fixture(&pool).await;+ let query = database_sql::compile_table_query(+ table.id,+ &ViewQuery::default(),+ &Catalog {+ tables: vec![table.clone()],+ scope: None,+ },+ )+ .unwrap();+ assert!(matches!(+ PgViewRows::new(pool)+ .page(&table, &query, TableVersion(-1), None, 2)+ .await,+ Err(ViewRowsError::Stale)+ ));+}++/// Price, Qty and Due on six rows, and four formulas over them:+/// Total = Price * Qty, Double = Total * 2, Ratio = Price / Qty and+/// Follow-up = Due + Qty. Only the inputs are written: the cell store+/// derives the formulas' cells as it inserts the rows.+async fn formula_fixture(pool: &PgPool) -> (QueryTable, Vec<RowId>) {+ use crate::domain::derivation::Derivation;+ use crate::domain::models::{ColumnConfig, NewDefinition};+ use models_databases::{Formula, Operator};+ use models_properties::DataType;+ let (repo, table, _) = fixture(pool).await;+ let [price, qty, due, total, double, ratio, follow_up] =+ [(); 7].map(|_| macro_uuid::generate_uuid_v7());+ let column = |id: Uuid| {+ Box::new(Formula::Column {+ column: ColumnId::from_uuid(id),+ })+ };+ let stored = |id: Uuid, name: &str, kind: ColumnKind| Column {+ id,+ placement: ColumnId::from_uuid(id),+ name: name.into(),+ kind,+ formula: None,+ };+ let derived = |id: Uuid, name: &str, kind: ColumnKind, formula: Formula| Column {+ formula: Some(formula),+ ..stored(id, name, kind)+ };+ let query_table = QueryTable {+ id: table.id,+ database_id: table.database_id,+ database: "Summer Offsite".into(),+ name: "Guests".into(),+ source: TableSource::Database,+ columns: vec![+ stored(price, "Price", ColumnKind::Number),+ stored(qty, "Qty", ColumnKind::Number),+ stored(due, "Due", ColumnKind::Date),+ derived(+ total,+ "Total",+ ColumnKind::Number,+ Formula::Binary {+ operator: Operator::Multiply,+ left: column(price),+ right: column(qty),+ },+ ),+ derived(+ double,+ "Double",+ ColumnKind::Number,+ Formula::Binary {+ operator: Operator::Multiply,+ left: column(total),+ right: Box::new(Formula::Number { value: 2.0 }),+ },+ ),+ derived(+ ratio,+ "Ratio",+ ColumnKind::Number,+ Formula::Binary {+ operator: Operator::Divide,+ left: column(price),+ right: column(qty),+ },+ ),+ derived(+ follow_up,+ "Follow-up",+ ColumnKind::Date,+ Formula::Binary {+ operator: Operator::Add,+ left: column(due),+ right: column(qty),+ },+ ),+ ],+ };+ let mut position: models_databases::position::Position = "80".parse().unwrap();+ let mut columns = Vec::new();+ for placed in &query_table.columns {+ position = models_databases::position::key_between(Some(&position), None).unwrap();+ columns.push(Write::CreateColumn {+ column: crate::domain::models::Column {+ protections: vec![],+ nullable: true,+ id: placed.placement,+ table_id: table.id,+ property_definition_id: placed.id,+ position: position.clone(),+ config: placed+ .formula+ .clone()+ .map(|formula| ColumnConfig::Derived { formula }),+ display_name: None,+ infer_type: false,+ },+ definition: Some(NewDefinition {+ id: placed.id,+ name: placed.name.clone(),+ data_type: match placed.kind {+ ColumnKind::Date => DataType::Date,+ _ => DataType::Number,+ },+ is_multi_select: false,+ specific_entity_type: None,+ options: vec![],+ }),+ });+ }+ commit(pool, table.database_id, columns).await;++ let number = |value: f64| PropertyValue::Num(value);+ let date = |value: &str| PropertyValue::Date(value.parse().unwrap());+ commit(+ pool,+ table.database_id,+ vec![Write::InsertRows {+ table_id: table.id,+ rows: vec![+ vec![+ (price, number(10.0)),+ (qty, number(3.0)),+ (due, date("2026-10-01T00:00:00Z")),+ ],+ vec![(price, number(5.0)), (due, date("2026-10-02T00:00:00Z"))],+ vec![],+ vec![+ (price, number(2.0)),+ (qty, number(50.0)),+ (due, date("2026-09-01T00:00:00Z")),+ ],+ vec![(price, number(7.0)), (qty, number(0.0))],+ vec![(qty, number(4.0)), (due, date("2026-10-03T00:00:00Z"))],+ ],+ restored: vec![],+ derivation: Derivation::reading(&query_table, |_| true),+ }],+ )+ .await;+ let rows = repo.row_refs(table.id).await.unwrap();+ (query_table, rows.into_iter().map(|row| row.id).collect())+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn formulas_filter_and_sort_with_the_engine_rules(pool: PgPool) {+ let (table, rows) = formula_fixture(&pool).await;+ let sorted = |column: usize, direction: SortDirection| ViewQuery {+ filter: None,+ sort: vec![SortKey {+ column: table.columns[column].placement,+ direction,+ }],+ };+ // Total: 30, 0 (empty Qty reads as 0), empty (every input empty), 100,+ // 0, 0 (empty Price reads as 0). Ties keep row order.+ assert_eq!(+ pages(&pool, &table, &sorted(3, SortDirection::Ascending)).await,+ vec![rows[1], rows[4], rows[5], rows[0], rows[3], rows[2]]+ );+ assert_eq!(+ pages(&pool, &table, &sorted(3, SortDirection::Descending)).await,+ vec![rows[3], rows[0], rows[1], rows[4], rows[5], rows[2]]+ );+ // Double reads Total, a formula itself: 60, 0, empty, 200, 0, 0.+ assert_eq!(+ pages(+ &pool,+ &table,+ &ViewQuery {+ filter: Some(FilterGroup {+ conjunction: Conjunction::And,+ conditions: vec![FilterNode::Condition(FilterCondition {+ column: table.columns[4].placement,+ test: FilterTest::Number {+ operator: NumberOperator::GreaterThanOrEqual,+ value: 60.0,+ },+ })],+ }),+ sort: vec![],+ }+ )+ .await,+ vec![rows[0], rows[3]]+ );+ // Ratio: 3.33, empty (5 / 0), empty, 0.04, empty (7 / 0), 0.+ assert_eq!(+ pages(&pool, &table, &sorted(5, SortDirection::Ascending)).await,+ vec![rows[5], rows[3], rows[0], rows[1], rows[2], rows[4]]+ );+ // Follow-up, Due + Qty days: Oct 4, Oct 2, empty, Oct 21, empty (no+ // Due), Oct 7.+ assert_eq!(+ pages(&pool, &table, &sorted(6, SortDirection::Ascending)).await,+ vec![rows[1], rows[0], rows[5], rows[3], rows[2], rows[4]]+ );+}++async fn values(+ pool: &PgPool,+ table: &QueryTable,+ rows: &[RowId],+ values: Vec<Option<PropertyValue>>,+) {+ for (row, value) in rows.iter().zip(values) {+ sqlx::query!(+ "INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values) VALUES ($1, $2, 'DATABASE_ROW', $3, $4) ON CONFLICT (entity_id, entity_type, property_definition_id) DO UPDATE SET values = EXCLUDED.values",+ macro_uuid::generate_uuid_v7(), row.to_string(), table.columns[0].id,+ value.map(|value| serde_json::to_value(value).unwrap()),+ ).execute(pool).await.unwrap();+ }+}++fn filtered(table: &QueryTable, test: FilterTest) -> ViewQuery {+ ViewQuery {+ filter: Some(FilterGroup {+ conjunction: Conjunction::And,+ conditions: vec![FilterNode::Condition(FilterCondition {+ column: table.columns[0].placement,+ test,+ })],+ }),+ sort: vec![],+ }+}++fn ascending(table: &QueryTable) -> ViewQuery {+ ViewQuery {+ filter: None,+ sort: vec![SortKey {+ column: table.columns[0].placement,+ direction: SortDirection::Ascending,+ }],+ }+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn text_filters_escape_wildcards_and_text_sorts_ignore_case(pool: PgPool) {+ use models_databases::views::{PresenceOperator, TextOperator};+ let (mut table, rows) = page_fixture(&pool).await;+ table.columns[0].kind = ColumnKind::Text;+ values(+ &pool,+ &table,+ &rows,+ [+ Some("beta"),+ Some("ALPHA"),+ None,+ Some("alpha"),+ Some("a_%\\z"),+ Some(""),+ ]+ .map(|value| value.map(|value| PropertyValue::Str(value.into())))+ .into(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[5], rows[4], rows[1], rows[3], rows[0], rows[2]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Text {+ operator: TextOperator::Contains,+ value: "_%\\".into()+ }+ )+ )+ .await,+ vec![rows[4]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Text {+ operator: TextOperator::Contains,+ value: "AlPhA".into()+ }+ )+ )+ .await,+ vec![rows[1], rows[3]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Text {+ operator: TextOperator::Is,+ value: "alpha".into()+ }+ )+ )+ .await,+ vec![rows[3]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Text {+ operator: TextOperator::IsNot,+ value: "alpha".into()+ }+ )+ )+ .await,+ vec![rows[0], rows[1], rows[2], rows[4], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Presence {+ operator: PresenceOperator::IsEmpty+ }+ )+ )+ .await,+ vec![rows[2]]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn checkbox_dates_and_links_keep_the_existing_cell_semantics(pool: PgPool) {+ use models_databases::views::{DateOperator, PresenceOperator, TextOperator};+ let (mut table, rows) = page_fixture(&pool).await;+ table.columns[0].kind = ColumnKind::Boolean;+ values(+ &pool,+ &table,+ &rows,+ [Some(true), Some(false), None, Some(true), Some(false), None]+ .map(|value| value.map(PropertyValue::Bool))+ .into(),+ )+ .await;+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(&table, FilterTest::Checkbox { checked: false })+ )+ .await,+ vec![rows[1], rows[2], rows[4], rows[5]]+ );+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[1], rows[4], rows[0], rows[3], rows[2], rows[5]]+ );+ table.columns[0].kind = ColumnKind::Date;+ values(+ &pool,+ &table,+ &rows,+ [+ Some("2026-01-02T00:00:00Z"),+ Some("2026-01-01T00:00:00+01:00"),+ None,+ Some("2026-01-01T00:00:00Z"),+ Some("2026-01-02T00:00:00Z"),+ None,+ ]+ .map(|value| value.map(|value| PropertyValue::Date(value.parse().unwrap())))+ .into(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[1], rows[3], rows[0], rows[4], rows[2], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Date {+ operator: DateOperator::OnOrBefore,+ value: "2026-01-01T00:00:00Z".parse().unwrap()+ }+ )+ )+ .await,+ vec![rows[1], rows[3]]+ );+ table.columns[0].kind = ColumnKind::Link;+ values(+ &pool,+ &table,+ &rows,+ vec![+ Some(PropertyValue::Link(vec![+ "https://a.test".into(),+ "https://b.test".into(),+ ])),+ Some(PropertyValue::Link(vec![])),+ None,+ None,+ None,+ None,+ ],+ )+ .await;+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Text {+ operator: TextOperator::Is,+ value: "https://a.test https://b.test".into()+ }+ )+ )+ .await,+ vec![rows[0]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Presence {+ operator: PresenceOperator::IsEmpty+ }+ )+ )+ .await,+ vec![rows[2], rows[3], rows[4], rows[5]]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn select_and_entity_pages_preserve_option_rank_membership_and_empty_sets(pool: PgPool) {+ use database_sql::catalog::{EntityKind, SelectOption};+ use models_databases::{OptionId, views::SetOperator};+ use models_properties::shared::{EntityReference, EntityType};+ let (mut table, rows) = page_fixture(&pool).await;+ let first = OptionId::from_uuid(Uuid::from_u128(201));+ let second = OptionId::from_uuid(Uuid::from_u128(202));+ let options = vec![+ SelectOption {+ id: second,+ label: "Second".into(),+ },+ SelectOption {+ id: first,+ label: "First".into(),+ },+ ];+ table.columns[0].kind = ColumnKind::Select {+ multi: false,+ options: options.clone(),+ };+ values(+ &pool,+ &table,+ &rows,+ vec![+ Some(vec![first]),+ Some(vec![second]),+ None,+ Some(vec![first]),+ Some(vec![]),+ Some(vec![second]),+ ]+ .into_iter()+ .map(|value| {+ value.map(|ids| {+ PropertyValue::SelectOption(ids.into_iter().map(OptionId::into_uuid).collect())+ })+ })+ .collect(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[1], rows[5], rows[0], rows[3], rows[2], rows[4]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Options {+ operator: SetOperator::IsAnyOf,+ options: vec![first, second],+ }+ )+ )+ .await,+ vec![rows[0], rows[1], rows[3], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Options {+ operator: SetOperator::IsNoneOf,+ options: vec![second]+ }+ )+ )+ .await,+ vec![rows[0], rows[2], rows[3], rows[4]]+ );+ table.columns[0].kind = ColumnKind::Select {+ multi: true,+ options,+ };+ values(+ &pool,+ &table,+ &rows,+ vec![+ Some(vec![first, second]),+ Some(vec![second]),+ None,+ Some(vec![first]),+ Some(vec![]),+ Some(vec![second, first]),+ ]+ .into_iter()+ .map(|value| {+ value.map(|ids| {+ PropertyValue::SelectOption(ids.into_iter().map(OptionId::into_uuid).collect())+ })+ })+ .collect(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[1], rows[5], rows[3], rows[0], rows[2], rows[4]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Options {+ operator: SetOperator::HasAny,+ options: vec![first],+ }+ )+ )+ .await,+ vec![rows[0], rows[3], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Options {+ operator: SetOperator::HasAll,+ options: vec![first, second]+ }+ )+ )+ .await,+ vec![rows[0], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Options {+ operator: SetOperator::HasNone,+ options: vec![first]+ }+ )+ )+ .await,+ vec![rows[1], rows[2], rows[4]]+ );+ table.columns[0].kind = ColumnKind::Entity {+ multi: true,+ target: EntityKind::User,+ };+ values(+ &pool,+ &table,+ &rows,+ vec![+ Some(vec!["Z"]),+ Some(vec!["a"]),+ None,+ Some(vec!["b", "a"]),+ Some(vec![]),+ Some(vec!["a", "b"]),+ ]+ .into_iter()+ .map(|value| {+ value.map(|ids| {+ PropertyValue::EntityRef(+ ids.into_iter()+ .map(|id| EntityReference {+ entity_id: id.into(),+ entity_type: EntityType::User,+ specific_message_id: None,+ })+ .collect(),+ )+ })+ })+ .collect(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[0], rows[1], rows[5], rows[3], rows[2], rows[4]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Entities {+ operator: SetOperator::HasAny,+ entities: vec!["a".into()],+ }+ )+ )+ .await,+ vec![rows[1], rows[3], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Entities {+ operator: SetOperator::HasNone,+ entities: vec!["a".into()]+ }+ )+ )+ .await,+ vec![rows[0], rows[2], rows[4]]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn unicode_case_mapping_matches_the_view_engine(pool: PgPool) {+ use models_databases::views::TextOperator;+ let (mut table, rows) = page_fixture(&pool).await;+ table.columns[0].kind = ColumnKind::Text;+ values(+ &pool,+ &table,+ &rows,+ [+ Some("ΟΣ"),+ Some("οσ"),+ Some("İ"),+ Some("i"),+ None,+ Some("É"),+ ]+ .map(|value| value.map(|value| PropertyValue::Str(value.into())))+ .into(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[3], rows[2], rows[5], rows[0], rows[1], rows[4]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Text {+ operator: TextOperator::Contains,+ value: "ΟΣ".into()+ }+ )+ )+ .await,+ vec![rows[0]]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn date_comparisons_preserve_sub_microsecond_precision(pool: PgPool) {+ use models_databases::views::DateOperator;+ let (mut table, rows) = page_fixture(&pool).await;+ table.columns[0].kind = ColumnKind::Date;+ values(+ &pool,+ &table,+ &rows,+ [+ Some("2026-01-01T00:00:00.000000002Z"),+ Some("2026-01-01T00:00:00.000000001Z"),+ None,+ Some("2026-01-01T00:00:00Z"),+ Some("2026-01-01T00:00:00.999999999Z"),+ None,+ ]+ .map(|value| value.map(|value| PropertyValue::Date(value.parse().unwrap())))+ .into(),+ )+ .await;+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[3], rows[1], rows[0], rows[4], rows[2], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &filtered(+ &table,+ FilterTest::Date {+ operator: DateOperator::OnOrBefore,+ value: "2026-01-01T00:00:00.000000001Z".parse().unwrap()+ }+ )+ )+ .await,+ vec![rows[1], rows[3]]+ );+}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn date_ranges_sort_by_start_then_end_and_filter_by_day_like_the_engine(pool: PgPool) {+ use models_databases::views::DateRangeOperator;+ use models_properties::shared::DateRange;+ let (mut table, rows) = page_fixture(&pool).await;+ table.columns[0].kind = ColumnKind::DateRange;+ let range = |start: &str, end: &str| {+ PropertyValue::DateRange(+ DateRange::new(start.parse().unwrap(), end.parse().unwrap()).unwrap(),+ )+ };+ values(+ &pool,+ &table,+ &rows,+ vec![+ Some(range("2026-01-03T00:00:00Z", "2026-01-05T00:00:00Z")),+ Some(range("2026-01-01T00:00:00Z", "2026-01-10T00:00:00Z")),+ None,+ Some(range("2026-01-01T00:00:00Z", "2026-01-02T00:00:00Z")),+ Some(range("2026-01-06T00:00:00Z", "2026-01-06T00:00:00Z")),+ None,+ ],+ )+ .await;+ // Start first, then end; empty ranges last.+ assert_eq!(+ pages(&pool, &table, &ascending(&table)).await,+ vec![rows[3], rows[1], rows[0], rows[4], rows[2], rows[5]]+ );+ let day = |operator: DateRangeOperator, day: &str| {+ filtered(+ &table,+ FilterTest::DateRange {+ operator,+ value: day.parse().unwrap(),+ },+ )+ };+ // Overlaps January 4.+ assert_eq!(+ pages(+ &pool,+ &table,+ &day(DateRangeOperator::Contains, "2026-01-04T00:00:00Z")+ )+ .await,+ vec![rows[0], rows[1]]+ );+ // Misses January 4, empty ranges included.+ assert_eq!(+ pages(+ &pool,+ &table,+ &day(DateRangeOperator::DoesNotContain, "2026-01-04T00:00:00Z")+ )+ .await,+ vec![rows[2], rows[3], rows[4], rows[5]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &day(DateRangeOperator::EndsBefore, "2026-01-03T00:00:00Z")+ )+ .await,+ vec![rows[3]]+ );+ assert_eq!(+ pages(+ &pool,+ &table,+ &day(DateRangeOperator::StartsAfter, "2026-01-05T00:00:00Z")+ )+ .await,+ vec![rows[4]]+ );+}diff --git a/crates/databases/src/outbound/pg_view_rows.rs b/crates/databases/src/outbound/pg_view_rows.rsnew file mode 100644index 0000000000..f166f51fa4--- /dev/null+++ b/crates/databases/src/outbound/pg_view_rows.rs@@ -0,0 +1,71 @@+//! PostgreSQL reads in a view's global order, with keyset pagination.++mod query;++use database_sql::{catalog::Table, resolve::SelectQuery};+use models_databases::{RowId, TableVersion};+use sqlx::PgPool;+use tracing::Instrument;++use crate::domain::view_rows::{ViewRowsError, ViewRowsRepository};++/// The database-row read model, including the property values that determine+/// membership and order. Only the selected identities are hydrated afterward.+pub struct PgViewRows {+ pool: PgPool,+}++impl PgViewRows {+ /// Use the primary so accepted edits are visible on the following read.+ pub fn new(pool: PgPool) -> Self {+ Self { pool }+ }+}++fn persistence(error: sqlx::Error) -> ViewRowsError {+ ViewRowsError::Infrastructure(rootcause::Report::new(error).into())+}++impl ViewRowsRepository for PgViewRows {+ #[tracing::instrument(name = "database.view_rows.page", skip_all, fields(database.table_id = %table.id, database.page_limit = limit, database.rows), err)]+ async fn page(+ &self,+ table: &Table,+ query: &SelectQuery,+ version: TableVersion,+ after: Option<RowId>,+ limit: u16,+ ) -> Result<Vec<RowId>, ViewRowsError> {+ let mut sql = query::page(table, query, after, limit)?;+ let mut transaction = self+ .pool+ .begin()+ .instrument(tracing::info_span!("database.view_rows.pool.acquire"))+ .await+ .map_err(persistence)?;+ sqlx::query!("SET TRANSACTION ISOLATION LEVEL REPEATABLE READ, READ ONLY")+ .execute(&mut *transaction)+ .await+ .map_err(persistence)?;+ let actual = sqlx::query_scalar!(+ "SELECT version FROM database_tables WHERE id = $1 AND database_id = $2",+ table.id.into_uuid(),+ table.database_id.into_uuid(),+ )+ .fetch_optional(&mut *transaction)+ .await+ .map_err(persistence)?;+ if actual != Some(version.0) {+ return Err(ViewRowsError::Stale);+ }+ let rows: Vec<uuid::Uuid> = sql+ .build_query_scalar()+ .fetch_all(&mut *transaction)+ .instrument(tracing::info_span!("database.view_rows.select"))+ .await+ .map_err(persistence)?;+ transaction.commit().await.map_err(persistence)?;+ tracing::Span::current().record("database.rows", rows.len());+ Ok(rows.into_iter().map(RowId::from_uuid).collect())+ }+}diff --git a/crates/databases/src/outbound/pg_view_rows/query.rs b/crates/databases/src/outbound/pg_view_rows/query.rsnew file mode 100644index 0000000000..9c709ba533--- /dev/null+++ b/crates/databases/src/outbound/pg_view_rows/query.rs@@ -0,0 +1,502 @@+//! Parameterized SQL for the engine's validated, single-table view plan.++use std::collections::BTreeSet;++use database_sql::{+ catalog::{Column, ColumnKind, Table, TableSource},+ resolve::{+ ComparisonOperator, Direction, Filter, OrderKey, SelectQuery, Value, row_position_key,+ },+};+use models_databases::RowId;+use sqlx::{Postgres, QueryBuilder};+use uuid::Uuid;++use crate::domain::view_rows::ViewRowsError;++type Sql = QueryBuilder<'static, Postgres>;++fn filter_columns(filter: &Filter, columns: &mut BTreeSet<Uuid>) {+ match filter {+ Filter::And(parts) | Filter::Or(parts) => {+ for part in parts {+ filter_columns(part, columns);+ }+ }+ Filter::Comparison { column, .. }+ | Filter::In { column, .. }+ | Filter::Has { column, .. }+ | Filter::IsNull { column, .. }+ | Filter::Like { column, .. } => {+ columns.insert(*column);+ }+ }+}++struct Columns<'a>(Vec<&'a Column>);++impl Columns<'_> {+ fn index(&self, id: Uuid) -> Result<usize, ViewRowsError> {+ self.0+ .iter()+ .position(|column| column.id == id)+ .ok_or(ViewRowsError::InvalidQuery)+ }++ fn kind(&self, id: Uuid) -> Result<&ColumnKind, ViewRowsError> {+ Ok(&self.0[self.index(id)?].kind)+ }++ fn reference(&self, sql: &mut Sql, id: Uuid, scalar: bool) -> Result<(), ViewRowsError> {+ let index = self.index(id)?;+ let array = matches!(+ self.0[index].kind,+ ColumnKind::Select { .. } | ColumnKind::Entity { .. }+ );+ if scalar && array {+ sql.push(format!(+ "(CASE WHEN cardinality(v{index}) = 1 THEN v{index}[1] END)"+ ));+ } else {+ sql.push(format!("v{index}"));+ }+ Ok(())+ }+}++/// Only IDs and compiler-generated aliases become SQL identifiers. Every+/// caller-supplied value, option ID and cursor is a bind parameter.+pub(super) fn page(+ table: &Table,+ query: &SelectQuery,+ after: Option<RowId>,+ limit: u16,+) -> Result<Sql, ViewRowsError> {+ if table.source != TableSource::Database+ || query.relations.len() != 1+ || query.relations[0].table != table.id+ || !query.joins.is_empty()+ || query.group_by.is_some()+ || query.distinct+ || !(1..=501).contains(&limit)+ {+ return Err(ViewRowsError::InvalidQuery);+ }+ let position = row_position_key(table.id);+ let mut needed = BTreeSet::new();+ if let Some(filter) = &query.where_ {+ filter_columns(filter, &mut needed);+ }+ for order in &query.order_by {+ let OrderKey::Column(column) = order.key else {+ return Err(ViewRowsError::InvalidQuery);+ };+ if column != position {+ needed.insert(column);+ }+ }+ let columns = Columns(+ needed+ .into_iter()+ .map(|id| {+ table+ .columns+ .iter()+ // A formula's values are stored in its definition like any cell.+ .find(|column| column.id == id)+ .ok_or(ViewRowsError::InvalidQuery)+ })+ .collect::<Result<_, _>>()?,+ );+ let mut sql = Sql::new("WITH cells AS NOT MATERIALIZED (SELECT r.id, r.position");+ for (index, column) in columns.0.iter().enumerate() {+ sql.push(", ");+ cell(&mut sql, index, &column.kind);+ sql.push(format!(" AS v{index}, p{index}.values AS raw{index}"));+ }+ sql.push(" FROM database_rows r");+ for (index, column) in columns.0.iter().enumerate() {+ // Keep both keys indexable: filters may start from properties, whereas+ // an unfiltered ordered page starts from database_rows.+ sql.push(format!(" LEFT JOIN entity_properties p{index} ON p{index}.entity_id = r.id::text AND r.id = CASE WHEN p{index}.entity_type = 'DATABASE_ROW' THEN p{index}.entity_id::uuid END AND p{index}.entity_type = 'DATABASE_ROW' AND p{index}.property_definition_id = "))+ .push_bind(column.id);+ }+ sql.push(" WHERE r.table_id = ")+ .push_bind(table.id.into_uuid());+ sql.push("), candidates AS NOT MATERIALIZED (SELECT cells.*");+ for (index, order) in query.order_by.iter().enumerate() {+ sql.push(", ");+ let OrderKey::Column(column) = order.key else {+ return Err(ViewRowsError::InvalidQuery);+ };+ if column == position {+ sql.push("position");+ } else {+ sort_value(&mut sql, &columns, column)?;+ }+ sql.push(format!(" AS k{index}"));+ }+ sql.push(" FROM cells");+ if let Some(filter) = &query.where_ {+ sql.push(" WHERE ");+ predicate(&mut sql, &columns, filter)?;+ }+ sql.push(") SELECT c.id FROM candidates c");+ if let Some(after) = after {+ sql.push(" CROSS JOIN (SELECT * FROM candidates WHERE id = ")+ .push_bind(after.into_uuid())+ .push(") pivot WHERE ");+ if query.order_by.len() == 1+ && query.order_by[0].key == OrderKey::Column(position)+ && query.order_by[0].direction == Direction::Ascending+ {+ sql.push("(c.k0, c.id) > (pivot.k0, pivot.id)");+ } else {+ seek(&mut sql, query);+ }+ }+ sql.push(" ORDER BY ");+ for (index, order) in query.order_by.iter().enumerate() {+ sql.push(format!(+ "c.k{index} {} NULLS LAST, ",+ direction(order.direction)+ ));+ }+ sql.push("c.id ASC LIMIT ").push_bind(i64::from(limit));+ Ok(sql)+}++fn direction(direction: Direction) -> &'static str {+ match direction {+ Direction::Ascending => "ASC",+ Direction::Descending => "DESC",+ }+}++/// Lexicographic continuation with empty values last in either direction.+fn seek(sql: &mut Sql, query: &SelectQuery) {+ sql.push("(");+ for at in 0..=query.order_by.len() {+ if at > 0 {+ sql.push(" OR ");+ }+ sql.push("(");+ for prefix in 0..at {+ sql.push(format!(+ "c.k{prefix} IS NOT DISTINCT FROM pivot.k{prefix} AND "+ ));+ }+ if let Some(order) = query.order_by.get(at) {+ let operator = match order.direction {+ Direction::Ascending => ">",+ Direction::Descending => "<",+ };+ sql.push(format!(+ "(c.k{at} {operator} pivot.k{at} OR (c.k{at} IS NULL AND pivot.k{at} IS NOT NULL))"+ ));+ } else {+ sql.push("c.id > pivot.id");+ }+ sql.push(")");+ }+ sql.push(")");+}++fn cell(sql: &mut Sql, index: usize, kind: &ColumnKind) {+ let property = format!("p{index}.values");+ let (tag, value) = match kind {+ ColumnKind::Text => ("String", format!("({property}->>'value') COLLATE \"C\"")),+ ColumnKind::Number => (+ "Number",+ format!("({property}->>'value')::double precision"),+ ),+ ColumnKind::Boolean => ("Boolean", format!("({property}->>'value')::boolean")),+ ColumnKind::Date => ("Date", date_value(&format!("({property}->>'value')"))),+ // `[start, end]`: arrays compare element by element, so a sort by+ // start then end and the keyset seek need nothing more.+ ColumnKind::DateRange => (+ "DateRange",+ format!(+ "ARRAY[{}, {}]",+ date_value(&format!("({property}->'value'->>'start')")),+ date_value(&format!("({property}->'value'->>'end')"))+ ),+ ),+ ColumnKind::Link => (+ "Link",+ format!(+ "COALESCE((SELECT string_agg(value, ' ' ORDER BY ordinal) FROM jsonb_array_elements_text({property}->'value') WITH ORDINALITY AS links(value, ordinal)), '') COLLATE \"C\""+ ),+ ),+ ColumnKind::Select { .. } => (+ "SelectOption",+ format!(+ "NULLIF(ARRAY(SELECT value FROM jsonb_array_elements_text({property}->'value') WITH ORDINALITY AS options(value, ordinal) ORDER BY ordinal), '{{}}'::text[]) COLLATE \"C\""+ ),+ ),+ ColumnKind::Entity { .. } => (+ "EntityReference",+ format!(+ "NULLIF(ARRAY(SELECT value->>'entity_id' FROM jsonb_array_elements({property}->'value') WITH ORDINALITY AS refs(value, ordinal) ORDER BY ordinal), '{{}}'::text[]) COLLATE \"C\""+ ),+ ),+ };+ sql.push(format!(+ "CASE WHEN {property}->>'type' = '{tag}' THEN {value} END"+ ));+}++// Properties serialize DateTime<Utc> as RFC3339 UTC. Keep the fractional+// seconds separate: PostgreSQL timestamps round to microseconds, but the+// view engine compares every stored nanosecond.+fn date_value(text: &str) -> String {+ format!(+ "(extract(epoch FROM (split_part(rtrim({text}, 'Z'), '.', 1) || 'Z')::timestamptz) + COALESCE(('0.' || substring({text} FROM '[.]([0-9]+)'))::numeric, 0))"+ )+}++fn sort_value(sql: &mut Sql, columns: &Columns<'_>, id: Uuid) -> Result<(), ViewRowsError> {+ let index = columns.index(id)?;+ match &columns.0[index].kind {+ ColumnKind::Text | ColumnKind::Link => {+ sql.push(format!(+ "lower(v{index} COLLATE \"und-x-icu\") COLLATE \"C\""+ ));+ }+ ColumnKind::Select { options, .. } => {+ sql.push(format!(+ "CASE WHEN v{index} IS NOT NULL THEN ARRAY(SELECT COALESCE(array_position("+ ));+ sql.push_bind(+ options+ .iter()+ .map(|option| option.id.to_string())+ .collect::<Vec<_>>(),+ );+ sql.push(format!("::text[], value), 2147483647) FROM unnest(v{index}) WITH ORDINALITY AS members(value, ordinal) ORDER BY ordinal) END"));+ }+ _ => {+ sql.push(format!("v{index}"));+ }+ }+ Ok(())+}++fn literal(sql: &mut Sql, value: &Value) -> Result<(), ViewRowsError> {+ match value {+ Value::Text(value) | Value::Entity(value) => {+ sql.push_bind(value.clone());+ }+ Value::Number(value) => {+ sql.push_bind(*value);+ }+ Value::Bool(value) => {+ sql.push_bind(*value);+ }+ Value::Date(value) => {+ sql.push("(")+ .push_bind(value.timestamp())+ .push("::numeric + ")+ .push_bind(i64::from(value.timestamp_subsec_nanos()))+ .push("::numeric / 1000000000)");+ }+ Value::Option(value) => {+ sql.push_bind(value.to_string());+ }+ Value::Options(_) | Value::Entities(_) | Value::DateRange(_) => {+ return Err(ViewRowsError::InvalidQuery);+ }+ }+ Ok(())+}++fn predicate(sql: &mut Sql, columns: &Columns<'_>, filter: &Filter) -> Result<(), ViewRowsError> {+ sql.push("(");+ // The typed expressions below retain scalar/list and empty-cell semantics.+ // Expose positive membership separately so the property GIN index can prune+ // candidates before PostgreSQL expands JSON arrays.+ match filter {+ Filter::Has {+ column,+ value,+ negated: false,+ } => {+ membership_index(sql, columns, *column, std::slice::from_ref(value))?;+ }+ Filter::In {+ column,+ values,+ negated: false,+ } => {+ membership_index(sql, columns, *column, values)?;+ }+ _ => {}+ }+ match filter {+ Filter::And(parts) | Filter::Or(parts) => {+ if parts.is_empty() {+ sql.push(if matches!(filter, Filter::And(_)) {+ "TRUE"+ } else {+ "FALSE"+ });+ }+ for (index, part) in parts.iter().enumerate() {+ if index > 0 {+ sql.push(if matches!(filter, Filter::And(_)) {+ " AND "+ } else {+ " OR "+ });+ }+ predicate(sql, columns, part)?;+ }+ }+ Filter::Comparison {+ column,+ operator,+ value,+ } if columns.kind(*column)? == &ColumnKind::DateRange => {+ range_comparison(sql, columns.index(*column)?, *operator, value)?;+ }+ Filter::Comparison {+ column,+ operator,+ value,+ } => {+ columns.reference(sql, *column, true)?;+ sql.push(match operator {+ ComparisonOperator::Equal => " = ",+ ComparisonOperator::NotEqual => " <> ",+ ComparisonOperator::Less => " < ",+ ComparisonOperator::LessOrEqual => " <= ",+ ComparisonOperator::Greater => " > ",+ ComparisonOperator::GreaterOrEqual => " >= ",+ });+ literal(sql, value)?;+ }+ Filter::In {+ column,+ values,+ negated,+ } => {+ if values.is_empty() {+ return Err(ViewRowsError::InvalidQuery);+ }+ columns.reference(sql, *column, true)?;+ sql.push(if *negated { " NOT IN (" } else { " IN (" });+ for (index, value) in values.iter().enumerate() {+ if index > 0 {+ sql.push(", ");+ }+ literal(sql, value)?;+ }+ sql.push(")");+ }+ Filter::Has {+ column,+ value,+ negated,+ } => {+ sql.push(if *negated {+ "NOT COALESCE("+ } else {+ "COALESCE("+ });+ literal(sql, value)?;+ sql.push(" = ANY(");+ columns.reference(sql, *column, false)?;+ sql.push("), FALSE)");+ }+ Filter::IsNull { column, negated } => {+ columns.reference(sql, *column, false)?;+ sql.push(if *negated { " IS NOT NULL" } else { " IS NULL" });+ }+ Filter::Like {+ column,+ pattern,+ escape,+ negated,+ } => {+ sql.push("lower(");+ columns.reference(sql, *column, true)?;+ sql.push(" COLLATE \"und-x-icu\") COLLATE \"C\"");+ sql.push(if *negated { " NOT LIKE " } else { " LIKE " });+ sql.push_bind(pattern.to_lowercase());+ sql.push(" ESCAPE ")+ .push_bind(escape.map(|value| value.to_string()).unwrap_or_default());+ }+ }+ sql.push(")");+ Ok(())+}++/// A range compared to a date the way the engine reads it+/// (`DateRange::compare_to`): `=` contains it, `<` ends before it, `<=`+/// starts on or before it, `>` starts after it and `>=` ends on or after it.+fn range_comparison(+ sql: &mut Sql,+ index: usize,+ operator: ComparisonOperator,+ value: &Value,+) -> Result<(), ViewRowsError> {+ let start = format!("v{index}[1]");+ let end = format!("v{index}[2]");+ let side = |sql: &mut Sql, bound: &str, symbol: &str| {+ sql.push(format!("{bound} {symbol} "));+ literal(sql, value)+ };+ match operator {+ ComparisonOperator::Equal => {+ sql.push("(");+ side(sql, &start, "<=")?;+ sql.push(" AND ");+ side(sql, &end, ">=")?;+ sql.push(")");+ }+ ComparisonOperator::NotEqual => {+ sql.push("(");+ side(sql, &end, "<")?;+ sql.push(" OR ");+ side(sql, &start, ">")?;+ sql.push(")");+ }+ ComparisonOperator::Less => side(sql, &end, "<")?,+ ComparisonOperator::LessOrEqual => side(sql, &start, "<=")?,+ ComparisonOperator::Greater => side(sql, &start, ">")?,+ ComparisonOperator::GreaterOrEqual => side(sql, &end, ">=")?,+ }+ Ok(())+}++fn membership_index(+ sql: &mut Sql,+ columns: &Columns<'_>,+ column: Uuid,+ values: &[Value],+) -> Result<(), ViewRowsError> {+ let index = columns.index(column)?;+ let needles = values+ .iter()+ .map(|value| match (&columns.0[index].kind, value) {+ (ColumnKind::Select { .. }, Value::Option(option)) => Some(serde_json::json!({+ "type": "SelectOption", "value": [option.to_string()]+ })),+ (ColumnKind::Entity { .. }, Value::Entity(entity)) => Some(serde_json::json!({+ "type": "EntityReference", "value": [{"entity_id": entity}]+ })),+ _ => None,+ })+ .collect::<Option<Vec<_>>>();+ if let Some(needles) = needles.filter(|values| !values.is_empty()) {+ sql.push("(");+ for (at, needle) in needles.into_iter().enumerate() {+ if at > 0 {+ sql.push(" OR ");+ }+ sql.push(format!("raw{index} @> ")).push_bind(needle);+ }+ sql.push(") AND ");+ }+ Ok(())+}diff --git a/crates/databases_sql/Cargo.toml b/crates/databases_sql/Cargo.tomlindex b661fd1c07..b7a3f2d19f 100644--- a/crates/databases_sql/Cargo.toml+++ b/crates/databases_sql/Cargo.toml@@ -19,11 +19,12 @@ utoipa = { workspace = true, optional = true }ai_toolset = { path = "../ai_toolset", optional = true }anyhow = { workspace = true, optional = true }async-trait = { workspace = true, optional = true }+base64 = { workspace = true }bot_id = { path = "../bot_id" }chrono = { workspace = true }contacts = { path = "../contacts", default-features = false, features = ["ports"] }database_sql = { path = "../database_sql", features = ["schema"] }-databases = { path = "../databases", default-features = false, features = ["ports"] }+databases = { path = "../databases", default-features = false, features = ["ports", "view_rows"] }email = { path = "../email", default-features = false }entity_access = { path = "../entity_access", default-features = false, features = ["ports"] }filter_ast = { path = "../filter_ast" }@@ -38,6 +39,7 @@ rootcause = { workspace = true }schemars = { workspace = true, optional = true }serde = { workspace = true }serde_json = { workspace = true }+sha2 = { workspace = true }soup = { path = "../soup", default-features = false, features = ["ports"] }thiserror = { workspace = true }tracing = { workspace = true }diff --git a/crates/databases_sql/src/lib.rs b/crates/databases_sql/src/lib.rsindex 29bfd7da4d..2ad8c2e623 100644--- a/crates/databases_sql/src/lib.rs+++ b/crates/databases_sql/src/lib.rs@@ -15,6 +15,7 @@ mod test_support;#[cfg(feature = "ai_tools")]pub mod toolset;mod view_only;+pub mod view_rows;pub use outcome::{ResultColumn, ResultSet, SqlOutcome, SqlStatement};pub use service::{ChartColumns, DatabasesSql, SqlError, SqlRequest};diff --git a/crates/databases_sql/src/service.rs b/crates/databases_sql/src/service.rsindex 2938f51d03..58f6f0fab7 100644--- a/crates/databases_sql/src/service.rs+++ b/crates/databases_sql/src/service.rs@@ -3,7 +3,7 @@mod schema;#[cfg(test)]-mod test;+pub(crate) mod test;use std::collections::HashMap;use std::sync::{Arc, Mutex, PoisonError};diff --git a/crates/databases_sql/src/service/test.rs b/crates/databases_sql/src/service/test.rsindex a3e8697439..95731390a8 100644--- a/crates/databases_sql/src/service/test.rs+++ b/crates/databases_sql/src/service/test.rs@@ -62,7 +62,7 @@ const LAUNCH: RowId = RowId::from_uuid(Uuid::from_u128(0xe004));/// `Offsite.Guests` (relating to `Venues.Halls`) and `Venues` are the/// owner's; the viewer can read `Offsite`; `Secret.Plans` is the/// stranger's alone.-fn world() -> Shared {+pub(crate) fn world() -> Shared {let reference = |id: &str| {PropertyValue::EntityRef(vec![EntityReference {entity_id: id.to_string(),diff --git a/crates/databases_sql/src/test_support.rs b/crates/databases_sql/src/test_support.rsindex dd80e6e8ed..2cbd2c12e7 100644--- a/crates/databases_sql/src/test_support.rs+++ b/crates/databases_sql/src/test_support.rs@@ -261,9 +261,16 @@ impl DatabasesService for FakeDatabases {}async fn get_database(&self,- _: EntityAccessReceipt<ViewAccessLevel>,+ receipt: EntityAccessReceipt<ViewAccessLevel>,) -> Result<DatabaseDetail, DatabaseError> {- unimplemented!("SQL reads every database at once")+ self.0+ .lock()+ .unwrap()+ .databases+ .iter()+ .find(|database| database.database.id.to_string() == receipt.entity().entity_id)+ .cloned()+ .ok_or(DatabaseError::NotFound)}async fn rename_database(&self,diff --git a/crates/databases_sql/src/view_rows.rs b/crates/databases_sql/src/view_rows.rsnew file mode 100644index 0000000000..48566a8e83--- /dev/null+++ b/crates/databases_sql/src/view_rows.rs@@ -0,0 +1,200 @@+//! Paged view reads: authorize the catalog, compile the existing view model,+//! and keep every cursor bound to the query and table version it started with.++#[cfg(test)]+mod test;++use std::{future::Future, sync::Arc};++use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};+use databases::domain::{+ models::{DatabaseError, RowId, TableId, TableVersion},+ ports::DatabasesService,+ view_rows::{ViewRowsError, ViewRowsRepository},+};+use entity_access::domain::models::{EntityAccessReceipt, ViewAccessLevel};+use models_databases::views::{ViewProblem, ViewQuery};+use serde::{Deserialize, Serialize};+use sha2::{Digest, Sha256};++use crate::catalog::ViewerCatalog;++/// The page size remains the same as the existing database row reader.+pub const VIEW_PAGE_LIMIT: u16 = 500;++/// One page of the rows a table view shows.+pub struct ViewRowsRequest {+ /// The table in the authorized database.+ pub table_id: TableId,+ /// All filtering and ordering, before the page boundary.+ pub query: ViewQuery,+ /// The preceding page's opaque continuation.+ pub cursor: Option<String>,+ /// How many rows to return, from one through 500.+ pub limit: u16,+}++/// Ordered row identities, ready for the canonical entity loader.+#[derive(Debug)]+pub struct ViewRowsPage {+ /// Identities in global view order.+ pub rows: Vec<RowId>,+ /// Absent once all matching rows have been read.+ pub next_cursor: Option<String>,+ /// The table version used by this read.+ pub version: TableVersion,+}++/// Why a view page could not be answered.+#[derive(Debug, thiserror::Error)]+pub enum ViewPageError {+ /// The authorized database could not be loaded.+ #[error(transparent)]+ Database(#[from] DatabaseError),+ /// The view does not fit its table's schema.+ #[error(transparent)]+ View(#[from] ViewProblem),+ /// The page size is out of bounds.+ #[error("page size must be between 1 and 500")]+ InvalidLimit,+ /// The cursor belongs to a different query or is malformed.+ #[error("invalid database view cursor")]+ InvalidCursor,+ /// The row read failed, including when its version became stale.+ #[error(transparent)]+ Read(#[from] ViewRowsError),+}++/// The use case exposed to GraphQL and other authenticated adapters.+pub trait ViewRowsService: Send + Sync + 'static {+ /// Read a page within the database the receipt authorizes.+ fn page(+ &self,+ receipt: EntityAccessReceipt<ViewAccessLevel>,+ request: ViewRowsRequest,+ ) -> impl Future<Output = Result<ViewRowsPage, ViewPageError>> + Send;+}++/// View compilation and cursor policy over the owning domains' ports.+pub struct DatabaseViewRows<Databases, Rows> {+ databases: Arc<Databases>,+ rows: Rows,+}++impl<Databases, Rows> DatabaseViewRows<Databases, Rows> {+ /// Compose catalog reads with the ordered row read model.+ pub fn new(databases: Arc<Databases>, rows: Rows) -> Self {+ Self { databases, rows }+ }+}++#[derive(Serialize, Deserialize)]+#[serde(deny_unknown_fields)]+struct Cursor {+ format: u8,+ query: String,+ schema: String,+ version: TableVersion,+ after: RowId,+}++fn query_key(table: TableId, query: &ViewQuery) -> Result<String, ViewPageError> {+ let json = serde_json::to_vec(&(table, query)).map_err(|_| ViewPageError::InvalidCursor)?;+ Ok(URL_SAFE_NO_PAD.encode(Sha256::digest(json)))+}++fn decode_cursor(encoded: &str, query: &str) -> Result<Cursor, ViewPageError> {+ if encoded.len() > 1024 {+ return Err(ViewPageError::InvalidCursor);+ }+ let bytes = URL_SAFE_NO_PAD+ .decode(encoded)+ .map_err(|_| ViewPageError::InvalidCursor)?;+ let cursor: Cursor =+ serde_json::from_slice(&bytes).map_err(|_| ViewPageError::InvalidCursor)?;+ if cursor.format != 1 || cursor.query != query {+ return Err(ViewPageError::InvalidCursor);+ }+ Ok(cursor)+}++impl<Databases: DatabasesService, Rows: ViewRowsRepository> ViewRowsService+ for DatabaseViewRows<Databases, Rows>+{+ #[tracing::instrument(name = "database.view_rows.read", skip_all, fields(database.table_id = %request.table_id, database.page_limit = request.limit, database.rows), err)]+ async fn page(+ &self,+ receipt: EntityAccessReceipt<ViewAccessLevel>,+ request: ViewRowsRequest,+ ) -> Result<ViewRowsPage, ViewPageError> {+ if !(1..=VIEW_PAGE_LIMIT).contains(&request.limit) {+ return Err(ViewPageError::InvalidLimit);+ }+ let detail = self.databases.get_database(receipt).await?;+ let catalog = ViewerCatalog::new(vec![detail], None);+ let query =+ database_sql::compile_table_query(request.table_id, &request.query, catalog.catalog())?;+ let (_, stored) = catalog+ .table(request.table_id)+ .ok_or(ViewProblem::UnknownTable {+ table: request.table_id,+ })?;+ let table = catalog+ .catalog()+ .tables+ .iter()+ .find(|table| table.id == request.table_id)+ .ok_or(ViewProblem::UnknownTable {+ table: request.table_id,+ })?;+ let version = stored.table.version;+ let key = query_key(request.table_id, &request.query)?;+ let schema = URL_SAFE_NO_PAD.encode(Sha256::digest(+ serde_json::to_vec(table).map_err(|_| ViewPageError::InvalidCursor)?,+ ));+ let cursor = request+ .cursor+ .as_deref()+ .map(|cursor| decode_cursor(cursor, &key))+ .transpose()?;+ if cursor+ .as_ref()+ .is_some_and(|cursor| cursor.version != version || cursor.schema != schema)+ {+ return Err(ViewRowsError::Stale.into());+ }+ let mut rows = self+ .rows+ .page(+ table,+ &query,+ version,+ cursor.map(|cursor| cursor.after),+ request.limit + 1,+ )+ .await?;+ let more = rows.len() > usize::from(request.limit);+ rows.truncate(usize::from(request.limit));+ let next_cursor = more+ .then(|| rows.last())+ .flatten()+ .map(|after| {+ serde_json::to_vec(&Cursor {+ format: 1,+ query: key,+ schema,+ version,+ after: *after,+ })+ .map(|bytes| URL_SAFE_NO_PAD.encode(bytes))+ .map_err(|_| ViewPageError::InvalidCursor)+ })+ .transpose()?;+ tracing::Span::current().record("database.rows", rows.len());+ Ok(ViewRowsPage {+ rows,+ next_cursor,+ version,+ })+ }+}diff --git a/crates/databases_sql/src/view_rows/test.rs b/crates/databases_sql/src/view_rows/test.rsnew file mode 100644index 0000000000..15f11c35da--- /dev/null+++ b/crates/databases_sql/src/view_rows/test.rs@@ -0,0 +1,172 @@+use std::sync::Mutex;++use database_sql::{catalog::Table, resolve::SelectQuery};+use databases::domain::{models::Viewer, receipt::database_receipt};+use models_databases::DatabaseId;+use uuid::Uuid;++use super::*;+use crate::test_support::{FakeAccess, FakeDatabases, Shared, VIEWER, user};++#[derive(Clone, Default)]+struct Rows(Arc<Mutex<Vec<Option<RowId>>>>);++const FIRST: RowId = RowId::from_uuid(Uuid::from_u128(101));+const SECOND: RowId = RowId::from_uuid(Uuid::from_u128(102));+const THIRD: RowId = RowId::from_uuid(Uuid::from_u128(103));++impl ViewRowsRepository for Rows {+ async fn page(+ &self,+ _: &Table,+ _: &SelectQuery,+ _: TableVersion,+ after: Option<RowId>,+ limit: u16,+ ) -> Result<Vec<RowId>, ViewRowsError> {+ assert_eq!(limit, 3);+ self.0.lock().unwrap().push(after);+ Ok(if after.is_none() {+ vec![FIRST, SECOND, THIRD]+ } else {+ vec![THIRD]+ })+ }+}++async fn receipt(world: &Shared, database: DatabaseId) -> EntityAccessReceipt<ViewAccessLevel> {+ database_receipt(+ &FakeAccess(world.clone()),+ &Viewer {+ user_id: user(VIEWER),+ acting_bot: None,+ },+ database,+ )+ .await+ .unwrap()+}++fn request(table_id: TableId, cursor: Option<String>) -> ViewRowsRequest {+ ViewRowsRequest {+ table_id,+ query: ViewQuery::default(),+ cursor,+ limit: 2,+ }+}++#[tokio::test]+async fn pages_continue_after_the_last_returned_row_and_stop_at_the_end() {+ let world = crate::service::test::world();+ let detail = world.lock().unwrap().databases[0].clone();+ let rows = Rows::default();+ let service = DatabaseViewRows::new(Arc::new(FakeDatabases(world.clone())), rows.clone());+ let first = service+ .page(+ receipt(&world, detail.database.id).await,+ request(detail.tables[0].table.id, None),+ )+ .await+ .unwrap();+ assert_eq!(first.rows, vec![FIRST, SECOND]);+ assert_eq!(first.version, TableVersion(1));+ assert!(first.next_cursor.is_some());+ let second = service+ .page(+ receipt(&world, detail.database.id).await,+ request(detail.tables[0].table.id, first.next_cursor),+ )+ .await+ .unwrap();+ assert_eq!(second.rows, vec![THIRD]);+ assert!(second.next_cursor.is_none());+ assert_eq!(*rows.0.lock().unwrap(), vec![None, Some(SECOND)]);+}++#[tokio::test]+async fn a_database_receipt_cannot_read_a_table_in_another_database() {+ let world = crate::service::test::world();+ let details = world.lock().unwrap().databases.clone();+ let rows = Rows::default();+ let service = DatabaseViewRows::new(Arc::new(FakeDatabases(world.clone())), rows.clone());+ let answer = service+ .page(+ receipt(&world, details[0].database.id).await,+ request(details[1].tables[0].table.id, None),+ )+ .await;+ assert!(matches!(+ answer,+ Err(ViewPageError::View(ViewProblem::UnknownTable { .. }))+ ));+ assert!(rows.0.lock().unwrap().is_empty());+}++#[tokio::test]+async fn a_cursor_rejects_a_changed_query_or_table_version() {+ let world = crate::service::test::world();+ let detail = world.lock().unwrap().databases[0].clone();+ let table_id = detail.tables[0].table.id;+ let rows = Rows::default();+ let service = DatabaseViewRows::new(Arc::new(FakeDatabases(world.clone())), rows.clone());+ let first = service+ .page(+ receipt(&world, detail.database.id).await,+ request(table_id, None),+ )+ .await+ .unwrap();+ let cursor = first.next_cursor.unwrap();+ let mut changed = request(table_id, Some(cursor.clone()));+ changed.query.sort.push(models_databases::views::SortKey {+ column: detail.tables[0].columns[0].column.id,+ direction: models_databases::views::SortDirection::Ascending,+ });+ assert!(matches!(+ service+ .page(receipt(&world, detail.database.id).await, changed)+ .await,+ Err(ViewPageError::InvalidCursor)+ ));+ world.lock().unwrap().databases[0].tables[0].table.version = TableVersion(2);+ assert!(matches!(+ service+ .page(+ receipt(&world, detail.database.id).await,+ request(table_id, Some(cursor))+ )+ .await,+ Err(ViewPageError::Read(ViewRowsError::Stale))+ ));+ assert_eq!(rows.0.lock().unwrap().len(), 1);+}++#[tokio::test]+async fn cursor_restarts_when_shared_column_metadata_changes_without_a_version_bump() {+ let world = crate::service::test::world();+ let detail = world.lock().unwrap().databases[0].clone();+ let rows = Rows::default();+ let service = DatabaseViewRows::new(Arc::new(FakeDatabases(world.clone())), rows.clone());+ let first = service+ .page(+ receipt(&world, detail.database.id).await,+ request(detail.tables[0].table.id, None),+ )+ .await+ .unwrap();+ world.lock().unwrap().databases[0].tables[0].columns[0]+ .definition+ .definition+ .display_name = "renamed".into();+ assert!(matches!(+ service+ .page(+ receipt(&world, detail.database.id).await,+ request(detail.tables[0].table.id, first.next_cursor)+ )+ .await,+ Err(ViewPageError::Read(ViewRowsError::Stale))+ ));+ assert_eq!(rows.0.lock().unwrap().len(), 1);+}diff --git a/crates/graphql_databases/Cargo.toml b/crates/graphql_databases/Cargo.tomlnew file mode 100644index 0000000000..83167addc4--- /dev/null+++ b/crates/graphql_databases/Cargo.toml@@ -0,0 +1,24 @@+[package]+name = "graphql_databases"+version = "0.1.0"+edition = "2024"+publish = false++[dependencies]+async-graphql = { workspace = true, features = ["dataloader", "chrono"] }+chrono = { workspace = true }+databases = { path = "../databases", features = ["ports", "view_rows"] }+databases_sql = { path = "../databases_sql" }+entity_access = { path = "../entity_access", default-features = false, features = ["ports"] }+futures = { workspace = true }+graphql_common = { path = "../graphql_common" }+graphql_soup = { path = "../graphql_soup" }+macro_user_id = { path = "../macro_user_id" }+model-entity = { path = "../model-entity" }+models_databases = { path = "../models_databases" }+tracing = { workspace = true }+workspace-hack = { version = "0.1", path = "../workspace-hack" }++[dev-dependencies]+rootcause = { workspace = true }+tokio = { workspace = true, features = ["macros", "rt"] }diff --git a/crates/graphql_databases/src/context.rs b/crates/graphql_databases/src/context.rsnew file mode 100644index 0000000000..1d4b3866b0--- /dev/null+++ b/crates/graphql_databases/src/context.rs@@ -0,0 +1,74 @@+//! The inbound receipt boundary and request capabilities supplied by DSS.++use std::{future::Future, pin::Pin, sync::Arc};++use databases::domain::{models::Viewer, receipt::database_receipt};+use databases_sql::view_rows::{ViewRowsPage, ViewRowsRequest, ViewRowsService};+use entity_access::domain::{models::ViewAccessLevel, ports::EntityAccessService};+use graphql_soup::SoupItemDataLoader;+use macro_user_id::user_id::MacroUserIdStr;+use models_databases::DatabaseId;++pub(crate) type PageFuture<'a> =+ Pin<Box<dyn Future<Output = async_graphql::Result<ViewRowsPage>> + Send + 'a>>;++pub(crate) trait DatabaseRowsApi: Send + Sync {+ fn page(+ &self,+ user: MacroUserIdStr<'static>,+ database: DatabaseId,+ request: ViewRowsRequest,+ ) -> PageFuture<'_>;+}++/// View-page service and primary entity loader, without schema-level generics.+#[derive(Clone)]+pub struct DatabaseRowsGraphqlContext {+ pub(crate) api: Arc<dyn DatabaseRowsApi>,+ pub(crate) loader: SoupItemDataLoader,+}++impl DatabaseRowsGraphqlContext {+ /// Compose the use case, receipt minting and primary row hydration.+ pub fn new<S: ViewRowsService, A: EntityAccessService>(+ service: S,+ access: Arc<A>,+ loader: SoupItemDataLoader,+ ) -> Self {+ Self {+ api: Arc::new(Adapter { service, access }),+ loader,+ }+ }+}++struct Adapter<S, A> {+ service: S,+ access: Arc<A>,+}++impl<S: ViewRowsService, A: EntityAccessService> DatabaseRowsApi for Adapter<S, A> {+ fn page(+ &self,+ user: MacroUserIdStr<'static>,+ database: DatabaseId,+ request: ViewRowsRequest,+ ) -> PageFuture<'_> {+ Box::pin(async move {+ let receipt = database_receipt::<ViewAccessLevel, _>(+ self.access.as_ref(),+ &Viewer {+ user_id: user,+ acting_bot: None,+ },+ database,+ )+ .await+ .map_err(|_| async_graphql::Error::new("not found"))?;+ self.service+ .page(receipt, request)+ .await+ .map_err(super::graphql_error)+ })+ }+}diff --git a/crates/graphql_databases/src/inputs.rs b/crates/graphql_databases/src/inputs.rsnew file mode 100644index 0000000000..c89fe390cf--- /dev/null+++ b/crates/graphql_databases/src/inputs.rs@@ -0,0 +1,355 @@+//! Typed GraphQL inputs mapped to the existing database view model.++use async_graphql::{Enum, ID, InputObject, OneofObject};+use chrono::{DateTime, Utc};+use databases_sql::view_rows::{VIEW_PAGE_LIMIT, ViewRowsRequest};+use graphql_common::parse_id;+use models_databases::{ColumnId, OptionId, TableId, views as model};++/// A table view and its optional continuation, with a bounded page size.+#[derive(InputObject)]+pub struct DatabaseViewRowsInput {+ /// The table within the authorized database.+ table_id: ID,+ /// The complete filtering and ordering definition.+ query: DatabaseViewQueryInput,+ /// The preceding page’s opaque cursor; omit for the first page.+ cursor: Option<String>,+ /// Page size from 1 through 500; defaults to 500.+ limit: Option<u16>,+}++impl DatabaseViewRowsInput {+ pub(crate) fn into_request(self) -> async_graphql::Result<ViewRowsRequest> {+ Ok(ViewRowsRequest {+ table_id: TableId::from_uuid(parse_id(self.table_id, "tableId")?),+ query: self.query.into_model()?,+ cursor: self.cursor,+ limit: self.limit.unwrap_or(VIEW_PAGE_LIMIT),+ })+ }+}++/// Filtering and ordering applied to the entire table before pagination.+#[derive(InputObject)]+struct DatabaseViewQueryInput {+ /// Optional filter; omission includes every row.+ filter: Option<DatabaseViewFilterGroup>,+ #[graphql(default)]+ /// Ordering keys in priority order; omission uses manual row order.+ sort: Vec<DatabaseViewSort>,+}++impl DatabaseViewQueryInput {+ fn into_model(self) -> async_graphql::Result<model::ViewQuery> {+ Ok(model::ViewQuery {+ filter: self.filter.map(|filter| filter.into_model(0)).transpose()?,+ sort: self+ .sort+ .into_iter()+ .map(|sort| {+ Ok(model::SortKey {+ column: ColumnId::from_uuid(parse_id(sort.column, "column")?),+ direction: sort.direction.into(),+ })+ })+ .collect::<async_graphql::Result<_>>()?,+ })+ }+}++/// One ordering key; earlier keys take precedence.+#[derive(InputObject)]+struct DatabaseViewSort {+ /// Column placement identity within the selected table.+ column: ID,+ /// Ascending or descending value order.+ direction: DatabaseViewSortDirection,+}++/// The direction of a view ordering key.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::SortDirection")]+enum DatabaseViewSortDirection {+ /// Order values from lowest to highest.+ Ascending,+ /// Order values from highest to lowest.+ Descending,+}++/// How the conditions within a filter group combine.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::Conjunction")]+enum DatabaseViewConjunction {+ /// Require every condition.+ And,+ /// Require at least one condition.+ Or,+}++/// A conjunction of column conditions or nested groups.+#[derive(InputObject)]+struct DatabaseViewFilterGroup {+ /// Require every condition or at least one condition.+ conjunction: DatabaseViewConjunction,+ /// Column tests or nested groups.+ conditions: Vec<DatabaseViewFilterNode>,+}++/// Exactly one column condition or nested filter group.+#[derive(OneofObject)]+enum DatabaseViewFilterNode {+ /// A predicate on one column.+ Condition(DatabaseViewFilterCondition),+ /// A nested group of predicates.+ Group(DatabaseViewFilterGroup),+}++/// A typed predicate on one column placement.+#[derive(InputObject)]+struct DatabaseViewFilterCondition {+ /// Column placement identity within the selected table.+ column: ID,+ /// The comparison appropriate for this column.+ test: DatabaseViewFilterTest,+}++impl DatabaseViewFilterGroup {+ fn into_model(self, depth: u8) -> async_graphql::Result<model::FilterGroup> {+ if depth > 16 || self.conditions.len() > 100 {+ return Err(async_graphql::Error::new(+ "database view filter is too complex",+ ));+ }+ Ok(model::FilterGroup {+ conjunction: self.conjunction.into(),+ conditions: self+ .conditions+ .into_iter()+ .map(|node| match node {+ DatabaseViewFilterNode::Group(group) => {+ group.into_model(depth + 1).map(model::FilterNode::Group)+ }+ DatabaseViewFilterNode::Condition(condition) => {+ Ok(model::FilterNode::Condition(model::FilterCondition {+ column: ColumnId::from_uuid(parse_id(condition.column, "column")?),+ test: condition.test.into_model()?,+ }))+ }+ })+ .collect::<async_graphql::Result<_>>()?,+ })+ }+}++/// Exactly one test compatible with the selected column type.+#[derive(OneofObject)]+enum DatabaseViewFilterTest {+ /// Test cell emptiness.+ Presence(DatabaseViewPresenceTest),+ /// Test a text or link cell.+ Text(DatabaseViewTextTest),+ /// Test a numeric cell.+ Number(DatabaseViewNumberTest),+ /// Test a date cell.+ Date(DatabaseViewDateTest),+ /// Test a date range cell against a day.+ DateRange(DatabaseViewDateRangeTest),+ /// Test a checkbox cell.+ Checkbox(DatabaseViewCheckboxTest),+ /// Test selected option identities.+ Options(DatabaseViewOptionsTest),+ /// Test referenced entity identities.+ Entities(DatabaseViewEntitiesTest),+}++/// Test whether the cell has a value.+#[derive(InputObject)]+struct DatabaseViewPresenceTest {+ /// The comparison to apply.+ operator: DatabaseViewPresenceOperator,+}+/// Compare a text or link cell with a string.+#[derive(InputObject)]+struct DatabaseViewTextTest {+ /// The comparison to apply.+ operator: DatabaseViewTextOperator,+ /// The value to compare the cell against.+ value: String,+}+/// Compare a numeric cell with a finite number.+#[derive(InputObject)]+struct DatabaseViewNumberTest {+ /// The comparison to apply.+ operator: DatabaseViewNumberOperator,+ /// The value to compare the cell against.+ value: f64,+}+/// Compare a date cell with a UTC instant.+#[derive(InputObject)]+struct DatabaseViewDateTest {+ /// The comparison to apply.+ operator: DatabaseViewDateOperator,+ /// The value to compare the cell against.+ value: DateTime<Utc>,+}+/// Compare a date range cell with the day starting at a UTC instant.+#[derive(InputObject)]+struct DatabaseViewDateRangeTest {+ /// The comparison to apply.+ operator: DatabaseViewDateRangeOperator,+ /// The start of the day compared against.+ value: DateTime<Utc>,+}+/// Match the checked or unchecked state of a cell.+#[derive(InputObject)]+struct DatabaseViewCheckboxTest {+ /// Whether the cell must be checked.+ checked: bool,+}+/// Compare selected option identities with a set of options.+#[derive(InputObject)]+struct DatabaseViewOptionsTest {+ /// The comparison to apply.+ operator: DatabaseViewSetOperator,+ /// Option identities from the selected column.+ options: Vec<ID>,+}+/// Compare referenced entity identities with a set of entities.+#[derive(InputObject)]+struct DatabaseViewEntitiesTest {+ /// The comparison to apply.+ operator: DatabaseViewSetOperator,+ /// Referenced entity identities.+ entities: Vec<String>,+}++/// The requested cell-presence condition.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::PresenceOperator")]+enum DatabaseViewPresenceOperator {+ /// The cell has no value.+ IsEmpty,+ /// The cell has a value.+ IsNotEmpty,+}+/// A text equality or substring comparison.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::TextOperator")]+enum DatabaseViewTextOperator {+ /// The value equals the comparison value.+ Is,+ /// The value differs from the comparison value.+ IsNot,+ /// The text contains the comparison string.+ Contains,+ /// The text does not contain the comparison string.+ DoesNotContain,+ /// The text starts with the comparison string.+ StartsWith,+ /// The text ends with the comparison string.+ EndsWith,+}+/// A numeric equality or ordering comparison.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::NumberOperator")]+enum DatabaseViewNumberOperator {+ /// The value equals the comparison value.+ Is,+ /// The value differs from the comparison value.+ IsNot,+ /// The number is greater than the comparison value.+ GreaterThan,+ /// The number is at least the comparison value.+ GreaterThanOrEqual,+ /// The number is less than the comparison value.+ LessThan,+ /// The number is at most the comparison value.+ LessThanOrEqual,+}+/// An ordering comparison between instants.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::DateOperator")]+enum DatabaseViewDateOperator {+ /// The instant is before the comparison instant.+ Before,+ /// The instant is after the comparison instant.+ After,+ /// The instant is at or before the comparison instant.+ OnOrBefore,+ /// The instant is at or after the comparison instant.+ OnOrAfter,+}+/// How a date range relates to a day.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::DateRangeOperator")]+enum DatabaseViewDateRangeOperator {+ /// The range overlaps the day.+ Contains,+ /// The range misses the day, or is empty.+ DoesNotContain,+ /// The range ends before the day.+ EndsBefore,+ /// The range ends on or after the day.+ EndsOnOrAfter,+ /// The range starts after the day.+ StartsAfter,+ /// The range starts on or before the day.+ StartsOnOrBefore,+}+/// A membership condition for single- or multi-valued cells.+#[derive(Enum, Copy, Clone, Eq, PartialEq)]+#[graphql(remote = "model::SetOperator")]+enum DatabaseViewSetOperator {+ /// The single-valued cell matches one supplied identity.+ IsAnyOf,+ /// The single-valued cell matches no supplied identity.+ IsNoneOf,+ /// The cell contains at least one supplied identity.+ HasAny,+ /// The cell contains every supplied identity.+ HasAll,+ /// The cell contains no supplied identity.+ HasNone,+}++impl DatabaseViewFilterTest {+ fn into_model(self) -> async_graphql::Result<model::FilterTest> {+ Ok(match self {+ Self::Presence(test) => model::FilterTest::Presence {+ operator: test.operator.into(),+ },+ Self::Text(test) => model::FilterTest::Text {+ operator: test.operator.into(),+ value: test.value,+ },+ Self::Number(test) => model::FilterTest::Number {+ operator: test.operator.into(),+ value: test.value,+ },+ Self::Date(test) => model::FilterTest::Date {+ operator: test.operator.into(),+ value: test.value,+ },+ Self::DateRange(test) => model::FilterTest::DateRange {+ operator: test.operator.into(),+ value: test.value,+ },+ Self::Checkbox(test) => model::FilterTest::Checkbox {+ checked: test.checked,+ },+ Self::Options(test) => model::FilterTest::Options {+ operator: test.operator.into(),+ options: test+ .options+ .into_iter()+ .map(|id| parse_id(id, "option").map(OptionId::from_uuid))+ .collect::<async_graphql::Result<_>>()?,+ },+ Self::Entities(test) => model::FilterTest::Entities {+ operator: test.operator.into(),+ entities: test.entities,+ },+ })+ }+}diff --git a/crates/graphql_databases/src/lib.rs b/crates/graphql_databases/src/lib.rsnew file mode 100644index 0000000000..1d9cd21e9b--- /dev/null+++ b/crates/graphql_databases/src/lib.rs@@ -0,0 +1,92 @@+//! The viewer's paginated database views, returning canonical Soup row entities.++#![deny(missing_docs)]++mod context;+mod inputs;++use async_graphql::{Context, ErrorExtensions, ID, SimpleObject};+use databases::domain::{models::DatabaseError, view_rows::ViewRowsError};+use databases_sql::view_rows::ViewPageError;+use graphql_common::{parse_id, require_authenticated_user};+use graphql_soup::{GraphqlSoupDatabaseRow, GraphqlSoupEntity, SoupEntityEdges};+use model_entity::EntityType;+use models_databases::DatabaseId;++pub use context::DatabaseRowsGraphqlContext;+pub use inputs::DatabaseViewRowsInput;++/// Membership and order belong to this page; row facts retain their shared identity.+#[derive(SimpleObject)]+pub struct DatabaseViewRowsPage<E: SoupEntityEdges> {+ /// Rows in global view order, hydrated as canonical Soup entities.+ items: Vec<GraphqlSoupDatabaseRow<E>>,+ /// Continuation for the next page; absent at the end of the result.+ next_cursor: Option<String>,+ /// Table version against which this page was selected.+ version: i64,+}++fn graphql_error(error: ViewPageError) -> async_graphql::Error {+ let (code, message) = match &error {+ ViewPageError::Read(ViewRowsError::Stale) => ("DATABASE_VIEW_STALE", error.to_string()),+ ViewPageError::View(_) => ("DATABASE_VIEW_INVALID", error.to_string()),+ ViewPageError::Read(ViewRowsError::Infrastructure(_))+ | ViewPageError::Database(DatabaseError::Repo(_)) => (+ "INTERNAL_SERVER_ERROR",+ "The database view could not be read.".into(),+ ),+ ViewPageError::Database(DatabaseError::NotFound | DatabaseError::Unauthorized) => {+ ("NOT_FOUND", "not found".into())+ }+ _ => ("BAD_USER_INPUT", error.to_string()),+ };+ tracing::warn!(error = ?error, code, "database view read failed");+ async_graphql::Error::new(message).extend_with(|_, extensions| extensions.set("code", code))+}++/// Resolve one authorized page, then hydrate its rows through the shared entity contract.+pub async fn resolve_database_view_rows<E: SoupEntityEdges>(+ ctx: &Context<'_>,+ database_id: ID,+ input: DatabaseViewRowsInput,+) -> async_graphql::Result<DatabaseViewRowsPage<E>> {+ let user = require_authenticated_user(ctx)?;+ let context = ctx.data::<DatabaseRowsGraphqlContext>()?;+ let database = DatabaseId::from_uuid(parse_id(database_id, "databaseId")?);+ let page = context+ .api+ .page(user.clone(), database, input.into_request()?)+ .await?;+ let keys = page+ .rows+ .iter()+ .map(|row| {+ (+ user.clone(),+ EntityType::DatabaseRow.with_entity_string(row.to_string()),+ )+ })+ .collect::<Vec<_>>();+ let loaded =+ futures::future::try_join_all(keys.into_iter().map(|key| context.loader.load_one(key)))+ .await?;+ let items = loaded+ .into_iter()+ .map(|item| {+ let item = item.ok_or_else(|| graphql_error(ViewRowsError::Stale.into()))?;+ match GraphqlSoupEntity::<E>::new_with_projection(item) {+ GraphqlSoupEntity::DatabaseRow(row) => Ok(row),+ _ => Err(async_graphql::Error::new("expected a database row")),+ }+ })+ .collect::<async_graphql::Result<Vec<_>>>()?;+ Ok(DatabaseViewRowsPage {+ items,+ next_cursor: page.next_cursor,+ version: page.version.0,+ })+}++#[cfg(test)]+mod test;diff --git a/crates/graphql_databases/src/test.rs b/crates/graphql_databases/src/test.rsnew file mode 100644index 0000000000..39a81aa107--- /dev/null+++ b/crates/graphql_databases/src/test.rs@@ -0,0 +1,18 @@+use super::*;++#[test]+fn repository_failures_are_internal_and_do_not_expose_details() {+ let error =+ graphql_error(DatabaseError::Repo(rootcause::report!("private connection details")).into());+ assert_eq!(error.message, "The database view could not be read.");+ assert_eq!(+ error.extensions.unwrap().get("code"),+ Some(&async_graphql::Value::from("INTERNAL_SERVER_ERROR"))+ );+ let missing = graphql_error(DatabaseError::NotFound.into());+ assert_eq!(missing.message, "not found");+ assert_eq!(+ missing.extensions.unwrap().get("code"),+ Some(&async_graphql::Value::from("NOT_FOUND"))+ );+}diff --git a/crates/macro_db_client/migrations/20261005193310_database_rows_created_cursor_index.sql b/crates/macro_db_client/migrations/20261005193310_database_rows_created_cursor_index.sqlnew file mode 100644index 0000000000..57dfd07293--- /dev/null+++ b/crates/macro_db_client/migrations/20261005193310_database_rows_created_cursor_index.sql@@ -0,0 +1,7 @@+-- no-transaction+-- Soup pages a table by (created_at, id::text). The text expression preserves+-- the shared Soup cursor ordering while allowing a bounded backward index scan.+-- Do not skip an existing name: an interrupted concurrent build can leave an+-- invalid index. Inspect and drop that index before retrying the migration.+CREATE INDEX CONCURRENTLY idx_database_rows_table_created_cursor+ ON database_rows (table_id, created_at, (id::text));diff --git a/crates/macro_db_client/migrations/20261005210518_database_rows_position_cursor_index.sql b/crates/macro_db_client/migrations/20261005210518_database_rows_position_cursor_index.sqlnew file mode 100644index 0000000000..6382b7ec64--- /dev/null+++ b/crates/macro_db_client/migrations/20261005210518_database_rows_position_cursor_index.sql@@ -0,0 +1,5 @@+-- no-transaction+-- The table view's stable manual order, including a unique cursor tie-breaker.+-- An existing name must fail so an interrupted, invalid index is not accepted.+CREATE INDEX CONCURRENTLY idx_database_rows_position_cursor+ ON database_rows (table_id, position, id);diff --git a/crates/macro_db_client/migrations/20261005224256_drop_database_rows_redundant_table_index.sql b/crates/macro_db_client/migrations/20261005224256_drop_database_rows_redundant_table_index.sqlnew file mode 100644index 0000000000..703f03d8b6--- /dev/null+++ b/crates/macro_db_client/migrations/20261005224256_drop_database_rows_redundant_table_index.sql@@ -0,0 +1,4 @@+-- no-transaction+-- The composite cursor indexes also cover table_id lookups and cascade deletes.+-- Keep this separate: CONCURRENTLY requires a single statement per SQLx migration.+DROP INDEX CONCURRENTLY IF EXISTS idx_database_rows_table;diff --git a/crates/properties/src/domain/database_cell_writer.rs b/crates/properties/src/domain/database_cell_writer.rsindex 291232820c..481f5c0e1e 100644--- a/crates/properties/src/domain/database_cell_writer.rs+++ b/crates/properties/src/domain/database_cell_writer.rs@@ -35,6 +35,19 @@ pub trait DatabaseCellWriter: DatabaseWriteTransaction {value: Option<PropertyValue>,) -> impl Future<Output = Result<(), Self::Err>> + Send;+ /// Set one property on many entities of one type, each to its value,+ /// or with `None` clear it: a statement each way however many entities+ /// there are, for a use case storing a whole column at once. Answers how+ /// many stored values changed. The caller holds the locks that serialize+ /// writers of these values.+ fn set_entity_values_in(+ &self,+ transaction: &mut Self::Transaction,+ entity_type: EntityType,+ property_definition_id: Uuid,+ values: &[(String, Option<PropertyValue>)],+ ) -> impl Future<Output = Result<u64, Self::Err>> + Send;+/// What these entities of one type hold, read inside the transaction so/// a use case sees what its own locks guard: each entity's properties/// with a value, or only those of `definitions` when given, asdiff --git a/crates/properties/src/domain/service_impl.rs b/crates/properties/src/domain/service_impl.rsindex 062a50d6db..280a91a001 100644--- a/crates/properties/src/domain/service_impl.rs+++ b/crates/properties/src/domain/service_impl.rs@@ -58,6 +58,16 @@ use helpers::{extract_option_ids_from_property_value, is_property_applicable_to, retain_caller_visible_tags,};+// Database cells must commit with their table version and change journal.+fn require_generic_property_target(entity_type: AccessEntityType) -> Result<(), PropertiesErr> {+ if entity_type == AccessEntityType::DatabaseRow {+ return Err(PropertiesErr::Validation(+ "Database row properties must be edited through database operations".into(),+ ));+ }+ Ok(())+}+struct PublishedEventActors {actor: Option<Actor<'static>>,on_behalf_of: Option<MacroUserIdStr<'static>>,@@ -645,6 +655,7 @@ wherevalue: Option<SetPropertyValue>,source: Option<PropertyChangeSource>,) -> Result<EntityPropertyWithDefinition, PropertiesErr> {+ require_generic_property_target(access.entity_type())?;let subject = self.resolve_subject(access).await?;let entity_id = access.entity_id();let entity_type = subject.storage_entity_type;@@ -878,6 +889,7 @@ whereproperty_definition_id: Uuid,option_id: Uuid,) -> Result<(), PropertiesErr> {+ require_generic_property_target(access.entity_type())?;let property_definition = self.repository.get_property_definition(property_definition_id)@@ -943,6 +955,7 @@ whereproperty_definition_id: Uuid,option_id: Uuid,) -> Result<(), PropertiesErr> {+ require_generic_property_target(access.entity_type())?;let subject = self.resolve_subject(access).await?;let mutation = self.repository@@ -982,6 +995,7 @@ whereaccess: &EditReceipt,updates: Vec<EntityPropertyOptionUpdate>,) -> Result<Vec<EntityPropertyOptionSelection>, PropertiesErr> {+ require_generic_property_target(access.entity_type())?;let subject = self.resolve_subject(access).await?;let entity_type = subject.storage_entity_type;@@ -1105,6 +1119,15 @@ wherelet mut outcomes: Vec<Option<EntityOptionUpdateOutcome>> =(0..subjects.len()).map(|_| None).collect();for index in order {+ if let Err(error) = require_generic_property_target(access[index].entity_type()) {+ outcomes[index] = Some(EntityOptionUpdateOutcome::Failed {+ message: match error {+ PropertiesErr::Validation(message) => message,+ other => other.to_string(),+ },+ });+ continue;+ }let subject = &subjects[index];let entity_id = subject.canonical_key.entity_id.as_str();let entity_type = subject.storage_entity_type;@@ -1886,6 +1909,7 @@ where#[tracing::instrument(skip(self, access), fields(entity_id = %access.entity_id(), entity_type = ?access.entity_type()), err)]async fn delete_entity_properties(&self, access: &EditReceipt) -> Result<(), PropertiesErr> {+ require_generic_property_target(access.entity_type())?;let subject = self.resolve_subject(access).await?;let entity_reference =EntityReference::new(access.entity_id().to_string(), subject.storage_entity_type);@@ -1934,6 +1958,7 @@ whereaccess: &EditReceipt,entity_property_id: Uuid,) -> Result<(), PropertiesErr> {+ require_generic_property_target(access.entity_type())?;let property_info = self.repository.lookup_entity_property(entity_property_id)diff --git a/crates/properties/src/domain/test.rs b/crates/properties/src/domain/test.rsindex 3208a53239..f80493da69 100644--- a/crates/properties/src/domain/test.rs+++ b/crates/properties/src/domain/test.rs@@ -1,5 +1,6 @@//! Unit tests for PropertiesServiceImpl using mockall-generated repo.+mod database_rows;mod initiatives;mod options_batch;mod project_agents;diff --git a/crates/properties/src/domain/test/database_rows.rs b/crates/properties/src/domain/test/database_rows.rsnew file mode 100644index 0000000000..33654efad7--- /dev/null+++ b/crates/properties/src/domain/test/database_rows.rs@@ -0,0 +1,64 @@+use super::*;++#[tokio::test]+async fn database_row_writes_require_the_versioned_database_service() {+ let service =+ service_with_event_broker(MockPropertiesRepo::new(), RecordingEventBroker::default());+ let access = edit_receipt("row", EntityType::DatabaseRow);+ let definition = Uuid::from_u128(1);+ let option = Uuid::from_u128(2);+ let answers = [+ service+ .set_entity_property(&access, definition, None)+ .await+ .map(|_| ()),+ service+ .add_entity_property_option(&access, definition, option)+ .await,+ service+ .remove_entity_property_option(&access, definition, option)+ .await,+ service+ .bulk_update_entity_property_options(&access, vec![])+ .await+ .map(|_| ()),+ service.delete_entity_properties(&access).await,+ service.delete_entity_property(&access, definition).await,+ ];+ for answer in answers {+ assert!(+ matches!(answer, Err(PropertiesErr::Validation(message)) if message == "Database row properties must be edited through database operations")+ );+ }+}++#[tokio::test]+async fn bulk_option_updates_reject_database_rows_without_writing_them() {+ let definition = Uuid::from_u128(1);+ let mut repository = MockPropertiesRepo::new();+ repository+ .expect_get_property_definition()+ .returning(move |_| {+ Box::pin(async move {+ Ok(Some(property_definition_for_event(+ definition,+ "Choices",+ DataType::String,+ true,+ )))+ })+ });+ let service = service_with_event_broker(repository, RecordingEventBroker::default());+ let outcomes = service+ .bulk_update_entities_property_options(+ &[edit_receipt("row", EntityType::DatabaseRow)],+ definition,+ vec![],+ vec![],+ )+ .await+ .unwrap();+ assert!(+ matches!(outcomes.as_slice(), [crate::domain::model::EntityOptionUpdateOutcome::Failed { message }] if message == "Database row properties must be edited through database operations")+ );+}diff --git a/crates/properties/src/outbound/database_cell_writer.rs b/crates/properties/src/outbound/database_cell_writer.rsindex 3ee1c7497e..def6e48f60 100644--- a/crates/properties/src/outbound/database_cell_writer.rs+++ b/crates/properties/src/outbound/database_cell_writer.rs@@ -52,6 +52,22 @@ impl DatabaseCellWriter for PropertiesPgRepo {Ok(())}+ async fn set_entity_values_in(+ &self,+ transaction: &mut Self::Transaction,+ entity_type: EntityType,+ property_definition_id: Uuid,+ values: &[(String, Option<PropertyValue>)],+ ) -> Result<u64, Self::Err> {+ entity_property_queries::set_entity_values_in_transaction(+ transaction,+ entity_type,+ property_definition_id,+ values,+ )+ .await+ }+async fn entity_values_in(&self,transaction: &mut Self::Transaction,diff --git a/crates/properties/src/outbound/entity_property_queries.rs b/crates/properties/src/outbound/entity_property_queries.rsindex a81f2e4583..f780b71154 100644--- a/crates/properties/src/outbound/entity_property_queries.rs+++ b/crates/properties/src/outbound/entity_property_queries.rs@@ -100,6 +100,94 @@ pub async fn entity_values_in_transaction(.collect())}+/// Set one property on many entities of one type inside a caller's+/// transaction, each to its value or, with `None`, cleared: a few+/// statements however many entities there are. A value already stored is+/// left untouched; answers how many stored values changed. The caller+/// serializes writers of these values.+pub async fn set_entity_values_in_transaction(+ tx: &mut sqlx::PgConnection,+ entity_type: EntityType,+ property_definition_id: Uuid,+ values: &[(String, Option<PropertyValue>)],+) -> Result<u64, PropertyQueryError> {+ let mut ids = Vec::new();+ let mut set = Vec::new();+ let mut stored = Vec::new();+ let mut cleared = Vec::new();+ let mut changed = 0;+ for (entity_id, value) in values {+ match value {+ Some(value) => {+ ids.push(macro_uuid::generate_uuid_v7());+ set.push(entity_id.clone());+ stored.push(serde_json::to_value(value)?);+ }+ None => cleared.push(entity_id.clone()),+ }+ }+ if !set.is_empty() {+ // Updating the cells already there first keeps the insert, and the+ // checks its triggers run per row, to the cells that are new.+ changed += sqlx::query!(+ r#"+ UPDATE entity_properties+ SET values = cell.value, updated_at = NOW()+ FROM UNNEST($1::text[], $2::jsonb[]) AS cell(entity_id, value)+ WHERE entity_properties.entity_type = $3+ AND entity_properties.property_definition_id = $4+ AND entity_properties.entity_id = cell.entity_id+ AND entity_properties.values IS DISTINCT FROM cell.value+ "#,+ &set,+ &stored,+ entity_type as EntityType,+ property_definition_id,+ )+ .execute(&mut *tx)+ .await?+ .rows_affected();+ changed += sqlx::query!(+ r#"+ INSERT INTO entity_properties (id, entity_id, entity_type, property_definition_id, values)+ SELECT cell.id, cell.entity_id, $3, $4, cell.value+ FROM UNNEST($1::uuid[], $2::text[], $5::jsonb[]) AS cell(id, entity_id, value)+ WHERE NOT EXISTS (+ SELECT 1 FROM entity_properties stored+ WHERE stored.entity_id = cell.entity_id+ AND stored.entity_type = $3+ AND stored.property_definition_id = $4+ )+ ON CONFLICT (entity_id, entity_type, property_definition_id)+ DO UPDATE SET values = EXCLUDED.values, updated_at = NOW()+ "#,+ &ids,+ &set,+ entity_type as EntityType,+ property_definition_id,+ &stored,+ )+ .execute(&mut *tx)+ .await?+ .rows_affected();+ }+ if !cleared.is_empty() {+ changed += sqlx::query!(+ r#"+ DELETE FROM entity_properties+ WHERE entity_type = $1 AND property_definition_id = $2 AND entity_id = ANY($3)+ "#,+ entity_type as EntityType,+ property_definition_id,+ &cleared,+ )+ .execute(&mut *tx)+ .await?+ .rows_affected();+ }+ Ok(changed)+}+/// Upsert an entity property value (insert or update)./// If the property doesn't exist, it will be created and attached to the entity./// If it exists, the value will be updated. The returned snapshot carries thediff --git a/crates/properties/src/outbound/entity_property_queries/test.rs b/crates/properties/src/outbound/entity_property_queries/test.rsindex e210829c4a..22d0282266 100644--- a/crates/properties/src/outbound/entity_property_queries/test.rs+++ b/crates/properties/src/outbound/entity_property_queries/test.rs@@ -298,3 +298,64 @@ async fn assignment_snapshot_waits_for_writers_using_only_row_locks(assert_eq!(reassigned.value, Some(assignment()));Ok(())}++#[sqlx::test(migrator = "MACRO_DB_MIGRATIONS")]+async fn one_statement_sets_and_clears_a_property_across_entities(+ pool: Pool<Postgres>,+) -> anyhow::Result<()> {+ let [first, second, third] = [(); 3].map(|_| Uuid::now_v7().to_string());+ let definition = SystemPropertyKey::ASSIGNEES_UUID;+ let mut tx = pool.begin().await?;+ let created = set_entity_values_in_transaction(+ &mut tx,+ EntityType::Task,+ definition,+ &[+ (first.clone(), Some(PropertyValue::Num(1.0))),+ (second.clone(), Some(PropertyValue::Num(2.0))),+ ],+ )+ .await?;+ assert_eq!(created, 2);+ // First changes, second is cleared, third is new.+ let changed = set_entity_values_in_transaction(+ &mut tx,+ EntityType::Task,+ definition,+ &[+ (first.clone(), Some(PropertyValue::Num(3.0))),+ (second.clone(), None),+ (third.clone(), Some(PropertyValue::Num(4.0))),+ ],+ )+ .await?;+ assert_eq!(changed, 3);+ let unchanged = set_entity_values_in_transaction(+ &mut tx,+ EntityType::Task,+ definition,+ &[+ (first.clone(), Some(PropertyValue::Num(3.0))),+ (third.clone(), Some(PropertyValue::Num(4.0))),+ ],+ )+ .await?;+ assert_eq!(unchanged, 0);++ let mut stored = entity_values_in_transaction(+ &mut tx,+ EntityType::Task,+ &[first.clone(), second, third.clone()],+ Some(&[definition]),+ )+ .await?;+ stored.sort_by(|left, right| left.0.cmp(&right.0));+ assert_eq!(+ stored,+ vec![+ (first, definition, PropertyValue::Num(3.0)),+ (third, definition, PropertyValue::Num(4.0)),+ ]+ );+ Ok(())+}diff --git a/crates/soup/src/outbound/pg_soup_repo/database_row/test.rs b/crates/soup/src/outbound/pg_soup_repo/database_row/test.rsindex cada8e871f..9fe15b3201 100644--- a/crates/soup/src/outbound/pg_soup_repo/database_row/test.rs+++ b/crates/soup/src/outbound/pg_soup_repo/database_row/test.rs@@ -1,4 +1,5 @@use super::*;+use crate::domain::models::sort::DynamicSortValue;use crate::outbound::pg_soup_repo::expanded::dynamic::{ExpandedDynamicCursorArgs, GroupedDynamicCursorArgs, expanded_dynamic_cursor_soup,expanded_dynamic_cursor_soup_grouped,@@ -16,7 +17,7 @@ use item_filters::ast::{use macro_db_migrator::MACRO_DB_MIGRATIONS;use macro_user_id::user_id::MacroUserIdStr;use models_grouping::{GroupByField, GroupingConfig};-use models_pagination::{Identify, Query, SimpleSortMethod};+use models_pagination::{Cursor, CursorVal, Identify, Query, SimpleSortMethod};use models_properties::service::property_value::PropertyValue;use models_soup::database_row::SoupDatabaseRow;use std::sync::Arc;@@ -336,6 +337,78 @@ async fn a_row_carries_every_cell_as_a_property(pool: PgPool) {);}+#[sqlx::test(fixtures("test/database_rows.sql"), migrator = "MACRO_DB_MIGRATIONS")]+async fn detail_joins_accept_mixed_soup_ids(pool: PgPool) {+ for detail in [+ include_str!("../expanded/dynamic/database_row_detail.sql"),+ include_str!("../expanded/dynamic/database_row_grouped_detail.sql"),+ ] {+ // Exercise the production fragments; SQLx cannot expand include_str! inside SQL.+ let query = format!(+ "WITH TopItems(item_type, id, sort_ts, sort_key, group_key, group_order, group_total_count, row_in_group) AS (+ VALUES+ ('chat', 'legacy-chat-id', now(), 0::double precision, 'group', NULL::text, 3::bigint, 1::bigint),+ ('project', '70000000-0000-0000-0000-000000000001', now(), 0, 'group', NULL, 3, 2),+ ('database_row', '70000000-0000-0000-0000-000000000003', now(), 0, 'group', NULL, 3, 3)+ ), GroupedItems AS (SELECT * FROM TopItems), details AS ({detail})+ SELECT id FROM details"+ );+ let ids: Vec<String> = sqlx::query_scalar(&query).fetch_all(&pool).await.unwrap();+ assert_eq!(ids, vec!["70000000-0000-0000-0000-000000000003"]);+ }+}++#[sqlx::test(fixtures("test/database_rows.sql"), migrator = "MACRO_DB_MIGRATIONS")]+async fn created_cursor_pages_equal_timestamps_without_skipping_rows(pool: PgPool) {+ sqlx::query!(+ "UPDATE database_rows SET created_at = '2026-01-03T00:00:00Z' WHERE table_id = $1",+ DEALS+ )+ .execute(&pool)+ .await+ .unwrap();+ let filter = rows_of(Some(Expr::val(DatabaseRowLiteral::TableId(DEALS))));+ let first = expanded_dynamic_cursor_soup(+ &pool,+ ExpandedDynamicCursorArgs {+ user_id: viewer(),+ limit: 2,+ cursor: Query::Sort(SimpleSortMethod::CreatedAt.into(), filter.clone()),+ exclude_frecency: false,+ },+ )+ .await+ .unwrap();+ assert_eq!(+ first.iter().map(Identify::id).collect::<Vec<_>>(),+ vec![DEAL_3, DEAL_2]+ );++ let second = expanded_dynamic_cursor_soup(+ &pool,+ ExpandedDynamicCursorArgs {+ user_id: viewer(),+ limit: 2,+ cursor: Query::Cursor(Cursor {+ id: DEAL_2,+ limit: 2,+ val: CursorVal {+ sort_type: SimpleSortMethod::CreatedAt.into(),+ last_val: DynamicSortValue::Timestamp("2026-01-03T00:00:00Z".parse().unwrap()),+ },+ filter,+ }),+ exclude_frecency: false,+ },+ )+ .await+ .unwrap();+ assert_eq!(+ second.iter().map(Identify::id).collect::<Vec<_>>(),+ vec![DEAL_1]+ );+}+#[sqlx::test(fixtures("test/database_rows.sql"), migrator = "MACRO_DB_MIGRATIONS")]async fn core_rows_stay_out_of_soup_even_with_a_stray_database_grant(pool: PgPool) {let database = Uuid::now_v7();diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rsindex 7898d97217..da98412142 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic.rs@@ -1542,6 +1542,7 @@ pub(in crate::outbound::pg_soup_repo) fn project_filter_is_impossible(filter_ast::ExprFrame::And(a, b) => a || b,filter_ast::ExprFrame::Or(a, b) => a && b,filter_ast::ExprFrame::Not(_) => false,+ filter_ast::ExprFrame::Literal(ProjectLiteral::ProjectIdSelf(id)) => id.is_nil(),filter_ast::ExprFrame::Literal(ProjectLiteral::Importance(false)) => true,filter_ast::ExprFrame::Literal(_) => false,})diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_detail.sql b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_detail.sqlindex 6ef3035e6f..fe313b9b29 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_detail.sql+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_detail.sql@@ -32,7 +32,8 @@'createdBy', r.created_by) as "database_row"FROM TopItems t- INNER JOIN database_rows r ON r.id::text = t.id+ -- Keep the indexed UUID column bare. Other Soup kinds can have legacy text ids.+ INNER JOIN database_rows r ON r.id = (CASE WHEN t.item_type = 'database_row' THEN t.id END)::uuidINNER JOIN database_tables row_table ON row_table.id = r.table_idINNER JOIN database_entities row_database ON row_database.database_id = row_table.database_idWHERE t.item_type = 'database_row'diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_grouped_detail.sql b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_grouped_detail.sqlindex 9730e1be71..304c003de9 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_grouped_detail.sql+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/database_row_grouped_detail.sql@@ -36,7 +36,8 @@gi.group_total_count as "group_total_count",gi.row_in_group as "row_in_group"FROM GroupedItems gi- INNER JOIN database_rows r ON r.id::text = gi.id+ -- Keep the indexed UUID column bare. Other Soup kinds can have legacy text ids.+ INNER JOIN database_rows r ON r.id = (CASE WHEN gi.item_type = 'database_row' THEN gi.id END)::uuidINNER JOIN database_tables row_table ON row_table.id = r.table_idINNER JOIN database_entities row_database ON row_database.database_id = row_table.database_idWHERE gi.item_type = 'database_row'diff --git a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rsindex 0c2d7699cf..6665aebb46 100644--- a/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs+++ b/crates/soup/src/outbound/pg_soup_repo/expanded/dynamic/test.rs@@ -2,6 +2,27 @@ use super::*;use macro_db_migrator::MACRO_DB_MIGRATIONS;use std::sync::Arc;+#[test]+fn nil_project_ids_are_impossible_but_parent_and_negated_filters_are_not() {+ let nil = Expr::val(ProjectLiteral::ProjectIdSelf(Uuid::nil()));+ let real = Expr::val(ProjectLiteral::ProjectIdSelf(Uuid::from_u128(1)));+ assert!(project_filter_is_impossible(Some(&nil)));+ assert!(project_filter_is_impossible(Some(&Expr::and(+ nil.clone(),+ real.clone()+ ))));+ assert!(!project_filter_is_impossible(Some(&Expr::or(+ nil.clone(),+ real.clone()+ ))));+ assert!(!project_filter_is_impossible(Some(&Expr::is_not(nil))));+ assert!(!project_filter_is_impossible(Some(&real)));+ assert!(!project_filter_is_impossible(Some(&Expr::val(+ ProjectLiteral::ProjectId(Uuid::nil())+ ))));+ assert!(!project_filter_is_impossible(None));+}+#[test]fn grouped_access_shape() {let filter = EntityFilterAst::mock_empty();diff --git a/crates/workspace-hack/Cargo.toml b/crates/workspace-hack/Cargo.tomlindex 1c36443118..0b9d7dfc99 100644--- a/crates/workspace-hack/Cargo.toml+++ b/crates/workspace-hack/Cargo.toml@@ -20,7 +20,7 @@ aes-gcm = { version = "0.10", features = ["zeroize"] }ahash = { version = "0.8", features = ["serde"] }aho-corasick = { version = "1" }anyhow = { version = "1" }-async-graphql = { version = "7", features = ["dataloader"] }+async-graphql = { version = "7", features = ["chrono", "dataloader"] }async-openai = { version = "0.36", features = ["audio", "byot", "chat-completion", "completions", "embedding"] }aws-config = { version = "1", features = ["behavior-version-latest"] }aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] }@@ -180,7 +180,7 @@ aes-gcm = { version = "0.10", features = ["zeroize"] }ahash = { version = "0.8", features = ["serde"] }aho-corasick = { version = "1" }anyhow = { version = "1" }-async-graphql = { version = "7", features = ["dataloader"] }+async-graphql = { version = "7", features = ["chrono", "dataloader"] }async-openai = { version = "0.36", features = ["audio", "byot", "chat-completion", "completions", "embedding"] }aws-config = { version = "1", features = ["behavior-version-latest"] }aws-credential-types = { version = "1", default-features = false, features = ["hardcoded-credentials", "test-util"] }diff --git a/docs/agent-guide/databases.md b/docs/agent-guide/databases.mdindex 6c4a2c4f89..01ab7d986a 100644--- a/docs/agent-guide/databases.md+++ b/docs/agent-guide/databases.md@@ -80,7 +80,9 @@ The table selector matches the inset project/channel navigation. In narrow panesscroll its tabs horizontally with a trackpad, mouse wheel, or the arrow buttons.The **+** menu stays visible, and focusing or selecting a table reveals its tab.-The grid’s sticky bottom summary shows the saved records in the current view.+The grid’s sticky bottom summary shows the loaded saved records in the current view.+While more pages remain, it says **records loaded** and calculations cover only+those records. Loading every page makes the summary cover the full view.Choose **+ Add calculation** in a column’s footer for filled/unique counts, orsum, average, min, and max on numeric columns. Numeric columns default to Sum.Calculation choices are local to the open table, not saved with the view.@@ -788,6 +790,35 @@ they do not wait for the background catalog refresh. A refused edit rolls backits optimistic state when no newer cache update has replaced it, then refreshes.A slow or failed refresh is not a reason to resend a successful mutation.+## Checking large tables locally++Use synthetic records on an isolated local stack and the+[database profiling harness](../../apps/web/scripts/database-profile/README.md).+Table views without formula columns filter and sort all stored records on the+server before fetching pages of 500. Scroll near the bottom or choose **Load more**+for the next page.+The footer reports loaded records while more pages remain. Only visible grid+rows are mounted, so count loaded records separately from DOM rows. Test a+filter whose matches occur beyond the first page, and a sort whose first record+would otherwise occur near the end of the table.++**New** opens the blank row even in a long table. A duplicated record outside+the loaded rows opens its record panel. A saved-record notice can also open a+record that is not loaded; this does not mean it fails the current filters.+Arrow keys and Tab must reveal and focus cells beyond the visible window.++Boards, tables containing formula columns, and arbitrary SQL still use the+existing engine reader with its 20,000-record cap. The engine computes formula+values before filtering and sorting. Board lane/card ordering is applied to that+result; check the incomplete-results notice when auditing those surfaces.++After loading, switch between tables and views several times, edit a cell,+open a select picker, add and delete a temporary record, and scroll. Confirm+that edits preserve focus and that background refreshes leave the grid mounted.+Count a browser crash or the database error boundary as a failed run, even if+the grid appeared before it. Keep heap/CPU profiling runs separate from timing+runs, and preserve existing local data.+### Embedded database editorsIn a document, `/database` creates a database and inserts its expanded editor.diff --git a/services/document_storage_service/Cargo.toml b/services/document_storage_service/Cargo.tomlindex 07c8ceb16c..c575d222d2 100644--- a/services/document_storage_service/Cargo.toml+++ b/services/document_storage_service/Cargo.toml@@ -157,6 +157,8 @@ github_pull_requests = { path = "../../crates/github_pull_requests", default-feagraphql_soup = { path = "../../crates/graphql_soup" }graphql_crm = { path = "../../crates/graphql_crm" }graphql_initiative = { path = "../../crates/graphql_initiative" }+graphql_databases = { path = "../../crates/graphql_databases" }+graphql_calendar = { path = "../../crates/graphql_calendar" }graphql_scheduled_action = { path = "../../crates/graphql_scheduled_action" }harnesses = { path = "../../crates/harnesses" }diff --git a/services/document_storage_service/src/api/context.rs b/services/document_storage_service/src/api/context.rsindex c1190368d0..84458e8295 100644--- a/services/document_storage_service/src/api/context.rs+++ b/services/document_storage_service/src/api/context.rs@@ -936,6 +936,7 @@ pub(crate) struct ApiContext {pub initiative_state: DssInitiativeState,pub graphql_crm_context: graphql_crm::CrmGraphqlContext,pub graphql_initiative_context: graphql_initiative::InitiativeGraphqlContext,+ pub graphql_database_rows_context: graphql_databases::DatabaseRowsGraphqlContext,pub graphql_scheduled_action_context: graphql_scheduled_action::ScheduledActionGraphqlContext,pub graphql_calendar_context: graphql_calendar::CalendarGraphqlContext,pub graphql_calendar_mutation_context: graphql_calendar::CalendarGraphqlMutationContext,diff --git a/services/document_storage_service/src/api/graphql_soup.rs b/services/document_storage_service/src/api/graphql_soup.rsindex 6d1bdf3d8d..2db74780d2 100644--- a/services/document_storage_service/src/api/graphql_soup.rs+++ b/services/document_storage_service/src/api/graphql_soup.rs@@ -203,6 +203,7 @@ fn insert_graphql_context_data()));data.insert(state.channel_service.clone());data.insert(state.graphql_initiative_context.clone());+ data.insert(state.graphql_database_rows_context.clone());data.insert(state.graphql_scheduled_action_context.clone());data.insert(state.graphql_calendar_context.clone());data.insert(state.graphql_calendar_mutation_context.clone());diff --git a/services/document_storage_service/src/server.rs b/services/document_storage_service/src/server.rsindex 8a9f5865cd..bb1b7ecc2c 100644--- a/services/document_storage_service/src/server.rs+++ b/services/document_storage_service/src/server.rs@@ -2195,6 +2195,18 @@ pub async fn run() -> anyhow::Result<()> {entity_access_service.as_ref().clone(),),),+ graphql_database_rows_context: graphql_databases::DatabaseRowsGraphqlContext::new(+ databases_sql::view_rows::DatabaseViewRows::new(+ databases_service.clone(),+ databases::outbound::pg_view_rows::PgViewRows::new(db.clone()),+ ),+ entity_access_service.clone(),+ graphql_soup::soup_item_loader(+ primary_entity_soup.clone(),+ Arc::new(email_service.clone()),+ entity_access_service.as_ref().clone(),+ ),+ ),graphql_primary_soup_item_loader: graphql_soup::PrimarySoupItemLoader(graphql_soup::soup_item_loader_with_team_access(primary_entity_soup,diff --git a/static_assets/schema.graphql b/static_assets/schema.graphqlindex e7d1304d9d..e19dc137c7 100644--- a/static_assets/schema.graphql+++ b/static_assets/schema.graphql@@ -603,6 +603,445 @@ input CreateSavedViewInput {sharedTeamId: ID}+"""+Match the checked or unchecked state of a cell.+"""+input DatabaseViewCheckboxTest {+ """+ Whether the cell must be checked.+ """+ checked: Boolean!+}++"""+How the conditions within a filter group combine.+"""+enum DatabaseViewConjunction {+ """+ Require every condition.+ """+ AND+ """+ Require at least one condition.+ """+ OR+}++"""+An ordering comparison between instants.+"""+enum DatabaseViewDateOperator {+ """+ The instant is before the comparison instant.+ """+ BEFORE+ """+ The instant is after the comparison instant.+ """+ AFTER+ """+ The instant is at or before the comparison instant.+ """+ ON_OR_BEFORE+ """+ The instant is at or after the comparison instant.+ """+ ON_OR_AFTER+}++"""+How a date range relates to a day.+"""+enum DatabaseViewDateRangeOperator {+ """+ The range overlaps the day.+ """+ CONTAINS+ """+ The range misses the day, or is empty.+ """+ DOES_NOT_CONTAIN+ """+ The range ends before the day.+ """+ ENDS_BEFORE+ """+ The range ends on or after the day.+ """+ ENDS_ON_OR_AFTER+ """+ The range starts after the day.+ """+ STARTS_AFTER+ """+ The range starts on or before the day.+ """+ STARTS_ON_OR_BEFORE+}++"""+Compare a date range cell with the day starting at a UTC instant.+"""+input DatabaseViewDateRangeTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewDateRangeOperator!+ """+ The start of the day compared against.+ """+ value: DateTime!+}++"""+Compare a date cell with a UTC instant.+"""+input DatabaseViewDateTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewDateOperator!+ """+ The value to compare the cell against.+ """+ value: DateTime!+}++"""+Compare referenced entity identities with a set of entities.+"""+input DatabaseViewEntitiesTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewSetOperator!+ """+ Referenced entity identities.+ """+ entities: [String!]!+}++"""+A typed predicate on one column placement.+"""+input DatabaseViewFilterCondition {+ """+ Column placement identity within the selected table.+ """+ column: ID!+ """+ The comparison appropriate for this column.+ """+ test: DatabaseViewFilterTest!+}++"""+A conjunction of column conditions or nested groups.+"""+input DatabaseViewFilterGroup {+ """+ Require every condition or at least one condition.+ """+ conjunction: DatabaseViewConjunction!+ """+ Column tests or nested groups.+ """+ conditions: [DatabaseViewFilterNode!]!+}++"""+Exactly one column condition or nested filter group.+"""+input DatabaseViewFilterNode @oneOf {+ """+ A predicate on one column.+ """+ condition: DatabaseViewFilterCondition+ """+ A nested group of predicates.+ """+ group: DatabaseViewFilterGroup+}++"""+Exactly one test compatible with the selected column type.+"""+input DatabaseViewFilterTest @oneOf {+ """+ Test cell emptiness.+ """+ presence: DatabaseViewPresenceTest+ """+ Test a text or link cell.+ """+ text: DatabaseViewTextTest+ """+ Test a numeric cell.+ """+ number: DatabaseViewNumberTest+ """+ Test a date cell.+ """+ date: DatabaseViewDateTest+ """+ Test a date range cell against a day.+ """+ dateRange: DatabaseViewDateRangeTest+ """+ Test a checkbox cell.+ """+ checkbox: DatabaseViewCheckboxTest+ """+ Test selected option identities.+ """+ options: DatabaseViewOptionsTest+ """+ Test referenced entity identities.+ """+ entities: DatabaseViewEntitiesTest+}++"""+A numeric equality or ordering comparison.+"""+enum DatabaseViewNumberOperator {+ """+ The value equals the comparison value.+ """+ IS+ """+ The value differs from the comparison value.+ """+ IS_NOT+ """+ The number is greater than the comparison value.+ """+ GREATER_THAN+ """+ The number is at least the comparison value.+ """+ GREATER_THAN_OR_EQUAL+ """+ The number is less than the comparison value.+ """+ LESS_THAN+ """+ The number is at most the comparison value.+ """+ LESS_THAN_OR_EQUAL+}++"""+Compare a numeric cell with a finite number.+"""+input DatabaseViewNumberTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewNumberOperator!+ """+ The value to compare the cell against.+ """+ value: Float!+}++"""+Compare selected option identities with a set of options.+"""+input DatabaseViewOptionsTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewSetOperator!+ """+ Option identities from the selected column.+ """+ options: [ID!]!+}++"""+The requested cell-presence condition.+"""+enum DatabaseViewPresenceOperator {+ """+ The cell has no value.+ """+ IS_EMPTY+ """+ The cell has a value.+ """+ IS_NOT_EMPTY+}++"""+Test whether the cell has a value.+"""+input DatabaseViewPresenceTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewPresenceOperator!+}++"""+Filtering and ordering applied to the entire table before pagination.+"""+input DatabaseViewQueryInput {+ """+ Optional filter; omission includes every row.+ """+ filter: DatabaseViewFilterGroup+ """+ Ordering keys in priority order; omission uses manual row order.+ """+ sort: [DatabaseViewSort!]! = []+}++"""+A table view and its optional continuation, with a bounded page size.+"""+input DatabaseViewRowsInput {+ """+ The table within the authorized database.+ """+ tableId: ID!+ """+ The complete filtering and ordering definition.+ """+ query: DatabaseViewQueryInput!+ """+ The preceding page’s opaque cursor; omit for the first page.+ """+ cursor: String+ """+ Page size from 1 through 500; defaults to 500.+ """+ limit: Int+}++"""+Membership and order belong to this page; row facts retain their shared identity.+"""+type DatabaseViewRowsPage {+ """+ Rows in global view order, hydrated as canonical Soup entities.+ """+ items: [GraphqlSoupDatabaseRow!]!+ """+ Continuation for the next page; absent at the end of the result.+ """+ nextCursor: String+ """+ Table version against which this page was selected.+ """+ version: Int!+}++"""+A membership condition for single- or multi-valued cells.+"""+enum DatabaseViewSetOperator {+ """+ The single-valued cell matches one supplied identity.+ """+ IS_ANY_OF+ """+ The single-valued cell matches no supplied identity.+ """+ IS_NONE_OF+ """+ The cell contains at least one supplied identity.+ """+ HAS_ANY+ """+ The cell contains every supplied identity.+ """+ HAS_ALL+ """+ The cell contains no supplied identity.+ """+ HAS_NONE+}++"""+One ordering key; earlier keys take precedence.+"""+input DatabaseViewSort {+ """+ Column placement identity within the selected table.+ """+ column: ID!+ """+ Ascending or descending value order.+ """+ direction: DatabaseViewSortDirection!+}++"""+The direction of a view ordering key.+"""+enum DatabaseViewSortDirection {+ """+ Order values from lowest to highest.+ """+ ASCENDING+ """+ Order values from highest to lowest.+ """+ DESCENDING+}++"""+A text equality or substring comparison.+"""+enum DatabaseViewTextOperator {+ """+ The value equals the comparison value.+ """+ IS+ """+ The value differs from the comparison value.+ """+ IS_NOT+ """+ The text contains the comparison string.+ """+ CONTAINS+ """+ The text does not contain the comparison string.+ """+ DOES_NOT_CONTAIN+ """+ The text starts with the comparison string.+ """+ STARTS_WITH+ """+ The text ends with the comparison string.+ """+ ENDS_WITH+}++"""+Compare a text or link cell with a string.+"""+input DatabaseViewTextTest {+ """+ The comparison to apply.+ """+ operator: DatabaseViewTextOperator!+ """+ The value to compare the cell against.+ """+ value: String!+}++"""+Implement the DateTime<Utc> scalar++The input/output is a string in RFC3339 format.+"""+scalar DateTime+"""Delete an event, one occurrence, or an occurrence onward."""@@ -9639,6 +10078,10 @@ type GraphqlUser {"""id: ID!"""+ A page filtered and ordered across the entire authorized table.+ """+ databaseViewRows(databaseId: ID!, input: DatabaseViewRowsInput!): DatabaseViewRowsPage!+ """One initiative accessible to the authenticated viewer."""initiative(initiativeId: ID!): GraphqlSoupInitiative!@@ -11374,6 +11817,10 @@ directive @oneOf on INPUT_OBJECTDirects the executor to skip this field or fragment when the `if` argument is true."""directive @skip(if: Boolean!) on FIELD | FRAGMENT_SPREAD | INLINE_FRAGMENT+"""+Provides a scalar specification URL for specifying the behavior of custom scalar types.+"""+directive @specifiedBy(url: String!) on SCALARschema {query: SoupQueryRootmutation: CompleteMutationRoot