A local-first personal finance app
[AI] migrations: pending messages DB (#8519)
Matiss Janis Aboltins2fc69915c21f
diff --git a/packages/desktop-client/src/sync-events.ts b/packages/desktop-client/src/sync-events.tsindex 595687e3b9..aafe5ed557 100644--- a/packages/desktop-client/src/sync-events.ts+++ b/packages/desktop-client/src/sync-events.ts@@ -1,4 +1,5 @@import { listen, send } from '@actual-app/core/platform/client/connection';+import type { ServerEvents } from '@actual-app/core/types/server-events';import type { QueryClient } from '@tanstack/react-query';import { t } from 'i18next';@@ -18,6 +19,34 @@ import { loadPrefs } from './prefs/prefsSlice';import type { AppStore } from './redux/store';import { signOut } from './users/usersSlice';+// Notifications for sync events that can fire during budget load+const syncNotifications: Partial<+ Record<ServerEvents['sync-event']['type'], () => Notification>+> = {+ 'deferred-messages': () => ({+ id: 'deferred-messages',+ type: 'message',+ // Sticky: firing during budget load, it would otherwise+ // auto-dismiss before the user can read it+ sticky: true,+ title: t('Update available'),+ message: t(+ 'Some synced changes were made with a newer version of Actual. ' ++ 'They will show up here once you update this app.',+ ),+ }),+ 'dropped-messages': () => ({+ id: 'dropped-messages',+ type: 'warning',+ sticky: true,+ title: t('Some changes could not be applied'),+ message: t(+ 'Some changes made on another device could not be applied ' ++ 'on this device, so your devices may show different values.',+ ),+ }),+};+export function listenForSyncEvent(store: AppStore, queryClient: QueryClient) {// TODO: Should this run on mobile too?const unlistenUnauthorized = listen('sync-event', async ({ type }) => {@@ -39,7 +68,20 @@ export function listenForSyncEvent(store: AppStore, queryClient: QueryClient) {const unlistenSuccess = listen('sync-event', event => {const prefs = store.getState().prefs.local;- if (!prefs || !prefs.id) {+ const hasBudget = Boolean(prefs && prefs.id);++ const makeNotification = syncNotifications[event.type];+ if (makeNotification) {+ // Accept these while a budget is loaded OR still loading (prefs+ // arrive only after the load completes); with neither, the user+ // has left the budget and the notification would lack context+ if (hasBudget || store.getState().app.loadingText != null) {+ store.dispatch(addNotification({ notification: makeNotification() }));+ }+ return;+ }++ if (!hasBudget) {// Do nothing if no budget is loadedreturn;}diff --git a/packages/desktop-client/src/util/error.ts b/packages/desktop-client/src/util/error.tsindex ed1bafa0dd..e09ef1e5b1 100644--- a/packages/desktop-client/src/util/error.ts+++ b/packages/desktop-client/src/util/error.ts@@ -1,4 +1,8 @@-import { getUnsafeZipMeta, toMB } from '@actual-app/core/shared/errors';+import {+ getUnsafeZipMeta,+ isDatabaseSchemaMismatch,+ toMB,+} from '@actual-app/core/shared/errors';import type { UnsafeZipMeta } from '@actual-app/core/shared/errors';import { t } from 'i18next';@@ -85,21 +89,6 @@ export function getUploadError({ reason, meta }: ErrorWithMeta) {}}-function isDatabaseSchemaMismatch(meta?: unknown): boolean {- if (- meta &&- typeof meta === 'object' &&- 'error' in meta &&- meta.error &&- typeof meta.error === 'object' &&- 'message' in meta.error &&- typeof meta.error.message === 'string'- ) {- return /no such (column|table)/i.test(meta.error.message);- }- return false;-}-function getSchemaMismatchError() {return t('This budget could not be loaded because it uses a newer database schema than this version of Actual supports. Make sure you are using the latest version, then try again.',diff --git a/packages/loot-core/migrations/1788468782000_add_messages_pending.js b/packages/loot-core/migrations/1788468782000_add_messages_pending.jsnew file mode 100644index 0000000000..d1144c897d--- /dev/null+++ b/packages/loot-core/migrations/1788468782000_add_messages_pending.js@@ -0,0 +1,5 @@+import { PENDING_MESSAGES_TABLE_SQL } from '#server/sync/messages-pending';++export default async function runMigration(db) {+ db.execQuery(PENDING_MESSAGES_TABLE_SQL);+}diff --git a/packages/loot-core/migrations/README.md b/packages/loot-core/migrations/README.mdnew file mode 100644index 0000000000..007e784977--- /dev/null+++ b/packages/loot-core/migrations/README.md@@ -0,0 +1,45 @@+# Database migrations++Actual upgrades local budget databases by running the SQL and JavaScript files in this directory in timestamp order. Each file name must start with a numeric id (typically the authoring timestamp in milliseconds), followed by a short description.++## Cross-version sync++Actual supports running different app versions against the same synced budget. A device on an older release may still receive sync messages produced by a newer release, and it must keep loading and syncing until it upgrades.++That only works if newer migrations are **additive-only**: they may extend the schema, but must not remove or rename anything an older client still reads or writes through sync.++Post-cutoff migrations (ids after `1780606215001`, see `ADDITIVE_ONLY_CUTOFF` in `src/server/migrate/migrations.ts`) are enforced by `src/server/migrate/additive-migrations.test.ts`. CI runs that test against the real migration chain.++## Allowed changes++After the cutoff, a migration may:++- Create a new table. Synced tables must use a single primary key column named `id`.+- Add a nullable column, or a `NOT NULL` column with a `DEFAULT`.+- Add a non-unique index.+- Drop or recreate a **view** (views are rebuilt on startup and are not synced).+- Create an internal (non-synced) table. Register it in `NON_SYNCED_TABLES` in `additive-migrations.test.ts`. Internal tables are only exempt from the synced-table rules at creation time; later changes are validated like any other table.++Use `CREATE TABLE IF NOT EXISTS` when a migration might run against a database that already has the table (for example after a migration id is renumbered).++## Disallowed changes++Do **not** drop or rename tables or columns. Do not rebuild a table to change an existing column's type, nullability, default, primary-key membership, `UNIQUE` constraints, or `CHECK` expressions. Do not add `UNIQUE` constraints or required columns without defaults to existing synced tables.++If a column is no longer used, stop reading it in application code and leave it in the database.++Renaming is not supported even via `ALTER TABLE ... RENAME`. Older clients address synced data by table and column name; a rename looks like a delete plus an add and breaks cross-version sync.++## Migration ids++- Use a current timestamp for new migrations. Do not reuse or backdate ids into the pre-cutoff era.+- A long-lived branch may merge after other migrations landed on `master`. If your migration id sorts below ids already on `master`, bump it to a new timestamp before merge so existing databases do not treat it as already applied.+- If you must change a migration that has already shipped, prefer a new migration file rather than editing the old one. If you renumber an id, make the SQL idempotent (`IF NOT EXISTS`, and so on).++## What the test does not cover++`additive-migrations.test.ts` compares the database **schema** before and after each post-cutoff migration. It does not inspect `UPDATE`/`INSERT` statements that rewrite existing row data.++A data migration that changes the meaning or format of values already stored in a column can pass the test but still confuse an older app version reading those rows or applying sync messages. Prefer leaving existing values as-is, or gate behavioral changes on the app version reading them.++For sync **values** (not just columns) from newer clients, see deferred message handling in `src/server/sync/` and `messages_pending`.diff --git a/packages/loot-core/src/server/budgetfiles/app.ts b/packages/loot-core/src/server/budgetfiles/app.tsindex f5e50384fa..0acd745ecc 100644--- a/packages/loot-core/src/server/budgetfiles/app.ts+++ b/packages/loot-core/src/server/budgetfiles/app.ts@@ -26,6 +26,7 @@ import {setSyncingMode,} from '#server/sync';import * as syncMigrations from '#server/sync/migrate';+import { replayPendingMessages } from '#server/sync/replay';import * as rules from '#server/transactions/transaction-rules';import { clearUndo } from '#server/undo';import { updateVersion } from '#server/update';@@ -597,6 +598,20 @@ async function _loadBudget(id: Budget['id']): Promise<{return result;}+ try {+ // Apply sync messages deferred from a newer app version, now that+ // migrations have run. Skipped without a sync server: serverless+ // edits aren't in the crdt log, so replay couldn't tell whether a+ // pending value is stale.+ if (getServer()) {+ replayPendingMessages();+ }+ } catch (e) {+ // Failing to replay shouldn't block loading the budget; the+ // messages stay pending+ captureException(e);+ }+await db.loadClock();if (prefs.getPrefs().resetClock) {diff --git a/packages/loot-core/src/server/db/types/index.ts b/packages/loot-core/src/server/db/types/index.tsindex 4991cb285c..3a68be278f 100644--- a/packages/loot-core/src/server/db/types/index.ts+++ b/packages/loot-core/src/server/db/types/index.ts@@ -109,6 +109,14 @@ export type DbCrdtMessage = {value: Uint8Array;};+export type DbPendingMessage = {+ timestamp: string;+ dataset: string;+ row: string;+ column: string;+ value: string;+};+export type DbNote = {id: string;note: string;diff --git a/packages/loot-core/src/server/migrate/additive-migrations.test.ts b/packages/loot-core/src/server/migrate/additive-migrations.test.tsnew file mode 100644index 0000000000..7aa56c4ce3--- /dev/null+++ b/packages/loot-core/src/server/migrate/additive-migrations.test.ts@@ -0,0 +1,362 @@+import * as nativeFs from 'fs';+import * as path from 'path';++import type { Database } from '@jlongster/sql.js';+import { describe, expect, it } from 'vitest';++import * as sqlite from '#platform/server/sqlite';++import {+ ADDITIVE_ONLY_CUTOFF,+ applyMigration,+ getMigrationId,+ getMigrationList,+ migrate,+ withMigrationsDir,+} from './migrations';+import { findAdditiveViolations, snapshotSchema } from './schema-diff';+import type { SchemaSnapshot } from './schema-diff';++// Migrations newer than ADDITIVE_ONLY_CUTOFF must be additive-only:+// clients tolerate budgets and sync messages from newer app versions+// (see `replayPendingMessages` and `checkDatabaseValidity`), which is+// only safe if newer migrations never remove or rename what older+// clients depend on. Enforced by running the real migration chain and+// diffing the actual schema around each new migration.++const MIGRATIONS_DIR = path.resolve(__dirname, '../../../migrations');+const INIT_SQL = path.resolve(__dirname, '../sql/init.sql');++// Internal (non-CRDT-synced) tables, exempt from the synced-table rules+// at creation only (see `findAdditiveViolations`). Add new internal+// tables here.+const NON_SYNCED_TABLES = new Set(['messages_pending']);++async function openTestDb(setupSql: string): Promise<Database> {+ await sqlite.init();+ const db = await sqlite.openDatabase(':memory:');+ sqlite.execQuery(db, setupSql);+ return db;+}++// The additive-only violations that `migrationSql` introduces on a+// database prepared with `setupSql`+async function violationsFor(+ setupSql: string,+ migrationSql: string,+): Promise<string[]> {+ const db = await openTestDb(setupSql);+ const before = snapshotSchema(db);+ sqlite.execQuery(db, migrationSql);+ const violations = findAdditiveViolations(+ before,+ snapshotSchema(db),+ NON_SYNCED_TABLES,+ );+ sqlite.closeDatabase(db);+ return violations;+}++describe('migrations are additive-only', () => {+ it('new migrations carry post-cutoff ids', async () => {+ // `checkDatabaseValidity` treats an unknown pre-cutoff id as+ // corruption, so merging a migration with a pre-cutoff id (e.g. a+ // long-lived PR keeping its original authoring timestamp) would+ // break every upgrade. The pre-cutoff set shipped long ago and is+ // frozen — if this count changed, renumber the new migration's id+ // to the present day.+ const ids = (await getMigrationList(MIGRATIONS_DIR)).map(getMigrationId);+ expect(ids.filter(id => id <= ADDITIVE_ONLY_CUTOFF).length).toBe(57);+ });++ it('every migration after the cutoff is additive-only', async () => {+ const db = await openTestDb(nativeFs.readFileSync(INIT_SQL, 'utf8'));++ const violations: string[] = [];+ let snapshot: SchemaSnapshot | null = null;+ for (const name of await getMigrationList(MIGRATIONS_DIR)) {+ if (getMigrationId(name) <= ADDITIVE_ONLY_CUTOFF) {+ await applyMigration(db, name, MIGRATIONS_DIR);+ snapshot = null;+ continue;+ }+ const before = snapshot ?? snapshotSchema(db);+ await applyMigration(db, name, MIGRATIONS_DIR);+ snapshot = snapshotSchema(db);+ violations.push(+ ...findAdditiveViolations(before, snapshot, NON_SYNCED_TABLES).map(+ violation => `${name}: ${violation}`,+ ),+ );+ }+ sqlite.closeDatabase(db);++ expect(+ violations,+ 'Migrations must be additive-only so that older clients keep ' ++ 'working: no dropping or renaming tables/columns, new columns ' ++ 'in synced tables must be nullable or have a DEFAULT, and new ' ++ 'synced tables need a single primary key named "id" (sync ' ++ 'builds rows one column at a time, addressed by id). If you ' ++ 'need to retire a column, stop reading it but leave it in ' ++ 'place. New internal (non-synced) tables go in ' ++ 'NON_SYNCED_TABLES in this test.',+ ).toEqual([]);+ });++ it('tolerates an unknown migration id interleaved below the newest known one', async () => {+ const db = await openTestDb(nativeFs.readFileSync(INIT_SQL, 'utf8'));+ await withMigrationsDir(MIGRATIONS_DIR, async () => {+ await migrate(db);+ // A newer release may ship a migration whose id (an authoring+ // timestamp) sorts below ids this version already knows — it is+ // still additive-era and must not fail validation+ sqlite.runQuery(db, 'INSERT INTO __migrations__ (id) VALUES (?)', [+ ADDITIVE_ONLY_CUTOFF + 1,+ ]);+ await migrate(db);+ });+ sqlite.closeDatabase(db);+ });++ const TABLE_FOO = 'CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT, b TEXT);';++ it.each([+ ['dropping a table', TABLE_FOO, 'DROP TABLE foo;'],+ ['renaming a table', TABLE_FOO, 'ALTER TABLE foo RENAME TO bar;'],+ ['renaming a column', TABLE_FOO, 'ALTER TABLE foo RENAME COLUMN a TO c;'],+ [+ 'dropping a column via table rebuild',+ TABLE_FOO,+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, b TEXT);+ INSERT INTO foo_new SELECT id, b FROM foo;+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'adding a required column without a default',+ TABLE_FOO,+ 'ALTER TABLE foo ADD COLUMN c TEXT NOT NULL;',+ ],+ [+ 'creating a synced table with required columns lacking defaults',+ TABLE_FOO,+ `CREATE TABLE gadgets+ (id TEXT PRIMARY KEY,+ name TEXT NOT NULL,+ size INTEGER NOT NULL,+ color TEXT,+ shape TEXT DEFAULT 'round');`,+ ],+ [+ 'creating a synced table whose primary key is not "id"',+ TABLE_FOO,+ 'CREATE TABLE gadgets (uuid TEXT PRIMARY KEY, name TEXT);',+ ],+ [+ 'creating a synced table with a composite primary key',+ TABLE_FOO,+ 'CREATE TABLE gadgets (id TEXT, name TEXT, PRIMARY KEY (id, name));',+ ],+ [+ 'creating a synced table with no primary key',+ TABLE_FOO,+ 'CREATE TABLE gadgets (name TEXT);',+ ],+ [+ 'making an existing column required via table rebuild',+ TABLE_FOO,+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT NOT NULL, b TEXT);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'changing primary-key membership via table rebuild',+ TABLE_FOO,+ `CREATE TABLE foo_new (id TEXT, a TEXT, b TEXT, PRIMARY KEY (id, a));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'adding a UNIQUE index to a synced table',+ TABLE_FOO,+ 'CREATE UNIQUE INDEX foo_a ON foo (a);',+ ],+ [+ 'adding a UNIQUE constraint via table rebuild',+ TABLE_FOO,+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT UNIQUE, b TEXT);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'adding a required column to an existing internal table',+ TABLE_FOO ++ 'CREATE TABLE messages_pending (timestamp TEXT PRIMARY KEY, dataset TEXT NOT NULL);',+ 'ALTER TABLE messages_pending ADD COLUMN extra TEXT NOT NULL;',+ ],+ [+ 'removing a UNIQUE constraint',+ TABLE_FOO + 'CREATE UNIQUE INDEX foo_a ON foo (a);',+ 'DROP INDEX foo_a;',+ ],+ [+ 'adding a CHECK constraint',+ TABLE_FOO,+ 'ALTER TABLE foo ADD COLUMN amount INTEGER CHECK(amount >= 0);',+ ],+ [+ 'making a column NOT NULL with a DEFAULT via table rebuild',+ TABLE_FOO,+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, a TEXT NOT NULL DEFAULT 'x', b TEXT);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'making a NOT NULL column nullable via table rebuild',+ "CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT NOT NULL DEFAULT 'x');",+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'removing a DEFAULT from a NOT NULL column via table rebuild',+ "CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT NOT NULL DEFAULT 'x');",+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT NOT NULL);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'changing a CHECK expression via table rebuild',+ 'CREATE TABLE foo (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount >= 0));',+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount > 0));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'changing a CHECK expression after a quoted paren',+ `CREATE TABLE foo+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'x'));`,+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'y'));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'changing a CHECK string literal case',+ `CREATE TABLE foo+ (id TEXT PRIMARY KEY, status TEXT CHECK(status <> 'ARCHIVED'));`,+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, status TEXT CHECK(status <> 'archived'));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'adding a NOT NULL column whose DEFAULT is an explicit NULL',+ TABLE_FOO,+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, a TEXT, b TEXT, c TEXT NOT NULL DEFAULT NULL);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ "changing an existing column's type via table rebuild",+ TABLE_FOO,+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a INTEGER, b TEXT);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ "changing an existing column's DEFAULT value via table rebuild",+ "CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT DEFAULT 'x');",+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT DEFAULT 'y');+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ ])('sanity check: flags %s', async (_case, setup, migration) => {+ expect(await violationsFor(setup, migration)).not.toEqual([]);+ });++ it.each([+ [+ 'adding nullable and defaulted columns',+ TABLE_FOO,+ `ALTER TABLE foo ADD COLUMN c TEXT;+ ALTER TABLE foo ADD COLUMN d TEXT NOT NULL DEFAULT 'x';`,+ ],+ [+ 'creating a table with nullable and defaulted columns',+ TABLE_FOO,+ `CREATE TABLE gadgets+ (id TEXT PRIMARY KEY, name TEXT, size INTEGER NOT NULL DEFAULT 0);`,+ ],+ [+ 'dropping and recreating a view',+ TABLE_FOO + 'CREATE VIEW v_foo AS SELECT id, a FROM foo;',+ `DROP VIEW v_foo;+ CREATE VIEW v_foo AS SELECT id, a, b FROM foo;`,+ ],+ [+ 'creating an internal (non-synced) table with required columns',+ TABLE_FOO,+ `CREATE TABLE messages_pending+ (timestamp TEXT PRIMARY KEY, dataset TEXT NOT NULL);`,+ ],+ [+ 'adding a plain (non-unique) index',+ TABLE_FOO,+ 'CREATE INDEX foo_a_idx ON foo (a);',+ ],+ [+ 'creating a synced table with a UNIQUE constraint',+ TABLE_FOO,+ 'CREATE TABLE gadgets (id TEXT PRIMARY KEY, name TEXT UNIQUE);',+ ],+ [+ 'creating a synced table with a CHECK constraint',+ TABLE_FOO,+ 'CREATE TABLE gadgets (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount >= 0));',+ ],+ [+ 'rebuilding a table with an identical CHECK, reformatted',+ 'CREATE TABLE foo (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount >= 0));',+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, amount INTEGER CHECK( amount >= 0 ));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'rebuilding a table with an identical CHECK containing a quoted paren',+ `CREATE TABLE foo+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'x'));`,+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'x'));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'rebuilding a table with an identical CHECK, reformatted keywords',+ `CREATE TABLE foo+ (id TEXT PRIMARY KEY, status TEXT CHECK(status <> 'ARCHIVED'));`,+ `CREATE TABLE foo_new+ (id TEXT PRIMARY KEY, status TEXT CHECK( STATUS <> 'ARCHIVED' ));+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ [+ 'adding a column whose quoted DEFAULT contains CHECK(...)',+ TABLE_FOO,+ "ALTER TABLE foo ADD COLUMN c TEXT DEFAULT 'CHECK(c > 0)';",+ ],+ [+ 'rebuilding a table replacing an explicit DEFAULT NULL with no DEFAULT',+ 'CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT DEFAULT NULL);',+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT);+ DROP TABLE foo;+ ALTER TABLE foo_new RENAME TO foo;`,+ ],+ ])('sanity check: allows %s', async (_case, setup, migration) => {+ expect(await violationsFor(setup, migration)).toEqual([]);+ });+});diff --git a/packages/loot-core/src/server/migrate/migrations.test.ts b/packages/loot-core/src/server/migrate/migrations.test.tsindex d66b12925d..04cef9619d 100644--- a/packages/loot-core/src/server/migrate/migrations.test.ts+++ b/packages/loot-core/src/server/migrate/migrations.test.ts@@ -1,4 +1,8 @@// @ts-strict-ignore+import * as nativeFs from 'fs';+import * as os from 'os';+import * as path from 'path';+import * as db from '#server/db';import {@@ -58,6 +62,103 @@ describe('Migrations', () => {);});+ test('tolerates migrations applied by a newer version of the app', async () => {+ return withMigrationsDir(+ __dirname + '/../../mocks/migrations',+ async () => {+ await migrate(db.getDatabase());++ // Simulate a migration applied by a newer version of the app+ // (its id is newer than anything this version knows about)+ db.runQuery('INSERT INTO __migrations__ (id) VALUES (9999999999999)');++ // Should not throw+ await migrate(db.getDatabase());++ const applied = await getAppliedMigrations(db.getDatabase());+ expect(applied).toContain(9999999999999);+ },+ );+ });++ test('rejects a newer unknown migration when a known one is missing', async () => {+ return withMigrationsDir(+ __dirname + '/../../mocks/migrations',+ async () => {+ // A database that skipped known migrations but somehow contains+ // one from a newer version — impossible via any legitimate flow+ // (append-only migrations mean the newer version knew ours too),+ // so it must be treated as corrupt, not migrated further+ db.runQuery('INSERT INTO __migrations__ (id) VALUES (1508717984291)');+ db.runQuery('INSERT INTO __migrations__ (id) VALUES (9999999999999)');++ await expect(migrate(db.getDatabase())).rejects.toThrow(+ 'out-of-sync-migrations',+ );+ },+ );+ });++ test('applies a pending migration whose id sorts below an applied one', async () => {+ const dir = nativeFs.mkdtempSync(+ path.join(os.tmpdir(), 'interleaved-migrations-'),+ );+ try {+ nativeFs.writeFileSync(+ path.join(dir, '1790000000001_a.sql'),+ 'CREATE TABLE interleave_a (id TEXT PRIMARY KEY);',+ );+ nativeFs.writeFileSync(+ path.join(dir, '1790000000003_c.sql'),+ 'CREATE TABLE interleave_c (id TEXT PRIMARY KEY);',+ );+ await withMigrationsDir(dir, async () => {+ await migrate(db.getDatabase());+ });++ // A later release ships a migration authored earlier: its id+ // sorts between two already-applied ones. The upgrade must treat+ // it as pending, not as an out-of-sync database.+ nativeFs.writeFileSync(+ path.join(dir, '1790000000002_b.sql'),+ 'CREATE TABLE interleave_b (id TEXT PRIMARY KEY);',+ );+ await withMigrationsDir(dir, async () => {+ await migrate(db.getDatabase());+ });++ const applied = await getAppliedMigrations(db.getDatabase());+ expect(applied).toContain(1790000000002);+ const desc = await db.first<{ name: string }>(+ "SELECT name FROM sqlite_master WHERE name = 'interleave_b'",+ );+ expect(desc?.name).toBe('interleave_b');+ } finally {+ nativeFs.rmSync(dir, { recursive: true, force: true });+ }+ });++ test('rejects a migrated database when no migrations exist on disk', async () => {+ await withMigrationsDir(__dirname + '/../../mocks/migrations', async () => {+ await migrate(db.getDatabase());+ });++ // An empty migrations directory — a broken install must not pass+ // validation just because every applied id looks unknown+ const dir = nativeFs.mkdtempSync(+ path.join(os.tmpdir(), 'empty-migrations-'),+ );+ try {+ await withMigrationsDir(dir, async () => {+ await expect(migrate(db.getDatabase())).rejects.toThrow(+ 'out-of-sync-migrations',+ );+ });+ } finally {+ nativeFs.rmSync(dir, { recursive: true, force: true });+ }+ });+test('app runs database migrations', async () => {return withMigrationsDir(__dirname + '/../../mocks/migrations',diff --git a/packages/loot-core/src/server/migrate/migrations.ts b/packages/loot-core/src/server/migrate/migrations.tsindex 50c6eb8110..8206610040 100644--- a/packages/loot-core/src/server/migrate/migrations.ts+++ b/packages/loot-core/src/server/migrate/migrations.ts@@ -9,6 +9,7 @@ import m1722717601000 from '#migrations/1722717601000_reports_move_selected_cateimport m1722804019000 from '#migrations/1722804019000_create_dashboard_table';import m1723665565000 from '#migrations/1723665565000_prefs';import m1765518577215 from '#migrations/1765518577215_multiple_dashboards';+import m1788468782000 from '#migrations/1788468782000_add_messages_pending';import * as fs from '#platform/server/fs';import { logger } from '#platform/server/log';import * as sqlite from '#platform/server/sqlite';@@ -22,6 +23,7 @@ const javascriptMigrations = {1722804019000: m1722804019000,1723665565000: m1723665565000,1765518577215: m1765518577215,+ 1788468782000: m1788468782000,};export async function withMigrationsDir(@@ -38,7 +40,7 @@ export function getMigrationsDir(): string {return MIGRATIONS_DIR;}-function getMigrationId(name: string): number {+export function getMigrationId(name: string): number {return parseInt(name.match(/^(\d)+/)[0]);}@@ -143,13 +145,36 @@ export async function applyMigration(]);}+// Migrations with ids after this point are additive-only, enforced by+// additive-migrations.test.ts. A database containing unknown applied+// ids from that era was touched by a newer release — including one+// whose migration id happens to sort below ids this version already+// knows (authored earlier, merged later) — and is still safe to open.+// Unknown ids from before this point indicate a corrupt or+// incompatible database.+export const ADDITIVE_ONLY_CUTOFF = 1780606215001;+function checkDatabaseValidity(appliedIds: number[],available: string[],): void {- if (appliedIds.length > available.length) {+ // A migrated database with no migrations on disk means the install is+ // broken — without this guard every applied id would count as+ // "unknown but tolerable" below and the checks would pass vacuously+ if (available.length === 0 && appliedIds.length > 0) {+ logger.error('No migrations found on disk for a migrated database:', {+ appliedIds,+ });+ throw new Error('out-of-sync-migrations');+ }++ const allAvailableIds = available.map(getMigrationId);+ const availableIds = new Set(allAvailableIds);+ const unknownIds = appliedIds.filter(id => !availableIds.has(id));++ if (unknownIds.some(id => id <= ADDITIVE_ONLY_CUTOFF)) {logger.error(- 'Database is out of sync with migrations (index past available):',+ 'Database is out of sync with migrations (unknown migration from before the additive-only era):',{appliedIds,available,@@ -158,8 +183,53 @@ function checkDatabaseValidity(throw new Error('out-of-sync-migrations');}- for (let i = 0; i < appliedIds.length; i++) {- if (appliedIds[i] !== getMigrationId(available[i])) {+ const knownAppliedIds = appliedIds.filter(id => availableIds.has(id));++ // A database touched by a newer version must already contain every+ // migration this app knows (append-only migrations guarantee the newer+ // version knew them all). A known migration missing next to an+ // unknown one means the database is corrupt — running it now, after+ // later migrations already ran, would be unsafe.+ if (unknownIds.length > 0 && knownAppliedIds.length !== available.length) {+ logger.error(+ 'Database is out of sync with migrations (missing known migration next to an unknown one):',+ {+ appliedIds,+ available,+ },+ );+ throw new Error('out-of-sync-migrations');+ }++ // Pre-cutoff migrations shipped strictly append-only, so the applied+ // ones must form an ordered prefix of the available list — a gap+ // there means the database is corrupt. Post-cutoff, a gap is just a+ // pending interleaved-id migration that `migrate` applies next.+ const preCutoffAvailableIds = allAvailableIds.filter(+ id => id <= ADDITIVE_ONLY_CUTOFF,+ );+ const preCutoffAppliedIds = knownAppliedIds.filter(+ id => id <= ADDITIVE_ONLY_CUTOFF,+ );++ // A post-cutoff migration only ever runs after the entire pre-cutoff+ // chain, so one applied next to a missing pre-cutoff migration means+ // the database is corrupt+ if (+ appliedIds.some(id => id > ADDITIVE_ONLY_CUTOFF) &&+ preCutoffAppliedIds.length !== preCutoffAvailableIds.length+ ) {+ logger.error(+ 'Database is out of sync with migrations (missing pre-cutoff migration next to an applied post-cutoff one):',+ {+ appliedIds,+ available,+ },+ );+ throw new Error('out-of-sync-migrations');+ }+ for (let i = 0; i < preCutoffAppliedIds.length; i++) {+ if (preCutoffAppliedIds[i] !== preCutoffAvailableIds[i]) {logger.error('Database is out of sync with migrations (migration id mismatch):',{diff --git a/packages/loot-core/src/server/migrate/schema-diff.ts b/packages/loot-core/src/server/migrate/schema-diff.tsnew file mode 100644index 0000000000..2d422a231f--- /dev/null+++ b/packages/loot-core/src/server/migrate/schema-diff.ts@@ -0,0 +1,257 @@+// Test-only: imported solely by additive-migrations.test.ts, which+// enforces the additive-only migration policy in CI. Nothing enforces+// that policy at runtime — see `checkDatabaseValidity` for what the app+// itself tolerates.+import type { Database } from '@jlongster/sql.js';++import * as sqlite from '#platform/server/sqlite';++// `required` = cannot insert a row without providing this column.+// `notNull` is tracked separately: NOT NULL with a DEFAULT still+// rejects explicit NULL writes. `defaultValue` is null when absent or+// an explicit NULL (they behave the same).+type ColumnFlags = {+ required: boolean;+ notNull: boolean;+ pk: boolean;+ type: string;+ defaultValue: string | null;+};++export type TableSnapshot = {+ columns: Map<string, ColumnFlags>;+ // Column lists of UNIQUE constraints/indexes, e.g. "a" or "a,b"+ uniques: Set<string>;+ // Normalized CHECK constraint expressions, sorted. PRAGMAs don't+ // expose CHECKs, so they are extracted from the canonical CREATE+ // TABLE text SQLite stores+ checks: string[];+};++// Collapses whitespace so formatting differences in SQL text don't+// register as changes+function normalizeSql(sql: string): string {+ return sql.replace(/\s+/g, ' ').trim();+}++// Quoted literals/identifiers (doubled quotes are escapes) and comments.+// The input is well-formed SQL from sqlite_master, so unterminated+// quotes/comments can't occur.+const NON_CODE =+ /'(?:[^']|'')*'|"(?:[^"]|"")*"|`(?:[^`]|``)*`|\[[^\]]*\]|--[^\n]*|\/\*[\s\S]*?\*\//g;++// Lowercases SQL tokens outside quoted literals/comments so formatting+// and keyword case differences don't register as CHECK changes, while+// preserving string literal contents (SQLite treats 'ARCHIVED' and+// 'archived' as different values).+function normalizeCheckExpression(expr: string): string {+ const normalized = normalizeSql(expr);+ let result = '';+ let lastIndex = 0;+ for (const match of normalized.matchAll(NON_CODE)) {+ const index = match.index!;+ result += normalized.slice(lastIndex, index).toLowerCase();+ result += match[0];+ lastIndex = index + match[0].length;+ }+ result += normalized.slice(lastIndex).toLowerCase();+ return result;+}++// Extracts each CHECK(...) expression by scanning to the balanced close+// paren, normalized (whitespace collapsed, SQL tokens lowercased outside+// quoted literals) so formatting differences don't register as changes.+// The scan runs on a masked copy with literals and comments blanked to+// spaces (length-preserving, so indexes line up) — a CHECK or paren+// inside one can't start or end a clause. Clause text is sliced from+// the original so literal contents survive intact.+function checkClauses(sql: string): string[] {+ const masked = sql.replace(NON_CODE, match => ' '.repeat(match.length));+ const clauses: string[] = [];+ const starts = /\bCHECK\s*\(/gi;+ let match;+ while ((match = starts.exec(masked))) {+ const start = match.index + match[0].length;+ let i = start;+ for (let depth = 1; i < masked.length && depth > 0; i++) {+ if (masked[i] === '(') {+ depth++;+ } else if (masked[i] === ')') {+ depth--;+ }+ }+ clauses.push(normalizeCheckExpression(sql.slice(start, i - 1)));+ }+ return clauses.sort();+}++// table -> its columns and constraints, from schema introspection+export type SchemaSnapshot = Map<string, TableSnapshot>;++export function snapshotSchema(db: Database): SchemaSnapshot {+ const tables = sqlite.runQuery<{ name: string; sql: string }>(+ db,+ "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",+ [],+ true,+ );+ return new Map(+ tables.map(({ name: table, sql }): [string, TableSnapshot] => {+ const columns = new Map(+ sqlite+ .runQuery<{+ name: string;+ type: string;+ notnull: number;+ dflt_value: string | null;+ pk: number;+ }>(db, `PRAGMA table_info("${table}")`, [], true)+ .map((column): [string, ColumnFlags] => {+ // PRAGMA reports the DEFAULT as verbatim SQL text; an+ // explicit `DEFAULT NULL` comes back as the string "NULL",+ // which is not a usable default for a NOT NULL column+ const rawDefault =+ column.dflt_value == null+ ? null+ : normalizeSql(column.dflt_value);+ const defaultValue =+ rawDefault?.toUpperCase() === 'NULL' ? null : rawDefault;+ return [+ column.name,+ {+ required:+ column.notnull !== 0 &&+ defaultValue == null &&+ column.pk === 0,+ notNull: column.notnull !== 0,+ pk: column.pk !== 0,+ type: normalizeSql(column.type).toLowerCase(),+ defaultValue,+ },+ ];+ }),+ );+ const uniques = new Set(+ sqlite+ .runQuery<{ name: string; unique: number; origin: string }>(+ db,+ `PRAGMA index_list("${table}")`,+ [],+ true,+ )+ .filter(index => index.unique !== 0 && index.origin !== 'pk')+ .map(index =>+ sqlite+ .runQuery<{ name: string }>(+ db,+ `PRAGMA index_info("${index.name}")`,+ [],+ true,+ )+ .map(column => column.name)+ .sort()+ .join(','),+ ),+ );+ return [table, { columns, uniques, checks: checkClauses(sql) }];+ }),+ );+}++// A schema change is additive-only when every column that existed+// before still exists after, unchanged, and no constraint changes in+// either direction — either direction, because tightening rejects data+// older clients hold while loosening lets newer clients write data+// older schemas reject on sync.+export function findAdditiveViolations(+ before: SchemaSnapshot,+ after: SchemaSnapshot,+ nonSyncedTables: Set<string>,+): string[] {+ const violations: string[] = [];++ for (const [table, { columns: beforeColumns }] of before) {+ const afterColumns = after.get(table)?.columns;+ for (const name of beforeColumns.keys()) {+ if (!afterColumns?.has(name)) {+ violations.push(+ `column "${table}.${name}" (or its table) was removed or renamed`,+ );+ }+ }+ }++ for (const [table, { columns: afterColumns, uniques, checks }] of after) {+ const beforeTable = before.get(table);++ // The internal-table exemption applies only at creation; once a+ // table exists, older app versions write it with the column lists+ // they were built with, so changes are validated regardless of+ // syncedness+ if (!beforeTable && nonSyncedTables.has(table)) {+ continue;+ }++ for (const [name, column] of afterColumns) {+ const prev = beforeTable?.columns.get(name);+ if (!prev) {+ if (column.required) {+ violations.push(+ `new column "${table}.${name}" is NOT NULL without a DEFAULT`,+ );+ }+ continue;+ }+ // A table rebuild can change an existing column in place; any+ // attribute change breaks one side of the version skew+ const changed = (+ ['notNull', 'pk', 'type', 'defaultValue'] as const+ ).filter(key => column[key] !== prev[key]);+ if (changed.length > 0) {+ violations.push(+ `existing column "${table}.${name}" changed: ${changed.join(', ')}`,+ );+ }+ }++ // Constraint changes only matter on existing tables — a brand-new+ // table gets the same constraints on every client that runs its+ // migration, so there is no version skew to protect against+ if (beforeTable) {+ for (const unique of uniques) {+ if (!beforeTable.uniques.has(unique)) {+ violations.push(+ `table "${table}" gained a UNIQUE constraint on (${unique})`,+ );+ }+ }+ for (const unique of beforeTable.uniques) {+ if (!uniques.has(unique)) {+ violations.push(+ `table "${table}" lost a UNIQUE constraint on (${unique})`,+ );+ }+ }++ if (beforeTable.checks.join(';') !== checks.join(';')) {+ violations.push(`table "${table}" changed its CHECK constraints`);+ }+ }++ if (!beforeTable) {+ // Sync addresses rows by their `id` column (see `apply` in+ // #server/sync), so a new synced table needs exactly that as its+ // primary key — no other name, no composite keys+ const pks = [...afterColumns]+ .filter(([, c]) => c.pk)+ .map(([name]) => name);+ if (pks.length !== 1 || pks[0] !== 'id') {+ violations.push(+ `new table "${table}" must have a single primary key named "id"`,+ );+ }+ }+ }++ return violations;+}diff --git a/packages/loot-core/src/server/sync/deferred.test.ts b/packages/loot-core/src/server/sync/deferred.test.tsnew file mode 100644index 0000000000..e447212df0--- /dev/null+++ b/packages/loot-core/src/server/sync/deferred.test.ts@@ -0,0 +1,402 @@+import { Timestamp } from '@actual-app/crdt';+import { afterEach, beforeEach, describe, expect, it } from 'vitest';++import * as db from '#server/db';++import { replayPendingMessages } from './replay';+import { deserializeValueSafe } from './serialization';++import { applyMessages, setSyncingMode } from './index';++beforeEach(() => {+ setSyncingMode('enabled');+ return global.emptyDatabase()();+});++afterEach(() => {+ setSyncingMode('disabled');+});++function sendTimestamp(): Timestamp {+ const timestamp = Timestamp.send();+ if (timestamp == null) {+ throw new Error('Timestamp.send() returned null');+ }+ return timestamp;+}++function getPending(): db.DbPendingMessage[] {+ return db.runQuery<db.DbPendingMessage>(+ 'SELECT * FROM messages_pending ORDER BY timestamp',+ [],+ true,+ );+}++describe('Deferred sync messages (newer schema)', () => {+ it('works even when the messages_pending migration never ran', async () => {+ // Code newer than the schema: the served migration files can lag+ // behind the app (stale service-worker cache, unrebuilt dev env)+ db.execQuery('DROP TABLE messages_pending');++ // Budget load must not throw...+ replayPendingMessages();++ // ...and deferral must still store the message, not fail the batch+ await applyMessages(+ [+ {+ dataset: 'transactions',+ row: 't1',+ column: 'brand_new_column',+ value: 'hello',+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ expect(getPending().length).toBe(1);+ });++ it('defers messages for unknown columns without failing the batch', async () => {+ await applyMessages(+ [+ {+ dataset: 'transactions',+ row: 't1',+ column: 'brand_new_column',+ value: 'hello',+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'transactions',+ row: 't1',+ column: 'amount',+ value: 1234,+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );++ // The known-column message must still create the row, even though+ // the unknown-column message for the same row came first+ const row = await db.first<{ id: string; amount: number }>(+ 'SELECT * FROM transactions WHERE id = ?',+ ['t1'],+ );+ expect(row?.amount).toBe(1234);++ // The unknown-column message is recorded as pending...+ const pending = getPending();+ expect(pending.length).toBe(1);+ expect(pending[0].dataset).toBe('transactions');+ expect(pending[0].column).toBe('brand_new_column');+ expect(pending[0].value).toBe('S:hello');++ // ...and both messages made it into the crdt log, so the merkle+ // stays in sync with other clients+ const crdt = db.runQuery<{ column: string }>(+ 'SELECT * FROM messages_crdt',+ [],+ true,+ );+ expect(crdt.length).toBe(2);+ });++ it('does not defer local messages: unknown schema is a bug and must fail', async () => {+ await expect(+ applyMessages([+ {+ dataset: 'transactions',+ row: 't1',+ column: 'brand_new_column',+ value: 'hello',+ timestamp: sendTimestamp(),+ },+ ]),+ ).rejects.toMatchObject({ reason: 'invalid-schema' });++ expect(getPending().length).toBe(0);+ });++ it('replays pending messages once the schema catches up, last write wins', async () => {+ await applyMessages(+ [+ {+ dataset: 'transactions',+ row: 't1',+ column: 'brand_new_column',+ value: 'old value',+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'transactions',+ row: 't1',+ column: 'amount',+ value: 1234,+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'transactions',+ row: 't1',+ column: 'brand_new_column',+ value: 'new value',+ timestamp: sendTimestamp(),+ },+ {+ // A row that only ever got a deferred message; replay has to+ // create it+ dataset: 'transactions',+ row: 't2',+ column: 'brand_new_column',+ value: 'other row',+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'gadgets',+ row: 'g1',+ column: 'name',+ value: 'flux capacitor',+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ // Repeated writes to the same cell coalesce to the newest value, so+ // t1's two brand_new_column messages leave one pending row+ const coalesced = getPending();+ expect(coalesced.length).toBe(3);+ expect(+ coalesced.find(p => p.row === 't1' && p.column === 'brand_new_column')+ ?.value,+ ).toBe('S:new value');++ // "Run the migration" that the newer client already has+ db.execQuery('ALTER TABLE transactions ADD COLUMN brand_new_column TEXT');+ replayPendingMessages();++ const t1 = await db.first<{ brand_new_column: string }>(+ 'SELECT * FROM transactions WHERE id = ?',+ ['t1'],+ );+ expect(t1?.brand_new_column).toBe('new value');++ const t2 = await db.first<{ brand_new_column: string }>(+ 'SELECT * FROM transactions WHERE id = ?',+ ['t2'],+ );+ expect(t2?.brand_new_column).toBe('other row');++ // The message for the still-unknown table stays pending...+ let pending = getPending();+ expect(pending.length).toBe(1);+ expect(pending[0].dataset).toBe('gadgets');++ // ...until a later upgrade adds it+ db.execQuery('CREATE TABLE gadgets (id TEXT PRIMARY KEY, name TEXT)');+ replayPendingMessages();++ const g1 = await db.first<{ name: string }>(+ 'SELECT * FROM gadgets WHERE id = ?',+ ['g1'],+ );+ expect(g1?.name).toBe('flux capacitor');+ pending = getPending();+ expect(pending.length).toBe(0);+ });++ it('drops a message that can never apply without blocking the rest', async () => {+ await applyMessages(+ [+ {+ dataset: 'gadgets',+ row: 'g1',+ column: 'size',+ value: -5,+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'gadgets',+ row: 'g2',+ column: 'size',+ value: 10,+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ expect(getPending().length).toBe(2);++ // The migration arrives with a CHECK constraint that the first+ // deferred value violates; replay must still apply the second one+ // and must not leave the failing message wedged in the queue+ db.execQuery(+ 'CREATE TABLE gadgets (id TEXT PRIMARY KEY, size INTEGER CHECK(size >= 0))',+ );+ replayPendingMessages();++ const g2 = await db.first<{ size: number }>(+ 'SELECT * FROM gadgets WHERE id = ?',+ ['g2'],+ );+ expect(g2?.size).toBe(10);+ expect(getPending().length).toBe(0);+ });++ it('defers an inbound message whose value uses a newer serialization format', async () => {+ await applyMessages(+ [+ {+ // Even though the column exists, the value can't be decoded —+ // the message must defer instead of wedging the sync+ dataset: 'transactions',+ row: 't1',+ column: 'notes',+ value: deserializeValueSafe('B:true'),+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );++ // Deferred with its original encoding intact, and acknowledged into+ // the crdt log with the same bytes the sender wrote+ const pending = getPending();+ expect(pending.length).toBe(1);+ expect(pending[0].value).toBe('B:true');+ const crdt = db.runQuery<{ value: string }>(+ 'SELECT * FROM messages_crdt',+ [],+ true,+ );+ expect(crdt.length).toBe(1);+ expect(String(crdt[0].value)).toBe('B:true');+ });++ it('applies dependency chains among deferred cells', async () => {+ await applyMessages(+ [+ {+ dataset: 'gadgets',+ row: 'g1',+ column: 'a',+ value: 'A',+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'gadgets',+ row: 'g1',+ column: 'b',+ value: 'B',+ timestamp: sendTimestamp(),+ },+ {+ dataset: 'gadgets',+ row: 'g1',+ column: 'c',+ value: 'C',+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ expect(getPending().length).toBe(3);++ // Row-level constraints form a chain: `a` needs `b` set, `b` needs+ // `c` set — timestamp order alone can't apply these in one pass, so+ // replay must keep retrying while passes make progress+ db.execQuery(`+ CREATE TABLE gadgets+ (id TEXT PRIMARY KEY, a TEXT, b TEXT, c TEXT,+ CHECK(a IS NULL OR b IS NOT NULL),+ CHECK(b IS NULL OR c IS NOT NULL))+ `);+ replayPendingMessages();++ const g1 = await db.first<{ a: string; b: string; c: string }>(+ 'SELECT * FROM gadgets WHERE id = ?',+ ['g1'],+ );+ expect(g1?.a).toBe('A');+ expect(g1?.b).toBe('B');+ expect(g1?.c).toBe('C');+ expect(getPending().length).toBe(0);+ });++ it('does not replay a deferred value over a newer applied write', async () => {+ await applyMessages(+ [+ {+ dataset: 'transactions',+ row: 't1',+ column: 'notes',+ value: deserializeValueSafe('B:old-format'),+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ // A later, decodable write to the same cell applies through normal+ // sync while the undecodable one sits pending+ await applyMessages(+ [+ {+ dataset: 'transactions',+ row: 't1',+ column: 'notes',+ value: 'newer value',+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ expect(getPending().length).toBe(1);++ replayPendingMessages();++ // Last write wins: the stale pending value is discarded, not applied+ const t1 = await db.first<{ notes: string }>(+ 'SELECT * FROM transactions WHERE id = ?',+ ['t1'],+ );+ expect(t1?.notes).toBe('newer value');+ expect(getPending().length).toBe(0);+ });++ it('keeps a message whose value uses a newer serialization format', async () => {+ await applyMessages(+ [+ {+ dataset: 'gadgets',+ row: 'g1',+ column: 'name',+ value: 'flux capacitor',+ timestamp: sendTimestamp(),+ },+ ],+ true,+ );+ // A value prefix this version's deserializeValue doesn't know,+ // as a newer app version could write it+ db.runQuery(+ `INSERT INTO messages_pending (timestamp, dataset, row, column, value)+ VALUES (?, 'gadgets', 'g2', 'name', 'B:true')`,+ [sendTimestamp().toString()],+ );++ db.execQuery('CREATE TABLE gadgets (id TEXT PRIMARY KEY, name TEXT)');+ replayPendingMessages();++ // The known-format message applied; the newer-format one stays+ // pending for a future upgrade instead of aborting the replay+ const g1 = await db.first<{ name: string }>(+ 'SELECT * FROM gadgets WHERE id = ?',+ ['g1'],+ );+ expect(g1?.name).toBe('flux capacitor');+ const pending = getPending();+ expect(pending.length).toBe(1);+ expect(pending[0].row).toBe('g2');+ });+});diff --git a/packages/loot-core/src/server/sync/index.ts b/packages/loot-core/src/server/sync/index.tsindex f22e0ab44f..ce928f16c2 100644--- a/packages/loot-core/src/server/sync/index.ts+++ b/packages/loot-core/src/server/sync/index.ts@@ -26,12 +26,20 @@ import * as sheet from '#server/sheet';import { resolveName } from '#server/spreadsheet/util';import * as undo from '#server/undo';import { once, sequential } from '#shared/async';+import { isMissingSchemaError } from '#shared/errors';import { getIn, setIn } from '#shared/util';import type { MetadataPrefs } from '#types/prefs';import * as encoder from './encoder';+import { PENDING_MESSAGES_TABLE_SQL } from './messages-pending';import { rebuildMerkleHash } from './repair';-import { isError } from './utils';+import {+ deserializeValueSafe,+ isUnknownFormatValue,+ serializeValue,+} from './serialization';+import type { UnknownFormatValue } from './serialization';+import { isError, notifyDeferredMessages, quoteSqlId } from './utils';export { makeTestMessage } from './make-test-message';export { resetSync } from './reset';@@ -77,34 +85,92 @@ export function checkSyncingMode(mode: SyncingMode): boolean {}}-function apply(msg: Message, prev?: boolean) {+// Record a message that can't be applied yet because it targets schema+// from a newer version. It's replayed by `replayPendingMessages` once a+// migration adds the missing table/column. Only the newest value per+// cell is kept — replay is last-write-wins per cell anyway, and this+// bounds the table while the client stays on an old version.+function deferMessage(msg: Message) {+ // The table may not exist yet when the served migration files are+ // older than the code — create it rather than failing the sync batch+ db.execQuery(PENDING_MESSAGES_TABLE_SQL);+ db.runQuery(+ db.cache(+ `INSERT INTO messages_pending (timestamp, dataset, row, column, value)+ VALUES (?, ?, ?, ?, ?)+ ON CONFLICT(dataset, row, column) DO UPDATE+ SET timestamp = excluded.timestamp, value = excluded.value+ WHERE excluded.timestamp > messages_pending.timestamp`,+ ),+ [+ msg.timestamp.toString(),+ msg.dataset,+ msg.row,+ msg.column,+ serializeValue(msg.value),+ ],+ );+}++// Returns false when the message was deferred because it references+// schema this client doesn't have yet (sent by a newer version).+// Deferral only makes sense for inbound messages; for locally-created+// messages a missing table/column is a bug and must surface as an+// apply-failure.+function apply(+ msg: Message,+ prev?: boolean,+ deferUnknownSchema?: boolean,+): boolean {const { dataset, row, column, value } = msg;if (dataset === 'prefs') {// Do nothing, it doesn't exist in the db+ } else if (dataset === 'spreadsheet_cells') {+ // Legacy dataset keyed by `name`, not `id`, so the write below+ // could never apply; it's a derived cache, so ignore the message+ } else if (isUnknownFormatValue(value)) {+ // The value was serialized by a newer version in a format this one+ // can't decode — defer it whole, like a missing table/column+ if (deferUnknownSchema) {+ deferMessage(msg);+ return false;+ }+ throw new SyncError('invalid-schema', {+ error: { message: 'Unknown value format: ' + value.raw, stack: '' },+ query: {+ sql: `INSERT INTO ${quoteSqlId(dataset)} (id, ${quoteSqlId(column)}) VALUES (?, ?)`,+ params: [row, value.raw],+ },+ });} else {let query;try {if (prev) {query = {- sql: `UPDATE ${dataset} SET ${column} = ? WHERE id = ?`,+ sql: `UPDATE ${quoteSqlId(dataset)} SET ${quoteSqlId(column)} = ? WHERE id = ?`,params: [value, row],};} else {query = {- sql: `INSERT INTO ${dataset} (id, ${column}) VALUES (?, ?)`,+ sql: `INSERT INTO ${quoteSqlId(dataset)} (id, ${quoteSqlId(column)}) VALUES (?, ?)`,params: [row, value],};}db.runQuery(db.cache(query.sql), query.params);} catch (error) {+ if (deferUnknownSchema && isMissingSchemaError(error)) {+ deferMessage(msg);+ return false;+ }throw new SyncError('invalid-schema', {error: { message: error.message, stack: error.stack },query,});}}+ return true;}// TODO: convert to `whereIn`@@ -117,7 +183,7 @@ async function fetchAll(table, ids) {for (let i = 0; i < ids.length; i += batchSize) {const partIds = ids.slice(i, i + batchSize);let sql;- let column = `${table}.id`;+ let column = `${quoteSqlId(table)}.id`;// We have to provide *mapped* data so the spreadsheet works. The functions// which trigger budget changes based on data changes assumes data has been@@ -131,7 +197,7 @@ async function fetchAll(table, ids) {`;column = 't.id';} else {- sql = `SELECT * FROM ${table}`;+ sql = `SELECT * FROM ${quoteSqlId(table)}`;}sql += ` WHERE `;@@ -141,6 +207,11 @@ async function fetchAll(table, ids) {const rows = db.runQuery(sql, partIds, true);results = results.concat(rows);} catch (error) {+ if (isMissingSchemaError(error)) {+ // The table comes from a newer version of the app; its messages+ // will be deferred by `apply`+ break;+ }throw new SyncError('invalid-schema', {error: {message: error.message,@@ -154,33 +225,6 @@ async function fetchAll(table, ids) {return results;}-export function serializeValue(value: string | number | null): string {- if (value === null) {- return '0:';- } else if (typeof value === 'number') {- return 'N:' + value;- } else if (typeof value === 'string') {- return 'S:' + value;- }-- throw new Error('Unserializable value type: ' + JSON.stringify(value));-}--export function deserializeValue(value: string): string | number | null {- const type = value[0];- switch (type) {- case '0':- return null;- case 'N':- return parseFloat(value.slice(2));- case 'S':- return value.slice(2);- default:- }-- throw new Error('Invalid type key for value: ' + value);-}-// TODO make this type stricter.type DataMap = Map<string, unknown>;type SyncListener = (oldData: DataMap, newData: DataMap) => unknown;@@ -255,10 +299,10 @@ export type Message = {old?: unknown;row: string;timestamp: Timestamp;- value: string | number | null;+ value: string | number | null | UnknownFormatValue;};-export const applyMessages = sequential(async (messages: Message[]) => {+async function _applyMessages(messages: Message[], deferUnknownSchema = false) {if (checkSyncingMode('import')) {applyMessagesForImport(messages);return undefined;@@ -335,6 +379,7 @@ export const applyMessages = sequential(async (messages: Message[]) => {// nothing is changed. This is critical to maintain consistency. We// also avoid any side effects to in-memory objects, and apply them// after this succeeds.+ const deferredMessages = new Set<Message>();db.transaction(() => {const added = new Set();@@ -342,32 +387,49 @@ export const applyMessages = sequential(async (messages: Message[]) => {const { dataset, row, column, timestamp, value } = msg;if (!msg.old) {- apply(msg, getIn(oldData, [dataset, row]) || added.has(dataset + row));+ const applied = apply(+ msg,+ getIn(oldData, [dataset, row]) || added.has(dataset + row),+ deferUnknownSchema,+ );- if (dataset === 'prefs') {- prefsToSet[row] = value;+ if (applied) {+ if (dataset === 'prefs') {+ // An unknown-format pref value can't be stored or replayed+ // (prefs aren't a table) — leave the local pref as-is+ if (!isUnknownFormatValue(value)) {+ prefsToSet[row] = value;+ }+ } else {+ // Keep track of which items have been added it in this sync+ // so it knows whether they already exist in the db or not. We+ // ignore any changes to the spreadsheet.+ added.add(dataset + row);++ // Special treatment for some synced prefs. Applied messages+ // only — an old or deferred message must not flip the+ // in-memory budget type+ if (dataset === 'preferences' && row === 'budgetType') {+ void setBudgetType(value);+ }+ }} else {- // Keep track of which items have been added it in this sync- // so it knows whether they already exist in the db or not. We- // ignore any changes to the spreadsheet.- added.add(dataset + row);+ // Deferred messages must not be tracked in `added`: their+ // row wasn't created, so a later message for a known column+ // still needs to INSERT it+ deferredMessages.add(msg);}}if (checkSyncingMode('enabled')) {db.runQuery(db.cache(`INSERT INTO messages_crdt (timestamp, dataset, row, column, value)- VALUES (?, ?, ?, ?, ?)`),+ VALUES (?, ?, ?, ?, ?)`),[timestamp.toString(), dataset, row, column, serializeValue(value)],);currentMerkle = merkle.insert(currentMerkle, timestamp);}-- // Special treatment for some synced prefs- if (dataset === 'preferences' && row === 'budgetType') {- void setBudgetType(value);- }}if (checkSyncingMode('enabled')) {@@ -433,7 +495,11 @@ export const applyMessages = sequential(async (messages: Message[]) => {_syncListeners.forEach(func => func(oldData, newData));- const tables = getTablesFromMessages(messages.filter(msg => !msg.old));+ // Only tables that actually changed — deferred messages wrote+ // nothing, so they must not trigger client cache invalidation+ const tables = getTablesFromMessages(+ messages.filter(msg => !msg.old && !deferredMessages.has(msg)),+ );app.events.emit('sync', {type: 'applied',tables,@@ -441,8 +507,20 @@ export const applyMessages = sequential(async (messages: Message[]) => {prevData: oldData,});- return messages;-});+ if (deferredMessages.size > 0) {+ notifyDeferredMessages();+ }++ // Deferred messages wrote nothing, so they don't count as received+ // — this also keeps their tables out of the `success` event in+ // `fullSync`. Old messages stay: they were processed (merkled),+ // just superseded.+ return deferredMessages.size === 0+ ? messages+ : messages.filter(msg => !deferredMessages.has(msg));+}++export const applyMessages = sequential(_applyMessages);export function receiveMessages(messages: Message[]): Promise<Message[]> {try {@@ -456,7 +534,9 @@ export function receiveMessages(messages: Message[]): Promise<Message[]> {throw e;}- return runMutator(() => applyMessages(messages));+ // Inbound messages may come from a newer version of the app, so+ // unknown-schema errors defer instead of failing the batch+ return runMutator(() => applyMessages(messages, true));}async function errorHandler(e: Error) {@@ -744,7 +824,7 @@ async function _fullSync(receivedMessages = await receiveMessages(res.messages.map(msg => ({...msg,- value: deserializeValue(msg.value as string),+ value: deserializeValueSafe(msg.value as string),})),);}diff --git a/packages/loot-core/src/server/sync/messages-pending.ts b/packages/loot-core/src/server/sync/messages-pending.tsnew file mode 100644index 0000000000..5bd56d1b33--- /dev/null+++ b/packages/loot-core/src/server/sync/messages-pending.ts@@ -0,0 +1,18 @@+// Sync messages deferred because they reference schema from a newer+// app version; see `deferMessage` and `replayPendingMessages`.+//+// The DDL is shared between the migration and the runtime: the code+// that uses the table also creates it on demand, so it keeps working+// even when the served migration files are older than the code (e.g. a+// stale service-worker cache or an unrebuilt dev environment).+// Dependency-free on purpose — the migration imports this too.+export const PENDING_MESSAGES_TABLE_SQL = `+ CREATE TABLE IF NOT EXISTS messages_pending+ (dataset TEXT NOT NULL,+ row TEXT NOT NULL,+ column TEXT NOT NULL,+ timestamp TEXT NOT NULL,+ -- Serialized string form, e.g. "S:hello" (see \`serializeValue\`)+ value TEXT NOT NULL,+ PRIMARY KEY (dataset, row, column));+`;diff --git a/packages/loot-core/src/server/sync/replay.ts b/packages/loot-core/src/server/sync/replay.tsnew file mode 100644index 0000000000..efd44be54f--- /dev/null+++ b/packages/loot-core/src/server/sync/replay.ts@@ -0,0 +1,194 @@+import { captureException } from '#platform/exceptions';+import { logger } from '#platform/server/log';+import * as db from '#server/db';+import { isMissingSchemaError } from '#shared/errors';++import { PENDING_MESSAGES_TABLE_SQL } from './messages-pending';+import { deserializeValue } from './serialization';+import {+ notifyDeferredMessages,+ notifyDroppedMessages,+ quoteSqlId,+ resetDeferredMessagesNotification,+} from './utils';++// SQLite failures that repeat identically on every attempt. Anything+// else non-schema (locked/full/interrupted/storage errors) may be+// transient and aborts the whole replay instead.+const DETERMINISTIC_SQLITE_ERROR =+ /constraint failed|datatype mismatch|cannot store|generated column|string or blob too big|syntax error/i;++type ReplayOutcome =+ | { outcome: 'applied' }+ | { outcome: 'newer-version' }+ | { outcome: 'failed'; error: Error };++// Deletes pending rows superseded by a newer acknowledged write in the+// crdt log (last write wins per cell — possible because an+// unknown-format value defers even when its column exists). Also used+// by `resetSync` so its discard warning only counts real losses.+export function deleteStalePendingMessages(): void {+ // The table may not exist yet when the served migration files are+ // older than the code (stale service-worker cache, unrebuilt dev+ // environment) — create it rather than failing every budget load+ db.execQuery(PENDING_MESSAGES_TABLE_SQL);+ db.runQuery(`+ DELETE FROM messages_pending+ WHERE EXISTS+ (SELECT 1 FROM messages_crdt c+ WHERE c.dataset = messages_pending.dataset+ AND c.row = messages_pending.row+ AND c.column = messages_pending.column+ AND c.timestamp > messages_pending.timestamp)+ `);+}++// Applies sync messages that were deferred because they referenced+// schema (or a value format) this client didn't have yet — see+// `deferMessage` in ./index.ts. Called on budget load right after+// migrations run, only when a sync server is configured. Messages that+// still can't apply stay pending for a future upgrade and re-trigger+// the update-required notification.+export function replayPendingMessages(): void {+ resetDeferredMessagesNotification();++ let appliedCount = 0;+ let newerVersionCount = 0;+ let failedCount = 0;++ const replay = () => {+ deleteStalePendingMessages();++ // Timestamp order gives last-write-wins per cell+ const pending = db.runQuery<db.DbPendingMessage>(+ 'SELECT * FROM messages_pending ORDER BY timestamp',+ [],+ true,+ );+ if (pending.length === 0) {+ return;+ }++ function applyOne(msg: db.DbPendingMessage): ReplayOutcome {+ let value;+ try {+ value = deserializeValue(msg.value);+ } catch {+ // A value format from an even newer version; decodes after the+ // user updates+ return { outcome: 'newer-version' };+ }+ try {+ // The same per-cell write as `apply` in ./index.ts. Not an+ // upsert: SQLite checks row-level CHECK constraints against the+ // INSERT candidate row (sibling columns NULL) before ON+ // CONFLICT fires, so an existing row must take the UPDATE path.+ const updated = Number(+ db.runQuery(+ db.cache(+ `UPDATE ${quoteSqlId(msg.dataset)} SET ${quoteSqlId(msg.column)} = ? WHERE id = ?`,+ ),+ [value, msg.row],+ ).changes,+ );+ if (updated === 0) {+ db.runQuery(+ db.cache(+ `INSERT INTO ${quoteSqlId(msg.dataset)} (id, ${quoteSqlId(msg.column)}) VALUES (?, ?)`,+ ),+ [msg.row, value],+ );+ }+ return { outcome: 'applied' };+ } catch (e) {+ if (isMissingSchemaError(e)) {+ // Still targets schema from an even newer version+ return { outcome: 'newer-version' };+ }+ const error = e instanceof Error ? e : new Error(String(e));+ if (!DETERMINISTIC_SQLITE_ERROR.test(error.message)) {+ // Possibly transient — and errors like disk-full force SQLite+ // to roll back the enclosing transaction, so continuing would+ // run in autocommit. Abort; everything stays pending.+ throw error;+ }+ return { outcome: 'failed', error };+ }+ }++ function deletePending(msg: db.DbPendingMessage) {+ db.runQuery(+ db.cache(+ 'DELETE FROM messages_pending WHERE dataset = ? AND row = ? AND column = ?',+ ),+ [msg.dataset, msg.row, msg.column],+ );+ }++ function processPass(+ msgs: db.DbPendingMessage[],+ ): Array<{ msg: db.DbPendingMessage; error: Error }> {+ const failed: Array<{ msg: db.DbPendingMessage; error: Error }> = [];+ for (const msg of msgs) {+ const result = applyOne(msg);+ if (result.outcome === 'applied') {+ appliedCount++;+ deletePending(msg);+ } else if (result.outcome === 'newer-version') {+ newerVersionCount++;+ } else {+ failed.push({ msg, error: result.error });+ }+ }+ return failed;+ }++ // Retry while passes make progress: a row-level constraint can need+ // later cells to fill the row first, including through dependency+ // chains. Terminates because each retained pass strictly shrinks+ // the failure list.+ let failed = processPass(pending);+ while (failed.length > 0) {+ const retried = processPass(failed.map(failure => failure.msg));+ if (retried.length === failed.length) {+ // No progress: these repeat identically on every load, so drop+ // them. The values stay in messages_crdt history, but this+ // device now diverges for those cells.+ for (const { msg, error } of retried) {+ captureException(error);+ failedCount++;+ deletePending(msg);+ }+ break;+ }+ failed = retried;+ }++ if (appliedCount > 0) {+ // The values were written outside the normal sync pipeline, so+ // force the spreadsheet cache to recompute+ db.runQuery('DELETE FROM kvcache_key');+ }+ };++ try {+ db.transaction(replay);+ } catch (e) {+ // Possibly-transient abort: nothing was dropped, everything retries+ // on the next load — telemetry only, no user notification+ captureException(e instanceof Error ? e : new Error(String(e)));+ return;+ }++ if (appliedCount > 0) {+ logger.info(+ `Applied ${appliedCount} sync message(s) deferred from a newer version`,+ );+ }+ if (failedCount > 0) {+ notifyDroppedMessages();+ }+ if (newerVersionCount > 0) {+ notifyDeferredMessages();+ }+}diff --git a/packages/loot-core/src/server/sync/reset.ts b/packages/loot-core/src/server/sync/reset.tsindex f1b183c011..e185e37c2b 100644--- a/packages/loot-core/src/server/sync/reset.ts+++ b/packages/loot-core/src/server/sync/reset.ts@@ -7,6 +7,9 @@ import * as db from '#server/db';import { runMutator } from '#server/mutators';import * as prefs from '#server/prefs';+import { deleteStalePendingMessages } from './replay';+import { notifyDroppedMessages } from './utils';+export async function resetSync(keyState?,): Promise<{ error?: { reason: string; meta?: unknown } }> {@@ -27,7 +30,20 @@ export async function resetSync(return { error };}+ // Resetting discards deferred newer-version messages for every+ // device; the user is warned afterwards via `notifyDroppedMessages`.+ // TODO: pre-reset confirmation dialog+ let discardedDeferredCount = 0;+await runMutator(async () => {+ // Deferred messages belong to the discarded message log; replaying+ // them later would resurrect rows hard-deleted below. Stale rows go+ // first, uncounted, so only real losses feed the warning.+ deleteStalePendingMessages();+ discardedDeferredCount = Number(+ db.runQuery('DELETE FROM messages_pending').changes,+ );+// TOOD: We could automatically generate the list of tables to// cleanup by looking at the schema//@@ -49,6 +65,10 @@ export async function resetSync(await db.loadClock();});+ if (discardedDeferredCount > 0) {+ notifyDroppedMessages();+ }+await prefs.savePrefs({groupId: null,lastSyncedTimestamp: null,diff --git a/packages/loot-core/src/server/sync/serialization.test.ts b/packages/loot-core/src/server/sync/serialization.test.tsnew file mode 100644index 0000000000..433289931b--- /dev/null+++ b/packages/loot-core/src/server/sync/serialization.test.ts@@ -0,0 +1,27 @@+import { describe, expect, it } from 'vitest';++import {+ deserializeValue,+ deserializeValueSafe,+ isUnknownFormatValue,+ serializeValue,+} from './serialization';++describe('sync value serialization', () => {+ it('round-trips known value types', () => {+ for (const value of [null, 0, -12.5, '', 'hello', 'N:tricky']) {+ expect(deserializeValue(serializeValue(value))).toBe(value);+ }+ });++ it('treats unknown and extended prefixes as unknown format', () => {+ // 'N16:' and 'Sx:' guard against first-character prefix collisions+ // with future tags — they must defer, not misdecode+ for (const raw of ['B:true', 'N16:123', 'Sx:hi', 'x', '']) {+ const value = deserializeValueSafe(raw);+ expect(isUnknownFormatValue(value)).toBe(true);+ // The original encoding survives for a future version to decode+ expect(serializeValue(value)).toBe(raw);+ }+ });+});diff --git a/packages/loot-core/src/server/sync/serialization.ts b/packages/loot-core/src/server/sync/serialization.tsnew file mode 100644index 0000000000..2f69901034--- /dev/null+++ b/packages/loot-core/src/server/sync/serialization.ts@@ -0,0 +1,68 @@+// Sync message values travel as tagged strings — "0:" for null, "N:"+// for numbers, "S:" for strings; newer app versions may introduce more+// prefixes.++// A value serialized by a newer version of the app with a type prefix+// this version doesn't know. Carried opaquely through the sync pipeline+// so the message can still be acknowledged into the crdt log and+// deferred with its original encoding intact (see `apply` in ./index.ts).+export type UnknownFormatValue = { kind: 'unknown-format'; raw: string };++export function isUnknownFormatValue(+ value: unknown,+): value is UnknownFormatValue {+ return (+ typeof value === 'object' &&+ value !== null &&+ 'kind' in value &&+ value.kind === 'unknown-format'+ );+}++export function serializeValue(+ value: string | number | null | UnknownFormatValue,+): string {+ if (isUnknownFormatValue(value)) {+ return value.raw;+ } else if (value === null) {+ return '0:';+ } else if (typeof value === 'number') {+ return 'N:' + value;+ } else if (typeof value === 'string') {+ return 'S:' + value;+ }++ throw new Error('Unserializable value type: ' + JSON.stringify(value));+}++export function deserializeValue(value: string): string | number | null {+ // The full prefix must match, not just the first character — a future+ // version's extended tag like "N16:..." must throw (and so defer via+ // `deserializeValueSafe`), not silently misdecode+ if (value[1] === ':') {+ switch (value[0]) {+ case '0':+ return null;+ case 'N':+ return parseFloat(value.slice(2));+ case 'S':+ return value.slice(2);+ default:+ }+ }++ throw new Error('Invalid type key for value: ' + value);+}++// For inbound messages, which may come from a newer version of the app:+// an unknown value format must not abort the sync — `apply` defers the+// message instead, exactly like a missing table/column+export function deserializeValueSafe(+ value: string,+): string | number | null | UnknownFormatValue {+ try {+ return deserializeValue(value);+ } catch {+ return { kind: 'unknown-format', raw: value };+ }+}diff --git a/packages/loot-core/src/server/sync/sync.property.test.ts b/packages/loot-core/src/server/sync/sync.property.test.tsindex d9a68460ec..d6e5beed2d 100644--- a/packages/loot-core/src/server/sync/sync.property.test.ts+++ b/packages/loot-core/src/server/sync/sync.property.test.ts@@ -9,6 +9,7 @@ import * as sheet from '#server/sheet';import * as mockSyncServer from '#server/tests/mockSyncServer';import * as encoder from './encoder';+import { serializeValue } from './serialization';import { isError } from './utils';import * as sync from './index';@@ -257,7 +258,7 @@ async function run(msgs) {Timestamp.zero,res.secondMessages.map(x => ({...x,- value: sync.serializeValue(x.value),+ value: serializeValue(x.value),timestamp: x.timestamp.toString(),})),),@@ -276,7 +277,7 @@ async function run(msgs) {Timestamp.zero,res.secondMessages.map(x => ({...x,- value: sync.serializeValue(x.value),+ value: serializeValue(x.value),timestamp: x.timestamp.toString(),})),),diff --git a/packages/loot-core/src/server/sync/utils.ts b/packages/loot-core/src/server/sync/utils.tsindex 802dcc1896..fd3c3d0171 100644--- a/packages/loot-core/src/server/sync/utils.ts+++ b/packages/loot-core/src/server/sync/utils.ts@@ -1,3 +1,34 @@+import { app } from '#server/main-app';+export function isError(value: unknown): value is { error: unknown } {return (value as { error: unknown }).error !== undefined;}++// Sync messages carry table/column names as data, and a newer app+// version may name a column with an SQL keyword — unquoted, that raises+// a syntax error instead of the missing-column error the deferral path+// relies on.+export function quoteSqlId(name: string): string {+ return '"' + name.replace(/"/g, '""') + '"';+}++// Notify once per loaded budget (reset on budget load); the client-side+// notification id additionally dedupes while it's displayed+let hasNotifiedDeferredMessages = false;++export function resetDeferredMessagesNotification() {+ hasNotifiedDeferredMessages = false;+}++export function notifyDeferredMessages() {+ if (!hasNotifiedDeferredMessages) {+ hasNotifiedDeferredMessages = true;+ app.events.emit('sync', { type: 'deferred-messages' });+ }+}++// A deferred change was dropped or discarded. Emitted per incident; the+// client-side notification id dedupes while one is displayed.+export function notifyDroppedMessages() {+ app.events.emit('sync', { type: 'dropped-messages' });+}diff --git a/packages/loot-core/src/shared/errors.ts b/packages/loot-core/src/shared/errors.tsindex 237654831e..7e9d45f0eb 100644--- a/packages/loot-core/src/shared/errors.ts+++ b/packages/loot-core/src/shared/errors.ts@@ -54,7 +54,18 @@ function getUnsafeZipError(meta: UnsafeZipMeta): string {// plain English strings. User-facing, translated equivalents live in the// desktop-client (`src/util/error.ts`).-function isDatabaseSchemaMismatch(meta?: unknown): boolean {+// A SQLite error meaning a query referenced a table or column that+// doesn't exist in this database's schema. SQLite reports a missing+// column as "no such column" in UPDATE/SELECT statements but as "has no+// column named" in INSERT statements.+export function isMissingSchemaError(error: unknown): boolean {+ return (+ error instanceof Error &&+ /no such (table|column)|has no column named/i.test(error.message)+ );+}++export function isDatabaseSchemaMismatch(meta?: unknown): boolean {if (meta &&typeof meta === 'object' &&@@ -64,7 +75,10 @@ function isDatabaseSchemaMismatch(meta?: unknown): boolean {'message' in meta.error &&typeof meta.error.message === 'string') {- return /no such (column|table)/i.test(meta.error.message);+ // Narrower than `isMissingSchemaError`: sync defers the INSERT form+ // ("has no column named"), so one reaching the UI means local+ // damage, not version skew+ return /no such (table|column)/i.test(meta.error.message);}return false;}diff --git a/packages/loot-core/src/types/server-events.ts b/packages/loot-core/src/types/server-events.tsindex 4956731c3e..b80534606c 100644--- a/packages/loot-core/src/types/server-events.ts+++ b/packages/loot-core/src/types/server-events.ts@@ -41,6 +41,17 @@ type SyncEvent = {| {type: 'unauthorized';}+ | {+ // Some received messages were written by a newer version of the+ // app and can't be shown until this client updates+ type: 'deferred-messages';+ }+ | {+ // A deferred change from another device was discarded — it could+ // never be applied, or a sync reset made this file the source of+ // truth without it+ type: 'dropped-messages';+ });type BackupUpdatedEvent = Backup[];diff --git a/upcoming-release-notes/cross-version-sync-compatibility.md b/upcoming-release-notes/cross-version-sync-compatibility.mdnew file mode 100644index 0000000000..a1933f59d8--- /dev/null+++ b/upcoming-release-notes/cross-version-sync-compatibility.md@@ -0,0 +1,6 @@+---+category: Enhancements+authors: [MatissJanis]+---++Keep syncing working when other devices run a newer version of Actual — their changes appear once you update the app