Skip to content

A local-first personal finance app

[AI] migrations: pending messages DB (#8519)

Matiss Janis Aboltins2fc69915c21f
diff --git a/packages/desktop-client/src/sync-events.ts b/packages/desktop-client/src/sync-events.ts
index 595687e3b9..aafe5ed557 100644
--- a/packages/desktop-client/src/sync-events.ts
+++ b/packages/desktop-client/src/sync-events.ts
@@ -1,4 +1,5 @@
import { listen, send } from '@actual-app/core/platform/client/connection';
+import type { ServerEvents } from '@actual-app/core/types/server-events';
import type { QueryClient } from '@tanstack/react-query';
import { t } from 'i18next';
@@ -18,6 +19,34 @@ import { loadPrefs } from './prefs/prefsSlice';
import type { AppStore } from './redux/store';
import { signOut } from './users/usersSlice';
+// Notifications for sync events that can fire during budget load
+const syncNotifications: Partial<
+ Record<ServerEvents['sync-event']['type'], () => Notification>
+> = {
+ 'deferred-messages': () => ({
+ id: 'deferred-messages',
+ type: 'message',
+ // Sticky: firing during budget load, it would otherwise
+ // auto-dismiss before the user can read it
+ sticky: true,
+ title: t('Update available'),
+ message: t(
+ 'Some synced changes were made with a newer version of Actual. ' +
+ 'They will show up here once you update this app.',
+ ),
+ }),
+ 'dropped-messages': () => ({
+ id: 'dropped-messages',
+ type: 'warning',
+ sticky: true,
+ title: t('Some changes could not be applied'),
+ message: t(
+ 'Some changes made on another device could not be applied ' +
+ 'on this device, so your devices may show different values.',
+ ),
+ }),
+};
+
export function listenForSyncEvent(store: AppStore, queryClient: QueryClient) {
// TODO: Should this run on mobile too?
const unlistenUnauthorized = listen('sync-event', async ({ type }) => {
@@ -39,7 +68,20 @@ export function listenForSyncEvent(store: AppStore, queryClient: QueryClient) {
const unlistenSuccess = listen('sync-event', event => {
const prefs = store.getState().prefs.local;
- if (!prefs || !prefs.id) {
+ const hasBudget = Boolean(prefs && prefs.id);
+
+ const makeNotification = syncNotifications[event.type];
+ if (makeNotification) {
+ // Accept these while a budget is loaded OR still loading (prefs
+ // arrive only after the load completes); with neither, the user
+ // has left the budget and the notification would lack context
+ if (hasBudget || store.getState().app.loadingText != null) {
+ store.dispatch(addNotification({ notification: makeNotification() }));
+ }
+ return;
+ }
+
+ if (!hasBudget) {
// Do nothing if no budget is loaded
return;
}
diff --git a/packages/desktop-client/src/util/error.ts b/packages/desktop-client/src/util/error.ts
index ed1bafa0dd..e09ef1e5b1 100644
--- a/packages/desktop-client/src/util/error.ts
+++ b/packages/desktop-client/src/util/error.ts
@@ -1,4 +1,8 @@
-import { getUnsafeZipMeta, toMB } from '@actual-app/core/shared/errors';
+import {
+ getUnsafeZipMeta,
+ isDatabaseSchemaMismatch,
+ toMB,
+} from '@actual-app/core/shared/errors';
import type { UnsafeZipMeta } from '@actual-app/core/shared/errors';
import { t } from 'i18next';
@@ -85,21 +89,6 @@ export function getUploadError({ reason, meta }: ErrorWithMeta) {
}
}
-function isDatabaseSchemaMismatch(meta?: unknown): boolean {
- if (
- meta &&
- typeof meta === 'object' &&
- 'error' in meta &&
- meta.error &&
- typeof meta.error === 'object' &&
- 'message' in meta.error &&
- typeof meta.error.message === 'string'
- ) {
- return /no such (column|table)/i.test(meta.error.message);
- }
- return false;
-}
-
function getSchemaMismatchError() {
return t(
'This budget could not be loaded because it uses a newer database schema than this version of Actual supports. Make sure you are using the latest version, then try again.',
diff --git a/packages/loot-core/migrations/1788468782000_add_messages_pending.js b/packages/loot-core/migrations/1788468782000_add_messages_pending.js
new file mode 100644
index 0000000000..d1144c897d
--- /dev/null
+++ b/packages/loot-core/migrations/1788468782000_add_messages_pending.js
@@ -0,0 +1,5 @@
+import { PENDING_MESSAGES_TABLE_SQL } from '#server/sync/messages-pending';
+
+export default async function runMigration(db) {
+ db.execQuery(PENDING_MESSAGES_TABLE_SQL);
+}
diff --git a/packages/loot-core/migrations/README.md b/packages/loot-core/migrations/README.md
new file mode 100644
index 0000000000..007e784977
--- /dev/null
+++ b/packages/loot-core/migrations/README.md
@@ -0,0 +1,45 @@
+# Database migrations
+
+Actual upgrades local budget databases by running the SQL and JavaScript files in this directory in timestamp order. Each file name must start with a numeric id (typically the authoring timestamp in milliseconds), followed by a short description.
+
+## Cross-version sync
+
+Actual supports running different app versions against the same synced budget. A device on an older release may still receive sync messages produced by a newer release, and it must keep loading and syncing until it upgrades.
+
+That only works if newer migrations are **additive-only**: they may extend the schema, but must not remove or rename anything an older client still reads or writes through sync.
+
+Post-cutoff migrations (ids after `1780606215001`, see `ADDITIVE_ONLY_CUTOFF` in `src/server/migrate/migrations.ts`) are enforced by `src/server/migrate/additive-migrations.test.ts`. CI runs that test against the real migration chain.
+
+## Allowed changes
+
+After the cutoff, a migration may:
+
+- Create a new table. Synced tables must use a single primary key column named `id`.
+- Add a nullable column, or a `NOT NULL` column with a `DEFAULT`.
+- Add a non-unique index.
+- Drop or recreate a **view** (views are rebuilt on startup and are not synced).
+- Create an internal (non-synced) table. Register it in `NON_SYNCED_TABLES` in `additive-migrations.test.ts`. Internal tables are only exempt from the synced-table rules at creation time; later changes are validated like any other table.
+
+Use `CREATE TABLE IF NOT EXISTS` when a migration might run against a database that already has the table (for example after a migration id is renumbered).
+
+## Disallowed changes
+
+Do **not** drop or rename tables or columns. Do not rebuild a table to change an existing column's type, nullability, default, primary-key membership, `UNIQUE` constraints, or `CHECK` expressions. Do not add `UNIQUE` constraints or required columns without defaults to existing synced tables.
+
+If a column is no longer used, stop reading it in application code and leave it in the database.
+
+Renaming is not supported even via `ALTER TABLE ... RENAME`. Older clients address synced data by table and column name; a rename looks like a delete plus an add and breaks cross-version sync.
+
+## Migration ids
+
+- Use a current timestamp for new migrations. Do not reuse or backdate ids into the pre-cutoff era.
+- A long-lived branch may merge after other migrations landed on `master`. If your migration id sorts below ids already on `master`, bump it to a new timestamp before merge so existing databases do not treat it as already applied.
+- If you must change a migration that has already shipped, prefer a new migration file rather than editing the old one. If you renumber an id, make the SQL idempotent (`IF NOT EXISTS`, and so on).
+
+## What the test does not cover
+
+`additive-migrations.test.ts` compares the database **schema** before and after each post-cutoff migration. It does not inspect `UPDATE`/`INSERT` statements that rewrite existing row data.
+
+A data migration that changes the meaning or format of values already stored in a column can pass the test but still confuse an older app version reading those rows or applying sync messages. Prefer leaving existing values as-is, or gate behavioral changes on the app version reading them.
+
+For sync **values** (not just columns) from newer clients, see deferred message handling in `src/server/sync/` and `messages_pending`.
diff --git a/packages/loot-core/src/server/budgetfiles/app.ts b/packages/loot-core/src/server/budgetfiles/app.ts
index f5e50384fa..0acd745ecc 100644
--- a/packages/loot-core/src/server/budgetfiles/app.ts
+++ b/packages/loot-core/src/server/budgetfiles/app.ts
@@ -26,6 +26,7 @@ import {
setSyncingMode,
} from '#server/sync';
import * as syncMigrations from '#server/sync/migrate';
+import { replayPendingMessages } from '#server/sync/replay';
import * as rules from '#server/transactions/transaction-rules';
import { clearUndo } from '#server/undo';
import { updateVersion } from '#server/update';
@@ -597,6 +598,20 @@ async function _loadBudget(id: Budget['id']): Promise<{
return result;
}
+ try {
+ // Apply sync messages deferred from a newer app version, now that
+ // migrations have run. Skipped without a sync server: serverless
+ // edits aren't in the crdt log, so replay couldn't tell whether a
+ // pending value is stale.
+ if (getServer()) {
+ replayPendingMessages();
+ }
+ } catch (e) {
+ // Failing to replay shouldn't block loading the budget; the
+ // messages stay pending
+ captureException(e);
+ }
+
await db.loadClock();
if (prefs.getPrefs().resetClock) {
diff --git a/packages/loot-core/src/server/db/types/index.ts b/packages/loot-core/src/server/db/types/index.ts
index 4991cb285c..3a68be278f 100644
--- a/packages/loot-core/src/server/db/types/index.ts
+++ b/packages/loot-core/src/server/db/types/index.ts
@@ -109,6 +109,14 @@ export type DbCrdtMessage = {
value: Uint8Array;
};
+export type DbPendingMessage = {
+ timestamp: string;
+ dataset: string;
+ row: string;
+ column: string;
+ value: string;
+};
+
export type DbNote = {
id: string;
note: string;
diff --git a/packages/loot-core/src/server/migrate/additive-migrations.test.ts b/packages/loot-core/src/server/migrate/additive-migrations.test.ts
new file mode 100644
index 0000000000..7aa56c4ce3
--- /dev/null
+++ b/packages/loot-core/src/server/migrate/additive-migrations.test.ts
@@ -0,0 +1,362 @@
+import * as nativeFs from 'fs';
+import * as path from 'path';
+
+import type { Database } from '@jlongster/sql.js';
+import { describe, expect, it } from 'vitest';
+
+import * as sqlite from '#platform/server/sqlite';
+
+import {
+ ADDITIVE_ONLY_CUTOFF,
+ applyMigration,
+ getMigrationId,
+ getMigrationList,
+ migrate,
+ withMigrationsDir,
+} from './migrations';
+import { findAdditiveViolations, snapshotSchema } from './schema-diff';
+import type { SchemaSnapshot } from './schema-diff';
+
+// Migrations newer than ADDITIVE_ONLY_CUTOFF must be additive-only:
+// clients tolerate budgets and sync messages from newer app versions
+// (see `replayPendingMessages` and `checkDatabaseValidity`), which is
+// only safe if newer migrations never remove or rename what older
+// clients depend on. Enforced by running the real migration chain and
+// diffing the actual schema around each new migration.
+
+const MIGRATIONS_DIR = path.resolve(__dirname, '../../../migrations');
+const INIT_SQL = path.resolve(__dirname, '../sql/init.sql');
+
+// Internal (non-CRDT-synced) tables, exempt from the synced-table rules
+// at creation only (see `findAdditiveViolations`). Add new internal
+// tables here.
+const NON_SYNCED_TABLES = new Set(['messages_pending']);
+
+async function openTestDb(setupSql: string): Promise<Database> {
+ await sqlite.init();
+ const db = await sqlite.openDatabase(':memory:');
+ sqlite.execQuery(db, setupSql);
+ return db;
+}
+
+// The additive-only violations that `migrationSql` introduces on a
+// database prepared with `setupSql`
+async function violationsFor(
+ setupSql: string,
+ migrationSql: string,
+): Promise<string[]> {
+ const db = await openTestDb(setupSql);
+ const before = snapshotSchema(db);
+ sqlite.execQuery(db, migrationSql);
+ const violations = findAdditiveViolations(
+ before,
+ snapshotSchema(db),
+ NON_SYNCED_TABLES,
+ );
+ sqlite.closeDatabase(db);
+ return violations;
+}
+
+describe('migrations are additive-only', () => {
+ it('new migrations carry post-cutoff ids', async () => {
+ // `checkDatabaseValidity` treats an unknown pre-cutoff id as
+ // corruption, so merging a migration with a pre-cutoff id (e.g. a
+ // long-lived PR keeping its original authoring timestamp) would
+ // break every upgrade. The pre-cutoff set shipped long ago and is
+ // frozen — if this count changed, renumber the new migration's id
+ // to the present day.
+ const ids = (await getMigrationList(MIGRATIONS_DIR)).map(getMigrationId);
+ expect(ids.filter(id => id <= ADDITIVE_ONLY_CUTOFF).length).toBe(57);
+ });
+
+ it('every migration after the cutoff is additive-only', async () => {
+ const db = await openTestDb(nativeFs.readFileSync(INIT_SQL, 'utf8'));
+
+ const violations: string[] = [];
+ let snapshot: SchemaSnapshot | null = null;
+ for (const name of await getMigrationList(MIGRATIONS_DIR)) {
+ if (getMigrationId(name) <= ADDITIVE_ONLY_CUTOFF) {
+ await applyMigration(db, name, MIGRATIONS_DIR);
+ snapshot = null;
+ continue;
+ }
+ const before = snapshot ?? snapshotSchema(db);
+ await applyMigration(db, name, MIGRATIONS_DIR);
+ snapshot = snapshotSchema(db);
+ violations.push(
+ ...findAdditiveViolations(before, snapshot, NON_SYNCED_TABLES).map(
+ violation => `${name}: ${violation}`,
+ ),
+ );
+ }
+ sqlite.closeDatabase(db);
+
+ expect(
+ violations,
+ 'Migrations must be additive-only so that older clients keep ' +
+ 'working: no dropping or renaming tables/columns, new columns ' +
+ 'in synced tables must be nullable or have a DEFAULT, and new ' +
+ 'synced tables need a single primary key named "id" (sync ' +
+ 'builds rows one column at a time, addressed by id). If you ' +
+ 'need to retire a column, stop reading it but leave it in ' +
+ 'place. New internal (non-synced) tables go in ' +
+ 'NON_SYNCED_TABLES in this test.',
+ ).toEqual([]);
+ });
+
+ it('tolerates an unknown migration id interleaved below the newest known one', async () => {
+ const db = await openTestDb(nativeFs.readFileSync(INIT_SQL, 'utf8'));
+ await withMigrationsDir(MIGRATIONS_DIR, async () => {
+ await migrate(db);
+ // A newer release may ship a migration whose id (an authoring
+ // timestamp) sorts below ids this version already knows — it is
+ // still additive-era and must not fail validation
+ sqlite.runQuery(db, 'INSERT INTO __migrations__ (id) VALUES (?)', [
+ ADDITIVE_ONLY_CUTOFF + 1,
+ ]);
+ await migrate(db);
+ });
+ sqlite.closeDatabase(db);
+ });
+
+ const TABLE_FOO = 'CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT, b TEXT);';
+
+ it.each([
+ ['dropping a table', TABLE_FOO, 'DROP TABLE foo;'],
+ ['renaming a table', TABLE_FOO, 'ALTER TABLE foo RENAME TO bar;'],
+ ['renaming a column', TABLE_FOO, 'ALTER TABLE foo RENAME COLUMN a TO c;'],
+ [
+ 'dropping a column via table rebuild',
+ TABLE_FOO,
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, b TEXT);
+ INSERT INTO foo_new SELECT id, b FROM foo;
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'adding a required column without a default',
+ TABLE_FOO,
+ 'ALTER TABLE foo ADD COLUMN c TEXT NOT NULL;',
+ ],
+ [
+ 'creating a synced table with required columns lacking defaults',
+ TABLE_FOO,
+ `CREATE TABLE gadgets
+ (id TEXT PRIMARY KEY,
+ name TEXT NOT NULL,
+ size INTEGER NOT NULL,
+ color TEXT,
+ shape TEXT DEFAULT 'round');`,
+ ],
+ [
+ 'creating a synced table whose primary key is not "id"',
+ TABLE_FOO,
+ 'CREATE TABLE gadgets (uuid TEXT PRIMARY KEY, name TEXT);',
+ ],
+ [
+ 'creating a synced table with a composite primary key',
+ TABLE_FOO,
+ 'CREATE TABLE gadgets (id TEXT, name TEXT, PRIMARY KEY (id, name));',
+ ],
+ [
+ 'creating a synced table with no primary key',
+ TABLE_FOO,
+ 'CREATE TABLE gadgets (name TEXT);',
+ ],
+ [
+ 'making an existing column required via table rebuild',
+ TABLE_FOO,
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT NOT NULL, b TEXT);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'changing primary-key membership via table rebuild',
+ TABLE_FOO,
+ `CREATE TABLE foo_new (id TEXT, a TEXT, b TEXT, PRIMARY KEY (id, a));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'adding a UNIQUE index to a synced table',
+ TABLE_FOO,
+ 'CREATE UNIQUE INDEX foo_a ON foo (a);',
+ ],
+ [
+ 'adding a UNIQUE constraint via table rebuild',
+ TABLE_FOO,
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT UNIQUE, b TEXT);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'adding a required column to an existing internal table',
+ TABLE_FOO +
+ 'CREATE TABLE messages_pending (timestamp TEXT PRIMARY KEY, dataset TEXT NOT NULL);',
+ 'ALTER TABLE messages_pending ADD COLUMN extra TEXT NOT NULL;',
+ ],
+ [
+ 'removing a UNIQUE constraint',
+ TABLE_FOO + 'CREATE UNIQUE INDEX foo_a ON foo (a);',
+ 'DROP INDEX foo_a;',
+ ],
+ [
+ 'adding a CHECK constraint',
+ TABLE_FOO,
+ 'ALTER TABLE foo ADD COLUMN amount INTEGER CHECK(amount >= 0);',
+ ],
+ [
+ 'making a column NOT NULL with a DEFAULT via table rebuild',
+ TABLE_FOO,
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, a TEXT NOT NULL DEFAULT 'x', b TEXT);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'making a NOT NULL column nullable via table rebuild',
+ "CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT NOT NULL DEFAULT 'x');",
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'removing a DEFAULT from a NOT NULL column via table rebuild',
+ "CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT NOT NULL DEFAULT 'x');",
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT NOT NULL);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'changing a CHECK expression via table rebuild',
+ 'CREATE TABLE foo (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount >= 0));',
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount > 0));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'changing a CHECK expression after a quoted paren',
+ `CREATE TABLE foo
+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'x'));`,
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'y'));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'changing a CHECK string literal case',
+ `CREATE TABLE foo
+ (id TEXT PRIMARY KEY, status TEXT CHECK(status <> 'ARCHIVED'));`,
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, status TEXT CHECK(status <> 'archived'));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'adding a NOT NULL column whose DEFAULT is an explicit NULL',
+ TABLE_FOO,
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, a TEXT, b TEXT, c TEXT NOT NULL DEFAULT NULL);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ "changing an existing column's type via table rebuild",
+ TABLE_FOO,
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a INTEGER, b TEXT);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ "changing an existing column's DEFAULT value via table rebuild",
+ "CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT DEFAULT 'x');",
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT DEFAULT 'y');
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ ])('sanity check: flags %s', async (_case, setup, migration) => {
+ expect(await violationsFor(setup, migration)).not.toEqual([]);
+ });
+
+ it.each([
+ [
+ 'adding nullable and defaulted columns',
+ TABLE_FOO,
+ `ALTER TABLE foo ADD COLUMN c TEXT;
+ ALTER TABLE foo ADD COLUMN d TEXT NOT NULL DEFAULT 'x';`,
+ ],
+ [
+ 'creating a table with nullable and defaulted columns',
+ TABLE_FOO,
+ `CREATE TABLE gadgets
+ (id TEXT PRIMARY KEY, name TEXT, size INTEGER NOT NULL DEFAULT 0);`,
+ ],
+ [
+ 'dropping and recreating a view',
+ TABLE_FOO + 'CREATE VIEW v_foo AS SELECT id, a FROM foo;',
+ `DROP VIEW v_foo;
+ CREATE VIEW v_foo AS SELECT id, a, b FROM foo;`,
+ ],
+ [
+ 'creating an internal (non-synced) table with required columns',
+ TABLE_FOO,
+ `CREATE TABLE messages_pending
+ (timestamp TEXT PRIMARY KEY, dataset TEXT NOT NULL);`,
+ ],
+ [
+ 'adding a plain (non-unique) index',
+ TABLE_FOO,
+ 'CREATE INDEX foo_a_idx ON foo (a);',
+ ],
+ [
+ 'creating a synced table with a UNIQUE constraint',
+ TABLE_FOO,
+ 'CREATE TABLE gadgets (id TEXT PRIMARY KEY, name TEXT UNIQUE);',
+ ],
+ [
+ 'creating a synced table with a CHECK constraint',
+ TABLE_FOO,
+ 'CREATE TABLE gadgets (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount >= 0));',
+ ],
+ [
+ 'rebuilding a table with an identical CHECK, reformatted',
+ 'CREATE TABLE foo (id TEXT PRIMARY KEY, amount INTEGER CHECK(amount >= 0));',
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, amount INTEGER CHECK( amount >= 0 ));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'rebuilding a table with an identical CHECK containing a quoted paren',
+ `CREATE TABLE foo
+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'x'));`,
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, a TEXT CHECK(a <> ')' AND a <> 'x'));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'rebuilding a table with an identical CHECK, reformatted keywords',
+ `CREATE TABLE foo
+ (id TEXT PRIMARY KEY, status TEXT CHECK(status <> 'ARCHIVED'));`,
+ `CREATE TABLE foo_new
+ (id TEXT PRIMARY KEY, status TEXT CHECK( STATUS <> 'ARCHIVED' ));
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ [
+ 'adding a column whose quoted DEFAULT contains CHECK(...)',
+ TABLE_FOO,
+ "ALTER TABLE foo ADD COLUMN c TEXT DEFAULT 'CHECK(c > 0)';",
+ ],
+ [
+ 'rebuilding a table replacing an explicit DEFAULT NULL with no DEFAULT',
+ 'CREATE TABLE foo (id TEXT PRIMARY KEY, a TEXT DEFAULT NULL);',
+ `CREATE TABLE foo_new (id TEXT PRIMARY KEY, a TEXT);
+ DROP TABLE foo;
+ ALTER TABLE foo_new RENAME TO foo;`,
+ ],
+ ])('sanity check: allows %s', async (_case, setup, migration) => {
+ expect(await violationsFor(setup, migration)).toEqual([]);
+ });
+});
diff --git a/packages/loot-core/src/server/migrate/migrations.test.ts b/packages/loot-core/src/server/migrate/migrations.test.ts
index d66b12925d..04cef9619d 100644
--- a/packages/loot-core/src/server/migrate/migrations.test.ts
+++ b/packages/loot-core/src/server/migrate/migrations.test.ts
@@ -1,4 +1,8 @@
// @ts-strict-ignore
+import * as nativeFs from 'fs';
+import * as os from 'os';
+import * as path from 'path';
+
import * as db from '#server/db';
import {
@@ -58,6 +62,103 @@ describe('Migrations', () => {
);
});
+ test('tolerates migrations applied by a newer version of the app', async () => {
+ return withMigrationsDir(
+ __dirname + '/../../mocks/migrations',
+ async () => {
+ await migrate(db.getDatabase());
+
+ // Simulate a migration applied by a newer version of the app
+ // (its id is newer than anything this version knows about)
+ db.runQuery('INSERT INTO __migrations__ (id) VALUES (9999999999999)');
+
+ // Should not throw
+ await migrate(db.getDatabase());
+
+ const applied = await getAppliedMigrations(db.getDatabase());
+ expect(applied).toContain(9999999999999);
+ },
+ );
+ });
+
+ test('rejects a newer unknown migration when a known one is missing', async () => {
+ return withMigrationsDir(
+ __dirname + '/../../mocks/migrations',
+ async () => {
+ // A database that skipped known migrations but somehow contains
+ // one from a newer version — impossible via any legitimate flow
+ // (append-only migrations mean the newer version knew ours too),
+ // so it must be treated as corrupt, not migrated further
+ db.runQuery('INSERT INTO __migrations__ (id) VALUES (1508717984291)');
+ db.runQuery('INSERT INTO __migrations__ (id) VALUES (9999999999999)');
+
+ await expect(migrate(db.getDatabase())).rejects.toThrow(
+ 'out-of-sync-migrations',
+ );
+ },
+ );
+ });
+
+ test('applies a pending migration whose id sorts below an applied one', async () => {
+ const dir = nativeFs.mkdtempSync(
+ path.join(os.tmpdir(), 'interleaved-migrations-'),
+ );
+ try {
+ nativeFs.writeFileSync(
+ path.join(dir, '1790000000001_a.sql'),
+ 'CREATE TABLE interleave_a (id TEXT PRIMARY KEY);',
+ );
+ nativeFs.writeFileSync(
+ path.join(dir, '1790000000003_c.sql'),
+ 'CREATE TABLE interleave_c (id TEXT PRIMARY KEY);',
+ );
+ await withMigrationsDir(dir, async () => {
+ await migrate(db.getDatabase());
+ });
+
+ // A later release ships a migration authored earlier: its id
+ // sorts between two already-applied ones. The upgrade must treat
+ // it as pending, not as an out-of-sync database.
+ nativeFs.writeFileSync(
+ path.join(dir, '1790000000002_b.sql'),
+ 'CREATE TABLE interleave_b (id TEXT PRIMARY KEY);',
+ );
+ await withMigrationsDir(dir, async () => {
+ await migrate(db.getDatabase());
+ });
+
+ const applied = await getAppliedMigrations(db.getDatabase());
+ expect(applied).toContain(1790000000002);
+ const desc = await db.first<{ name: string }>(
+ "SELECT name FROM sqlite_master WHERE name = 'interleave_b'",
+ );
+ expect(desc?.name).toBe('interleave_b');
+ } finally {
+ nativeFs.rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ test('rejects a migrated database when no migrations exist on disk', async () => {
+ await withMigrationsDir(__dirname + '/../../mocks/migrations', async () => {
+ await migrate(db.getDatabase());
+ });
+
+ // An empty migrations directory — a broken install must not pass
+ // validation just because every applied id looks unknown
+ const dir = nativeFs.mkdtempSync(
+ path.join(os.tmpdir(), 'empty-migrations-'),
+ );
+ try {
+ await withMigrationsDir(dir, async () => {
+ await expect(migrate(db.getDatabase())).rejects.toThrow(
+ 'out-of-sync-migrations',
+ );
+ });
+ } finally {
+ nativeFs.rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
test('app runs database migrations', async () => {
return withMigrationsDir(
__dirname + '/../../mocks/migrations',
diff --git a/packages/loot-core/src/server/migrate/migrations.ts b/packages/loot-core/src/server/migrate/migrations.ts
index 50c6eb8110..8206610040 100644
--- a/packages/loot-core/src/server/migrate/migrations.ts
+++ b/packages/loot-core/src/server/migrate/migrations.ts
@@ -9,6 +9,7 @@ import m1722717601000 from '#migrations/1722717601000_reports_move_selected_cate
import m1722804019000 from '#migrations/1722804019000_create_dashboard_table';
import m1723665565000 from '#migrations/1723665565000_prefs';
import m1765518577215 from '#migrations/1765518577215_multiple_dashboards';
+import m1788468782000 from '#migrations/1788468782000_add_messages_pending';
import * as fs from '#platform/server/fs';
import { logger } from '#platform/server/log';
import * as sqlite from '#platform/server/sqlite';
@@ -22,6 +23,7 @@ const javascriptMigrations = {
1722804019000: m1722804019000,
1723665565000: m1723665565000,
1765518577215: m1765518577215,
+ 1788468782000: m1788468782000,
};
export async function withMigrationsDir(
@@ -38,7 +40,7 @@ export function getMigrationsDir(): string {
return MIGRATIONS_DIR;
}
-function getMigrationId(name: string): number {
+export function getMigrationId(name: string): number {
return parseInt(name.match(/^(\d)+/)[0]);
}
@@ -143,13 +145,36 @@ export async function applyMigration(
]);
}
+// Migrations with ids after this point are additive-only, enforced by
+// additive-migrations.test.ts. A database containing unknown applied
+// ids from that era was touched by a newer release — including one
+// whose migration id happens to sort below ids this version already
+// knows (authored earlier, merged later) — and is still safe to open.
+// Unknown ids from before this point indicate a corrupt or
+// incompatible database.
+export const ADDITIVE_ONLY_CUTOFF = 1780606215001;
+
function checkDatabaseValidity(
appliedIds: number[],
available: string[],
): void {
- if (appliedIds.length > available.length) {
+ // A migrated database with no migrations on disk means the install is
+ // broken — without this guard every applied id would count as
+ // "unknown but tolerable" below and the checks would pass vacuously
+ if (available.length === 0 && appliedIds.length > 0) {
+ logger.error('No migrations found on disk for a migrated database:', {
+ appliedIds,
+ });
+ throw new Error('out-of-sync-migrations');
+ }
+
+ const allAvailableIds = available.map(getMigrationId);
+ const availableIds = new Set(allAvailableIds);
+ const unknownIds = appliedIds.filter(id => !availableIds.has(id));
+
+ if (unknownIds.some(id => id <= ADDITIVE_ONLY_CUTOFF)) {
logger.error(
- 'Database is out of sync with migrations (index past available):',
+ 'Database is out of sync with migrations (unknown migration from before the additive-only era):',
{
appliedIds,
available,
@@ -158,8 +183,53 @@ function checkDatabaseValidity(
throw new Error('out-of-sync-migrations');
}
- for (let i = 0; i < appliedIds.length; i++) {
- if (appliedIds[i] !== getMigrationId(available[i])) {
+ const knownAppliedIds = appliedIds.filter(id => availableIds.has(id));
+
+ // A database touched by a newer version must already contain every
+ // migration this app knows (append-only migrations guarantee the newer
+ // version knew them all). A known migration missing next to an
+ // unknown one means the database is corrupt — running it now, after
+ // later migrations already ran, would be unsafe.
+ if (unknownIds.length > 0 && knownAppliedIds.length !== available.length) {
+ logger.error(
+ 'Database is out of sync with migrations (missing known migration next to an unknown one):',
+ {
+ appliedIds,
+ available,
+ },
+ );
+ throw new Error('out-of-sync-migrations');
+ }
+
+ // Pre-cutoff migrations shipped strictly append-only, so the applied
+ // ones must form an ordered prefix of the available list — a gap
+ // there means the database is corrupt. Post-cutoff, a gap is just a
+ // pending interleaved-id migration that `migrate` applies next.
+ const preCutoffAvailableIds = allAvailableIds.filter(
+ id => id <= ADDITIVE_ONLY_CUTOFF,
+ );
+ const preCutoffAppliedIds = knownAppliedIds.filter(
+ id => id <= ADDITIVE_ONLY_CUTOFF,
+ );
+
+ // A post-cutoff migration only ever runs after the entire pre-cutoff
+ // chain, so one applied next to a missing pre-cutoff migration means
+ // the database is corrupt
+ if (
+ appliedIds.some(id => id > ADDITIVE_ONLY_CUTOFF) &&
+ preCutoffAppliedIds.length !== preCutoffAvailableIds.length
+ ) {
+ logger.error(
+ 'Database is out of sync with migrations (missing pre-cutoff migration next to an applied post-cutoff one):',
+ {
+ appliedIds,
+ available,
+ },
+ );
+ throw new Error('out-of-sync-migrations');
+ }
+ for (let i = 0; i < preCutoffAppliedIds.length; i++) {
+ if (preCutoffAppliedIds[i] !== preCutoffAvailableIds[i]) {
logger.error(
'Database is out of sync with migrations (migration id mismatch):',
{
diff --git a/packages/loot-core/src/server/migrate/schema-diff.ts b/packages/loot-core/src/server/migrate/schema-diff.ts
new file mode 100644
index 0000000000..2d422a231f
--- /dev/null
+++ b/packages/loot-core/src/server/migrate/schema-diff.ts
@@ -0,0 +1,257 @@
+// Test-only: imported solely by additive-migrations.test.ts, which
+// enforces the additive-only migration policy in CI. Nothing enforces
+// that policy at runtime — see `checkDatabaseValidity` for what the app
+// itself tolerates.
+import type { Database } from '@jlongster/sql.js';
+
+import * as sqlite from '#platform/server/sqlite';
+
+// `required` = cannot insert a row without providing this column.
+// `notNull` is tracked separately: NOT NULL with a DEFAULT still
+// rejects explicit NULL writes. `defaultValue` is null when absent or
+// an explicit NULL (they behave the same).
+type ColumnFlags = {
+ required: boolean;
+ notNull: boolean;
+ pk: boolean;
+ type: string;
+ defaultValue: string | null;
+};
+
+export type TableSnapshot = {
+ columns: Map<string, ColumnFlags>;
+ // Column lists of UNIQUE constraints/indexes, e.g. "a" or "a,b"
+ uniques: Set<string>;
+ // Normalized CHECK constraint expressions, sorted. PRAGMAs don't
+ // expose CHECKs, so they are extracted from the canonical CREATE
+ // TABLE text SQLite stores
+ checks: string[];
+};
+
+// Collapses whitespace so formatting differences in SQL text don't
+// register as changes
+function normalizeSql(sql: string): string {
+ return sql.replace(/\s+/g, ' ').trim();
+}
+
+// Quoted literals/identifiers (doubled quotes are escapes) and comments.
+// The input is well-formed SQL from sqlite_master, so unterminated
+// quotes/comments can't occur.
+const NON_CODE =
+ /'(?:[^']|'')*'|"(?:[^"]|"")*"|`(?:[^`]|``)*`|\[[^\]]*\]|--[^\n]*|\/\*[\s\S]*?\*\//g;
+
+// Lowercases SQL tokens outside quoted literals/comments so formatting
+// and keyword case differences don't register as CHECK changes, while
+// preserving string literal contents (SQLite treats 'ARCHIVED' and
+// 'archived' as different values).
+function normalizeCheckExpression(expr: string): string {
+ const normalized = normalizeSql(expr);
+ let result = '';
+ let lastIndex = 0;
+ for (const match of normalized.matchAll(NON_CODE)) {
+ const index = match.index!;
+ result += normalized.slice(lastIndex, index).toLowerCase();
+ result += match[0];
+ lastIndex = index + match[0].length;
+ }
+ result += normalized.slice(lastIndex).toLowerCase();
+ return result;
+}
+
+// Extracts each CHECK(...) expression by scanning to the balanced close
+// paren, normalized (whitespace collapsed, SQL tokens lowercased outside
+// quoted literals) so formatting differences don't register as changes.
+// The scan runs on a masked copy with literals and comments blanked to
+// spaces (length-preserving, so indexes line up) — a CHECK or paren
+// inside one can't start or end a clause. Clause text is sliced from
+// the original so literal contents survive intact.
+function checkClauses(sql: string): string[] {
+ const masked = sql.replace(NON_CODE, match => ' '.repeat(match.length));
+ const clauses: string[] = [];
+ const starts = /\bCHECK\s*\(/gi;
+ let match;
+ while ((match = starts.exec(masked))) {
+ const start = match.index + match[0].length;
+ let i = start;
+ for (let depth = 1; i < masked.length && depth > 0; i++) {
+ if (masked[i] === '(') {
+ depth++;
+ } else if (masked[i] === ')') {
+ depth--;
+ }
+ }
+ clauses.push(normalizeCheckExpression(sql.slice(start, i - 1)));
+ }
+ return clauses.sort();
+}
+
+// table -> its columns and constraints, from schema introspection
+export type SchemaSnapshot = Map<string, TableSnapshot>;
+
+export function snapshotSchema(db: Database): SchemaSnapshot {
+ const tables = sqlite.runQuery<{ name: string; sql: string }>(
+ db,
+ "SELECT name, sql FROM sqlite_master WHERE type = 'table' AND name NOT LIKE 'sqlite_%'",
+ [],
+ true,
+ );
+ return new Map(
+ tables.map(({ name: table, sql }): [string, TableSnapshot] => {
+ const columns = new Map(
+ sqlite
+ .runQuery<{
+ name: string;
+ type: string;
+ notnull: number;
+ dflt_value: string | null;
+ pk: number;
+ }>(db, `PRAGMA table_info("${table}")`, [], true)
+ .map((column): [string, ColumnFlags] => {
+ // PRAGMA reports the DEFAULT as verbatim SQL text; an
+ // explicit `DEFAULT NULL` comes back as the string "NULL",
+ // which is not a usable default for a NOT NULL column
+ const rawDefault =
+ column.dflt_value == null
+ ? null
+ : normalizeSql(column.dflt_value);
+ const defaultValue =
+ rawDefault?.toUpperCase() === 'NULL' ? null : rawDefault;
+ return [
+ column.name,
+ {
+ required:
+ column.notnull !== 0 &&
+ defaultValue == null &&
+ column.pk === 0,
+ notNull: column.notnull !== 0,
+ pk: column.pk !== 0,
+ type: normalizeSql(column.type).toLowerCase(),
+ defaultValue,
+ },
+ ];
+ }),
+ );
+ const uniques = new Set(
+ sqlite
+ .runQuery<{ name: string; unique: number; origin: string }>(
+ db,
+ `PRAGMA index_list("${table}")`,
+ [],
+ true,
+ )
+ .filter(index => index.unique !== 0 && index.origin !== 'pk')
+ .map(index =>
+ sqlite
+ .runQuery<{ name: string }>(
+ db,
+ `PRAGMA index_info("${index.name}")`,
+ [],
+ true,
+ )
+ .map(column => column.name)
+ .sort()
+ .join(','),
+ ),
+ );
+ return [table, { columns, uniques, checks: checkClauses(sql) }];
+ }),
+ );
+}
+
+// A schema change is additive-only when every column that existed
+// before still exists after, unchanged, and no constraint changes in
+// either direction — either direction, because tightening rejects data
+// older clients hold while loosening lets newer clients write data
+// older schemas reject on sync.
+export function findAdditiveViolations(
+ before: SchemaSnapshot,
+ after: SchemaSnapshot,
+ nonSyncedTables: Set<string>,
+): string[] {
+ const violations: string[] = [];
+
+ for (const [table, { columns: beforeColumns }] of before) {
+ const afterColumns = after.get(table)?.columns;
+ for (const name of beforeColumns.keys()) {
+ if (!afterColumns?.has(name)) {
+ violations.push(
+ `column "${table}.${name}" (or its table) was removed or renamed`,
+ );
+ }
+ }
+ }
+
+ for (const [table, { columns: afterColumns, uniques, checks }] of after) {
+ const beforeTable = before.get(table);
+
+ // The internal-table exemption applies only at creation; once a
+ // table exists, older app versions write it with the column lists
+ // they were built with, so changes are validated regardless of
+ // syncedness
+ if (!beforeTable && nonSyncedTables.has(table)) {
+ continue;
+ }
+
+ for (const [name, column] of afterColumns) {
+ const prev = beforeTable?.columns.get(name);
+ if (!prev) {
+ if (column.required) {
+ violations.push(
+ `new column "${table}.${name}" is NOT NULL without a DEFAULT`,
+ );
+ }
+ continue;
+ }
+ // A table rebuild can change an existing column in place; any
+ // attribute change breaks one side of the version skew
+ const changed = (
+ ['notNull', 'pk', 'type', 'defaultValue'] as const
+ ).filter(key => column[key] !== prev[key]);
+ if (changed.length > 0) {
+ violations.push(
+ `existing column "${table}.${name}" changed: ${changed.join(', ')}`,
+ );
+ }
+ }
+
+ // Constraint changes only matter on existing tables — a brand-new
+ // table gets the same constraints on every client that runs its
+ // migration, so there is no version skew to protect against
+ if (beforeTable) {
+ for (const unique of uniques) {
+ if (!beforeTable.uniques.has(unique)) {
+ violations.push(
+ `table "${table}" gained a UNIQUE constraint on (${unique})`,
+ );
+ }
+ }
+ for (const unique of beforeTable.uniques) {
+ if (!uniques.has(unique)) {
+ violations.push(
+ `table "${table}" lost a UNIQUE constraint on (${unique})`,
+ );
+ }
+ }
+
+ if (beforeTable.checks.join(';') !== checks.join(';')) {
+ violations.push(`table "${table}" changed its CHECK constraints`);
+ }
+ }
+
+ if (!beforeTable) {
+ // Sync addresses rows by their `id` column (see `apply` in
+ // #server/sync), so a new synced table needs exactly that as its
+ // primary key — no other name, no composite keys
+ const pks = [...afterColumns]
+ .filter(([, c]) => c.pk)
+ .map(([name]) => name);
+ if (pks.length !== 1 || pks[0] !== 'id') {
+ violations.push(
+ `new table "${table}" must have a single primary key named "id"`,
+ );
+ }
+ }
+ }
+
+ return violations;
+}
diff --git a/packages/loot-core/src/server/sync/deferred.test.ts b/packages/loot-core/src/server/sync/deferred.test.ts
new file mode 100644
index 0000000000..e447212df0
--- /dev/null
+++ b/packages/loot-core/src/server/sync/deferred.test.ts
@@ -0,0 +1,402 @@
+import { Timestamp } from '@actual-app/crdt';
+import { afterEach, beforeEach, describe, expect, it } from 'vitest';
+
+import * as db from '#server/db';
+
+import { replayPendingMessages } from './replay';
+import { deserializeValueSafe } from './serialization';
+
+import { applyMessages, setSyncingMode } from './index';
+
+beforeEach(() => {
+ setSyncingMode('enabled');
+ return global.emptyDatabase()();
+});
+
+afterEach(() => {
+ setSyncingMode('disabled');
+});
+
+function sendTimestamp(): Timestamp {
+ const timestamp = Timestamp.send();
+ if (timestamp == null) {
+ throw new Error('Timestamp.send() returned null');
+ }
+ return timestamp;
+}
+
+function getPending(): db.DbPendingMessage[] {
+ return db.runQuery<db.DbPendingMessage>(
+ 'SELECT * FROM messages_pending ORDER BY timestamp',
+ [],
+ true,
+ );
+}
+
+describe('Deferred sync messages (newer schema)', () => {
+ it('works even when the messages_pending migration never ran', async () => {
+ // Code newer than the schema: the served migration files can lag
+ // behind the app (stale service-worker cache, unrebuilt dev env)
+ db.execQuery('DROP TABLE messages_pending');
+
+ // Budget load must not throw...
+ replayPendingMessages();
+
+ // ...and deferral must still store the message, not fail the batch
+ await applyMessages(
+ [
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'brand_new_column',
+ value: 'hello',
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ expect(getPending().length).toBe(1);
+ });
+
+ it('defers messages for unknown columns without failing the batch', async () => {
+ await applyMessages(
+ [
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'brand_new_column',
+ value: 'hello',
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'amount',
+ value: 1234,
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+
+ // The known-column message must still create the row, even though
+ // the unknown-column message for the same row came first
+ const row = await db.first<{ id: string; amount: number }>(
+ 'SELECT * FROM transactions WHERE id = ?',
+ ['t1'],
+ );
+ expect(row?.amount).toBe(1234);
+
+ // The unknown-column message is recorded as pending...
+ const pending = getPending();
+ expect(pending.length).toBe(1);
+ expect(pending[0].dataset).toBe('transactions');
+ expect(pending[0].column).toBe('brand_new_column');
+ expect(pending[0].value).toBe('S:hello');
+
+ // ...and both messages made it into the crdt log, so the merkle
+ // stays in sync with other clients
+ const crdt = db.runQuery<{ column: string }>(
+ 'SELECT * FROM messages_crdt',
+ [],
+ true,
+ );
+ expect(crdt.length).toBe(2);
+ });
+
+ it('does not defer local messages: unknown schema is a bug and must fail', async () => {
+ await expect(
+ applyMessages([
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'brand_new_column',
+ value: 'hello',
+ timestamp: sendTimestamp(),
+ },
+ ]),
+ ).rejects.toMatchObject({ reason: 'invalid-schema' });
+
+ expect(getPending().length).toBe(0);
+ });
+
+ it('replays pending messages once the schema catches up, last write wins', async () => {
+ await applyMessages(
+ [
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'brand_new_column',
+ value: 'old value',
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'amount',
+ value: 1234,
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'brand_new_column',
+ value: 'new value',
+ timestamp: sendTimestamp(),
+ },
+ {
+ // A row that only ever got a deferred message; replay has to
+ // create it
+ dataset: 'transactions',
+ row: 't2',
+ column: 'brand_new_column',
+ value: 'other row',
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'gadgets',
+ row: 'g1',
+ column: 'name',
+ value: 'flux capacitor',
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ // Repeated writes to the same cell coalesce to the newest value, so
+ // t1's two brand_new_column messages leave one pending row
+ const coalesced = getPending();
+ expect(coalesced.length).toBe(3);
+ expect(
+ coalesced.find(p => p.row === 't1' && p.column === 'brand_new_column')
+ ?.value,
+ ).toBe('S:new value');
+
+ // "Run the migration" that the newer client already has
+ db.execQuery('ALTER TABLE transactions ADD COLUMN brand_new_column TEXT');
+ replayPendingMessages();
+
+ const t1 = await db.first<{ brand_new_column: string }>(
+ 'SELECT * FROM transactions WHERE id = ?',
+ ['t1'],
+ );
+ expect(t1?.brand_new_column).toBe('new value');
+
+ const t2 = await db.first<{ brand_new_column: string }>(
+ 'SELECT * FROM transactions WHERE id = ?',
+ ['t2'],
+ );
+ expect(t2?.brand_new_column).toBe('other row');
+
+ // The message for the still-unknown table stays pending...
+ let pending = getPending();
+ expect(pending.length).toBe(1);
+ expect(pending[0].dataset).toBe('gadgets');
+
+ // ...until a later upgrade adds it
+ db.execQuery('CREATE TABLE gadgets (id TEXT PRIMARY KEY, name TEXT)');
+ replayPendingMessages();
+
+ const g1 = await db.first<{ name: string }>(
+ 'SELECT * FROM gadgets WHERE id = ?',
+ ['g1'],
+ );
+ expect(g1?.name).toBe('flux capacitor');
+ pending = getPending();
+ expect(pending.length).toBe(0);
+ });
+
+ it('drops a message that can never apply without blocking the rest', async () => {
+ await applyMessages(
+ [
+ {
+ dataset: 'gadgets',
+ row: 'g1',
+ column: 'size',
+ value: -5,
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'gadgets',
+ row: 'g2',
+ column: 'size',
+ value: 10,
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ expect(getPending().length).toBe(2);
+
+ // The migration arrives with a CHECK constraint that the first
+ // deferred value violates; replay must still apply the second one
+ // and must not leave the failing message wedged in the queue
+ db.execQuery(
+ 'CREATE TABLE gadgets (id TEXT PRIMARY KEY, size INTEGER CHECK(size >= 0))',
+ );
+ replayPendingMessages();
+
+ const g2 = await db.first<{ size: number }>(
+ 'SELECT * FROM gadgets WHERE id = ?',
+ ['g2'],
+ );
+ expect(g2?.size).toBe(10);
+ expect(getPending().length).toBe(0);
+ });
+
+ it('defers an inbound message whose value uses a newer serialization format', async () => {
+ await applyMessages(
+ [
+ {
+ // Even though the column exists, the value can't be decoded —
+ // the message must defer instead of wedging the sync
+ dataset: 'transactions',
+ row: 't1',
+ column: 'notes',
+ value: deserializeValueSafe('B:true'),
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+
+ // Deferred with its original encoding intact, and acknowledged into
+ // the crdt log with the same bytes the sender wrote
+ const pending = getPending();
+ expect(pending.length).toBe(1);
+ expect(pending[0].value).toBe('B:true');
+ const crdt = db.runQuery<{ value: string }>(
+ 'SELECT * FROM messages_crdt',
+ [],
+ true,
+ );
+ expect(crdt.length).toBe(1);
+ expect(String(crdt[0].value)).toBe('B:true');
+ });
+
+ it('applies dependency chains among deferred cells', async () => {
+ await applyMessages(
+ [
+ {
+ dataset: 'gadgets',
+ row: 'g1',
+ column: 'a',
+ value: 'A',
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'gadgets',
+ row: 'g1',
+ column: 'b',
+ value: 'B',
+ timestamp: sendTimestamp(),
+ },
+ {
+ dataset: 'gadgets',
+ row: 'g1',
+ column: 'c',
+ value: 'C',
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ expect(getPending().length).toBe(3);
+
+ // Row-level constraints form a chain: `a` needs `b` set, `b` needs
+ // `c` set — timestamp order alone can't apply these in one pass, so
+ // replay must keep retrying while passes make progress
+ db.execQuery(`
+ CREATE TABLE gadgets
+ (id TEXT PRIMARY KEY, a TEXT, b TEXT, c TEXT,
+ CHECK(a IS NULL OR b IS NOT NULL),
+ CHECK(b IS NULL OR c IS NOT NULL))
+ `);
+ replayPendingMessages();
+
+ const g1 = await db.first<{ a: string; b: string; c: string }>(
+ 'SELECT * FROM gadgets WHERE id = ?',
+ ['g1'],
+ );
+ expect(g1?.a).toBe('A');
+ expect(g1?.b).toBe('B');
+ expect(g1?.c).toBe('C');
+ expect(getPending().length).toBe(0);
+ });
+
+ it('does not replay a deferred value over a newer applied write', async () => {
+ await applyMessages(
+ [
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'notes',
+ value: deserializeValueSafe('B:old-format'),
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ // A later, decodable write to the same cell applies through normal
+ // sync while the undecodable one sits pending
+ await applyMessages(
+ [
+ {
+ dataset: 'transactions',
+ row: 't1',
+ column: 'notes',
+ value: 'newer value',
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ expect(getPending().length).toBe(1);
+
+ replayPendingMessages();
+
+ // Last write wins: the stale pending value is discarded, not applied
+ const t1 = await db.first<{ notes: string }>(
+ 'SELECT * FROM transactions WHERE id = ?',
+ ['t1'],
+ );
+ expect(t1?.notes).toBe('newer value');
+ expect(getPending().length).toBe(0);
+ });
+
+ it('keeps a message whose value uses a newer serialization format', async () => {
+ await applyMessages(
+ [
+ {
+ dataset: 'gadgets',
+ row: 'g1',
+ column: 'name',
+ value: 'flux capacitor',
+ timestamp: sendTimestamp(),
+ },
+ ],
+ true,
+ );
+ // A value prefix this version's deserializeValue doesn't know,
+ // as a newer app version could write it
+ db.runQuery(
+ `INSERT INTO messages_pending (timestamp, dataset, row, column, value)
+ VALUES (?, 'gadgets', 'g2', 'name', 'B:true')`,
+ [sendTimestamp().toString()],
+ );
+
+ db.execQuery('CREATE TABLE gadgets (id TEXT PRIMARY KEY, name TEXT)');
+ replayPendingMessages();
+
+ // The known-format message applied; the newer-format one stays
+ // pending for a future upgrade instead of aborting the replay
+ const g1 = await db.first<{ name: string }>(
+ 'SELECT * FROM gadgets WHERE id = ?',
+ ['g1'],
+ );
+ expect(g1?.name).toBe('flux capacitor');
+ const pending = getPending();
+ expect(pending.length).toBe(1);
+ expect(pending[0].row).toBe('g2');
+ });
+});
diff --git a/packages/loot-core/src/server/sync/index.ts b/packages/loot-core/src/server/sync/index.ts
index f22e0ab44f..ce928f16c2 100644
--- a/packages/loot-core/src/server/sync/index.ts
+++ b/packages/loot-core/src/server/sync/index.ts
@@ -26,12 +26,20 @@ import * as sheet from '#server/sheet';
import { resolveName } from '#server/spreadsheet/util';
import * as undo from '#server/undo';
import { once, sequential } from '#shared/async';
+import { isMissingSchemaError } from '#shared/errors';
import { getIn, setIn } from '#shared/util';
import type { MetadataPrefs } from '#types/prefs';
import * as encoder from './encoder';
+import { PENDING_MESSAGES_TABLE_SQL } from './messages-pending';
import { rebuildMerkleHash } from './repair';
-import { isError } from './utils';
+import {
+ deserializeValueSafe,
+ isUnknownFormatValue,
+ serializeValue,
+} from './serialization';
+import type { UnknownFormatValue } from './serialization';
+import { isError, notifyDeferredMessages, quoteSqlId } from './utils';
export { makeTestMessage } from './make-test-message';
export { resetSync } from './reset';
@@ -77,34 +85,92 @@ export function checkSyncingMode(mode: SyncingMode): boolean {
}
}
-function apply(msg: Message, prev?: boolean) {
+// Record a message that can't be applied yet because it targets schema
+// from a newer version. It's replayed by `replayPendingMessages` once a
+// migration adds the missing table/column. Only the newest value per
+// cell is kept — replay is last-write-wins per cell anyway, and this
+// bounds the table while the client stays on an old version.
+function deferMessage(msg: Message) {
+ // The table may not exist yet when the served migration files are
+ // older than the code — create it rather than failing the sync batch
+ db.execQuery(PENDING_MESSAGES_TABLE_SQL);
+ db.runQuery(
+ db.cache(
+ `INSERT INTO messages_pending (timestamp, dataset, row, column, value)
+ VALUES (?, ?, ?, ?, ?)
+ ON CONFLICT(dataset, row, column) DO UPDATE
+ SET timestamp = excluded.timestamp, value = excluded.value
+ WHERE excluded.timestamp > messages_pending.timestamp`,
+ ),
+ [
+ msg.timestamp.toString(),
+ msg.dataset,
+ msg.row,
+ msg.column,
+ serializeValue(msg.value),
+ ],
+ );
+}
+
+// Returns false when the message was deferred because it references
+// schema this client doesn't have yet (sent by a newer version).
+// Deferral only makes sense for inbound messages; for locally-created
+// messages a missing table/column is a bug and must surface as an
+// apply-failure.
+function apply(
+ msg: Message,
+ prev?: boolean,
+ deferUnknownSchema?: boolean,
+): boolean {
const { dataset, row, column, value } = msg;
if (dataset === 'prefs') {
// Do nothing, it doesn't exist in the db
+ } else if (dataset === 'spreadsheet_cells') {
+ // Legacy dataset keyed by `name`, not `id`, so the write below
+ // could never apply; it's a derived cache, so ignore the message
+ } else if (isUnknownFormatValue(value)) {
+ // The value was serialized by a newer version in a format this one
+ // can't decode — defer it whole, like a missing table/column
+ if (deferUnknownSchema) {
+ deferMessage(msg);
+ return false;
+ }
+ throw new SyncError('invalid-schema', {
+ error: { message: 'Unknown value format: ' + value.raw, stack: '' },
+ query: {
+ sql: `INSERT INTO ${quoteSqlId(dataset)} (id, ${quoteSqlId(column)}) VALUES (?, ?)`,
+ params: [row, value.raw],
+ },
+ });
} else {
let query;
try {
if (prev) {
query = {
- sql: `UPDATE ${dataset} SET ${column} = ? WHERE id = ?`,
+ sql: `UPDATE ${quoteSqlId(dataset)} SET ${quoteSqlId(column)} = ? WHERE id = ?`,
params: [value, row],
};
} else {
query = {
- sql: `INSERT INTO ${dataset} (id, ${column}) VALUES (?, ?)`,
+ sql: `INSERT INTO ${quoteSqlId(dataset)} (id, ${quoteSqlId(column)}) VALUES (?, ?)`,
params: [row, value],
};
}
db.runQuery(db.cache(query.sql), query.params);
} catch (error) {
+ if (deferUnknownSchema && isMissingSchemaError(error)) {
+ deferMessage(msg);
+ return false;
+ }
throw new SyncError('invalid-schema', {
error: { message: error.message, stack: error.stack },
query,
});
}
}
+ return true;
}
// TODO: convert to `whereIn`
@@ -117,7 +183,7 @@ async function fetchAll(table, ids) {
for (let i = 0; i < ids.length; i += batchSize) {
const partIds = ids.slice(i, i + batchSize);
let sql;
- let column = `${table}.id`;
+ let column = `${quoteSqlId(table)}.id`;
// We have to provide *mapped* data so the spreadsheet works. The functions
// which trigger budget changes based on data changes assumes data has been
@@ -131,7 +197,7 @@ async function fetchAll(table, ids) {
`;
column = 't.id';
} else {
- sql = `SELECT * FROM ${table}`;
+ sql = `SELECT * FROM ${quoteSqlId(table)}`;
}
sql += ` WHERE `;
@@ -141,6 +207,11 @@ async function fetchAll(table, ids) {
const rows = db.runQuery(sql, partIds, true);
results = results.concat(rows);
} catch (error) {
+ if (isMissingSchemaError(error)) {
+ // The table comes from a newer version of the app; its messages
+ // will be deferred by `apply`
+ break;
+ }
throw new SyncError('invalid-schema', {
error: {
message: error.message,
@@ -154,33 +225,6 @@ async function fetchAll(table, ids) {
return results;
}
-export function serializeValue(value: string | number | null): string {
- if (value === null) {
- return '0:';
- } else if (typeof value === 'number') {
- return 'N:' + value;
- } else if (typeof value === 'string') {
- return 'S:' + value;
- }
-
- throw new Error('Unserializable value type: ' + JSON.stringify(value));
-}
-
-export function deserializeValue(value: string): string | number | null {
- const type = value[0];
- switch (type) {
- case '0':
- return null;
- case 'N':
- return parseFloat(value.slice(2));
- case 'S':
- return value.slice(2);
- default:
- }
-
- throw new Error('Invalid type key for value: ' + value);
-}
-
// TODO make this type stricter.
type DataMap = Map<string, unknown>;
type SyncListener = (oldData: DataMap, newData: DataMap) => unknown;
@@ -255,10 +299,10 @@ export type Message = {
old?: unknown;
row: string;
timestamp: Timestamp;
- value: string | number | null;
+ value: string | number | null | UnknownFormatValue;
};
-export const applyMessages = sequential(async (messages: Message[]) => {
+async function _applyMessages(messages: Message[], deferUnknownSchema = false) {
if (checkSyncingMode('import')) {
applyMessagesForImport(messages);
return undefined;
@@ -335,6 +379,7 @@ export const applyMessages = sequential(async (messages: Message[]) => {
// nothing is changed. This is critical to maintain consistency. We
// also avoid any side effects to in-memory objects, and apply them
// after this succeeds.
+ const deferredMessages = new Set<Message>();
db.transaction(() => {
const added = new Set();
@@ -342,32 +387,49 @@ export const applyMessages = sequential(async (messages: Message[]) => {
const { dataset, row, column, timestamp, value } = msg;
if (!msg.old) {
- apply(msg, getIn(oldData, [dataset, row]) || added.has(dataset + row));
+ const applied = apply(
+ msg,
+ getIn(oldData, [dataset, row]) || added.has(dataset + row),
+ deferUnknownSchema,
+ );
- if (dataset === 'prefs') {
- prefsToSet[row] = value;
+ if (applied) {
+ if (dataset === 'prefs') {
+ // An unknown-format pref value can't be stored or replayed
+ // (prefs aren't a table) — leave the local pref as-is
+ if (!isUnknownFormatValue(value)) {
+ prefsToSet[row] = value;
+ }
+ } else {
+ // Keep track of which items have been added it in this sync
+ // so it knows whether they already exist in the db or not. We
+ // ignore any changes to the spreadsheet.
+ added.add(dataset + row);
+
+ // Special treatment for some synced prefs. Applied messages
+ // only — an old or deferred message must not flip the
+ // in-memory budget type
+ if (dataset === 'preferences' && row === 'budgetType') {
+ void setBudgetType(value);
+ }
+ }
} else {
- // Keep track of which items have been added it in this sync
- // so it knows whether they already exist in the db or not. We
- // ignore any changes to the spreadsheet.
- added.add(dataset + row);
+ // Deferred messages must not be tracked in `added`: their
+ // row wasn't created, so a later message for a known column
+ // still needs to INSERT it
+ deferredMessages.add(msg);
}
}
if (checkSyncingMode('enabled')) {
db.runQuery(
db.cache(`INSERT INTO messages_crdt (timestamp, dataset, row, column, value)
- VALUES (?, ?, ?, ?, ?)`),
+ VALUES (?, ?, ?, ?, ?)`),
[timestamp.toString(), dataset, row, column, serializeValue(value)],
);
currentMerkle = merkle.insert(currentMerkle, timestamp);
}
-
- // Special treatment for some synced prefs
- if (dataset === 'preferences' && row === 'budgetType') {
- void setBudgetType(value);
- }
}
if (checkSyncingMode('enabled')) {
@@ -433,7 +495,11 @@ export const applyMessages = sequential(async (messages: Message[]) => {
_syncListeners.forEach(func => func(oldData, newData));
- const tables = getTablesFromMessages(messages.filter(msg => !msg.old));
+ // Only tables that actually changed — deferred messages wrote
+ // nothing, so they must not trigger client cache invalidation
+ const tables = getTablesFromMessages(
+ messages.filter(msg => !msg.old && !deferredMessages.has(msg)),
+ );
app.events.emit('sync', {
type: 'applied',
tables,
@@ -441,8 +507,20 @@ export const applyMessages = sequential(async (messages: Message[]) => {
prevData: oldData,
});
- return messages;
-});
+ if (deferredMessages.size > 0) {
+ notifyDeferredMessages();
+ }
+
+ // Deferred messages wrote nothing, so they don't count as received
+ // — this also keeps their tables out of the `success` event in
+ // `fullSync`. Old messages stay: they were processed (merkled),
+ // just superseded.
+ return deferredMessages.size === 0
+ ? messages
+ : messages.filter(msg => !deferredMessages.has(msg));
+}
+
+export const applyMessages = sequential(_applyMessages);
export function receiveMessages(messages: Message[]): Promise<Message[]> {
try {
@@ -456,7 +534,9 @@ export function receiveMessages(messages: Message[]): Promise<Message[]> {
throw e;
}
- return runMutator(() => applyMessages(messages));
+ // Inbound messages may come from a newer version of the app, so
+ // unknown-schema errors defer instead of failing the batch
+ return runMutator(() => applyMessages(messages, true));
}
async function errorHandler(e: Error) {
@@ -744,7 +824,7 @@ async function _fullSync(
receivedMessages = await receiveMessages(
res.messages.map(msg => ({
...msg,
- value: deserializeValue(msg.value as string),
+ value: deserializeValueSafe(msg.value as string),
})),
);
}
diff --git a/packages/loot-core/src/server/sync/messages-pending.ts b/packages/loot-core/src/server/sync/messages-pending.ts
new file mode 100644
index 0000000000..5bd56d1b33
--- /dev/null
+++ b/packages/loot-core/src/server/sync/messages-pending.ts
@@ -0,0 +1,18 @@
+// Sync messages deferred because they reference schema from a newer
+// app version; see `deferMessage` and `replayPendingMessages`.
+//
+// The DDL is shared between the migration and the runtime: the code
+// that uses the table also creates it on demand, so it keeps working
+// even when the served migration files are older than the code (e.g. a
+// stale service-worker cache or an unrebuilt dev environment).
+// Dependency-free on purpose — the migration imports this too.
+export const PENDING_MESSAGES_TABLE_SQL = `
+ CREATE TABLE IF NOT EXISTS messages_pending
+ (dataset TEXT NOT NULL,
+ row TEXT NOT NULL,
+ column TEXT NOT NULL,
+ timestamp TEXT NOT NULL,
+ -- Serialized string form, e.g. "S:hello" (see \`serializeValue\`)
+ value TEXT NOT NULL,
+ PRIMARY KEY (dataset, row, column));
+`;
diff --git a/packages/loot-core/src/server/sync/replay.ts b/packages/loot-core/src/server/sync/replay.ts
new file mode 100644
index 0000000000..efd44be54f
--- /dev/null
+++ b/packages/loot-core/src/server/sync/replay.ts
@@ -0,0 +1,194 @@
+import { captureException } from '#platform/exceptions';
+import { logger } from '#platform/server/log';
+import * as db from '#server/db';
+import { isMissingSchemaError } from '#shared/errors';
+
+import { PENDING_MESSAGES_TABLE_SQL } from './messages-pending';
+import { deserializeValue } from './serialization';
+import {
+ notifyDeferredMessages,
+ notifyDroppedMessages,
+ quoteSqlId,
+ resetDeferredMessagesNotification,
+} from './utils';
+
+// SQLite failures that repeat identically on every attempt. Anything
+// else non-schema (locked/full/interrupted/storage errors) may be
+// transient and aborts the whole replay instead.
+const DETERMINISTIC_SQLITE_ERROR =
+ /constraint failed|datatype mismatch|cannot store|generated column|string or blob too big|syntax error/i;
+
+type ReplayOutcome =
+ | { outcome: 'applied' }
+ | { outcome: 'newer-version' }
+ | { outcome: 'failed'; error: Error };
+
+// Deletes pending rows superseded by a newer acknowledged write in the
+// crdt log (last write wins per cell — possible because an
+// unknown-format value defers even when its column exists). Also used
+// by `resetSync` so its discard warning only counts real losses.
+export function deleteStalePendingMessages(): void {
+ // The table may not exist yet when the served migration files are
+ // older than the code (stale service-worker cache, unrebuilt dev
+ // environment) — create it rather than failing every budget load
+ db.execQuery(PENDING_MESSAGES_TABLE_SQL);
+ db.runQuery(`
+ DELETE FROM messages_pending
+ WHERE EXISTS
+ (SELECT 1 FROM messages_crdt c
+ WHERE c.dataset = messages_pending.dataset
+ AND c.row = messages_pending.row
+ AND c.column = messages_pending.column
+ AND c.timestamp > messages_pending.timestamp)
+ `);
+}
+
+// Applies sync messages that were deferred because they referenced
+// schema (or a value format) this client didn't have yet — see
+// `deferMessage` in ./index.ts. Called on budget load right after
+// migrations run, only when a sync server is configured. Messages that
+// still can't apply stay pending for a future upgrade and re-trigger
+// the update-required notification.
+export function replayPendingMessages(): void {
+ resetDeferredMessagesNotification();
+
+ let appliedCount = 0;
+ let newerVersionCount = 0;
+ let failedCount = 0;
+
+ const replay = () => {
+ deleteStalePendingMessages();
+
+ // Timestamp order gives last-write-wins per cell
+ const pending = db.runQuery<db.DbPendingMessage>(
+ 'SELECT * FROM messages_pending ORDER BY timestamp',
+ [],
+ true,
+ );
+ if (pending.length === 0) {
+ return;
+ }
+
+ function applyOne(msg: db.DbPendingMessage): ReplayOutcome {
+ let value;
+ try {
+ value = deserializeValue(msg.value);
+ } catch {
+ // A value format from an even newer version; decodes after the
+ // user updates
+ return { outcome: 'newer-version' };
+ }
+ try {
+ // The same per-cell write as `apply` in ./index.ts. Not an
+ // upsert: SQLite checks row-level CHECK constraints against the
+ // INSERT candidate row (sibling columns NULL) before ON
+ // CONFLICT fires, so an existing row must take the UPDATE path.
+ const updated = Number(
+ db.runQuery(
+ db.cache(
+ `UPDATE ${quoteSqlId(msg.dataset)} SET ${quoteSqlId(msg.column)} = ? WHERE id = ?`,
+ ),
+ [value, msg.row],
+ ).changes,
+ );
+ if (updated === 0) {
+ db.runQuery(
+ db.cache(
+ `INSERT INTO ${quoteSqlId(msg.dataset)} (id, ${quoteSqlId(msg.column)}) VALUES (?, ?)`,
+ ),
+ [msg.row, value],
+ );
+ }
+ return { outcome: 'applied' };
+ } catch (e) {
+ if (isMissingSchemaError(e)) {
+ // Still targets schema from an even newer version
+ return { outcome: 'newer-version' };
+ }
+ const error = e instanceof Error ? e : new Error(String(e));
+ if (!DETERMINISTIC_SQLITE_ERROR.test(error.message)) {
+ // Possibly transient — and errors like disk-full force SQLite
+ // to roll back the enclosing transaction, so continuing would
+ // run in autocommit. Abort; everything stays pending.
+ throw error;
+ }
+ return { outcome: 'failed', error };
+ }
+ }
+
+ function deletePending(msg: db.DbPendingMessage) {
+ db.runQuery(
+ db.cache(
+ 'DELETE FROM messages_pending WHERE dataset = ? AND row = ? AND column = ?',
+ ),
+ [msg.dataset, msg.row, msg.column],
+ );
+ }
+
+ function processPass(
+ msgs: db.DbPendingMessage[],
+ ): Array<{ msg: db.DbPendingMessage; error: Error }> {
+ const failed: Array<{ msg: db.DbPendingMessage; error: Error }> = [];
+ for (const msg of msgs) {
+ const result = applyOne(msg);
+ if (result.outcome === 'applied') {
+ appliedCount++;
+ deletePending(msg);
+ } else if (result.outcome === 'newer-version') {
+ newerVersionCount++;
+ } else {
+ failed.push({ msg, error: result.error });
+ }
+ }
+ return failed;
+ }
+
+ // Retry while passes make progress: a row-level constraint can need
+ // later cells to fill the row first, including through dependency
+ // chains. Terminates because each retained pass strictly shrinks
+ // the failure list.
+ let failed = processPass(pending);
+ while (failed.length > 0) {
+ const retried = processPass(failed.map(failure => failure.msg));
+ if (retried.length === failed.length) {
+ // No progress: these repeat identically on every load, so drop
+ // them. The values stay in messages_crdt history, but this
+ // device now diverges for those cells.
+ for (const { msg, error } of retried) {
+ captureException(error);
+ failedCount++;
+ deletePending(msg);
+ }
+ break;
+ }
+ failed = retried;
+ }
+
+ if (appliedCount > 0) {
+ // The values were written outside the normal sync pipeline, so
+ // force the spreadsheet cache to recompute
+ db.runQuery('DELETE FROM kvcache_key');
+ }
+ };
+
+ try {
+ db.transaction(replay);
+ } catch (e) {
+ // Possibly-transient abort: nothing was dropped, everything retries
+ // on the next load — telemetry only, no user notification
+ captureException(e instanceof Error ? e : new Error(String(e)));
+ return;
+ }
+
+ if (appliedCount > 0) {
+ logger.info(
+ `Applied ${appliedCount} sync message(s) deferred from a newer version`,
+ );
+ }
+ if (failedCount > 0) {
+ notifyDroppedMessages();
+ }
+ if (newerVersionCount > 0) {
+ notifyDeferredMessages();
+ }
+}
diff --git a/packages/loot-core/src/server/sync/reset.ts b/packages/loot-core/src/server/sync/reset.ts
index f1b183c011..e185e37c2b 100644
--- a/packages/loot-core/src/server/sync/reset.ts
+++ b/packages/loot-core/src/server/sync/reset.ts
@@ -7,6 +7,9 @@ import * as db from '#server/db';
import { runMutator } from '#server/mutators';
import * as prefs from '#server/prefs';
+import { deleteStalePendingMessages } from './replay';
+import { notifyDroppedMessages } from './utils';
+
export async function resetSync(
keyState?,
): Promise<{ error?: { reason: string; meta?: unknown } }> {
@@ -27,7 +30,20 @@ export async function resetSync(
return { error };
}
+ // Resetting discards deferred newer-version messages for every
+ // device; the user is warned afterwards via `notifyDroppedMessages`.
+ // TODO: pre-reset confirmation dialog
+ let discardedDeferredCount = 0;
+
await runMutator(async () => {
+ // Deferred messages belong to the discarded message log; replaying
+ // them later would resurrect rows hard-deleted below. Stale rows go
+ // first, uncounted, so only real losses feed the warning.
+ deleteStalePendingMessages();
+ discardedDeferredCount = Number(
+ db.runQuery('DELETE FROM messages_pending').changes,
+ );
+
// TOOD: We could automatically generate the list of tables to
// cleanup by looking at the schema
//
@@ -49,6 +65,10 @@ export async function resetSync(
await db.loadClock();
});
+ if (discardedDeferredCount > 0) {
+ notifyDroppedMessages();
+ }
+
await prefs.savePrefs({
groupId: null,
lastSyncedTimestamp: null,
diff --git a/packages/loot-core/src/server/sync/serialization.test.ts b/packages/loot-core/src/server/sync/serialization.test.ts
new file mode 100644
index 0000000000..433289931b
--- /dev/null
+++ b/packages/loot-core/src/server/sync/serialization.test.ts
@@ -0,0 +1,27 @@
+import { describe, expect, it } from 'vitest';
+
+import {
+ deserializeValue,
+ deserializeValueSafe,
+ isUnknownFormatValue,
+ serializeValue,
+} from './serialization';
+
+describe('sync value serialization', () => {
+ it('round-trips known value types', () => {
+ for (const value of [null, 0, -12.5, '', 'hello', 'N:tricky']) {
+ expect(deserializeValue(serializeValue(value))).toBe(value);
+ }
+ });
+
+ it('treats unknown and extended prefixes as unknown format', () => {
+ // 'N16:' and 'Sx:' guard against first-character prefix collisions
+ // with future tags — they must defer, not misdecode
+ for (const raw of ['B:true', 'N16:123', 'Sx:hi', 'x', '']) {
+ const value = deserializeValueSafe(raw);
+ expect(isUnknownFormatValue(value)).toBe(true);
+ // The original encoding survives for a future version to decode
+ expect(serializeValue(value)).toBe(raw);
+ }
+ });
+});
diff --git a/packages/loot-core/src/server/sync/serialization.ts b/packages/loot-core/src/server/sync/serialization.ts
new file mode 100644
index 0000000000..2f69901034
--- /dev/null
+++ b/packages/loot-core/src/server/sync/serialization.ts
@@ -0,0 +1,68 @@
+// Sync message values travel as tagged strings — "0:" for null, "N:"
+// for numbers, "S:" for strings; newer app versions may introduce more
+// prefixes.
+
+// A value serialized by a newer version of the app with a type prefix
+// this version doesn't know. Carried opaquely through the sync pipeline
+// so the message can still be acknowledged into the crdt log and
+// deferred with its original encoding intact (see `apply` in ./index.ts).
+export type UnknownFormatValue = { kind: 'unknown-format'; raw: string };
+
+export function isUnknownFormatValue(
+ value: unknown,
+): value is UnknownFormatValue {
+ return (
+ typeof value === 'object' &&
+ value !== null &&
+ 'kind' in value &&
+ value.kind === 'unknown-format'
+ );
+}
+
+export function serializeValue(
+ value: string | number | null | UnknownFormatValue,
+): string {
+ if (isUnknownFormatValue(value)) {
+ return value.raw;
+ } else if (value === null) {
+ return '0:';
+ } else if (typeof value === 'number') {
+ return 'N:' + value;
+ } else if (typeof value === 'string') {
+ return 'S:' + value;
+ }
+
+ throw new Error('Unserializable value type: ' + JSON.stringify(value));
+}
+
+export function deserializeValue(value: string): string | number | null {
+ // The full prefix must match, not just the first character — a future
+ // version's extended tag like "N16:..." must throw (and so defer via
+ // `deserializeValueSafe`), not silently misdecode
+ if (value[1] === ':') {
+ switch (value[0]) {
+ case '0':
+ return null;
+ case 'N':
+ return parseFloat(value.slice(2));
+ case 'S':
+ return value.slice(2);
+ default:
+ }
+ }
+
+ throw new Error('Invalid type key for value: ' + value);
+}
+
+// For inbound messages, which may come from a newer version of the app:
+// an unknown value format must not abort the sync — `apply` defers the
+// message instead, exactly like a missing table/column
+export function deserializeValueSafe(
+ value: string,
+): string | number | null | UnknownFormatValue {
+ try {
+ return deserializeValue(value);
+ } catch {
+ return { kind: 'unknown-format', raw: value };
+ }
+}
diff --git a/packages/loot-core/src/server/sync/sync.property.test.ts b/packages/loot-core/src/server/sync/sync.property.test.ts
index d9a68460ec..d6e5beed2d 100644
--- a/packages/loot-core/src/server/sync/sync.property.test.ts
+++ b/packages/loot-core/src/server/sync/sync.property.test.ts
@@ -9,6 +9,7 @@ import * as sheet from '#server/sheet';
import * as mockSyncServer from '#server/tests/mockSyncServer';
import * as encoder from './encoder';
+import { serializeValue } from './serialization';
import { isError } from './utils';
import * as sync from './index';
@@ -257,7 +258,7 @@ async function run(msgs) {
Timestamp.zero,
res.secondMessages.map(x => ({
...x,
- value: sync.serializeValue(x.value),
+ value: serializeValue(x.value),
timestamp: x.timestamp.toString(),
})),
),
@@ -276,7 +277,7 @@ async function run(msgs) {
Timestamp.zero,
res.secondMessages.map(x => ({
...x,
- value: sync.serializeValue(x.value),
+ value: serializeValue(x.value),
timestamp: x.timestamp.toString(),
})),
),
diff --git a/packages/loot-core/src/server/sync/utils.ts b/packages/loot-core/src/server/sync/utils.ts
index 802dcc1896..fd3c3d0171 100644
--- a/packages/loot-core/src/server/sync/utils.ts
+++ b/packages/loot-core/src/server/sync/utils.ts
@@ -1,3 +1,34 @@
+import { app } from '#server/main-app';
+
export function isError(value: unknown): value is { error: unknown } {
return (value as { error: unknown }).error !== undefined;
}
+
+// Sync messages carry table/column names as data, and a newer app
+// version may name a column with an SQL keyword — unquoted, that raises
+// a syntax error instead of the missing-column error the deferral path
+// relies on.
+export function quoteSqlId(name: string): string {
+ return '"' + name.replace(/"/g, '""') + '"';
+}
+
+// Notify once per loaded budget (reset on budget load); the client-side
+// notification id additionally dedupes while it's displayed
+let hasNotifiedDeferredMessages = false;
+
+export function resetDeferredMessagesNotification() {
+ hasNotifiedDeferredMessages = false;
+}
+
+export function notifyDeferredMessages() {
+ if (!hasNotifiedDeferredMessages) {
+ hasNotifiedDeferredMessages = true;
+ app.events.emit('sync', { type: 'deferred-messages' });
+ }
+}
+
+// A deferred change was dropped or discarded. Emitted per incident; the
+// client-side notification id dedupes while one is displayed.
+export function notifyDroppedMessages() {
+ app.events.emit('sync', { type: 'dropped-messages' });
+}
diff --git a/packages/loot-core/src/shared/errors.ts b/packages/loot-core/src/shared/errors.ts
index 237654831e..7e9d45f0eb 100644
--- a/packages/loot-core/src/shared/errors.ts
+++ b/packages/loot-core/src/shared/errors.ts
@@ -54,7 +54,18 @@ function getUnsafeZipError(meta: UnsafeZipMeta): string {
// plain English strings. User-facing, translated equivalents live in the
// desktop-client (`src/util/error.ts`).
-function isDatabaseSchemaMismatch(meta?: unknown): boolean {
+// A SQLite error meaning a query referenced a table or column that
+// doesn't exist in this database's schema. SQLite reports a missing
+// column as "no such column" in UPDATE/SELECT statements but as "has no
+// column named" in INSERT statements.
+export function isMissingSchemaError(error: unknown): boolean {
+ return (
+ error instanceof Error &&
+ /no such (table|column)|has no column named/i.test(error.message)
+ );
+}
+
+export function isDatabaseSchemaMismatch(meta?: unknown): boolean {
if (
meta &&
typeof meta === 'object' &&
@@ -64,7 +75,10 @@ function isDatabaseSchemaMismatch(meta?: unknown): boolean {
'message' in meta.error &&
typeof meta.error.message === 'string'
) {
- return /no such (column|table)/i.test(meta.error.message);
+ // Narrower than `isMissingSchemaError`: sync defers the INSERT form
+ // ("has no column named"), so one reaching the UI means local
+ // damage, not version skew
+ return /no such (table|column)/i.test(meta.error.message);
}
return false;
}
diff --git a/packages/loot-core/src/types/server-events.ts b/packages/loot-core/src/types/server-events.ts
index 4956731c3e..b80534606c 100644
--- a/packages/loot-core/src/types/server-events.ts
+++ b/packages/loot-core/src/types/server-events.ts
@@ -41,6 +41,17 @@ type SyncEvent = {
| {
type: 'unauthorized';
}
+ | {
+ // Some received messages were written by a newer version of the
+ // app and can't be shown until this client updates
+ type: 'deferred-messages';
+ }
+ | {
+ // A deferred change from another device was discarded — it could
+ // never be applied, or a sync reset made this file the source of
+ // truth without it
+ type: 'dropped-messages';
+ }
);
type BackupUpdatedEvent = Backup[];
diff --git a/upcoming-release-notes/cross-version-sync-compatibility.md b/upcoming-release-notes/cross-version-sync-compatibility.md
new file mode 100644
index 0000000000..a1933f59d8
--- /dev/null
+++ b/upcoming-release-notes/cross-version-sync-compatibility.md
@@ -0,0 +1,6 @@
+---
+category: Enhancements
+authors: [MatissJanis]
+---
+
+Keep syncing working when other devices run a newer version of Actual — their changes appear once you update the app