Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
fix: preserve Signal message rowids in the snapshot (#11149)
Adam Eivy83cfca6f6942
diff --git a/server/services/signalSnapshot.js b/server/services/signalSnapshot.jsindex 0f1203f7d1..e65be2696e 100644--- a/server/services/signalSnapshot.js+++ b/server/services/signalSnapshot.js@@ -9,17 +9,16 @@* Capture opens the source with the bundled SQLCipher driver* (`better-sqlite3-multiple-ciphers`, SQLCipher 4 compatibility, raw 64-hex* key — the same key form Signal uses). One connection holds a deferred read- * transaction so a checkpoint cannot discard frames mid-capture. A second- * read-only connection materializes that committed view with SQLite- * `VACUUM INTO`, which includes WAL frames and works against a read-only- * source. `backup()` is not used: this driver rejects an encrypted source- * backing up into a plaintext target, and it does not ship `sqlcipher_export`.+ * transaction so a checkpoint cannot discard frames mid-capture. Ordinary+ * tables are then copied into a new plaintext database, including each+ * source `rowid`. That rowid is the ingestion cursor, and SQLite may+ * renumber it during `VACUUM` for tables without an INTEGER PRIMARY KEY.+ * `backup()` is not used either: this driver rejects an encrypted source+ * backing up into a new file, and it does not ship `sqlcipher_export`.*- * The private copy is then `rekey`ed with an empty key, which yields a- * standard plaintext SQLite file. Rekey runs only on that copy. The source is- * opened `readonly` and this module never issues `wal_checkpoint`,- * `journal_mode`, or any other write against it. The WAL index may record a- * reader mark; the database file and the WAL must stay byte-identical.+ * The source is opened `readonly`. This module never issues `wal_checkpoint`,+ * `journal_mode`, `VACUUM`, or any other write against it. The WAL index may+ * record a reader mark; the database file and the WAL must stay byte-identical.** If the driver cannot be loaded, or the snapshot is not a single integrity-* clean committed view, the result is an explicit failure. Callers leave the@@ -72,8 +71,46 @@ function resolveConstructor(loaded) {return null;}-function sqlLiteral(value) {- return `'${String(value).replaceAll("'", "''")}'`;+function quoteIdent(name) {+ return `"${String(name).replaceAll('"', '""')}"`;+}++// INTEGER PRIMARY KEY is the rowid alias and survives a normal INSERT.+// Any other rowid is not, and the sync cursor needs the source value.+function isIntegerPrimaryKey(info) {+ const pk = info.filter((column) => column.pk > 0);+ return pk.length === 1 && String(pk[0].type).trim().toLowerCase() === 'integer';+}++function copyOrdinaryTables(source, dest) {+ const tables = source.prepare(`+ SELECT name, sql FROM sqlite_master+ WHERE type = 'table' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%'+ ORDER BY name+ `).all();+ dest.pragma('foreign_keys = OFF');+ const copyAll = dest.transaction(() => {+ for (const table of tables) {+ if (/^\s*create\s+virtual\b/i.test(table.sql)) continue;+ dest.exec(table.sql);+ const info = source.prepare(`PRAGMA table_xinfo(${quoteIdent(table.name)})`).all()+ .filter((column) => !column.hidden);+ const names = info.map((column) => quoteIdent(column.name));+ if (names.length === 0) continue;+ // WITHOUT ROWID tables have no rowid to preserve. Every other table+ // copies the source rowid unless an INTEGER PRIMARY KEY already is one.+ const withoutRowid = /\bwithout\s+rowid\b/i.test(table.sql);+ const selectCols = withoutRowid || isIntegerPrimaryKey(info) ? names : ['rowid', ...names];+ const insert = dest.prepare(+ `INSERT INTO ${quoteIdent(table.name)} (${selectCols.join(', ')}) VALUES (${selectCols.map(() => '?').join(', ')})`,+ );+ const rows = source.prepare(+ `SELECT ${selectCols.join(', ')} FROM ${quoteIdent(table.name)}`,+ ).raw(true);+ for (const row of rows.iterate()) insert.run(...row);+ }+ });+ copyAll();}function applyKey(db, key) {@@ -150,33 +187,23 @@ export async function captureConsistentSnapshot(sourcePath, key, options = {}) {const tmp = await mkdtemp(join(options.tmpRoot || tmpdir(), 'portos-signal-'));const plainPath = join(tmp, 'db.plain.sqlite');let holder;- let exporter;- let owned;let plain;try {holder = openReadonly(Database, sourcePath);applyKey(holder, key);holder.exec('BEGIN');- // Establishes the read mark and rejects a wrong key before any export.+ // Establishes the read mark and rejects a wrong key before any copy.holder.prepare('SELECT count(*) AS n FROM sqlite_master').get();if (options.onReadLocked) await options.onReadLocked();- exporter = openReadonly(Database, sourcePath);- applyKey(exporter, key);- exporter.exec(`VACUUM INTO ${sqlLiteral(plainPath)}`);- exporter.close();- exporter = null;+ plain = new Database(plainPath);+ copyOrdinaryTables(holder, plain);+ plain.close();+ plain = null;holder.exec('ROLLBACK');holder.close();holder = null;- owned = new Database(plainPath, { fileMustExist: true });- applyKey(owned, key);- owned.prepare('SELECT count(*) AS n FROM sqlite_master').get();- owned.pragma(`rekey=''`);- owned.close();- owned = null;-const header = await readFile(plainPath);if (header.length < 16 || header.subarray(0, 16).toString('latin1') !== SQLITE_MAGIC) {await rm(tmp, { recursive: true, force: true });@@ -195,8 +222,6 @@ export async function captureConsistentSnapshot(sourcePath, key, options = {}) {return { ok: true, plaintextPath: plainPath, tmpDir: tmp };} catch (err) {await closeQuietly(plain);- await closeQuietly(owned);- await closeQuietly(exporter);await closeQuietly(holder);await rm(tmp, { recursive: true, force: true }).catch(() => {});return classifyDriverError(err);diff --git a/server/services/signalSnapshot.test.js b/server/services/signalSnapshot.test.jsindex 7b181d887d..25eac81246 100644--- a/server/services/signalSnapshot.test.js+++ b/server/services/signalSnapshot.test.js@@ -135,6 +135,45 @@ describe('captureConsistentSnapshot', () => {}});+ it('keeps source rowids after a delete so a newer WAL row stays above an old cursor', async () => {+ const dir = scratch();+ const src = join(dir, 'db.sqlite');+ const key = randomBytes(32).toString('hex');+ const writer = new Database(src);+ applyKey(writer, key);+ writer.pragma('journal_mode = WAL');+ writer.pragma('wal_autocheckpoint = 0');+ writer.exec(`+ CREATE TABLE messages (id TEXT, body TEXT);+ CREATE TABLE lookup (name TEXT PRIMARY KEY, value TEXT) WITHOUT ROWID;+ INSERT INTO lookup VALUES ('k', 'v');+ INSERT INTO messages(rowid, id, body) VALUES (1, 'm1', 'old'), (2, 'm2', 'kept');+ `);+ writer.pragma('wal_checkpoint(TRUNCATE)');+ writer.exec(`+ DELETE FROM messages WHERE id = 'm1';+ INSERT INTO messages(rowid, id, body) VALUES (4, 'm4', 'wal-only');+ `);+ try {+ const snap = await captureConsistentSnapshot(src, key, { tmpRoot: dir });+ expect(snap.ok, snap.error).toBe(true);+ const plain = new DatabaseSync(snap.plaintextPath, { readOnly: true });+ try {+ const rows = plain.prepare('SELECT rowid, id FROM messages ORDER BY rowid').all()+ .map((row) => ({ rowid: Number(row.rowid), id: row.id }));+ expect(rows).toEqual([+ { rowid: 2, id: 'm2' },+ { rowid: 4, id: 'm4' },+ ]);+ expect(plain.prepare('SELECT name, value FROM lookup').all()).toEqual([{ name: 'k', value: 'v' }]);+ } finally {+ plain.close();+ }+ } finally {+ writer.close();+ }+ });+it('reports setup failure and leaves no temp copy when the driver cannot load', async () => {const dir = scratch();const result = await captureConsistentSnapshot(join(dir, 'missing.sqlite'), 'ab'.repeat(32), {diff --git a/server/services/signalSync.wal.test.js b/server/services/signalSync.wal.test.jsindex 077429afa1..fa5feb8e92 100644--- a/server/services/signalSync.wal.test.js+++ b/server/services/signalSync.wal.test.js@@ -126,6 +126,47 @@ describe('runSync WAL snapshot', () => {expect(recordEventsMock.mock.calls[1][0]).toEqual([]);});+ it('ingests a WAL message inserted after an older row was deleted', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'signal-sync-rowid-'));+ const path = join(dir, 'db.sqlite');+ const local = new Database(path);+ local.pragma(`cipher='sqlcipher'`);+ local.pragma('legacy=4');+ local.pragma(`key="x'${key}'"`);+ local.pragma('journal_mode = WAL');+ local.pragma('wal_autocheckpoint = 0');+ local.exec(`+ CREATE TABLE conversations (+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT+ );+ CREATE TABLE messages (+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT+ );+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'first');+ INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'second');+ `);+ const previous = process.env.SIGNAL_DB_PATH;+ process.env.SIGNAL_DB_PATH = path;+ try {+ const first = await runSync();+ expect(first.ok, first.error).toBe(true);+ expect(first.cursorRowid).toBe(2);+ local.exec(`DELETE FROM messages WHERE id = 'm1'`);+ local.exec(`INSERT INTO messages VALUES ('m3', 'c1', 'incoming', 1700000002000, 1700000002000, 'after-delete')`);+ recordEventsMock.mockClear();+ const second = await runSync();+ expect(second.ok, second.error).toBe(true);+ expect(second.scanned).toBe(1);+ expect(second.cursorRowid).toBeGreaterThan(first.cursorRowid);+ expect(recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey)).toEqual(['signal:m3']);+ } finally {+ process.env.SIGNAL_DB_PATH = previous;+ local.close();+ rmSync(dir, { recursive: true, force: true });+ }+ });+it('leaves the cursor and activity calls untouched when the key is rejected', async () => {const first = await runSync();expect(first.ok, first.error).toBe(true);