Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: preserve Signal message rowids in the snapshot (#11149)

Adam Eivy83cfca6f6942
diff --git a/server/services/signalSnapshot.js b/server/services/signalSnapshot.js
index 0f1203f7d1..e65be2696e 100644
--- a/server/services/signalSnapshot.js
+++ b/server/services/signalSnapshot.js
@@ -9,17 +9,16 @@
* Capture opens the source with the bundled SQLCipher driver
* (`better-sqlite3-multiple-ciphers`, SQLCipher 4 compatibility, raw 64-hex
* key — the same key form Signal uses). One connection holds a deferred read
- * transaction so a checkpoint cannot discard frames mid-capture. A second
- * read-only connection materializes that committed view with SQLite
- * `VACUUM INTO`, which includes WAL frames and works against a read-only
- * source. `backup()` is not used: this driver rejects an encrypted source
- * backing up into a plaintext target, and it does not ship `sqlcipher_export`.
+ * transaction so a checkpoint cannot discard frames mid-capture. Ordinary
+ * tables are then copied into a new plaintext database, including each
+ * source `rowid`. That rowid is the ingestion cursor, and SQLite may
+ * renumber it during `VACUUM` for tables without an INTEGER PRIMARY KEY.
+ * `backup()` is not used either: this driver rejects an encrypted source
+ * backing up into a new file, and it does not ship `sqlcipher_export`.
*
- * The private copy is then `rekey`ed with an empty key, which yields a
- * standard plaintext SQLite file. Rekey runs only on that copy. The source is
- * opened `readonly` and this module never issues `wal_checkpoint`,
- * `journal_mode`, or any other write against it. The WAL index may record a
- * reader mark; the database file and the WAL must stay byte-identical.
+ * The source is opened `readonly`. This module never issues `wal_checkpoint`,
+ * `journal_mode`, `VACUUM`, or any other write against it. The WAL index may
+ * record a reader mark; the database file and the WAL must stay byte-identical.
*
* If the driver cannot be loaded, or the snapshot is not a single integrity-
* clean committed view, the result is an explicit failure. Callers leave the
@@ -72,8 +71,46 @@ function resolveConstructor(loaded) {
return null;
}
-function sqlLiteral(value) {
- return `'${String(value).replaceAll("'", "''")}'`;
+function quoteIdent(name) {
+ return `"${String(name).replaceAll('"', '""')}"`;
+}
+
+// INTEGER PRIMARY KEY is the rowid alias and survives a normal INSERT.
+// Any other rowid is not, and the sync cursor needs the source value.
+function isIntegerPrimaryKey(info) {
+ const pk = info.filter((column) => column.pk > 0);
+ return pk.length === 1 && String(pk[0].type).trim().toLowerCase() === 'integer';
+}
+
+function copyOrdinaryTables(source, dest) {
+ const tables = source.prepare(`
+ SELECT name, sql FROM sqlite_master
+ WHERE type = 'table' AND sql IS NOT NULL AND name NOT LIKE 'sqlite_%'
+ ORDER BY name
+ `).all();
+ dest.pragma('foreign_keys = OFF');
+ const copyAll = dest.transaction(() => {
+ for (const table of tables) {
+ if (/^\s*create\s+virtual\b/i.test(table.sql)) continue;
+ dest.exec(table.sql);
+ const info = source.prepare(`PRAGMA table_xinfo(${quoteIdent(table.name)})`).all()
+ .filter((column) => !column.hidden);
+ const names = info.map((column) => quoteIdent(column.name));
+ if (names.length === 0) continue;
+ // WITHOUT ROWID tables have no rowid to preserve. Every other table
+ // copies the source rowid unless an INTEGER PRIMARY KEY already is one.
+ const withoutRowid = /\bwithout\s+rowid\b/i.test(table.sql);
+ const selectCols = withoutRowid || isIntegerPrimaryKey(info) ? names : ['rowid', ...names];
+ const insert = dest.prepare(
+ `INSERT INTO ${quoteIdent(table.name)} (${selectCols.join(', ')}) VALUES (${selectCols.map(() => '?').join(', ')})`,
+ );
+ const rows = source.prepare(
+ `SELECT ${selectCols.join(', ')} FROM ${quoteIdent(table.name)}`,
+ ).raw(true);
+ for (const row of rows.iterate()) insert.run(...row);
+ }
+ });
+ copyAll();
}
function applyKey(db, key) {
@@ -150,33 +187,23 @@ export async function captureConsistentSnapshot(sourcePath, key, options = {}) {
const tmp = await mkdtemp(join(options.tmpRoot || tmpdir(), 'portos-signal-'));
const plainPath = join(tmp, 'db.plain.sqlite');
let holder;
- let exporter;
- let owned;
let plain;
try {
holder = openReadonly(Database, sourcePath);
applyKey(holder, key);
holder.exec('BEGIN');
- // Establishes the read mark and rejects a wrong key before any export.
+ // Establishes the read mark and rejects a wrong key before any copy.
holder.prepare('SELECT count(*) AS n FROM sqlite_master').get();
if (options.onReadLocked) await options.onReadLocked();
- exporter = openReadonly(Database, sourcePath);
- applyKey(exporter, key);
- exporter.exec(`VACUUM INTO ${sqlLiteral(plainPath)}`);
- exporter.close();
- exporter = null;
+ plain = new Database(plainPath);
+ copyOrdinaryTables(holder, plain);
+ plain.close();
+ plain = null;
holder.exec('ROLLBACK');
holder.close();
holder = null;
- owned = new Database(plainPath, { fileMustExist: true });
- applyKey(owned, key);
- owned.prepare('SELECT count(*) AS n FROM sqlite_master').get();
- owned.pragma(`rekey=''`);
- owned.close();
- owned = null;
-
const header = await readFile(plainPath);
if (header.length < 16 || header.subarray(0, 16).toString('latin1') !== SQLITE_MAGIC) {
await rm(tmp, { recursive: true, force: true });
@@ -195,8 +222,6 @@ export async function captureConsistentSnapshot(sourcePath, key, options = {}) {
return { ok: true, plaintextPath: plainPath, tmpDir: tmp };
} catch (err) {
await closeQuietly(plain);
- await closeQuietly(owned);
- await closeQuietly(exporter);
await closeQuietly(holder);
await rm(tmp, { recursive: true, force: true }).catch(() => {});
return classifyDriverError(err);
diff --git a/server/services/signalSnapshot.test.js b/server/services/signalSnapshot.test.js
index 7b181d887d..25eac81246 100644
--- a/server/services/signalSnapshot.test.js
+++ b/server/services/signalSnapshot.test.js
@@ -135,6 +135,45 @@ describe('captureConsistentSnapshot', () => {
}
});
+ it('keeps source rowids after a delete so a newer WAL row stays above an old cursor', async () => {
+ const dir = scratch();
+ const src = join(dir, 'db.sqlite');
+ const key = randomBytes(32).toString('hex');
+ const writer = new Database(src);
+ applyKey(writer, key);
+ writer.pragma('journal_mode = WAL');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.exec(`
+ CREATE TABLE messages (id TEXT, body TEXT);
+ CREATE TABLE lookup (name TEXT PRIMARY KEY, value TEXT) WITHOUT ROWID;
+ INSERT INTO lookup VALUES ('k', 'v');
+ INSERT INTO messages(rowid, id, body) VALUES (1, 'm1', 'old'), (2, 'm2', 'kept');
+ `);
+ writer.pragma('wal_checkpoint(TRUNCATE)');
+ writer.exec(`
+ DELETE FROM messages WHERE id = 'm1';
+ INSERT INTO messages(rowid, id, body) VALUES (4, 'm4', 'wal-only');
+ `);
+ try {
+ const snap = await captureConsistentSnapshot(src, key, { tmpRoot: dir });
+ expect(snap.ok, snap.error).toBe(true);
+ const plain = new DatabaseSync(snap.plaintextPath, { readOnly: true });
+ try {
+ const rows = plain.prepare('SELECT rowid, id FROM messages ORDER BY rowid').all()
+ .map((row) => ({ rowid: Number(row.rowid), id: row.id }));
+ expect(rows).toEqual([
+ { rowid: 2, id: 'm2' },
+ { rowid: 4, id: 'm4' },
+ ]);
+ expect(plain.prepare('SELECT name, value FROM lookup').all()).toEqual([{ name: 'k', value: 'v' }]);
+ } finally {
+ plain.close();
+ }
+ } finally {
+ writer.close();
+ }
+ });
+
it('reports setup failure and leaves no temp copy when the driver cannot load', async () => {
const dir = scratch();
const result = await captureConsistentSnapshot(join(dir, 'missing.sqlite'), 'ab'.repeat(32), {
diff --git a/server/services/signalSync.wal.test.js b/server/services/signalSync.wal.test.js
index 077429afa1..fa5feb8e92 100644
--- a/server/services/signalSync.wal.test.js
+++ b/server/services/signalSync.wal.test.js
@@ -126,6 +126,47 @@ describe('runSync WAL snapshot', () => {
expect(recordEventsMock.mock.calls[1][0]).toEqual([]);
});
+ it('ingests a WAL message inserted after an older row was deleted', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'signal-sync-rowid-'));
+ const path = join(dir, 'db.sqlite');
+ const local = new Database(path);
+ local.pragma(`cipher='sqlcipher'`);
+ local.pragma('legacy=4');
+ local.pragma(`key="x'${key}'"`);
+ local.pragma('journal_mode = WAL');
+ local.pragma('wal_autocheckpoint = 0');
+ local.exec(`
+ CREATE TABLE conversations (
+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT
+ );
+ CREATE TABLE messages (
+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT
+ );
+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');
+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'first');
+ INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'second');
+ `);
+ const previous = process.env.SIGNAL_DB_PATH;
+ process.env.SIGNAL_DB_PATH = path;
+ try {
+ const first = await runSync();
+ expect(first.ok, first.error).toBe(true);
+ expect(first.cursorRowid).toBe(2);
+ local.exec(`DELETE FROM messages WHERE id = 'm1'`);
+ local.exec(`INSERT INTO messages VALUES ('m3', 'c1', 'incoming', 1700000002000, 1700000002000, 'after-delete')`);
+ recordEventsMock.mockClear();
+ const second = await runSync();
+ expect(second.ok, second.error).toBe(true);
+ expect(second.scanned).toBe(1);
+ expect(second.cursorRowid).toBeGreaterThan(first.cursorRowid);
+ expect(recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey)).toEqual(['signal:m3']);
+ } finally {
+ process.env.SIGNAL_DB_PATH = previous;
+ local.close();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
it('leaves the cursor and activity calls untouched when the key is rejected', async () => {
const first = await runSync();
expect(first.ok, first.error).toBe(true);