Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

merge origin/main and keep the Signal WAL snapshot (#11149)

Adam Eivyd90bb702e0a6
diff --cc server/services/signalSync.runSync.test.js
index 0000000000,3a233f95a1..60fed5e9f4
mode 000000,100644..100644
--- a/server/services/signalSync.runSync.test.js
+++ b/server/services/signalSync.runSync.test.js
@@@ -1,0 -1,160 +1,166 @@@
+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+ import { DatabaseSync } from 'node:sqlite';
+ import crypto from 'node:crypto';
+ import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';
+ import { join } from 'node:path';
+ import { createTempDataRoot } from '../lib/mockPathsDataRoot.js';
+ import { decryptSqlcipherDatabase, deriveSqlcipherKeys, sqlcipherPageHmac,
+ SQLCIPHER_PAGE_SIZE, SQLCIPHER_RESERVE_BYTES, SQLCIPHER_SALT_BYTES,
+ SQLCIPHER_IV_BYTES } from '../lib/signalCrypto.js';
+
+ const mocks = vi.hoisted(() => ({
+ root: '', snapshots: [], recordEvents: vi.fn(), autoLogTouchpoints: vi.fn(),
+ }));
+ vi.mock('../lib/fileUtils.js', async (importOriginal) => ({
+ ...await importOriginal(),
+ dataPath: (...parts) => join(mocks.root, ...parts),
+ }));
+ vi.mock('node:fs/promises', async (importOriginal) => {
+ const actual = await importOriginal();
+ return { ...actual, mkdtemp: async (...args) => {
+ const path = await actual.mkdtemp(...args);
+ mocks.snapshots.push(path);
+ return path;
+ } };
+ });
+ vi.mock('./settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));
+ vi.mock('./userTimezone.js', () => ({ getUserTimezone: vi.fn(async () => 'UTC') }));
+ vi.mock('./humanActivity.js', async (importOriginal) => ({
+ ...await importOriginal(), recordEvents: mocks.recordEvents,
+ }));
+ vi.mock('./tribe.js', () => ({ autoLogTouchpoints: mocks.autoLogTouchpoints }));
+
+ import { checkSetup, runSync, readSyncState } from './signalSync.js';
+
+ const KEY = 'ab'.repeat(32); // invented fixture key; never reads the keychain
+ const BODY = 'x'.repeat(512);
+ let encrypted;
+ let plaintext;
+
+ // Build a genuinely queryable SQLite database with SQLCipher's 80-byte reserve.
+ // An empty SQLite page has no cells to relocate: changing its reserve and empty
+ // content-area pointer before creating tables makes SQLite allocate all later
+ // pages with the correct usable size. No external SQLCipher/Python dependency.
+ function createFixture() {
+ const path = join(mocks.root, 'fixture.sqlite');
+ let db = new DatabaseSync(path);
+ db.exec('PRAGMA page_size=4096; VACUUM');
+ db.close();
+ const empty = readFileSync(path);
+ empty[20] = SQLCIPHER_RESERVE_BYTES;
+ empty.writeUInt16BE(SQLCIPHER_PAGE_SIZE - SQLCIPHER_RESERVE_BYTES, 105);
+ writeFileSync(path, empty);
+ db = new DatabaseSync(path);
+ db.exec(`
+ CREATE TABLE conversations(id TEXT, e164 TEXT, type TEXT, name TEXT, profileName TEXT);
+ CREATE TABLE messages(id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT);
+ INSERT INTO conversations VALUES ('fixture-chat', '+15550000000', 'private', 'Example Contact', '');
+ `);
+ db.prepare('INSERT INTO messages(rowid, id, conversationId, type, sent_at, body) VALUES (?,?,?,?,?,?)')
+ .run(8, 'fixture-message', 'fixture-chat', 'incoming', Date.UTC(2024, 5, 1, 12), BODY);
+ expect(db.prepare('PRAGMA integrity_check').get().integrity_check).toBe('ok');
+ db.close();
+ return readFileSync(path);
+ }
+
+ // Encryption uses the production KDF pinned by signalCrypto's independent
+ // known-answer vectors; this fixture exercises service wiring, not KDF accuracy.
+ function encrypt(plain) {
+ const salt = Buffer.alloc(SQLCIPHER_SALT_BYTES, 7);
+ const { encKey, hmacKey } = deriveSqlcipherKeys(KEY, salt);
+ const out = Buffer.alloc(plain.length);
+ for (let page = 1; page <= plain.length / SQLCIPHER_PAGE_SIZE; page += 1) {
+ const start = (page - 1) * SQLCIPHER_PAGE_SIZE;
+ const bodyStart = start + (page === 1 ? SQLCIPHER_SALT_BYTES : 0);
+ const end = start + SQLCIPHER_PAGE_SIZE - SQLCIPHER_RESERVE_BYTES;
+ const iv = Buffer.alloc(SQLCIPHER_IV_BYTES, page);
+ const cipher = crypto.createCipheriv('aes-256-cbc', encKey, iv);
+ cipher.setAutoPadding(false);
+ const ciphertext = Buffer.concat([cipher.update(plain.subarray(bodyStart, end)), cipher.final()]);
+ if (page === 1) salt.copy(out, start);
+ ciphertext.copy(out, bodyStart);
+ iv.copy(out, end);
+ sqlcipherPageHmac(hmacKey, Buffer.concat([ciphertext, iv]), page).copy(out, end + SQLCIPHER_IV_BYTES);
+ }
+ return out;
+ }
+
+ beforeEach(() => {
+ mocks.root = createTempDataRoot('signal-auth-test-');
+ mocks.snapshots = [];
+ mkdirSync(join(mocks.root, 'sql'));
+ writeFileSync(join(mocks.root, 'config.json'), JSON.stringify({ key: KEY }));
+ vi.stubEnv('SIGNAL_DIR', mocks.root);
+ vi.stubEnv('SIGNAL_CONFIG_PATH', join(mocks.root, 'config.json'));
+ vi.stubEnv('SIGNAL_DB_PATH', join(mocks.root, 'sql', 'db.sqlite'));
+ plaintext = createFixture();
+ encrypted = encrypt(plaintext);
+ writeFileSync(process.env.SIGNAL_DB_PATH, encrypted);
+ mocks.recordEvents.mockReset().mockImplementation(async (events) => ({ recorded: events.length, skipped: 0 }));
+ mocks.autoLogTouchpoints.mockReset().mockResolvedValue({ created: 1, matched: 1 });
+ });
+ afterEach(() => {
+ vi.unstubAllEnvs();
+ rmSync(mocks.root, { recursive: true, force: true });
+ vi.restoreAllMocks();
+ });
+
+ function expectSnapshotsRemoved() {
+ expect(mocks.snapshots.length).toBeGreaterThan(0);
+ expect(mocks.snapshots.every((path) => !existsSync(path))).toBe(true);
+ }
+
+ describe('Signal authenticated snapshot ingestion', () => {
+ it('accepts a multipage database and does not reingest messages on replay', async () => {
+ expect(plaintext.length).toBeGreaterThan(SQLCIPHER_PAGE_SIZE);
+ expect(await checkSetup()).toMatchObject({ ok: true, messageCount: 1 });
+ expect(await runSync()).toMatchObject({ ok: true, recorded: 1, cursorRowid: 8 });
+ expect(mocks.recordEvents.mock.calls[0][0]).toMatchObject([
+ { dedupeKey: 'signal:fixture-message', kind: 'message.received' },
+ ]);
+ expect(mocks.autoLogTouchpoints.mock.calls[0][0]).toHaveLength(1);
+ expect(await runSync()).toMatchObject({ ok: true, recorded: 0, scanned: 0, cursorRowid: 8 });
+ expect(mocks.recordEvents.mock.calls[1][0]).toEqual([]);
+ expect((await readSyncState()).cursorRowid).toBe(8);
+ expectSnapshotsRemoved();
+ });
+
+ it('refuses queryable later-page body corruption before writes and preserves the prior cursor', async () => {
+ const statePath = join(mocks.root, 'signal-sync-state.json');
+ const state = JSON.stringify({ cursorRowid: 7, lastRunAt: '2024-01-01T00:00:00Z', lastResult: { ok: true } });
+ writeFileSync(statePath, state);
+ // Flip a ciphertext bit well inside the long body, beyond page 1. CBC
+ // damages text bytes while leaving the cell header and B-tree readable.
+ const bodyOffset = plaintext.indexOf(Buffer.from(BODY));
+ expect(bodyOffset).toBeGreaterThan(SQLCIPHER_PAGE_SIZE);
+ encrypted[bodyOffset + 128] ^= 1;
+ writeFileSync(process.env.SIGNAL_DB_PATH, encrypted);
+ const partial = decryptSqlcipherDatabase(encrypted, KEY, { verify: 'first' });
+ expect(partial.ok).toBe(true);
+ const readablePath = join(mocks.root, 'damaged-readable.sqlite');
+ writeFileSync(readablePath, partial.plaintext);
+ const readable = new DatabaseSync(readablePath, { readOnly: true });
+ try {
+ expect(readable.prepare('PRAGMA integrity_check').get().integrity_check).toBe('ok');
+ expect(readable.prepare('SELECT body FROM messages').get().body).not.toBe(BODY);
+ } finally { readable.close(); }
+
++ // verify:'first' still produces a readable database whose integrity_check
++ // passes and whose body is wrong. The SQLCipher snapshot must refuse that
++ // file (the driver reports the damaged page as malformed) and must not
++ // ingest it or move the cursor. The old main-file copy named this as a
++ // page HMAC mismatch; the snapshot failure is the same refusal.
+ vi.spyOn(console, 'error').mockImplementation(() => {});
+ for (const result of [await checkSetup(), await runSync()]) {
+ expect(result.ok).toBe(false);
- expect(result.error).toMatch(/Page-[2-9]\d* HMAC mismatch/);
- expect(result.remediation).toMatch(/Close Signal Desktop and retry/);
++ expect(['auth', 'snapshot']).toContain(result.reason);
++ expect(result.error).toMatch(/malformed|hmac|decryption failed|integrity/i);
++ expect(result.remediation).toMatch(/left unchanged|Nothing was ingested|not yet supported/);
+ }
+ expect(mocks.recordEvents).not.toHaveBeenCalled();
+ expect(mocks.autoLogTouchpoints).not.toHaveBeenCalled();
+ expect(readFileSync(statePath, 'utf8')).toBe(state);
+ expect((await readSyncState()).cursorRowid).toBe(7);
+ expectSnapshotsRemoved();
+ });
+ });
diff --cc server/services/signalSync.wal.test.js
index fa5feb8e92,0000000000..3d9848cd8f
mode 100644,000000..100644
--- a/server/services/signalSync.wal.test.js
+++ b/server/services/signalSync.wal.test.js
@@@ -1,186 -1,0 +1,201 @@@
+/**
+ * Signal sync consumes the consistent snapshot (#11149): both the checkpointed
+ * message and the committed WAL-only message are ingested, a second pass is a
+ * no-op, and an authentication failure leaves the cursor and activity calls
+ * untouched. The database is an encrypted fixture, not a live Signal profile.
+ */
+import { createRequire } from 'node:module';
+import { createHash } from 'node:crypto';
- import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
++import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+
- const { recordEventsMock, autoLogTouchpointsMock } = vi.hoisted(() => ({
++const { recordEventsMock, autoLogTouchpointsMock, snapshotDirs } = vi.hoisted(() => ({
+ recordEventsMock: vi.fn(),
+ autoLogTouchpointsMock: vi.fn(),
++ snapshotDirs: [],
+}));
+
+const dataRoot = vi.hoisted(() => {
+ const { mkdtempSync: mkdtemp } = require('node:fs');
+ const { tmpdir: osTmp } = require('node:os');
+ const { join: joinPath } = require('node:path');
+ return mkdtemp(joinPath(osTmp(), 'signal-sync-wal-'));
+});
+
+vi.mock('../lib/fileUtils.js', async () => {
+ const actual = await vi.importActual('../lib/fileUtils.js');
+ const { join: joinPath } = await import('node:path');
+ return {
+ ...actual,
+ dataPath: (...segments) => joinPath(dataRoot, ...segments),
+ };
+});
+
+vi.mock('./humanActivity.js', async () => {
+ const actual = await vi.importActual('./humanActivity.js');
+ return { ...actual, recordEvents: recordEventsMock };
+});
+
+vi.mock('./tribe.js', () => ({
+ autoLogTouchpoints: autoLogTouchpointsMock,
+}));
+
++// Only this file's snapshot dirs. A global tmpdir scan races other Signal tests.
++vi.mock('node:fs/promises', async (importOriginal) => {
++ const actual = await importOriginal();
++ return {
++ ...actual,
++ mkdtemp: async (...args) => {
++ const path = await actual.mkdtemp(...args);
++ snapshotDirs.push(path);
++ return path;
++ },
++ };
++});
++
+import { SQLCIPHER_DRIVER_PACKAGE } from './signalSnapshot.js';
+import { runSync } from './signalSync.js';
+
+const driverRequire = createRequire(import.meta.url);
+const Database = driverRequire(SQLCIPHER_DRIVER_PACKAGE);
+
+const dbDir = mkdtempSync(join(tmpdir(), 'signal-sync-wal-db-'));
+const dbPath = join(dbDir, 'db.sqlite');
+const configPath = join(dbDir, 'config.json');
+const key = 'ab'.repeat(32);
+let writer;
+
+function digest(path) {
+ return createHash('sha256').update(readFileSync(path)).digest('hex');
+}
+
- function signalTemps() {
- return readdirSync(tmpdir()).filter((name) => name.startsWith('portos-signal-'));
++function expectOwnSnapshotsRemoved() {
++ expect(snapshotDirs.length).toBeGreaterThan(0);
++ expect(snapshotDirs.every((path) => !existsSync(path))).toBe(true);
+}
+
+beforeAll(() => {
+ writer = new Database(dbPath);
+ writer.pragma(`cipher='sqlcipher'`);
+ writer.pragma('legacy=4');
+ writer.pragma(`key="x'${key}'"`);
+ writer.pragma('journal_mode = WAL');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.exec(`
+ CREATE TABLE conversations (
+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT
+ );
+ CREATE TABLE messages (
+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT
+ );
+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');
+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'checkpointed');
+ `);
+ writer.pragma('wal_checkpoint(TRUNCATE)');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.exec(`INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'wal-only')`);
+ writeFileSync(configPath, JSON.stringify({ key }));
+ process.env.SIGNAL_DB_PATH = dbPath;
+ process.env.SIGNAL_CONFIG_PATH = configPath;
+});
+
+afterAll(() => {
+ try { writer?.close(); } catch { /* already closed */ }
+ delete process.env.SIGNAL_DB_PATH;
+ delete process.env.SIGNAL_CONFIG_PATH;
+ rmSync(dbDir, { recursive: true, force: true });
+ rmSync(dataRoot, { recursive: true, force: true });
+});
+
+beforeEach(() => {
+ recordEventsMock.mockReset();
+ autoLogTouchpointsMock.mockReset();
+ recordEventsMock.mockResolvedValue({ recorded: 2, skipped: 0 });
+ autoLogTouchpointsMock.mockResolvedValue({ created: 1, matched: 1 });
++ snapshotDirs.length = 0;
+ rmSync(join(dataRoot, 'signal-sync-state.json'), { force: true });
+ writeFileSync(configPath, JSON.stringify({ key }));
+});
+
+describe('runSync WAL snapshot', () => {
+ it('ingests both committed messages once, then stays idempotent, without writing the source', async () => {
+ const wal = `${dbPath}-wal`;
+ const before = { db: digest(dbPath), wal: digest(wal) };
- const tempsBefore = new Set(signalTemps());
+
+ const first = await runSync();
+ expect(first.ok, first.error).toBe(true);
+ expect(first.scanned).toBe(2);
+ expect(first.cursorRowid).toBe(2);
+ const recorded = recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey).sort();
+ expect(recorded).toEqual(['signal:m1', 'signal:m2']);
+ expect(digest(dbPath)).toBe(before.db);
+ expect(digest(wal)).toBe(before.wal);
- expect(signalTemps().filter((name) => !tempsBefore.has(name))).toEqual([]);
++ expectOwnSnapshotsRemoved();
+
+ const second = await runSync();
+ expect(second.ok).toBe(true);
+ expect(second.scanned).toBe(0);
+ expect(second.cursorRowid).toBe(2);
+ expect(recordEventsMock.mock.calls[1][0]).toEqual([]);
+ });
+
+ it('ingests a WAL message inserted after an older row was deleted', async () => {
+ const dir = mkdtempSync(join(tmpdir(), 'signal-sync-rowid-'));
+ const path = join(dir, 'db.sqlite');
+ const local = new Database(path);
+ local.pragma(`cipher='sqlcipher'`);
+ local.pragma('legacy=4');
+ local.pragma(`key="x'${key}'"`);
+ local.pragma('journal_mode = WAL');
+ local.pragma('wal_autocheckpoint = 0');
+ local.exec(`
+ CREATE TABLE conversations (
+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT
+ );
+ CREATE TABLE messages (
+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT
+ );
+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');
+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'first');
+ INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'second');
+ `);
+ const previous = process.env.SIGNAL_DB_PATH;
+ process.env.SIGNAL_DB_PATH = path;
+ try {
+ const first = await runSync();
+ expect(first.ok, first.error).toBe(true);
+ expect(first.cursorRowid).toBe(2);
+ local.exec(`DELETE FROM messages WHERE id = 'm1'`);
+ local.exec(`INSERT INTO messages VALUES ('m3', 'c1', 'incoming', 1700000002000, 1700000002000, 'after-delete')`);
+ recordEventsMock.mockClear();
+ const second = await runSync();
+ expect(second.ok, second.error).toBe(true);
+ expect(second.scanned).toBe(1);
+ expect(second.cursorRowid).toBeGreaterThan(first.cursorRowid);
+ expect(recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey)).toEqual(['signal:m3']);
+ } finally {
+ process.env.SIGNAL_DB_PATH = previous;
+ local.close();
+ rmSync(dir, { recursive: true, force: true });
+ }
+ });
+
+ it('leaves the cursor and activity calls untouched when the key is rejected', async () => {
+ const first = await runSync();
+ expect(first.ok, first.error).toBe(true);
+ const calls = recordEventsMock.mock.calls.length;
+
+ writeFileSync(configPath, JSON.stringify({ key: 'cd'.repeat(32) }));
+ const failed = await runSync();
+ expect(failed.ok).toBe(false);
+ expect(failed.reason).toBe('auth');
+ expect(failed.error).toMatch(/decryption failed/i);
+
+ const state = JSON.parse(readFileSync(join(dataRoot, 'signal-sync-state.json'), 'utf8'));
+ expect(state.cursorRowid).toBe(first.cursorRowid);
+ expect(recordEventsMock.mock.calls.length).toBe(calls);
- expect(signalTemps()).toEqual([]);
++ expectOwnSnapshotsRemoved();
+ });
+});