Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS
JavaScript ai-agentsdashboarddeveloper-toolsdigital-twinquantified-selfsecond-brainself-hostedtailscale
merge origin/main and keep the Signal WAL snapshot (#11149)
Adam Eivyd90bb702e0a6
diff --cc server/services/signalSync.runSync.test.jsindex 0000000000,3a233f95a1..60fed5e9f4mode 000000,100644..100644--- a/server/services/signalSync.runSync.test.js+++ b/server/services/signalSync.runSync.test.js@@@ -1,0 -1,160 +1,166 @@@+ import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+ import { DatabaseSync } from 'node:sqlite';+ import crypto from 'node:crypto';+ import { existsSync, mkdirSync, readFileSync, writeFileSync, rmSync } from 'node:fs';+ import { join } from 'node:path';+ import { createTempDataRoot } from '../lib/mockPathsDataRoot.js';+ import { decryptSqlcipherDatabase, deriveSqlcipherKeys, sqlcipherPageHmac,+ SQLCIPHER_PAGE_SIZE, SQLCIPHER_RESERVE_BYTES, SQLCIPHER_SALT_BYTES,+ SQLCIPHER_IV_BYTES } from '../lib/signalCrypto.js';++ const mocks = vi.hoisted(() => ({+ root: '', snapshots: [], recordEvents: vi.fn(), autoLogTouchpoints: vi.fn(),+ }));+ vi.mock('../lib/fileUtils.js', async (importOriginal) => ({+ ...await importOriginal(),+ dataPath: (...parts) => join(mocks.root, ...parts),+ }));+ vi.mock('node:fs/promises', async (importOriginal) => {+ const actual = await importOriginal();+ return { ...actual, mkdtemp: async (...args) => {+ const path = await actual.mkdtemp(...args);+ mocks.snapshots.push(path);+ return path;+ } };+ });+ vi.mock('./settings.js', () => ({ getSettings: vi.fn(async () => ({})) }));+ vi.mock('./userTimezone.js', () => ({ getUserTimezone: vi.fn(async () => 'UTC') }));+ vi.mock('./humanActivity.js', async (importOriginal) => ({+ ...await importOriginal(), recordEvents: mocks.recordEvents,+ }));+ vi.mock('./tribe.js', () => ({ autoLogTouchpoints: mocks.autoLogTouchpoints }));++ import { checkSetup, runSync, readSyncState } from './signalSync.js';++ const KEY = 'ab'.repeat(32); // invented fixture key; never reads the keychain+ const BODY = 'x'.repeat(512);+ let encrypted;+ let plaintext;++ // Build a genuinely queryable SQLite database with SQLCipher's 80-byte reserve.+ // An empty SQLite page has no cells to relocate: changing its reserve and empty+ // content-area pointer before creating tables makes SQLite allocate all later+ // pages with the correct usable size. No external SQLCipher/Python dependency.+ function createFixture() {+ const path = join(mocks.root, 'fixture.sqlite');+ let db = new DatabaseSync(path);+ db.exec('PRAGMA page_size=4096; VACUUM');+ db.close();+ const empty = readFileSync(path);+ empty[20] = SQLCIPHER_RESERVE_BYTES;+ empty.writeUInt16BE(SQLCIPHER_PAGE_SIZE - SQLCIPHER_RESERVE_BYTES, 105);+ writeFileSync(path, empty);+ db = new DatabaseSync(path);+ db.exec(`+ CREATE TABLE conversations(id TEXT, e164 TEXT, type TEXT, name TEXT, profileName TEXT);+ CREATE TABLE messages(id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT);+ INSERT INTO conversations VALUES ('fixture-chat', '+15550000000', 'private', 'Example Contact', '');+ `);+ db.prepare('INSERT INTO messages(rowid, id, conversationId, type, sent_at, body) VALUES (?,?,?,?,?,?)')+ .run(8, 'fixture-message', 'fixture-chat', 'incoming', Date.UTC(2024, 5, 1, 12), BODY);+ expect(db.prepare('PRAGMA integrity_check').get().integrity_check).toBe('ok');+ db.close();+ return readFileSync(path);+ }++ // Encryption uses the production KDF pinned by signalCrypto's independent+ // known-answer vectors; this fixture exercises service wiring, not KDF accuracy.+ function encrypt(plain) {+ const salt = Buffer.alloc(SQLCIPHER_SALT_BYTES, 7);+ const { encKey, hmacKey } = deriveSqlcipherKeys(KEY, salt);+ const out = Buffer.alloc(plain.length);+ for (let page = 1; page <= plain.length / SQLCIPHER_PAGE_SIZE; page += 1) {+ const start = (page - 1) * SQLCIPHER_PAGE_SIZE;+ const bodyStart = start + (page === 1 ? SQLCIPHER_SALT_BYTES : 0);+ const end = start + SQLCIPHER_PAGE_SIZE - SQLCIPHER_RESERVE_BYTES;+ const iv = Buffer.alloc(SQLCIPHER_IV_BYTES, page);+ const cipher = crypto.createCipheriv('aes-256-cbc', encKey, iv);+ cipher.setAutoPadding(false);+ const ciphertext = Buffer.concat([cipher.update(plain.subarray(bodyStart, end)), cipher.final()]);+ if (page === 1) salt.copy(out, start);+ ciphertext.copy(out, bodyStart);+ iv.copy(out, end);+ sqlcipherPageHmac(hmacKey, Buffer.concat([ciphertext, iv]), page).copy(out, end + SQLCIPHER_IV_BYTES);+ }+ return out;+ }++ beforeEach(() => {+ mocks.root = createTempDataRoot('signal-auth-test-');+ mocks.snapshots = [];+ mkdirSync(join(mocks.root, 'sql'));+ writeFileSync(join(mocks.root, 'config.json'), JSON.stringify({ key: KEY }));+ vi.stubEnv('SIGNAL_DIR', mocks.root);+ vi.stubEnv('SIGNAL_CONFIG_PATH', join(mocks.root, 'config.json'));+ vi.stubEnv('SIGNAL_DB_PATH', join(mocks.root, 'sql', 'db.sqlite'));+ plaintext = createFixture();+ encrypted = encrypt(plaintext);+ writeFileSync(process.env.SIGNAL_DB_PATH, encrypted);+ mocks.recordEvents.mockReset().mockImplementation(async (events) => ({ recorded: events.length, skipped: 0 }));+ mocks.autoLogTouchpoints.mockReset().mockResolvedValue({ created: 1, matched: 1 });+ });+ afterEach(() => {+ vi.unstubAllEnvs();+ rmSync(mocks.root, { recursive: true, force: true });+ vi.restoreAllMocks();+ });++ function expectSnapshotsRemoved() {+ expect(mocks.snapshots.length).toBeGreaterThan(0);+ expect(mocks.snapshots.every((path) => !existsSync(path))).toBe(true);+ }++ describe('Signal authenticated snapshot ingestion', () => {+ it('accepts a multipage database and does not reingest messages on replay', async () => {+ expect(plaintext.length).toBeGreaterThan(SQLCIPHER_PAGE_SIZE);+ expect(await checkSetup()).toMatchObject({ ok: true, messageCount: 1 });+ expect(await runSync()).toMatchObject({ ok: true, recorded: 1, cursorRowid: 8 });+ expect(mocks.recordEvents.mock.calls[0][0]).toMatchObject([+ { dedupeKey: 'signal:fixture-message', kind: 'message.received' },+ ]);+ expect(mocks.autoLogTouchpoints.mock.calls[0][0]).toHaveLength(1);+ expect(await runSync()).toMatchObject({ ok: true, recorded: 0, scanned: 0, cursorRowid: 8 });+ expect(mocks.recordEvents.mock.calls[1][0]).toEqual([]);+ expect((await readSyncState()).cursorRowid).toBe(8);+ expectSnapshotsRemoved();+ });++ it('refuses queryable later-page body corruption before writes and preserves the prior cursor', async () => {+ const statePath = join(mocks.root, 'signal-sync-state.json');+ const state = JSON.stringify({ cursorRowid: 7, lastRunAt: '2024-01-01T00:00:00Z', lastResult: { ok: true } });+ writeFileSync(statePath, state);+ // Flip a ciphertext bit well inside the long body, beyond page 1. CBC+ // damages text bytes while leaving the cell header and B-tree readable.+ const bodyOffset = plaintext.indexOf(Buffer.from(BODY));+ expect(bodyOffset).toBeGreaterThan(SQLCIPHER_PAGE_SIZE);+ encrypted[bodyOffset + 128] ^= 1;+ writeFileSync(process.env.SIGNAL_DB_PATH, encrypted);+ const partial = decryptSqlcipherDatabase(encrypted, KEY, { verify: 'first' });+ expect(partial.ok).toBe(true);+ const readablePath = join(mocks.root, 'damaged-readable.sqlite');+ writeFileSync(readablePath, partial.plaintext);+ const readable = new DatabaseSync(readablePath, { readOnly: true });+ try {+ expect(readable.prepare('PRAGMA integrity_check').get().integrity_check).toBe('ok');+ expect(readable.prepare('SELECT body FROM messages').get().body).not.toBe(BODY);+ } finally { readable.close(); }+++ // verify:'first' still produces a readable database whose integrity_check++ // passes and whose body is wrong. The SQLCipher snapshot must refuse that++ // file (the driver reports the damaged page as malformed) and must not++ // ingest it or move the cursor. The old main-file copy named this as a++ // page HMAC mismatch; the snapshot failure is the same refusal.+ vi.spyOn(console, 'error').mockImplementation(() => {});+ for (const result of [await checkSetup(), await runSync()]) {+ expect(result.ok).toBe(false);- expect(result.error).toMatch(/Page-[2-9]\d* HMAC mismatch/);- expect(result.remediation).toMatch(/Close Signal Desktop and retry/);++ expect(['auth', 'snapshot']).toContain(result.reason);++ expect(result.error).toMatch(/malformed|hmac|decryption failed|integrity/i);++ expect(result.remediation).toMatch(/left unchanged|Nothing was ingested|not yet supported/);+ }+ expect(mocks.recordEvents).not.toHaveBeenCalled();+ expect(mocks.autoLogTouchpoints).not.toHaveBeenCalled();+ expect(readFileSync(statePath, 'utf8')).toBe(state);+ expect((await readSyncState()).cursorRowid).toBe(7);+ expectSnapshotsRemoved();+ });+ });diff --cc server/services/signalSync.wal.test.jsindex fa5feb8e92,0000000000..3d9848cd8fmode 100644,000000..100644--- a/server/services/signalSync.wal.test.js+++ b/server/services/signalSync.wal.test.js@@@ -1,186 -1,0 +1,201 @@@+/**+ * Signal sync consumes the consistent snapshot (#11149): both the checkpointed+ * message and the committed WAL-only message are ingested, a second pass is a+ * no-op, and an authentication failure leaves the cursor and activity calls+ * untouched. The database is an encrypted fixture, not a live Signal profile.+ */+import { createRequire } from 'node:module';+import { createHash } from 'node:crypto';- import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';++import { existsSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from 'node:fs';+import { tmpdir } from 'node:os';+import { join } from 'node:path';+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';+- const { recordEventsMock, autoLogTouchpointsMock } = vi.hoisted(() => ({++const { recordEventsMock, autoLogTouchpointsMock, snapshotDirs } = vi.hoisted(() => ({+ recordEventsMock: vi.fn(),+ autoLogTouchpointsMock: vi.fn(),++ snapshotDirs: [],+}));++const dataRoot = vi.hoisted(() => {+ const { mkdtempSync: mkdtemp } = require('node:fs');+ const { tmpdir: osTmp } = require('node:os');+ const { join: joinPath } = require('node:path');+ return mkdtemp(joinPath(osTmp(), 'signal-sync-wal-'));+});++vi.mock('../lib/fileUtils.js', async () => {+ const actual = await vi.importActual('../lib/fileUtils.js');+ const { join: joinPath } = await import('node:path');+ return {+ ...actual,+ dataPath: (...segments) => joinPath(dataRoot, ...segments),+ };+});++vi.mock('./humanActivity.js', async () => {+ const actual = await vi.importActual('./humanActivity.js');+ return { ...actual, recordEvents: recordEventsMock };+});++vi.mock('./tribe.js', () => ({+ autoLogTouchpoints: autoLogTouchpointsMock,+}));+++// Only this file's snapshot dirs. A global tmpdir scan races other Signal tests.++vi.mock('node:fs/promises', async (importOriginal) => {++ const actual = await importOriginal();++ return {++ ...actual,++ mkdtemp: async (...args) => {++ const path = await actual.mkdtemp(...args);++ snapshotDirs.push(path);++ return path;++ },++ };++});+++import { SQLCIPHER_DRIVER_PACKAGE } from './signalSnapshot.js';+import { runSync } from './signalSync.js';++const driverRequire = createRequire(import.meta.url);+const Database = driverRequire(SQLCIPHER_DRIVER_PACKAGE);++const dbDir = mkdtempSync(join(tmpdir(), 'signal-sync-wal-db-'));+const dbPath = join(dbDir, 'db.sqlite');+const configPath = join(dbDir, 'config.json');+const key = 'ab'.repeat(32);+let writer;++function digest(path) {+ return createHash('sha256').update(readFileSync(path)).digest('hex');+}+- function signalTemps() {- return readdirSync(tmpdir()).filter((name) => name.startsWith('portos-signal-'));++function expectOwnSnapshotsRemoved() {++ expect(snapshotDirs.length).toBeGreaterThan(0);++ expect(snapshotDirs.every((path) => !existsSync(path))).toBe(true);+}++beforeAll(() => {+ writer = new Database(dbPath);+ writer.pragma(`cipher='sqlcipher'`);+ writer.pragma('legacy=4');+ writer.pragma(`key="x'${key}'"`);+ writer.pragma('journal_mode = WAL');+ writer.pragma('wal_autocheckpoint = 0');+ writer.exec(`+ CREATE TABLE conversations (+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT+ );+ CREATE TABLE messages (+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT+ );+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'checkpointed');+ `);+ writer.pragma('wal_checkpoint(TRUNCATE)');+ writer.pragma('wal_autocheckpoint = 0');+ writer.exec(`INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'wal-only')`);+ writeFileSync(configPath, JSON.stringify({ key }));+ process.env.SIGNAL_DB_PATH = dbPath;+ process.env.SIGNAL_CONFIG_PATH = configPath;+});++afterAll(() => {+ try { writer?.close(); } catch { /* already closed */ }+ delete process.env.SIGNAL_DB_PATH;+ delete process.env.SIGNAL_CONFIG_PATH;+ rmSync(dbDir, { recursive: true, force: true });+ rmSync(dataRoot, { recursive: true, force: true });+});++beforeEach(() => {+ recordEventsMock.mockReset();+ autoLogTouchpointsMock.mockReset();+ recordEventsMock.mockResolvedValue({ recorded: 2, skipped: 0 });+ autoLogTouchpointsMock.mockResolvedValue({ created: 1, matched: 1 });++ snapshotDirs.length = 0;+ rmSync(join(dataRoot, 'signal-sync-state.json'), { force: true });+ writeFileSync(configPath, JSON.stringify({ key }));+});++describe('runSync WAL snapshot', () => {+ it('ingests both committed messages once, then stays idempotent, without writing the source', async () => {+ const wal = `${dbPath}-wal`;+ const before = { db: digest(dbPath), wal: digest(wal) };- const tempsBefore = new Set(signalTemps());++ const first = await runSync();+ expect(first.ok, first.error).toBe(true);+ expect(first.scanned).toBe(2);+ expect(first.cursorRowid).toBe(2);+ const recorded = recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey).sort();+ expect(recorded).toEqual(['signal:m1', 'signal:m2']);+ expect(digest(dbPath)).toBe(before.db);+ expect(digest(wal)).toBe(before.wal);- expect(signalTemps().filter((name) => !tempsBefore.has(name))).toEqual([]);++ expectOwnSnapshotsRemoved();++ const second = await runSync();+ expect(second.ok).toBe(true);+ expect(second.scanned).toBe(0);+ expect(second.cursorRowid).toBe(2);+ expect(recordEventsMock.mock.calls[1][0]).toEqual([]);+ });++ it('ingests a WAL message inserted after an older row was deleted', async () => {+ const dir = mkdtempSync(join(tmpdir(), 'signal-sync-rowid-'));+ const path = join(dir, 'db.sqlite');+ const local = new Database(path);+ local.pragma(`cipher='sqlcipher'`);+ local.pragma('legacy=4');+ local.pragma(`key="x'${key}'"`);+ local.pragma('journal_mode = WAL');+ local.pragma('wal_autocheckpoint = 0');+ local.exec(`+ CREATE TABLE conversations (+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT+ );+ CREATE TABLE messages (+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT+ );+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'first');+ INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'second');+ `);+ const previous = process.env.SIGNAL_DB_PATH;+ process.env.SIGNAL_DB_PATH = path;+ try {+ const first = await runSync();+ expect(first.ok, first.error).toBe(true);+ expect(first.cursorRowid).toBe(2);+ local.exec(`DELETE FROM messages WHERE id = 'm1'`);+ local.exec(`INSERT INTO messages VALUES ('m3', 'c1', 'incoming', 1700000002000, 1700000002000, 'after-delete')`);+ recordEventsMock.mockClear();+ const second = await runSync();+ expect(second.ok, second.error).toBe(true);+ expect(second.scanned).toBe(1);+ expect(second.cursorRowid).toBeGreaterThan(first.cursorRowid);+ expect(recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey)).toEqual(['signal:m3']);+ } finally {+ process.env.SIGNAL_DB_PATH = previous;+ local.close();+ rmSync(dir, { recursive: true, force: true });+ }+ });++ it('leaves the cursor and activity calls untouched when the key is rejected', async () => {+ const first = await runSync();+ expect(first.ok, first.error).toBe(true);+ const calls = recordEventsMock.mock.calls.length;++ writeFileSync(configPath, JSON.stringify({ key: 'cd'.repeat(32) }));+ const failed = await runSync();+ expect(failed.ok).toBe(false);+ expect(failed.reason).toBe('auth');+ expect(failed.error).toMatch(/decryption failed/i);++ const state = JSON.parse(readFileSync(join(dataRoot, 'signal-sync-state.json'), 'utf8'));+ expect(state.cursorRowid).toBe(first.cursorRowid);+ expect(recordEventsMock.mock.calls.length).toBe(calls);- expect(signalTemps()).toEqual([]);++ expectOwnSnapshotsRemoved();+ });+});