Skip to content

Self-hosted, Self-Improving, Federated Everything App — dev management, agent orchestration, persistent mind, digital twin, second brain, and creative pipeline from a single dashboard over Tailscale to port the user into the OS

fix: read a consistent Signal snapshot including committed WAL messages (#11149)

Adam Eivyb70b0c0b89cf
diff --git a/.env.example b/.env.example
index dbe6772400..61e242cda4 100644
--- a/.env.example
+++ b/.env.example
@@ -342,6 +342,13 @@ PGPASSWORD=portos
# Signal sync SQLCipher database path (default: SIGNAL_DIR/sql/db.sqlite)
# SIGNAL_DB_PATH=/path/to/signal/sql/db.sqlite
+#
+# Capture opens that file read-only with the bundled SQLCipher driver
+# (better-sqlite3-multiple-ciphers, SQLCipher 4 compatibility, raw 64-hex key)
+# and exports a private plaintext snapshot so committed WAL frames are included.
+# It never writes or checkpoints the live database or its WAL. Shipped prebuilds
+# cover macOS, Linux (glibc and musl), and Windows, on x64 and arm64. If the
+# prebuild cannot load, sync returns a setup error and leaves the cursor unchanged.
# Test/CI hook that supplies the "Signal Safe Storage" password instead of
# shelling out to the macOS keychain (signalSync.js). Leave unset on a real
diff --git a/README.md b/README.md
index 79242ccfd2..cd3d5acbd6 100644
--- a/README.md
+++ b/README.md
@@ -188,7 +188,7 @@ Unified inbox for everything that talks back at you — email, chat, agent-gener
- **Inbox / Drafts / Sync** — Aggregated message inbox with AI-generated drafts you can approve, edit, or discard before sending. Message content is untrusted input, and the prompt-injection boundary around it is explicit ([Messages security model](./docs/features/messages-security.md))
- **iMessage** — Local `chat.db` sync (requires Full Disk Access) with contact resolution, blocklist, and spam triage; feeds Tribe and the Activity Timeline
-- **Signal** — Read-only sync from a local Signal Desktop install (SQLCipher key via Keychain) into the same surfaces
+- **Signal** — Read-only SQLCipher snapshot of a local Signal Desktop database (key via Keychain), including committed write-ahead-log messages, into the same surfaces. Does not write or checkpoint Signal's database.
- **OpenClaw** — In-app chat with external operator agents (computer-use, Anthropic operator, etc.) with app context, attachments, and streaming responses ([OpenClaw docs](./docs/features/openclaw-operator-chat.md))
- **Social Agents** — Personality-driven agents that handle specific accounts/personas
- **Stacker News** — Territory management, untrusted content safety, and account stewardship ([Stacker News docs](./docs/features/stacker-news.md))
diff --git a/client/src/components/messages/SignalTab.jsx b/client/src/components/messages/SignalTab.jsx
index 03a1c5993b..22553b098e 100644
--- a/client/src/components/messages/SignalTab.jsx
+++ b/client/src/components/messages/SignalTab.jsx
@@ -63,10 +63,11 @@ export function SignalTab() {
</div>
<p className="text-sm text-gray-400 mb-4">
Reads your local Signal Desktop chat database
- (<code className="text-gray-300">~/Library/Application Support/Signal</code>) — decrypting its
- SQLCipher store with the keychain-wrapped key — and feeds both Tribe touchpoints and the activity timeline.
- Machine-local; nothing federates to peers. Signal never stores an API for this, so PortOS reads a
- temporary decrypted <strong>copy</strong> and never touches the live database.
+ (<code className="text-gray-300">~/Library/Application Support/Signal</code>) through a read-only
+ SQLCipher snapshot, so committed messages still in the write-ahead log are included, and feeds both
+ Tribe touchpoints and the activity timeline. Machine-local; nothing federates to peers. PortOS does
+ not write or checkpoint the live database. The snapshot uses the bundled SQLCipher driver; if that
+ driver cannot load, sync stops with a setup error and leaves the previous cursor in place.
This is the most fragile source: if a Signal update changes its key or storage format, ingestion
degrades to an actionable error rather than failing hard.
</p>
diff --git a/client/src/services/apiSignal.js b/client/src/services/apiSignal.js
index 9257bec51a..3b441ef317 100644
--- a/client/src/services/apiSignal.js
+++ b/client/src/services/apiSignal.js
@@ -1,7 +1,7 @@
import { request } from './apiCore.js';
-// Signal Desktop ingestion (#2154) — read the SQLCipher-encrypted chat DB (via
-// the keychain-wrapped key) into the Tribe touchpoint log + activity timeline.
+// Signal Desktop ingestion (#2154, #11149) — read-only SQLCipher snapshot of the
+// local chat DB (keychain-wrapped key, committed WAL frames included).
// Machine-local; OFF by default. These wrappers drive the settings tab.
export const getSignalStatus = (options = {}) => request('/signal/status', options);
export const checkSignalSetup = (options = {}) => request('/signal/setup-check', options);
diff --git a/docs/features/product-surfaces.md b/docs/features/product-surfaces.md
index 89fdc8eb44..16a5837569 100644
--- a/docs/features/product-surfaces.md
+++ b/docs/features/product-surfaces.md
@@ -176,7 +176,7 @@ Unified communication channels, local voice assistant, community stewardship, an
|---|---|---|---|
| **Comms Inbox & Drafts** | `/messages/inbox`, `/messages/drafts`, `/messages/sync` | Centralized message triage, drafting assistant, and cross-channel message synchronization. | [Messages Security](./messages-security.md) |
| **iMessage Sync & Contacts** | `/messages/imessage`, `/messages/contacts` | Local macOS `chat.db` SQLite ingestion, sender identity resolution, spam filtering, and address book contact merging. | [Messages Security](./messages-security.md) |
-| **Signal Integration** | `/settings/signal` | Encrypted local sync from Signal Desktop SQLCipher database. | [Messages Security](./messages-security.md) |
+| **Signal Integration** | `/messages/signal` | Read-only SQLCipher snapshot of the local Signal Desktop database, including committed WAL messages, into Tribe and the activity timeline. Fails setup instead of ingesting a partial copy when the bundled driver cannot capture a consistent view. | [Messages Security](./messages-security.md) |
| **Beeper Bridge** | `/messages/beeper`, `/messages/beeper?settings=1` | Local Beeper Desktop bridge mirroring WhatsApp, Discord, Telegram, and other bridged networks into a machine-local store, with a human-gated one-message-at-a-time outbox; the settings drawer connects an account and configures scheduled sync. | [Beeper](./beeper.md) |
| **Telegram & X (Twitter)** | `/settings/telegram`, `/x` | Social channel monitoring, outreach staging, engagement diagnostics, and shadowban checks. | — |
| **Stacker News Stewardship** | `/stacker-news` | Community territory moderation, Bitcoin Lightning tipping, post review, and encrypted credential storage. | [Stacker News](./stacker-news.md) |
diff --git a/docs/features/product-surfaces.test.js b/docs/features/product-surfaces.test.js
index f395f5ca30..646d92c77a 100644
--- a/docs/features/product-surfaces.test.js
+++ b/docs/features/product-surfaces.test.js
@@ -11,8 +11,6 @@
* Scope is deliberately narrower than "every NAV_COMMANDS path" — this change
* only touches the Comms section, and a repo-wide version of this guard would
* fail today on pre-existing gaps this change did not introduce:
- * - /messages/signal — the doc row still names the pre-#30 previousPath
- * `/settings/signal`, not the live path.
* - /messages/config — no row at all.
* - /messages/imessage?settings=1 — no row at all (the bare /messages/imessage
* row does not cover the settings-drawer deep link).
@@ -32,7 +30,6 @@ const SECTION_HEADING = '## 6. Communications, Voice, Social, & Spatial UI';
// Pre-existing gaps this change did not introduce — see the file comment above.
const KNOWN_UNRELATED_GAPS = new Set([
- '/messages/signal',
'/messages/config',
'/messages/imessage?settings=1',
]);
diff --git a/scripts/trusted-rebuilds.test.js b/scripts/trusted-rebuilds.test.js
index 2bdb334202..7dbd069214 100644
--- a/scripts/trusted-rebuilds.test.js
+++ b/scripts/trusted-rebuilds.test.js
@@ -169,10 +169,17 @@ describe('trusted rebuild allowlist', () => {
// 3 and the package already ships in Vue 3 mode. Keep the hook blocked rather
// than expanding the lifecycle-script allowlist for a no-op.
const VUE_DEMI_VUE3_NOOP = ['vue-demi'];
+ // better-sqlite3-multiple-ciphers publishes `gypfile: false` and ships
+ // prebuilt binaries (macOS, Linux glibc and musl, Windows; x64 and arm64).
+ // `binding.gyp` is the maintainer's source-build recipe, which this scan
+ // still sees. `npm rebuild` is unnecessary and can replace the prebuild the
+ // Signal snapshot loader uses. A missing prebuild fails that capture closed
+ // (setup error) instead of compiling during install.
+ const SIGNAL_SQLCIPHER_PREBUILD = ['better-sqlite3-multiple-ciphers'];
const DELIBERATELY_UNBUILT = {
root: FSEVENTS_STALE_FLAG,
client: [...FSEVENTS_STALE_FLAG, ...VUE_DEMI_VUE3_NOOP],
- server: FSEVENTS_STALE_FLAG
+ server: [...FSEVENTS_STALE_FLAG, ...SIGNAL_SQLCIPHER_PREBUILD]
};
it('detects install hooks at all, so the accounting assertion is not vacuous', () => {
diff --git a/server/lib/README.md b/server/lib/README.md
index cceb194705..61f7add6e7 100644
--- a/server/lib/README.md
+++ b/server/lib/README.md
@@ -654,7 +654,7 @@ The barrel `server/lib/index.js` is a machine-checkable enumeration of every pub
| `ports.js` | Canonical `PORTS` object (mirroring `ecosystem.config.cjs`, pinned by `ports.test.js`), `DEFAULT_PEER_PORT`, and `resolvePostgresPort(pgMode)`. A pure leaf: `client/src/lib/ports.js` re-exports it, so nothing here may read `process.env` at module scope — the env-derived origins live in `portosUrls.js`. |
| `portosUrls.js` | `PORTOS_UI_URL` / `PORTOS_API_URL` — the origins PortOS interpolates into prompts and links, resolved from `PORTOS_*` / `PORT` at module load. Split out of `ports.js` so that module stays free of module-scope `process.env` reads and can be re-exported to the client. |
| `platform.js` | Platform/OS detection helpers — listening-port probes plus `isAppleSilicon()` (arm64 darwin; gates MLX model features, detect at the route boundary). |
-| `signalCrypto.js` | Pure, dependency-free crypto for reading Signal Desktop's encrypted chat DB (#2154): SQLCipher-4 page decryption (`decryptSqlcipherDatabase`, `deriveSqlcipherKeys`, `sqlcipherPageHmac` — PBKDF2-SHA512 HMAC key + AES-256-CBC per page + HMAC-SHA512 verify → plaintext SQLite buffer the built-in `node:sqlite` can open) and Chromium/Electron `safeStorage` unwrap (`decryptSafeStorageValue`, `deriveSafeStorageKey` — macOS AES-128-CBC + PBKDF2-SHA1 over the keychain password). All functions return `{ ok, ... }` reports (never throw) for graceful degradation. Consumed by `services/signalSync.js`. |
+| `signalCrypto.js` | Pure, dependency-free crypto for Signal Desktop (#2154, #11149): SQLCipher-4 page decryption (`decryptSqlcipherDatabase`, `deriveSqlcipherKeys`, `sqlcipherPageHmac`) and Chromium/Electron `safeStorage` unwrap (`decryptSafeStorageValue`, `deriveSafeStorageKey`). Live sync does not decrypt a raw main-file copy — that misses committed WAL frames — and instead uses `services/signalSnapshot.js`. `isRawHexKey` still gates the key. All functions return `{ ok, ... }` reports (never throw). |
| `timezone.js` | Timezone utilities for scheduling. `getLocalParts(utcDate, timezone)` / `getUtcOffsetMs(utcDate, timezone)` are the primitives; `nextLocalTime(afterMs, hours, minutes, timezone)` finds the next UTC instant matching a local HH:MM. `anchorLocalMidnightUtc(dayStr, tz)` resolves the first UTC instant belonging to a local date, including zones whose DST transition skips 00:00; `localDayWindowUtc(timezone, atDate?)` exposes an inclusive ISO-string window, while `localDayRangeUtc(dateStr, timezone)` returns a validated half-open `{ start, end }` `Date` pair ending at the next local-date boundary so 23h/25h transition days remain exact. Also owns `HHMM_RE`/`HHMM_STRICT_RE`, `parseHHMM`, `dailyCronFromHHMM(time)` (the one strict "HH:MM" → `"M H * * *"` conversion every daily-reminder scheduler registers through, null on malformed input so the caller cancels rather than firing in some other slot), and `isWithinTimeWindow` (mirrored client-side in `client/src/utils/timeWindow.js`). |
| `tribeCadence.js` | Authoritative, pure Tribe care-cadence rules (single source of truth, mirrored to `client/src/lib/tribeCadence.js`): `cadenceStatus(entity)` → `{ state: external/missing/overdue/soon/steady, daysRemaining, daysOverdue }`, `daysSinceDate(dateStr)`, `DEFAULT_CADENCE_DAYS` (45), `SOON_WINDOW_DAYS` (7). Consumed by `personCadenceStatus` / `getCareSummary` in `services/tribe.js` (proactive alert + Care widget) and by the client Tribe page/map. |
| `tribeMatch.js` | Pure, deterministic matcher mapping a calendar attendee / message counterpart (`{ email, phone, name }`) to a tracked Tribe person for auto-logged touchpoints (#2033, #2151): `buildPersonMatchIndex(people)` → `{ byIdentifier, byPhone, byName }`, `matchPerson(identity, index)` (email/handle authoritative, then E.164 phone, then exact unique name fallback — no fuzzy matching), `matchPeople(identities, index)` → de-duplicated Set of personIds, `normalizeIdentifier(value)`, `normalizePhone(value)` (E.164 normalization for iMessage/Signal handles), `identityFromHandle(handle)` (classify a raw `chat.db` handle into email-or-phone). Also `resolveLinkedPersonId(tribePersonId, tribePersonDeleted)` — the shared soft-delete predicate a `beeper_participants` row's `tribe_person_id` cache must pass through everywhere it is shaped, so a soft-deleted Tribe person reads as unlinked in both `services/beeperConversations.js` and `services/beeperTribe.js`. Consumed by `autoLogTouchpoints` in `services/tribe.js` from the calendar, message, and iMessage sync producers. |
diff --git a/server/lib/signalCrypto.js b/server/lib/signalCrypto.js
index 873f7ed7d4..5a62efc42f 100644
--- a/server/lib/signalCrypto.js
+++ b/server/lib/signalCrypto.js
@@ -8,12 +8,13 @@
*
* 1. **SQLCipher 4 page decryption.** Signal stores its chat DB as a
* SQLCipher-4 file (`sql/db.sqlite`). `node:sqlite` cannot open it (no
- * SQLCipher support), and shelling to a `sqlcipher` CLI would add an
- * external tool dependency that most machines lack. Instead we decrypt the
- * file page-by-page here (PBKDF2-SHA512 for the HMAC key + AES-256-CBC per
- * page + HMAC-SHA512 verification) into a *plaintext* standard SQLite file,
- * which the built-in `node:sqlite` then opens read-only. This keeps the read
- * path self-contained and testable.
+ * SQLCipher support). These helpers decrypt a buffer page-by-page
+ * (PBKDF2-SHA512 for the HMAC key + AES-256-CBC per page + HMAC-SHA512
+ * verification) into a plaintext SQLite buffer. Live sync does not use
+ * them to read the on-disk file: a main-file decrypt misses committed WAL
+ * frames (#11149). `services/signalSnapshot.js` opens the live database
+ * read-only through the bundled SQLCipher driver and exports one committed
+ * snapshot instead. Key format checks (`isRawHexKey`) still live here.
*
* 2. **Chromium `safeStorage` decryption.** Signal ≥6.2 no longer stores the DB
* key in plaintext; `config.json` carries an `encryptedKey` wrapped by
diff --git a/server/package-lock.json b/server/package-lock.json
index b7bb36efa4..937721afd1 100644
--- a/server/package-lock.json
+++ b/server/package-lock.json
@@ -13,6 +13,7 @@
"@googleapis/calendar": "20.1.0",
"@googleapis/gmail": "22.0.1",
"@novnc/novnc": "1.7.0",
+ "better-sqlite3-multiple-ciphers": "13.0.3",
"chokidar": "5.0.0",
"compression": "1.8.2",
"express": "5.2.1",
@@ -1443,6 +1444,27 @@
"node": ">=10.0.0"
}
},
+ "node_modules/better-sqlite3-multiple-ciphers": {
+ "version": "13.0.3",
+ "resolved": "https://registry.npmjs.org/better-sqlite3-multiple-ciphers/-/better-sqlite3-multiple-ciphers-13.0.3.tgz",
+ "integrity": "sha512-UYabM82r1J84TLWc/SszoHs6XopWpl/2HCg3Nui1JUaFXg/VLswzkPowYiRhK/4CftI8dgtikwyZQecMldrGxQ==",
+ "license": "MIT",
+ "dependencies": {
+ "node-addon-api": "^8.0.0"
+ },
+ "engines": {
+ "node": ">=22"
+ }
+ },
+ "node_modules/better-sqlite3-multiple-ciphers/node_modules/node-addon-api": {
+ "version": "8.9.2",
+ "resolved": "https://registry.npmjs.org/node-addon-api/-/node-addon-api-8.9.2.tgz",
+ "integrity": "sha512-VijLXbi3UACN69I0JVXJsX4tjACjNoQDgv2gTF6sx2wWEi8tkSg2eX8p5gSIFi8z2+DL3oHmY6OyKce38SDolg==",
+ "license": "MIT",
+ "engines": {
+ "node": "^18 || ^20 || >= 21"
+ }
+ },
"node_modules/bignumber.js": {
"version": "9.3.1",
"resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.3.1.tgz",
diff --git a/server/package.json b/server/package.json
index 991268d83d..d21e884a72 100644
--- a/server/package.json
+++ b/server/package.json
@@ -27,6 +27,7 @@
"@googleapis/calendar": "20.1.0",
"@googleapis/gmail": "22.0.1",
"@novnc/novnc": "1.7.0",
+ "better-sqlite3-multiple-ciphers": "13.0.3",
"chokidar": "5.0.0",
"compression": "1.8.2",
"express": "5.2.1",
diff --git a/server/services/signalSnapshot.js b/server/services/signalSnapshot.js
new file mode 100644
index 0000000000..0f1203f7d1
--- /dev/null
+++ b/server/services/signalSnapshot.js
@@ -0,0 +1,204 @@
+/**
+ * Read-only SQLCipher snapshot of a live Signal Desktop database (#11149).
+ *
+ * Signal Desktop keeps `sql/db.sqlite` in WAL mode. A copy of that main file
+ * alone misses committed frames that have not been checkpointed, and a copy
+ * taken while a checkpoint is running is not a transactional snapshot. Both
+ * can look like a valid database and still drop messages.
+ *
+ * Capture opens the source with the bundled SQLCipher driver
+ * (`better-sqlite3-multiple-ciphers`, SQLCipher 4 compatibility, raw 64-hex
+ * key — the same key form Signal uses). One connection holds a deferred read
+ * transaction so a checkpoint cannot discard frames mid-capture. A second
+ * read-only connection materializes that committed view with SQLite
+ * `VACUUM INTO`, which includes WAL frames and works against a read-only
+ * source. `backup()` is not used: this driver rejects an encrypted source
+ * backing up into a plaintext target, and it does not ship `sqlcipher_export`.
+ *
+ * The private copy is then `rekey`ed with an empty key, which yields a
+ * standard plaintext SQLite file. Rekey runs only on that copy. The source is
+ * opened `readonly` and this module never issues `wal_checkpoint`,
+ * `journal_mode`, or any other write against it. The WAL index may record a
+ * reader mark; the database file and the WAL must stay byte-identical.
+ *
+ * If the driver cannot be loaded, or the snapshot is not a single integrity-
+ * clean committed view, the result is an explicit failure. Callers leave the
+ * ingestion cursor where it was. Temporary files are removed on failure;
+ * the caller removes them after a successful read.
+ */
+import { createRequire } from 'node:module';
+import { mkdtemp, readFile, rm, stat } from 'node:fs/promises';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+
+import { isRawHexKey } from '../lib/signalCrypto.js';
+
+const require = createRequire(import.meta.url);
+
+export const SQLCIPHER_DRIVER_PACKAGE = 'better-sqlite3-multiple-ciphers';
+
+const SQLITE_MAGIC = 'SQLite format 3\0';
+const BUSY_TIMEOUT_MS = 5000;
+
+const SETUP_REMEDIATION = 'Signal snapshot capture needs the bundled SQLCipher driver (better-sqlite3-multiple-ciphers) and its shipped prebuild for this platform (macOS, Linux glibc or musl, and Windows; x64 and arm64). Reinstall server dependencies. PortOS will not ingest a partial copy of the live database.';
+const AUTH_REMEDIATION = 'Signal database decryption failed — the key or SQLCipher format may have changed in this Signal version (not yet supported).';
+const SNAPSHOT_REMEDIATION = 'Could not take one consistent read-only snapshot of the Signal database. Nothing was ingested and the sync cursor was left unchanged; retry when Signal is not checkpointing.';
+
+function setupFailure(err) {
+ const detail = err?.message || 'SQLCipher driver unavailable';
+ return {
+ ok: false,
+ reason: 'setup',
+ error: `SQLCipher driver unavailable: ${detail}`,
+ remediation: SETUP_REMEDIATION,
+ };
+}
+
+function authFailure(error) {
+ return { ok: false, reason: 'auth', error, remediation: AUTH_REMEDIATION };
+}
+
+function snapshotFailure(error) {
+ return { ok: false, reason: 'snapshot', error, remediation: SNAPSHOT_REMEDIATION };
+}
+
+function loadDefaultDriver() {
+ return require(SQLCIPHER_DRIVER_PACKAGE);
+}
+
+function resolveConstructor(loaded) {
+ if (typeof loaded === 'function') return loaded;
+ if (loaded && typeof loaded.default === 'function') return loaded.default;
+ return null;
+}
+
+function sqlLiteral(value) {
+ return `'${String(value).replaceAll("'", "''")}'`;
+}
+
+function applyKey(db, key) {
+ // isRawHexKey already accepted this value. Trim so a wrapped config key
+ // cannot change the SQLCipher raw-key literal. The body is 64 hex chars.
+ const hex = String(key).trim();
+ db.pragma(`busy_timeout = ${BUSY_TIMEOUT_MS}`);
+ db.pragma(`cipher='sqlcipher'`);
+ db.pragma('legacy=4');
+ db.pragma(`key="x'${hex}'"`);
+}
+
+function openReadonly(Database, sourcePath) {
+ return new Database(sourcePath, { readonly: true, fileMustExist: true });
+}
+
+function classifyDriverError(err) {
+ const message = err?.message || String(err);
+ if (/not a database|hmac|cipher|encrypt|invalid key|file is not a database/i.test(message)) {
+ return authFailure(`Signal database decryption failed: ${message}`);
+ }
+ return snapshotFailure(`Signal snapshot failed: ${message}`);
+}
+
+async function closeQuietly(db) {
+ if (!db) return;
+ try {
+ if (db.inTransaction) db.exec('ROLLBACK');
+ } catch { /* already closed or no transaction */ }
+ try { db.close(); } catch { /* already closed */ }
+}
+
+/**
+ * Materialize a private plaintext snapshot of `sourcePath`.
+ *
+ * `options.loadDriver` overrides the package load (tests for a missing
+ * prebuild). `options.tmpRoot` relocates the temp directory. `options.onReadLocked`
+ * runs while the source read transaction is held, so a test can checkpoint
+ * concurrently. Returns `{ ok: true, plaintextPath, tmpDir }` or
+ * `{ ok: false, reason, error, remediation }`. Never throws.
+ */
+export async function captureConsistentSnapshot(sourcePath, key, options = {}) {
+ if (!isRawHexKey(key)) {
+ return authFailure('Signal SQLCipher key is not a 64-hex-char raw key');
+ }
+
+ let Database;
+ try {
+ const loaded = (options.loadDriver || loadDefaultDriver)();
+ Database = resolveConstructor(loaded);
+ if (!Database) throw new Error('driver did not export a database constructor');
+ } catch (err) {
+ return setupFailure(err);
+ }
+
+ let info;
+ try {
+ info = await stat(sourcePath);
+ } catch (err) {
+ const missing = err?.code === 'ENOENT';
+ return {
+ ok: false,
+ reason: missing ? 'missing' : 'snapshot',
+ error: missing ? `Signal database not found at ${sourcePath}` : `Cannot access ${sourcePath}: ${err?.message || err}`,
+ remediation: missing
+ ? 'Signal Desktop has no chat database yet (never opened, or a different profile).'
+ : 'Could not read the Signal database file — check file permissions for the process running PortOS.',
+ };
+ }
+ if (!info.isFile() || info.size === 0) {
+ return snapshotFailure('Signal database snapshot is empty');
+ }
+
+ const tmp = await mkdtemp(join(options.tmpRoot || tmpdir(), 'portos-signal-'));
+ const plainPath = join(tmp, 'db.plain.sqlite');
+ let holder;
+ let exporter;
+ let owned;
+ let plain;
+ try {
+ holder = openReadonly(Database, sourcePath);
+ applyKey(holder, key);
+ holder.exec('BEGIN');
+ // Establishes the read mark and rejects a wrong key before any export.
+ holder.prepare('SELECT count(*) AS n FROM sqlite_master').get();
+ if (options.onReadLocked) await options.onReadLocked();
+
+ exporter = openReadonly(Database, sourcePath);
+ applyKey(exporter, key);
+ exporter.exec(`VACUUM INTO ${sqlLiteral(plainPath)}`);
+ exporter.close();
+ exporter = null;
+ holder.exec('ROLLBACK');
+ holder.close();
+ holder = null;
+
+ owned = new Database(plainPath, { fileMustExist: true });
+ applyKey(owned, key);
+ owned.prepare('SELECT count(*) AS n FROM sqlite_master').get();
+ owned.pragma(`rekey=''`);
+ owned.close();
+ owned = null;
+
+ const header = await readFile(plainPath);
+ if (header.length < 16 || header.subarray(0, 16).toString('latin1') !== SQLITE_MAGIC) {
+ await rm(tmp, { recursive: true, force: true });
+ return snapshotFailure('Signal snapshot export did not produce a plaintext database');
+ }
+
+ plain = new Database(plainPath, { readonly: true, fileMustExist: true });
+ const check = plain.pragma('integrity_check');
+ plain.close();
+ plain = null;
+ const clean = Array.isArray(check) && check.length === 1 && check[0]?.integrity_check === 'ok';
+ if (!clean) {
+ await rm(tmp, { recursive: true, force: true });
+ return snapshotFailure('Signal snapshot failed integrity check');
+ }
+ return { ok: true, plaintextPath: plainPath, tmpDir: tmp };
+ } catch (err) {
+ await closeQuietly(plain);
+ await closeQuietly(owned);
+ await closeQuietly(exporter);
+ await closeQuietly(holder);
+ await rm(tmp, { recursive: true, force: true }).catch(() => {});
+ return classifyDriverError(err);
+ }
+}
diff --git a/server/services/signalSnapshot.test.js b/server/services/signalSnapshot.test.js
new file mode 100644
index 0000000000..7b181d887d
--- /dev/null
+++ b/server/services/signalSnapshot.test.js
@@ -0,0 +1,168 @@
+/**
+ * Consistent Signal capture (#11149). Builds a real SQLCipher WAL database with
+ * the bundled driver — no live Signal profile. One message is checkpointed into
+ * the main file; a second is committed while the writer stays open, so it exists
+ * only in the WAL. Copying the main file alone must not be treated as caught up.
+ */
+import { createRequire } from 'node:module';
+import { createHash, randomBytes } from 'node:crypto';
+import { copyFileSync, existsSync, mkdtempSync, readFileSync, readdirSync, rmSync, statSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { DatabaseSync } from 'node:sqlite';
+import { afterEach, describe, expect, it } from 'vitest';
+
+import { captureConsistentSnapshot, SQLCIPHER_DRIVER_PACKAGE } from './signalSnapshot.js';
+
+const require = createRequire(import.meta.url);
+const Database = require(SQLCIPHER_DRIVER_PACKAGE);
+
+const temps = [];
+
+function scratch() {
+ const dir = mkdtempSync(join(tmpdir(), 'signal-snapshot-test-'));
+ temps.push(dir);
+ return dir;
+}
+
+afterEach(() => {
+ while (temps.length) rmSync(temps.pop(), { recursive: true, force: true });
+});
+
+function digest(path) {
+ return createHash('sha256').update(readFileSync(path)).digest('hex');
+}
+
+function applyKey(db, key) {
+ db.pragma(`cipher='sqlcipher'`);
+ db.pragma('legacy=4');
+ db.pragma(`key="x'${key}'"`);
+}
+
+/**
+ * Encrypted WAL database. `writer` stays open so Signal-style autocheckpoint on
+ * close cannot fold the second message into the main file.
+ */
+function buildLiveWal(dir) {
+ const src = join(dir, 'db.sqlite');
+ const key = randomBytes(32).toString('hex');
+ const writer = new Database(src);
+ applyKey(writer, key);
+ writer.pragma('journal_mode = WAL');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.pragma('busy_timeout = 200');
+ writer.exec(`
+ CREATE TABLE conversations (
+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT
+ );
+ CREATE TABLE messages (
+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT
+ );
+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');
+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'checkpointed');
+ `);
+ const checkpoint = writer.pragma('wal_checkpoint(TRUNCATE)');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.exec(`INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'wal-only')`);
+ return { src, key, writer, checkpoint };
+}
+
+function messageIds(dbPath) {
+ const db = new DatabaseSync(dbPath, { readOnly: true });
+ try {
+ return db.prepare('SELECT id FROM messages ORDER BY rowid').all().map((row) => row.id);
+ } finally {
+ db.close();
+ }
+}
+
+describe('captureConsistentSnapshot', () => {
+ it('ingests the checkpointed row and the committed WAL-only row without modifying the source', async () => {
+ const dir = scratch();
+ const { src, key, writer } = buildLiveWal(dir);
+ const wal = `${src}-wal`;
+ expect(existsSync(wal)).toBe(true);
+ expect(statSync(wal).size).toBeGreaterThan(32);
+
+ const mainOnly = join(dir, 'main-only.sqlite');
+ copyFileSync(src, mainOnly);
+ const stale = new Database(mainOnly, { readonly: true, fileMustExist: true });
+ applyKey(stale, key);
+ const staleIds = stale.prepare('SELECT id FROM messages ORDER BY rowid').all().map((row) => row.id);
+ const staleIntegrity = stale.pragma('integrity_check');
+ stale.close();
+ expect(staleIds).toEqual(['m1']);
+ expect(staleIntegrity).toEqual([{ integrity_check: 'ok' }]);
+
+ const before = { db: digest(src), wal: digest(wal) };
+ try {
+ const snap = await captureConsistentSnapshot(src, key, { tmpRoot: dir });
+ expect(snap.ok, snap.error).toBe(true);
+ expect(messageIds(snap.plaintextPath)).toEqual(['m1', 'm2']);
+ expect(digest(src)).toBe(before.db);
+ expect(digest(wal)).toBe(before.wal);
+ } finally {
+ writer.close();
+ }
+ });
+
+ it('returns one integrity-clean committed view or an explicit failure while a checkpointer runs', async () => {
+ const dir = scratch();
+ const { src, key, writer } = buildLiveWal(dir);
+ try {
+ const snap = await captureConsistentSnapshot(src, key, {
+ tmpRoot: dir,
+ onReadLocked() {
+ writer.pragma('wal_checkpoint(RESTART)');
+ },
+ });
+ if (snap.ok) {
+ const plain = new DatabaseSync(snap.plaintextPath, { readOnly: true });
+ try {
+ expect(plain.prepare('PRAGMA integrity_check').all()).toEqual([{ integrity_check: 'ok' }]);
+ const ids = plain.prepare('SELECT id, body FROM messages ORDER BY rowid').all();
+ expect(ids.map((row) => row.id)).toEqual(['m1', 'm2']);
+ expect(ids.every((row) => row.body)).toBe(true);
+ } finally {
+ plain.close();
+ }
+ } else {
+ expect(snap.reason).toBe('snapshot');
+ expect(readdirSync(dir).some((name) => name.startsWith('portos-signal-'))).toBe(false);
+ }
+ } finally {
+ writer.close();
+ }
+ });
+
+ it('reports setup failure and leaves no temp copy when the driver cannot load', async () => {
+ const dir = scratch();
+ const result = await captureConsistentSnapshot(join(dir, 'missing.sqlite'), 'ab'.repeat(32), {
+ tmpRoot: dir,
+ loadDriver() {
+ throw new Error('no prebuild for this platform');
+ },
+ });
+ expect(result.ok).toBe(false);
+ expect(result.reason).toBe('setup');
+ expect(result.remediation).toContain(SQLCIPHER_DRIVER_PACKAGE);
+ expect(readdirSync(dir)).toEqual([]);
+ });
+
+ it('rejects a bad key without leaving a temp copy or changing the source', async () => {
+ const dir = scratch();
+ const { src, writer } = buildLiveWal(dir);
+ const wal = `${src}-wal`;
+ const before = { db: digest(src), wal: digest(wal) };
+ try {
+ const result = await captureConsistentSnapshot(src, 'cd'.repeat(32), { tmpRoot: dir });
+ expect(result.ok).toBe(false);
+ expect(result.reason).toBe('auth');
+ expect(digest(src)).toBe(before.db);
+ expect(digest(wal)).toBe(before.wal);
+ expect(readdirSync(dir).some((name) => name.startsWith('portos-signal-'))).toBe(false);
+ } finally {
+ writer.close();
+ }
+ });
+});
diff --git a/server/services/signalSync.js b/server/services/signalSync.js
index e21d354cad..2626baee0d 100644
--- a/server/services/signalSync.js
+++ b/server/services/signalSync.js
@@ -9,17 +9,20 @@
* schema, a missing key, or a failed decryption surfaces an actionable sync-status
* error and NEVER crashes or retry-loops.
*
- * Design constraints (docs/plans/2026-07-04-human-activity-tracking.md):
+ * Design constraints (docs/plans/2026-07-04-human-activity-tracking.md, #11149):
*
- * - **Zero new dependencies.** The SQLCipher-4 page decryption + Chromium
- * safeStorage key-unwrap are implemented in pure `node:crypto`
- * (`server/lib/signalCrypto.js`) — `node:sqlite` cannot open a SQLCipher file
- * and shelling to a `sqlcipher` CLI would add a tool most machines lack. We
- * decrypt to a plaintext temp copy, then read THAT with the built-in
- * `node:sqlite` (readOnly). The macOS keychain read shells to the system
- * `security` CLI (no dependency).
- * - **Never touch Signal's live file.** The DB is copied to a temp snapshot before
- * decryption; the original `db.sqlite` is opened only for the `cp`.
+ * - **Consistent snapshot, not a main-file copy.** Signal runs in WAL mode, so
+ * committed messages can live only in the WAL until the next checkpoint.
+ * `signalSnapshot.js` opens the live file read-only with the bundled
+ * SQLCipher driver, holds a read transaction, and exports one committed
+ * view (WAL frames included) to a private plaintext database. `node:sqlite`
+ * then reads that plaintext file. Key unwrap stays in `server/lib/signalCrypto.js`.
+ * The macOS keychain read shells to the system `security` CLI.
+ * - **Fail closed.** If the driver prebuild cannot load, or the export is not
+ * one integrity-clean snapshot, sync returns a setup/snapshot error and does
+ * not advance the cursor. A stale main-file copy is not a fallback.
+ * - **Never write Signal's live database or WAL.** No checkpoint, no rekey, no
+ * journal-mode change on the source. Only the private temp copy is decrypted.
* - **Machine-local.** The incremental cursor lives in a `data/` JSON file — the DB
* and its rowids are per-machine. Derived events land in the already-machine-local
* `human_activity_events` table (excluded from peer sync).
@@ -35,16 +38,15 @@
import { DatabaseSync } from 'node:sqlite';
import { promisify } from 'node:util';
import { execFile } from '../lib/childProcess.js';
-import { copyFile, mkdtemp, writeFile, rm } from 'node:fs/promises';
-import { statSync } from 'node:fs';
-import { tmpdir } from 'node:os';
+import { rm } from 'node:fs/promises';
import { join } from 'node:path';
import { expandHome, dataPath, atomicWrite, tryReadFile, safeJSONParse } from '../lib/fileUtils.js';
import { getUserTimezone } from './userTimezone.js';
import { identityFromHandle } from '../lib/tribeMatch.js';
-import { decryptSqlcipherDatabase, decryptSafeStorageValue, isRawHexKey } from '../lib/signalCrypto.js';
+import { decryptSafeStorageValue, isRawHexKey } from '../lib/signalCrypto.js';
import { shortSummary, localDayKey } from './humanActivity.js';
+import { captureConsistentSnapshot } from './signalSnapshot.js';
import { getSettings } from './settings.js';
const execFileAsync = promisify(execFile);
@@ -313,42 +315,14 @@ export async function resolveDbKey() {
}
// ---------------------------------------------------------------------------
-// DB access — copy → decrypt → open plaintext with node:sqlite (readOnly).
+// DB access — read-only SQLCipher snapshot, then node:sqlite on the plaintext.
// ---------------------------------------------------------------------------
-// Copy the live DB to a temp snapshot, decrypt it to a sibling plaintext file,
-// and return both paths for cleanup. Never touches Signal's live file beyond the
-// read for `cp`. Returns { ok, plaintextPath, tmpDir } or an error report.
+// Export one committed view of the live database, including WAL frames, to a
+// private plaintext file. Returns { ok, plaintextPath, tmpDir } or an error
+// report. Does not write the source database or WAL. See signalSnapshot.js.
async function openDecryptedSnapshot(key) {
- const src = dbPath();
- try { statSync(src); } catch (err) {
- const missing = err?.code === 'ENOENT';
- return {
- ok: false,
- error: missing ? `Signal database not found at ${src}` : `Cannot access ${src}: ${err?.message || err}`,
- remediation: missing ? 'Signal Desktop has no chat database yet (never opened, or a different profile).' : 'Could not read the Signal database file — check file permissions for the process running PortOS.',
- };
- }
- const tmp = await mkdtemp(join(tmpdir(), 'portos-signal-'));
- const encPath = join(tmp, 'db.enc.sqlite');
- const plainPath = join(tmp, 'db.plain.sqlite');
- await copyFile(src, encPath);
- const encBuf = await tryReadFile(encPath, { encoding: null });
- if (!encBuf || !encBuf.length) {
- await rm(tmp, { recursive: true, force: true });
- return { ok: false, error: 'Signal database snapshot is empty', remediation: 'The Signal database copy was empty — try again with Signal closed.' };
- }
- const decrypted = decryptSqlcipherDatabase(encBuf, key, { verify: 'first' });
- if (!decrypted.ok) {
- await rm(tmp, { recursive: true, force: true });
- return {
- ok: false,
- error: `Signal database decryption failed: ${decrypted.error}`,
- remediation: 'Signal database decryption failed — the key or SQLCipher format may have changed in this Signal version (not yet supported).',
- };
- }
- await writeFile(plainPath, decrypted.plaintext);
- return { ok: true, plaintextPath: plainPath, tmpDir: tmp };
+ return captureConsistentSnapshot(dbPath(), key);
}
// Confirm the decrypted DB carries the Signal schema we understand. On an
@@ -424,7 +398,7 @@ export async function checkSetup() {
}
const snapshot = await openDecryptedSnapshot(keyResult.key);
if (!snapshot.ok) {
- return { ok: false, dbPath: dbPath(), error: snapshot.error, remediation: snapshot.remediation };
+ return { ok: false, dbPath: dbPath(), error: snapshot.error, remediation: snapshot.remediation, reason: snapshot.reason };
}
let db;
try {
@@ -473,7 +447,7 @@ async function doRunSync() {
const snapshot = await openDecryptedSnapshot(keyResult.key);
if (!snapshot.ok) {
console.error(`❌ Signal sync: ${snapshot.error}`);
- return { ok: false, error: snapshot.error, remediation: snapshot.remediation };
+ return { ok: false, error: snapshot.error, remediation: snapshot.remediation, reason: snapshot.reason };
}
let db;
diff --git a/server/services/signalSync.wal.test.js b/server/services/signalSync.wal.test.js
new file mode 100644
index 0000000000..077429afa1
--- /dev/null
+++ b/server/services/signalSync.wal.test.js
@@ -0,0 +1,145 @@
+/**
+ * Signal sync consumes the consistent snapshot (#11149): both the checkpointed
+ * message and the committed WAL-only message are ingested, a second pass is a
+ * no-op, and an authentication failure leaves the cursor and activity calls
+ * untouched. The database is an encrypted fixture, not a live Signal profile.
+ */
+import { createRequire } from 'node:module';
+import { createHash } from 'node:crypto';
+import { mkdtempSync, readFileSync, readdirSync, rmSync, writeFileSync } from 'node:fs';
+import { tmpdir } from 'node:os';
+import { join } from 'node:path';
+import { afterAll, beforeAll, beforeEach, describe, expect, it, vi } from 'vitest';
+
+const { recordEventsMock, autoLogTouchpointsMock } = vi.hoisted(() => ({
+ recordEventsMock: vi.fn(),
+ autoLogTouchpointsMock: vi.fn(),
+}));
+
+const dataRoot = vi.hoisted(() => {
+ const { mkdtempSync: mkdtemp } = require('node:fs');
+ const { tmpdir: osTmp } = require('node:os');
+ const { join: joinPath } = require('node:path');
+ return mkdtemp(joinPath(osTmp(), 'signal-sync-wal-'));
+});
+
+vi.mock('../lib/fileUtils.js', async () => {
+ const actual = await vi.importActual('../lib/fileUtils.js');
+ const { join: joinPath } = await import('node:path');
+ return {
+ ...actual,
+ dataPath: (...segments) => joinPath(dataRoot, ...segments),
+ };
+});
+
+vi.mock('./humanActivity.js', async () => {
+ const actual = await vi.importActual('./humanActivity.js');
+ return { ...actual, recordEvents: recordEventsMock };
+});
+
+vi.mock('./tribe.js', () => ({
+ autoLogTouchpoints: autoLogTouchpointsMock,
+}));
+
+import { SQLCIPHER_DRIVER_PACKAGE } from './signalSnapshot.js';
+import { runSync } from './signalSync.js';
+
+const driverRequire = createRequire(import.meta.url);
+const Database = driverRequire(SQLCIPHER_DRIVER_PACKAGE);
+
+const dbDir = mkdtempSync(join(tmpdir(), 'signal-sync-wal-db-'));
+const dbPath = join(dbDir, 'db.sqlite');
+const configPath = join(dbDir, 'config.json');
+const key = 'ab'.repeat(32);
+let writer;
+
+function digest(path) {
+ return createHash('sha256').update(readFileSync(path)).digest('hex');
+}
+
+function signalTemps() {
+ return readdirSync(tmpdir()).filter((name) => name.startsWith('portos-signal-'));
+}
+
+beforeAll(() => {
+ writer = new Database(dbPath);
+ writer.pragma(`cipher='sqlcipher'`);
+ writer.pragma('legacy=4');
+ writer.pragma(`key="x'${key}'"`);
+ writer.pragma('journal_mode = WAL');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.exec(`
+ CREATE TABLE conversations (
+ id TEXT PRIMARY KEY, e164 TEXT, type TEXT, name TEXT, profileName TEXT
+ );
+ CREATE TABLE messages (
+ id TEXT, conversationId TEXT, type TEXT, sent_at INTEGER, received_at INTEGER, body TEXT
+ );
+ INSERT INTO conversations VALUES ('c1', '+15551230001', 'private', 'Ada', 'Ada');
+ INSERT INTO messages VALUES ('m1', 'c1', 'incoming', 1700000000000, 1700000000000, 'checkpointed');
+ `);
+ writer.pragma('wal_checkpoint(TRUNCATE)');
+ writer.pragma('wal_autocheckpoint = 0');
+ writer.exec(`INSERT INTO messages VALUES ('m2', 'c1', 'outgoing', 1700000001000, 1700000001000, 'wal-only')`);
+ writeFileSync(configPath, JSON.stringify({ key }));
+ process.env.SIGNAL_DB_PATH = dbPath;
+ process.env.SIGNAL_CONFIG_PATH = configPath;
+});
+
+afterAll(() => {
+ try { writer?.close(); } catch { /* already closed */ }
+ delete process.env.SIGNAL_DB_PATH;
+ delete process.env.SIGNAL_CONFIG_PATH;
+ rmSync(dbDir, { recursive: true, force: true });
+ rmSync(dataRoot, { recursive: true, force: true });
+});
+
+beforeEach(() => {
+ recordEventsMock.mockReset();
+ autoLogTouchpointsMock.mockReset();
+ recordEventsMock.mockResolvedValue({ recorded: 2, skipped: 0 });
+ autoLogTouchpointsMock.mockResolvedValue({ created: 1, matched: 1 });
+ rmSync(join(dataRoot, 'signal-sync-state.json'), { force: true });
+ writeFileSync(configPath, JSON.stringify({ key }));
+});
+
+describe('runSync WAL snapshot', () => {
+ it('ingests both committed messages once, then stays idempotent, without writing the source', async () => {
+ const wal = `${dbPath}-wal`;
+ const before = { db: digest(dbPath), wal: digest(wal) };
+ const tempsBefore = new Set(signalTemps());
+
+ const first = await runSync();
+ expect(first.ok, first.error).toBe(true);
+ expect(first.scanned).toBe(2);
+ expect(first.cursorRowid).toBe(2);
+ const recorded = recordEventsMock.mock.calls[0][0].map((row) => row.dedupeKey).sort();
+ expect(recorded).toEqual(['signal:m1', 'signal:m2']);
+ expect(digest(dbPath)).toBe(before.db);
+ expect(digest(wal)).toBe(before.wal);
+ expect(signalTemps().filter((name) => !tempsBefore.has(name))).toEqual([]);
+
+ const second = await runSync();
+ expect(second.ok).toBe(true);
+ expect(second.scanned).toBe(0);
+ expect(second.cursorRowid).toBe(2);
+ expect(recordEventsMock.mock.calls[1][0]).toEqual([]);
+ });
+
+ it('leaves the cursor and activity calls untouched when the key is rejected', async () => {
+ const first = await runSync();
+ expect(first.ok, first.error).toBe(true);
+ const calls = recordEventsMock.mock.calls.length;
+
+ writeFileSync(configPath, JSON.stringify({ key: 'cd'.repeat(32) }));
+ const failed = await runSync();
+ expect(failed.ok).toBe(false);
+ expect(failed.reason).toBe('auth');
+ expect(failed.error).toMatch(/decryption failed/i);
+
+ const state = JSON.parse(readFileSync(join(dataRoot, 'signal-sync-state.json'), 'utf8'));
+ expect(state.cursorRowid).toBe(first.cursorRowid);
+ expect(recordEventsMock.mock.calls.length).toBe(calls);
+ expect(signalTemps()).toEqual([]);
+ });
+});