Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(wallet): refuse to seal custodial keys under a key production does not hold

Nicholas433581f82ca4
diff --git a/api/_lib/secret-box.js b/api/_lib/secret-box.js
index 7c957da21..450a72fd4 100644
--- a/api/_lib/secret-box.js
+++ b/api/_lib/secret-box.js
@@ -128,11 +128,83 @@ async function deriveKey(secret, salt) {
);
}
+// ── Write-key guard ─────────────────────────────────────────────────────────
+//
+// Why this exists: 24 custodial wallets in the production database are sealed
+// under a key production never held. They were not lost in a rotation. A
+// deployment that shared the database but carried its own key (a Vercel-era
+// preview in 2026-06, dev and preview servers in 2026-08 and 2026-09) wrote them,
+// and non-production code is allowed to encrypt under whatever key it has. The
+// first time anyone noticed was a customer who could not withdraw 99.95 USDC.
+//
+// The guard binds a database to the key that writes into it. The first write
+// records an HMAC fingerprint of the key in app_settings; every later write
+// compares against it and refuses on a mismatch, so a process holding the wrong
+// key fails loudly at provisioning instead of silently minting an unopenable
+// wallet. A rotation done by the runbook (outgoing key kept in
+// WALLET_ENCRYPTION_KEY_PREVIOUS) moves the fingerprint forward instead of
+// refusing. The fingerprint is a one-way HMAC, so storing it reveals nothing.
+export const WRITE_KEY_SETTING = 'secret_box_write_key';
+const FINGERPRINT_LABEL = new TextEncoder().encode('three.ws/secret-box/write-key-fingerprint/v1');
+
+export async function secretBoxKeyFingerprint(secret) {
+ const key = await subtle.importKey('raw', new TextEncoder().encode(secret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);
+ return Buffer.from(await subtle.sign('HMAC', key, FINGERPRINT_LABEL)).toString('hex');
+}
+
+export class SecretBoxKeyMismatchError extends Error {
+ constructor() {
+ super(
+ '[secret-box] refusing to encrypt: this process holds a different encryption key than ' +
+ 'the one bound to this database, so anything it sealed could never be opened by ' +
+ 'production. Point it at its own database, or give it the production key.',
+ );
+ this.name = 'SecretBoxKeyMismatchError';
+ this.code = 'secret_box_key_mismatch';
+ }
+}
+
+/**
+ * Bind `sql`'s database to `secret`, or throw SecretBoxKeyMismatchError.
+ * Exported for tests and for operator tooling that seeds the binding.
+ */
+export async function verifyWriteKey(sql, secret, retired = []) {
+ const fingerprint = await secretBoxKeyFingerprint(secret);
+ const value = JSON.stringify({ fingerprint, bound_at: new Date().toISOString() });
+ await sql`INSERT INTO app_settings (key, value) VALUES (${WRITE_KEY_SETTING}, ${value}::jsonb) ON CONFLICT (key) DO NOTHING`;
+ const [row] = await sql`SELECT value FROM app_settings WHERE key = ${WRITE_KEY_SETTING}`;
+ const bound = row?.value?.fingerprint;
+ if (bound === fingerprint) return { fingerprint, rotated: false };
+ const retiredPrints = await Promise.all(retired.map(secretBoxKeyFingerprint));
+ if (bound && retiredPrints.includes(bound)) {
+ await sql`UPDATE app_settings SET value = ${value}::jsonb WHERE key = ${WRITE_KEY_SETTING} AND value->>'fingerprint' = ${bound}`;
+ return { fingerprint, rotated: true };
+ }
+ throw new SecretBoxKeyMismatchError();
+}
+
+let _verifiedWriteKey = null;
+
+// Unit tests encrypt against mocked or absent databases, and a process with no
+// database has nowhere to strand a wallet, so both skip the guard. A failed
+// check is not cached: a transient database error retries on the next write.
+async function guardWriteKey(secret) {
+ if (process.env.VITEST || _verifiedWriteKey === secret) return;
+ let hasDb = false;
+ try { hasDb = Boolean(env.DATABASE_URL); } catch { hasDb = false; }
+ if (!hasDb) return;
+ const { sql } = await import('./db.js');
+ await verifyWriteKey(sql, secret, retiredSecrets());
+ _verifiedWriteKey = secret;
+}
+
// v2 layout: "v2:" + base64( salt[16] || iv[12] || ciphertext+tag ).
export async function encryptSecret(plaintext) {
const salt = randomBytes(16);
const iv = randomBytes(12);
- const key = await deriveKey(walletMasterSecret(), salt);
+ const secret = walletMasterSecret();
+ await guardWriteKey(secret);
+ const key = await deriveKey(secret, salt);
const data = new TextEncoder().encode(plaintext);
const ct = await subtle.encrypt({ name: 'AES-GCM', iv }, key, data);
const buf = new Uint8Array(salt.length + iv.length + ct.byteLength);
diff --git a/docs/ops/stranded-wallets.md b/docs/ops/stranded-wallets.md
index 270d27673..95b7f2e27 100644
--- a/docs/ops/stranded-wallets.md
+++ b/docs/ops/stranded-wallets.md
@@ -1,12 +1,12 @@
# Stranded custodial wallets: the standing owner decision
-Some custodial Solana wallets on three.ws are sealed under an encryption key the
-platform no longer holds. Their SOL is visible on chain, keeps rendering in the
-product, and can never be signed for again. Two of those wallets belong to
-customers who cannot withdraw.
+Some custodial Solana wallets on three.ws are sealed under an encryption key
+production does not hold. Their balance is visible on chain and keeps rendering in
+the product, but cannot be signed for until the sealing key is found. Two of those
+wallets belong to customers who cannot withdraw; one also holds 99.94637 USDC.
-This is the decision brief. It carries the measurement, why recovery is
-impossible, what each option costs, and the exact commands for whichever option
+This is the decision brief. It carries the measurement, the recovery path, what each
+option costs if recovery fails, and the exact commands for whichever option
the owner picks. It does not move money: crediting, contacting, or writing off a
customer balance is the owner's call.
@@ -16,26 +16,76 @@ decision that incident left open.
## What happened
-The 2026-07 Vercel to Cloud Run migration rotated `WALLET_ENCRYPTION_KEY`. Every
-custodial secret written under the retired value became permanently unopenable
-the moment the new key went live, because nothing kept a copy of the old one.
-
-`api/_lib/secret-box.js` has since gained a retired-key read path
-(`WALLET_ENCRYPTION_KEY_PREVIOUS`, comma separated, tried newest first, and a
-legacy `JWT_SECRET` candidate for v1 records). That path works. It has nothing to
-put in it:
-
-- Secret Manager holds exactly ONE version of `WALLET_ENCRYPTION_KEY`, created on
- migration day. There is no prior version to roll back to.
-- The migration-era `JWT_SECRET` and the current one were both tried against the
- sealed ciphertext during the 2026-07 investigation. Neither decrypted.
-- AES-GCM authenticates, so there is no "try harder" path: a wrong key throws
- rather than returning plausible-but-wrong material. There is no brute force
- worth attempting against a >=32-character master secret.
-
-Recovery is therefore not slow or expensive. It is impossible. The only open
-questions are what we owe the affected customers and what we do with the
-platform's own sealed dust.
+**Corrected 2026-09-16.** This brief used to say the 2026-07 Vercel to Cloud Run
+migration rotated `WALLET_ENCRYPTION_KEY` and lost the old value. The measurement
+says otherwise:
+
+- Every wallet the production API wrote in June 2026, including a "My First Agent"
+ created the same day as the stranded customer wallet, opens with **today's**
+ production key. Production's key never changed.
+- The stranded customer wallet failed its first withdrawal on 2026-06-30, a week
+ **before** the migration, while production held that same key.
+- Wallets kept being sealed after the migration: 13 QA agents on 2026-08-16 (headless
+ Chrome) and one on 2026-09-03 (a session whose user agent was `home15-preview`).
+- Every `three-ws-api` Cloud Run revision since 2026-07-07, `agent-sniper`, and the
+ `wallet-encryption-key` / `agent-orders-*` secrets carry one key value, the current
+ one. None of them opens a sealed wallet.
+
+The real cause: **a deployment that shared the production database but carried its
+own key** wrote those wallets. `secret-box.js` only requires the dedicated key when
+`VERCEL_ENV` or `NODE_ENV` is `production`, so a preview or dev server pointed at
+production silently encrypted under whatever `WALLET_ENCRYPTION_KEY` or `JWT_SECRET`
+it had. In June that was a Vercel-era deployment other than production (a preview
+environment or a second Vercel project on the same database); in August and
+September, dev and preview servers.
+
+That changes the recovery question. The keys were never in Google Cloud, so
+searching it again finds nothing. They may still exist where those deployments
+kept their env: **the Vercel dashboard** (every project and every environment that
+had this `DATABASE_URL`, Settings > Environment Variables; non-sensitive values are
+viewable there) and any local `.env` a developer ran against production. A database
+backup does not help on its own: it holds the same ciphertext, never the key.
+
+### It cannot happen again
+
+`encryptSecret` now binds a database to the key that writes into it
+(`verifyWriteKey` in `api/_lib/secret-box.js`). The first write records an HMAC
+fingerprint of the key in `app_settings` under `secret_box_write_key`; any later
+process holding a different key is refused with `secret_box_key_mismatch` before it
+can seal anything. A runbook rotation (outgoing key kept in
+`WALLET_ENCRYPTION_KEY_PREVIOUS`) moves the binding forward instead of refusing.
+Production was bound on 2026-09-16 with
+`node scripts/recover-sealed-wallets.mjs --bind`.
+
+## Recovering a sealed wallet (no funds move)
+
+`scripts/recover-sealed-wallets.mjs` opens a sealed secret with a candidate key,
+proves it signs for the stored address, and re-encrypts it under the production key
+with a compare-and-swap on the old ciphertext. The address never changes, so the
+owner's balance simply becomes withdrawable. Candidate keys go in on stdin, one per
+line, and are never printed.
+
+```bash
+export DATABASE_URL="$(node scripts/read-service-env.mjs '^DATABASE_URL$' --raw)"
+export WALLET_ENCRYPTION_KEY="$(node scripts/read-service-env.mjs '^WALLET_ENCRYPTION_KEY$' --raw)"
+node scripts/recover-sealed-wallets.mjs --list # the sealed set
+node scripts/recover-sealed-wallets.mjs < candidate-keys.txt # dry run: which key opens which wallet
+node scripts/recover-sealed-wallets.mjs --apply < candidate-keys.txt
+```
+
+Try the `WALLET_ENCRYPTION_KEY` and `JWT_SECRET` of every Vercel project and
+environment that pointed at this database. Delete the candidate file afterwards.
+
+### The funded customer wallet is a real customer
+
+Checked on chain 2026-09-16 for `GemVS5fT958FKRe5fpgizohUYUKE8cUDueEdmB1bmXnm`
+(agent `5e05f68f-eead-4ef9-b6b4-fc85ea73bbe9`): 10 + 89.94527 USDC and 0.25 SOL
+arrived on 2026-06-27 from `58uQ7w8qPvDcD4WsUqtZqCkiMyJrz1fEeMZmuRPaPbZv`. Minutes
+later a look-alike address (`58uQT1Y1…PbZv`) and `5811kC…5bZv` sent dust: that is
+address poisoning aimed at the customer. The account was created from Phantom's iOS
+in-app browser, and every withdrawal asked for either the genuine funding wallet or
+the owner's own login wallet `7jWU2UBX7nKEFSXndbjR3NoSmgeqhL6Fh9fA1Vd7GaRe`, never
+the poisoned address. Nothing indicates abuse.
## Measured state
diff --git a/scripts/recover-sealed-wallets.mjs b/scripts/recover-sealed-wallets.mjs
new file mode 100644
index 000000000..5d3680847
--- /dev/null
+++ b/scripts/recover-sealed-wallets.mjs
@@ -0,0 +1,199 @@
+#!/usr/bin/env node
+// Recover custodial wallets sealed under a key production does not hold, in place.
+//
+// A sealed wallet is not a lost wallet. Its ciphertext is intact in the database;
+// only the key that sealed it is missing from production. Hand this script that
+// key and it opens the secret, proves the secret really belongs to the stored
+// address, and re-encrypts it under the CURRENT production key. The address does
+// not change and no funds move: the owner's balance simply becomes withdrawable.
+//
+// Where the missing keys came from (measured 2026-09-16): every sealed wallet was
+// written by a deployment that shared the production database but carried its own
+// WALLET_ENCRYPTION_KEY or JWT_SECRET. June 2026 wallets came from a Vercel-era
+// deployment other than production (a preview environment or a second project on
+// the same database); August and September ones from dev and preview servers.
+// None of those values exist in Secret Manager or any Cloud Run revision. The
+// place to look is the Vercel dashboard (Project > Settings > Environment
+// Variables, every project and every environment that pointed at this database)
+// and any local .env a developer ran against production. secret-box.js now
+// refuses such writes, so the set can only shrink.
+//
+// Candidate keys are read from STDIN, one per line, so they never land in shell
+// history or a process listing. Values are never printed.
+//
+// Usage (production key and database pulled from the live service):
+// export DATABASE_URL="$(node scripts/read-service-env.mjs '^DATABASE_URL$' --raw)"
+// export WALLET_ENCRYPTION_KEY="$(node scripts/read-service-env.mjs '^WALLET_ENCRYPTION_KEY$' --raw)"
+// node scripts/recover-sealed-wallets.mjs --list # which wallets are sealed
+// node scripts/recover-sealed-wallets.mjs < candidate-keys.txt # dry run: which keys open which wallets
+// node scripts/recover-sealed-wallets.mjs --apply < candidate-keys.txt
+// node scripts/recover-sealed-wallets.mjs --bind # bind the database to the production key
+//
+// Exit codes: 0 ok, 2 bad invocation, 3 no production key configured.
+
+import { readFileSync } from 'node:fs';
+import { webcrypto } from 'node:crypto';
+import { sql } from '../api/_lib/db.js';
+import { decryptSecret, encryptSecret, secretBoxKeyCandidates, verifyWriteKey } from '../api/_lib/secret-box.js';
+
+const args = new Set(process.argv.slice(2));
+const APPLY = args.has('--apply');
+const LIST = args.has('--list');
+const BIND = args.has('--bind');
+const subtle = webcrypto.subtle;
+
+const FIELDS = [
+ { field: 'encrypted_solana_secret', kind: 'solana' },
+ { field: 'encrypted_wallet_key', kind: 'evm' },
+];
+
+// Same derivation as secret-box.js, but against one explicit candidate instead of
+// the environment's candidate list.
+async function openWith(ciphertext, secret) {
+ if (!ciphertext.startsWith('v2:')) return null;
+ const raw = Buffer.from(ciphertext.slice(3), 'base64');
+ const base = await subtle.importKey('raw', new TextEncoder().encode(secret), 'HKDF', false, ['deriveKey']);
+ const key = await subtle.deriveKey(
+ { name: 'HKDF', hash: 'SHA-256', salt: raw.subarray(0, 16), info: new Uint8Array(0) },
+ base,
+ { name: 'AES-GCM', length: 256 },
+ false,
+ ['decrypt'],
+ );
+ try {
+ const plain = await subtle.decrypt({ name: 'AES-GCM', iv: raw.subarray(16, 28) }, key, raw.subarray(28));
+ return new TextDecoder().decode(plain);
+ } catch {
+ return null;
+ }
+}
+
+async function opensInProduction(ciphertext) {
+ try {
+ await decryptSecret(ciphertext);
+ return true;
+ } catch {
+ return false;
+ }
+}
+
+// A decrypt that succeeds under the wrong record would be caught by AES-GCM, but
+// the address check also proves the stored address is the one the secret signs for.
+async function addressOf(kind, plaintext) {
+ if (kind === 'solana') {
+ const { Keypair } = await import('@solana/web3.js');
+ return Keypair.fromSecretKey(Buffer.from(plaintext, 'base64')).publicKey.toBase58();
+ }
+ const { computeAddress } = await import('ethers');
+ return computeAddress(plaintext);
+}
+
+function readCandidates() {
+ if (process.stdin.isTTY) return [];
+ return [...new Set(readFileSync(0, 'utf8').split('\n').map((l) => l.replace(/\r$/, '')).filter((l) => l.trim().length >= 16))];
+}
+
+async function sealedRecords() {
+ const rows = await sql`
+ SELECT id, name, user_id, created_at, deleted_at, wallet_address, meta
+ FROM agent_identities
+ WHERE meta ? 'encrypted_solana_secret' OR meta ? 'encrypted_wallet_key'
+ ORDER BY created_at
+ `;
+ const sealed = [];
+ for (const row of rows) {
+ for (const { field, kind } of FIELDS) {
+ const ciphertext = row.meta?.[field];
+ if (typeof ciphertext !== 'string' || !ciphertext) continue;
+ if (await opensInProduction(ciphertext)) continue;
+ sealed.push({
+ agentId: row.id,
+ name: row.name,
+ createdAt: new Date(row.created_at).toISOString(),
+ deleted: Boolean(row.deleted_at),
+ field,
+ kind,
+ ciphertext,
+ address: kind === 'solana' ? row.meta.solana_address : row.wallet_address,
+ });
+ }
+ }
+ return sealed;
+}
+
+async function rewrap(record, plaintext) {
+ const fresh = await encryptSecret(plaintext);
+ if ((await decryptSecret(fresh)) !== plaintext) throw new Error('re-encrypted secret did not round-trip');
+ // Compare-and-swap on the old ciphertext so a concurrent writer is never clobbered.
+ const updated = await sql`
+ UPDATE agent_identities
+ SET meta = jsonb_set(meta, ${[record.field]}::text[], to_jsonb(${fresh}::text))
+ || jsonb_build_object('custody_key_rewrapped_at', now())
+ WHERE id = ${record.agentId} AND meta->>${record.field} = ${record.ciphertext}
+ RETURNING id
+ `;
+ return updated.length === 1;
+}
+
+async function main() {
+ if (secretBoxKeyCandidates().length === 0) {
+ console.error('No production key configured. Export WALLET_ENCRYPTION_KEY from the live service first (see header).');
+ process.exit(3);
+ }
+ const [current] = secretBoxKeyCandidates();
+
+ if (BIND) {
+ const result = await verifyWriteKey(sql, current);
+ console.log(`database bound to the production key (fingerprint ${result.fingerprint.slice(0, 12)}...)`);
+ return;
+ }
+
+ const sealed = await sealedRecords();
+ console.log(`${sealed.length} sealed secret(s) across ${new Set(sealed.map((r) => r.agentId)).size} agent(s)`);
+
+ if (LIST) {
+ for (const r of sealed) {
+ console.log(` ${r.agentId} ${r.createdAt.slice(0, 10)} ${r.kind.padEnd(6)} ${r.address || '(no address)'}${r.deleted ? ' [deleted]' : ''} ${r.name}`);
+ }
+ return;
+ }
+
+ const candidates = readCandidates();
+ if (candidates.length === 0) {
+ console.error('No candidate keys on stdin. Pipe them in, one per line, or pass --list or --bind.');
+ process.exit(2);
+ }
+ console.log(`${candidates.length} candidate key(s) supplied; ${APPLY ? 'APPLY: matching secrets will be re-encrypted' : 'dry run'}`);
+
+ let recovered = 0;
+ for (const record of sealed) {
+ let match = null;
+ for (let i = 0; i < candidates.length && !match; i++) {
+ const plaintext = await openWith(record.ciphertext, candidates[i]);
+ if (plaintext !== null) match = { index: i, plaintext };
+ }
+ const label = `${record.agentId} ${record.kind} ${record.address || ''}`;
+ if (!match) {
+ console.log(` sealed ${label}`);
+ continue;
+ }
+ const derived = await addressOf(record.kind, match.plaintext);
+ if (record.address && derived !== record.address) {
+ console.log(` MISMATCH ${label} opens with candidate #${match.index + 1} but signs for ${derived}; left untouched`);
+ continue;
+ }
+ if (!APPLY) {
+ console.log(` opens ${label} with candidate #${match.index + 1}`);
+ continue;
+ }
+ const ok = await rewrap(record, match.plaintext);
+ console.log(` ${ok ? 'recovered ' : 'CHANGED '} ${label}${ok ? '' : ' (record changed underneath; rerun)'}`);
+ if (ok) recovered++;
+ }
+ if (APPLY) console.log(`${recovered} secret(s) re-encrypted under the production key`);
+}
+
+main().catch((err) => {
+ console.error(err?.message || err);
+ process.exit(1);
+});
diff --git a/tests/secret-box.test.js b/tests/secret-box.test.js
index 1239e0550..9c31e7722 100644
--- a/tests/secret-box.test.js
+++ b/tests/secret-box.test.js
@@ -105,3 +105,61 @@ describe('secret-box encrypt/decrypt', () => {
process.env.JWT_SECRET = JWT; // restore for other tests
});
});
+
+// In-memory app_settings standing in for the database, interpreting exactly the
+// three statements verifyWriteKey issues.
+function fakeSettingsDb() {
+ const rows = new Map();
+ const sql = async (strings, ...values) => {
+ const text = strings.join('?');
+ if (text.startsWith('INSERT')) {
+ if (!rows.has(values[0])) rows.set(values[0], JSON.parse(values[1]));
+ return [];
+ }
+ if (text.startsWith('SELECT')) return rows.has(values[0]) ? [{ value: rows.get(values[0]) }] : [];
+ if (text.startsWith('UPDATE')) {
+ if (rows.get(values[1])?.fingerprint === values[2]) rows.set(values[1], JSON.parse(values[0]));
+ return [];
+ }
+ throw new Error(`unexpected statement: ${text}`);
+ };
+ return { sql, rows };
+}
+
+describe('secret-box write-key guard', () => {
+ const PROD = 'production-wallet-key-000000000000000000000';
+ const ROGUE = 'preview-server-own-key-11111111111111111111';
+
+ it('binds a database to the first key that writes into it', async () => {
+ const { verifyWriteKey, secretBoxKeyFingerprint, WRITE_KEY_SETTING } = await load();
+ const db = fakeSettingsDb();
+ await expect(verifyWriteKey(db.sql, PROD)).resolves.toEqual({ fingerprint: await secretBoxKeyFingerprint(PROD), rotated: false });
+ expect(db.rows.get(WRITE_KEY_SETTING).fingerprint).toBe(await secretBoxKeyFingerprint(PROD));
+ await expect(verifyWriteKey(db.sql, PROD)).resolves.toMatchObject({ rotated: false });
+ });
+
+ it('refuses a process holding a different key than the bound one', async () => {
+ const { verifyWriteKey, SecretBoxKeyMismatchError } = await load();
+ const db = fakeSettingsDb();
+ await verifyWriteKey(db.sql, PROD);
+ await expect(verifyWriteKey(db.sql, ROGUE)).rejects.toBeInstanceOf(SecretBoxKeyMismatchError);
+ await expect(verifyWriteKey(db.sql, ROGUE, ['some-other-retired-key-2222222222222'])).rejects.toMatchObject({ code: 'secret_box_key_mismatch' });
+ });
+
+ it('moves the binding forward on a runbook rotation (outgoing key kept as retired)', async () => {
+ const { verifyWriteKey, secretBoxKeyFingerprint, WRITE_KEY_SETTING } = await load();
+ const db = fakeSettingsDb();
+ await verifyWriteKey(db.sql, PROD);
+ const NEXT = 'rotated-production-key-33333333333333333333';
+ await expect(verifyWriteKey(db.sql, NEXT, [PROD])).resolves.toMatchObject({ rotated: true });
+ expect(db.rows.get(WRITE_KEY_SETTING).fingerprint).toBe(await secretBoxKeyFingerprint(NEXT));
+ await expect(verifyWriteKey(db.sql, PROD)).rejects.toMatchObject({ code: 'secret_box_key_mismatch' });
+ });
+
+ it('never stores anything that reveals the key', async () => {
+ const { verifyWriteKey, WRITE_KEY_SETTING } = await load();
+ const db = fakeSettingsDb();
+ await verifyWriteKey(db.sql, PROD);
+ expect(JSON.stringify(db.rows.get(WRITE_KEY_SETTING))).not.toContain(PROD);
+ });
+});