Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(wallet): refuse to seal custodial keys under a key production does not hold
Nicholas433581f82ca4
diff --git a/api/_lib/secret-box.js b/api/_lib/secret-box.jsindex 7c957da21..450a72fd4 100644--- a/api/_lib/secret-box.js+++ b/api/_lib/secret-box.js@@ -128,11 +128,83 @@ async function deriveKey(secret, salt) {);}+// ── Write-key guard ─────────────────────────────────────────────────────────+//+// Why this exists: 24 custodial wallets in the production database are sealed+// under a key production never held. They were not lost in a rotation. A+// deployment that shared the database but carried its own key (a Vercel-era+// preview in 2026-06, dev and preview servers in 2026-08 and 2026-09) wrote them,+// and non-production code is allowed to encrypt under whatever key it has. The+// first time anyone noticed was a customer who could not withdraw 99.95 USDC.+//+// The guard binds a database to the key that writes into it. The first write+// records an HMAC fingerprint of the key in app_settings; every later write+// compares against it and refuses on a mismatch, so a process holding the wrong+// key fails loudly at provisioning instead of silently minting an unopenable+// wallet. A rotation done by the runbook (outgoing key kept in+// WALLET_ENCRYPTION_KEY_PREVIOUS) moves the fingerprint forward instead of+// refusing. The fingerprint is a one-way HMAC, so storing it reveals nothing.+export const WRITE_KEY_SETTING = 'secret_box_write_key';+const FINGERPRINT_LABEL = new TextEncoder().encode('three.ws/secret-box/write-key-fingerprint/v1');++export async function secretBoxKeyFingerprint(secret) {+ const key = await subtle.importKey('raw', new TextEncoder().encode(secret), { name: 'HMAC', hash: 'SHA-256' }, false, ['sign']);+ return Buffer.from(await subtle.sign('HMAC', key, FINGERPRINT_LABEL)).toString('hex');+}++export class SecretBoxKeyMismatchError extends Error {+ constructor() {+ super(+ '[secret-box] refusing to encrypt: this process holds a different encryption key than ' ++ 'the one bound to this database, so anything it sealed could never be opened by ' ++ 'production. Point it at its own database, or give it the production key.',+ );+ this.name = 'SecretBoxKeyMismatchError';+ this.code = 'secret_box_key_mismatch';+ }+}++/**+ * Bind `sql`'s database to `secret`, or throw SecretBoxKeyMismatchError.+ * Exported for tests and for operator tooling that seeds the binding.+ */+export async function verifyWriteKey(sql, secret, retired = []) {+ const fingerprint = await secretBoxKeyFingerprint(secret);+ const value = JSON.stringify({ fingerprint, bound_at: new Date().toISOString() });+ await sql`INSERT INTO app_settings (key, value) VALUES (${WRITE_KEY_SETTING}, ${value}::jsonb) ON CONFLICT (key) DO NOTHING`;+ const [row] = await sql`SELECT value FROM app_settings WHERE key = ${WRITE_KEY_SETTING}`;+ const bound = row?.value?.fingerprint;+ if (bound === fingerprint) return { fingerprint, rotated: false };+ const retiredPrints = await Promise.all(retired.map(secretBoxKeyFingerprint));+ if (bound && retiredPrints.includes(bound)) {+ await sql`UPDATE app_settings SET value = ${value}::jsonb WHERE key = ${WRITE_KEY_SETTING} AND value->>'fingerprint' = ${bound}`;+ return { fingerprint, rotated: true };+ }+ throw new SecretBoxKeyMismatchError();+}++let _verifiedWriteKey = null;++// Unit tests encrypt against mocked or absent databases, and a process with no+// database has nowhere to strand a wallet, so both skip the guard. A failed+// check is not cached: a transient database error retries on the next write.+async function guardWriteKey(secret) {+ if (process.env.VITEST || _verifiedWriteKey === secret) return;+ let hasDb = false;+ try { hasDb = Boolean(env.DATABASE_URL); } catch { hasDb = false; }+ if (!hasDb) return;+ const { sql } = await import('./db.js');+ await verifyWriteKey(sql, secret, retiredSecrets());+ _verifiedWriteKey = secret;+}+// v2 layout: "v2:" + base64( salt[16] || iv[12] || ciphertext+tag ).export async function encryptSecret(plaintext) {const salt = randomBytes(16);const iv = randomBytes(12);- const key = await deriveKey(walletMasterSecret(), salt);+ const secret = walletMasterSecret();+ await guardWriteKey(secret);+ const key = await deriveKey(secret, salt);const data = new TextEncoder().encode(plaintext);const ct = await subtle.encrypt({ name: 'AES-GCM', iv }, key, data);const buf = new Uint8Array(salt.length + iv.length + ct.byteLength);diff --git a/docs/ops/stranded-wallets.md b/docs/ops/stranded-wallets.mdindex 270d27673..95b7f2e27 100644--- a/docs/ops/stranded-wallets.md+++ b/docs/ops/stranded-wallets.md@@ -1,12 +1,12 @@# Stranded custodial wallets: the standing owner decision-Some custodial Solana wallets on three.ws are sealed under an encryption key the-platform no longer holds. Their SOL is visible on chain, keeps rendering in the-product, and can never be signed for again. Two of those wallets belong to-customers who cannot withdraw.+Some custodial Solana wallets on three.ws are sealed under an encryption key+production does not hold. Their balance is visible on chain and keeps rendering in+the product, but cannot be signed for until the sealing key is found. Two of those+wallets belong to customers who cannot withdraw; one also holds 99.94637 USDC.-This is the decision brief. It carries the measurement, why recovery is-impossible, what each option costs, and the exact commands for whichever option+This is the decision brief. It carries the measurement, the recovery path, what each+option costs if recovery fails, and the exact commands for whichever optionthe owner picks. It does not move money: crediting, contacting, or writing off acustomer balance is the owner's call.@@ -16,26 +16,76 @@ decision that incident left open.## What happened-The 2026-07 Vercel to Cloud Run migration rotated `WALLET_ENCRYPTION_KEY`. Every-custodial secret written under the retired value became permanently unopenable-the moment the new key went live, because nothing kept a copy of the old one.--`api/_lib/secret-box.js` has since gained a retired-key read path-(`WALLET_ENCRYPTION_KEY_PREVIOUS`, comma separated, tried newest first, and a-legacy `JWT_SECRET` candidate for v1 records). That path works. It has nothing to-put in it:--- Secret Manager holds exactly ONE version of `WALLET_ENCRYPTION_KEY`, created on- migration day. There is no prior version to roll back to.-- The migration-era `JWT_SECRET` and the current one were both tried against the- sealed ciphertext during the 2026-07 investigation. Neither decrypted.-- AES-GCM authenticates, so there is no "try harder" path: a wrong key throws- rather than returning plausible-but-wrong material. There is no brute force- worth attempting against a >=32-character master secret.--Recovery is therefore not slow or expensive. It is impossible. The only open-questions are what we owe the affected customers and what we do with the-platform's own sealed dust.+**Corrected 2026-09-16.** This brief used to say the 2026-07 Vercel to Cloud Run+migration rotated `WALLET_ENCRYPTION_KEY` and lost the old value. The measurement+says otherwise:++- Every wallet the production API wrote in June 2026, including a "My First Agent"+ created the same day as the stranded customer wallet, opens with **today's**+ production key. Production's key never changed.+- The stranded customer wallet failed its first withdrawal on 2026-06-30, a week+ **before** the migration, while production held that same key.+- Wallets kept being sealed after the migration: 13 QA agents on 2026-08-16 (headless+ Chrome) and one on 2026-09-03 (a session whose user agent was `home15-preview`).+- Every `three-ws-api` Cloud Run revision since 2026-07-07, `agent-sniper`, and the+ `wallet-encryption-key` / `agent-orders-*` secrets carry one key value, the current+ one. None of them opens a sealed wallet.++The real cause: **a deployment that shared the production database but carried its+own key** wrote those wallets. `secret-box.js` only requires the dedicated key when+`VERCEL_ENV` or `NODE_ENV` is `production`, so a preview or dev server pointed at+production silently encrypted under whatever `WALLET_ENCRYPTION_KEY` or `JWT_SECRET`+it had. In June that was a Vercel-era deployment other than production (a preview+environment or a second Vercel project on the same database); in August and+September, dev and preview servers.++That changes the recovery question. The keys were never in Google Cloud, so+searching it again finds nothing. They may still exist where those deployments+kept their env: **the Vercel dashboard** (every project and every environment that+had this `DATABASE_URL`, Settings > Environment Variables; non-sensitive values are+viewable there) and any local `.env` a developer ran against production. A database+backup does not help on its own: it holds the same ciphertext, never the key.++### It cannot happen again++`encryptSecret` now binds a database to the key that writes into it+(`verifyWriteKey` in `api/_lib/secret-box.js`). The first write records an HMAC+fingerprint of the key in `app_settings` under `secret_box_write_key`; any later+process holding a different key is refused with `secret_box_key_mismatch` before it+can seal anything. A runbook rotation (outgoing key kept in+`WALLET_ENCRYPTION_KEY_PREVIOUS`) moves the binding forward instead of refusing.+Production was bound on 2026-09-16 with+`node scripts/recover-sealed-wallets.mjs --bind`.++## Recovering a sealed wallet (no funds move)++`scripts/recover-sealed-wallets.mjs` opens a sealed secret with a candidate key,+proves it signs for the stored address, and re-encrypts it under the production key+with a compare-and-swap on the old ciphertext. The address never changes, so the+owner's balance simply becomes withdrawable. Candidate keys go in on stdin, one per+line, and are never printed.++```bash+export DATABASE_URL="$(node scripts/read-service-env.mjs '^DATABASE_URL$' --raw)"+export WALLET_ENCRYPTION_KEY="$(node scripts/read-service-env.mjs '^WALLET_ENCRYPTION_KEY$' --raw)"+node scripts/recover-sealed-wallets.mjs --list # the sealed set+node scripts/recover-sealed-wallets.mjs < candidate-keys.txt # dry run: which key opens which wallet+node scripts/recover-sealed-wallets.mjs --apply < candidate-keys.txt+```++Try the `WALLET_ENCRYPTION_KEY` and `JWT_SECRET` of every Vercel project and+environment that pointed at this database. Delete the candidate file afterwards.++### The funded customer wallet is a real customer++Checked on chain 2026-09-16 for `GemVS5fT958FKRe5fpgizohUYUKE8cUDueEdmB1bmXnm`+(agent `5e05f68f-eead-4ef9-b6b4-fc85ea73bbe9`): 10 + 89.94527 USDC and 0.25 SOL+arrived on 2026-06-27 from `58uQ7w8qPvDcD4WsUqtZqCkiMyJrz1fEeMZmuRPaPbZv`. Minutes+later a look-alike address (`58uQT1Y1…PbZv`) and `5811kC…5bZv` sent dust: that is+address poisoning aimed at the customer. The account was created from Phantom's iOS+in-app browser, and every withdrawal asked for either the genuine funding wallet or+the owner's own login wallet `7jWU2UBX7nKEFSXndbjR3NoSmgeqhL6Fh9fA1Vd7GaRe`, never+the poisoned address. Nothing indicates abuse.## Measured statediff --git a/scripts/recover-sealed-wallets.mjs b/scripts/recover-sealed-wallets.mjsnew file mode 100644index 000000000..5d3680847--- /dev/null+++ b/scripts/recover-sealed-wallets.mjs@@ -0,0 +1,199 @@+#!/usr/bin/env node+// Recover custodial wallets sealed under a key production does not hold, in place.+//+// A sealed wallet is not a lost wallet. Its ciphertext is intact in the database;+// only the key that sealed it is missing from production. Hand this script that+// key and it opens the secret, proves the secret really belongs to the stored+// address, and re-encrypts it under the CURRENT production key. The address does+// not change and no funds move: the owner's balance simply becomes withdrawable.+//+// Where the missing keys came from (measured 2026-09-16): every sealed wallet was+// written by a deployment that shared the production database but carried its own+// WALLET_ENCRYPTION_KEY or JWT_SECRET. June 2026 wallets came from a Vercel-era+// deployment other than production (a preview environment or a second project on+// the same database); August and September ones from dev and preview servers.+// None of those values exist in Secret Manager or any Cloud Run revision. The+// place to look is the Vercel dashboard (Project > Settings > Environment+// Variables, every project and every environment that pointed at this database)+// and any local .env a developer ran against production. secret-box.js now+// refuses such writes, so the set can only shrink.+//+// Candidate keys are read from STDIN, one per line, so they never land in shell+// history or a process listing. Values are never printed.+//+// Usage (production key and database pulled from the live service):+// export DATABASE_URL="$(node scripts/read-service-env.mjs '^DATABASE_URL$' --raw)"+// export WALLET_ENCRYPTION_KEY="$(node scripts/read-service-env.mjs '^WALLET_ENCRYPTION_KEY$' --raw)"+// node scripts/recover-sealed-wallets.mjs --list # which wallets are sealed+// node scripts/recover-sealed-wallets.mjs < candidate-keys.txt # dry run: which keys open which wallets+// node scripts/recover-sealed-wallets.mjs --apply < candidate-keys.txt+// node scripts/recover-sealed-wallets.mjs --bind # bind the database to the production key+//+// Exit codes: 0 ok, 2 bad invocation, 3 no production key configured.++import { readFileSync } from 'node:fs';+import { webcrypto } from 'node:crypto';+import { sql } from '../api/_lib/db.js';+import { decryptSecret, encryptSecret, secretBoxKeyCandidates, verifyWriteKey } from '../api/_lib/secret-box.js';++const args = new Set(process.argv.slice(2));+const APPLY = args.has('--apply');+const LIST = args.has('--list');+const BIND = args.has('--bind');+const subtle = webcrypto.subtle;++const FIELDS = [+ { field: 'encrypted_solana_secret', kind: 'solana' },+ { field: 'encrypted_wallet_key', kind: 'evm' },+];++// Same derivation as secret-box.js, but against one explicit candidate instead of+// the environment's candidate list.+async function openWith(ciphertext, secret) {+ if (!ciphertext.startsWith('v2:')) return null;+ const raw = Buffer.from(ciphertext.slice(3), 'base64');+ const base = await subtle.importKey('raw', new TextEncoder().encode(secret), 'HKDF', false, ['deriveKey']);+ const key = await subtle.deriveKey(+ { name: 'HKDF', hash: 'SHA-256', salt: raw.subarray(0, 16), info: new Uint8Array(0) },+ base,+ { name: 'AES-GCM', length: 256 },+ false,+ ['decrypt'],+ );+ try {+ const plain = await subtle.decrypt({ name: 'AES-GCM', iv: raw.subarray(16, 28) }, key, raw.subarray(28));+ return new TextDecoder().decode(plain);+ } catch {+ return null;+ }+}++async function opensInProduction(ciphertext) {+ try {+ await decryptSecret(ciphertext);+ return true;+ } catch {+ return false;+ }+}++// A decrypt that succeeds under the wrong record would be caught by AES-GCM, but+// the address check also proves the stored address is the one the secret signs for.+async function addressOf(kind, plaintext) {+ if (kind === 'solana') {+ const { Keypair } = await import('@solana/web3.js');+ return Keypair.fromSecretKey(Buffer.from(plaintext, 'base64')).publicKey.toBase58();+ }+ const { computeAddress } = await import('ethers');+ return computeAddress(plaintext);+}++function readCandidates() {+ if (process.stdin.isTTY) return [];+ return [...new Set(readFileSync(0, 'utf8').split('\n').map((l) => l.replace(/\r$/, '')).filter((l) => l.trim().length >= 16))];+}++async function sealedRecords() {+ const rows = await sql`+ SELECT id, name, user_id, created_at, deleted_at, wallet_address, meta+ FROM agent_identities+ WHERE meta ? 'encrypted_solana_secret' OR meta ? 'encrypted_wallet_key'+ ORDER BY created_at+ `;+ const sealed = [];+ for (const row of rows) {+ for (const { field, kind } of FIELDS) {+ const ciphertext = row.meta?.[field];+ if (typeof ciphertext !== 'string' || !ciphertext) continue;+ if (await opensInProduction(ciphertext)) continue;+ sealed.push({+ agentId: row.id,+ name: row.name,+ createdAt: new Date(row.created_at).toISOString(),+ deleted: Boolean(row.deleted_at),+ field,+ kind,+ ciphertext,+ address: kind === 'solana' ? row.meta.solana_address : row.wallet_address,+ });+ }+ }+ return sealed;+}++async function rewrap(record, plaintext) {+ const fresh = await encryptSecret(plaintext);+ if ((await decryptSecret(fresh)) !== plaintext) throw new Error('re-encrypted secret did not round-trip');+ // Compare-and-swap on the old ciphertext so a concurrent writer is never clobbered.+ const updated = await sql`+ UPDATE agent_identities+ SET meta = jsonb_set(meta, ${[record.field]}::text[], to_jsonb(${fresh}::text))+ || jsonb_build_object('custody_key_rewrapped_at', now())+ WHERE id = ${record.agentId} AND meta->>${record.field} = ${record.ciphertext}+ RETURNING id+ `;+ return updated.length === 1;+}++async function main() {+ if (secretBoxKeyCandidates().length === 0) {+ console.error('No production key configured. Export WALLET_ENCRYPTION_KEY from the live service first (see header).');+ process.exit(3);+ }+ const [current] = secretBoxKeyCandidates();++ if (BIND) {+ const result = await verifyWriteKey(sql, current);+ console.log(`database bound to the production key (fingerprint ${result.fingerprint.slice(0, 12)}...)`);+ return;+ }++ const sealed = await sealedRecords();+ console.log(`${sealed.length} sealed secret(s) across ${new Set(sealed.map((r) => r.agentId)).size} agent(s)`);++ if (LIST) {+ for (const r of sealed) {+ console.log(` ${r.agentId} ${r.createdAt.slice(0, 10)} ${r.kind.padEnd(6)} ${r.address || '(no address)'}${r.deleted ? ' [deleted]' : ''} ${r.name}`);+ }+ return;+ }++ const candidates = readCandidates();+ if (candidates.length === 0) {+ console.error('No candidate keys on stdin. Pipe them in, one per line, or pass --list or --bind.');+ process.exit(2);+ }+ console.log(`${candidates.length} candidate key(s) supplied; ${APPLY ? 'APPLY: matching secrets will be re-encrypted' : 'dry run'}`);++ let recovered = 0;+ for (const record of sealed) {+ let match = null;+ for (let i = 0; i < candidates.length && !match; i++) {+ const plaintext = await openWith(record.ciphertext, candidates[i]);+ if (plaintext !== null) match = { index: i, plaintext };+ }+ const label = `${record.agentId} ${record.kind} ${record.address || ''}`;+ if (!match) {+ console.log(` sealed ${label}`);+ continue;+ }+ const derived = await addressOf(record.kind, match.plaintext);+ if (record.address && derived !== record.address) {+ console.log(` MISMATCH ${label} opens with candidate #${match.index + 1} but signs for ${derived}; left untouched`);+ continue;+ }+ if (!APPLY) {+ console.log(` opens ${label} with candidate #${match.index + 1}`);+ continue;+ }+ const ok = await rewrap(record, match.plaintext);+ console.log(` ${ok ? 'recovered ' : 'CHANGED '} ${label}${ok ? '' : ' (record changed underneath; rerun)'}`);+ if (ok) recovered++;+ }+ if (APPLY) console.log(`${recovered} secret(s) re-encrypted under the production key`);+}++main().catch((err) => {+ console.error(err?.message || err);+ process.exit(1);+});diff --git a/tests/secret-box.test.js b/tests/secret-box.test.jsindex 1239e0550..9c31e7722 100644--- a/tests/secret-box.test.js+++ b/tests/secret-box.test.js@@ -105,3 +105,61 @@ describe('secret-box encrypt/decrypt', () => {process.env.JWT_SECRET = JWT; // restore for other tests});});++// In-memory app_settings standing in for the database, interpreting exactly the+// three statements verifyWriteKey issues.+function fakeSettingsDb() {+ const rows = new Map();+ const sql = async (strings, ...values) => {+ const text = strings.join('?');+ if (text.startsWith('INSERT')) {+ if (!rows.has(values[0])) rows.set(values[0], JSON.parse(values[1]));+ return [];+ }+ if (text.startsWith('SELECT')) return rows.has(values[0]) ? [{ value: rows.get(values[0]) }] : [];+ if (text.startsWith('UPDATE')) {+ if (rows.get(values[1])?.fingerprint === values[2]) rows.set(values[1], JSON.parse(values[0]));+ return [];+ }+ throw new Error(`unexpected statement: ${text}`);+ };+ return { sql, rows };+}++describe('secret-box write-key guard', () => {+ const PROD = 'production-wallet-key-000000000000000000000';+ const ROGUE = 'preview-server-own-key-11111111111111111111';++ it('binds a database to the first key that writes into it', async () => {+ const { verifyWriteKey, secretBoxKeyFingerprint, WRITE_KEY_SETTING } = await load();+ const db = fakeSettingsDb();+ await expect(verifyWriteKey(db.sql, PROD)).resolves.toEqual({ fingerprint: await secretBoxKeyFingerprint(PROD), rotated: false });+ expect(db.rows.get(WRITE_KEY_SETTING).fingerprint).toBe(await secretBoxKeyFingerprint(PROD));+ await expect(verifyWriteKey(db.sql, PROD)).resolves.toMatchObject({ rotated: false });+ });++ it('refuses a process holding a different key than the bound one', async () => {+ const { verifyWriteKey, SecretBoxKeyMismatchError } = await load();+ const db = fakeSettingsDb();+ await verifyWriteKey(db.sql, PROD);+ await expect(verifyWriteKey(db.sql, ROGUE)).rejects.toBeInstanceOf(SecretBoxKeyMismatchError);+ await expect(verifyWriteKey(db.sql, ROGUE, ['some-other-retired-key-2222222222222'])).rejects.toMatchObject({ code: 'secret_box_key_mismatch' });+ });++ it('moves the binding forward on a runbook rotation (outgoing key kept as retired)', async () => {+ const { verifyWriteKey, secretBoxKeyFingerprint, WRITE_KEY_SETTING } = await load();+ const db = fakeSettingsDb();+ await verifyWriteKey(db.sql, PROD);+ const NEXT = 'rotated-production-key-33333333333333333333';+ await expect(verifyWriteKey(db.sql, NEXT, [PROD])).resolves.toMatchObject({ rotated: true });+ expect(db.rows.get(WRITE_KEY_SETTING).fingerprint).toBe(await secretBoxKeyFingerprint(NEXT));+ await expect(verifyWriteKey(db.sql, PROD)).rejects.toMatchObject({ code: 'secret_box_key_mismatch' });+ });++ it('never stores anything that reveals the key', async () => {+ const { verifyWriteKey, WRITE_KEY_SETTING } = await load();+ const db = fakeSettingsDb();+ await verifyWriteKey(db.sql, PROD);+ expect(JSON.stringify(db.rows.get(WRITE_KEY_SETTING))).not.toContain(PROD);+ });+});