Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(custodial): share one platform-account boundary between the sweepback and the audit

nirholasf85284230b02
diff --git a/api/_lib/custodial-key-health.js b/api/_lib/custodial-key-health.js
new file mode 100644
index 000000000..5cd721265
--- /dev/null
+++ b/api/_lib/custodial-key-health.js
@@ -0,0 +1,338 @@
+// @ts-check
+// api/_lib/custodial-key-health.js
+//
+// One measurement of "how much custody have we lost to a key rotation?", shared
+// by every surface that asks the question.
+//
+// A custodial wallet whose `encrypted_solana_secret` no longer decrypts is not a
+// degraded wallet, it is a destroyed one: the SOL stays visible on chain and can
+// never be signed for again. The 2026-07 Vercel to Cloud Run migration rotated
+// WALLET_ENCRYPTION_KEY with no read path back to the retired value, and the
+// wallets written under it were sealed permanently (docs/ops/wallet-key-migration.md).
+// The balance keeps rendering in the product, the treasury self-heal keeps
+// planning to reclaim it, and nothing in either path knows the key is gone.
+//
+// This module is the single source of that number so the CLI audit
+// (scripts/audit-custodial-key-health.mjs) and the ops board
+// (/api/ops/payment-outcomes → `stranded_custody`) cannot drift apart. It is
+// READ-ONLY: it decrypts in memory to test the key, never writes, never signs,
+// never broadcasts.
+//
+// Two invariants the callers depend on:
+// • Ownership is classified with the SAME predicate the reclaim leg enforces
+// in SQL (isPlatformOwnedAgent below, re-exported by economy-sweepback.js).
+// The audit script used to carry its own copy with a different owner email,
+// which filed 12 platform wallets as CUSTOMER ones and would have inflated
+// the customer support obligation in exactly the report written to size it.
+// • A wallet whose balance was never READ is never counted as zero. Coalescing
+// "unread" into "0" is what let this audit certify "0 SOL stranded" on
+// 2026-08-09 while two wallets held 0.12 SOL behind a dead key: every total
+// here sums confirmed reads only, and `counts.stranded_unread` is the flag
+// that says a zero is not yet trustworthy.
+
+import { sql } from './db.js';
+import { decryptSecret } from './secret-box.js';
+
+/** The platform's own agent-owner account. Anything else is a customer. */
+export const PLATFORM_AGENT_OWNER_EMAIL = '[email protected]';
+/** Per-agent mailbox domain the platform issues its own agents. */
+export const PLATFORM_AGENT_EMAIL_SUFFIX = '@agents.three.ws';
+
+/** Balances are read in getMultipleAccounts batches of this size. */
+const BALANCE_CHUNK = 100;
+const LAMPORTS_PER_SOL = 1_000_000_000;
+
+/**
+ * True only for the platform's own agent-owner accounts. Anything else (a real
+ * signup, a wallet-auth account, an unknown value) is a customer and is out.
+ *
+ * @param {string|null|undefined} ownerEmail
+ * @returns {boolean}
+ */
+export function isPlatformOwnedAgent(ownerEmail) {
+ if (!ownerEmail || typeof ownerEmail !== 'string') return false;
+ const email = ownerEmail.trim().toLowerCase();
+ return email === PLATFORM_AGENT_OWNER_EMAIL || email.endsWith(PLATFORM_AGENT_EMAIL_SUFFIX);
+}
+
+const round9 = (n) => Number(Number(n).toFixed(9));
+
+/**
+ * Every custodial wallet the platform stores a secret for, oldest first.
+ *
+ * @param {{platformOnly?:boolean}} [opts]
+ * @returns {Promise<Array<{agent_id:string,name:string,created_at:string|Date,address:string,secret:string,owner:string}>>}
+ */
+export async function readCustodialWalletRows({ platformOnly = false } = {}) {
+ const rows = await sql`
+ SELECT a.id AS agent_id,
+ a.name AS name,
+ a.created_at AS created_at,
+ a.meta->>'solana_address' AS address,
+ a.meta->>'encrypted_solana_secret' AS secret,
+ LOWER(u.email) AS owner
+ FROM agent_identities a
+ JOIN users u ON u.id = a.user_id
+ WHERE a.deleted_at IS NULL
+ AND a.meta->>'solana_address' IS NOT NULL
+ AND a.meta->>'encrypted_solana_secret' IS NOT NULL
+ ORDER BY a.created_at ASC
+ `;
+ return platformOnly ? rows.filter((r) => isPlatformOwnedAgent(r.owner)) : rows;
+}
+
+/**
+ * Test every stored secret against the live key candidates. Decrypt IS the test:
+ * AES-GCM authenticates, so a wrong key throws rather than returning wrong
+ * plaintext, and the recovered material is never used beyond checking it exists.
+ *
+ * @param {Array<{secret:string}>} rows mutated in place with `decryptable` + `reason`
+ * @param {(secret:string)=>Promise<string|Uint8Array|null>} [decrypt] injectable for tests
+ * @returns {Promise<Array<any>>} the same rows
+ */
+export async function classifyCustodialSecrets(rows, decrypt = decryptSecret) {
+ for (const w of rows) {
+ try {
+ const plain = await decrypt(w.secret);
+ w.decryptable = Boolean(plain && plain.length > 0);
+ w.reason = w.decryptable ? null : 'empty_plaintext';
+ } catch (e) {
+ w.decryptable = false;
+ w.reason = e?.name === 'OperationError' ? 'wrong_key' : e?.message || 'decrypt_failed';
+ }
+ }
+ return rows;
+}
+
+/**
+ * Read on-chain SOL for every wallet, 100 addresses per RPC call. An address
+ * whose chunk failed gets NO map entry, which is what keeps an unread balance
+ * from summing as a confirmed zero downstream.
+ *
+ * @param {import('@solana/web3.js').Connection} connection
+ * @param {Array<{address:string}>} wallets
+ * @returns {Promise<{balances:Map<string,number>, readErrors:Array<{address:string,reason:string}>}>}
+ */
+export async function readCustodialBalances(connection, wallets) {
+ const { PublicKey } = await import('@solana/web3.js');
+ const balances = new Map();
+ const readErrors = [];
+ for (let i = 0; i < wallets.length; i += BALANCE_CHUNK) {
+ const chunk = wallets.slice(i, i + BALANCE_CHUNK);
+ let keys;
+ try {
+ keys = chunk.map((w) => new PublicKey(w.address));
+ } catch (e) {
+ for (const w of chunk) readErrors.push({ address: w.address, reason: `bad_address: ${e?.message}` });
+ continue;
+ }
+ try {
+ const infos = await connection.getMultipleAccountsInfo(keys);
+ infos.forEach((info, idx) => balances.set(chunk[idx].address, (info?.lamports ?? 0) / LAMPORTS_PER_SOL));
+ } catch (e) {
+ for (const w of chunk) readErrors.push({ address: w.address, reason: `rpc_error: ${e?.message}` });
+ }
+ }
+ return { balances, readErrors };
+}
+
+/**
+ * Turn classified wallets + confirmed balances into the report every caller
+ * renders. Pure: no DB, no RPC, so the counting rules are unit-testable.
+ *
+ * @param {object} args
+ * @param {Array<{agent_id?:string,name?:string,address:string,owner:string,created_at?:any,decryptable?:boolean,reason?:string|null}>} args.wallets
+ * @param {Map<string,number>} args.balances confirmed reads only
+ * @param {Array<{address:string,reason:string}>} [args.readErrors]
+ * @param {string} [args.rpc] label for the lane the balances came from
+ * @param {number|Date} [args.now]
+ * @param {number} [args.topN]
+ * @param {number|null} [args.keyCandidates] how many keys a decrypt had to try
+ * @returns {object}
+ */
+export function summarizeCustodialKeyHealth({ wallets, balances, readErrors = [], rpc = 'unknown', now = Date.now(), topN = 20, keyCandidates = null }) {
+ const sum = (list) => list.reduce((t, w) => (balances.has(w.address) ? t + balances.get(w.address) : t), 0);
+ const unread = (list) => list.filter((w) => !balances.has(w.address));
+
+ const platform = wallets.filter((w) => isPlatformOwnedAgent(w.owner));
+ const customer = wallets.filter((w) => !isPlatformOwnedAgent(w.owner));
+ const stranded = wallets.filter((w) => !w.decryptable);
+ // "Funded" requires a CONFIRMED positive balance; a stranded wallet whose
+ // balance we never read is neither funded nor cleared, it is unknown.
+ const strandedFunded = stranded.filter((w) => balances.has(w.address) && balances.get(w.address) > 0);
+ const strandedUnread = unread(stranded);
+ const describe = (w) => ({
+ agent_id: w.agent_id || null,
+ name: w.name || null,
+ address: w.address,
+ owner: w.owner,
+ reason: w.reason,
+ created_at: w.created_at || null,
+ platform: isPlatformOwnedAgent(w.owner),
+ });
+
+ return {
+ checked_at: new Date(now).toISOString(),
+ rpc,
+ wallets: wallets.length,
+ // How many keys a decrypt had to try. 1 is the healthy steady state; a
+ // higher count means retired keys are still configured for the migration.
+ key_candidates: keyCandidates,
+ read_errors: readErrors.length,
+ decryptable: wallets.length - stranded.length,
+ undecryptable: stranded.length,
+ sol: {
+ // Every figure below sums CONFIRMED reads only (sum() skips addresses
+ // with no balances entry). Never trust them as complete while
+ // counts.stranded_unread is above zero.
+ total: round9(sum(wallets)),
+ decryptable: round9(sum(wallets.filter((w) => w.decryptable))),
+ stranded: round9(sum(stranded)),
+ stranded_platform: round9(sum(stranded.filter((w) => isPlatformOwnedAgent(w.owner)))),
+ stranded_customer: round9(sum(stranded.filter((w) => !isPlatformOwnedAgent(w.owner)))),
+ },
+ counts: {
+ platform: platform.length,
+ customer: customer.length,
+ stranded_platform: stranded.filter((w) => isPlatformOwnedAgent(w.owner)).length,
+ stranded_customer: stranded.filter((w) => !isPlatformOwnedAgent(w.owner)).length,
+ stranded_funded: strandedFunded.length,
+ stranded_unread: strandedUnread.length,
+ },
+ top_stranded: strandedFunded
+ .slice()
+ .sort((a, b) => (balances.get(b.address) || 0) - (balances.get(a.address) || 0))
+ .slice(0, topN)
+ .map((w) => ({ ...describe(w), sol: round9(balances.get(w.address) || 0) })),
+ unread_stranded: strandedUnread.map(describe),
+ };
+}
+
+/**
+ * The whole measurement: read the fleet, test every key, read every balance.
+ *
+ * @param {object} [args]
+ * @param {import('@solana/web3.js').Connection} [args.connection] defaults to the rotating production lane
+ * @param {boolean} [args.platformOnly]
+ * @param {string|null} [args.rpcUrl]
+ * @param {number} [args.topN]
+ * @param {number|null} [args.keyCandidates]
+ * @returns {Promise<object>} the report shape summarizeCustodialKeyHealth builds
+ */
+export async function gatherCustodialKeyHealth({ connection, platformOnly = false, rpcUrl = null, topN = 20, keyCandidates = null } = {}) {
+ const wallets = await readCustodialWalletRows({ platformOnly });
+ await classifyCustodialSecrets(wallets);
+ // The rotating multi-lane connection production reads balances through, not a
+ // single bare endpoint: a script pinned to one RPC URL dies the moment that
+ // lane rate-limits the caller (observed 2026-08-09, and the failure was
+ // invisible because unread balances silently summed as zero).
+ let conn = connection;
+ if (!conn) {
+ const { solanaConnection } = await import('./solana/connection.js');
+ conn = solanaConnection({ url: rpcUrl || process.env.SOLANA_RPC_URL || null });
+ }
+ const { balances, readErrors } = await readCustodialBalances(conn, wallets);
+ return summarizeCustodialKeyHealth({
+ wallets,
+ balances,
+ readErrors,
+ rpc: rpcUrl || process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
+ topN,
+ keyCandidates,
+ });
+}
+
+/**
+ * How long a stranded-custody snapshot stays good. Custody loss is a rotation
+ * event, not a live metric: it changes when someone rotates a key or a sealed
+ * wallet receives a deposit, never between two dashboard refreshes. Six hours
+ * keeps an ops board honest while making the read effectively free (one fleet
+ * scan per instance per quarter-day instead of one per request).
+ */
+export const STRANDED_SNAPSHOT_TTL_MS = 6 * 60 * 60 * 1000;
+
+/** @type {{at:number, panel:object}|null} */
+let cached = null;
+/** @type {Promise<object>|null} */
+let inFlight = null;
+
+/**
+ * The ops-board panel: stranded wallet count, SOL, and the platform/customer
+ * split, cached for STRANDED_SNAPSHOT_TTL_MS and de-duplicated across concurrent
+ * requests so a polled dashboard never turns into fleet-wide RPC load.
+ *
+ * @param {object} [args]
+ * @param {number} [args.ttlMs]
+ * @param {boolean} [args.force] recompute even if the cache is warm
+ * @param {() => Promise<object>} [args.gather] injectable for tests
+ * @param {number} [args.now]
+ * @returns {Promise<object>}
+ */
+export async function strandedCustodyPanel({ ttlMs = STRANDED_SNAPSHOT_TTL_MS, force = false, gather = gatherCustodialKeyHealth, now = Date.now() } = {}) {
+ if (!force && cached && now - cached.at < ttlMs) {
+ return { ...cached.panel, cache_age_seconds: Math.round((now - cached.at) / 1000) };
+ }
+ if (!inFlight) {
+ inFlight = (async () => {
+ const report = await gather();
+ const panel = buildStrandedPanel(report);
+ cached = { at: Date.now(), panel };
+ return panel;
+ })().finally(() => {
+ inFlight = null;
+ });
+ }
+ const panel = await inFlight;
+ return { ...panel, cache_age_seconds: 0 };
+}
+
+/** Drop the memoized snapshot. Tests and a forced ops refresh use this. */
+export function resetStrandedCustodyCache() {
+ cached = null;
+ inFlight = null;
+}
+
+/**
+ * Reduce a full key-health report to the board panel. Pure.
+ *
+ * @param {object} report as returned by summarizeCustodialKeyHealth
+ * @returns {object}
+ */
+export function buildStrandedPanel(report) {
+ const counts = report?.counts || {};
+ const sol = report?.sol || {};
+ const unread = counts.stranded_unread || 0;
+ const funded = counts.stranded_funded || 0;
+ // A total is only a total when every sealed wallet actually got a balance
+ // read. While any is unread the SOL figures are a FLOOR, and the panel says
+ // so rather than letting a partial read render as a clean number.
+ const status = unread > 0 ? 'unknown' : funded > 0 ? 'stranded' : 'clear';
+ return {
+ status,
+ checked_at: report?.checked_at || null,
+ wallets: report?.wallets ?? null,
+ undecryptable: report?.undecryptable ?? 0,
+ stranded_funded: funded,
+ stranded_unread: unread,
+ sol_stranded: sol.stranded ?? 0,
+ sol_stranded_platform: sol.stranded_platform ?? 0,
+ sol_stranded_customer: sol.stranded_customer ?? 0,
+ customer_wallets_stranded: (report?.top_stranded || []).filter((w) => !w.platform).length,
+ detail:
+ status === 'clear'
+ ? 'No funded custodial wallet sits behind an undecryptable key.'
+ : status === 'unknown'
+ ? `${unread} sealed wallet(s) never got a balance read, so the stranded total is a floor, not a measurement.`
+ : `${funded} custodial wallet(s) hold ${sol.stranded ?? 0} SOL behind a retired encryption key (platform ${sol.stranded_platform ?? 0}, customer ${sol.stranded_customer ?? 0}).`,
+ // Customer money that cannot be withdrawn is a support obligation, not an
+ // ops number: the board names the decision doc rather than leaving the
+ // reader to rediscover that recovery is impossible.
+ brief: (sol.stranded_customer || 0) > 0 ? 'docs/ops/stranded-wallets.md' : null,
+ top_stranded: (report?.top_stranded || []).slice(0, 10).map((w) => ({
+ address: w.address,
+ sol: w.sol,
+ platform: w.platform,
+ reason: w.reason,
+ })),
+ };
+}
diff --git a/api/_lib/economy-sweepback.js b/api/_lib/economy-sweepback.js
index 455ebb31c..606b79136 100644
--- a/api/_lib/economy-sweepback.js
+++ b/api/_lib/economy-sweepback.js
@@ -31,6 +31,11 @@ import { sendSol, LAMPORTS_PER_SOL } from './avatar-wallet.js';
import { submitProtected } from './execution-engine.js';
import { MIN_OPERATIONAL_WALLET_SOL } from './agent-trade-guards.js';
import { antiOscillationFloorSol } from './agent-funding-policy.js';
+import {
+ PLATFORM_AGENT_OWNER_EMAIL,
+ PLATFORM_AGENT_EMAIL_SUFFIX,
+ isPlatformOwnedAgent,
+} from './custodial-key-health.js';
// Same float the treasury-topup cron refills to — sweep only above it.
const DEFAULT_REFILL_MULTIPLE = 3;
@@ -450,10 +455,10 @@ export async function reclaimIdleSol({ connection, network = 'mainnet', dryRun =
// customer's agent is never touched, and the SQL gate plus a redundant in-JS
// re-check make that true even if the query is later edited.
-/** House account that owns the platform's own (non-bot) agents. */
-export const PLATFORM_AGENT_OWNER_EMAIL = '[email protected]';
-/** Email suffix of the platform-created circulation bot accounts. */
-export const PLATFORM_AGENT_EMAIL_SUFFIX = '@agents.three.ws';
+// One definition, shared with the custodial key-health audit: two copies of this
+// boundary drifted apart once already (the audit carried a different house-account
+// address and filed 12 platform wallets as customer ones).
+export { PLATFORM_AGENT_OWNER_EMAIL, PLATFORM_AGENT_EMAIL_SUFFIX, isPlatformOwnedAgent };
/** Floor left on an idle platform agent — enough to sign its own transactions. */
const AGENT_IDLE_FLOOR_SOL = num('AGENT_RECLAIM_IDLE_FLOOR_SOL', 0.005);
@@ -583,17 +588,6 @@ export function agentCandidateFromRow(row, sol) {
};
}
-/**
- * True only for the platform's own agent-owner accounts. Anything else (a real
- * signup, a wallet-auth account, an unknown value) is a customer and is out.
- * @param {string|null|undefined} ownerEmail
- */
-export function isPlatformOwnedAgent(ownerEmail) {
- if (!ownerEmail || typeof ownerEmail !== 'string') return false;
- const email = ownerEmail.trim().toLowerCase();
- return email === PLATFORM_AGENT_OWNER_EMAIL || email.endsWith(PLATFORM_AGENT_EMAIL_SUFFIX);
-}
-
/**
* Pull idle SOL out of PLATFORM-OWNED agent custody wallets back to the economy
* master, so the fee wallet refills itself from capital the platform already owns
diff --git a/api/_lib/ops/subsystem-health.js b/api/_lib/ops/subsystem-health.js
index 4304f8d5d..8a85b2b3d 100644
--- a/api/_lib/ops/subsystem-health.js
+++ b/api/_lib/ops/subsystem-health.js
@@ -466,12 +466,21 @@ export function classifyOkxChatBotBeat(beat, now = Date.now()) {
}
const reported = typeof meta.health === 'string' ? meta.health : 'unknown';
+ const host = typeof meta.host === 'string' && meta.host ? meta.host : null;
if (reported === 'ok') {
- return {
- ...base,
- status: 'ok',
- detail: `online (${meta.activeClients ?? '?'} XMTP client(s), provider=${beat.mode || meta.provider || 'unknown'})`,
- };
+ const online = `online (${meta.activeClients ?? '?'} XMTP client(s), provider=${beat.mode || meta.provider || 'unknown'}${host ? `, host=${host}` : ''})`;
+ // A host that beats but cannot survive on its own is a stopgap, not the
+ // fix. Reporting it green is how "the bot is up" and "the bot is up until
+ // this workspace sleeps" became the same sentence.
+ if (meta.hostDurable === false) {
+ return {
+ ...base,
+ status: 'degraded',
+ detail: `${online}; that host is a stopgap and dies with its workspace, so chat delivery is not durable yet`,
+ hint: 'Deploy the always-on host: gcloud builds submit --config workers/okx-chat-bot/cloudbuild.yaml . (see workers/okx-chat-bot/README.md).',
+ };
+ }
+ return { ...base, status: 'ok', detail: online };
}
if (reported === 'unknown') {
return { ...base, status: 'unknown', detail: String(meta.detail || 'bot could not determine its own state') };
diff --git a/scripts/audit-custodial-key-health.mjs b/scripts/audit-custodial-key-health.mjs
index 1e2869ee0..a57acec5e 100644
--- a/scripts/audit-custodial-key-health.mjs
+++ b/scripts/audit-custodial-key-health.mjs
@@ -15,7 +15,9 @@
// A key rotation is the usual cause. `scripts/rekey-stale-launch-wallets.mjs`
// documents one: the WALLET_ENCRYPTION_KEY changed during the Vercel to Cloud
// Run migration in 2026-07, and every wallet written under the retired key
-// became unreadable. This script measures the blast radius of that class.
+// became unreadable. This script measures the blast radius of that class;
+// `docs/ops/stranded-wallets.md` carries the standing owner decision on the
+// customer balances it finds.
//
// READ-ONLY. It decrypts in memory to test the key, never writes, never signs,
// never broadcasts. Balances are read with getMultipleAccounts (100 per call) so
@@ -49,11 +51,6 @@ const PLATFORM_ONLY = process.argv.includes('--platform-only');
const emitJson = console.log.bind(console);
if (AS_JSON) console.log = (...args) => console.error(...args);
-// Same ownership markers the reclaim leg uses (api/_lib/economy-sweepback.js):
-// platform agents live under one owner email or the agent-email suffix.
-const PLATFORM_OWNER_EMAIL = '[email protected]';
-const PLATFORM_EMAIL_SUFFIX = '@agents.three.ws';
-const LAMPORTS_PER_SOL = 1_000_000_000;
// Where an operator actually gets the key. The playbook order in CLAUDE.md:
// local dotfiles first, then the running service, then Secret Manager.
@@ -78,10 +75,12 @@ for (const file of ['.env', '.env.local']) {
}
}
-const { sql } = await import('../api/_lib/db.js');
-const { decryptSecret, secretBoxKeyCandidates } = await import('../api/_lib/secret-box.js');
-const { PublicKey } = await import('@solana/web3.js');
-const { solanaConnection } = await import('../api/_lib/solana/connection.js');
+// The measurement itself lives in api/_lib/custodial-key-health.js, shared with
+// the ops board (`stranded_custody` in /api/ops/payment-outcomes), so the number
+// an operator reads here and the number the board renders cannot drift apart.
+// This file is the CLI around it: env loading, the key guard, and rendering.
+const { secretBoxKeyCandidates } = await import('../api/_lib/secret-box.js');
+const { gatherCustodialKeyHealth } = await import('../api/_lib/custodial-key-health.js');
// A shell with NO decryption key configured cannot tell "sealed under a retired
// key" from "this machine was never given the key". Without this check the
@@ -115,138 +114,14 @@ if (keyCandidates.length === 0) {
process.exit(3);
}
-
-const rows = await sql`
- SELECT a.id AS agent_id,
- a.name AS name,
- a.created_at AS created_at,
- a.meta->>'solana_address' AS address,
- a.meta->>'encrypted_solana_secret' AS secret,
- LOWER(u.email) AS owner
- FROM agent_identities a
- JOIN users u ON u.id = a.user_id
- WHERE a.deleted_at IS NULL
- AND a.meta->>'solana_address' IS NOT NULL
- AND a.meta->>'encrypted_solana_secret' IS NOT NULL
- ORDER BY a.created_at ASC
-`;
-
-const isPlatform = (owner) =>
- owner === PLATFORM_OWNER_EMAIL || (owner || '').endsWith(PLATFORM_EMAIL_SUFFIX);
-
-const wallets = rows.filter((r) => (PLATFORM_ONLY ? isPlatform(r.owner) : true));
-
-// Decrypt is the whole test: AES-GCM authenticates, so a wrong key throws rather
-// than returning wrong plaintext. We never touch the recovered material beyond
-// checking that it exists.
-for (const w of wallets) {
- try {
- const plain = await decryptSecret(w.secret);
- w.decryptable = Boolean(plain && plain.length > 0);
- w.reason = w.decryptable ? null : 'empty_plaintext';
- } catch (e) {
- w.decryptable = false;
- w.reason = e?.name === 'OperationError' ? 'wrong_key' : e?.message || 'decrypt_failed';
- }
-}
-
-// Same rotating multi-lane connection production reads balances through
-// (api/_lib/solana/connection.js), not a single bare endpoint. A script that
-// hits exactly one RPC URL dies the moment that one lane rate-limits or
-// blocks the caller's IP (observed 2026-08-09: SOLANA_RPC_URL pinned to a
-// single free lane returned 403 for every wallet), and the failure was
-// invisible because unread balances silently summed as zero (see below).
-const connection = solanaConnection({ url: process.env.SOLANA_RPC_URL || null });
-
-const balances = new Map();
-const readErrors = [];
-for (let i = 0; i < wallets.length; i += 100) {
- const chunk = wallets.slice(i, i + 100);
- let keys;
- try {
- keys = chunk.map((w) => new PublicKey(w.address));
- } catch (e) {
- for (const w of chunk) readErrors.push({ address: w.address, reason: `bad_address: ${e?.message}` });
- continue;
- }
- try {
- const infos = await connection.getMultipleAccountsInfo(keys);
- infos.forEach((info, idx) => balances.set(chunk[idx].address, (info?.lamports ?? 0) / LAMPORTS_PER_SOL));
- } catch (e) {
- for (const w of chunk) readErrors.push({ address: w.address, reason: `rpc_error: ${e?.message}` });
- }
-}
-
-// balances.has() is load-bearing: a wallet whose chunk hit an RPC error has no
-// entry at all, and must never be treated as a confirmed zero balance. Coalescing
-// "unread" into "0" is exactly the bug that let this audit report "0 SOL
-// stranded" while two undecryptable wallets actually held 0.12 SOL (2026-08-09;
-// caught only because the treasury-topup cron's *live* reclaim attempt, which
-// reads balances through Cloud Run's own multi-lane connection, tried to sweep
-// them and hit secret_undecryptable on wallets this audit had certified as
-// zero). `sum()` therefore only totals wallets with a confirmed read, and the
-// caller must check `unreadCount` before trusting a "0 stranded" verdict.
-const sum = (list) => list.reduce((t, w) => (balances.has(w.address) ? t + balances.get(w.address) : t), 0);
-const unread = (list) => list.filter((w) => !balances.has(w.address));
-const round = (n) => Number(n.toFixed(9));
-
-const platform = wallets.filter((w) => isPlatform(w.owner));
-const customer = wallets.filter((w) => !isPlatform(w.owner));
-const stranded = wallets.filter((w) => !w.decryptable);
-// "Funded" requires a CONFIRMED positive balance; a stranded wallet whose
-// balance we never read is neither funded nor cleared, it is unknown.
-const strandedFunded = stranded.filter((w) => balances.has(w.address) && balances.get(w.address) > 0);
-const strandedUnread = unread(stranded);
-
-const report = {
- checked_at: new Date().toISOString(),
- rpc: process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
- wallets: wallets.length,
- // How many keys decryptSecret() had to try. 1 is the healthy steady state; a
- // higher count means retired keys are still configured for the migration.
- key_candidates: keyCandidates.length,
- read_errors: readErrors.length,
- decryptable: wallets.length - stranded.length,
- undecryptable: stranded.length,
- sol: {
- // Every figure below is a sum over CONFIRMED reads only (sum() skips
- // addresses with no balances.has() entry) - never trust these as
- // complete while unread_stranded_count > 0 below.
- total: round(sum(wallets)),
- decryptable: round(sum(wallets.filter((w) => w.decryptable))),
- stranded: round(sum(stranded)),
- stranded_platform: round(sum(stranded.filter((w) => isPlatform(w.owner)))),
- stranded_customer: round(sum(stranded.filter((w) => !isPlatform(w.owner)))),
- },
- counts: {
- platform: platform.length,
- customer: customer.length,
- stranded_platform: stranded.filter((w) => isPlatform(w.owner)).length,
- stranded_customer: stranded.filter((w) => !isPlatform(w.owner)).length,
- stranded_funded: strandedFunded.length,
- stranded_unread: strandedUnread.length,
- },
- top_stranded: strandedFunded
- .sort((a, b) => (balances.get(b.address) || 0) - (balances.get(a.address) || 0))
- .slice(0, 20)
- .map((w) => ({
- name: w.name,
- address: w.address,
- owner: w.owner,
- sol: round(balances.get(w.address) || 0),
- reason: w.reason,
- created_at: w.created_at,
- platform: isPlatform(w.owner),
- })),
- unread_stranded: strandedUnread.map((w) => ({
- name: w.name,
- address: w.address,
- owner: w.owner,
- reason: w.reason,
- created_at: w.created_at,
- platform: isPlatform(w.owner),
- })),
-};
+// One fleet scan: read every stored wallet, test every key, read every balance
+// through the rotating multi-lane connection production uses. Totals sum only
+// CONFIRMED balance reads, so an RPC failure reports `stranded_unread` instead of
+// a confident zero (the 2026-08-09 blind spot).
+const report = await gatherCustodialKeyHealth({
+ platformOnly: PLATFORM_ONLY,
+ keyCandidates: keyCandidates.length,
+});
if (AS_JSON) {
emitJson(JSON.stringify(report, null, 2));
@@ -292,5 +167,6 @@ if (report.wallets > 0 && report.undecryptable === report.wallets) {
console.log('');
console.log(' Customer funds are stranded. This is a support obligation, not just an');
console.log(' ops number: those users cannot withdraw. Escalate before anything else.');
+ console.log(' Decision brief: docs/ops/stranded-wallets.md');
}
process.exit(0);
diff --git a/tests/migrate-plaintext-secrets.test.js b/tests/migrate-plaintext-secrets.test.js
new file mode 100644
index 000000000..cb1f89195
--- /dev/null
+++ b/tests/migrate-plaintext-secrets.test.js
@@ -0,0 +1,81 @@
+// Cover the one decision in scripts/migrate-plaintext-secrets.mjs that carries
+// real risk: which Cloud Run env vars are credentials that must move into Secret
+// Manager, and which are public identifiers that should stay as plaintext config.
+//
+// A false negative leaves a wallet key readable by anyone with run.services.get.
+// A false positive buries a public address behind an IAM grant and makes the
+// service config unreadable to an operator for no gain. Both cases are named
+// below with the real env vars this platform runs on (api/_lib/solana-signers.js,
+// docs/ops/gcp-production.md).
+
+import { describe, it, expect } from 'vitest';
+import { classify, defaultSecretName } from '../scripts/migrate-plaintext-secrets.mjs';
+
+describe('credential classifier', () => {
+ it('treats the wallet secret keys as credentials', () => {
+ for (const name of [
+ 'ECONOMY_MASTER_SECRET_BASE58',
+ 'X402_TREASURY_SECRET_BASE58',
+ 'X402_SEED_SOLANA_SECRET_BASE58',
+ 'LAUNCHER_MASTER_SECRET_KEY_B64',
+ 'PUMP_CRON_RELAYER_SECRET_KEY_B64',
+ 'PLATFORM_TREASURY_KEYPAIR',
+ 'WALLET_ENCRYPTION_KEY',
+ 'JWT_SECRET',
+ 'CRON_SECRET',
+ ]) {
+ expect(classify(name).secret, name).toBe(true);
+ }
+ });
+
+ it('treats API tokens and connection strings as credentials', () => {
+ for (const name of [
+ 'UPSTASH_REDIS_REST_TOKEN',
+ 'ANTHROPIC_API_KEY',
+ 'TELEGRAM_BOT_TOKEN',
+ 'DATABASE_URL',
+ 'A2A_PAYER_SOLANA_SECRET',
+ ]) {
+ expect(classify(name).secret, name).toBe(true);
+ }
+ });
+
+ it('leaves public identifiers as plaintext config', () => {
+ for (const name of [
+ 'ECONOMY_MASTER_ADDRESS',
+ 'X402_FEE_PAYER_SOLANA',
+ 'THREE_TOKEN_MINT',
+ 'NEXT_PUBLIC_API_KEY',
+ ]) {
+ expect(classify(name).secret, name).toBe(false);
+ }
+ });
+
+ it('leaves a handle that merely names a credential', () => {
+ for (const name of ['KMS_KEY_RING', 'VANITY_KEY_ID', 'GCS_KEY_FILE', 'WALLET_SECRET_NAME']) {
+ expect(classify(name).secret, name).toBe(false);
+ }
+ });
+
+ it('flags a provider URL for a human call instead of filing it as public config', () => {
+ for (const name of ['SOLANA_RPC_URL', 'UPSTASH_REDIS_REST_URL', 'HELIUS_RPC_ENDPOINT']) {
+ const verdict = classify(name);
+ expect(verdict.secret, name).toBe(false);
+ expect(verdict.review, name).toBe(true);
+ }
+ });
+
+ it('always reports a reason, so a wrong call is visible before --apply', () => {
+ for (const name of ['ECONOMY_MASTER_SECRET_BASE58', 'THREE_TOKEN_MINT']) {
+ expect(classify(name).reason).toBeTruthy();
+ }
+ });
+});
+
+describe('secret naming', () => {
+ it('matches the shape the existing secrets on this project already use', () => {
+ expect(defaultSecretName('X402_FEE_PAYER_SECRET_BASE58')).toBe('x402-fee-payer-secret-base58');
+ expect(defaultSecretName('UPSTASH_REDIS_REST_TOKEN')).toBe('upstash-redis-rest-token');
+ expect(defaultSecretName('ECONOMY_MASTER_SECRET_BASE58')).toBe('economy-master-secret-base58');
+ });
+});