Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(custody): report an unreadable key as unknown rather than as stranded funds

nirholasab4ff394cf9d
diff --git a/api/_lib/custodial-key-health.js b/api/_lib/custodial-key-health.js
index 5cd721265..59328061f 100644
--- a/api/_lib/custodial-key-health.js
+++ b/api/_lib/custodial-key-health.js
@@ -31,7 +31,7 @@
// that says a zero is not yet trustworthy.
import { sql } from './db.js';
-import { decryptSecret } from './secret-box.js';
+import { decryptSecret, secretBoxKeyCandidates } from './secret-box.js';
/** The platform's own agent-owner account. Anything else is a customer. */
export const PLATFORM_AGENT_OWNER_EMAIL = '[email protected]';
@@ -220,6 +220,11 @@ export function summarizeCustodialKeyHealth({ wallets, balances, readErrors = []
* @returns {Promise<object>} the report shape summarizeCustodialKeyHealth builds
*/
export async function gatherCustodialKeyHealth({ connection, platformOnly = false, rpcUrl = null, topN = 20, keyCandidates = null } = {}) {
+ // How many keys a decrypt may try, straight from secret-box's own precedence
+ // so this can never drift from what decryptSecret() actually does. Zero means
+ // this process holds no key at all, and every verdict below would be an
+ // artifact of that rather than a fact about custody.
+ const keys = keyCandidates == null ? secretBoxKeyCandidates().length : keyCandidates;
const wallets = await readCustodialWalletRows({ platformOnly });
await classifyCustodialSecrets(wallets);
// The rotating multi-lane connection production reads balances through, not a
@@ -238,7 +243,7 @@ export async function gatherCustodialKeyHealth({ connection, platformOnly = fals
readErrors,
rpc: rpcUrl || process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',
topN,
- keyCandidates,
+ keyCandidates: keys,
});
}
@@ -276,7 +281,11 @@ export async function strandedCustodyPanel({ ttlMs = STRANDED_SNAPSHOT_TTL_MS, f
inFlight = (async () => {
const report = await gather();
const panel = buildStrandedPanel(report);
- cached = { at: Date.now(), panel };
+ // Stamped with the clock the CALLER passed, not a second reading of the
+ // wall clock: the age this panel reports and the TTL it is judged
+ // against then come from one timeline, which is also what makes the
+ // cache testable without faking timers.
+ cached = { at: now, panel };
return panel;
})().finally(() => {
inFlight = null;
@@ -303,31 +312,59 @@ export function buildStrandedPanel(report) {
const sol = report?.sol || {};
const unread = counts.stranded_unread || 0;
const funded = counts.stranded_funded || 0;
- // A total is only a total when every sealed wallet actually got a balance
- // read. While any is unread the SOL figures are a FLOOR, and the panel says
- // so rather than letting a partial read render as a clean number.
- const status = unread > 0 ? 'unknown' : funded > 0 ? 'stranded' : 'clear';
+ const wallets = report?.wallets ?? 0;
+ const undecryptable = report?.undecryptable ?? 0;
+ const keys = report?.key_candidates;
+
+ // Three ways a stranded number can be a lie, each answered before the honest
+ // verdicts below:
+ // • no key configured at all: every wallet reads sealed here regardless of
+ // production state (secret-box fails closed in production, so this is a
+ // misconfigured non-prod process, not a custody event).
+ // • EVERY wallet failed: almost never 700 separately sealed wallets, almost
+ // always one wrong key for this deployment. Reporting it as a mass
+ // customer incident sends operators after the wrong problem.
+ // • some sealed wallet never got a balance read: the totals are a floor.
+ // Only when none of those hold is the SOL figure a measurement.
+ let status = 'clear';
+ let reason = 'clear';
+ let detail = 'No funded custodial wallet sits behind an undecryptable key.';
+ if (keys === 0) {
+ status = 'unknown';
+ reason = 'no_decryption_key';
+ detail = 'This process holds no custodial decryption key, so every wallet reads as sealed here. The reading says nothing about custody.';
+ } else if (wallets > 0 && undecryptable === wallets) {
+ status = 'unknown';
+ reason = 'key_mismatch';
+ detail = `All ${wallets} custodial wallets failed to decrypt under the ${keys ?? 'configured'} key(s). A fleet-wide failure means the key is wrong for this deployment, not that every wallet is sealed.`;
+ } else if (unread > 0) {
+ status = 'unknown';
+ reason = 'partial_read';
+ detail = `${unread} sealed wallet(s) never got a balance read, so the stranded total is a floor, not a measurement.`;
+ } else if (funded > 0) {
+ status = 'stranded';
+ reason = 'sealed';
+ detail = `${funded} custodial wallet(s) hold ${sol.stranded ?? 0} SOL behind a retired encryption key (platform ${sol.stranded_platform ?? 0}, customer ${sol.stranded_customer ?? 0}).`;
+ }
+
return {
status,
+ reason,
checked_at: report?.checked_at || null,
+ key_candidates: keys ?? null,
wallets: report?.wallets ?? null,
- undecryptable: report?.undecryptable ?? 0,
+ undecryptable,
stranded_funded: funded,
stranded_unread: unread,
sol_stranded: sol.stranded ?? 0,
sol_stranded_platform: sol.stranded_platform ?? 0,
sol_stranded_customer: sol.stranded_customer ?? 0,
customer_wallets_stranded: (report?.top_stranded || []).filter((w) => !w.platform).length,
- detail:
- status === 'clear'
- ? 'No funded custodial wallet sits behind an undecryptable key.'
- : status === 'unknown'
- ? `${unread} sealed wallet(s) never got a balance read, so the stranded total is a floor, not a measurement.`
- : `${funded} custodial wallet(s) hold ${sol.stranded ?? 0} SOL behind a retired encryption key (platform ${sol.stranded_platform ?? 0}, customer ${sol.stranded_customer ?? 0}).`,
+ detail,
// Customer money that cannot be withdrawn is a support obligation, not an
// ops number: the board names the decision doc rather than leaving the
// reader to rediscover that recovery is impossible.
- brief: (sol.stranded_customer || 0) > 0 ? 'docs/ops/stranded-wallets.md' : null,
+ brief: status === 'stranded' && (sol.stranded_customer || 0) > 0 ? 'docs/ops/stranded-wallets.md' : null,
top_stranded: (report?.top_stranded || []).slice(0, 10).map((w) => ({
address: w.address,
sol: w.sol,
diff --git a/tests/avatars-list-scale.test.js b/tests/avatars-list-scale.test.js
new file mode 100644
index 000000000..7c49e5622
--- /dev/null
+++ b/tests/avatars-list-scale.test.js
@@ -0,0 +1,127 @@
+// The owner-visible listing and the public gallery totals are the two queries
+// whose cost used to grow with the whole avatar catalog rather than with the
+// page being served. Both were measured against the live 58k-row table:
+//
+// listAvatars({ includePublic: true }) 48.7 ms / 7,605 heap pages read,
+// because `(owner_id = $1 or visibility = 'public')` is unindexable and
+// Postgres seq-scanned and sorted every surviving row.
+// searchPublicAvatars({ withTotals }) 42 ms sequential scan for the
+// count + view-count sum, on every single request.
+//
+// These tests lock in the shapes that fixed them: a UNION ALL of two indexed
+// branches, and a cached aggregate. They assert on the SQL the module emits
+// rather than on timings, so they stay meaningful without a database.
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+process.env.S3_PUBLIC_DOMAIN ||= 'https://cdn.test';
+process.env.S3_BUCKET ||= 'test-bucket';
+
+const calls = [];
+vi.mock('../api/_lib/db.js', () => ({
+ sql: vi.fn(async (text, params) => {
+ calls.push({ text: typeof text === 'string' ? text : String(text), params });
+ return [];
+ }),
+ isDbUnavailableError: () => false,
+ isDbCapacityError: () => false,
+}));
+
+const cacheWrapCalls = [];
+vi.mock('../api/_lib/cache.js', () => ({
+ cacheWrap: vi.fn(async (key, ttl, fn) => {
+ cacheWrapCalls.push({ key, ttl });
+ return fn();
+ }),
+}));
+
+const { listAvatars, searchPublicAvatars } = await import('../api/_lib/avatars.js');
+
+const USER = '11111111-2222-3333-4444-555555555555';
+
+function lastSelect() {
+ return calls[calls.length - 1];
+}
+
+beforeEach(() => {
+ calls.length = 0;
+ cacheWrapCalls.length = 0;
+});
+
+describe('listAvatars row selection', () => {
+ it('unions two indexed branches instead of an unindexable OR', async () => {
+ await listAvatars({ userId: USER, limit: 50, includePublic: true });
+ const { text, params } = lastSelect();
+
+ expect(text).toContain('union all');
+ // The predicate that forced the seq scan must be gone.
+ expect(text).not.toMatch(/owner_id = \$1 or/);
+ // Owner branch and public branch, each ordered + limited on its own so
+ // Postgres can serve them from avatars_owner_idx / avatars_public_idx.
+ expect(text).toContain("where owner_id = $1 and deleted_at is null order by created_at desc");
+ expect(text).toContain("where visibility = 'public' and owner_id is distinct from $1");
+ // limit+1 drives both branches and the outer merge, so hasMore still works.
+ expect(params[params.length - 1]).toBe(51);
+ });
+
+ it('excludes the caller from the public branch with is-distinct-from', async () => {
+ // A plain `owner_id <> $1` drops every NULL-owner row from the catalog.
+ await listAvatars({ userId: USER, includePublic: true });
+ expect(lastSelect().text).toContain('owner_id is distinct from $1');
+ expect(lastSelect().text).not.toContain('owner_id <> $1');
+ });
+
+ it('keeps a single indexed branch when the caller wants only their own', async () => {
+ await listAvatars({ userId: USER, limit: 10 });
+ const { text, params } = lastSelect();
+ expect(text).not.toContain('union all');
+ expect(text).toContain('where owner_id = $1');
+ expect(params).toEqual([USER, 11]);
+ });
+
+ it('applies visibility and cursor filters to every branch', async () => {
+ await listAvatars({
+ userId: USER,
+ includePublic: true,
+ visibility: 'public',
+ cursor: '2026-09-01T00:00:00.000Z',
+ limit: 20,
+ });
+ const { text, params } = lastSelect();
+ // Both branches share the filters, so a cursor page can never return rows
+ // from one branch that the other already passed.
+ expect(text.match(/visibility = \$2/g)?.length).toBe(2);
+ expect(text.match(/created_at < \$3/g)?.length).toBe(2);
+ expect(params).toEqual([USER, 'public', new Date('2026-09-01T00:00:00.000Z'), 21]);
+ });
+
+ it('joins the wide column list to the union result, not to a scan', async () => {
+ await listAvatars({ userId: USER, includePublic: true });
+ const { text } = lastSelect();
+ expect(text).toContain('join avatars a on a.id = v.id');
+ expect(text).toContain('order by v.created_at desc');
+ });
+});
+
+describe('public gallery totals', () => {
+ it('caches the catalog-wide aggregate for the endpoint cache window', async () => {
+ await searchPublicAvatars({ limit: 24, withTotals: true });
+ expect(cacheWrapCalls).toHaveLength(1);
+ expect(cacheWrapCalls[0].ttl).toBe(60);
+ expect(cacheWrapCalls[0].key).toMatch(/^avatars:public-totals:[0-9a-f]{40}$/);
+ });
+
+ it('keys the cache by the filter set so a filtered view never reads unfiltered numbers', async () => {
+ await searchPublicAvatars({ limit: 24, withTotals: true });
+ await searchPublicAvatars({ limit: 24, withTotals: true, rigged: 'rigged' });
+ await searchPublicAvatars({ limit: 24, withTotals: true, tag: 'knight' });
+ const keys = cacheWrapCalls.map((c) => c.key);
+ expect(new Set(keys).size).toBe(3);
+ });
+
+ it('runs no aggregate at all when totals were not asked for', async () => {
+ await searchPublicAvatars({ limit: 24 });
+ expect(cacheWrapCalls).toHaveLength(0);
+ expect(calls.some((c) => c.text.includes('count(*)'))).toBe(false);
+ });
+});
diff --git a/tests/custodial-key-health.test.js b/tests/custodial-key-health.test.js
new file mode 100644
index 000000000..2884281ca
--- /dev/null
+++ b/tests/custodial-key-health.test.js
@@ -0,0 +1,236 @@
+// The custody loss measurement (api/_lib/custodial-key-health.js): the one
+// number the CLI audit and the ops board both read.
+//
+// Every fence here is a defect that already shipped:
+//
+// 1. an unread balance summed as a confirmed zero, so the audit certified
+// "0 SOL stranded" on 2026-08-09 while two wallets held 0.12 SOL behind a
+// dead key. A wallet with no balance read must never reach a total.
+// 2. the audit carried its own ownership predicate with a different house
+// account address than the reclaim leg enforces in SQL, filing 12 platform
+// wallets as CUSTOMER ones. The split feeds a customer support obligation,
+// so it has to be the reclaim leg's exact boundary.
+// 3. the board must not turn a polled dashboard into a fleet-wide RPC scan:
+// the panel is snapshot-cached and concurrent reads share one scan.
+
+import { describe, it, expect, beforeEach, vi } from 'vitest';
+
+vi.mock('../api/_lib/db.js', () => ({ sql: async () => [] }));
+vi.mock('../api/_lib/secret-box.js', () => ({
+ decryptSecret: async () => 'plaintext',
+ secretBoxKeyCandidates: () => ['key'],
+}));
+
+const {
+ isPlatformOwnedAgent,
+ classifyCustodialSecrets,
+ summarizeCustodialKeyHealth,
+ buildStrandedPanel,
+ strandedCustodyPanel,
+ resetStrandedCustodyCache,
+ PLATFORM_AGENT_OWNER_EMAIL,
+} = await import('../api/_lib/custodial-key-health.js');
+
+const wallet = (address, owner, decryptable, extra = {}) => ({
+ agent_id: `ag-${address}`,
+ name: address,
+ address,
+ owner,
+ decryptable,
+ reason: decryptable ? null : 'wrong_key',
+ ...extra,
+});
+
+const HOUSE = PLATFORM_AGENT_OWNER_EMAIL;
+const BOT = '[email protected]';
+const CUSTOMER = '[email protected]';
+
+describe('ownership boundary', () => {
+ it('counts the house account and the circulation bots as platform', () => {
+ expect(isPlatformOwnedAgent(HOUSE)).toBe(true);
+ expect(isPlatformOwnedAgent(BOT)).toBe(true);
+ expect(isPlatformOwnedAgent(' [email protected] ')).toBe(true);
+ });
+
+ it('counts everything else as a customer, including a lookalike domain', () => {
+ expect(isPlatformOwnedAgent(CUSTOMER)).toBe(false);
+ expect(isPlatformOwnedAgent('[email protected]')).toBe(false);
+ // The suffix must match at the end, not anywhere: an attacker-chosen
+ // local part must never buy platform classification.
+ expect(isPlatformOwnedAgent('[email protected]')).toBe(false);
+ expect(isPlatformOwnedAgent(null)).toBe(false);
+ expect(isPlatformOwnedAgent(undefined)).toBe(false);
+ });
+});
+
+describe('secret classification', () => {
+ it('marks a wallet undecryptable with the WebCrypto failure named, not a generic error', async () => {
+ const rows = [{ secret: 'sealed' }, { secret: 'fine' }];
+ const err = Object.assign(new Error('The operation failed'), { name: 'OperationError' });
+ const rowsOut = await classifyCustodialSecrets(rows, async (s) => {
+ if (s === 'sealed') throw err;
+ return 'plaintext';
+ });
+ expect(rowsOut[0].decryptable).toBe(false);
+ expect(rowsOut[0].reason).toBe('wrong_key');
+ expect(rowsOut[1].decryptable).toBe(true);
+ expect(rowsOut[1].reason).toBe(null);
+ });
+
+ it('treats an empty plaintext as undecryptable rather than a healthy wallet', async () => {
+ const rows = await classifyCustodialSecrets([{ secret: 'empty' }], async () => '');
+ expect(rows[0].decryptable).toBe(false);
+ expect(rows[0].reason).toBe('empty_plaintext');
+ });
+});
+
+describe('summary counting rules', () => {
+ it('never sums a wallet whose balance was not read', () => {
+ const wallets = [
+ wallet('SEALED_READ', CUSTOMER, false),
+ wallet('SEALED_UNREAD', CUSTOMER, false),
+ wallet('OK', HOUSE, true),
+ ];
+ const balances = new Map([['SEALED_READ', 0.25], ['OK', 1]]);
+ const r = summarizeCustodialKeyHealth({
+ wallets, balances,
+ readErrors: [{ address: 'SEALED_UNREAD', reason: 'rpc_error: 403' }],
+ });
+ // 0.25, not 0.25 plus an invented zero for the wallet nobody could read.
+ expect(r.sol.stranded).toBe(0.25);
+ expect(r.counts.stranded_unread).toBe(1);
+ expect(r.counts.stranded_funded).toBe(1);
+ expect(r.unread_stranded.map((w) => w.address)).toEqual(['SEALED_UNREAD']);
+ expect(r.read_errors).toBe(1);
+ });
+
+ it('splits stranded SOL by the reclaim leg’s ownership boundary', () => {
+ const wallets = [
+ wallet('BOT', BOT, false),
+ wallet('HOUSE', HOUSE, false),
+ wallet('CUST', CUSTOMER, false),
+ ];
+ const balances = new Map([['BOT', 0.07], ['HOUSE', 0.03], ['CUST', 0.35]]);
+ const r = summarizeCustodialKeyHealth({ wallets, balances });
+ expect(r.counts.stranded_platform).toBe(2);
+ expect(r.counts.stranded_customer).toBe(1);
+ expect(r.sol.stranded_platform).toBe(0.1);
+ expect(r.sol.stranded_customer).toBe(0.35);
+ expect(r.sol.stranded).toBe(0.45);
+ });
+
+ it('ranks the stranded list by balance and keeps every wallet identifiable', () => {
+ const wallets = [wallet('SMALL', CUSTOMER, false), wallet('BIG', CUSTOMER, false)];
+ const balances = new Map([['SMALL', 0.01], ['BIG', 1.2]]);
+ const r = summarizeCustodialKeyHealth({ wallets, balances });
+ expect(r.top_stranded.map((w) => w.address)).toEqual(['BIG', 'SMALL']);
+ expect(r.top_stranded[0]).toMatchObject({ sol: 1.2, platform: false, reason: 'wrong_key' });
+ expect(r.top_stranded[0].agent_id).toBe('ag-BIG');
+ });
+
+ it('reports a zero-balance sealed wallet as sealed but NOT funded', () => {
+ const wallets = [wallet('EMPTY', CUSTOMER, false)];
+ const r = summarizeCustodialKeyHealth({ wallets, balances: new Map([['EMPTY', 0]]) });
+ expect(r.undecryptable).toBe(1);
+ expect(r.counts.stranded_funded).toBe(0);
+ expect(r.counts.stranded_unread).toBe(0);
+ expect(r.sol.stranded).toBe(0);
+ });
+});
+
+describe('ops panel verdict', () => {
+ const report = (over = {}) => ({
+ checked_at: '2026-09-02T00:00:00.000Z',
+ wallets: 725,
+ key_candidates: 1,
+ undecryptable: 8,
+ sol: { stranded: 0.49, stranded_platform: 0.14, stranded_customer: 0.35 },
+ counts: { stranded_funded: 8, stranded_unread: 0 },
+ top_stranded: [
+ { address: 'A', sol: 0.35, platform: false, reason: 'wrong_key' },
+ { address: 'B', sol: 0.14, platform: true, reason: 'wrong_key' },
+ ],
+ ...over,
+ });
+
+ it('says stranded, splits the SOL, and points at the owner decision brief', () => {
+ const p = buildStrandedPanel(report());
+ expect(p.status).toBe('stranded');
+ expect(p.sol_stranded_customer).toBe(0.35);
+ expect(p.customer_wallets_stranded).toBe(1);
+ expect(p.brief).toBe('docs/ops/stranded-wallets.md');
+ });
+
+ it('refuses to call a partial read a measurement', () => {
+ const p = buildStrandedPanel(report({ counts: { stranded_funded: 8, stranded_unread: 3 } }));
+ expect(p.status).toBe('unknown');
+ expect(p.reason).toBe('partial_read');
+ expect(p.detail).toMatch(/floor, not a measurement/);
+ });
+
+ it('refuses to read a keyless process as a fleet of sealed wallets', () => {
+ // secret-box fails closed in production, so a zero-candidate process is a
+ // misconfigured non-prod one. Reporting its 100% failure rate as custody
+ // loss would manufacture an incident out of a missing env var.
+ const p = buildStrandedPanel(report({ key_candidates: 0, undecryptable: 725, counts: { stranded_funded: 700, stranded_unread: 0 } }));
+ expect(p.status).toBe('unknown');
+ expect(p.reason).toBe('no_decryption_key');
+ });
+
+ it('calls a fleet-wide decrypt failure a wrong key, not a mass customer incident', () => {
+ const p = buildStrandedPanel(report({ undecryptable: 725, counts: { stranded_funded: 168, stranded_unread: 0 } }));
+ expect(p.status).toBe('unknown');
+ expect(p.reason).toBe('key_mismatch');
+ expect(p.brief).toBe(null);
+ });
+
+ it('says clear only when nothing funded is sealed, and then names no brief', () => {
+ const p = buildStrandedPanel(report({
+ undecryptable: 0, counts: { stranded_funded: 0, stranded_unread: 0 },
+ sol: { stranded: 0, stranded_platform: 0, stranded_customer: 0 }, top_stranded: [],
+ }));
+ expect(p.status).toBe('clear');
+ expect(p.brief).toBe(null);
+ });
+});
+
+describe('snapshot cache', () => {
+ beforeEach(() => resetStrandedCustodyCache());
+
+ it('scans once and serves the snapshot for the whole TTL', async () => {
+ const gather = vi.fn(async () => ({ wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} }));
+ await strandedCustodyPanel({ gather, now: 1_000 });
+ const second = await strandedCustodyPanel({ gather, now: 1_000 + 60_000 });
+ expect(gather).toHaveBeenCalledTimes(1);
+ expect(second.cache_age_seconds).toBe(60);
+ });
+
+ it('rescans once the snapshot ages past the TTL', async () => {
+ const gather = vi.fn(async () => ({ wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} }));
+ await strandedCustodyPanel({ gather, ttlMs: 1_000, now: 0 });
+ await strandedCustodyPanel({ gather, ttlMs: 1_000, now: Date.now() + 5_000 });
+ expect(gather).toHaveBeenCalledTimes(2);
+ });
+
+ it('collapses concurrent reads into one fleet scan', async () => {
+ let release;
+ const gate = new Promise((r) => { release = r; });
+ const gather = vi.fn(async () => {
+ await gate;
+ return { wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} };
+ });
+ const both = Promise.all([strandedCustodyPanel({ gather }), strandedCustodyPanel({ gather })]);
+ release();
+ await both;
+ expect(gather).toHaveBeenCalledTimes(1);
+ });
+
+ it('does not cache a failed scan, so a transient outage is retried not frozen', async () => {
+ const gather = vi.fn()
+ .mockRejectedValueOnce(new Error('rpc down'))
+ .mockResolvedValueOnce({ wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} });
+ await expect(strandedCustodyPanel({ gather })).rejects.toThrow('rpc down');
+ await expect(strandedCustodyPanel({ gather })).resolves.toMatchObject({ status: 'clear' });
+ expect(gather).toHaveBeenCalledTimes(2);
+ });
+});