Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(custody): report an unreadable key as unknown rather than as stranded funds
nirholasab4ff394cf9d
diff --git a/api/_lib/custodial-key-health.js b/api/_lib/custodial-key-health.jsindex 5cd721265..59328061f 100644--- a/api/_lib/custodial-key-health.js+++ b/api/_lib/custodial-key-health.js@@ -31,7 +31,7 @@// that says a zero is not yet trustworthy.import { sql } from './db.js';-import { decryptSecret } from './secret-box.js';+import { decryptSecret, secretBoxKeyCandidates } from './secret-box.js';/** The platform's own agent-owner account. Anything else is a customer. */export const PLATFORM_AGENT_OWNER_EMAIL = '[email protected]';@@ -220,6 +220,11 @@ export function summarizeCustodialKeyHealth({ wallets, balances, readErrors = []* @returns {Promise<object>} the report shape summarizeCustodialKeyHealth builds*/export async function gatherCustodialKeyHealth({ connection, platformOnly = false, rpcUrl = null, topN = 20, keyCandidates = null } = {}) {+ // How many keys a decrypt may try, straight from secret-box's own precedence+ // so this can never drift from what decryptSecret() actually does. Zero means+ // this process holds no key at all, and every verdict below would be an+ // artifact of that rather than a fact about custody.+ const keys = keyCandidates == null ? secretBoxKeyCandidates().length : keyCandidates;const wallets = await readCustodialWalletRows({ platformOnly });await classifyCustodialSecrets(wallets);// The rotating multi-lane connection production reads balances through, not a@@ -238,7 +243,7 @@ export async function gatherCustodialKeyHealth({ connection, platformOnly = falsreadErrors,rpc: rpcUrl || process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com',topN,- keyCandidates,+ keyCandidates: keys,});}@@ -276,7 +281,11 @@ export async function strandedCustodyPanel({ ttlMs = STRANDED_SNAPSHOT_TTL_MS, finFlight = (async () => {const report = await gather();const panel = buildStrandedPanel(report);- cached = { at: Date.now(), panel };+ // Stamped with the clock the CALLER passed, not a second reading of the+ // wall clock: the age this panel reports and the TTL it is judged+ // against then come from one timeline, which is also what makes the+ // cache testable without faking timers.+ cached = { at: now, panel };return panel;})().finally(() => {inFlight = null;@@ -303,31 +312,59 @@ export function buildStrandedPanel(report) {const sol = report?.sol || {};const unread = counts.stranded_unread || 0;const funded = counts.stranded_funded || 0;- // A total is only a total when every sealed wallet actually got a balance- // read. While any is unread the SOL figures are a FLOOR, and the panel says- // so rather than letting a partial read render as a clean number.- const status = unread > 0 ? 'unknown' : funded > 0 ? 'stranded' : 'clear';+ const wallets = report?.wallets ?? 0;+ const undecryptable = report?.undecryptable ?? 0;+ const keys = report?.key_candidates;++ // Three ways a stranded number can be a lie, each answered before the honest+ // verdicts below:+ // • no key configured at all: every wallet reads sealed here regardless of+ // production state (secret-box fails closed in production, so this is a+ // misconfigured non-prod process, not a custody event).+ // • EVERY wallet failed: almost never 700 separately sealed wallets, almost+ // always one wrong key for this deployment. Reporting it as a mass+ // customer incident sends operators after the wrong problem.+ // • some sealed wallet never got a balance read: the totals are a floor.+ // Only when none of those hold is the SOL figure a measurement.+ let status = 'clear';+ let reason = 'clear';+ let detail = 'No funded custodial wallet sits behind an undecryptable key.';+ if (keys === 0) {+ status = 'unknown';+ reason = 'no_decryption_key';+ detail = 'This process holds no custodial decryption key, so every wallet reads as sealed here. The reading says nothing about custody.';+ } else if (wallets > 0 && undecryptable === wallets) {+ status = 'unknown';+ reason = 'key_mismatch';+ detail = `All ${wallets} custodial wallets failed to decrypt under the ${keys ?? 'configured'} key(s). A fleet-wide failure means the key is wrong for this deployment, not that every wallet is sealed.`;+ } else if (unread > 0) {+ status = 'unknown';+ reason = 'partial_read';+ detail = `${unread} sealed wallet(s) never got a balance read, so the stranded total is a floor, not a measurement.`;+ } else if (funded > 0) {+ status = 'stranded';+ reason = 'sealed';+ detail = `${funded} custodial wallet(s) hold ${sol.stranded ?? 0} SOL behind a retired encryption key (platform ${sol.stranded_platform ?? 0}, customer ${sol.stranded_customer ?? 0}).`;+ }+return {status,+ reason,checked_at: report?.checked_at || null,+ key_candidates: keys ?? null,wallets: report?.wallets ?? null,- undecryptable: report?.undecryptable ?? 0,+ undecryptable,stranded_funded: funded,stranded_unread: unread,sol_stranded: sol.stranded ?? 0,sol_stranded_platform: sol.stranded_platform ?? 0,sol_stranded_customer: sol.stranded_customer ?? 0,customer_wallets_stranded: (report?.top_stranded || []).filter((w) => !w.platform).length,- detail:- status === 'clear'- ? 'No funded custodial wallet sits behind an undecryptable key.'- : status === 'unknown'- ? `${unread} sealed wallet(s) never got a balance read, so the stranded total is a floor, not a measurement.`- : `${funded} custodial wallet(s) hold ${sol.stranded ?? 0} SOL behind a retired encryption key (platform ${sol.stranded_platform ?? 0}, customer ${sol.stranded_customer ?? 0}).`,+ detail,// Customer money that cannot be withdrawn is a support obligation, not an// ops number: the board names the decision doc rather than leaving the// reader to rediscover that recovery is impossible.- brief: (sol.stranded_customer || 0) > 0 ? 'docs/ops/stranded-wallets.md' : null,+ brief: status === 'stranded' && (sol.stranded_customer || 0) > 0 ? 'docs/ops/stranded-wallets.md' : null,top_stranded: (report?.top_stranded || []).slice(0, 10).map((w) => ({address: w.address,sol: w.sol,diff --git a/tests/avatars-list-scale.test.js b/tests/avatars-list-scale.test.jsnew file mode 100644index 000000000..7c49e5622--- /dev/null+++ b/tests/avatars-list-scale.test.js@@ -0,0 +1,127 @@+// The owner-visible listing and the public gallery totals are the two queries+// whose cost used to grow with the whole avatar catalog rather than with the+// page being served. Both were measured against the live 58k-row table:+//+// listAvatars({ includePublic: true }) 48.7 ms / 7,605 heap pages read,+// because `(owner_id = $1 or visibility = 'public')` is unindexable and+// Postgres seq-scanned and sorted every surviving row.+// searchPublicAvatars({ withTotals }) 42 ms sequential scan for the+// count + view-count sum, on every single request.+//+// These tests lock in the shapes that fixed them: a UNION ALL of two indexed+// branches, and a cached aggregate. They assert on the SQL the module emits+// rather than on timings, so they stay meaningful without a database.++import { describe, it, expect, beforeEach, vi } from 'vitest';++process.env.S3_PUBLIC_DOMAIN ||= 'https://cdn.test';+process.env.S3_BUCKET ||= 'test-bucket';++const calls = [];+vi.mock('../api/_lib/db.js', () => ({+ sql: vi.fn(async (text, params) => {+ calls.push({ text: typeof text === 'string' ? text : String(text), params });+ return [];+ }),+ isDbUnavailableError: () => false,+ isDbCapacityError: () => false,+}));++const cacheWrapCalls = [];+vi.mock('../api/_lib/cache.js', () => ({+ cacheWrap: vi.fn(async (key, ttl, fn) => {+ cacheWrapCalls.push({ key, ttl });+ return fn();+ }),+}));++const { listAvatars, searchPublicAvatars } = await import('../api/_lib/avatars.js');++const USER = '11111111-2222-3333-4444-555555555555';++function lastSelect() {+ return calls[calls.length - 1];+}++beforeEach(() => {+ calls.length = 0;+ cacheWrapCalls.length = 0;+});++describe('listAvatars row selection', () => {+ it('unions two indexed branches instead of an unindexable OR', async () => {+ await listAvatars({ userId: USER, limit: 50, includePublic: true });+ const { text, params } = lastSelect();++ expect(text).toContain('union all');+ // The predicate that forced the seq scan must be gone.+ expect(text).not.toMatch(/owner_id = \$1 or/);+ // Owner branch and public branch, each ordered + limited on its own so+ // Postgres can serve them from avatars_owner_idx / avatars_public_idx.+ expect(text).toContain("where owner_id = $1 and deleted_at is null order by created_at desc");+ expect(text).toContain("where visibility = 'public' and owner_id is distinct from $1");+ // limit+1 drives both branches and the outer merge, so hasMore still works.+ expect(params[params.length - 1]).toBe(51);+ });++ it('excludes the caller from the public branch with is-distinct-from', async () => {+ // A plain `owner_id <> $1` drops every NULL-owner row from the catalog.+ await listAvatars({ userId: USER, includePublic: true });+ expect(lastSelect().text).toContain('owner_id is distinct from $1');+ expect(lastSelect().text).not.toContain('owner_id <> $1');+ });++ it('keeps a single indexed branch when the caller wants only their own', async () => {+ await listAvatars({ userId: USER, limit: 10 });+ const { text, params } = lastSelect();+ expect(text).not.toContain('union all');+ expect(text).toContain('where owner_id = $1');+ expect(params).toEqual([USER, 11]);+ });++ it('applies visibility and cursor filters to every branch', async () => {+ await listAvatars({+ userId: USER,+ includePublic: true,+ visibility: 'public',+ cursor: '2026-09-01T00:00:00.000Z',+ limit: 20,+ });+ const { text, params } = lastSelect();+ // Both branches share the filters, so a cursor page can never return rows+ // from one branch that the other already passed.+ expect(text.match(/visibility = \$2/g)?.length).toBe(2);+ expect(text.match(/created_at < \$3/g)?.length).toBe(2);+ expect(params).toEqual([USER, 'public', new Date('2026-09-01T00:00:00.000Z'), 21]);+ });++ it('joins the wide column list to the union result, not to a scan', async () => {+ await listAvatars({ userId: USER, includePublic: true });+ const { text } = lastSelect();+ expect(text).toContain('join avatars a on a.id = v.id');+ expect(text).toContain('order by v.created_at desc');+ });+});++describe('public gallery totals', () => {+ it('caches the catalog-wide aggregate for the endpoint cache window', async () => {+ await searchPublicAvatars({ limit: 24, withTotals: true });+ expect(cacheWrapCalls).toHaveLength(1);+ expect(cacheWrapCalls[0].ttl).toBe(60);+ expect(cacheWrapCalls[0].key).toMatch(/^avatars:public-totals:[0-9a-f]{40}$/);+ });++ it('keys the cache by the filter set so a filtered view never reads unfiltered numbers', async () => {+ await searchPublicAvatars({ limit: 24, withTotals: true });+ await searchPublicAvatars({ limit: 24, withTotals: true, rigged: 'rigged' });+ await searchPublicAvatars({ limit: 24, withTotals: true, tag: 'knight' });+ const keys = cacheWrapCalls.map((c) => c.key);+ expect(new Set(keys).size).toBe(3);+ });++ it('runs no aggregate at all when totals were not asked for', async () => {+ await searchPublicAvatars({ limit: 24 });+ expect(cacheWrapCalls).toHaveLength(0);+ expect(calls.some((c) => c.text.includes('count(*)'))).toBe(false);+ });+});diff --git a/tests/custodial-key-health.test.js b/tests/custodial-key-health.test.jsnew file mode 100644index 000000000..2884281ca--- /dev/null+++ b/tests/custodial-key-health.test.js@@ -0,0 +1,236 @@+// The custody loss measurement (api/_lib/custodial-key-health.js): the one+// number the CLI audit and the ops board both read.+//+// Every fence here is a defect that already shipped:+//+// 1. an unread balance summed as a confirmed zero, so the audit certified+// "0 SOL stranded" on 2026-08-09 while two wallets held 0.12 SOL behind a+// dead key. A wallet with no balance read must never reach a total.+// 2. the audit carried its own ownership predicate with a different house+// account address than the reclaim leg enforces in SQL, filing 12 platform+// wallets as CUSTOMER ones. The split feeds a customer support obligation,+// so it has to be the reclaim leg's exact boundary.+// 3. the board must not turn a polled dashboard into a fleet-wide RPC scan:+// the panel is snapshot-cached and concurrent reads share one scan.++import { describe, it, expect, beforeEach, vi } from 'vitest';++vi.mock('../api/_lib/db.js', () => ({ sql: async () => [] }));+vi.mock('../api/_lib/secret-box.js', () => ({+ decryptSecret: async () => 'plaintext',+ secretBoxKeyCandidates: () => ['key'],+}));++const {+ isPlatformOwnedAgent,+ classifyCustodialSecrets,+ summarizeCustodialKeyHealth,+ buildStrandedPanel,+ strandedCustodyPanel,+ resetStrandedCustodyCache,+ PLATFORM_AGENT_OWNER_EMAIL,+} = await import('../api/_lib/custodial-key-health.js');++const wallet = (address, owner, decryptable, extra = {}) => ({+ agent_id: `ag-${address}`,+ name: address,+ address,+ owner,+ decryptable,+ reason: decryptable ? null : 'wrong_key',+ ...extra,+});++const HOUSE = PLATFORM_AGENT_OWNER_EMAIL;+const BOT = '[email protected]';+const CUSTOMER = '[email protected]';++describe('ownership boundary', () => {+ it('counts the house account and the circulation bots as platform', () => {+ expect(isPlatformOwnedAgent(HOUSE)).toBe(true);+ expect(isPlatformOwnedAgent(BOT)).toBe(true);+ expect(isPlatformOwnedAgent(' [email protected] ')).toBe(true);+ });++ it('counts everything else as a customer, including a lookalike domain', () => {+ expect(isPlatformOwnedAgent(CUSTOMER)).toBe(false);+ expect(isPlatformOwnedAgent('[email protected]')).toBe(false);+ // The suffix must match at the end, not anywhere: an attacker-chosen+ // local part must never buy platform classification.+ expect(isPlatformOwnedAgent('[email protected]')).toBe(false);+ expect(isPlatformOwnedAgent(null)).toBe(false);+ expect(isPlatformOwnedAgent(undefined)).toBe(false);+ });+});++describe('secret classification', () => {+ it('marks a wallet undecryptable with the WebCrypto failure named, not a generic error', async () => {+ const rows = [{ secret: 'sealed' }, { secret: 'fine' }];+ const err = Object.assign(new Error('The operation failed'), { name: 'OperationError' });+ const rowsOut = await classifyCustodialSecrets(rows, async (s) => {+ if (s === 'sealed') throw err;+ return 'plaintext';+ });+ expect(rowsOut[0].decryptable).toBe(false);+ expect(rowsOut[0].reason).toBe('wrong_key');+ expect(rowsOut[1].decryptable).toBe(true);+ expect(rowsOut[1].reason).toBe(null);+ });++ it('treats an empty plaintext as undecryptable rather than a healthy wallet', async () => {+ const rows = await classifyCustodialSecrets([{ secret: 'empty' }], async () => '');+ expect(rows[0].decryptable).toBe(false);+ expect(rows[0].reason).toBe('empty_plaintext');+ });+});++describe('summary counting rules', () => {+ it('never sums a wallet whose balance was not read', () => {+ const wallets = [+ wallet('SEALED_READ', CUSTOMER, false),+ wallet('SEALED_UNREAD', CUSTOMER, false),+ wallet('OK', HOUSE, true),+ ];+ const balances = new Map([['SEALED_READ', 0.25], ['OK', 1]]);+ const r = summarizeCustodialKeyHealth({+ wallets, balances,+ readErrors: [{ address: 'SEALED_UNREAD', reason: 'rpc_error: 403' }],+ });+ // 0.25, not 0.25 plus an invented zero for the wallet nobody could read.+ expect(r.sol.stranded).toBe(0.25);+ expect(r.counts.stranded_unread).toBe(1);+ expect(r.counts.stranded_funded).toBe(1);+ expect(r.unread_stranded.map((w) => w.address)).toEqual(['SEALED_UNREAD']);+ expect(r.read_errors).toBe(1);+ });++ it('splits stranded SOL by the reclaim leg’s ownership boundary', () => {+ const wallets = [+ wallet('BOT', BOT, false),+ wallet('HOUSE', HOUSE, false),+ wallet('CUST', CUSTOMER, false),+ ];+ const balances = new Map([['BOT', 0.07], ['HOUSE', 0.03], ['CUST', 0.35]]);+ const r = summarizeCustodialKeyHealth({ wallets, balances });+ expect(r.counts.stranded_platform).toBe(2);+ expect(r.counts.stranded_customer).toBe(1);+ expect(r.sol.stranded_platform).toBe(0.1);+ expect(r.sol.stranded_customer).toBe(0.35);+ expect(r.sol.stranded).toBe(0.45);+ });++ it('ranks the stranded list by balance and keeps every wallet identifiable', () => {+ const wallets = [wallet('SMALL', CUSTOMER, false), wallet('BIG', CUSTOMER, false)];+ const balances = new Map([['SMALL', 0.01], ['BIG', 1.2]]);+ const r = summarizeCustodialKeyHealth({ wallets, balances });+ expect(r.top_stranded.map((w) => w.address)).toEqual(['BIG', 'SMALL']);+ expect(r.top_stranded[0]).toMatchObject({ sol: 1.2, platform: false, reason: 'wrong_key' });+ expect(r.top_stranded[0].agent_id).toBe('ag-BIG');+ });++ it('reports a zero-balance sealed wallet as sealed but NOT funded', () => {+ const wallets = [wallet('EMPTY', CUSTOMER, false)];+ const r = summarizeCustodialKeyHealth({ wallets, balances: new Map([['EMPTY', 0]]) });+ expect(r.undecryptable).toBe(1);+ expect(r.counts.stranded_funded).toBe(0);+ expect(r.counts.stranded_unread).toBe(0);+ expect(r.sol.stranded).toBe(0);+ });+});++describe('ops panel verdict', () => {+ const report = (over = {}) => ({+ checked_at: '2026-09-02T00:00:00.000Z',+ wallets: 725,+ key_candidates: 1,+ undecryptable: 8,+ sol: { stranded: 0.49, stranded_platform: 0.14, stranded_customer: 0.35 },+ counts: { stranded_funded: 8, stranded_unread: 0 },+ top_stranded: [+ { address: 'A', sol: 0.35, platform: false, reason: 'wrong_key' },+ { address: 'B', sol: 0.14, platform: true, reason: 'wrong_key' },+ ],+ ...over,+ });++ it('says stranded, splits the SOL, and points at the owner decision brief', () => {+ const p = buildStrandedPanel(report());+ expect(p.status).toBe('stranded');+ expect(p.sol_stranded_customer).toBe(0.35);+ expect(p.customer_wallets_stranded).toBe(1);+ expect(p.brief).toBe('docs/ops/stranded-wallets.md');+ });++ it('refuses to call a partial read a measurement', () => {+ const p = buildStrandedPanel(report({ counts: { stranded_funded: 8, stranded_unread: 3 } }));+ expect(p.status).toBe('unknown');+ expect(p.reason).toBe('partial_read');+ expect(p.detail).toMatch(/floor, not a measurement/);+ });++ it('refuses to read a keyless process as a fleet of sealed wallets', () => {+ // secret-box fails closed in production, so a zero-candidate process is a+ // misconfigured non-prod one. Reporting its 100% failure rate as custody+ // loss would manufacture an incident out of a missing env var.+ const p = buildStrandedPanel(report({ key_candidates: 0, undecryptable: 725, counts: { stranded_funded: 700, stranded_unread: 0 } }));+ expect(p.status).toBe('unknown');+ expect(p.reason).toBe('no_decryption_key');+ });++ it('calls a fleet-wide decrypt failure a wrong key, not a mass customer incident', () => {+ const p = buildStrandedPanel(report({ undecryptable: 725, counts: { stranded_funded: 168, stranded_unread: 0 } }));+ expect(p.status).toBe('unknown');+ expect(p.reason).toBe('key_mismatch');+ expect(p.brief).toBe(null);+ });++ it('says clear only when nothing funded is sealed, and then names no brief', () => {+ const p = buildStrandedPanel(report({+ undecryptable: 0, counts: { stranded_funded: 0, stranded_unread: 0 },+ sol: { stranded: 0, stranded_platform: 0, stranded_customer: 0 }, top_stranded: [],+ }));+ expect(p.status).toBe('clear');+ expect(p.brief).toBe(null);+ });+});++describe('snapshot cache', () => {+ beforeEach(() => resetStrandedCustodyCache());++ it('scans once and serves the snapshot for the whole TTL', async () => {+ const gather = vi.fn(async () => ({ wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} }));+ await strandedCustodyPanel({ gather, now: 1_000 });+ const second = await strandedCustodyPanel({ gather, now: 1_000 + 60_000 });+ expect(gather).toHaveBeenCalledTimes(1);+ expect(second.cache_age_seconds).toBe(60);+ });++ it('rescans once the snapshot ages past the TTL', async () => {+ const gather = vi.fn(async () => ({ wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} }));+ await strandedCustodyPanel({ gather, ttlMs: 1_000, now: 0 });+ await strandedCustodyPanel({ gather, ttlMs: 1_000, now: Date.now() + 5_000 });+ expect(gather).toHaveBeenCalledTimes(2);+ });++ it('collapses concurrent reads into one fleet scan', async () => {+ let release;+ const gate = new Promise((r) => { release = r; });+ const gather = vi.fn(async () => {+ await gate;+ return { wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} };+ });+ const both = Promise.all([strandedCustodyPanel({ gather }), strandedCustodyPanel({ gather })]);+ release();+ await both;+ expect(gather).toHaveBeenCalledTimes(1);+ });++ it('does not cache a failed scan, so a transient outage is retried not frozen', async () => {+ const gather = vi.fn()+ .mockRejectedValueOnce(new Error('rpc down'))+ .mockResolvedValueOnce({ wallets: 3, undecryptable: 0, key_candidates: 1, counts: { stranded_funded: 0, stranded_unread: 0 }, sol: {} });+ await expect(strandedCustodyPanel({ gather })).rejects.toThrow('rpc down');+ await expect(strandedCustodyPanel({ gather })).resolves.toMatchObject({ status: 'clear' });+ expect(gather).toHaveBeenCalledTimes(2);+ });+});