Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(x402): pause settling from the failure itself when the sponsor cannot pay rent
nirholas38e862cdd925
diff --git a/api/_lib/ops/x402-settle-health.js b/api/_lib/ops/x402-settle-health.jsindex 94edaf07c..0ed06e0ae 100644--- a/api/_lib/ops/x402-settle-health.js+++ b/api/_lib/ops/x402-settle-health.js@@ -210,7 +210,7 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips* reconciliation below re-attributes status-only 5xx faults to the governor /* floor, clamped by min() so a window skew between the two logs can never* invent faults or attribute more than actually happened.- * @param {Array<{ success: boolean, paid: boolean, reason: string, n: number }>} buckets+ * @param {Array<{ success: boolean, paid: boolean, reason: string, rent?: boolean, n: number }>} buckets* @param {{ minAttempts?: number,* facilitatorRejects?: { governor?: number, floor?: number } }} [opts]* @returns {{ status: 'ok'|'degraded'|'down'|'unknown', settled: number,@@ -240,6 +240,19 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, facif (b.reason === NO_SOLANA_ACCEPT) noSolanaAccept += n;if (SPONSOR_FLOOR.test(String(b.reason || ''))) floorSignals += n;if (FEE_GOVERNOR.test(String(b.reason || ''))) governorSkips += n;+ // A rent-exemption failure on the fee payer wears a rail-shaped reason+ // token (`simulation_failed`, `sweep_broadcast_failed`) while being the+ // opposite of a rail fault: the transaction never reached the rail because+ // the sponsor could not pay for it. Counting it as rail is what produced+ // the wrong hint on 2026-08-28, when 3 hours of a dry sponsor were reported+ // as `cause: rail` and the operator was pointed at duplicate signatures and+ // RPC preflight instead of at a wallet holding 0.000899 SOL. The flag is+ // computed from the full error_msg because the `:`-token this groups by+ // throws the rent detail away.+ if (b.rent) {+ floorSignals += n;+ continue;+ }if (isRailFault(b.reason)) {faults += n;faultBy[b.reason] = (faultBy[b.reason] || 0) + n;@@ -274,6 +287,29 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac.sort((a, b) => b.n - a.n);if (attempts < minAttempts) {+ // A sponsor under its floor is a HARD stop, not an unjudgeable window. Every+ // settle it touches fails closed, so the attempts that would have been+ // judged are precisely the ones the empty wallet prevented, and moving them+ // out of `faults` (which is correct: they never reached the rail) must not+ // let the sensor answer `unknown`. On 2026-08-28 that would have reported a+ // three-hour, zero-settle outage as "too few attempts to judge". Down rather+ // than degraded, per the precedence in diagnoseSettleDrop: under the floor+ // nothing settles at all, where a spent governor budget still settles at the+ // paced rate.+ if (floorSignals >= minAttempts) {+ const { cause, hint } = diagnoseSettleDrop({+ noSolanaAccept, floorSignals, governorSkips, settled, faults,+ });+ return {+ status: 'down',+ settled, faults, attempts, rate: null, faultClasses,+ cause, noSolanaAccept, floorSignals, governorSkips,+ detail:+ `settle halted: ${floorSignals} attempt(s) refused with the sponsor under its SOL floor ` ++ `in ${WINDOW_INTERVAL}, ${settled} settled`,+ hint,+ };+ }// A throttled rail is not an unjudgeable one. When the reason there is// nothing to judge is that the governor skipped the calls, say so and report// `degraded`: the funding action is identical to the low-rate case, and@@ -354,15 +390,17 @@ export async function gatherX402SettleHealth() {const base = { name: /** @type {const} */ ('x402_settle'), label: 'x402 settlement success' };try {const { sql } = await import('../db.js');- const buckets = /** @type {Array<{ success: boolean, paid: boolean, reason: string, n: number }>} */ (+ const buckets = /** @type {Array<{ success: boolean, paid: boolean, reason: string, rent?: boolean, n: number }>} */ (await sql`SELECT success,(amount_atomic > 0) AS paid,COALESCE(NULLIF(split_part(error_msg, ':', 1), ''), 'none') AS reason,+ ((error_msg ILIKE '%InsufficientFundsForRent%' AND error_msg ILIKE '%"account_index":0%')+ OR error_msg ILIKE '%account (0) with insufficient funds for rent%') AS rent,count(*)::int AS nFROM x402_autonomous_logWHERE ts >= now() - ${WINDOW_INTERVAL}::interval- GROUP BY success, paid, reason+ GROUP BY success, paid, reason, rent`);// The facilitator's own book for the same window: the ring log only carriesdiff --git a/api/_lib/x402/pipelines/ring-rebalance.js b/api/_lib/x402/pipelines/ring-rebalance.jsindex ca2e4a4c5..345789f47 100644--- a/api/_lib/x402/pipelines/ring-rebalance.js+++ b/api/_lib/x402/pipelines/ring-rebalance.js@@ -36,6 +36,7 @@ import { solanaConnection } from '../../solana/connection.js';import { blockhashKey, getRecentBlockhashInfo, mintDecimals } from '../../solana/read-guards.js';import { loadSeedKeypair, USDC_MINT } from '../pay.js';import { ECONOMY_MASTER_ADDRESS } from '../../economy-master.js';+import { noteSponsorRentFailure } from '../self-facilitator.js';const log = logger('x402-ring-rebalance');@@ -256,6 +257,11 @@ export async function run(ctx = {}) {try {signature = await conn.sendRawTransaction(vtx.serialize(), { skipPreflight: false, maxRetries: 5 });} catch (err) {+ // The sweep is fee-paid by the same sponsor the settle path uses, so a+ // rent-exemption rejection here is the same platform condition and must+ // reach the same guard. 86 of the 95 faults measured on 2026-08-28 arrived+ // through this branch, not through settle simulation.+ noteSponsorRentFailure(err?.message || err, feePayerKp.publicKey.toBase58());return { success: false, amountAtomic: 0, errorMsg: `sweep_broadcast_failed:${String(err?.message || err).slice(0, 200)}` };}diff --git a/api/_lib/x402/self-facilitator.js b/api/_lib/x402/self-facilitator.jsindex f5a4571df..2fa0ed0df 100644--- a/api/_lib/x402/self-facilitator.js+++ b/api/_lib/x402/self-facilitator.js@@ -399,6 +399,59 @@ function noteFloorState(lamports, now) {_floorState = { below: lamports < SPONSOR_SOL_FLOOR_LAMPORTS, at: now };}+/**+ * Is this simulation error the chain saying the FEE PAYER cannot afford the fee+ * without dropping below rent exemption?+ *+ * Account index 0 of a compiled Solana message is the fee payer, by definition,+ * so `InsufficientFundsForRent { account_index: 0 }` is never about the buyer's+ * token balance: it is the sponsor being too poor to sign. Both spellings the+ * RPCs use are matched, the structured `err` object and the human sentence that+ * reaches the sweep path inside a SendTransactionError message.+ * @param {unknown} simErr+ * @returns {boolean}+ */+export function isFeePayerRentFailure(simErr) {+ if (simErr == null) return false;+ if (typeof simErr === 'object') {+ return Number(/** @type {any} */ (simErr)?.InsufficientFundsForRent?.account_index) === 0;+ }+ const s = String(simErr);+ if (/InsufficientFundsForRent/.test(s)) return /"account_index"\s*:\s*0/.test(s);+ return /account \(0\) with insufficient funds for rent/i.test(s);+}++/**+ * Trip the floor guard from a settle failure instead of a balance read.+ *+ * The guard was only ever written by getBalance, which is exactly the call that+ * stops answering when the Solana RPC lanes are over quota, and+ * refreshSponsorFloorState fails open on that error by design. So the two+ * failures compounded: on 2026-08-28 all four paid lanes were exhausted at once+ * while the sponsor sat at 0.000899 SOL against a 0.02 floor, the guard never+ * learned it, and the ring spent three hours making payments that could not+ * settle. 95 attempts, 0 settled, every one of them dying on this exact error.+ *+ * A rent-exemption failure on our own fee payer is strictly better evidence than+ * the balance read it replaces: it is free, it is the chain's own verdict, and+ * it arrives precisely when the RPC is too degraded to answer anything else.+ *+ * Only OUR sponsor counts. In a self-pay settle the buyer is the fee payer, and+ * tripping the platform-wide guard because one buyer is broke would withdraw the+ * Solana accept from every 402 challenge on the site.+ * @param {unknown} simErr+ * @param {string|null|undefined} feePayer the transaction's fee payer+ * @param {number} [now]+ * @returns {boolean} true when the guard was tripped+ */+export function noteSponsorRentFailure(simErr, feePayer, now = Date.now()) {+ if (!isFeePayerRentFailure(simErr)) return false;+ const sponsor = env.X402_FEE_PAYER_SOLANA;+ if (!sponsor || !feePayer || String(feePayer) !== String(sponsor)) return false;+ _floorState = { below: true, at: now };+ return true;+}+// Keep the floor state warm on instances that never settle.//// The state above is per-process and was only written by the settle path, so a@@ -769,6 +822,10 @@ async function assertSettleable({ tx, connection, requirement, decoded }) {const simErr = sim?.value?.err ?? null;if (simErr) {const s = typeof simErr === 'object' ? JSON.stringify(simErr) : String(simErr);+ // A dry sponsor is a platform condition, not this payment's fault: record+ // it so the 402 challenge stops advertising a Solana accept that cannot+ // settle, rather than rejecting one payment and inviting the next.+ noteSponsorRentFailure(simErr, decoded?.feePayer);return { ok: false, reason: `simulation_failed:${s}`.slice(0, 160) };}return { ok: true };diff --git a/docs/ops/production-log-triage.md b/docs/ops/production-log-triage.mdindex 755bf4648..2ef651cae 100644--- a/docs/ops/production-log-triage.md+++ b/docs/ops/production-log-triage.md@@ -196,6 +196,35 @@ HTTP 502/503 GET|POST /api/x402/*, /api/mcp ua: threews-x402-autonomous/1.0 orwindow, or `rail` for real payment-rail faults) and `detail` names thewithdrawal, so the sensor no longer points at the facilitator for a fundingproblem.+- **The loud variant: a rail-shaped storm that is really a dry sponsor+ (`InsufficientFundsForRent`).** Observed 2026-08-28. This is the quiet+ variant's opposite and it is easy to misread, because every symptom points at+ the rail. The sponsor held 0.000899107 SOL against the 0.02 SOL floor, which+ is 0.0000082 SOL of spendable headroom, less than two transaction fees, so+ every transaction it fee-paid failed at simulation:+ `simulation_failed:{"InsufficientFundsForRent":{"account_index":0}}` on the+ verify path and `sweep_broadcast_failed:Simulation failed ... account (0) with+ insufficient funds for rent` on the sweep path. **Account index 0 of a+ compiled Solana message is the fee payer**, so that error is never about the+ buyer's USDC: it is the sponsor being too poor to sign.++ Why the quiet variant's guard did not catch it: `sponsorKnownBelowFloor()` was+ only ever written by `getBalance`, and all four paid Solana RPC lanes were over+ quota in the same window. `refreshSponsorFloorState()` fails open on an RPC+ error by design, so the guard never learned the sponsor was dry and the ring+ spent three hours attempting payments that could not settle: 95 attempts, 0+ settled. Since 2026-08-28 the settle path reads the verdict off the failure+ itself (`noteSponsorRentFailure()` in+ [self-facilitator.js](../../api/_lib/x402/self-facilitator.js)), which costs no+ RPC call and works precisely when the RPC is too degraded to answer one, and+ the sensor counts a fee-payer rent failure as a floor signal rather than a rail+ fault. Before that fix healthz reported `cause: rail` and its hint sent the+ reader to duplicate signatures and RPC preflight.++ **Confirm in one command, no logs needed:**+ `curl -s https://api.mainnet-beta.solana.com -X POST -H 'content-type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["WwwuGbqHrwF5RG89KhUbmRWEvjnRH9k5kVM5p7T3WwW"]}'`+ Under 20,000,000 lamports is below the floor; under ~900,000 the wallet cannot+ pay a single fee. Fix is the owner top-up below.- **Not a code bug.** The economy-rebalance keypair crash (assigned`loadSignerKeypair`'s wrapper to `keypair`, read `.publicKey` of undefined)is fixed and live in commit `bb02839f9`. Whendiff --git a/tests/api/x402-settle-health.test.js b/tests/api/x402-settle-health.test.jsindex 08025bc09..c540264e3 100644--- a/tests/api/x402-settle-health.test.js+++ b/tests/api/x402-settle-health.test.js@@ -372,3 +372,63 @@ describe('classifySettleBuckets: facilitator-book reconciliation of status-onlyexpect(without.cause).toBe('rail');});});++describe('classifySettleBuckets: a dry sponsor is not a rail fault', () => {+ // The 2026-08-28 outage, reproduced from the real aggregation. The sponsor+ // held 0.000899107 SOL against a 0.02 floor, so every transaction it fee-paid+ // died on InsufficientFundsForRent at account index 0. The reason TOKEN those+ // rows carry is rail-shaped (`sweep_broadcast_failed`, `simulation_failed`)+ // because the rent detail lives past the first `:`, so the sensor counted 95+ // rail faults and answered `cause: rail`. The hint then sent the operator to+ // hunt duplicate signatures and RPC preflight faults instead of funding a+ // wallet that had 0.0000082 SOL of spendable headroom.+ const rentWave = [+ { success: false, paid: false, reason: 'sweep_broadcast_failed', rent: true, n: 86 },+ { success: false, paid: false, reason: 'simulation_failed', rent: true, n: 5 },+ { success: false, paid: false, reason: 'rpc_preflight_failed', rent: false, n: 4 },+ ];++ it('names the sponsor floor, not the rail, when the fee payer cannot afford rent', () => {+ const v = classifySettleBuckets(rentWave);+ expect(v.cause).toBe('sponsor_floor');+ expect(v.floorSignals).toBe(91);+ // The 4 genuine preflight faults stay on the rail where they belong.+ expect(v.faults).toBe(4);+ expect(v.hint).toMatch(/sponsor/i);+ });++ it('reports DOWN, never UNKNOWN, when the floor is why there is nothing to judge', () => {+ // Moving rent rows out of `faults` is correct (they never reached the rail)+ // but it drops `attempts` under MIN_ATTEMPTS. Without the floor branch the+ // sensor would answer "too few attempts to judge" for a three-hour outage.+ const v = classifySettleBuckets([+ { success: false, paid: false, reason: 'sweep_broadcast_failed', rent: true, n: 95 },+ ]);+ expect(v.status).toBe('down');+ expect(v.cause).toBe('sponsor_floor');+ expect(v.detail).toMatch(/sponsor under its SOL floor/);+ });++ it('a handful of rent rows on an otherwise idle ring still reads UNKNOWN', () => {+ // The floor branch keys off the same MIN_ATTEMPTS as every other verdict, so+ // a couple of stray rows cannot declare an outage on a quiet ring.+ const v = classifySettleBuckets([+ { success: false, paid: false, reason: 'simulation_failed', rent: true, n: 3 },+ ]);+ expect(v.status).toBe('unknown');+ });++ it('a bucket with no rent flag behaves exactly as it always did', () => {+ const withFlag = classifySettleBuckets([+ { success: true, paid: true, n: 90 },+ { success: false, paid: false, reason: 'http_502', rent: false, n: 10 },+ ]);+ const without = classifySettleBuckets([+ { success: true, paid: true, n: 90 },+ { success: false, paid: false, reason: 'http_502', n: 10 },+ ]);+ expect(withFlag.status).toBe(without.status);+ expect(withFlag.faults).toBe(without.faults);+ expect(withFlag.rate).toBe(without.rate);+ });+});diff --git a/tests/x402-sponsor-rent-floor.test.js b/tests/x402-sponsor-rent-floor.test.jsnew file mode 100644index 000000000..198629ae8--- /dev/null+++ b/tests/x402-sponsor-rent-floor.test.js@@ -0,0 +1,110 @@+// Cover for the sponsor floor guard learning about a dry fee payer from a+// SETTLE FAILURE rather than from a balance read+// (api/_lib/x402/self-facilitator.js).+//+// Measured in production on 2026-08-28. The sponsor wallet sat at 0.000899107+// SOL against a 0.02 SOL floor, which is 0.0000082 SOL of spendable headroom,+// less than two transaction fees. Every transaction it fee-paid therefore died+// at simulation with InsufficientFundsForRent on account index 0.+//+// The guard that exists to stop exactly this never fired, because the only thing+// that ever wrote it was getBalance, and all four paid Solana RPC lanes were+// over quota at the same moment. refreshSponsorFloorState fails open on an RPC+// error by design, so the two failures compounded: the ring spent three hours+// making payments that could not settle. 95 attempts, 0 settled, and healthz+// reported the cause as `rail`, which sends an operator hunting duplicate+// signatures instead of funding the wallet.+//+// The fix reads the verdict off the failure itself. It costs no RPC call and it+// is available precisely when the RPC is too degraded to answer one.++import { describe, it, expect, beforeEach } from 'vitest';++const SPONSOR = 'WwwuGbqHrwF5RG89KhUbmRWEvjnRH9k5kVM5p7T3WwW';+const OTHER = 'wwwwwDxFWRn7grgr3Esrsg5C6NvDoDHSA4gaCffccrU';++process.env.X402_FEE_PAYER_SOLANA = SPONSOR;++const { isFeePayerRentFailure, noteSponsorRentFailure, sponsorKnownBelowFloor } = await import(+ '../api/_lib/x402/self-facilitator.js'+);++// The two spellings the RPCs actually produced, kept verbatim from the+// x402_self_facilitator_log and x402_autonomous_log rows of that outage.+const SIM_ERR_OBJECT = { InsufficientFundsForRent: { account_index: 0 } };+const SWEEP_MESSAGE =+ 'Simulation failed. \nMessage: Transaction simulation failed: Transaction results in an ' ++ 'account (0) with insufficient funds for rent. \n\nCatch the `SendTransactionError` and call `getLogs()`';++describe('isFeePayerRentFailure', () => {+ it('recognizes the structured simulation error', () => {+ expect(isFeePayerRentFailure(SIM_ERR_OBJECT)).toBe(true);+ expect(isFeePayerRentFailure(JSON.stringify(SIM_ERR_OBJECT))).toBe(true);+ });++ it('recognizes the sentence the sweep path receives', () => {+ expect(isFeePayerRentFailure(SWEEP_MESSAGE)).toBe(true);+ });++ it('ignores rent failures against an account that is not the fee payer', () => {+ // Account index 0 is the fee payer by definition. Any other index is some+ // other account in the message and says nothing about the sponsor.+ expect(isFeePayerRentFailure({ InsufficientFundsForRent: { account_index: 3 } })).toBe(false);+ });++ it('ignores unrelated failures and empty input', () => {+ expect(isFeePayerRentFailure({ InstructionError: [0, { Custom: 1 }] })).toBe(false);+ expect(isFeePayerRentFailure('blockhash not found')).toBe(false);+ expect(isFeePayerRentFailure(null)).toBe(false);+ expect(isFeePayerRentFailure(undefined)).toBe(false);+ expect(isFeePayerRentFailure('')).toBe(false);+ });+});++describe('noteSponsorRentFailure', () => {+ // The guard holds for 60s and clears by timestamp, so each case runs on its+ // own clock far past any previous trip rather than sharing a reset hook.+ // There is deliberately no way to force-clear it: the expiry IS the recovery+ // path, and a test that reached around it would not be testing the guard.+ const BASE = 1_787_877_609_500;+ let tick = 0;+ beforeEach(() => {+ tick += 1;+ });+ const clock = () => BASE + tick * 600_000;++ it('trips the floor guard when our own sponsor is the broke fee payer', () => {+ const now = clock();+ expect(sponsorKnownBelowFloor(now)).toBe(false);+ expect(noteSponsorRentFailure(SIM_ERR_OBJECT, SPONSOR, now)).toBe(true);+ expect(sponsorKnownBelowFloor(now)).toBe(true);+ });++ it('trips from the sweep path message too, which carried 86 of the 95 faults', () => {+ const now = clock();+ expect(noteSponsorRentFailure(SWEEP_MESSAGE, SPONSOR, now)).toBe(true);+ expect(sponsorKnownBelowFloor(now)).toBe(true);+ });++ it('does NOT trip when the broke fee payer is a self-paying buyer', () => {+ // In a self-pay settle the buyer is the fee payer. Tripping the+ // platform-wide guard because one buyer is broke would withdraw the Solana+ // accept from every 402 challenge on the site: a self-inflicted outage.+ const now = clock();+ expect(noteSponsorRentFailure(SIM_ERR_OBJECT, OTHER, now)).toBe(false);+ expect(sponsorKnownBelowFloor(now)).toBe(false);+ });++ it('does NOT trip on a failure that is not about rent', () => {+ const now = clock();+ expect(noteSponsorRentFailure({ InstructionError: [0, { Custom: 1 }] }, SPONSOR, now)).toBe(false);+ expect(sponsorKnownBelowFloor(now)).toBe(false);+ });++ it('expires so a topped-up sponsor starts settling again without a redeploy', () => {+ const now = clock();+ noteSponsorRentFailure(SIM_ERR_OBJECT, SPONSOR, now);+ expect(sponsorKnownBelowFloor(now + 59_000)).toBe(true);+ expect(sponsorKnownBelowFloor(now + 61_000)).toBe(false);+ });+});