Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(x402): pause settling from the failure itself when the sponsor cannot pay rent

nirholas38e862cdd925
diff --git a/api/_lib/ops/x402-settle-health.js b/api/_lib/ops/x402-settle-health.js
index 94edaf07c..0ed06e0ae 100644
--- a/api/_lib/ops/x402-settle-health.js
+++ b/api/_lib/ops/x402-settle-health.js
@@ -210,7 +210,7 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips
* reconciliation below re-attributes status-only 5xx faults to the governor /
* floor, clamped by min() so a window skew between the two logs can never
* invent faults or attribute more than actually happened.
- * @param {Array<{ success: boolean, paid: boolean, reason: string, n: number }>} buckets
+ * @param {Array<{ success: boolean, paid: boolean, reason: string, rent?: boolean, n: number }>} buckets
* @param {{ minAttempts?: number,
* facilitatorRejects?: { governor?: number, floor?: number } }} [opts]
* @returns {{ status: 'ok'|'degraded'|'down'|'unknown', settled: number,
@@ -240,6 +240,19 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac
if (b.reason === NO_SOLANA_ACCEPT) noSolanaAccept += n;
if (SPONSOR_FLOOR.test(String(b.reason || ''))) floorSignals += n;
if (FEE_GOVERNOR.test(String(b.reason || ''))) governorSkips += n;
+ // A rent-exemption failure on the fee payer wears a rail-shaped reason
+ // token (`simulation_failed`, `sweep_broadcast_failed`) while being the
+ // opposite of a rail fault: the transaction never reached the rail because
+ // the sponsor could not pay for it. Counting it as rail is what produced
+ // the wrong hint on 2026-08-28, when 3 hours of a dry sponsor were reported
+ // as `cause: rail` and the operator was pointed at duplicate signatures and
+ // RPC preflight instead of at a wallet holding 0.000899 SOL. The flag is
+ // computed from the full error_msg because the `:`-token this groups by
+ // throws the rent detail away.
+ if (b.rent) {
+ floorSignals += n;
+ continue;
+ }
if (isRailFault(b.reason)) {
faults += n;
faultBy[b.reason] = (faultBy[b.reason] || 0) + n;
@@ -274,6 +287,29 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac
.sort((a, b) => b.n - a.n);
if (attempts < minAttempts) {
+ // A sponsor under its floor is a HARD stop, not an unjudgeable window. Every
+ // settle it touches fails closed, so the attempts that would have been
+ // judged are precisely the ones the empty wallet prevented, and moving them
+ // out of `faults` (which is correct: they never reached the rail) must not
+ // let the sensor answer `unknown`. On 2026-08-28 that would have reported a
+ // three-hour, zero-settle outage as "too few attempts to judge". Down rather
+ // than degraded, per the precedence in diagnoseSettleDrop: under the floor
+ // nothing settles at all, where a spent governor budget still settles at the
+ // paced rate.
+ if (floorSignals >= minAttempts) {
+ const { cause, hint } = diagnoseSettleDrop({
+ noSolanaAccept, floorSignals, governorSkips, settled, faults,
+ });
+ return {
+ status: 'down',
+ settled, faults, attempts, rate: null, faultClasses,
+ cause, noSolanaAccept, floorSignals, governorSkips,
+ detail:
+ `settle halted: ${floorSignals} attempt(s) refused with the sponsor under its SOL floor ` +
+ `in ${WINDOW_INTERVAL}, ${settled} settled`,
+ hint,
+ };
+ }
// A throttled rail is not an unjudgeable one. When the reason there is
// nothing to judge is that the governor skipped the calls, say so and report
// `degraded`: the funding action is identical to the low-rate case, and
@@ -354,15 +390,17 @@ export async function gatherX402SettleHealth() {
const base = { name: /** @type {const} */ ('x402_settle'), label: 'x402 settlement success' };
try {
const { sql } = await import('../db.js');
- const buckets = /** @type {Array<{ success: boolean, paid: boolean, reason: string, n: number }>} */ (
+ const buckets = /** @type {Array<{ success: boolean, paid: boolean, reason: string, rent?: boolean, n: number }>} */ (
await sql`
SELECT success,
(amount_atomic > 0) AS paid,
COALESCE(NULLIF(split_part(error_msg, ':', 1), ''), 'none') AS reason,
+ ((error_msg ILIKE '%InsufficientFundsForRent%' AND error_msg ILIKE '%"account_index":0%')
+ OR error_msg ILIKE '%account (0) with insufficient funds for rent%') AS rent,
count(*)::int AS n
FROM x402_autonomous_log
WHERE ts >= now() - ${WINDOW_INTERVAL}::interval
- GROUP BY success, paid, reason
+ GROUP BY success, paid, reason, rent
`
);
// The facilitator's own book for the same window: the ring log only carries
diff --git a/api/_lib/x402/pipelines/ring-rebalance.js b/api/_lib/x402/pipelines/ring-rebalance.js
index ca2e4a4c5..345789f47 100644
--- a/api/_lib/x402/pipelines/ring-rebalance.js
+++ b/api/_lib/x402/pipelines/ring-rebalance.js
@@ -36,6 +36,7 @@ import { solanaConnection } from '../../solana/connection.js';
import { blockhashKey, getRecentBlockhashInfo, mintDecimals } from '../../solana/read-guards.js';
import { loadSeedKeypair, USDC_MINT } from '../pay.js';
import { ECONOMY_MASTER_ADDRESS } from '../../economy-master.js';
+import { noteSponsorRentFailure } from '../self-facilitator.js';
const log = logger('x402-ring-rebalance');
@@ -256,6 +257,11 @@ export async function run(ctx = {}) {
try {
signature = await conn.sendRawTransaction(vtx.serialize(), { skipPreflight: false, maxRetries: 5 });
} catch (err) {
+ // The sweep is fee-paid by the same sponsor the settle path uses, so a
+ // rent-exemption rejection here is the same platform condition and must
+ // reach the same guard. 86 of the 95 faults measured on 2026-08-28 arrived
+ // through this branch, not through settle simulation.
+ noteSponsorRentFailure(err?.message || err, feePayerKp.publicKey.toBase58());
return { success: false, amountAtomic: 0, errorMsg: `sweep_broadcast_failed:${String(err?.message || err).slice(0, 200)}` };
}
diff --git a/api/_lib/x402/self-facilitator.js b/api/_lib/x402/self-facilitator.js
index f5a4571df..2fa0ed0df 100644
--- a/api/_lib/x402/self-facilitator.js
+++ b/api/_lib/x402/self-facilitator.js
@@ -399,6 +399,59 @@ function noteFloorState(lamports, now) {
_floorState = { below: lamports < SPONSOR_SOL_FLOOR_LAMPORTS, at: now };
}
+/**
+ * Is this simulation error the chain saying the FEE PAYER cannot afford the fee
+ * without dropping below rent exemption?
+ *
+ * Account index 0 of a compiled Solana message is the fee payer, by definition,
+ * so `InsufficientFundsForRent { account_index: 0 }` is never about the buyer's
+ * token balance: it is the sponsor being too poor to sign. Both spellings the
+ * RPCs use are matched, the structured `err` object and the human sentence that
+ * reaches the sweep path inside a SendTransactionError message.
+ * @param {unknown} simErr
+ * @returns {boolean}
+ */
+export function isFeePayerRentFailure(simErr) {
+ if (simErr == null) return false;
+ if (typeof simErr === 'object') {
+ return Number(/** @type {any} */ (simErr)?.InsufficientFundsForRent?.account_index) === 0;
+ }
+ const s = String(simErr);
+ if (/InsufficientFundsForRent/.test(s)) return /"account_index"\s*:\s*0/.test(s);
+ return /account \(0\) with insufficient funds for rent/i.test(s);
+}
+
+/**
+ * Trip the floor guard from a settle failure instead of a balance read.
+ *
+ * The guard was only ever written by getBalance, which is exactly the call that
+ * stops answering when the Solana RPC lanes are over quota, and
+ * refreshSponsorFloorState fails open on that error by design. So the two
+ * failures compounded: on 2026-08-28 all four paid lanes were exhausted at once
+ * while the sponsor sat at 0.000899 SOL against a 0.02 floor, the guard never
+ * learned it, and the ring spent three hours making payments that could not
+ * settle. 95 attempts, 0 settled, every one of them dying on this exact error.
+ *
+ * A rent-exemption failure on our own fee payer is strictly better evidence than
+ * the balance read it replaces: it is free, it is the chain's own verdict, and
+ * it arrives precisely when the RPC is too degraded to answer anything else.
+ *
+ * Only OUR sponsor counts. In a self-pay settle the buyer is the fee payer, and
+ * tripping the platform-wide guard because one buyer is broke would withdraw the
+ * Solana accept from every 402 challenge on the site.
+ * @param {unknown} simErr
+ * @param {string|null|undefined} feePayer the transaction's fee payer
+ * @param {number} [now]
+ * @returns {boolean} true when the guard was tripped
+ */
+export function noteSponsorRentFailure(simErr, feePayer, now = Date.now()) {
+ if (!isFeePayerRentFailure(simErr)) return false;
+ const sponsor = env.X402_FEE_PAYER_SOLANA;
+ if (!sponsor || !feePayer || String(feePayer) !== String(sponsor)) return false;
+ _floorState = { below: true, at: now };
+ return true;
+}
+
// Keep the floor state warm on instances that never settle.
//
// The state above is per-process and was only written by the settle path, so a
@@ -769,6 +822,10 @@ async function assertSettleable({ tx, connection, requirement, decoded }) {
const simErr = sim?.value?.err ?? null;
if (simErr) {
const s = typeof simErr === 'object' ? JSON.stringify(simErr) : String(simErr);
+ // A dry sponsor is a platform condition, not this payment's fault: record
+ // it so the 402 challenge stops advertising a Solana accept that cannot
+ // settle, rather than rejecting one payment and inviting the next.
+ noteSponsorRentFailure(simErr, decoded?.feePayer);
return { ok: false, reason: `simulation_failed:${s}`.slice(0, 160) };
}
return { ok: true };
diff --git a/docs/ops/production-log-triage.md b/docs/ops/production-log-triage.md
index 755bf4648..2ef651cae 100644
--- a/docs/ops/production-log-triage.md
+++ b/docs/ops/production-log-triage.md
@@ -196,6 +196,35 @@ HTTP 502/503 GET|POST /api/x402/*, /api/mcp ua: threews-x402-autonomous/1.0 or
window, or `rail` for real payment-rail faults) and `detail` names the
withdrawal, so the sensor no longer points at the facilitator for a funding
problem.
+- **The loud variant: a rail-shaped storm that is really a dry sponsor
+ (`InsufficientFundsForRent`).** Observed 2026-08-28. This is the quiet
+ variant's opposite and it is easy to misread, because every symptom points at
+ the rail. The sponsor held 0.000899107 SOL against the 0.02 SOL floor, which
+ is 0.0000082 SOL of spendable headroom, less than two transaction fees, so
+ every transaction it fee-paid failed at simulation:
+ `simulation_failed:{"InsufficientFundsForRent":{"account_index":0}}` on the
+ verify path and `sweep_broadcast_failed:Simulation failed ... account (0) with
+ insufficient funds for rent` on the sweep path. **Account index 0 of a
+ compiled Solana message is the fee payer**, so that error is never about the
+ buyer's USDC: it is the sponsor being too poor to sign.
+
+ Why the quiet variant's guard did not catch it: `sponsorKnownBelowFloor()` was
+ only ever written by `getBalance`, and all four paid Solana RPC lanes were over
+ quota in the same window. `refreshSponsorFloorState()` fails open on an RPC
+ error by design, so the guard never learned the sponsor was dry and the ring
+ spent three hours attempting payments that could not settle: 95 attempts, 0
+ settled. Since 2026-08-28 the settle path reads the verdict off the failure
+ itself (`noteSponsorRentFailure()` in
+ [self-facilitator.js](../../api/_lib/x402/self-facilitator.js)), which costs no
+ RPC call and works precisely when the RPC is too degraded to answer one, and
+ the sensor counts a fee-payer rent failure as a floor signal rather than a rail
+ fault. Before that fix healthz reported `cause: rail` and its hint sent the
+ reader to duplicate signatures and RPC preflight.
+
+ **Confirm in one command, no logs needed:**
+ `curl -s https://api.mainnet-beta.solana.com -X POST -H 'content-type: application/json' -d '{"jsonrpc":"2.0","id":1,"method":"getBalance","params":["WwwuGbqHrwF5RG89KhUbmRWEvjnRH9k5kVM5p7T3WwW"]}'`
+ Under 20,000,000 lamports is below the floor; under ~900,000 the wallet cannot
+ pay a single fee. Fix is the owner top-up below.
- **Not a code bug.** The economy-rebalance keypair crash (assigned
`loadSignerKeypair`'s wrapper to `keypair`, read `.publicKey` of undefined)
is fixed and live in commit `bb02839f9`. When
diff --git a/tests/api/x402-settle-health.test.js b/tests/api/x402-settle-health.test.js
index 08025bc09..c540264e3 100644
--- a/tests/api/x402-settle-health.test.js
+++ b/tests/api/x402-settle-health.test.js
@@ -372,3 +372,63 @@ describe('classifySettleBuckets: facilitator-book reconciliation of status-only
expect(without.cause).toBe('rail');
});
});
+
+describe('classifySettleBuckets: a dry sponsor is not a rail fault', () => {
+ // The 2026-08-28 outage, reproduced from the real aggregation. The sponsor
+ // held 0.000899107 SOL against a 0.02 floor, so every transaction it fee-paid
+ // died on InsufficientFundsForRent at account index 0. The reason TOKEN those
+ // rows carry is rail-shaped (`sweep_broadcast_failed`, `simulation_failed`)
+ // because the rent detail lives past the first `:`, so the sensor counted 95
+ // rail faults and answered `cause: rail`. The hint then sent the operator to
+ // hunt duplicate signatures and RPC preflight faults instead of funding a
+ // wallet that had 0.0000082 SOL of spendable headroom.
+ const rentWave = [
+ { success: false, paid: false, reason: 'sweep_broadcast_failed', rent: true, n: 86 },
+ { success: false, paid: false, reason: 'simulation_failed', rent: true, n: 5 },
+ { success: false, paid: false, reason: 'rpc_preflight_failed', rent: false, n: 4 },
+ ];
+
+ it('names the sponsor floor, not the rail, when the fee payer cannot afford rent', () => {
+ const v = classifySettleBuckets(rentWave);
+ expect(v.cause).toBe('sponsor_floor');
+ expect(v.floorSignals).toBe(91);
+ // The 4 genuine preflight faults stay on the rail where they belong.
+ expect(v.faults).toBe(4);
+ expect(v.hint).toMatch(/sponsor/i);
+ });
+
+ it('reports DOWN, never UNKNOWN, when the floor is why there is nothing to judge', () => {
+ // Moving rent rows out of `faults` is correct (they never reached the rail)
+ // but it drops `attempts` under MIN_ATTEMPTS. Without the floor branch the
+ // sensor would answer "too few attempts to judge" for a three-hour outage.
+ const v = classifySettleBuckets([
+ { success: false, paid: false, reason: 'sweep_broadcast_failed', rent: true, n: 95 },
+ ]);
+ expect(v.status).toBe('down');
+ expect(v.cause).toBe('sponsor_floor');
+ expect(v.detail).toMatch(/sponsor under its SOL floor/);
+ });
+
+ it('a handful of rent rows on an otherwise idle ring still reads UNKNOWN', () => {
+ // The floor branch keys off the same MIN_ATTEMPTS as every other verdict, so
+ // a couple of stray rows cannot declare an outage on a quiet ring.
+ const v = classifySettleBuckets([
+ { success: false, paid: false, reason: 'simulation_failed', rent: true, n: 3 },
+ ]);
+ expect(v.status).toBe('unknown');
+ });
+
+ it('a bucket with no rent flag behaves exactly as it always did', () => {
+ const withFlag = classifySettleBuckets([
+ { success: true, paid: true, n: 90 },
+ { success: false, paid: false, reason: 'http_502', rent: false, n: 10 },
+ ]);
+ const without = classifySettleBuckets([
+ { success: true, paid: true, n: 90 },
+ { success: false, paid: false, reason: 'http_502', n: 10 },
+ ]);
+ expect(withFlag.status).toBe(without.status);
+ expect(withFlag.faults).toBe(without.faults);
+ expect(withFlag.rate).toBe(without.rate);
+ });
+});
diff --git a/tests/x402-sponsor-rent-floor.test.js b/tests/x402-sponsor-rent-floor.test.js
new file mode 100644
index 000000000..198629ae8
--- /dev/null
+++ b/tests/x402-sponsor-rent-floor.test.js
@@ -0,0 +1,110 @@
+// Cover for the sponsor floor guard learning about a dry fee payer from a
+// SETTLE FAILURE rather than from a balance read
+// (api/_lib/x402/self-facilitator.js).
+//
+// Measured in production on 2026-08-28. The sponsor wallet sat at 0.000899107
+// SOL against a 0.02 SOL floor, which is 0.0000082 SOL of spendable headroom,
+// less than two transaction fees. Every transaction it fee-paid therefore died
+// at simulation with InsufficientFundsForRent on account index 0.
+//
+// The guard that exists to stop exactly this never fired, because the only thing
+// that ever wrote it was getBalance, and all four paid Solana RPC lanes were
+// over quota at the same moment. refreshSponsorFloorState fails open on an RPC
+// error by design, so the two failures compounded: the ring spent three hours
+// making payments that could not settle. 95 attempts, 0 settled, and healthz
+// reported the cause as `rail`, which sends an operator hunting duplicate
+// signatures instead of funding the wallet.
+//
+// The fix reads the verdict off the failure itself. It costs no RPC call and it
+// is available precisely when the RPC is too degraded to answer one.
+
+import { describe, it, expect, beforeEach } from 'vitest';
+
+const SPONSOR = 'WwwuGbqHrwF5RG89KhUbmRWEvjnRH9k5kVM5p7T3WwW';
+const OTHER = 'wwwwwDxFWRn7grgr3Esrsg5C6NvDoDHSA4gaCffccrU';
+
+process.env.X402_FEE_PAYER_SOLANA = SPONSOR;
+
+const { isFeePayerRentFailure, noteSponsorRentFailure, sponsorKnownBelowFloor } = await import(
+ '../api/_lib/x402/self-facilitator.js'
+);
+
+// The two spellings the RPCs actually produced, kept verbatim from the
+// x402_self_facilitator_log and x402_autonomous_log rows of that outage.
+const SIM_ERR_OBJECT = { InsufficientFundsForRent: { account_index: 0 } };
+const SWEEP_MESSAGE =
+ 'Simulation failed. \nMessage: Transaction simulation failed: Transaction results in an ' +
+ 'account (0) with insufficient funds for rent. \n\nCatch the `SendTransactionError` and call `getLogs()`';
+
+describe('isFeePayerRentFailure', () => {
+ it('recognizes the structured simulation error', () => {
+ expect(isFeePayerRentFailure(SIM_ERR_OBJECT)).toBe(true);
+ expect(isFeePayerRentFailure(JSON.stringify(SIM_ERR_OBJECT))).toBe(true);
+ });
+
+ it('recognizes the sentence the sweep path receives', () => {
+ expect(isFeePayerRentFailure(SWEEP_MESSAGE)).toBe(true);
+ });
+
+ it('ignores rent failures against an account that is not the fee payer', () => {
+ // Account index 0 is the fee payer by definition. Any other index is some
+ // other account in the message and says nothing about the sponsor.
+ expect(isFeePayerRentFailure({ InsufficientFundsForRent: { account_index: 3 } })).toBe(false);
+ });
+
+ it('ignores unrelated failures and empty input', () => {
+ expect(isFeePayerRentFailure({ InstructionError: [0, { Custom: 1 }] })).toBe(false);
+ expect(isFeePayerRentFailure('blockhash not found')).toBe(false);
+ expect(isFeePayerRentFailure(null)).toBe(false);
+ expect(isFeePayerRentFailure(undefined)).toBe(false);
+ expect(isFeePayerRentFailure('')).toBe(false);
+ });
+});
+
+describe('noteSponsorRentFailure', () => {
+ // The guard holds for 60s and clears by timestamp, so each case runs on its
+ // own clock far past any previous trip rather than sharing a reset hook.
+ // There is deliberately no way to force-clear it: the expiry IS the recovery
+ // path, and a test that reached around it would not be testing the guard.
+ const BASE = 1_787_877_609_500;
+ let tick = 0;
+ beforeEach(() => {
+ tick += 1;
+ });
+ const clock = () => BASE + tick * 600_000;
+
+ it('trips the floor guard when our own sponsor is the broke fee payer', () => {
+ const now = clock();
+ expect(sponsorKnownBelowFloor(now)).toBe(false);
+ expect(noteSponsorRentFailure(SIM_ERR_OBJECT, SPONSOR, now)).toBe(true);
+ expect(sponsorKnownBelowFloor(now)).toBe(true);
+ });
+
+ it('trips from the sweep path message too, which carried 86 of the 95 faults', () => {
+ const now = clock();
+ expect(noteSponsorRentFailure(SWEEP_MESSAGE, SPONSOR, now)).toBe(true);
+ expect(sponsorKnownBelowFloor(now)).toBe(true);
+ });
+
+ it('does NOT trip when the broke fee payer is a self-paying buyer', () => {
+ // In a self-pay settle the buyer is the fee payer. Tripping the
+ // platform-wide guard because one buyer is broke would withdraw the Solana
+ // accept from every 402 challenge on the site: a self-inflicted outage.
+ const now = clock();
+ expect(noteSponsorRentFailure(SIM_ERR_OBJECT, OTHER, now)).toBe(false);
+ expect(sponsorKnownBelowFloor(now)).toBe(false);
+ });
+
+ it('does NOT trip on a failure that is not about rent', () => {
+ const now = clock();
+ expect(noteSponsorRentFailure({ InstructionError: [0, { Custom: 1 }] }, SPONSOR, now)).toBe(false);
+ expect(sponsorKnownBelowFloor(now)).toBe(false);
+ });
+
+ it('expires so a topped-up sponsor starts settling again without a redeploy', () => {
+ const now = clock();
+ noteSponsorRentFailure(SIM_ERR_OBJECT, SPONSOR, now);
+ expect(sponsorKnownBelowFloor(now + 59_000)).toBe(true);
+ expect(sponsorKnownBelowFloor(now + 61_000)).toBe(false);
+ });
+});