Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(x402): size the seed cron to the fee budget so it stops flooding the facilitator with refused settles

nirholasc7befebdd33b
diff --git a/api/_lib/ops/x402-settle-health.js b/api/_lib/ops/x402-settle-health.js
index 9a5b3a152..f32847e53 100644
--- a/api/_lib/ops/x402-settle-health.js
+++ b/api/_lib/ops/x402-settle-health.js
@@ -249,6 +249,41 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips
};
}
+// The admission gate exists so a spent fee budget costs one skipped call, not a
+// signed transfer plus a simulated verify refused at the end. A caller that goes
+// around it leaves no trace in the ring log (it is not the ring), so the rate
+// this sensor reports stays green while the facilitator refuses thousands of
+// settles an hour. Measured on production 2026-09-29: x402_settle read `ok, 97%`
+// while 43,848 of 46,375 facilitator settle attempts in 24h were governor
+// refusals, about 95% of them the x402 seed cron firing 60 sponsor-mode tips
+// every two minutes against a budget that funded one.
+//
+// Degraded, not down: the calls that asked the gate still settle. It fires only
+// when the bypass is both material (MIN_ATTEMPTS) and larger than everything the
+// ring settled, so the residue of a cross-instance admission race (a handful an
+// hour, each still recorded) never trips it.
+function gateBypassVerdict(verdict, { gateBypass, settled, minAttempts }) {
+ if (!(gateBypass >= minAttempts && gateBypass > settled)) return null;
+ return {
+ ...verdict,
+ status: /** @type {const} */ ('degraded'),
+ cause: /** @type {const} */ ('fee_governor'),
+ mechanism: /** @type {const} */ ('gate_bypass'),
+ detail:
+ `${verdict.detail}; ${gateBypass} settle(s) refused by the fee governor at the facilitator ` +
+ 'from a caller that skipped the admission gate',
+ hint:
+ 'A caller is building, signing and verifying payments the wallet fee governor then refuses at ' +
+ 'settle (`fee_runway_exhausted` in x402_self_facilitator_log with no matching ring row). Each one ' +
+ 'costs a simulated verify and a 503 for nothing. Find it: group the same window of ' +
+ 'x402_self_facilitator_log by payer and amount_atomic (verify rows always carry the payer), then ' +
+ 'match the request logs by user agent. Fix the caller, not the budget: size the batch with ' +
+ 'reserveFeeAdmissions() or gate each call with assessFeeAdmission() / admitSponsorSettle() ' +
+ '(api/_lib/x402/wallet-fee-meter.js, api/_lib/x402/pay.js). Funding the fee wallet only moves ' +
+ 'the line such a caller runs into. See docs/x402-ring-economy.md "The wallet fee governor".',
+ };
+}
+
/**
* Classify pre-aggregated settle buckets into a subsystem verdict. Pure — no DB,
* no clock — so the thresholds and the rail-fault split are unit-testable
@@ -270,7 +305,8 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips
* @returns {{ status: 'ok'|'degraded'|'down'|'unknown', settled: number,
* faults: number, attempts: number, rate: number|null,
* faultClasses: Array<{ reason: string, n: number }>, detail: string,
- * mechanism?: 'accept_withdrawn'|'settle_refused'|'paced'|'rail', hint?: string }}
+ * mechanism?: 'accept_withdrawn'|'settle_refused'|'paced'|'gate_bypass'|'rail', hint?: string,
+ * gateBypass?: number }}
*/
export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, facilitatorRejects } = {}) {
let settled = 0;
@@ -333,8 +369,15 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac
}
return drained;
};
- governorSkips += drainStatusFaults(facilitatorRejects?.governor);
+ const facilitatorGovernor = Math.max(0, Math.floor(Number(facilitatorRejects?.governor) || 0));
+ const drainedGovernor = drainStatusFaults(facilitatorGovernor);
+ governorSkips += drainedGovernor;
floorSignals += drainStatusFaults(facilitatorRejects?.floor);
+ // Governor refusals the facilitator booked that NO ring row accounts for: a
+ // caller that signed, verified and submitted a settle the budget could not
+ // fund, without asking the caller-side admission gate first and without
+ // writing to x402_autonomous_log. See gateBypassVerdict().
+ const gateBypass = facilitatorGovernor - drainedGovernor;
const attempts = settled + faults;
const faultClasses = Object.entries(faultBy)
@@ -385,11 +428,12 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac
hint,
};
}
- return {
- status: 'unknown',
- settled, faults, attempts, rate: null, faultClasses, governorSkips,
+ const quiet = {
+ status: /** @type {const} */ ('unknown'),
+ settled, faults, attempts, rate: null, faultClasses, governorSkips, gateBypass,
detail: `only ${attempts} settle attempts in ${WINDOW_INTERVAL}, too few to judge`,
};
+ return gateBypassVerdict(quiet, { gateBypass, settled, minAttempts }) || quiet;
}
const rate = settled / attempts;
@@ -404,11 +448,12 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac
// A healthy rate with the governor pacing behind it is still healthy: the
// settles that ran, ran. Carry the skip count so a wallet quietly sliding
// toward its budget shows up before the rate does.
- return {
- status: 'ok', settled, faults, attempts, rate, faultClasses,
- noSolanaAccept, floorSignals, governorSkips,
+ const healthy = {
+ status: /** @type {const} */ ('ok'), settled, faults, attempts, rate, faultClasses,
+ noSolanaAccept, floorSignals, governorSkips, gateBypass,
detail: governorSkips > 0 ? `${base}; ${governorSkips} paced by the fee governor` : base,
};
+ return gateBypassVerdict(healthy, { gateBypass, settled, minAttempts }) || healthy;
}
const status = rate < DOWN_RATE ? 'down' : 'degraded';
const { cause, mechanism, hint } = diagnoseSettleDrop({
@@ -435,6 +480,7 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac
noSolanaAccept,
floorSignals,
governorSkips,
+ gateBypass,
detail: `${base}; ${faults} rail faults${topFaults ? ` (${topFaults})` : ''}${causeNote}`,
hint,
};
@@ -511,6 +557,9 @@ export async function gatherX402SettleHealth() {
// Deliberate spend-pacing, not failure. Read it next to `rate`: the two
// together separate "the rail broke" from "the rail is out of budget".
governorSkips: v.governorSkips ?? 0,
+ // Governor refusals at the facilitator with no ring row behind them:
+ // callers that skipped the admission gate (see gateBypassVerdict).
+ gateBypass: v.gateBypass ?? 0,
},
};
} catch (err) {
diff --git a/api/_lib/x402/pay.js b/api/_lib/x402/pay.js
index b6fedf40d..d32354208 100644
--- a/api/_lib/x402/pay.js
+++ b/api/_lib/x402/pay.js
@@ -154,6 +154,25 @@ export function expectedFeeLamports({ selfPay, priorityMicrolamports = 0, cuLimi
return SIGNATURE_FEE_LAMPORTS * signatures + priorityLamports;
}
+// Caller-side fee admission for a hand-built SPONSOR-mode payment, for the
+// pipelines that build their own transaction instead of calling payX402() (an
+// audit that must replay one exact proof, a builder-code echo, an MCP canary).
+// The ring payer and the autonomous loop already ask the governor before signing;
+// these did not, so each call past a spent budget paid for an ATA read, a
+// signature and a simulated verify and was then refused at settle. Priced at the
+// same worst case as payX402's own admission for this nonce, and fails open the
+// same way (see assessFeeAdmission).
+export async function admitSponsorSettle({ accept, connection = null, nonce = 0 }) {
+ const { microLamports, cuLimit } = ringFeeConfig(nonce, { selfPay: false });
+ return assessFeeAdmission({
+ feeWalletB58: accept?.extra?.feePayer || null,
+ estFeeLamports: expectedFeeLamports({
+ selfPay: false, priorityMicrolamports: microLamports, cuLimit,
+ }),
+ connection,
+ });
+}
+
// Load the autonomous payer keypair. Seed wallet preferred; agent wallet is the
// documented fallback. In non-prod a local test wallet file is honored so the
// loop and manual tests can run without env wiring.
diff --git a/api/_lib/x402/pipelines/builder-code-attribution.js b/api/_lib/x402/pipelines/builder-code-attribution.js
index 513bbffe1..679fb3d04 100644
--- a/api/_lib/x402/pipelines/builder-code-attribution.js
+++ b/api/_lib/x402/pipelines/builder-code-attribution.js
@@ -67,6 +67,7 @@ import {
parseSolanaAccept,
buildPaymentTx,
bootstrapSolanaContext,
+ admitSponsorSettle,
USDC_MINT,
} from '../pay.js';
@@ -269,6 +270,8 @@ async function settleAttributed({ row, buyer, conn, blockhash, mintInfo, remaini
const amountAtomic = Number(accept.amount || 0);
if (amountAtomic > (remainingCap ?? Infinity)) { out.error = 'cap_would_exceed'; return out; }
+ const admission = await admitSponsorSettle({ accept, connection: conn });
+ if (!admission.ok) { out.error = admission.reason || 'fee_runway_exhausted'; return out; }
// Echo the declared app code (anti-tamper: the server rejects a mismatch) and
// self-attribute the wallet + service so the settlement CBOR records who
diff --git a/api/_lib/x402/pipelines/payment-proof-idempotency-audit.js b/api/_lib/x402/pipelines/payment-proof-idempotency-audit.js
index f295a63fa..7c4154066 100644
--- a/api/_lib/x402/pipelines/payment-proof-idempotency-audit.js
+++ b/api/_lib/x402/pipelines/payment-proof-idempotency-audit.js
@@ -64,6 +64,7 @@ import {
parseSolanaAccept,
buildPaymentTx,
fetchWithTimeout,
+ admitSponsorSettle,
USDC_MINT,
SOLANA_RPC,
} from '../pay.js';
@@ -237,6 +238,8 @@ export async function runIdempotencyAudit(ctx = {}) {
const amountAtomic = Number(accept.amount || 0);
if (!(amountAtomic > 0)) return fail('zero_price');
if (amountAtomic > remainingCap) return fail('cap_would_exceed', { skipped: true });
+ const admission = await admitSponsorSettle({ accept, connection: conn });
+ if (!admission.ok) return fail(admission.reason || 'fee_runway_exhausted', { skipped: true });
// ── Step 2: build ONE signed payment proof carrying a payment-identifier ───
// The same base64 header is replayed verbatim in step 4 — that single proof
diff --git a/api/_lib/x402/pipelines/streaming-mcp-health.js b/api/_lib/x402/pipelines/streaming-mcp-health.js
index 91ae16ed6..9a339094d 100644
--- a/api/_lib/x402/pipelines/streaming-mcp-health.js
+++ b/api/_lib/x402/pipelines/streaming-mcp-health.js
@@ -60,6 +60,7 @@ import {
parseSolanaAccept,
buildPaymentTx,
fetchWithTimeout,
+ admitSponsorSettle,
USDC_MINT,
SOLANA_RPC,
} from '../pay.js';
@@ -295,6 +296,8 @@ export async function runStreamingMcpHealth(ctx = {}) {
const amountAtomic = Number(accept.amount || 0);
if (!(amountAtomic > 0)) return fail('zero_price');
if (amountAtomic > remainingCap) return fail('cap_would_exceed', { skipped: true });
+ const admission = await admitSponsorSettle({ accept, connection: conn });
+ if (!admission.ok) return fail(admission.reason || 'fee_runway_exhausted', { skipped: true });
// ── Step 2: build the signed payment tx + X-PAYMENT envelope ───────────────
let xPayment;
diff --git a/api/_lib/x402/self-facilitator.js b/api/_lib/x402/self-facilitator.js
index f0f68f601..a11e62248 100644
--- a/api/_lib/x402/self-facilitator.js
+++ b/api/_lib/x402/self-facilitator.js
@@ -834,6 +834,7 @@ export async function settleRingPayment({
reason: `fee_wallet_below_floor:${solLamports}<${floorFor}`,
sponsorSolLamports: solLamports,
feePayer: decoded.feePayer,
+ payer,
selfPay,
};
}
@@ -859,6 +860,7 @@ export async function settleRingPayment({
sponsorSolLamports: solLamports,
estFeeLamports,
feePayer: decoded.feePayer,
+ payer,
selfPay,
};
}
@@ -884,6 +886,10 @@ export async function settleRingPayment({
reason: meterVerdict.reason || 'fee_runway_exhausted',
sponsorSolLamports: solLamports,
feePayer: decoded.feePayer,
+ // Recorded so the facilitator's reject book names WHO was refused.
+ // Without it, 43,848 governor refusals in a day logged payer NULL and
+ // the flood could only be traced through the matching verify rows.
+ payer,
selfPay,
};
}
diff --git a/api/_lib/x402/wallet-fee-governor.js b/api/_lib/x402/wallet-fee-governor.js
index a28e129d7..17505556d 100644
--- a/api/_lib/x402/wallet-fee-governor.js
+++ b/api/_lib/x402/wallet-fee-governor.js
@@ -133,3 +133,33 @@ export function assessWalletFeeBudget({ spentTodayLamports, budgetLamports, next
}
return { ok: true, reason: null };
}
+
+// ── Batch sizing ────────────────────────────────────────────────────────────────
+// How many settles of `feeLamports` each still fit in today's budget, after
+// leaving `keepLamports` of headroom untouched for other tenants of the wallet.
+//
+// assessWalletFeeBudget() answers yes/no for ONE settle. A batch caller (the x402
+// seed cron fires up to 120 sponsor-mode payments in parallel) needs the count,
+// and asking one at a time is how it used to fire 60 when the budget funded one:
+// measured on production 2026-09-29, 3,600 of 3,660 seed payments in two hours
+// were refused at the facilitator with `fee_runway_exhausted` after each had
+// paid for a signed transfer and a simulated verify. Same arithmetic as the
+// single-settle verdict, so the two can never disagree about the boundary:
+// `n` settles are admitted exactly when spent + n*fee <= budget - keep.
+//
+// Unknown spend (ledger unreadable) returns `max`, the same fail-open contract as
+// assessWalletFeeBudget: the settle path's SOL floor stays the hard protection.
+export function admissibleSettleCount({
+ spentTodayLamports, budgetLamports, feeLamports, keepLamports = 0, max = Infinity,
+}) {
+ const cap = Math.max(0, Math.floor(Number(max)));
+ const ceiling = Number.isFinite(cap) ? cap : Infinity;
+ if (!Number.isFinite(spentTodayLamports)) return ceiling;
+ const fee = Math.max(0, Number(feeLamports) || 0);
+ const keep = Math.max(0, Number(keepLamports) || 0);
+ const headroom = Number(budgetLamports) - spentTodayLamports - keep;
+ if (!(headroom >= 0)) return 0;
+ // A zero fee cannot exhaust anything; the caller's own cap is the only limit.
+ if (fee === 0) return ceiling;
+ return Math.min(ceiling, Math.floor(headroom / fee));
+}
diff --git a/api/_lib/x402/wallet-fee-meter.js b/api/_lib/x402/wallet-fee-meter.js
index bed371247..78a4a015f 100644
--- a/api/_lib/x402/wallet-fee-meter.js
+++ b/api/_lib/x402/wallet-fee-meter.js
@@ -34,6 +34,7 @@ import {
walletFeeGovernorConfig,
walletDailyFeeBudgetLamports,
assessWalletFeeBudget,
+ admissibleSettleCount,
pacedFeeBudgetLamports,
utcDayElapsedFraction,
} from './wallet-fee-governor.js';
@@ -192,20 +193,16 @@ async function readAdmissionSnapshot({ feeWalletB58, connection, cfg, now, day }
return entry;
}
-export async function assessFeeAdmission({
- feeWalletB58, estFeeLamports = 0, connection = null, config, now = Date.now(),
-} = {}) {
- const cfg = config || walletFeeGovernorConfig();
- if (!cfg.enabled || !feeWalletB58) return ADMIT;
-
+// The snapshot every admission decision is made against: the cached one while it
+// is fresh and from today, otherwise one shared balance + ledger read. Resolves
+// `{ entry: null }` for every fail-open case (ungoverned wallet, unreadable
+// allowlist, unreadable balance), so callers admit without pricing anything.
+async function admissionSnapshot({ feeWalletB58, connection, cfg, now }) {
// Never serve a verdict across a UTC day boundary: the budget resets at
// midnight and a stale refusal would hold the rail shut into the new day.
const day = utcDay(now);
const hit = _admissionCache.get(feeWalletB58);
- if (hit && hit.day === day && now - hit.at < hit.ttl) {
- const verdict = snapshotVerdict(hit, estFeeLamports, now);
- return { ok: verdict.ok, reason: verdict.reason, cached: true };
- }
+ if (hit && hit.day === day && now - hit.at < hit.ttl) return { entry: hit, cached: true, hit: true };
// Only platform-controlled wallets are governed. An external buyer self-paying
// through our facilitator spends its own SOL; refusing it would refuse revenue.
@@ -213,9 +210,9 @@ export async function assessFeeAdmission({
try {
allowed = await governedWallets(now);
} catch {
- return ADMIT;
+ return { entry: null };
}
- if (!allowed || !allowed.has(feeWalletB58)) return ADMIT;
+ if (!allowed || !allowed.has(feeWalletB58)) return { entry: null };
let pending = _admissionInflight.get(feeWalletB58);
const leader = !pending;
@@ -224,20 +221,100 @@ export async function assessFeeAdmission({
.finally(() => _admissionInflight.delete(feeWalletB58));
_admissionInflight.set(feeWalletB58, pending);
}
- const entry = await pending;
- // Unreadable balance: fail open, exactly like the settle-path meter.
+ // A null entry is an unreadable balance: fail open, like the settle-path meter.
+ return { entry: await pending, cached: !leader, hit: false };
+}
+
+export async function assessFeeAdmission({
+ feeWalletB58, estFeeLamports = 0, connection = null, config, now = Date.now(),
+} = {}) {
+ const cfg = config || walletFeeGovernorConfig();
+ if (!cfg.enabled || !feeWalletB58) return ADMIT;
+
+ const { entry, cached, hit } = await admissionSnapshot({ feeWalletB58, connection, cfg, now });
if (!entry) return ADMIT;
const verdict = snapshotVerdict(entry, estFeeLamports, now);
+ if (hit) return { ok: verdict.ok, reason: verdict.reason, cached: true };
return {
ok: verdict.ok,
reason: verdict.reason,
budgetLamports: entry.budget,
spentTodayLamports: entry.spent,
- cached: !leader,
+ cached,
};
}
+// Batch twin of assessFeeAdmission(): how many of `count` settles, each costing
+// up to `estFeeLamports`, may this caller start right now?
+//
+// Why a batch caller cannot just loop over assessFeeAdmission(): it has no way to
+// say "only spend what is left over". The x402 seed cron exists to keep the
+// activity feed moving, not to buy anything, so it must YIELD the fee wallet's
+// budget to the pipelines that pay for real data. Measured on production
+// 2026-09-29: the seeder fired 60 sponsor-mode tips every two minutes, landed
+// about 720 settles a day, and in doing so spent the sponsor's paced budget out
+// from under the health, oracle, volume, sniper and feed pipelines, which the
+// caller-side gate then skipped about 7,400 times a day. `keepLamports` is the
+// headroom this caller leaves untouched for everyone else.
+//
+// Admitted fees are debited from the shared snapshot exactly like single
+// admissions, so other callers on this instance see the budget shrink at once.
+// When the budget cannot fund even one settle regardless of `keepLamports`, the
+// snapshot turns into the same cached refusal assessFeeAdmission() would record.
+// Fails OPEN on every unreadable input (returns `count`), same contract as the
+// single gate: the settle path's meter and SOL floor stay the real protection.
+export async function reserveFeeAdmissions({
+ feeWalletB58, estFeeLamports = 0, count = 1, keepLamports = 0,
+ connection = null, config, now = Date.now(),
+} = {}) {
+ const want = Math.max(0, Math.floor(Number(count) || 0));
+ const cfg = config || walletFeeGovernorConfig();
+ if (!cfg.enabled || !feeWalletB58 || want === 0) {
+ return { admitted: want, governed: false, reason: null };
+ }
+
+ const { entry } = await admissionSnapshot({ feeWalletB58, connection, cfg, now });
+ if (!entry) return { admitted: want, governed: false, reason: null };
+
+ const fee = Math.max(0, Number(estFeeLamports) || 0);
+ const spent = entry.spent + entry.admitted;
+ const base = {
+ governed: true,
+ budgetLamports: entry.budget,
+ spentTodayLamports: spent,
+ keepLamports: Math.max(0, Number(keepLamports) || 0),
+ };
+ if (!entry.ok) return { ...base, admitted: 0, reason: entry.reason };
+
+ const admitted = admissibleSettleCount({
+ spentTodayLamports: spent,
+ budgetLamports: entry.budget,
+ feeLamports: fee,
+ keepLamports,
+ max: want,
+ });
+ if (admitted > 0) {
+ entry.admitted += admitted * fee;
+ return { ...base, admitted, reason: null };
+ }
+ // Nothing fits. Distinguish "the budget is spent" (the same refusal every
+ // caller should now see, so it is cached as one) from "the budget has room,
+ // but only inside the headroom kept for other tenants" (not a refusal for
+ // anyone else, so the snapshot stays admitting).
+ const single = snapshotVerdict(entry, fee, now);
+ if (single.ok) {
+ // snapshotVerdict debited one fee for a settle this caller is not starting.
+ entry.admitted -= fee;
+ return {
+ ...base,
+ admitted: 0,
+ reason: `fee_budget_kept_for_paid_calls:${spent}+${fee}>${entry.budget}-${base.keepLamports}`,
+ };
+ }
+ return { ...base, admitted: 0, reason: single.reason };
+}
+
// Single source of truth for "what fee budget applies to this wallet right now".
// The caller-side admission gate and the settle-path meter MUST answer that
// question identically: a laxer admission re-creates the wasted handshakes the
diff --git a/api/cron/x402-seed-cron.js b/api/cron/x402-seed-cron.js
index 86de41c40..2ea94d0b7 100644
--- a/api/cron/x402-seed-cron.js
+++ b/api/cron/x402-seed-cron.js
@@ -1,14 +1,17 @@
// GET /api/cron/x402-seed-cron
//
-// Per-minute cron that seeds the x402 activity feed with 60 real Solana
-// micropayments per tick — one per second equivalent, fired in parallel.
+// Economy-tick cron that seeds the x402 activity feed with up to 60 real Solana
+// micropayments per tick, fired in parallel. The sponsor pays each payment's
+// SOL fee, so the batch is capped at what the wallet fee governor will admit
+// after leaving X402_SEED_FEE_RESERVE_LAMPORTS for the paid pipelines.
//
// Each invocation:
// 1. Loads the seeder keypair (X402_SEED_SOLANA_SECRET_BASE58 or fallback).
// 2. Probes /api/x402/dance-tip once to get live payment requirements.
-// 3. Fetches a single blockhash + mint info shared across all 60 transactions.
-// 4. Builds 60 signed USDC TransferChecked transactions (synchronous).
-// 5. Fires all 60 in parallel against dance-tip with X-PAYMENT headers.
+// 3. Fetches a single blockhash + mint info shared across the batch.
+// 4. Sizes the batch to the payer's balance and the sponsor's fee budget.
+// 5. Builds that many signed TransferChecked transactions (synchronous).
+// 6. Fires them in parallel against dance-tip with X-PAYMENT headers.
//
// The payments are real on-chain USDC transfers from the seeder wallet.
// The Solana x402 facilitator co-signs (feePayer) and broadcasts each tx.
@@ -18,6 +21,8 @@
// X402_SEED_SOLANA_SECRET_BASE58 base58 64-byte ed25519 seeder keypair
// X402_SEED_ENABLED set to 'false' to pause (default: enabled)
// X402_SEED_BATCH_SIZE calls per tick (default: 60)
+// X402_SEED_FEE_RESERVE_LAMPORTS fee-wallet budget the seeder leaves for
+// paid pipelines (default: 200000)
// The transactions come from the ONE shared ring payment builder
// (api/_lib/x402/pay.js), whose nonce-based (priority price, CU limit) spread
@@ -38,7 +43,9 @@ import { logger } from '../_lib/usage.js';
import { SPONSOR_SOL_FLOOR_LAMPORTS } from '../_lib/x402/self-facilitator.js';
import {
loadSeedKeypair, fetchWithTimeout, parseSolanaAccept, buildPaymentTx,
+ ringFeeConfig, expectedFeeLamports,
} from '../_lib/x402/pay.js';
+import { reserveFeeAdmissions } from '../_lib/x402/wallet-fee-meter.js';
import { readPayerUsdcAtomic } from './x402-autonomous-loop.js';
import { requireCron } from '../_lib/cron-auth.js';
@@ -104,6 +111,30 @@ export function planSeedBatch({ sponsorSolLamports, payerUsdcAtomic, priceAtomic
return { skip: null, batch: Math.min(batchSize, affordable) };
}
+// The seeder's fee headroom: how much of the sponsor's daily fee budget it must
+// leave untouched for the pipelines that pay for real data. Default 200,000
+// lamports, twenty sponsor-mode settles, the same size as the governor's
+// top-of-day pacing slice. 0 lets the seeder compete for every lamport.
+export function seedFeeReserveLamports(e = process.env) {
+ const raw = String(e.X402_SEED_FEE_RESERVE_LAMPORTS ?? '').trim();
+ const n = Number(raw);
+ return raw && Number.isFinite(n) && n >= 0 ? Math.floor(n) : 200_000;
+}
+
+// Worst-case SOL fee of one seed payment. The batch uses nonces 0..batch-1 in
+// sponsor mode, so the ceiling over that range is what every admission is priced
+// at: the same expectedFeeLamports() math payX402 prices its own admissions with.
+export function seedFeeEstimateLamports(batch) {
+ let worst = 0;
+ for (let i = 0; i < Math.max(1, batch); i += 1) {
+ const { microLamports, cuLimit } = ringFeeConfig(i, { selfPay: false });
+ worst = Math.max(worst, expectedFeeLamports({
+ selfPay: false, priorityMicrolamports: microLamports, cuLimit,
+ }));
+ }
+ return worst;
+}
+
// Push an entry to the shared x402 activity feed in Redis.
async function pushFeedEntry(entry) {
const r = getRedis();
@@ -242,10 +273,41 @@ export default wrapCron(async (req, res) => {
});
}
+ // ── Pre-flight: fit the batch inside the sponsor's fee budget ─────────────
+ // The balance gate above only proves the sponsor is over its hard floor. The
+ // wallet fee governor refuses anything past today's paced budget, and this
+ // cron built, signed and verified 60 payments a tick regardless: measured on
+ // production 2026-09-29, 3,600 of 3,660 seed payments in two hours were
+ // refused at the facilitator with `fee_runway_exhausted`, about 95% of every
+ // settle attempt the platform made, each after a simulated verify. The seeder
+ // sends only what the governor will admit, and leaves the reserve for the
+ // pipelines that buy real data from the same wallet.
+ const feeEstimate = seedFeeEstimateLamports(plan.batch);
+ const admission = await reserveFeeAdmissions({
+ feeWalletB58: accept.extra.feePayer,
+ estFeeLamports: feeEstimate,
+ count: plan.batch,
+ keepLamports: seedFeeReserveLamports(),
+ connection: conn,
+ });
+ if (admission.admitted < 1) {
+ return json(res, 200, {
+ ok: false,
+ skipped: true,
+ reason: 'fee_budget',
+ detail: admission.reason,
+ feePayer: accept.extra.feePayer,
+ budget_lamports: admission.budgetLamports ?? null,
+ spent_today_lamports: admission.spentTodayLamports ?? null,
+ reserve_lamports: admission.keepLamports ?? null,
+ fee_estimate_lamports: feeEstimate,
+ });
+ }
+
// ── Step 3: build the signed transactions (synchronous) ───────────────────
// The nonce spread is what makes each batch member a distinct transaction;
// see ringFeeConfig in api/_lib/x402/pay.js for why it costs ~nothing.
- const affordable = plan.batch;
+ const affordable = Math.min(plan.batch, admission.admitted);
const txBases = Array.from({ length: affordable }, (_, index) =>
buildPaymentTx({ accept, buyer, blockhash, mintInfo, receiverAtaExists, nonce: index }),
);
@@ -304,6 +366,7 @@ export default wrapCron(async (req, res) => {
log.info('x402_seed_tick', {
batch: affordable,
configured_batch: BATCH_SIZE,
+ fee_admitted: admission.admitted,
succeeded: succeeded.length,
failed,
payer: payerAddress,
diff --git a/docs/ops/production-log-triage.md b/docs/ops/production-log-triage.md
index 80b9961b6..f408f18c0 100644
--- a/docs/ops/production-log-triage.md
+++ b/docs/ops/production-log-triage.md
@@ -320,8 +320,16 @@ HTTP 502/503 GET|POST /api/x402/*, /api/mcp ua: threews-x402-autonomous/1.0 or
| `accept_withdrawn` | `sponsorKnownBelowFloor()` dropped the Solana accept, so no payment is ever attempted | the 402 challenge (`/api/x402/three-intel`), NOT the facilitator: it never saw these calls |
| `settle_refused` | the accept is on every challenge, buyers sign real payments, our facilitator turns each away at the floor gate | the facilitator's reject book: `fee_wallet_below_floor:<held><<floor>` names the wallet and both numbers |
| `paced` | the wallet fee governor spent today's budget on purpose | `GET /api/x402/runway-lab` |
+ | `gate_bypass` | a caller outside the ring signs and verifies payments the governor then refuses at settle, without asking the admission gate first (`metrics.gateBypass` counts them) | `x402_self_facilitator_log`: group the window's `verify` rows by `payer` and `amount_atomic`, then match the request logs by user agent |
| `rail` | genuine payment-rail faults | the settle logs, per the greps above |
+ `gate_bypass` was production on 2026-09-29, invisible under `x402_settle: ok, 97%`:
+ 43,848 of 46,375 facilitator settle attempts in 24h were `fee_runway_exhausted`,
+ and 95% of them were the x402 seed cron (`threews-x402-seed/1.0`, 60 sponsor-mode
+ $0.001 tips every two minutes against a budget that funded about one). The fix is
+ in the caller, not the budget: funding the fee wallet only moves the line such a
+ caller runs into.
+
On 2026-09-09 production was `settle_refused` verbatim: **0** `no_solana_accept`,
164 `fee_wallet_below_floor:1167627<2000000`, and 126 `http_402` recorded by the
ring on the paid replay (a 402 there means the payment we built was rejected,
diff --git a/docs/x402-ring-economy.md b/docs/x402-ring-economy.md
index e6f46a09e..7c3cfc501 100644
--- a/docs/x402-ring-economy.md
+++ b/docs/x402-ring-economy.md
@@ -130,6 +130,23 @@ decision logic in
every call in the window, and under intraday pacing (where the unlocked budget
sits at the spent line all day) that let about 1,700 doomed handshakes an hour
reach the settle path to be refused there, beside about 230 that settled.
+- **Every caller that builds its own payment asks too.** Code that signs a
+ transaction itself instead of calling `payX402()` must still ask the governor
+ first. One sponsor-mode payment: `admitSponsorSettle({ accept, connection })`
+ from `api/_lib/x402/pay.js` (the streaming MCP canary, the builder-code and the
+ payment-proof idempotency pipelines use it). A batch:
+ `reserveFeeAdmissions({ feeWalletB58, estFeeLamports, count, keepLamports })`
+ from `api/_lib/x402/wallet-fee-meter.js`, which returns how many settles fit
+ after leaving `keepLamports` of the budget for everyone else. The x402 seed
+ cron (`api/cron/x402-seed-cron.js`) sizes its batch with it and leaves
+ `X402_SEED_FEE_RESERVE_LAMPORTS` (default 200,000, about twenty sponsor-mode
+ settles) for the pipelines that buy real data, so seeding the activity feed
+ never starves them. Before 2026-09-29 the seeder fired 60 tips every two minutes
+ whatever the budget: 43,662 of the day's 43,848 facilitator governor refusals
+ were its $0.001 tips, while the health, oracle, volume and feed pipelines were
+ skipped about 7,400 times a day on the same wallet. `/api/healthz` now reports
+ that shape as `x402_settle` `degraded` with `metrics.mechanism: gate_bypass`
+ (governor refusals at the facilitator with no ring row behind them).
- **Platform wallets only.** The meter governs only wallets in
`ringAllowedAddresses()`. An external organic buyer self-paying through this
facilitator spends its own SOL and is always admitted.
diff --git a/tests/api/x402-settle-health.test.js b/tests/api/x402-settle-health.test.js
index c2651b916..e3315ff6f 100644
--- a/tests/api/x402-settle-health.test.js
+++ b/tests/api/x402-settle-health.test.js
@@ -601,3 +601,59 @@ describe('readSettleHealth surfaces the mechanism the runbook tells operators to
expect(metricsBlock).toMatch(/cause: v\.cause/);
});
});
+
+describe('classifySettleBuckets: a caller that skips the admission gate is not a healthy rail', () => {
+ // Production 2026-09-29, scaled to 3h: the ring's own rows read 97% while the
+ // facilitator refused about 5,500 governor-paced settles from the x402 seed
+ // cron, which never asks the admission gate and never writes a ring row.
+ const ringRows = [
+ { success: true, paid: true, reason: 'none', n: 262 },
+ { success: false, paid: true, reason: 'This operation was aborted', n: 7 },
+ { success: false, paid: false, reason: 'fee_runway_exhausted', n: 924 },
+ ];
+
+ it('degrades with mechanism gate_bypass when unattributed refusals outnumber settles', () => {
+ const v = classifySettleBuckets(ringRows, { facilitatorRejects: { governor: 5_480 } });
+ expect(v.rate).toBeGreaterThan(0.9);
+ expect(v.gateBypass).toBe(5_480);
+ expect(v.status).toBe('degraded');
+ expect(v.cause).toBe('fee_governor');
+ expect(v.mechanism).toBe('gate_bypass');
+ expect(v.detail).toMatch(/5480 settle\(s\) refused by the fee governor at the facilitator/);
+ expect(v.hint).toMatch(/reserveFeeAdmissions/);
+ });
+
+ it('counts only refusals no ring row accounts for', () => {
+ const rows = [...ringRows, { success: false, paid: true, reason: 'http_503', n: 300 }];
+ const v = classifySettleBuckets(rows, { facilitatorRejects: { governor: 320 } });
+ // 300 of the 320 are the ring's own http_503 replays; 20 remain unattributed,
+ // fewer than the 262 the ring settled, so a healthy rail stays ok.
+ expect(v.gateBypass).toBe(20);
+ expect(v.status).toBe('ok');
+ expect(v.mechanism).toBeUndefined();
+ });
+
+ it('the cross-instance race residue never trips it', () => {
+ // About 8 an hour of $0.01 governor refusals survived the caller-side gate
+ // in the same 24h, the residue of per-instance admission snapshots.
+ const v = classifySettleBuckets(ringRows, { facilitatorRejects: { governor: 24 } });
+ expect(v.status).toBe('ok');
+ });
+
+ it('an idle ring with a bypassing caller is degraded, not unknown', () => {
+ const v = classifySettleBuckets([], { facilitatorRejects: { governor: 1_800 } });
+ expect(v.status).toBe('degraded');
+ expect(v.mechanism).toBe('gate_bypass');
+ });
+
+ it('a rail that is already failing keeps its own diagnosis and carries the count', () => {
+ const failing = [
+ { success: true, paid: true, reason: 'none', n: 100 },
+ { success: false, paid: true, reason: 'broadcast_failed', n: 200 },
+ ];
+ const v = classifySettleBuckets(failing, { facilitatorRejects: { governor: 5_000 } });
+ expect(v.status).toBe('down');
+ expect(v.mechanism).toBe('rail');
+ expect(v.gateBypass).toBe(5_000);
+ });
+});
diff --git a/tests/x402-fee-admission.test.js b/tests/x402-fee-admission.test.js
index 7006c3f4d..5bf0b2917 100644
--- a/tests/x402-fee-admission.test.js
+++ b/tests/x402-fee-admission.test.js
@@ -46,6 +46,7 @@ vi.mock('../api/_lib/x402/self-facilitator.js', () => ({
const {
assessFeeAdmission,
+ reserveFeeAdmissions,
facilitatorFeeMeter,
resetWalletFeeMeterCaches,
} = await import('../api/_lib/x402/wallet-fee-meter.js');
@@ -304,3 +305,87 @@ describe('parity with the settle-path meter', () => {
});
}
});
+
+// The batch gate the x402 seed cron sizes its tick with. Production 2026-09-29:
+// 60 sponsor-mode tips every two minutes against a budget that funded about one,
+// 43,662 governor refusals a day at the facilitator, and the paid pipelines on
+// the same wallet skipped about 7,400 times a day because the seeder got there
+// first. reserveFeeAdmissions admits only what fits after the kept headroom.
+describe('reserveFeeAdmissions: batch sizing against the shared snapshot', () => {
+ const reserve = (overrides = {}) => reserveFeeAdmissions({
+ feeWalletB58: GOV, estFeeLamports: 10_000, count: 60, config: CFG, ...overrides,
+ });
+
+ it('admits only the settles the budget can fund', async () => {
+ h.sql.mockResolvedValue([{ spent: '970000' }]);
+ const r = await reserve();
+ expect(r).toMatchObject({ admitted: 3, governed: true, reason: null, budgetLamports: 1_000_000 });
+ });
+
+ it('leaves the kept headroom, without refusing anyone else', async () => {
+ h.sql.mockResolvedValue([{ spent: '900000' }]);
+ const r = await reserve({ keepLamports: 200_000 });
+ expect(r.admitted).toBe(0);
+ expect(r.reason).toMatch(/^fee_budget_kept_for_paid_calls:/);
+ // The budget still has 100,000 lamports; a paid call must still get in.
+ const paid = await admit({ estFeeLamports: 10_000 });
+ expect(paid.ok).toBe(true);
+ });
+
+ it('debits admitted fees from the snapshot other callers read', async () => {
+ h.sql.mockResolvedValue([{ spent: '950000' }]);
+ expect((await reserve()).admitted).toBe(5);
+ // The seeder took every settle the budget funds; the next caller is refused
+ // from the same snapshot without another ledger read.
+ const next = await admit({ estFeeLamports: 10_000 });
+ expect(next.ok).toBe(false);
+ expect(next.reason).toMatch(/^fee_runway_exhausted:/);
+ expect(h.sql).toHaveBeenCalledTimes(1);
+ });
+
+ it('an exhausted budget becomes the same cached refusal every caller sees', async () => {
+ h.sql.mockResolvedValue([{ spent: '1000000' }]);
+ const r = await reserve({ keepLamports: 200_000 });
+ expect(r.admitted).toBe(0);
+ expect(r.reason).toMatch(/^fee_runway_exhausted:/);
+ const next = await admit({ estFeeLamports: 5_000 });
+ expect(next).toMatchObject({ ok: false, cached: true });
+ });
+
+ it('fails open: ungoverned wallet, unreadable balance, disabled governor', async () => {
+ expect((await reserve({ feeWalletB58: ORGANIC })).admitted).toBe(60);
+ expect((await reserve({ config: { ...CFG, enabled: false } })).admitted).toBe(60);
+ resetWalletFeeMeterCaches();
+ h.sponsorSolLamports.mockRejectedValue(new Error('rpc down'));
+ expect((await reserve()).admitted).toBe(60);
+ });
+});
+
+// Pipelines that hand-build a sponsor-mode payment (the MCP canary, the
+// builder-code and payment-proof audits) ask through admitSponsorSettle(). It
+// must price at the sponsor-mode worst case, two signatures, because that is
+// what the settle-path meter will charge against the same budget.
+describe('admitSponsorSettle: hand-built sponsor payments ask the same gate', () => {
+ it('prices at the two-signature sponsor fee and gates the accept\'s fee payer', async () => {
+ const { admitSponsorSettle } = await import('../api/_lib/x402/pay.js');
+ // 995,000 spent of a 1,000,000 budget: a self-pay fee (5,000) would fit,
+ // the sponsor-mode fee (10,000) does not.
+ h.sql.mockResolvedValue([{ spent: '995000' }]);
+ vi.stubEnv('X402_WALLET_FEE_RUNWAY_DAYS', '1');
+ vi.stubEnv('X402_WALLET_FEE_MIN_BUDGET_LAMPORTS', '0');
+ vi.stubEnv('X402_WALLET_FEE_PACE_DAY', 'false');
+ try {
+ const v = await admitSponsorSettle({ accept: { extra: { feePayer: GOV } } });
+ expect(v.ok).toBe(false);
+ expect(v.reason).toBe('fee_runway_exhausted:995000+10000>1000000');
+ } finally {
+ vi.unstubAllEnvs();
+ }
+ });
+
+ it('admits when the accept names no fee payer to price', async () => {
+ const { admitSponsorSettle } = await import('../api/_lib/x402/pay.js');
+ expect((await admitSponsorSettle({ accept: { extra: {} } })).ok).toBe(true);
+ expect(h.sponsorSolLamports).not.toHaveBeenCalled();
+ });
+});
diff --git a/tests/x402-seed-batch-plan.test.js b/tests/x402-seed-batch-plan.test.js
index 23e6b69f0..a860e2f17 100644
--- a/tests/x402-seed-batch-plan.test.js
+++ b/tests/x402-seed-batch-plan.test.js
@@ -20,12 +20,14 @@
import { describe, it, expect, beforeAll } from 'vitest';
let planSeedBatch;
+let seedFeeReserveLamports;
+let seedFeeEstimateLamports;
let FLOOR;
beforeAll(async () => {
process.env.X402_ASSET_MINT_SOLANA = 'EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v';
process.env.CRON_SECRET = 'test-secret';
- ({ planSeedBatch } = await import('../api/cron/x402-seed-cron.js'));
+ ({ planSeedBatch, seedFeeReserveLamports, seedFeeEstimateLamports } = await import('../api/cron/x402-seed-cron.js'));
({ SPONSOR_SOL_FLOOR_LAMPORTS: FLOOR } = await import('../api/_lib/x402/self-facilitator.js'));
});
@@ -101,3 +103,30 @@ describe('planSeedBatch — the seeder preflight', () => {
}
});
});
+
+// The fee-budget half of the preflight. The balance gate above proves the sponsor
+// is over its hard floor; it says nothing about the wallet fee governor's daily
+// budget, and on 2026-09-29 that gap was 3,600 of 3,660 seed payments in two
+// hours refused at the facilitator. The cron now sizes the batch with
+// reserveFeeAdmissions(), priced and reserved by these two helpers.
+describe('seed fee pricing and reserve', () => {
+ it('prices every batch member at the sponsor-mode worst case the facilitator meters', () => {
+ // Two signatures, and sponsor-mode price slots that floor to zero priority
+ // lamports: exactly the `+10000` in the production refusal string.
+ expect(seedFeeEstimateLamports(60)).toBe(10_000);
+ expect(seedFeeEstimateLamports(120)).toBe(10_000);
+ expect(seedFeeEstimateLamports(0)).toBe(10_000);
+ });
+
+ it('leaves twenty sponsor settles of budget for paid pipelines by default', () => {
+ expect(seedFeeReserveLamports({})).toBe(200_000);
+ expect(seedFeeReserveLamports({ X402_SEED_FEE_RESERVE_LAMPORTS: '0' })).toBe(0);
+ expect(seedFeeReserveLamports({ X402_SEED_FEE_RESERVE_LAMPORTS: '50000' })).toBe(50_000);
+ });
+
+ it('ignores an unusable or blank reserve instead of zeroing it', () => {
+ for (const v of ['-1', 'abc', '', ' ']) {
+ expect(seedFeeReserveLamports({ X402_SEED_FEE_RESERVE_LAMPORTS: v })).toBe(200_000);
+ }
+ });
+});
diff --git a/tests/x402-wallet-fee-governor.test.js b/tests/x402-wallet-fee-governor.test.js
index 8b2bb5e82..f7ec2013c 100644
--- a/tests/x402-wallet-fee-governor.test.js
+++ b/tests/x402-wallet-fee-governor.test.js
@@ -12,6 +12,7 @@ import {
walletFeeGovernorConfig,
walletDailyFeeBudgetLamports,
assessWalletFeeBudget,
+ admissibleSettleCount,
pacedFeeBudgetLamports,
utcDayElapsedFraction,
} from '../api/_lib/x402/wallet-fee-governor.js';
@@ -223,3 +224,60 @@ describe('recurrence guard: default config sustains a full day at measured burn'
expect(prod).toBeGreaterThanOrEqual(base * 3);
});
});
+
+// A batch caller (the x402 seed cron) needs HOW MANY settles fit, not a yes/no
+// for one. Production on 2026-09-29 is the reason: the seeder fired 60 per tick
+// against a paced budget that funded about one, and 3,600 of 3,660 were refused
+// at the facilitator. The count must agree with assessWalletFeeBudget exactly at
+// the boundary, or the batch and the settle path disagree about the last settle.
+describe('admissibleSettleCount', () => {
+ it('counts the settles that fit, boundary inclusive', () => {
+ expect(admissibleSettleCount({
+ spentTodayLamports: 2_000_000, budgetLamports: 2_050_000, feeLamports: 10_000,
+ })).toBe(5);
+ expect(admissibleSettleCount({
+ spentTodayLamports: 2_000_000, budgetLamports: 2_049_999, feeLamports: 10_000,
+ })).toBe(4);
+ });
+
+ it('agrees with the single-settle verdict at every step up to the boundary', () => {
+ const budget = 1_000_000;
+ const fee = 10_000;
+ for (const spent of [0, 500_000, 989_999, 990_000, 990_001, 1_000_000, 1_200_000]) {
+ const n = admissibleSettleCount({ spentTodayLamports: spent, budgetLamports: budget, feeLamports: fee });
+ // n settles fit, and the (n+1)th is exactly the one the settle path refuses.
+ if (n > 0) {
+ expect(assessWalletFeeBudget({
+ spentTodayLamports: spent + (n - 1) * fee, budgetLamports: budget, nextFeeLamports: fee,
+ }).ok).toBe(true);
+ }
+ expect(assessWalletFeeBudget({
+ spentTodayLamports: spent + n * fee, budgetLamports: budget, nextFeeLamports: fee,
+ }).ok).toBe(false);
+ }
+ });
+
+ it('leaves the kept headroom for other tenants', () => {
+ expect(admissibleSettleCount({
+ spentTodayLamports: 0, budgetLamports: 300_000, feeLamports: 10_000, keepLamports: 200_000,
+ })).toBe(10);
+ expect(admissibleSettleCount({
+ spentTodayLamports: 150_000, budgetLamports: 300_000, feeLamports: 10_000, keepLamports: 200_000,
+ })).toBe(0);
+ });
+
+ it('never exceeds the caller cap, and returns 0 once spend passes the budget', () => {
+ expect(admissibleSettleCount({
+ spentTodayLamports: 0, budgetLamports: 10_000_000, feeLamports: 10_000, max: 60,
+ })).toBe(60);
+ expect(admissibleSettleCount({
+ spentTodayLamports: 2_780_275, budgetLamports: 2_661_971, feeLamports: 10_000, max: 60,
+ })).toBe(0);
+ });
+
+ it('fails OPEN on unknown spend, like the single verdict', () => {
+ expect(admissibleSettleCount({
+ spentTodayLamports: NaN, budgetLamports: 0, feeLamports: 10_000, max: 60,
+ })).toBe(60);
+ });
+});