Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(x402): stop the settle sensor calling a floor refusal a withdrawn accept
nirholas30e83da5c28f
diff --git a/api/_lib/ops/x402-settle-health.js b/api/_lib/ops/x402-settle-health.jsindex 169fa7e2f..44489810e 100644--- a/api/_lib/ops/x402-settle-health.js+++ b/api/_lib/ops/x402-settle-health.js@@ -132,6 +132,17 @@ const SPONSOR_FLOOR = /^(settlement temporarily unavailable|fee_wallet_below_flo* read as "settle 22%" while faults sat at their normal* ~100/h and `no_solana_accept` went 0 to 374/h.*+ * FLOOR REFUSED the accept is still advertised and buyers still sign real+ * payments, but OUR OWN facilitator refuses every one of them+ * at the fee-wallet floor gate. Same wallet, same remedy as+ * WITHDRAWN, opposite evidence trail: the proof is in the+ * facilitator's reject book, not in the challenge. Measured+ * 2026-09-09: 0 no_solana_accept, 164+ * `fee_wallet_below_floor:1167627<2000000`, and 126 `http_402`+ * from paid replays the gate turned away. Telling that+ * operator "do NOT start at the facilitator" points them away+ * from the only table that names the wallet and both numbers.+ ** BUDGET PACED nothing failed and nothing was withdrawn: the wallet fee* governor spent today's fee budget and is refusing the rest* of the day's settles on purpose. Look at the fee wallet's@@ -143,7 +154,8 @@ const SPONSOR_FLOOR = /^(settlement temporarily unavailable|fee_wallet_below_flo* not the benign "ring chose not to pay" the rate deliberately excludes.* @param {{ noSolanaAccept: number, floorSignals: number, governorSkips?: number,* settled: number, faults: number }} s- * @returns {{ cause: 'sponsor_floor'|'fee_governor'|'rail', hint: string }}+ * @returns {{ cause: 'sponsor_floor'|'fee_governor'|'rail',+ * mechanism: 'accept_withdrawn'|'settle_refused'|'paced'|'rail', hint: string }}*/export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips = 0, settled, faults }) {// A floor refusal is proof on its own. Absent that, treat the home chain going@@ -155,14 +167,37 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips// wallet, but under the floor EVERY settle fails closed, while a spent budget// still settles at the paced rate. Report the harder stop when both are lit.if (floorSignals > 0 || (noSolanaAccept > 0 && noSolanaAccept > settled)) {- return {- cause: /** @type {const} */ ('sponsor_floor'),- hint:- 'The Solana accept is being WITHDRAWN, not rejected: ' ++ // Same wallet, same remedy, two opposite evidence trails. Reporting the+ // withdrawal unconditionally was wrong exactly when `noSolanaAccept` is 0:+ // the accept was still on every challenge, buyers signed real payments, and+ // our own facilitator turned each one away at the floor gate. The remedy+ // paragraph below is shared because the fix is identical; only the sentence+ // naming the mechanism and the place to look differs.+ const withdrawn = noSolanaAccept > 0;+ const mechanism = /** @type {const} */ (withdrawn ? 'accept_withdrawn' : 'settle_refused');+ const opening = withdrawn+ ? 'The Solana accept is being WITHDRAWN, not rejected: ' +`${noSolanaAccept} no_solana_accept + ${floorSignals} floor refusal(s) against ${faults} rail faults. ` +'sponsorKnownBelowFloor() drops Solana from every 402 challenge while the sponsor sits under ' +'X402_SPONSOR_SOL_FLOOR_LAMPORTS, so the ring has nothing payable and settlements stop. ' +- 'Do NOT start at the facilitator. Check the sponsor balance, then let the free self-heal run: ' ++ 'Do NOT start at the facilitator: with no accept on the challenge it never saw these calls. ' ++ 'Check the sponsor balance, then let the free self-heal run: '+ : 'Settles are being REFUSED at the sponsor floor, not withdrawn: ' ++ `0 no_solana_accept + ${floorSignals} floor refusal(s) against ${faults} rail faults. ` ++ 'The Solana accept is still on every challenge and buyers are signing real payments; our own ' ++ 'facilitator rejects each one at the floor gate, because the sponsor fee wallet sits under ' ++ 'X402_SPONSOR_SOL_FLOOR_LAMPORTS. That is why the ring records them as `http_402` on the paid ' ++ 'replay. START at the facilitator here: its reject book is the only ' ++ 'place that names the wallet and both numbers ' ++ '(`fee_wallet_below_floor:<held><<floor>` in x402_self_facilitator_log.reject_reason, also ' ++ 'under x402.self_facilitator.settle.fail_reasons on /api/healthz). Read the held number off ' ++ 'that token before anything else: it is the wallet the money has to reach. Then let the free ' ++ 'self-heal run: ';+ return {+ cause: /** @type {const} */ ('sponsor_floor'),+ mechanism,+ hint:+ opening +'POST /api/cron/treasury-topup?dry=1 (Bearer CRON_SECRET) to see the plan, then without ?dry=1 ' +'to apply. READ agent_reclaim.failed in that plan before you trust its total: a wallet whose ' +'secret does not decrypt is reported at stage `recover` and its SOL is unreachable until ' +@@ -182,6 +217,7 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkipsif (governorSkips > faults && governorSkips > 0) {return {cause: /** @type {const} */ ('fee_governor'),+ mechanism: /** @type {const} */ ('paced'),hint:'This is a GOVERNED THROTTLE, not a rail fault: ' +`${governorSkips} settle(s) skipped by the wallet fee governor against ${faults} rail fault(s). ` +@@ -197,6 +233,7 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips}return {cause: /** @type {const} */ ('rail'),+ mechanism: /** @type {const} */ ('rail'),hint:'Payments are being rejected at settle. Check the self-facilitator: ' +'`npm run logs -- -s three-ws-api --grep "settle_failed" --since 3h`. ' +@@ -230,7 +267,8 @@ export function diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips* facilitatorRejects?: { governor?: number, floor?: number } }} [opts]* @returns {{ status: 'ok'|'degraded'|'down'|'unknown', settled: number,* faults: number, attempts: number, rate: number|null,- * faultClasses: Array<{ reason: string, n: number }>, detail: string, hint?: string }}+ * faultClasses: Array<{ reason: string, n: number }>, detail: string,+ * mechanism?: 'accept_withdrawn'|'settle_refused'|'paced'|'rail', hint?: string }}*/export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, facilitatorRejects } = {}) {let settled = 0;@@ -312,13 +350,13 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac// nothing settles at all, where a spent governor budget still settles at the// paced rate.if (floorSignals >= minAttempts) {- const { cause, hint } = diagnoseSettleDrop({+ const { cause, mechanism, hint } = diagnoseSettleDrop({noSolanaAccept, floorSignals, governorSkips, settled, faults,});return {status: 'down',settled, faults, attempts, rate: null, faultClasses,- cause, noSolanaAccept, floorSignals, governorSkips,+ cause, mechanism, noSolanaAccept, floorSignals, governorSkips,detail:`settle halted: ${floorSignals} attempt(s) refused with the sponsor under its SOL floor ` +`in ${WINDOW_INTERVAL}, ${settled} settled`,@@ -332,13 +370,13 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac// verdict. The threshold is the same MIN_ATTEMPTS, so a handful of skips on// a genuinely idle ring still reads `unknown`.if (governorSkips >= minAttempts) {- const { cause, hint } = diagnoseSettleDrop({+ const { cause, mechanism, hint } = diagnoseSettleDrop({noSolanaAccept, floorSignals, governorSkips, settled, faults,});return {status: 'degraded',settled, faults, attempts, rate: null, faultClasses,- cause, noSolanaAccept, floorSignals, governorSkips,+ cause, mechanism, noSolanaAccept, floorSignals, governorSkips,detail:`settle throttled: ${governorSkips} call(s) skipped by the wallet fee governor, ` +`${attempts} attempt(s) in ${WINDOW_INTERVAL}`,@@ -371,23 +409,31 @@ export function classifySettleBuckets(buckets, { minAttempts = MIN_ATTEMPTS, fac};}const status = rate < DOWN_RATE ? 'down' : 'degraded';- const { cause, hint } = diagnoseSettleDrop({ noSolanaAccept, floorSignals, governorSkips, settled, faults });- // Name the withdrawal in `detail` too: the hint is one field deep in the JSON,+ const { cause, mechanism, hint } = diagnoseSettleDrop({+ noSolanaAccept, floorSignals, governorSkips, settled, faults,+ });+ // Name the mechanism in `detail` too: the hint is one field deep in the JSON,// but `detail` is what the dashboard row, the digest and /api/status all print.- const withdrawn =- cause === 'sponsor_floor'+ // It has to agree with the counter printed beside it. "Solana accept withdrawn+ // (0 no_solana_accept)" was one clause contradicting itself, on the row an+ // operator reads first.+ const causeNote =+ mechanism === 'accept_withdrawn'? `; Solana accept withdrawn (${noSolanaAccept} no_solana_accept, sponsor under SOL floor)`- : cause === 'fee_governor'- ? `; ${governorSkips} call(s) paced by the fee governor (budget spent, not a rail fault)`- : '';+ : mechanism === 'settle_refused'+ ? `; ${floorSignals} settle(s) refused at the fee-wallet floor (accept still advertised)`+ : mechanism === 'paced'+ ? `; ${governorSkips} call(s) paced by the fee governor (budget spent, not a rail fault)`+ : '';return {status,settled, faults, attempts, rate, faultClasses,cause,+ mechanism,noSolanaAccept,floorSignals,governorSkips,- detail: `${base}; ${faults} rail faults${topFaults ? ` (${topFaults})` : ''}${withdrawn}`,+ detail: `${base}; ${faults} rail faults${topFaults ? ` (${topFaults})` : ''}${causeNote}`,hint,};}diff --git a/data/changelog.json b/data/changelog.jsonindex 2ee93e16b..7b0f9ee24 100644--- a/data/changelog.json+++ b/data/changelog.json@@ -1,6 +1,15 @@{"$comment": "Curated changelog entries for three.ws, the editorial layer of the public changelog. New PAGE launches are tracked automatically via the `added` field in data/pages.json; this file is for everything else holders care about: improvements, fixes, SDK releases, trading/payments capabilities, security work. scripts/build-page-index.mjs merges both sources into CHANGELOG.md, public/changelog.json, and public/changelog.xml. Newest first. Allowed tags: feature, improvement, fix, sdk, infra, docs, security. `link` is optional and must be a live three.ws path.","entries": [+ {+ "date": "2026-09-09",+ "title": "The payments status page now says which of two opposite outages it is looking at",+ "summary": "When agent payments stop settling, the status page names the cause. Two very different situations shared one description: either the platform stops offering Solana payment on its price quotes, so nothing is ever attempted, or it keeps offering them and then turns each signed payment away at the final check. Both are fixed by topping up the same wallet, but the evidence sits in opposite places, and the old wording sent whoever was on call to the wrong one. It could even report that Solana had been withdrawn from quotes while the counter printed beside it read zero withdrawals. The status page now names the mechanism and points at the record that actually holds the proof, and the operations runbook carries a table for telling the two apart.",+ "tags": [+ "fix",+ "infra"+ ]+ },{"date": "2026-09-09","title": "Trading finally counts toward your daily streak",diff --git a/docs/ops/production-log-triage.md b/docs/ops/production-log-triage.mdindex 36ba5ff1e..be04cccc6 100644--- a/docs/ops/production-log-triage.md+++ b/docs/ops/production-log-triage.md@@ -294,6 +294,26 @@ HTTP 502/503 GET|POST /api/x402/*, /api/mcp ua: threews-x402-autonomous/1.0 orwindow, or `rail` for real payment-rail faults) and `detail` names thewithdrawal, so the sensor no longer points at the facilitator for a fundingproblem.+- **The third variant, and the one that reads as the quiet variant but is not:+ the accept is still advertised and every settle is REFUSED at the floor.**+ Measured 2026-09-09. `cause` is `sponsor_floor` here too and the fix is the+ same wallet, but the evidence lives in the opposite place, so check+ `metrics.mechanism` before you follow either hint:++ | `metrics.mechanism` | What is happening | Where the proof is |+ |---|---|---|+ | `accept_withdrawn` | `sponsorKnownBelowFloor()` dropped the Solana accept, so no payment is ever attempted | the 402 challenge (`/api/x402/three-intel`), NOT the facilitator: it never saw these calls |+ | `settle_refused` | the accept is on every challenge, buyers sign real payments, our facilitator turns each away at the floor gate | the facilitator's reject book: `fee_wallet_below_floor:<held><<floor>` names the wallet and both numbers |+ | `paced` | the wallet fee governor spent today's budget on purpose | `GET /api/x402/runway-lab` |+ | `rail` | genuine payment-rail faults | the settle logs, per the greps above |++ On 2026-09-09 production was `settle_refused` verbatim: **0** `no_solana_accept`,+ 164 `fee_wallet_below_floor:1167627<2000000`, and 126 `http_402` recorded by the+ ring on the paid replay (a 402 there means the payment we built was rejected,+ which is why the reject book and not the challenge is the place to look). Before+ the split, that window printed `Solana accept withdrawn (0 no_solana_accept …)`,+ a clause contradicting its own counter, and told the reader not to start at the+ facilitator.- **The loud variant: a rail-shaped storm that is really a dry sponsor(`InsufficientFundsForRent`).** Observed 2026-08-28. This is the quietvariant's opposite and it is easy to misread, because every symptom points atdiff --git a/tests/api/x402-settle-health.test.js b/tests/api/x402-settle-health.test.jsindex a0815bda7..1da7511d1 100644--- a/tests/api/x402-settle-health.test.js+++ b/tests/api/x402-settle-health.test.js@@ -480,3 +480,106 @@ describe('classifySettleBuckets: a dry sponsor is not a rail fault', () => {expect(withFlag.rate).toBe(without.rate);});});++describe('classifySettleBuckets: a floor refusal is not a withdrawn accept', () => {+ // Both shapes are `cause: sponsor_floor` and both are fixed by getting SOL to+ // the same wallet, so one hint covered them for weeks. They are opposite+ // situations to the person reading it:+ //+ // WITHDRAWN the challenge carries no Solana accept, so nothing is ever+ // attempted and the facilitator never sees these calls.+ // REFUSED the accept is still advertised, buyers sign real payments, and+ // our own facilitator turns each one away at the floor gate.+ //+ // Production on 2026-09-09 was the REFUSED shape verbatim: 0 no_solana_accept,+ // 164 `fee_wallet_below_floor:1167627<2000000`, 126 `http_402` from paid+ // replays. The row read "Solana accept withdrawn (0 no_solana_accept...)", one+ // clause contradicting its own counter, and the hint said "Do NOT start at the+ // facilitator", which is the only place the wallet and both numbers are named.+ const refusedAtFloor = [+ { success: false, paid: false, reason: 'http_402', n: 126 },+ { success: false, paid: false, reason: 'This operation was aborted', n: 6 },+ { success: false, paid: false, reason: 'fee_wallet_below_floor', n: 25 },+ ];++ it('names the refusal, not a withdrawal, when no_solana_accept is zero', () => {+ const v = classifySettleBuckets(refusedAtFloor);+ expect(v.cause).toBe('sponsor_floor');+ expect(v.mechanism).toBe('settle_refused');+ expect(v.noSolanaAccept).toBe(0);+ expect(v.floorSignals).toBe(25);+ expect(v.hint).toMatch(/REFUSED at the sponsor floor, not withdrawn/);+ // The correction that matters: send the operator TO the reject book.+ expect(v.hint).toMatch(/START at the facilitator/);+ expect(v.hint).toMatch(/fee_wallet_below_floor:<held><<floor>/);+ expect(v.hint).not.toMatch(/Do NOT start at the facilitator/);+ expect(v.hint).not.toMatch(/WITHDRAWN/);+ });++ it('the detail row never claims a withdrawal beside a zero counter', () => {+ const v = classifySettleBuckets(refusedAtFloor);+ expect(v.detail).toMatch(/25 settle\(s\) refused at the fee-wallet floor \(accept still advertised\)/);+ expect(v.detail).not.toMatch(/withdrawn/i);+ expect(v.detail).not.toMatch(/0 no_solana_accept/);+ });++ it('keeps the withdrawn wording, and its counter, when the accept really is gone', () => {+ const v = classifySettleBuckets([+ { success: true, paid: true, reason: 'none', n: 10 },+ { success: false, paid: false, reason: 'http_502', n: 40 },+ { success: false, paid: false, reason: 'no_solana_accept', n: 374 },+ ]);+ expect(v.cause).toBe('sponsor_floor');+ expect(v.mechanism).toBe('accept_withdrawn');+ expect(v.hint).toMatch(/WITHDRAWN, not rejected/);+ expect(v.hint).toMatch(/Do NOT start at the facilitator/);+ expect(v.detail).toMatch(/Solana accept withdrawn \(374 no_solana_accept/);+ });++ it('a flapping floor with both signals reports the withdrawal, the harder stop', () => {+ // When the accept is gone for part of the window AND the settles that got+ // through were refused, the withdrawal is what an operator has to know: a+ // call that never carried a payable accept never reached the reject book,+ // so the facilitator-first instruction would under-count the outage.+ const v = classifySettleBuckets([+ { success: true, paid: true, reason: 'none', n: 5 },+ { success: false, paid: false, reason: 'http_502', n: 40 },+ { success: false, paid: false, reason: 'no_solana_accept', n: 200 },+ { success: false, paid: false, reason: 'fee_wallet_below_floor', n: 12 },+ ]);+ expect(v.mechanism).toBe('accept_withdrawn');+ expect(v.hint).toMatch(/WITHDRAWN, not rejected/);+ });++ it('both shapes still carry the identical funding remedy', () => {+ const refused = classifySettleBuckets(refusedAtFloor);+ const withdrawn = classifySettleBuckets([+ { success: true, paid: true, reason: 'none', n: 10 },+ { success: false, paid: false, reason: 'http_502', n: 40 },+ { success: false, paid: false, reason: 'no_solana_accept', n: 374 },+ ]);+ for (const v of [refused, withdrawn]) {+ expect(v.hint).toMatch(/treasury-topup\?dry=1/);+ expect(v.hint).toMatch(/agent_reclaim\.failed/);+ expect(v.hint).toMatch(/secret_undecryptable/);+ expect(v.hint).toMatch(/skipped_floor_held_sol 0/);+ expect(v.hint).toMatch(/docs\/ops\/production-log-triage\.md/);+ }+ });++ it('the governor and rail verdicts carry their own mechanism, not a floor one', () => {+ const paced = classifySettleBuckets([+ { success: true, paid: true, reason: 'none', n: 10 },+ { success: false, paid: false, reason: 'http_502', n: 20 },+ { success: false, paid: false, reason: 'fee_runway_exhausted', n: 400 },+ ]);+ expect(paced.cause).toBe('fee_governor');+ expect(paced.mechanism).toBe('paced');+ const rail = classifySettleBuckets([+ { success: true, paid: true, reason: 'none', n: 10 },+ { success: false, paid: false, reason: 'http_502', n: 90 },+ ]);+ expect(rail.cause).toBe('rail');+ expect(rail.mechanism).toBe('rail');+ });+});