Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

31.4 KB
Raw Blame
import { describe, it, expect } from 'vitest';
import { readFile } from 'node:fs/promises';
import {
	isRailFault,
	classifySettleBuckets,
} from '../../api/_lib/ops/x402-settle-health.js';

// The sensor that would have caught the 2026-07 502 wave on day one. The bucket
// shapes here are the real `x402_autonomous_log` aggregation observed in prod
// (GROUP BY success, amount>0 AS paid, first `:`-token of error_msg).

describe('isRailFault — only payment-rail faults count against the settle rate', () => {
	it('counts facilitator/settle 5xx and payment-rejected 402', () => {
		for (const r of ['http_500', 'http_502', 'http_503', 'http_504', 'http_402']) {
			expect(isRailFault(r)).toBe(true);
		}
	});

	it('counts RPC / broadcast / confirm / simulation / timeout signatures', () => {
		for (const r of [
			'broadcast_failed', 'settle_failed', 'not_confirmed', 'rpc_preflight_failed',
			'insufficient_source', 'This operation was aborted',
		]) {
			expect(isRailFault(r)).toBe(true);
		}
	});

	it('does NOT count caller/endpoint errors — these are not settle failures', () => {
		// The ring POSTing to a GET-only endpoint or passing rejected args. ~800/6h
		// in prod; counting them would peg the sensor red and bury real regressions.
		for (const r of ['http_400', 'http_404', 'http_405', 'http_409']) {
			expect(isRailFault(r)).toBe(false);
		}
	});

	it('does NOT count benign guards or downstream notes', () => {
		for (const r of [
			'cap_would_exceed', 'insufficient_payer_usdc', 'wallet_unconfigured',
			'breaker_tripped', 'config_missing', 'sniper_intel_calls_failed', 'none', '',
		]) {
			expect(isRailFault(r)).toBe(false);
		}
	});

	it('classifies the fix\'s own precise error format (broadcast_failed:already_processed:sig)', () => {
		// gatherX402SettleHealth splits on ':' → first token 'broadcast_failed'.
		expect(isRailFault('broadcast_failed')).toBe(true);
	});
});

describe('classifySettleBuckets — verdict from real bucket shapes', () => {
	// The live 3h window during the 502 wave: 344 settled, 121 http_502 + 7 http_402
	// rail faults, plus noise that must be excluded.
	const liveOutageBuckets = [
		{ success: true, paid: true, reason: 'none', n: 344 },
		{ success: false, paid: true, reason: 'http_502', n: 100 },
		{ success: false, paid: false, reason: 'http_502', n: 21 },
		{ success: false, paid: false, reason: 'http_402', n: 7 },
		// Excluded noise:
		{ success: false, paid: true, reason: 'http_400', n: 260 },
		{ success: false, paid: true, reason: 'http_405', n: 190 },
		{ success: false, paid: false, reason: 'cap_would_exceed', n: 52 },
		{ success: false, paid: false, reason: 'insufficient_payer_usdc', n: 39 },
		{ success: true, paid: false, reason: 'none', n: 93 }, // free/ok calls
	];

	it('reads the live 502 wave as DEGRADED with http_502 as the top fault', () => {
		const v = classifySettleBuckets(liveOutageBuckets);
		expect(v.status).toBe('degraded');
		expect(v.settled).toBe(344);
		expect(v.faults).toBe(128); // 100 + 21 + 7 — client errors and guards excluded
		expect(v.attempts).toBe(472);
		expect(v.rate).toBeCloseTo(0.729, 2);
		expect(v.faultClasses[0]).toEqual({ reason: 'http_502', n: 121 });
		expect(v.detail).toMatch(/http_502×121/);
		expect(v.hint).toMatch(/settle_failed/);
	});

	it('a clean rail reads OK — successes only, no rail faults', () => {
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 400 },
			{ success: false, paid: false, reason: 'cap_would_exceed', n: 30 },
			{ success: false, paid: true, reason: 'http_405', n: 50 }, // caller error, excluded
		]);
		expect(v.status).toBe('ok');
		expect(v.rate).toBe(1);
		expect(v.faults).toBe(0);
	});

	it('a rail collapse (<50% settling) reads DOWN', () => {
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 30 },
			{ success: false, paid: true, reason: 'http_502', n: 80 },
		]);
		expect(v.status).toBe('down');
		expect(v.rate).toBeCloseTo(0.273, 2);
	});

	it('too few attempts reads UNKNOWN and never pages', () => {
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 5 },
			{ success: false, paid: true, reason: 'http_502', n: 3 },
		]);
		expect(v.status).toBe('unknown');
		expect(v.rate).toBeNull();
		expect(v.detail).toMatch(/too few to judge/);
	});

	it('an idle window (all guards, nothing settled) is UNKNOWN, not an outage', () => {
		// The exact quiet-hour shape: ring low on USDC, choosing not to pay.
		const v = classifySettleBuckets([
			{ success: false, paid: false, reason: 'insufficient_payer_usdc', n: 15 },
			{ success: false, paid: false, reason: 'cap_would_exceed', n: 21 },
			{ success: true, paid: false, reason: 'none', n: 12 },
		]);
		expect(v.status).toBe('unknown');
		expect(v.attempts).toBe(0);
	});

	it('empty / malformed input is UNKNOWN, never a throw', () => {
		expect(classifySettleBuckets([]).status).toBe('unknown');
		expect(classifySettleBuckets(null).status).toBe('unknown');
		expect(classifySettleBuckets(undefined).status).toBe('unknown');
	});

	// The 2026-07-29 outage. Rail faults sat at their normal level all day while
	// settlements fell 750/h → 25/h, because the sponsor dropped under its SOL
	// floor and buildRequirements() withdrew the Solana accept from every
	// challenge. The rate alone reads identically to a facilitator rejection, and
	// the old hint sent an operator to the facilitator, where the 503s look flat
	// and nothing explains the drop.
	const acceptWithdrawnBuckets = [
		{ success: true, paid: true, reason: 'none', n: 87 },
		{ success: false, paid: false, reason: 'http_503', n: 154 },
		{ success: false, paid: false, reason: 'http_402', n: 133 },
		{ success: false, paid: false, reason: 'rpc_error', n: 12 },
		{ success: false, paid: false, reason: 'This operation was aborted', n: 8 },
		// The mechanism, which the rate deliberately does not count:
		{ success: false, paid: false, reason: 'no_solana_accept', n: 1059 },
		{ success: false, paid: false, reason: 'settlement temporarily unavailable', n: 26 },
		// Excluded noise, same as any window:
		{ success: false, paid: false, reason: 'datapoint_sweep_calls_failed', n: 24 },
		{ success: false, paid: false, reason: 'cap_would_exceed', n: 8 },
	];

	it('blames the sponsor floor, not the facilitator, when the Solana accept is withdrawn', () => {
		const v = classifySettleBuckets(acceptWithdrawnBuckets);
		expect(v.status).toBe('down');
		expect(v.cause).toBe('sponsor_floor');
		expect(v.noSolanaAccept).toBe(1059);
		expect(v.floorSignals).toBe(26);
		// The rate itself is unchanged: no_solana_accept and the 503-prose floor
		// refusals must not become rail faults, or the percentage stops meaning
		// "of the settles we attempted, how many landed".
		expect(v.settled).toBe(87);
		expect(v.faults).toBe(307);
		expect(v.detail).toMatch(/Solana accept withdrawn \(1059 no_solana_accept/);
		expect(v.hint).toMatch(/WITHDRAWN, not rejected/);
		expect(v.hint).toMatch(/treasury-topup/);
		// And it must NOT send the operator down the facilitator path.
		expect(v.hint).not.toMatch(/Payments are being rejected at settle/);
	});

	it('the sponsor-floor hint refuses to promise a self-heal that sealed wallets cannot deliver', () => {
		// The hint used to end at "Owner SOL is needed only when every reclaim
		// source reports at_or_below_floor". On this platform most of the
		// reclaimable SOL sits in wallets encrypted under a key retired in the
		// 2026-07 host migration: they report `secret_undecryptable`, never
		// `at_or_below_floor`, so that sentence read as "the cron will fix it" for
		// a condition no cron can fix. Two sessions lost time to it.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 40 },
			{ success: false, paid: false, reason: 'http_502', n: 60 },
			{ success: false, paid: false, reason: 'fee_wallet_below_floor', n: 1 },
		]);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.hint).toMatch(/secret_undecryptable/);
		expect(v.hint).toMatch(/agent_reclaim\.failed/);
		expect(v.hint).toMatch(/WALLET_ENCRYPTION_KEY/);
		// "only when ... at_or_below_floor" was the exact false promise. It must
		// not come back.
		expect(v.hint).not.toMatch(/only when every reclaim source reports at_or_below_floor\./);
	});

	it('still blames the rail when faults rose and Solana is payable', () => {
		const v = classifySettleBuckets(liveOutageBuckets);
		expect(v.cause).toBe('rail');
		expect(v.noSolanaAccept).toBe(0);
		expect(v.floorSignals).toBe(0);
		expect(v.hint).toMatch(/Payments are being rejected at settle/);
		expect(v.detail).not.toMatch(/withdrawn/i);
	});

	it('a single floor refusal is enough to name the cause, even under heavy rail noise', () => {
		// The flapping case: sponsorKnownBelowFloor() is cache-backed and fail-open,
		// so some settles still land and no_solana_accept can be lower than settled.
		// One 503-prose floor refusal is still proof of what is happening.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 40 },
			{ success: false, paid: false, reason: 'http_502', n: 60 },
			{ success: false, paid: false, reason: 'no_solana_accept', n: 3 },
			{ success: false, paid: false, reason: 'fee_wallet_below_floor', n: 1 },
		]);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.floorSignals).toBe(1);
	});

	it('no_solana_accept below the settled count alone does not cry sponsor floor', () => {
		// A handful of third-party endpoints that only advertise Base is normal and
		// is genuinely the ring choosing not to pay. Do not hijack the diagnosis.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 60 },
			{ success: false, paid: false, reason: 'http_502', n: 40 },
			{ success: false, paid: false, reason: 'no_solana_accept', n: 5 },
		]);
		expect(v.cause).toBe('rail');
		expect(v.noSolanaAccept).toBe(5);
	});

	it('boundary: exactly 90% is OK, just under is DEGRADED', () => {
		const ok = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 90 },
			{ success: false, paid: true, reason: 'http_502', n: 10 },
		]);
		expect(ok.status).toBe('ok');
		const degraded = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 89 },
			{ success: false, paid: true, reason: 'http_502', n: 11 },
		]);
		expect(degraded.status).toBe('degraded');
	});
});

// ── The 2026-08-01 fee_runway_exhausted wave ────────────────────────────────
// The shape: the wallet fee governor spends the fee wallet's daily SOL budget,
// then refuses every remaining settle of the day. Measured on production that
// morning: 85,265 governed refusals against 562 rail-shaped failures, a settle
// rate of 25.9%, and a `rail` verdict whose hint sent the reader to the
// facilitator to debug duplicate signatures. The money answer (fund the fee
// wallet) was nowhere in the output.
//
// Both regimes below are the SAME condition and want the SAME action, so both
// must name it. Before the caller-side admission check the refusal came back as
// an http_502 and inflated the fault count; after it, the ring skips the call
// and the counts collapse toward zero instead. A sensor that only understood the
// first regime would go quiet exactly when the fix landed.
describe('classifySettleBuckets: a governed throttle is not a rail fault', () => {
	it('names the fee governor when its skips outnumber the rail faults', () => {
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 60 },
			{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 900 },
			{ success: false, paid: true, reason: 'http_502', n: 40 },
		]);
		expect(v.cause).toBe('fee_governor');
		expect(v.governorSkips).toBe(900);
		// The skips stay out of the rate: they are not attempts that failed, they
		// are calls we chose not to make.
		expect(v.settled).toBe(60);
		expect(v.faults).toBe(40);
		expect(v.detail).toMatch(/900 call\(s\) paced by the fee governor/);
		expect(v.hint).toMatch(/GOVERNED THROTTLE/);
		expect(v.hint).toMatch(/runway-lab/);
		// It must not send anyone at the facilitator or at the sponsor floor knob.
		expect(v.hint).not.toMatch(/Payments are being rejected at settle/);
		expect(v.hint).toMatch(/do NOT lower the sponsor floor/i);
	});

	it('reports DEGRADED, not UNKNOWN, when pacing is why there is nothing to judge', () => {
		// The post-fix regime: the caller-side admission check skips the calls, so
		// attempts fall under MIN_ATTEMPTS. `unknown` here would hide a rail that
		// is 100% paced shut behind the same verdict a sleeping ring gets.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 4 },
			{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 1200 },
		]);
		expect(v.status).toBe('degraded');
		expect(v.cause).toBe('fee_governor');
		expect(v.rate).toBeNull();
		expect(v.detail).toMatch(/1200 call\(s\) skipped by the wallet fee governor/);
	});

	it('a genuinely idle ring still reads UNKNOWN, a few skips are the governor working', () => {
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 3 },
			{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 6 },
		]);
		expect(v.status).toBe('unknown');
		expect(v.governorSkips).toBe(6);
	});

	it('the hard SOL floor outranks the governor when both are lit', () => {
		// Both want SOL in the same wallet, but under the floor every settle fails
		// closed while a spent budget still settles at the paced rate. Report the
		// harder stop.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 20 },
			{ success: false, paid: false, reason: 'http_502', n: 40 },
			{ success: false, paid: false, reason: 'fee_wallet_below_floor', n: 2 },
			{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 500 },
		]);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.governorSkips).toBe(500);
	});

	it('pacing under a healthy rate stays OK and still reports the skip count', () => {
		// The steady state this fix is aiming for: a small, funded budget paced
		// across the day, everything it admits settling cleanly.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 248 },
			{ success: false, paid: true, reason: 'http_502', n: 6 },
			{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 3100 },
		]);
		expect(v.status).toBe('ok');
		expect(v.rate).toBeGreaterThan(0.9);
		expect(v.governorSkips).toBe(3100);
		expect(v.detail).toMatch(/3100 paced by the fee governor/);
	});

	it('the governor reason is never counted as a rail fault, in any variant', () => {
		// pay.js records the reason with its arithmetic appended; the SQL bucket
		// splits on ':' so the sensor sees the bare token. Cover both.
		expect(isRailFault('fee_runway_exhausted')).toBe(false);
		expect(isRailFault('fee_runway_exhausted:10022298+10000>10000000')).toBe(false);
	});
});

// The ring log is reason-blind for refusals that arrive over HTTP: a settle the
// governor refused reaches x402_autonomous_log as a bare http_503 (pre
// 2026-08-06: http_502) and used to read as a rail fault. On 2026-08-05
// production this showed `cause: "rail"` at settle 26.1% while the facilitator
// book held 75k+ fee_runway_exhausted rejects: the operator was sent to debug
// a healthy rail. `facilitatorRejects` carries that book into the classifier.
describe('classifySettleBuckets: facilitator-book reconciliation of status-only faults', () => {
	// The live 2026-08-05 shape, scaled: settles landing, a 502/503 wall, and the
	// facilitator book attributing nearly all of it to the governor.
	const reasonBlindOutage = [
		{ success: true, paid: true, reason: 'none', n: 443 },
		{ success: false, paid: true, reason: 'http_502', n: 1172 },
		{ success: false, paid: true, reason: 'http_503', n: 24 },
		{ success: false, paid: true, reason: 'settle_failed', n: 29 },
	];

	it('re-attributes governor-refused 5xx rows so the rate judges the rail alone', () => {
		const v = classifySettleBuckets(reasonBlindOutage, {
			facilitatorRejects: { governor: 1180 },
		});
		// 503 drains first, then 502: 24 + 1156 attributed, 16 http_502 + 29
		// settle_failed remain genuine rail faults.
		expect(v.governorSkips).toBe(1180);
		expect(v.faults).toBe(45);
		// The rate now judges the rail alone (443/488, not 443/1668), and per the
		// established pacing semantics a healthy residual rail reads OK with the
		// governed volume named in the detail, instead of yesterday's down/rail.
		expect(v.rate).toBeGreaterThan(0.9);
		expect(v.status).toBe('ok');
		expect(v.detail).toMatch(/1180 paced by the fee governor/);
	});

	it('clamps attribution at the observed status faults, never inventing negatives', () => {
		// Window skew between the two logs can put more rejects in the book than
		// 5xx rows in the ring log. min() caps the drain; faults never go below the
		// non-status rail signatures.
		const v = classifySettleBuckets(reasonBlindOutage, {
			facilitatorRejects: { governor: 50_000, floor: 10_000 },
		});
		expect(v.faults).toBe(29);
		expect(v.governorSkips).toBe(1196);
		expect(v.floorSignals).toBe(0);
		expect(v.faultClasses).toEqual([{ reason: 'settle_failed', n: 29 }]);
	});

	it('floor rejects drain what the governor left and outrank it in the verdict', () => {
		const v = classifySettleBuckets(reasonBlindOutage, {
			facilitatorRejects: { governor: 1000, floor: 150 },
		});
		// governor drains 24 http_503 + 976 http_502; floor drains 150 more 502s.
		expect(v.governorSkips).toBe(1000);
		expect(v.floorSignals).toBe(150);
		expect(v.faults).toBe(75);
		expect(v.cause).toBe('sponsor_floor');
	});

	it('a genuine rail outage with an empty facilitator book still blames the rail', () => {
		const v = classifySettleBuckets(reasonBlindOutage, { facilitatorRejects: { governor: 0, floor: 0 } });
		expect(v.cause).toBe('rail');
		expect(v.faults).toBe(1225);
		expect(v.status).toBe('down');
	});

	it('omitting facilitatorRejects behaves exactly as before', () => {
		const withOpt = classifySettleBuckets(reasonBlindOutage, {});
		const without = classifySettleBuckets(reasonBlindOutage);
		expect(withOpt).toEqual(without);
		expect(without.cause).toBe('rail');
	});
});

describe('classifySettleBuckets: a dry sponsor is not a rail fault', () => {
	// The 2026-08-28 outage, reproduced from the real aggregation. The sponsor
	// held 0.000899107 SOL against a 0.02 floor, so every transaction it fee-paid
	// died on InsufficientFundsForRent at account index 0. The reason TOKEN those
	// rows carry is rail-shaped (`sweep_broadcast_failed`, `simulation_failed`)
	// because the rent detail lives past the first `:`, so the sensor counted 95
	// rail faults and answered `cause: rail`. The hint then sent the operator to
	// hunt duplicate signatures and RPC preflight faults instead of funding a
	// wallet that had 0.0000082 SOL of spendable headroom.
	const rentWave = [
		{ success: false, paid: false, reason: 'sweep_broadcast_failed', rent: true, n: 86 },
		{ success: false, paid: false, reason: 'simulation_failed', rent: true, n: 5 },
		{ success: false, paid: false, reason: 'rpc_preflight_failed', rent: false, n: 4 },
	];

	it('names the sponsor floor, not the rail, when the fee payer cannot afford rent', () => {
		const v = classifySettleBuckets(rentWave);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.floorSignals).toBe(91);
		// The 4 genuine preflight faults stay on the rail where they belong.
		expect(v.faults).toBe(4);
		expect(v.hint).toMatch(/sponsor/i);
	});

	it('reports DOWN, never UNKNOWN, when the floor is why there is nothing to judge', () => {
		// Moving rent rows out of `faults` is correct (they never reached the rail)
		// but it drops `attempts` under MIN_ATTEMPTS. Without the floor branch the
		// sensor would answer "too few attempts to judge" for a three-hour outage.
		const v = classifySettleBuckets([
			{ success: false, paid: false, reason: 'sweep_broadcast_failed', rent: true, n: 95 },
		]);
		expect(v.status).toBe('down');
		expect(v.cause).toBe('sponsor_floor');
		expect(v.detail).toMatch(/sponsor under its SOL floor/);
	});

	it('a handful of rent rows on an otherwise idle ring still reads UNKNOWN', () => {
		// The floor branch keys off the same MIN_ATTEMPTS as every other verdict, so
		// a couple of stray rows cannot declare an outage on a quiet ring.
		const v = classifySettleBuckets([
			{ success: false, paid: false, reason: 'simulation_failed', rent: true, n: 3 },
		]);
		expect(v.status).toBe('unknown');
	});

	it('classifies sponsor_fee_unfunded as floor even with no rent flag set', () => {
		// The verify path now raises the fee-payer verdict as its own reason class
		// (api/_lib/x402/self-facilitator.js), so the token itself carries the
		// meaning and the sensor no longer depends on a `rent` flag reconstructed
		// from the full error_msg. That matters because the old spelling,
		// `simulation_failed`, matches RAIL_SIGNATURE's /simulation/: a caller that
		// aggregates by token alone counted a starved sponsor as a rail fault.
		const v = classifySettleBuckets([
			{ success: false, paid: false, reason: 'sponsor_fee_unfunded', n: 95 },
		]);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.floorSignals).toBe(95);
		expect(v.faults).toBe(0);
	});

	it('leaves the buyer-side twin off the sponsor floor', () => {
		// `payer_fee_unfunded` is a buyer whose OWN wallet cannot pay the fee. It is
		// not the platform's to fund, so it must never trip the sponsor hint that
		// tells an operator to send SOL.
		const v = classifySettleBuckets([
			{ success: false, paid: false, reason: 'payer_fee_unfunded', n: 95 },
		]);
		expect(v.floorSignals ?? 0).toBe(0);
		expect(v.cause).not.toBe('sponsor_floor');
		expect(v.hint ?? '').not.toMatch(/treasury-topup/);
	});

	it('a bucket with no rent flag behaves exactly as it always did', () => {
		const withFlag = classifySettleBuckets([
			{ success: true, paid: true, n: 90 },
			{ success: false, paid: false, reason: 'http_502', rent: false, n: 10 },
		]);
		const without = classifySettleBuckets([
			{ success: true, paid: true, n: 90 },
			{ success: false, paid: false, reason: 'http_502', n: 10 },
		]);
		expect(withFlag.status).toBe(without.status);
		expect(withFlag.faults).toBe(without.faults);
		expect(withFlag.rate).toBe(without.rate);
	});
});

describe('classifySettleBuckets: a floor refusal is not a withdrawn accept', () => {
	// Both shapes are `cause: sponsor_floor` and both are fixed by getting SOL to
	// the same wallet, so one hint covered them for weeks. They are opposite
	// situations to the person reading it:
	//
	//   WITHDRAWN  the challenge carries no Solana accept, so nothing is ever
	//              attempted and the facilitator never sees these calls.
	//   REFUSED    the accept is still advertised, buyers sign real payments, and
	//              our own facilitator turns each one away at the floor gate.
	//
	// Production on 2026-09-09 was the REFUSED shape verbatim: 0 no_solana_accept,
	// 164 `fee_wallet_below_floor:1167627<2000000`, 126 `http_402` from paid
	// replays. The row read "Solana accept withdrawn (0 no_solana_accept...)", one
	// clause contradicting its own counter, and the hint said "Do NOT start at the
	// facilitator", which is the only place the wallet and both numbers are named.
	const refusedAtFloor = [
		{ success: false, paid: false, reason: 'http_402', n: 126 },
		{ success: false, paid: false, reason: 'This operation was aborted', n: 6 },
		{ success: false, paid: false, reason: 'fee_wallet_below_floor', n: 25 },
	];

	it('names the refusal, not a withdrawal, when no_solana_accept is zero', () => {
		const v = classifySettleBuckets(refusedAtFloor);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.mechanism).toBe('settle_refused');
		expect(v.noSolanaAccept).toBe(0);
		expect(v.floorSignals).toBe(25);
		expect(v.hint).toMatch(/REFUSED at the sponsor floor, not withdrawn/);
		// The correction that matters: send the operator TO the reject book.
		expect(v.hint).toMatch(/START at the facilitator/);
		expect(v.hint).toMatch(/fee_wallet_below_floor:<held><<floor>/);
		expect(v.hint).not.toMatch(/Do NOT start at the facilitator/);
		expect(v.hint).not.toMatch(/WITHDRAWN/);
	});

	it('the detail row never claims a withdrawal beside a zero counter', () => {
		const v = classifySettleBuckets(refusedAtFloor);
		expect(v.detail).toMatch(/25 settle\(s\) refused at the fee-wallet floor \(accept still advertised\)/);
		expect(v.detail).not.toMatch(/withdrawn/i);
		expect(v.detail).not.toMatch(/0 no_solana_accept/);
	});

	it('keeps the withdrawn wording, and its counter, when the accept really is gone', () => {
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 10 },
			{ success: false, paid: false, reason: 'http_502', n: 40 },
			{ success: false, paid: false, reason: 'no_solana_accept', n: 374 },
		]);
		expect(v.cause).toBe('sponsor_floor');
		expect(v.mechanism).toBe('accept_withdrawn');
		expect(v.hint).toMatch(/WITHDRAWN, not rejected/);
		expect(v.hint).toMatch(/Do NOT start at the facilitator/);
		expect(v.detail).toMatch(/Solana accept withdrawn \(374 no_solana_accept/);
	});

	it('a flapping floor with both signals reports the withdrawal, the harder stop', () => {
		// When the accept is gone for part of the window AND the settles that got
		// through were refused, the withdrawal is what an operator has to know: a
		// call that never carried a payable accept never reached the reject book,
		// so the facilitator-first instruction would under-count the outage.
		const v = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 5 },
			{ success: false, paid: false, reason: 'http_502', n: 40 },
			{ success: false, paid: false, reason: 'no_solana_accept', n: 200 },
			{ success: false, paid: false, reason: 'fee_wallet_below_floor', n: 12 },
		]);
		expect(v.mechanism).toBe('accept_withdrawn');
		expect(v.hint).toMatch(/WITHDRAWN, not rejected/);
	});

	it('both shapes still carry the identical funding remedy', () => {
		const refused = classifySettleBuckets(refusedAtFloor);
		const withdrawn = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 10 },
			{ success: false, paid: false, reason: 'http_502', n: 40 },
			{ success: false, paid: false, reason: 'no_solana_accept', n: 374 },
		]);
		for (const v of [refused, withdrawn]) {
			expect(v.hint).toMatch(/treasury-topup\?dry=1/);
			expect(v.hint).toMatch(/agent_reclaim\.failed/);
			expect(v.hint).toMatch(/secret_undecryptable/);
			expect(v.hint).toMatch(/skipped_floor_held_sol 0/);
			expect(v.hint).toMatch(/docs\/ops\/production-log-triage\.md/);
		}
	});

	it('the governor and rail verdicts carry their own mechanism, not a floor one', () => {
		const paced = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 10 },
			{ success: false, paid: false, reason: 'http_502', n: 20 },
			{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 400 },
		]);
		expect(paced.cause).toBe('fee_governor');
		expect(paced.mechanism).toBe('paced');
		const rail = classifySettleBuckets([
			{ success: true, paid: true, reason: 'none', n: 10 },
			{ success: false, paid: false, reason: 'http_502', n: 90 },
		]);
		expect(rail.cause).toBe('rail');
		expect(rail.mechanism).toBe('rail');
	});
});

describe('readSettleHealth surfaces the mechanism the runbook tells operators to read', () => {
	// The field is computed in the classifier and consumed by
	// docs/ops/production-log-triage.md as `metrics.mechanism`. It reached the
	// healthz payload only because it is listed in the metrics builder, and a
	// documented field that is silently absent is worse than no field: it sends
	// the reader looking for something that is not there. Pin the wiring.
	it('lists mechanism in the metrics the health check builds', async () => {
		const src = await readFile(
			new URL('../../api/_lib/ops/x402-settle-health.js', import.meta.url),
			'utf8',
		);
		const metricsBlock = src.slice(src.indexOf('metrics: {'), src.indexOf('metrics: {') + 1400);
		expect(metricsBlock).toMatch(/mechanism: v\.mechanism/);
		expect(metricsBlock).toMatch(/cause: v\.cause/);
	});
});

describe('classifySettleBuckets: a caller that skips the admission gate is not a healthy rail', () => {
	// Production 2026-09-29, scaled to 3h: the ring's own rows read 97% while the
	// facilitator refused about 5,500 governor-paced settles from the x402 seed
	// cron, which never asks the admission gate and never writes a ring row.
	const ringRows = [
		{ success: true, paid: true, reason: 'none', n: 262 },
		{ success: false, paid: true, reason: 'This operation was aborted', n: 7 },
		{ success: false, paid: false, reason: 'fee_runway_exhausted', n: 924 },
	];

	it('degrades with mechanism gate_bypass when unattributed refusals outnumber settles', () => {
		const v = classifySettleBuckets(ringRows, { facilitatorRejects: { governor: 5_480 } });
		expect(v.rate).toBeGreaterThan(0.9);
		expect(v.gateBypass).toBe(5_480);
		expect(v.status).toBe('degraded');
		expect(v.cause).toBe('fee_governor');
		expect(v.mechanism).toBe('gate_bypass');
		expect(v.detail).toMatch(/5480 settle\(s\) refused by the fee governor at the facilitator/);
		expect(v.hint).toMatch(/reserveFeeAdmissions/);
	});

	it('counts only refusals no ring row accounts for', () => {
		const rows = [...ringRows, { success: false, paid: true, reason: 'http_503', n: 300 }];
		const v = classifySettleBuckets(rows, { facilitatorRejects: { governor: 320 } });
		// 300 of the 320 are the ring's own http_503 replays; 20 remain unattributed,
		// fewer than the 262 the ring settled, so a healthy rail stays ok.
		expect(v.gateBypass).toBe(20);
		expect(v.status).toBe('ok');
		expect(v.mechanism).toBeUndefined();
	});

	it('the cross-instance race residue never trips it', () => {
		// About 8 an hour of $0.01 governor refusals survived the caller-side gate
		// in the same 24h, the residue of per-instance admission snapshots.
		const v = classifySettleBuckets(ringRows, { facilitatorRejects: { governor: 24 } });
		expect(v.status).toBe('ok');
	});

	it('an idle ring with a bypassing caller is degraded, not unknown', () => {
		const v = classifySettleBuckets([], { facilitatorRejects: { governor: 1_800 } });
		expect(v.status).toBe('degraded');
		expect(v.mechanism).toBe('gate_bypass');
	});

	it('a rail that is already failing keeps its own diagnosis and carries the count', () => {
		const failing = [
			{ success: true, paid: true, reason: 'none', n: 100 },
			{ success: false, paid: true, reason: 'broadcast_failed', n: 200 },
		];
		const v = classifySettleBuckets(failing, { facilitatorRejects: { governor: 5_000 } });
		expect(v.status).toBe('down');
		expect(v.mechanism).toBe('rail');
		expect(v.gateBypass).toBe(5_000);
	});
});

describe('the settle sensor ignores pipelines that never pay', () => {
	it('excludes the external uptime monitor, and that monitor is still free', async () => {
		const { UNPAID_PROBE_PIPELINES } = await import('../../api/_lib/ops/x402-settle-health.js');
		expect(UNPAID_PROBE_PIPELINES).toContain('reliability');
		// The exclusion is only honest while every registry entry in these pipelines
		// is priced at zero. A paid entry added under one of them must fail here
		// rather than vanish from the settle rate.
		const registry = await readFile('api/_lib/x402/autonomous-registry.js', 'utf8');
		for (const pipeline of UNPAID_PROBE_PIPELINES) {
			const entries = registry.split(/\n\t\{\n/).filter((block) => block.includes(`pipeline: '${pipeline}'`));
			expect(entries.length).toBeGreaterThan(0);
			for (const block of entries) expect(block).toMatch(/price_atomic:\s*0\b/);
		}
		const source = await readFile('api/_lib/ops/x402-settle-health.js', 'utf8');
		expect(source).toMatch(/<> ALL\(\$\{\[\.\.\.UNPAID_PROBE_PIPELINES\]\}::text\[\]\)/);
	});
});