Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(brownout): prove each declared fallback by breaking the upstream it protects against

nirholasaa7782a9978b
diff --git a/data/brownout.json b/data/brownout.json
index 2c4897514..f1a5ab3af 100644
--- a/data/brownout.json
+++ b/data/brownout.json
@@ -8,57 +8,103 @@
"title": "The intel heatmap keeps its tiles when pump.fun throttles",
"surface": "/intel",
"endpoint": "/api/intel/heatmap",
- "query": { "limit": 6 },
- "bust": { "param": "limit", "min": 6, "max": 40 },
+ "query": {
+ "limit": 6
+ },
+ "bust": {
+ "param": "limit",
+ "warm": 6,
+ "attempt": 40
+ },
"warm": true,
- "break": { "intel-heatmap": "http:429" },
+ "break": {
+ "intel-heatmap": "http:429"
+ },
"expect": {
"status": 200,
- "tier": ["cache", "stale", "fallback"],
+ "tier": [
+ "cache",
+ "stale",
+ "fallback"
+ ],
"degraded": true,
- "exercised": ["intel-heatmap"],
- "jsonHas": ["tokens"]
+ "exercised": [
+ "intel-heatmap"
+ ],
+ "jsonHas": [
+ "tokens"
+ ]
},
"why": "pump.fun answers a burst with 429 far more often than it goes down, and the board is the whole page. The retry honours Retry-After and the last-good tier carries the tiles through what is left, so a throttle costs freshness rather than the page."
},
{
- "id": "fear-greed-survives-its-source",
- "title": "The market page keeps its fear and greed reading when alternative.me is down",
- "surface": "/markets",
- "endpoint": "/api/coin/global",
- "warm": true,
- "break": { "alternative-me-fng": "network" },
+ "id": "coin-page-survives-coingecko",
+ "title": "A coin page still renders when CoinGecko is rate limiting",
+ "surface": "/coin/:id",
+ "endpoint": "/api/coin/detail",
+ "query": {
+ "id": "avalanche-2"
+ },
+ "warm": false,
+ "break": {
+ "coingecko": "http:429"
+ },
"expect": {
"status": 200,
- "exercised": ["alternative-me-fng"]
+ "degraded": true,
+ "exercised": [
+ "coingecko"
+ ],
+ "jsonHas": [
+ "coin"
+ ]
},
- "why": "A single keyless index feed. The page is assembled from several independent reads, so one of them failing must cost that one figure and nothing else."
+ "why": "CoinGecko's keyless tier is shared per egress IP and throttles on its own schedule, which is why /coin/:id 502'd for hours on 2026-07-28. CoinPaprika supplies the profile and DefiLlama the price series, so a throttle costs the page its primary source and nothing else."
},
{
- "id": "news-archive-survives-gcs",
- "title": "The news archive still answers when its bucket does not",
- "surface": "/news",
- "endpoint": "/api/news/archive",
- "query": { "limit": 5 },
- "warm": true,
- "break": { "news-archive": "http:503" },
+ "id": "holders-survive-a-dead-rpc-lane",
+ "title": "Holder concentration survives a Solana RPC lane refusing",
+ "surface": "/token/:mint",
+ "endpoint": "/api/crypto/holders",
+ "query": {
+ "address": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump"
+ },
+ "warm": false,
+ "break": {
+ "solana-rpc": "http:429"
+ },
"expect": {
- "status": [200, 503],
- "exercised": ["news-archive"]
+ "status": [
+ 200,
+ 503
+ ],
+ "exercised": [
+ "solana-rpc"
+ ]
},
- "why": "Archive data is immutable once written, which is what makes a remembered copy exactly as correct as a live one. With nothing ever cached the honest answer is a retryable 503, so both outcomes are accepted and the tier says which happened."
+ "why": "Every lane in the rotation throttles at some point, and free lanes refuse specific call shapes outright. The rotation walks them, demotes a lane per method rather than benching it wholesale, and answers an idempotent read from its last-good body when the whole chain is down."
},
{
"id": "token-page-survives-dexscreener",
"title": "A token page still resolves when DexScreener is throttling",
"surface": "/ca2x402",
"endpoint": "/api/ca2x402/resolve",
- "query": { "mint": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump" },
+ "query": {
+ "mint": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump"
+ },
"warm": true,
- "break": { "dexscreener:tokens": "http:429" },
+ "break": {
+ "dexscreener:tokens": "http:429"
+ },
"expect": {
- "status": [200, 404, 503],
- "exercised": ["dexscreener:tokens"]
+ "status": [
+ 200,
+ 404,
+ 503
+ ],
+ "exercised": [
+ "dexscreener:tokens"
+ ]
},
"why": "One keyless read used to decide whether a token exists at all, so a throttle made a live token read as not found. It now retries behind a breaker with a last-known-good payload, and a definitive miss is reserved for cases where a source actually spoke."
}
diff --git a/public/brownout.json b/public/brownout.json
new file mode 100644
index 000000000..c4366f49f
--- /dev/null
+++ b/public/brownout.json
@@ -0,0 +1,171 @@
+{
+ "generated_at": "2026-08-28T03:33:28.080Z",
+ "base": "http://127.0.0.1:8791",
+ "summary": "Every fallback below has been executed against a running server with the upstream it protects against genuinely broken.",
+ "counts": {
+ "pass": 4
+ },
+ "proven": 4,
+ "total": 4,
+ "results": [
+ {
+ "id": "heatmap-survives-pump-throttle",
+ "title": "The intel heatmap keeps its tiles when pump.fun throttles",
+ "surface": "/intel",
+ "endpoint": "/api/intel/heatmap",
+ "broke": {
+ "intel-heatmap": "http:429"
+ },
+ "why": "pump.fun answers a burst with 429 far more often than it goes down, and the board is the whole page. The retry honours Retry-After and the last-good tier carries the tiles through what is left, so a throttle costs freshness rather than the page.",
+ "verdict": "pass",
+ "problems": [],
+ "ms": 4626,
+ "observed": {
+ "status": 200,
+ "tier": "stale",
+ "degraded": true,
+ "failedSources": 3,
+ "trace": [
+ {
+ "name": "intel-heatmap",
+ "outcome": "429",
+ "ms": 2120
+ },
+ {
+ "name": "intel-heatmap",
+ "outcome": "429",
+ "ms": 2226
+ },
+ {
+ "name": "lastgood:heatmap:meta:FeMbDoX7R1Psc4GEcvJdsbNbZA",
+ "outcome": "stale",
+ "ms": 19354
+ },
+ {
+ "name": "intel-heatmap",
+ "outcome": "429",
+ "ms": 2273
+ }
+ ]
+ }
+ },
+ {
+ "id": "coin-page-survives-coingecko",
+ "title": "A coin page still renders when CoinGecko is rate limiting",
+ "surface": "/coin/:id",
+ "endpoint": "/api/coin/detail",
+ "broke": {
+ "coingecko": "http:429"
+ },
+ "why": "CoinGecko's keyless tier is shared per egress IP and throttles on its own schedule, which is why /coin/:id 502'd for hours on 2026-07-28. CoinPaprika supplies the profile and DefiLlama the price series, so a throttle costs the page its primary source and nothing else.",
+ "verdict": "pass",
+ "problems": [],
+ "ms": 12,
+ "observed": {
+ "status": 200,
+ "tier": "live",
+ "degraded": true,
+ "failedSources": 1,
+ "trace": [
+ {
+ "name": "coingecko",
+ "outcome": "429",
+ "ms": 1
+ }
+ ]
+ }
+ },
+ {
+ "id": "holders-survive-a-dead-rpc-lane",
+ "title": "Holder concentration survives a Solana RPC lane refusing",
+ "surface": "/token/:mint",
+ "endpoint": "/api/crypto/holders",
+ "broke": {
+ "solana-rpc": "http:429"
+ },
+ "why": "Every lane in the rotation throttles at some point, and free lanes refuse specific call shapes outright. The rotation walks them, demotes a lane per method rather than benching it wholesale, and answers an idempotent read from its last-good body when the whole chain is down.",
+ "verdict": "pass",
+ "problems": [],
+ "ms": 20,
+ "observed": {
+ "status": 200,
+ "tier": "stale",
+ "degraded": true,
+ "failedSources": 17,
+ "trace": [
+ {
+ "name": "solana-rpc:solana-rpc.publicnode.com",
+ "outcome": "429",
+ "ms": 1
+ },
+ {
+ "name": "solana-rpc:api.mainnet-beta.solana.com",
+ "outcome": "429",
+ "ms": 0
+ },
+ {
+ "name": "solana-rpc:solana.leorpc.com",
+ "outcome": "429",
+ "ms": 0
+ },
+ {
+ "name": "solana-rpc:public.rpc.solanavibestation.com",
+ "outcome": "429",
+ "ms": 3
+ },
+ {
+ "name": "solana-rpc:api.tatum.io",
+ "outcome": "429",
+ "ms": 0
+ },
+ {
+ "name": "solana-rpc:solana-mainnet.gateway.tatum.io",
+ "outcome": "429",
+ "ms": 0
+ },
+ {
+ "name": "solana-rpc:last-good",
+ "outcome": "stale",
+ "ms": 97466
+ },
+ {
+ "name": "solana-rpc:solana-rpc.publicnode.com",
+ "outcome": "429",
+ "ms": 0
+ }
+ ]
+ }
+ },
+ {
+ "id": "token-page-survives-dexscreener",
+ "title": "A token page still resolves when DexScreener is throttling",
+ "surface": "/ca2x402",
+ "endpoint": "/api/ca2x402/resolve",
+ "broke": {
+ "dexscreener:tokens": "http:429"
+ },
+ "why": "One keyless read used to decide whether a token exists at all, so a throttle made a live token read as not found. It now retries behind a breaker with a last-known-good payload, and a definitive miss is reserved for cases where a source actually spoke.",
+ "verdict": "pass",
+ "problems": [],
+ "ms": 2252,
+ "observed": {
+ "status": 200,
+ "tier": "stale",
+ "degraded": true,
+ "failedSources": 1,
+ "trace": [
+ {
+ "name": "dexscreener:tokens",
+ "outcome": "429",
+ "ms": 2155
+ },
+ {
+ "name": "lastgood:token-market:fembdox7r1psc4gecvjdsbnbza",
+ "outcome": "stale",
+ "ms": 2166
+ }
+ ]
+ }
+ }
+ ]
+}
diff --git a/scripts/prove-brownout.mjs b/scripts/prove-brownout.mjs
index da193bdc3..7a2fb76b5 100644
--- a/scripts/prove-brownout.mjs
+++ b/scripts/prove-brownout.mjs
@@ -86,16 +86,17 @@ export function wasExercised(prov, name) {
}
/** Build the request URL, applying the contract's cache-busting parameter. */
-export function contractUrl(base, contract, seed = Date.now()) {
+export function contractUrl(base, contract, phase = 'attempt') {
const url = new URL(contract.endpoint, base);
for (const [k, v] of Object.entries(contract.query || {})) url.searchParams.set(k, String(v));
const bust = contract.bust;
if (bust?.param) {
- // A warm cache is the single biggest source of a false green here, so the
- // request is deliberately steered off whatever key the last one used. The
- // range is the contract's, because only it knows what stays meaningful.
- const span = Math.max(1, (bust.max ?? 10) - (bust.min ?? 1) + 1);
- url.searchParams.set(bust.param, String((bust.min ?? 1) + (seed % span)));
+ // A warm cache is the single biggest source of a false green here, so warm
+ // and attempt are steered onto DIFFERENT keys, and which value goes where
+ // is the contract's to state rather than the prover's to guess. Caches are
+ // not symmetric: the heatmap answers any smaller limit out of a larger
+ // cached field, so only warm-small then attempt-large actually misses.
+ url.searchParams.set(bust.param, String(phase === 'warm' ? bust.warm : bust.attempt));
}
return url.toString();
}
@@ -149,21 +150,27 @@ export function judge(contract, { status, prov, body, chaosStatus }) {
async function runContract(base, token, contract, timeoutMs) {
const started = Date.now();
- const seed = Math.floor(Math.random() * 1e6);
// A contract whose fallback is a last-good tier needs something to fall back
// TO. One clean request first is not cheating: it is the state every real
// user's request arrives in, and a cold process with no memory is a different
// contract (the honest 503) that other entries assert.
+ //
+ // The warm-up deliberately uses a DIFFERENT bust value than the attempt. Warm
+ // and attempt on the same key would populate the very fresh-cache entry the
+ // attempt then hits, and the fault would never reach an upstream: the exact
+ // false green this prover exists to refuse. Different keys leave the
+ // attempt's fresh slot cold while still filling the shared per-entity
+ // last-good tiers that a real user's request would have filled.
if (contract.warm) {
try {
- await fetch(contractUrl(base, contract, seed), { signal: AbortSignal.timeout(timeoutMs) });
+ await fetch(contractUrl(base, contract, 'warm'), { signal: AbortSignal.timeout(timeoutMs) });
} catch {
/* a failed warm-up is not a verdict; the real attempt below is */
}
}
- const url = contractUrl(base, contract, seed);
+ const url = contractUrl(base, contract, 'attempt');
let res;
try {
res = await fetch(url, {
diff --git a/tests/brownout-prover.test.js b/tests/brownout-prover.test.js
new file mode 100644
index 000000000..2208cf400
--- /dev/null
+++ b/tests/brownout-prover.test.js
@@ -0,0 +1,125 @@
+// The prover's judgement, tested directly. Its whole value rests on refusing to
+// call a fallback proven when the fault never reached the code, so that refusal
+// is the first thing covered here.
+import { describe, it, expect } from 'vitest';
+import { parseProvenance, wasExercised, judge, contractUrl, chaosDirective } from '../scripts/prove-brownout.mjs';
+
+const headers = (summary, trace) =>
+ new Headers({ ...(summary ? { 'x-brownout': summary } : {}), ...(trace ? { 'x-brownout-trace': trace } : {}) });
+
+describe('parseProvenance', () => {
+ it('reads the summary and the per-source trace', () => {
+ const p = parseProvenance(
+ headers('v=1;tier=stale;sources=3;ok=1;failed=2;ms=512;degraded=1', 'birdeye;o=429;t=412, dex;o=ok;t=88'),
+ );
+ expect(p).toMatchObject({ tier: 'stale', degraded: true, ok: 1, failed: 2, ms: 512 });
+ expect(p.sources).toEqual([
+ { name: 'birdeye', outcome: '429', ms: 412 },
+ { name: 'dex', outcome: 'ok', ms: 88 },
+ ]);
+ });
+
+ it('treats a missing header as "recorded nothing", not as an error', () => {
+ const p = parseProvenance(headers(null, null));
+ expect(p.tier).toBeNull();
+ expect(p.sources).toEqual([]);
+ expect(p.degraded).toBe(false);
+ });
+});
+
+describe('wasExercised', () => {
+ const prov = parseProvenance(headers('v=1;tier=live', 'birdeye:txs;o=429;t=1, dex;o=ok;t=2'));
+
+ it('counts a source that failed', () => {
+ expect(wasExercised(prov, 'birdeye:txs')).toBe(true);
+ });
+
+ it('matches a sub-scoped source from the provider name the contract broke', () => {
+ // A contract says "break birdeye"; the code calls it as birdeye:txs.
+ expect(wasExercised(prov, 'birdeye')).toBe(true);
+ });
+
+ it('does not count a source that succeeded, or one that never appeared', () => {
+ expect(wasExercised(prov, 'dex')).toBe(false);
+ expect(wasExercised(prov, 'coingecko')).toBe(false);
+ });
+});
+
+describe('judge', () => {
+ const contract = {
+ expect: { status: 200, tier: ['stale', 'fallback'], degraded: true, exercised: ['birdeye'], jsonHas: ['tokens'] },
+ };
+ const good = {
+ status: 200,
+ prov: parseProvenance(headers('v=1;tier=stale;ok=1;failed=1;degraded=1', 'birdeye;o=429;t=1, cache;o=ok;t=1')),
+ body: { tokens: [] },
+ chaosStatus: 'applied;faults=1',
+ };
+
+ it('passes a contract whose fault landed and whose expectations held', () => {
+ expect(judge(contract, good)).toMatchObject({ verdict: 'pass', problems: [] });
+ });
+
+ it('refuses to pass a fallback whose fault never reached the code', () => {
+ // The single most important behaviour here: a warm cache answered, no
+ // upstream was called, and calling that a pass would certify a fallback
+ // that never ran.
+ const unexercised = { ...good, prov: parseProvenance(headers('v=1;tier=cache;ok=1;failed=0', null)) };
+ const out = judge(contract, unexercised);
+ expect(out.verdict).toBe('not_exercised');
+ expect(out.problems[0]).toMatch(/never appears as a failed source/);
+ });
+
+ it('refuses when the server declined the directive, rather than reading the result', () => {
+ const refused = { ...good, chaosStatus: 'refused;reason=bad_token' };
+ expect(judge(contract, refused)).toMatchObject({ verdict: 'not_exercised' });
+ });
+
+ it('reports each broken expectation separately', () => {
+ const bad = {
+ status: 502,
+ prov: parseProvenance(headers('v=1;tier=live;ok=0;failed=1', 'birdeye;o=429;t=1')),
+ body: {},
+ chaosStatus: 'applied;faults=1',
+ };
+ const out = judge(contract, bad);
+ expect(out.verdict).toBe('fail');
+ expect(out.problems.join(' ')).toMatch(/status 502/);
+ expect(out.problems.join(' ')).toMatch(/tier live/);
+ expect(out.problems.join(' ')).toMatch(/missing `tokens`/);
+ });
+
+ it('accepts any of several allowed statuses', () => {
+ const c = { expect: { status: [200, 503], exercised: [] } };
+ for (const status of [200, 503]) {
+ expect(judge(c, { ...good, status }).verdict).toBe('pass');
+ }
+ expect(judge(c, { ...good, status: 500 }).verdict).toBe('fail');
+ });
+});
+
+describe('contractUrl', () => {
+ const contract = {
+ endpoint: '/api/intel/heatmap',
+ query: { limit: 6 },
+ bust: { param: 'limit', warm: 6, attempt: 40 },
+ };
+
+ it('sends warm and attempt to different cache keys, in the direction the contract states', () => {
+ // Caches are not symmetric: the heatmap answers any smaller limit out of a
+ // larger cached field, so only warm-small then attempt-large actually misses.
+ expect(contractUrl('http://x', contract, 'warm')).toContain('limit=6');
+ expect(contractUrl('http://x', contract, 'attempt')).toContain('limit=40');
+ });
+
+ it('applies the plain query when a contract declares no bust', () => {
+ const url = contractUrl('http://x', { endpoint: '/api/coin/detail', query: { id: 'solana' } });
+ expect(url).toBe('http://x/api/coin/detail?id=solana');
+ });
+});
+
+describe('chaosDirective', () => {
+ it('renders the header a contract asks for', () => {
+ expect(chaosDirective({ break: { birdeye: 'http:429', dex: 'timeout' } })).toBe('birdeye=http:429, dex=timeout');
+ });
+});