Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(brownout): prove each declared fallback by breaking the upstream it protects against
nirholasaa7782a9978b
diff --git a/data/brownout.json b/data/brownout.jsonindex 2c4897514..f1a5ab3af 100644--- a/data/brownout.json+++ b/data/brownout.json@@ -8,57 +8,103 @@"title": "The intel heatmap keeps its tiles when pump.fun throttles","surface": "/intel","endpoint": "/api/intel/heatmap",- "query": { "limit": 6 },- "bust": { "param": "limit", "min": 6, "max": 40 },+ "query": {+ "limit": 6+ },+ "bust": {+ "param": "limit",+ "warm": 6,+ "attempt": 40+ },"warm": true,- "break": { "intel-heatmap": "http:429" },+ "break": {+ "intel-heatmap": "http:429"+ },"expect": {"status": 200,- "tier": ["cache", "stale", "fallback"],+ "tier": [+ "cache",+ "stale",+ "fallback"+ ],"degraded": true,- "exercised": ["intel-heatmap"],- "jsonHas": ["tokens"]+ "exercised": [+ "intel-heatmap"+ ],+ "jsonHas": [+ "tokens"+ ]},"why": "pump.fun answers a burst with 429 far more often than it goes down, and the board is the whole page. The retry honours Retry-After and the last-good tier carries the tiles through what is left, so a throttle costs freshness rather than the page."},{- "id": "fear-greed-survives-its-source",- "title": "The market page keeps its fear and greed reading when alternative.me is down",- "surface": "/markets",- "endpoint": "/api/coin/global",- "warm": true,- "break": { "alternative-me-fng": "network" },+ "id": "coin-page-survives-coingecko",+ "title": "A coin page still renders when CoinGecko is rate limiting",+ "surface": "/coin/:id",+ "endpoint": "/api/coin/detail",+ "query": {+ "id": "avalanche-2"+ },+ "warm": false,+ "break": {+ "coingecko": "http:429"+ },"expect": {"status": 200,- "exercised": ["alternative-me-fng"]+ "degraded": true,+ "exercised": [+ "coingecko"+ ],+ "jsonHas": [+ "coin"+ ]},- "why": "A single keyless index feed. The page is assembled from several independent reads, so one of them failing must cost that one figure and nothing else."+ "why": "CoinGecko's keyless tier is shared per egress IP and throttles on its own schedule, which is why /coin/:id 502'd for hours on 2026-07-28. CoinPaprika supplies the profile and DefiLlama the price series, so a throttle costs the page its primary source and nothing else."},{- "id": "news-archive-survives-gcs",- "title": "The news archive still answers when its bucket does not",- "surface": "/news",- "endpoint": "/api/news/archive",- "query": { "limit": 5 },- "warm": true,- "break": { "news-archive": "http:503" },+ "id": "holders-survive-a-dead-rpc-lane",+ "title": "Holder concentration survives a Solana RPC lane refusing",+ "surface": "/token/:mint",+ "endpoint": "/api/crypto/holders",+ "query": {+ "address": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump"+ },+ "warm": false,+ "break": {+ "solana-rpc": "http:429"+ },"expect": {- "status": [200, 503],- "exercised": ["news-archive"]+ "status": [+ 200,+ 503+ ],+ "exercised": [+ "solana-rpc"+ ]},- "why": "Archive data is immutable once written, which is what makes a remembered copy exactly as correct as a live one. With nothing ever cached the honest answer is a retryable 503, so both outcomes are accepted and the tier says which happened."+ "why": "Every lane in the rotation throttles at some point, and free lanes refuse specific call shapes outright. The rotation walks them, demotes a lane per method rather than benching it wholesale, and answers an idempotent read from its last-good body when the whole chain is down."},{"id": "token-page-survives-dexscreener","title": "A token page still resolves when DexScreener is throttling","surface": "/ca2x402","endpoint": "/api/ca2x402/resolve",- "query": { "mint": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump" },+ "query": {+ "mint": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump"+ },"warm": true,- "break": { "dexscreener:tokens": "http:429" },+ "break": {+ "dexscreener:tokens": "http:429"+ },"expect": {- "status": [200, 404, 503],- "exercised": ["dexscreener:tokens"]+ "status": [+ 200,+ 404,+ 503+ ],+ "exercised": [+ "dexscreener:tokens"+ ]},"why": "One keyless read used to decide whether a token exists at all, so a throttle made a live token read as not found. It now retries behind a breaker with a last-known-good payload, and a definitive miss is reserved for cases where a source actually spoke."}diff --git a/public/brownout.json b/public/brownout.jsonnew file mode 100644index 000000000..c4366f49f--- /dev/null+++ b/public/brownout.json@@ -0,0 +1,171 @@+{+ "generated_at": "2026-08-28T03:33:28.080Z",+ "base": "http://127.0.0.1:8791",+ "summary": "Every fallback below has been executed against a running server with the upstream it protects against genuinely broken.",+ "counts": {+ "pass": 4+ },+ "proven": 4,+ "total": 4,+ "results": [+ {+ "id": "heatmap-survives-pump-throttle",+ "title": "The intel heatmap keeps its tiles when pump.fun throttles",+ "surface": "/intel",+ "endpoint": "/api/intel/heatmap",+ "broke": {+ "intel-heatmap": "http:429"+ },+ "why": "pump.fun answers a burst with 429 far more often than it goes down, and the board is the whole page. The retry honours Retry-After and the last-good tier carries the tiles through what is left, so a throttle costs freshness rather than the page.",+ "verdict": "pass",+ "problems": [],+ "ms": 4626,+ "observed": {+ "status": 200,+ "tier": "stale",+ "degraded": true,+ "failedSources": 3,+ "trace": [+ {+ "name": "intel-heatmap",+ "outcome": "429",+ "ms": 2120+ },+ {+ "name": "intel-heatmap",+ "outcome": "429",+ "ms": 2226+ },+ {+ "name": "lastgood:heatmap:meta:FeMbDoX7R1Psc4GEcvJdsbNbZA",+ "outcome": "stale",+ "ms": 19354+ },+ {+ "name": "intel-heatmap",+ "outcome": "429",+ "ms": 2273+ }+ ]+ }+ },+ {+ "id": "coin-page-survives-coingecko",+ "title": "A coin page still renders when CoinGecko is rate limiting",+ "surface": "/coin/:id",+ "endpoint": "/api/coin/detail",+ "broke": {+ "coingecko": "http:429"+ },+ "why": "CoinGecko's keyless tier is shared per egress IP and throttles on its own schedule, which is why /coin/:id 502'd for hours on 2026-07-28. CoinPaprika supplies the profile and DefiLlama the price series, so a throttle costs the page its primary source and nothing else.",+ "verdict": "pass",+ "problems": [],+ "ms": 12,+ "observed": {+ "status": 200,+ "tier": "live",+ "degraded": true,+ "failedSources": 1,+ "trace": [+ {+ "name": "coingecko",+ "outcome": "429",+ "ms": 1+ }+ ]+ }+ },+ {+ "id": "holders-survive-a-dead-rpc-lane",+ "title": "Holder concentration survives a Solana RPC lane refusing",+ "surface": "/token/:mint",+ "endpoint": "/api/crypto/holders",+ "broke": {+ "solana-rpc": "http:429"+ },+ "why": "Every lane in the rotation throttles at some point, and free lanes refuse specific call shapes outright. The rotation walks them, demotes a lane per method rather than benching it wholesale, and answers an idempotent read from its last-good body when the whole chain is down.",+ "verdict": "pass",+ "problems": [],+ "ms": 20,+ "observed": {+ "status": 200,+ "tier": "stale",+ "degraded": true,+ "failedSources": 17,+ "trace": [+ {+ "name": "solana-rpc:solana-rpc.publicnode.com",+ "outcome": "429",+ "ms": 1+ },+ {+ "name": "solana-rpc:api.mainnet-beta.solana.com",+ "outcome": "429",+ "ms": 0+ },+ {+ "name": "solana-rpc:solana.leorpc.com",+ "outcome": "429",+ "ms": 0+ },+ {+ "name": "solana-rpc:public.rpc.solanavibestation.com",+ "outcome": "429",+ "ms": 3+ },+ {+ "name": "solana-rpc:api.tatum.io",+ "outcome": "429",+ "ms": 0+ },+ {+ "name": "solana-rpc:solana-mainnet.gateway.tatum.io",+ "outcome": "429",+ "ms": 0+ },+ {+ "name": "solana-rpc:last-good",+ "outcome": "stale",+ "ms": 97466+ },+ {+ "name": "solana-rpc:solana-rpc.publicnode.com",+ "outcome": "429",+ "ms": 0+ }+ ]+ }+ },+ {+ "id": "token-page-survives-dexscreener",+ "title": "A token page still resolves when DexScreener is throttling",+ "surface": "/ca2x402",+ "endpoint": "/api/ca2x402/resolve",+ "broke": {+ "dexscreener:tokens": "http:429"+ },+ "why": "One keyless read used to decide whether a token exists at all, so a throttle made a live token read as not found. It now retries behind a breaker with a last-known-good payload, and a definitive miss is reserved for cases where a source actually spoke.",+ "verdict": "pass",+ "problems": [],+ "ms": 2252,+ "observed": {+ "status": 200,+ "tier": "stale",+ "degraded": true,+ "failedSources": 1,+ "trace": [+ {+ "name": "dexscreener:tokens",+ "outcome": "429",+ "ms": 2155+ },+ {+ "name": "lastgood:token-market:fembdox7r1psc4gecvjdsbnbza",+ "outcome": "stale",+ "ms": 2166+ }+ ]+ }+ }+ ]+}diff --git a/scripts/prove-brownout.mjs b/scripts/prove-brownout.mjsindex da193bdc3..7a2fb76b5 100644--- a/scripts/prove-brownout.mjs+++ b/scripts/prove-brownout.mjs@@ -86,16 +86,17 @@ export function wasExercised(prov, name) {}/** Build the request URL, applying the contract's cache-busting parameter. */-export function contractUrl(base, contract, seed = Date.now()) {+export function contractUrl(base, contract, phase = 'attempt') {const url = new URL(contract.endpoint, base);for (const [k, v] of Object.entries(contract.query || {})) url.searchParams.set(k, String(v));const bust = contract.bust;if (bust?.param) {- // A warm cache is the single biggest source of a false green here, so the- // request is deliberately steered off whatever key the last one used. The- // range is the contract's, because only it knows what stays meaningful.- const span = Math.max(1, (bust.max ?? 10) - (bust.min ?? 1) + 1);- url.searchParams.set(bust.param, String((bust.min ?? 1) + (seed % span)));+ // A warm cache is the single biggest source of a false green here, so warm+ // and attempt are steered onto DIFFERENT keys, and which value goes where+ // is the contract's to state rather than the prover's to guess. Caches are+ // not symmetric: the heatmap answers any smaller limit out of a larger+ // cached field, so only warm-small then attempt-large actually misses.+ url.searchParams.set(bust.param, String(phase === 'warm' ? bust.warm : bust.attempt));}return url.toString();}@@ -149,21 +150,27 @@ export function judge(contract, { status, prov, body, chaosStatus }) {async function runContract(base, token, contract, timeoutMs) {const started = Date.now();- const seed = Math.floor(Math.random() * 1e6);// A contract whose fallback is a last-good tier needs something to fall back// TO. One clean request first is not cheating: it is the state every real// user's request arrives in, and a cold process with no memory is a different// contract (the honest 503) that other entries assert.+ //+ // The warm-up deliberately uses a DIFFERENT bust value than the attempt. Warm+ // and attempt on the same key would populate the very fresh-cache entry the+ // attempt then hits, and the fault would never reach an upstream: the exact+ // false green this prover exists to refuse. Different keys leave the+ // attempt's fresh slot cold while still filling the shared per-entity+ // last-good tiers that a real user's request would have filled.if (contract.warm) {try {- await fetch(contractUrl(base, contract, seed), { signal: AbortSignal.timeout(timeoutMs) });+ await fetch(contractUrl(base, contract, 'warm'), { signal: AbortSignal.timeout(timeoutMs) });} catch {/* a failed warm-up is not a verdict; the real attempt below is */}}- const url = contractUrl(base, contract, seed);+ const url = contractUrl(base, contract, 'attempt');let res;try {res = await fetch(url, {diff --git a/tests/brownout-prover.test.js b/tests/brownout-prover.test.jsnew file mode 100644index 000000000..2208cf400--- /dev/null+++ b/tests/brownout-prover.test.js@@ -0,0 +1,125 @@+// The prover's judgement, tested directly. Its whole value rests on refusing to+// call a fallback proven when the fault never reached the code, so that refusal+// is the first thing covered here.+import { describe, it, expect } from 'vitest';+import { parseProvenance, wasExercised, judge, contractUrl, chaosDirective } from '../scripts/prove-brownout.mjs';++const headers = (summary, trace) =>+ new Headers({ ...(summary ? { 'x-brownout': summary } : {}), ...(trace ? { 'x-brownout-trace': trace } : {}) });++describe('parseProvenance', () => {+ it('reads the summary and the per-source trace', () => {+ const p = parseProvenance(+ headers('v=1;tier=stale;sources=3;ok=1;failed=2;ms=512;degraded=1', 'birdeye;o=429;t=412, dex;o=ok;t=88'),+ );+ expect(p).toMatchObject({ tier: 'stale', degraded: true, ok: 1, failed: 2, ms: 512 });+ expect(p.sources).toEqual([+ { name: 'birdeye', outcome: '429', ms: 412 },+ { name: 'dex', outcome: 'ok', ms: 88 },+ ]);+ });++ it('treats a missing header as "recorded nothing", not as an error', () => {+ const p = parseProvenance(headers(null, null));+ expect(p.tier).toBeNull();+ expect(p.sources).toEqual([]);+ expect(p.degraded).toBe(false);+ });+});++describe('wasExercised', () => {+ const prov = parseProvenance(headers('v=1;tier=live', 'birdeye:txs;o=429;t=1, dex;o=ok;t=2'));++ it('counts a source that failed', () => {+ expect(wasExercised(prov, 'birdeye:txs')).toBe(true);+ });++ it('matches a sub-scoped source from the provider name the contract broke', () => {+ // A contract says "break birdeye"; the code calls it as birdeye:txs.+ expect(wasExercised(prov, 'birdeye')).toBe(true);+ });++ it('does not count a source that succeeded, or one that never appeared', () => {+ expect(wasExercised(prov, 'dex')).toBe(false);+ expect(wasExercised(prov, 'coingecko')).toBe(false);+ });+});++describe('judge', () => {+ const contract = {+ expect: { status: 200, tier: ['stale', 'fallback'], degraded: true, exercised: ['birdeye'], jsonHas: ['tokens'] },+ };+ const good = {+ status: 200,+ prov: parseProvenance(headers('v=1;tier=stale;ok=1;failed=1;degraded=1', 'birdeye;o=429;t=1, cache;o=ok;t=1')),+ body: { tokens: [] },+ chaosStatus: 'applied;faults=1',+ };++ it('passes a contract whose fault landed and whose expectations held', () => {+ expect(judge(contract, good)).toMatchObject({ verdict: 'pass', problems: [] });+ });++ it('refuses to pass a fallback whose fault never reached the code', () => {+ // The single most important behaviour here: a warm cache answered, no+ // upstream was called, and calling that a pass would certify a fallback+ // that never ran.+ const unexercised = { ...good, prov: parseProvenance(headers('v=1;tier=cache;ok=1;failed=0', null)) };+ const out = judge(contract, unexercised);+ expect(out.verdict).toBe('not_exercised');+ expect(out.problems[0]).toMatch(/never appears as a failed source/);+ });++ it('refuses when the server declined the directive, rather than reading the result', () => {+ const refused = { ...good, chaosStatus: 'refused;reason=bad_token' };+ expect(judge(contract, refused)).toMatchObject({ verdict: 'not_exercised' });+ });++ it('reports each broken expectation separately', () => {+ const bad = {+ status: 502,+ prov: parseProvenance(headers('v=1;tier=live;ok=0;failed=1', 'birdeye;o=429;t=1')),+ body: {},+ chaosStatus: 'applied;faults=1',+ };+ const out = judge(contract, bad);+ expect(out.verdict).toBe('fail');+ expect(out.problems.join(' ')).toMatch(/status 502/);+ expect(out.problems.join(' ')).toMatch(/tier live/);+ expect(out.problems.join(' ')).toMatch(/missing `tokens`/);+ });++ it('accepts any of several allowed statuses', () => {+ const c = { expect: { status: [200, 503], exercised: [] } };+ for (const status of [200, 503]) {+ expect(judge(c, { ...good, status }).verdict).toBe('pass');+ }+ expect(judge(c, { ...good, status: 500 }).verdict).toBe('fail');+ });+});++describe('contractUrl', () => {+ const contract = {+ endpoint: '/api/intel/heatmap',+ query: { limit: 6 },+ bust: { param: 'limit', warm: 6, attempt: 40 },+ };++ it('sends warm and attempt to different cache keys, in the direction the contract states', () => {+ // Caches are not symmetric: the heatmap answers any smaller limit out of a+ // larger cached field, so only warm-small then attempt-large actually misses.+ expect(contractUrl('http://x', contract, 'warm')).toContain('limit=6');+ expect(contractUrl('http://x', contract, 'attempt')).toContain('limit=40');+ });++ it('applies the plain query when a contract declares no bust', () => {+ const url = contractUrl('http://x', { endpoint: '/api/coin/detail', query: { id: 'solana' } });+ expect(url).toBe('http://x/api/coin/detail?id=solana');+ });+});++describe('chaosDirective', () => {+ it('renders the header a contract asks for', () => {+ expect(chaosDirective({ break: { birdeye: 'http:429', dex: 'timeout' } })).toBe('birdeye=http:429, dex=timeout');+ });+});