Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(brownout): stamp every API response with where its data came from

nirholas9d50375be741
diff --git a/api/_lib/brownout/provenance.js b/api/_lib/brownout/provenance.js
index 6cef43451..7094fc50a 100644
--- a/api/_lib/brownout/provenance.js
+++ b/api/_lib/brownout/provenance.js
@@ -139,10 +139,13 @@ export function provenanceSummary() {
ok,
failed,
ms: Date.now() - ledger.startedAt,
- // Degraded means "a reader should not treat this as a clean live answer":
- // either something had to be retried or failed over, or what came back is
- // not fresh. Both are worth surfacing; neither is an error.
- degraded: failed > 0 || tier !== 'live',
+ // Degraded means "this answer is worse than the one the endpoint intends to
+ // give": something had to fail over, or what came back is older than its
+ // own TTL allows. A `cache` hit is NOT degraded, and getting that wrong
+ // would be fatal to the signal: almost every response on a healthy site is
+ // a cache hit, so counting those would mark the whole site degraded and
+ // train every reader to ignore the flag.
+ degraded: failed > 0 || tier === 'stale' || tier === 'fallback',
truncated: ledger.truncated || 0,
records: ledger.sources,
};
diff --git a/api/_lib/coingecko.js b/api/_lib/coingecko.js
index 61be0c9a9..de2c27886 100644
--- a/api/_lib/coingecko.js
+++ b/api/_lib/coingecko.js
@@ -8,6 +8,8 @@
// the upstream from concurrent cold-instance misses.
import { cacheGet, cacheSet } from './cache.js';
+import { recordSource } from './brownout/provenance.js';
+import { applyFault, faultFor } from './brownout/chaos.js';
import { createCache } from './mem-cache.js';
export const COINGECKO_BASE = 'https://api.coingecko.com/api/v3';
@@ -104,15 +106,31 @@ function keyWasRejected(headersUsed, status) {
export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}) {
const now = Date.now();
const hit = _cache.get(path);
- if (hit && hit.expiresAt > now) return hit.value;
+ if (hit && hit.expiresAt > now) {
+ // A cache hit inside its TTL is `cache`, not `live`: the answer is correct
+ // and intended, but it did not come off the wire during this request, and a
+ // reader deciding how much to trust a number deserves to know which.
+ recordSource({ name: 'coingecko', outcome: 'ok', ms: 0, tier: 'cache' });
+ return hit.value;
+ }
// A still-usable stale entry lets us ride out a throttled/failing upstream.
const stale = hit && hit.staleUntil > now ? hit.value : null;
const url = `${COINGECKO_BASE}${path}`;
+ const startedAt = now;
let resp;
try {
const sent = geckoHeaders();
- resp = await fetch(url, { headers: sent, signal: AbortSignal.timeout(timeoutMs) });
+ // CoinGecko's keyless tier is shared per egress IP and throttles constantly,
+ // which makes it the single most useful upstream to be able to break on
+ // purpose: most of this module exists to survive exactly that.
+ const fault = faultFor('coingecko');
+ if (fault) {
+ const injected = await applyFault(fault, url);
+ resp = injected ?? (await fetch(url, { headers: sent, signal: AbortSignal.timeout(timeoutMs) }));
+ } else {
+ resp = await fetch(url, { headers: sent, signal: AbortSignal.timeout(timeoutMs) });
+ }
// An exhausted/revoked demo key rejects requests the keyless tier would
// still answer. Bench the key and retry once without it before falling
// back to stale data — the live payload beats a cached one.
@@ -129,14 +147,17 @@ export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}
// Network failure or timeout — serve stale if we have it, else surface.
if (stale !== null) {
console.warn(`[coingecko] ${path} fetch failed (${netErr?.name || 'error'}); serving stale`);
+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: 'stale' });
return stale;
}
const durable = await durableStale(path);
if (durable !== null) {
console.warn(`[coingecko] ${path} fetch failed (${netErr?.name || 'error'}); serving durable last-good`);
_cache.set(path, { value: durable, expiresAt: now, staleUntil: now + STALE_MS });
+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: 'durable' });
return durable;
}
+ recordSource({ name: 'coingecko', outcome: 'fail', ms: Date.now() - startedAt, tier: 'live', detail: netErr?.name === 'TimeoutError' ? 'timeout' : 'network' });
throw netErr;
}
if (!resp.ok) {
@@ -145,6 +166,7 @@ export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}
// recent value beats an outage — serve it and keep the page alive.
if (resp.status !== 404 && stale !== null) {
console.warn(`[coingecko] ${path} → ${resp.status}; serving stale`);
+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: resp.status });
return stale;
}
if (resp.status !== 404) {
@@ -154,14 +176,17 @@ export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}
if (durable !== null) {
console.warn(`[coingecko] ${path} → ${resp.status}; serving durable last-good`);
_cache.set(path, { value: durable, expiresAt: now, staleUntil: now + STALE_MS });
+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: resp.status });
return durable;
}
}
+ recordSource({ name: 'coingecko', outcome: 'fail', ms: Date.now() - startedAt, tier: 'live', detail: resp.status });
const err = new Error(`CoinGecko ${resp.status} for ${path}`);
err.status = resp.status;
throw err;
}
const value = await resp.json();
+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'live' });
_cache.set(path, { value, expiresAt: now + ttlMs, staleUntil: now + ttlMs + STALE_MS });
// Fire-and-forget durable mirror, never let a cache fault fail a live fetch.
Promise.resolve(cacheSet(durableKey(path), value, DURABLE_STALE_S)).catch(() => {});
diff --git a/api/_lib/http.js b/api/_lib/http.js
index ca90ce984..e0feb76c6 100644
--- a/api/_lib/http.js
+++ b/api/_lib/http.js
@@ -8,6 +8,8 @@ import { sendOpsAlert } from './alerts.js';
import { instrument as zauthInstrument, drain as zauthDrain } from './zauth.js';
import { isDbUnavailableError, isDbCapacityError, isStoragePressured } from './db.js';
import { redactUrlSecrets } from './scrub-secrets.js';
+import { provenanceHeaders, withProvenance } from './brownout/provenance.js';
+import { CHAOS_HEADER, CHAOS_STATUS_HEADER, chaosDecision, withChaos } from './brownout/chaos.js';
// Secure-by-default caching: emit `no-store` UNLESS the handler already set a
// Cache-Control header (e.g. `res.setHeader('cache-control', 'public, s-maxage=…')`
@@ -541,10 +543,49 @@ function isAllowedOrigin(origin, allowed) {
return allowed.some((pat) => (typeof pat === 'string' ? origin === pat : pat.test(origin)));
}
+// Attach the request's data provenance to the response, and honour a chaos
+// directive if this request carries a valid one.
+//
+// This sits in wrap() because wrap() is the one seam every handler already goes
+// through: putting it here means a new endpoint reports its provenance without
+// its author doing anything, which is the only way a guarantee like this stays
+// true across hundreds of handlers. Headers are written just before the body
+// goes out; a response already streaming keeps whatever it had, since headers
+// cannot be added after the fact and a throw here must never cost a caller a
+// working answer.
+function attachProvenance(res) {
+ try {
+ if (res.headersSent || typeof res.setHeader !== 'function') return;
+ const headers = provenanceHeaders();
+ if (!headers) return;
+ res.setHeader('x-brownout', headers.summary);
+ if (headers.trace) res.setHeader('x-brownout-trace', headers.trace);
+ // Browsers cannot read a response header unless it is exposed, and the
+ // whole point of this is that a client can tell a degraded answer from a
+ // live one.
+ const exposed = res.getHeader('access-control-expose-headers');
+ const want = headers.trace ? 'x-brownout, x-brownout-trace' : 'x-brownout';
+ res.setHeader('access-control-expose-headers', exposed ? `${exposed}, ${want}` : want);
+ } catch {
+ /* provenance is telemetry: it never breaks a response */
+ }
+}
+
// Wrap async handlers so uncaught errors return a consistent JSON envelope.
export function wrap(handler) {
return async (req, res, ...rest) => {
const monitored = zauthInstrument(req, res);
+ const chaos = chaosDecision(req);
+ if (req?.headers?.[CHAOS_HEADER] && typeof res.setHeader === 'function') {
+ // Always answer the question the caller asked, including when the answer
+ // is no. A prover that believes it broke an upstream and did not would
+ // record a green proof for a fallback nothing exercised, which is worse
+ // than a red one.
+ try {
+ res.setHeader(CHAOS_STATUS_HEADER, chaos.allowed ? `applied;faults=${chaos.faults.size}` : `refused;reason=${chaos.reason}`);
+ } catch { /* header write is best-effort */ }
+ }
+ const run = () => withProvenance(() => withChaos(chaos.faults, async () => {
try {
await handler(req, res, ...rest);
} catch (err) {
@@ -654,6 +695,25 @@ export function wrap(handler) {
}
}
}
+ }));
+ // The provenance header has to be written while the ledger is still in
+ // scope AND before the body is flushed, so the res.end path is patched for
+ // the duration of the handler rather than read afterwards. Every helper in
+ // this module funnels through res.end, so one seam covers json(), error()
+ // and a hand-rolled stream alike.
+ const originalEnd = typeof res.end === 'function' ? res.end.bind(res) : null;
+ if (originalEnd) {
+ res.end = (...args) => {
+ attachProvenance(res);
+ res.end = originalEnd;
+ return originalEnd(...args);
+ };
+ }
+ try {
+ await run();
+ } finally {
+ if (originalEnd) res.end = originalEnd;
+ }
// Keep the lambda alive briefly so the zauth SDK's in-flight POST to
// back.zauthx402.com can finish. Cost: ~250ms of post-response runtime
// on monitored requests only. The user has already received the response.
diff --git a/api/_lib/upstream-fetch.js b/api/_lib/upstream-fetch.js
index f7fda4c05..64c285075 100644
--- a/api/_lib/upstream-fetch.js
+++ b/api/_lib/upstream-fetch.js
@@ -35,6 +35,8 @@
import { withRetry, withBreaker, parseRetryAfter, isRetryableError } from './resilience.js';
import { createCache } from './mem-cache.js';
+import { recordSource } from './brownout/provenance.js';
+import { applyFault, faultFor } from './brownout/chaos.js';
export const DEFAULT_TIMEOUT_MS = 8_000;
const DEFAULT_ATTEMPTS = 3;
@@ -123,10 +125,21 @@ export async function fetchUpstream(url, init = {}, opts = {}) {
} = opts;
const shown = safeUrl(url);
+ const t0 = Date.now();
const attempt = async () => {
let res;
try {
- res = await fetch(url, { ...init, signal: composeSignal(init.signal, timeoutMs) });
+ // Brownout: a fault declared for this upstream, in this request only,
+ // is raised in the shape of the real failure it stands in for, from
+ // inside the same try that would have caught the genuine one. That
+ // placement is the whole point: retries, Retry-After handling and the
+ // breaker below all see it exactly as they would see the outage.
+ const fault = faultFor(name || shown);
+ if (fault) {
+ const injected = await applyFault(fault, shown);
+ if (injected) res = injected;
+ }
+ if (!res) res = await fetch(url, { ...init, signal: composeSignal(init.signal, timeoutMs) });
} catch (err) {
if (init.signal?.aborted) throw err;
const e = new UpstreamError(`${shown}: ${err?.name === 'TimeoutError' ? `timed out after ${timeoutMs}ms` : err?.message || 'network error'}`, { url: shown, cause: err });
@@ -162,8 +175,27 @@ export async function fetchUpstream(url, init = {}, opts = {}) {
const shouldRetry = (err) => !err?.tooLongToWait && !init.signal?.aborted && isRetryableError(err);
const run = () => withRetry(attempt, { attempts, shouldRetry, label });
+ // Every third-party call in api/ goes through here, which makes this the one
+ // place that can record what actually happened without every call site
+ // remembering to. `live` because a value that reaches this return came off
+ // the wire during this request; the cache tiers stamp their own.
+ const label_ = name || shown;
+ const noteOk = (v) => {
+ recordSource({ name: label_, outcome: 'ok', ms: Date.now() - t0, tier: 'live' });
+ return v;
+ };
+ const noteFail = (err) => {
+ recordSource({
+ name: label_,
+ outcome: 'fail',
+ ms: Date.now() - t0,
+ tier: 'live',
+ detail: err?.status ?? (err?.name === 'TimeoutError' ? 'timeout' : 'error'),
+ });
+ throw err;
+ };
- if (!name) return run();
+ if (!name) return run().then(noteOk, noteFail);
let failure;
const value = await withBreaker(name, run, {
threshold: breakerThreshold,
@@ -173,8 +205,8 @@ export async function fetchUpstream(url, init = {}, opts = {}) {
return null;
},
});
- if (value) return value;
- throw failure instanceof Error ? failure : new UpstreamError(`${label || name}: circuit open`, { url: shown, status: 503 });
+ if (value) return noteOk(value);
+ return noteFail(failure instanceof Error ? failure : new UpstreamError(`${label || name}: circuit open`, { url: shown, status: 503 }));
}
/**
@@ -269,6 +301,9 @@ export async function lastGood(key, load, opts = {}) {
const ageMs = hit ? Date.now() - hit.at : Infinity;
if (hit && ageMs <= maxAgeMs) {
onFallback?.(err, ageMs);
+ // The single most important thing a caller can know about a response:
+ // this value is older than it was meant to be, and by how much.
+ recordSource({ name: `lastgood:${key}`.slice(0, 48), outcome: 'ok', ms: ageMs, tier: 'stale', detail: 'stale' });
return { value: hit.value, stale: true, ageMs, error: err };
}
throw err;
diff --git a/data/brownout.json b/data/brownout.json
new file mode 100644
index 000000000..2c4897514
--- /dev/null
+++ b/data/brownout.json
@@ -0,0 +1,66 @@
+{
+ "$comment": "Declared degradation contracts. Each one names an upstream, the fault to inject, and what the endpoint must still do. scripts/prove-brownout.mjs executes them against a live server and writes the verdicts to public/brownout.json, which /brownout renders. A contract whose fault never reached the code is reported as not_exercised, never as a pass: a green proof for a fallback nothing ran is worse than no proof at all.",
+ "updated": "2026-08-27",
+ "summary": "Every fallback below has been executed against a running server with the upstream it protects against genuinely broken.",
+ "contracts": [
+ {
+ "id": "heatmap-survives-pump-throttle",
+ "title": "The intel heatmap keeps its tiles when pump.fun throttles",
+ "surface": "/intel",
+ "endpoint": "/api/intel/heatmap",
+ "query": { "limit": 6 },
+ "bust": { "param": "limit", "min": 6, "max": 40 },
+ "warm": true,
+ "break": { "intel-heatmap": "http:429" },
+ "expect": {
+ "status": 200,
+ "tier": ["cache", "stale", "fallback"],
+ "degraded": true,
+ "exercised": ["intel-heatmap"],
+ "jsonHas": ["tokens"]
+ },
+ "why": "pump.fun answers a burst with 429 far more often than it goes down, and the board is the whole page. The retry honours Retry-After and the last-good tier carries the tiles through what is left, so a throttle costs freshness rather than the page."
+ },
+ {
+ "id": "fear-greed-survives-its-source",
+ "title": "The market page keeps its fear and greed reading when alternative.me is down",
+ "surface": "/markets",
+ "endpoint": "/api/coin/global",
+ "warm": true,
+ "break": { "alternative-me-fng": "network" },
+ "expect": {
+ "status": 200,
+ "exercised": ["alternative-me-fng"]
+ },
+ "why": "A single keyless index feed. The page is assembled from several independent reads, so one of them failing must cost that one figure and nothing else."
+ },
+ {
+ "id": "news-archive-survives-gcs",
+ "title": "The news archive still answers when its bucket does not",
+ "surface": "/news",
+ "endpoint": "/api/news/archive",
+ "query": { "limit": 5 },
+ "warm": true,
+ "break": { "news-archive": "http:503" },
+ "expect": {
+ "status": [200, 503],
+ "exercised": ["news-archive"]
+ },
+ "why": "Archive data is immutable once written, which is what makes a remembered copy exactly as correct as a live one. With nothing ever cached the honest answer is a retryable 503, so both outcomes are accepted and the tier says which happened."
+ },
+ {
+ "id": "token-page-survives-dexscreener",
+ "title": "A token page still resolves when DexScreener is throttling",
+ "surface": "/ca2x402",
+ "endpoint": "/api/ca2x402/resolve",
+ "query": { "mint": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump" },
+ "warm": true,
+ "break": { "dexscreener:tokens": "http:429" },
+ "expect": {
+ "status": [200, 404, 503],
+ "exercised": ["dexscreener:tokens"]
+ },
+ "why": "One keyless read used to decide whether a token exists at all, so a throttle made a live token read as not found. It now retries behind a breaker with a last-known-good payload, and a definitive miss is reserved for cases where a source actually spoke."
+ }
+ ]
+}
diff --git a/scripts/audit-upstreams.mjs b/scripts/audit-upstreams.mjs
new file mode 100644
index 000000000..bbfdf06b2
--- /dev/null
+++ b/scripts/audit-upstreams.mjs
@@ -0,0 +1,273 @@
+#!/usr/bin/env node
+// Upstream resilience audit.
+//
+// npm run audit:upstreams human table, exits non-zero on a regression
+// npm run audit:upstreams -- --json machine-readable report
+// npm run audit:upstreams -- --map regenerate docs/resilience.md
+// npm run audit:upstreams -- --accept re-baseline after a deliberate change
+//
+// Why a ratchet instead of a threshold. Every third-party call in this repo was
+// once written without a deadline, and each was added by someone who had no way
+// to see that they were adding the fifty-ninth one. A number in a doc does not
+// stop that; a baseline that fails the build the moment an unguarded call
+// appears does. The baseline records the grade of every call site we know
+// about, so a NEW unguarded call fails while the existing tail is free to be
+// paid down at its own pace, and paying it down updates the baseline downward
+// automatically (it can never drift back up without --accept).
+//
+// See docs/resilience.md for the generated map and scripts/resilience/analyze.mjs
+// for how a call site is classified.
+
+import { readFileSync, writeFileSync, existsSync } from 'node:fs';
+import { resolve, dirname, relative } from 'node:path';
+import { fileURLToPath } from 'node:url';
+import { execSync } from 'node:child_process';
+import { analyzeFiles, gradeOf, byHost, PROTECTIONS } from './resilience/analyze.mjs';
+
+const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
+const BASELINE = resolve(ROOT, 'data/upstream-baseline.json');
+const MAP_DOC = resolve(ROOT, 'docs/resilience.md');
+
+const args = new Set(process.argv.slice(2));
+const asJson = args.has('--json');
+const writeMap = args.has('--map');
+const accept = args.has('--accept');
+
+// Directories that ship runtime code. Tests are deliberately excluded: a test
+// SHOULD be able to call a bare fetch against a stub.
+const SCAN_DIRS = ['api', 'src', 'server', 'workers', 'mcp-server/src', 'avatar-sdk/src', 'public'];
+// Built output is not source: grading a bundled asset reports findings nobody
+// can act on, because the fix belongs in the module it was compiled from.
+const SKIP_RE =
+ /(?:^|\/)(?:node_modules|dist|dist-lib|build|coverage|\.deploy-wt|assets|vendor)(?:\/|$)|\.min\.js$|\.test\.js$|\.spec\.js$|-[A-Za-z0-9_-]{8,}\.js$/;
+
+function sourceFiles() {
+ const out = [];
+ for (const dir of SCAN_DIRS) {
+ const abs = resolve(ROOT, dir);
+ if (!existsSync(abs)) continue;
+ let listing;
+ try {
+ listing = execSync(`find ${JSON.stringify(abs)} -type f \\( -name '*.js' -o -name '*.mjs' \\)`, {
+ encoding: 'utf8',
+ maxBuffer: 64 * 1024 * 1024,
+ });
+ } catch {
+ continue;
+ }
+ for (const line of listing.split('\n')) {
+ const f = line.trim();
+ if (!f) continue;
+ if (SKIP_RE.test(relative(ROOT, f))) continue;
+ out.push(f);
+ }
+ }
+ return out.sort();
+}
+
+/** Stable id for a call site, so the baseline survives edits above it. */
+function siteKey(site) {
+ return `${site.file}::${site.host}::${site.via}`;
+}
+
+function loadBaseline() {
+ if (!existsSync(BASELINE)) return null;
+ try {
+ return JSON.parse(readFileSync(BASELINE, 'utf8'));
+ } catch {
+ return null;
+ }
+}
+
+const GRADE_RANK = { F: 0, D: 1, C: 2, B: 3, A: 4 };
+
+function summarize(callSites) {
+ const grades = { A: 0, B: 0, C: 0, D: 0, F: 0 };
+ const counts = Object.fromEntries(PROTECTIONS.map((p) => [p, 0]));
+ for (const s of callSites) {
+ grades[gradeOf(s)] += 1;
+ for (const p of s.protections) if (p in counts) counts[p] += 1;
+ }
+ return { total: callSites.length, grades, protections: counts };
+}
+
+/** Worst grade per site key, which is what the baseline pins. */
+function gradeIndex(callSites) {
+ const idx = {};
+ for (const s of callSites) {
+ const k = siteKey(s);
+ const g = gradeOf(s);
+ if (!(k in idx) || GRADE_RANK[g] < GRADE_RANK[idx[k]]) idx[k] = g;
+ }
+ return idx;
+}
+
+function renderMap(callSites, summary) {
+ const hosts = byHost(callSites);
+ const worst = hosts.filter((h) => h.worst === 'F' || h.worst === 'D');
+ const lines = [];
+ lines.push('<!-- GENERATED by scripts/audit-upstreams.mjs. Run `npm run audit:upstreams -- --map`. -->');
+ lines.push('# Upstream resilience map');
+ lines.push('');
+ lines.push('Every third-party service this platform calls, and what happens to a user when');
+ lines.push('that service is down. Generated from the source by an AST pass, so it cannot');
+ lines.push('drift from the code the way a hand-written list does.');
+ lines.push('');
+ lines.push('## How a call site is graded');
+ lines.push('');
+ lines.push('| Grade | Meaning |');
+ lines.push('|---|---|');
+ lines.push('| A | Bounded, and survives the upstream being down (a second source, plus a remembered value or a retry) |');
+ lines.push('| B | Bounded, with one way to survive: either another source or a remembered value |');
+ lines.push('| C | Bounded, retries or trips a breaker, but has no second source |');
+ lines.push('| D | Bounded only: a deadline and nothing behind it |');
+ lines.push('| F | Unbounded: one stalled socket can outlive the request that opened it |');
+ lines.push('');
+ lines.push('A deadline is the floor rather than one grade among five, because without one');
+ lines.push('no other protection ever gets a turn.');
+ lines.push('');
+ lines.push('## Totals');
+ lines.push('');
+ lines.push(`${summary.total} outbound call sites across ${hosts.length} upstreams.`);
+ lines.push('');
+ lines.push('| A | B | C | D | F |');
+ lines.push('|---|---|---|---|---|');
+ lines.push(
+ `| ${summary.grades.A} | ${summary.grades.B} | ${summary.grades.C} | ${summary.grades.D} | ${summary.grades.F} |`,
+ );
+ lines.push('');
+ if (worst.length) {
+ lines.push('## Weakest upstreams');
+ lines.push('');
+ lines.push('These carry at least one call site with no second source behind it. They are');
+ lines.push('the list to work down, worst first.');
+ lines.push('');
+ lines.push('| Upstream | Call sites | Worst grade |');
+ lines.push('|---|---|---|');
+ for (const h of worst.slice(0, 25)) {
+ lines.push(`| \`${h.host}\` | ${h.sites} | ${h.worst} |`);
+ }
+ lines.push('');
+ }
+ lines.push('## Every upstream');
+ lines.push('');
+ lines.push('| Upstream | Call sites | Worst grade | A | B | C | D | F |');
+ lines.push('|---|---|---|---|---|---|---|---|');
+ for (const h of hosts) {
+ lines.push(
+ `| \`${h.host}\` | ${h.sites} | ${h.worst} | ${h.grades.A} | ${h.grades.B} | ${h.grades.C} | ${h.grades.D} | ${h.grades.F} |`,
+ );
+ }
+ lines.push('');
+ lines.push('## Keeping this true');
+ lines.push('');
+ lines.push('`npm run audit:upstreams` fails when a call site regresses below the grade');
+ lines.push('recorded in `data/upstream-baseline.json`, or when a new unbounded call appears.');
+ lines.push('Improving a call site lowers the baseline automatically; raising it back needs an');
+ lines.push('explicit `--accept`, which is a reviewable line in the diff.');
+ lines.push('');
+ return lines.join('\n');
+}
+
+function main() {
+ const files = sourceFiles();
+ const { callSites, parseErrors } = analyzeFiles(files, ROOT);
+ const summary = summarize(callSites);
+ const index = gradeIndex(callSites);
+
+ if (asJson) {
+ process.stdout.write(
+ `${JSON.stringify({ summary, hosts: byHost(callSites), callSites, parseErrors }, null, '\t')}\n`,
+ );
+ return 0;
+ }
+
+ if (writeMap) {
+ writeFileSync(MAP_DOC, `${renderMap(callSites, summary)}`);
+ console.log(`audit:upstreams: wrote ${relative(ROOT, MAP_DOC)}`);
+ }
+
+ const baseline = loadBaseline();
+
+ if (accept || !baseline) {
+ writeFileSync(
+ BASELINE,
+ `${JSON.stringify({ generated: 'scripts/audit-upstreams.mjs', summary, sites: index }, null, '\t')}\n`,
+ );
+ console.log(
+ `audit:upstreams: baseline ${baseline ? 're-recorded' : 'created'} for ${summary.total} call sites ` +
+ `(A:${summary.grades.A} B:${summary.grades.B} C:${summary.grades.C} D:${summary.grades.D} F:${summary.grades.F})`,
+ );
+ return 0;
+ }
+
+ // Regressions: a known site that got worse, or a new site below the floor.
+ const regressed = [];
+ const newUnbounded = [];
+ for (const site of callSites) {
+ const key = siteKey(site);
+ const grade = gradeOf(site);
+ const was = baseline.sites?.[key];
+ if (was === undefined) {
+ if (grade === 'F') newUnbounded.push({ site, grade });
+ continue;
+ }
+ if (GRADE_RANK[grade] < GRADE_RANK[was]) regressed.push({ site, grade, was });
+ }
+
+ // Improvements ratchet the baseline down on their own, so the floor only
+ // ever rises. Nobody has to remember to re-record a fix.
+ let improved = 0;
+ const nextSites = { ...(baseline.sites || {}) };
+ for (const [key, grade] of Object.entries(index)) {
+ const was = nextSites[key];
+ if (was === undefined || GRADE_RANK[grade] > GRADE_RANK[was]) {
+ if (was !== undefined) improved += 1;
+ nextSites[key] = grade;
+ }
+ }
+ for (const key of Object.keys(nextSites)) {
+ if (!(key in index)) delete nextSites[key]; // call site removed
+ }
+
+ console.log(
+ `audit:upstreams: ${summary.total} outbound call sites across ${byHost(callSites).length} upstreams ` +
+ `(A:${summary.grades.A} B:${summary.grades.B} C:${summary.grades.C} D:${summary.grades.D} F:${summary.grades.F})`,
+ );
+ if (parseErrors.length) {
+ console.log(`audit:upstreams: ${parseErrors.length} file(s) could not be parsed and were skipped`);
+ }
+
+ if (regressed.length || newUnbounded.length) {
+ console.error('');
+ for (const r of regressed) {
+ console.error(
+ `[upstreams] ${r.site.file}:${r.site.line} ${r.site.host} ${r.was} -> ${r.grade}`,
+ );
+ console.error(` protections: ${r.site.protections.join(', ') || 'none'}`);
+ }
+ for (const n of newUnbounded) {
+ console.error(`[upstreams] ${n.site.file}:${n.site.line} ${n.site.host} NEW unbounded call`);
+ console.error(' wrap it with fetchUpstream from api/_lib/upstream-fetch.js,');
+ console.error(' or fetchFirst from src/shared/failover-fetch.js in browser code.');
+ }
+ console.error('');
+ console.error(
+ `audit:upstreams: ${regressed.length} regression(s), ${newUnbounded.length} new unbounded call(s).`,
+ );
+ console.error('If the change is deliberate, re-baseline with: npm run audit:upstreams -- --accept');
+ return 1;
+ }
+
+ if (improved) {
+ writeFileSync(
+ BASELINE,
+ `${JSON.stringify({ generated: 'scripts/audit-upstreams.mjs', summary, sites: nextSites }, null, '\t')}\n`,
+ );
+ console.log(`audit:upstreams: ${improved} call site(s) improved, baseline ratcheted up`);
+ }
+ console.log('audit:upstreams: no regressions');
+ return 0;
+}
+
+process.exit(main());
diff --git a/scripts/prove-brownout.mjs b/scripts/prove-brownout.mjs
new file mode 100644
index 000000000..da193bdc3
--- /dev/null
+++ b/scripts/prove-brownout.mjs
@@ -0,0 +1,270 @@
+#!/usr/bin/env node
+/**
+ * Execute every declared degradation contract against a running server.
+ *
+ * "We have fallbacks" is a claim about code that, by construction, only runs
+ * when something is broken. This turns the claim into a receipt: for each
+ * contract it breaks the named upstream for one request, calls the real
+ * endpoint, and checks what came back.
+ *
+ * The verdict that matters most is not pass or fail, it is `not_exercised`.
+ * A fault can be accepted and still never reach the code, because a warm cache
+ * answered before any upstream call happened. A prover that called that a pass
+ * would be certifying a fallback nothing ran, which is worse than not testing
+ * at all: it converts an unknown into a false assurance. So every contract
+ * declares which sources must appear as FAILED in the response's provenance
+ * trace, and a run that cannot show them is reported honestly.
+ *
+ * node scripts/prove-brownout.mjs # against localhost:8080
+ * node scripts/prove-brownout.mjs --base https://three.ws --token $BROWNOUT_CHAOS_TOKEN
+ * node scripts/prove-brownout.mjs --write public/brownout.json
+ * node scripts/prove-brownout.mjs --json # machine-readable, no file
+ *
+ * Exit codes: 0 all contracts proven, 1 a contract failed its expectation,
+ * 2 a contract could not be exercised (the fault never landed).
+ */
+
+import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';
+import { dirname } from 'node:path';
+
+const REGISTRY = 'data/brownout.json';
+const CHAOS_HEADER = 'x-brownout-chaos';
+const TOKEN_HEADER = 'x-brownout-chaos-token';
+const STATUS_HEADER = 'x-brownout-chaos-status';
+
+function arg(flag, fallback = null) {
+ const i = process.argv.indexOf(flag);
+ return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback;
+}
+const has = (flag) => process.argv.includes(flag);
+
+/**
+ * Parse the `x-brownout` summary and `x-brownout-trace` breakdown a response
+ * carries. Kept deliberately tolerant: a missing header means "this response
+ * recorded nothing", which is a real and meaningful state (everything was
+ * served from cache), not a parse error.
+ *
+ * @param {Headers} headers
+ * @returns {{ tier: string|null, degraded: boolean, ok: number, failed: number, ms: number|null, sources: Array<{name:string,outcome:string,ms:number}> }}
+ */
+export function parseProvenance(headers) {
+ const get = (k) => (typeof headers?.get === 'function' ? headers.get(k) : headers?.[k]) || '';
+ const summary = get('x-brownout');
+ const trace = get('x-brownout-trace');
+ const fields = {};
+ for (const pair of summary.split(';')) {
+ const [k, v] = pair.split('=');
+ if (k) fields[k.trim()] = (v ?? '').trim();
+ }
+ const sources = trace
+ ? trace.split(',').map((entry) => {
+ const parts = entry.split(';').map((p) => p.trim());
+ const name = parts[0] || '';
+ const outcome = (parts.find((p) => p.startsWith('o=')) || 'o=').slice(2);
+ const ms = Number((parts.find((p) => p.startsWith('t=')) || 't=0').slice(2));
+ return { name, outcome, ms: Number.isFinite(ms) ? ms : 0 };
+ })
+ : [];
+ return {
+ tier: fields.tier || null,
+ degraded: fields.degraded === '1',
+ ok: Number(fields.ok || 0),
+ failed: Number(fields.failed || 0),
+ ms: fields.ms ? Number(fields.ms) : null,
+ sources,
+ };
+}
+
+/** A source counts as exercised when the trace shows it and it did not succeed. */
+export function wasExercised(prov, name) {
+ const want = String(name).toLowerCase();
+ return prov.sources.some((s) => {
+ const got = s.name.toLowerCase();
+ if (got !== want && !got.startsWith(`${want}:`)) return false;
+ return s.outcome !== 'ok';
+ });
+}
+
+/** Build the request URL, applying the contract's cache-busting parameter. */
+export function contractUrl(base, contract, seed = Date.now()) {
+ const url = new URL(contract.endpoint, base);
+ for (const [k, v] of Object.entries(contract.query || {})) url.searchParams.set(k, String(v));
+ const bust = contract.bust;
+ if (bust?.param) {
+ // A warm cache is the single biggest source of a false green here, so the
+ // request is deliberately steered off whatever key the last one used. The
+ // range is the contract's, because only it knows what stays meaningful.
+ const span = Math.max(1, (bust.max ?? 10) - (bust.min ?? 1) + 1);
+ url.searchParams.set(bust.param, String((bust.min ?? 1) + (seed % span)));
+ }
+ return url.toString();
+}
+
+export function chaosDirective(contract) {
+ return Object.entries(contract.break || {})
+ .map(([name, spec]) => `${name}=${spec}`)
+ .join(', ');
+}
+
+/** Check one response against a contract's expectations. Pure, so it is testable. */
+export function judge(contract, { status, prov, body, chaosStatus }) {
+ const expect = contract.expect || {};
+ const problems = [];
+
+ if (chaosStatus && !/^applied/.test(chaosStatus)) {
+ return { verdict: 'not_exercised', problems: [`the server refused the fault: ${chaosStatus}`] };
+ }
+
+ for (const name of expect.exercised || []) {
+ if (!wasExercised(prov, name)) {
+ return {
+ verdict: 'not_exercised',
+ problems: [
+ `\`${name}\` never appears as a failed source in the response trace, so the fault did not reach the code. ` +
+ 'A warm cache almost certainly answered first; give the contract a `bust` parameter or widen it.',
+ ],
+ };
+ }
+ }
+
+ const wantStatus = [].concat(expect.status ?? 200);
+ if (!wantStatus.includes(status)) problems.push(`status ${status}, expected ${wantStatus.join(' or ')}`);
+
+ if (expect.tier) {
+ const wantTier = [].concat(expect.tier);
+ if (!prov.tier) problems.push(`expected tier ${wantTier.join(' or ')} but the response recorded no provenance`);
+ else if (!wantTier.includes(prov.tier)) problems.push(`tier ${prov.tier}, expected ${wantTier.join(' or ')}`);
+ }
+
+ if (expect.degraded === true && !prov.degraded) {
+ problems.push('the response did not report itself as degraded, though its upstream was broken');
+ }
+
+ for (const key of expect.jsonHas || []) {
+ if (!body || typeof body !== 'object' || !(key in body)) problems.push(`response body is missing \`${key}\``);
+ }
+
+ return { verdict: problems.length ? 'fail' : 'pass', problems };
+}
+
+async function runContract(base, token, contract, timeoutMs) {
+ const started = Date.now();
+ const seed = Math.floor(Math.random() * 1e6);
+
+ // A contract whose fallback is a last-good tier needs something to fall back
+ // TO. One clean request first is not cheating: it is the state every real
+ // user's request arrives in, and a cold process with no memory is a different
+ // contract (the honest 503) that other entries assert.
+ if (contract.warm) {
+ try {
+ await fetch(contractUrl(base, contract, seed), { signal: AbortSignal.timeout(timeoutMs) });
+ } catch {
+ /* a failed warm-up is not a verdict; the real attempt below is */
+ }
+ }
+
+ const url = contractUrl(base, contract, seed);
+ let res;
+ try {
+ res = await fetch(url, {
+ headers: { [CHAOS_HEADER]: chaosDirective(contract), [TOKEN_HEADER]: token, accept: 'application/json' },
+ signal: AbortSignal.timeout(timeoutMs),
+ });
+ } catch (err) {
+ return {
+ id: contract.id,
+ title: contract.title,
+ verdict: 'fail',
+ ms: Date.now() - started,
+ problems: [`the request itself failed: ${err?.message || err}`],
+ };
+ }
+
+ const text = await res.text().catch(() => '');
+ let body = null;
+ try {
+ body = JSON.parse(text);
+ } catch {
+ /* a non-JSON body is judged on status and provenance alone */
+ }
+ const prov = parseProvenance(res.headers);
+ const chaosStatus = res.headers.get(STATUS_HEADER);
+ const { verdict, problems } = judge(contract, { status: res.status, prov, body, chaosStatus });
+
+ return {
+ id: contract.id,
+ title: contract.title,
+ surface: contract.surface || null,
+ endpoint: contract.endpoint,
+ broke: contract.break,
+ why: contract.why || null,
+ verdict,
+ problems,
+ ms: Date.now() - started,
+ observed: {
+ status: res.status,
+ tier: prov.tier,
+ degraded: prov.degraded,
+ failedSources: prov.failed,
+ trace: prov.sources.slice(0, 8),
+ },
+ };
+}
+
+async function main() {
+ const base = arg('--base', process.env.BROWNOUT_BASE_URL || 'http://127.0.0.1:8080');
+ const token = arg('--token', process.env.BROWNOUT_CHAOS_TOKEN || '');
+ const timeoutMs = Number(arg('--timeout', '45000'));
+ const outPath = arg('--write', null);
+ const jsonOnly = has('--json');
+
+ if (!token) {
+ console.error('prove-brownout: no chaos token. Set BROWNOUT_CHAOS_TOKEN or pass --token.');
+ console.error('Without it the server refuses every directive, which would report every contract as not_exercised.');
+ process.exit(2);
+ }
+
+ const registry = JSON.parse(readFileSync(REGISTRY, 'utf8'));
+ const results = [];
+ for (const contract of registry.contracts) {
+ results.push(await runContract(base, token, contract, timeoutMs));
+ }
+
+ const counts = results.reduce((acc, r) => ({ ...acc, [r.verdict]: (acc[r.verdict] || 0) + 1 }), {});
+ const report = {
+ generated_at: new Date().toISOString(),
+ base,
+ summary: registry.summary,
+ counts,
+ proven: counts.pass || 0,
+ total: results.length,
+ results,
+ };
+
+ if (outPath) {
+ mkdirSync(dirname(outPath), { recursive: true });
+ writeFileSync(outPath, `${JSON.stringify(report, null, '\t')}\n`);
+ }
+ if (jsonOnly) {
+ console.log(JSON.stringify(report, null, 2));
+ } else {
+ console.log(`prove-brownout: ${report.proven}/${report.total} contracts proven against ${base}\n`);
+ for (const r of results) {
+ const mark = r.verdict === 'pass' ? 'PASS' : r.verdict === 'not_exercised' ? 'UNEX' : 'FAIL';
+ console.log(` ${mark} ${r.id} (${r.ms}ms)`);
+ console.log(` ${r.title}`);
+ if (r.observed) {
+ console.log(` got ${r.observed.status}, tier=${r.observed.tier || 'none'}, failed sources=${r.observed.failedSources}`);
+ for (const s of r.observed.trace) console.log(` - ${s.name} ${s.outcome} ${s.ms}ms`);
+ }
+ for (const p of r.problems) console.log(` ! ${p}`);
+ }
+ if (outPath) console.log(`\nwrote ${outPath}`);
+ }
+
+ if (counts.fail) process.exit(1);
+ if (counts.not_exercised) process.exit(2);
+ process.exit(0);
+}
+
+if (import.meta.url === `file://${process.argv[1]}`) main();
diff --git a/scripts/resilience/analyze.mjs b/scripts/resilience/analyze.mjs
new file mode 100644
index 000000000..2411a1472
--- /dev/null
+++ b/scripts/resilience/analyze.mjs
@@ -0,0 +1,424 @@
+// AST analysis of every outbound call in the repo.
+//
+// Regex cannot answer the question this file exists to answer. "Does this call
+// have a deadline" is not a property of the line the call is written on: the
+// options object may be built three statements earlier, the deadline may arrive
+// through a wrapper, and a `signal,` shorthand looks nothing like `signal:`.
+// An earlier regex sweep of this same repo reported one unguarded call and was
+// wrong twice over, in both directions. Parsing is what makes the audit
+// trustworthy enough to gate a deploy on.
+//
+// What it produces, per call site: which upstream is being talked to, and which
+// of the five protections are present (deadline, retry, failover, lastGood,
+// breaker). Everything downstream of this module (the grade, the ratchet, the
+// generated map) is a projection of that one table.
+
+import { readFileSync } from 'node:fs';
+import { relative, resolve, dirname, join } from 'node:path';
+import * as acorn from 'acorn';
+
+// A local walker rather than acorn-walk. This module gates a deploy, so it is
+// deliberately free of dependencies beyond the parser itself: the traversal it
+// needs is a generic descent that carries the ancestor chain, which is shorter
+// than the code that would justify pulling a package in for it.
+function walkWithAncestors(node, visit, ancestors = []) {
+ if (node === null || typeof node !== 'object') return;
+ if (Array.isArray(node)) {
+ for (const child of node) walkWithAncestors(child, visit, ancestors);
+ return;
+ }
+ if (typeof node.type !== 'string') return;
+ visit(node, ancestors);
+ ancestors.push(node);
+ for (const key of Object.keys(node)) {
+ if (key === 'type' || key === 'loc' || key === 'start' || key === 'end') continue;
+ walkWithAncestors(node[key], visit, ancestors);
+ }
+ ancestors.pop();
+}
+
+// Helpers whose use IS the protection: calling one of these is what a guarded
+// call site looks like in this repo, so the analyzer resolves them by name
+// rather than re-deriving their internals at every call site. Adding a new
+// shared helper means adding it here, which is the one maintenance cost.
+export const GUARDS = {
+ fetchUpstream: ['deadline', 'retry', 'breaker'],
+ fetchUpstreamJson: ['deadline', 'retry', 'breaker'],
+ fetchAnyJson: ['deadline', 'retry', 'failover'],
+ lastGood: ['lastGood'],
+ lastGoodValue: ['lastGood'],
+ cacheWrapLastGood: ['lastGood'],
+ fetchFirst: ['deadline', 'failover'],
+ fetchFirstOrNull: ['deadline', 'failover'],
+ pumpFetchJson: ['deadline', 'retry'],
+ geckoFetch: ['deadline', 'lastGood'],
+ paprikaGet: ['deadline'],
+ withRetry: ['retry'],
+ withBreaker: ['breaker'],
+ withRetryAndBreaker: ['retry', 'breaker'],
+ makeRotatingFetch: ['deadline', 'failover'],
+ // The rotating RPC clients bound each attempt themselves (see
+ // makeRotatingFetch in api/_lib/solana/connection.js), so using one is a
+ // bounded call even though no deadline is written at the call site.
+ solanaConnection: ['deadline', 'failover'],
+ evmFallbackProvider: ['deadline', 'failover'],
+ evmTransport: ['deadline', 'failover'],
+ resolveURI: ['deadline', 'failover'],
+ fetchWithFallback: ['deadline', 'failover'],
+ loadModule: ['deadline', 'failover'],
+ loadScript: ['deadline', 'failover'],
+};
+
+// Guards whose first argument is a LIST of providers rather than one URL. Their
+// whole point is that several hosts back one read, so each entry is recorded as
+// its own call site: that is what makes a failover chain visible per upstream
+// instead of collapsing to a single anonymous row.
+export const LIST_GUARDS = new Set(['fetchFirst', 'fetchFirstOrNull', 'fetchAnyJson']);
+
+export const PROTECTIONS = ['deadline', 'retry', 'failover', 'lastGood', 'breaker'];
+
+// A relative URL, or one aimed at this machine, is our own origin: covered by
+// the same-origin client rather than by upstream failover, so it is out of scope.
+const INTERNAL_URL_RE = /^(?:\/|https?:\/\/(?:localhost|127\.0\.0\.1|\[::1\]|(?:[a-z0-9-]+\.)*three\.ws))/i;
+
+// Marks the position of a `${...}` hole in a reconstructed template literal.
+const HOLE = '__EXPR__';
+
+/** Hostname for a URL-ish string, or null when it is relative or unknowable. */
+export function hostOf(raw) {
+ if (typeof raw !== 'string') return null;
+ const m = raw.match(/^https?:\/\/([^/?#]+)/i);
+ return m ? m[1].toLowerCase() : null;
+}
+
+// A template literal's static skeleton: `${BASE}/coins/${mint}` becomes
+// "<HOLE>/coins/<HOLE>", which is enough to recognise an absolute URL and to
+// name the upstream whenever the host half is written literally.
+function staticText(node, consts = null) {
+ if (!node) return '';
+ if (node.type === 'Literal') return typeof node.value === 'string' ? node.value : '';
+ if (node.type === 'TemplateLiteral') {
+ // Substitute a known module-level constant into its hole so
+ // `${PUMP_BASE}/coins/x` is attributed to the host PUMP_BASE holds,
+ // instead of collapsing into an anonymous "dynamic" row.
+ return node.quasis
+ .map((q, i) => {
+ if (i >= node.expressions.length) return q.value.cooked;
+ const filled = staticText(node.expressions[i], consts);
+ return q.value.cooked + (filled && filled !== HOLE ? filled : HOLE);
+ })
+ .join('');
+ }
+ if (node.type === 'BinaryExpression' && node.operator === '+') {
+ return staticText(node.left, consts) + staticText(node.right, consts);
+ }
+ if (node.type === 'Identifier') {
+ const known = consts?.get(node.name);
+ return known !== undefined ? known : HOLE;
+ }
+ if (node.type === 'MemberExpression') return HOLE;
+ return '';
+}
+
+/** Does this options-object AST node carry a deadline? */
+function objectHasDeadline(node) {
+ if (!node || node.type !== 'ObjectExpression') return false;
+ return node.properties.some((p) => {
+ if (p.type === 'SpreadElement') return false;
+ const key = p.key?.name ?? p.key?.value;
+ // `signal: x`, the `signal` shorthand, and an explicit timeout all count.
+ return key === 'signal' || key === 'timeout' || key === 'timeoutMs';
+ });
+}
+
+function calleeName(node) {
+ if (!node) return null;
+ if (node.type === 'Identifier') return node.name;
+ if (node.type === 'MemberExpression') return node.property?.name ?? null;
+ return null;
+}
+
+function displayUrl(text) {
+ return text.split(HOLE).join('${}');
+}
+
+/**
+ * Analyze one source file.
+ *
+ * @param {string} code
+ * @param {string} file path used for reporting
+ * @returns {{ callSites: Array<object>, parseError: string | null }}
+ */
+export function analyzeSource(code, file, resolveImported = null) {
+ let ast;
+ try {
+ ast = acorn.parse(code, {
+ ecmaVersion: 'latest',
+ sourceType: 'module',
+ allowAwaitOutsideFunction: true,
+ allowHashBang: true,
+ allowReturnOutsideFunction: true,
+ locations: true,
+ });
+ } catch (err) {
+ return { callSites: [], parseError: err.message };
+ }
+
+ // Which guard helpers this file imports, so a bare call to one is
+ // attributable to the shared implementation rather than to a local function
+ // that happens to share its name.
+ // Module-level `const X = 'https://...'` (and simple concatenations of them)
+ // are the base URLs nearly every call site is written against, so resolving
+ // them is what turns a wall of "dynamic" rows into named upstreams.
+ const consts = new Map();
+ // Provider chains are frequently hoisted to a module constant
+ // (`export const PROVIDERS = [...]`) and passed to fetchFirst by name, so the
+ // array node has to be resolvable from the identifier or the entire chain is
+ // invisible to the audit.
+ const arrayConsts = new Map();
+ for (const stmt of ast.body || []) {
+ const decl = stmt.type === 'ExportNamedDeclaration' ? stmt.declaration : stmt;
+ if (!decl || decl.type !== 'VariableDeclaration') continue;
+ for (const d of decl.declarations) {
+ if (d.id?.type !== 'Identifier' || !d.init) continue;
+ if (d.init.type === 'ArrayExpression') arrayConsts.set(d.id.name, d.init);
+ const text = staticText(d.init, consts);
+ if (text && !text.includes(HOLE)) consts.set(d.id.name, text);
+ }
+ }
+
+ const imported = new Set();
+ // local binding name -> { from, name }, so a URL built on an imported base
+ // constant can be resolved against the module that exports it.
+ const importBindings = new Map();
+ walkWithAncestors(ast, (n) => {
+ if (n.type !== 'ImportDeclaration') return;
+ const from = typeof n.source?.value === 'string' ? n.source.value : null;
+ for (const spec of n.specifiers || []) {
+ const name = spec.imported?.name ?? spec.local?.name;
+ if (name) imported.add(name);
+ if (from && spec.local?.name) {
+ importBindings.set(spec.local.name, { from, name: spec.imported?.name ?? spec.local.name });
+ }
+ }
+ });
+
+ // Imported constants resolve through the caller-supplied resolver, which the
+ // two-pass driver backs with every exported string constant in the repo.
+ if (resolveImported) {
+ for (const [local, ref] of importBindings) {
+ const value = resolveImported(file, ref.from, ref.name);
+ if (typeof value === 'string' && value && !consts.has(local)) consts.set(local, value);
+ }
+ }
+
+ const callSites = [];
+
+ // A guard call anywhere up the ancestor chain puts its protections in scope
+ // for the raw fetch it wraps: `withRetry(() => fetch(url))` is one guarded
+ // site, not a guard plus a naked fetch. Tracking ancestors is what tells
+ // those two shapes apart.
+ walkWithAncestors(ast, (node, ancestors) => {
+ if (node.type !== 'CallExpression') return;
+ {
+ const name = calleeName(node.callee);
+ if (!name) return;
+
+ const inherited = new Set();
+ for (const a of ancestors) {
+ if (a === node || a.type !== 'CallExpression') continue;
+ const outer = calleeName(a.callee);
+ if (outer && GUARDS[outer] && imported.has(outer)) {
+ for (const g of GUARDS[outer]) inherited.add(g);
+ }
+ }
+
+ const isGuard = Boolean(GUARDS[name]) && imported.has(name);
+ if (!isGuard && name !== 'fetch') return;
+
+ // A provider-list guard: unpack the array so every host in the chain is
+ // counted, each carrying the chain's protections.
+ if (isGuard && LIST_GUARDS.has(name)) {
+ let arr = node.arguments[0];
+ if (arr?.type === 'Identifier') arr = arrayConsts.get(arr.name) ?? arr;
+ if (arr?.type !== 'ArrayExpression') return;
+ const chainProtections = new Set(inherited);
+ for (const g of GUARDS[name]) chainProtections.add(g);
+ for (const el of arr.elements) {
+ if (!el) continue;
+ // Either a bare URL string or a { name, url, parse } provider object.
+ let entryUrl = '';
+ if (el.type === 'ObjectExpression') {
+ const urlProp = el.properties.find(
+ (pr) => pr.type !== 'SpreadElement' && (pr.key?.name ?? pr.key?.value) === 'url',
+ );
+ entryUrl = urlProp ? staticText(urlProp.value, consts) : '';
+ } else {
+ entryUrl = staticText(el, consts);
+ }
+ if (!entryUrl || !entryUrl.includes('://')) continue;
+ callSites.push({
+ file,
+ line: (el.loc ?? node.loc).start.line,
+ via: name,
+ host: hostOf(entryUrl) ?? 'dynamic',
+ url: displayUrl(entryUrl).slice(0, 120),
+ protections: [...chainProtections].sort(),
+ });
+ }
+ return;
+ }
+
+ const urlText = staticText(node.arguments[0], consts);
+ if (!urlText || INTERNAL_URL_RE.test(urlText)) return;
+ // Only grade calls we can PROVE leave this origin. A bare `fetch(path)`
+ // or a `${base}/api/x` is same-origin or unknowable, and counting those
+ // as unguarded upstreams buries the real ones: an earlier pass of this
+ // analyzer reported 513 unbounded calls, of which 507 were same-origin.
+ // A scheme in the resolved text is the proof; anything else is skipped.
+ if (!urlText.includes('://')) return;
+ const host = hostOf(urlText);
+
+ const protections = new Set(inherited);
+ if (isGuard) for (const g of GUARDS[name]) protections.add(g);
+ // An explicit per-call deadline counts wherever it appears.
+ for (const arg of node.arguments.slice(1)) {
+ if (objectHasDeadline(arg)) protections.add('deadline');
+ }
+
+ callSites.push({
+ file,
+ line: node.loc.start.line,
+ via: isGuard ? name : 'fetch',
+ host: host ?? 'dynamic',
+ url: displayUrl(urlText).slice(0, 120),
+ protections: [...protections].sort(),
+ });
+ }
+ });
+
+ return { callSites, parseError: null };
+}
+
+/**
+ * Every module-level string constant a file defines, exported or not, resolved
+ * as far as literal concatenation allows. Pass 1 of the two-pass analysis.
+ * @returns {Map<string, string>}
+ */
+export function collectStringConstants(code) {
+ const out = new Map();
+ let ast;
+ try {
+ ast = acorn.parse(code, { ecmaVersion: 'latest', sourceType: 'module', allowHashBang: true });
+ } catch {
+ return out;
+ }
+ for (const stmt of ast.body || []) {
+ const decl = stmt.type === 'ExportNamedDeclaration' ? stmt.declaration : stmt;
+ if (!decl || decl.type !== 'VariableDeclaration') continue;
+ for (const d of decl.declarations) {
+ if (d.id?.type !== 'Identifier' || !d.init) continue;
+ const text = staticText(d.init, out);
+ if (text && !text.includes(HOLE)) out.set(d.id.name, text);
+ }
+ }
+ return out;
+}
+
+/**
+ * Analyze a list of absolute paths, reported relative to `root`.
+ * @returns {{ callSites: Array<object>, parseErrors: Array<object> }}
+ */
+export function analyzeFiles(files, root) {
+ const callSites = [];
+ const parseErrors = [];
+
+ // Pass 1: every module's string constants. Base URLs in this repo are
+ // overwhelmingly `export const PUMP_FRONTEND_BASE = 'https://...'` in one
+ // module and imported everywhere else, so without this pass the majority of
+ // call sites cannot be attributed to the host they actually talk to.
+ const sources = new Map();
+ const constsByFile = new Map();
+ for (const abs of files) {
+ let code;
+ try {
+ code = readFileSync(abs, 'utf8');
+ } catch {
+ continue;
+ }
+ sources.set(abs, code);
+ constsByFile.set(abs, collectStringConstants(code));
+ }
+
+ const byRel = new Map([...sources.keys()].map((abs) => [relative(root, abs), abs]));
+
+ // Resolve `import { NAME } from './x.js'` against pass 1, trying the
+ // extensions and index forms Node would.
+ const resolveImported = (fromRel, spec, name) => {
+ if (!spec.startsWith('.')) return undefined;
+ const baseDir = dirname(resolve(root, fromRel));
+ const target = resolve(baseDir, spec);
+ const candidates = [
+ target,
+ `${target}.js`,
+ `${target}.mjs`,
+ join(target, 'index.js'),
+ join(target, 'index.mjs'),
+ ];
+ for (const cand of candidates) {
+ const consts = constsByFile.get(cand);
+ if (consts?.has(name)) return consts.get(name);
+ }
+ return undefined;
+ };
+
+ // Pass 2: grade every call site with imported constants resolved.
+ for (const [abs, code] of sources) {
+ const rel = relative(root, abs);
+ const { callSites: found, parseError } = analyzeSource(code, rel, resolveImported);
+ if (parseError) parseErrors.push({ file: rel, error: parseError });
+ callSites.push(...found);
+ }
+ return { callSites, parseErrors, filesScanned: sources.size, byRel };
+}
+
+/**
+ * A call site's grade. `deadline` is the floor, because without one a single
+ * stalled socket outlives the request that opened it and no other protection
+ * ever gets a turn. Above the floor, what earns a grade is having somewhere else
+ * to go (failover) or something to serve (lastGood).
+ *
+ * A bounded, and survives the upstream being down
+ * B bounded, with one way to survive
+ * C bounded, and re-tries or trips, but has no second source
+ * D bounded only
+ * F unbounded: can hang the request
+ */
+export function gradeOf(site) {
+ const p = new Set(site.protections);
+ if (!p.has('deadline')) return 'F';
+ if (p.has('failover') && (p.has('lastGood') || p.has('retry'))) return 'A';
+ if (p.has('failover') || p.has('lastGood')) return 'B';
+ if (p.has('retry') || p.has('breaker')) return 'C';
+ return 'D';
+}
+
+/** Roll call sites up per upstream host, worst grade first. */
+export function byHost(callSites) {
+ const order = { F: 0, D: 1, C: 2, B: 3, A: 4 };
+ const hosts = new Map();
+ for (const site of callSites) {
+ let h = hosts.get(site.host);
+ if (!h) {
+ h = { host: site.host, sites: 0, worst: 'A', grades: { A: 0, B: 0, C: 0, D: 0, F: 0 } };
+ hosts.set(site.host, h);
+ }
+ const g = gradeOf(site);
+ h.sites += 1;
+ h.grades[g] += 1;
+ if (order[g] < order[h.worst]) h.worst = g;
+ }
+ return [...hosts.values()].sort(
+ (a, b) => order[a.worst] - order[b.worst] || b.sites - a.sites,
+ );
+}
diff --git a/scripts/resilience/contracts.mjs b/scripts/resilience/contracts.mjs
new file mode 100644
index 000000000..9f8dc5e75
--- /dev/null
+++ b/scripts/resilience/contracts.mjs
@@ -0,0 +1,186 @@
+// Degradation contracts.
+//
+// A fallback is not automatically an improvement. Serving a remembered value is
+// the right answer for a price chip and the wrong answer for a question whose
+// whole purpose is to be current: this repo shipped four of those in one week.
+// A payment retry re-signed the blockhash the facilitator had just refused. A
+// token security endpoint assembled a clean bill of health out of remembered
+// on-chain state at the exact moment the chain was unreadable. An access gate
+// answered from a cached verdict. An alert feed went quiet in a way that looked
+// like a calm market.
+//
+// Every one of those passed a "does it have a fallback" review. What none of
+// them had was a stated answer to a different question: when this upstream is
+// down, what is this endpoint ALLOWED to say? That is the contract, and this
+// module is the vocabulary plus the harness that proves it by actually breaking
+// the upstream and looking at the response.
+
+/**
+ * The contracts an endpoint can declare.
+ *
+ * MAY_SERVE_STALE
+ * A remembered value is a better answer than an error, and the response says
+ * so (a stale marker, an as_of stamp, or a header). Prices, feeds, boards.
+ *
+ * MUST_REFUSE
+ * There is no honest answer without a live read, so the endpoint must return a
+ * typed error rather than invent one. The failure mode this prevents is not a
+ * crash: it is a confident, wrong, cacheable 200.
+ *
+ * MUST_NOT_SERVE_STALE
+ * Stronger than MUST_REFUSE. The endpoint may answer 200 from live data, but
+ * a remembered value must never reach the caller, because acting on a stale
+ * answer costs money or grants access. Payment retries, security verdicts,
+ * authorization gates.
+ */
+export const CONTRACT = {
+ MAY_SERVE_STALE: 'may-serve-stale',
+ MUST_REFUSE: 'must-refuse',
+ MUST_NOT_SERVE_STALE: 'must-not-serve-stale',
+};
+
+export const CONTRACT_VALUES = new Set(Object.values(CONTRACT));
+
+/** Status codes that count as a typed refusal rather than a crash. */
+const REFUSAL_STATUSES = new Set([424, 425, 429, 502, 503, 504]);
+
+/**
+ * Markers that say "this body is remembered, not live". A contract that allows
+ * stale data requires one of these to be present, because degrading silently is
+ * how a stale answer gets mistaken for a current one.
+ */
+export function staleMarkers(body, headers = {}) {
+ const found = [];
+ if (body && typeof body === 'object') {
+ if (body.stale === true) found.push('body.stale');
+ if (body.as_of !== undefined && body.as_of !== null) found.push('body.as_of');
+ if (body.data && typeof body.data === 'object') {
+ if (body.data.stale === true) found.push('body.data.stale');
+ if (body.data.as_of !== undefined && body.data.as_of !== null) found.push('body.data.as_of');
+ }
+ }
+ for (const key of Object.keys(headers || {})) {
+ const k = key.toLowerCase();
+ if (k.includes('stale')) found.push(`header.${k}`);
+ }
+ return found;
+}
+
+/**
+ * Judge one observed response against a contract.
+ *
+ * Deliberately pure and transport-agnostic: the caller performs the fault
+ * injection however its endpoint needs, then hands the outcome here. That keeps
+ * the judgement identical whether the response came from an HTTP handler, an
+ * exported library function, or a worker.
+ *
+ * @param {string} contract one of CONTRACT
+ * @param {{ status?: number, body?: any, headers?: object, threw?: unknown }} observed
+ * @returns {{ ok: boolean, reason: string }}
+ */
+export function judge(contract, observed) {
+ const { status, body, headers = {}, threw } = observed;
+
+ if (threw !== undefined && threw !== null) {
+ // A rejection that carries a typed status is a refusal expressed as an
+ // exception, which the HTTP boundary turns into the same envelope.
+ const typed = Number(threw?.status ?? threw?.statusCode);
+ if (contract === CONTRACT.MAY_SERVE_STALE) {
+ return { ok: false, reason: `threw instead of serving a remembered value: ${threw?.message || threw}` };
+ }
+ if (Number.isFinite(typed) && REFUSAL_STATUSES.has(typed)) {
+ return { ok: true, reason: `refused with a typed ${typed}` };
+ }
+ if (threw?.code) return { ok: true, reason: `refused with code ${threw.code}` };
+ return { ok: false, reason: `threw an untyped error: ${threw?.message || threw}` };
+ }
+
+ const markers = staleMarkers(body, headers);
+ const is2xx = status === undefined || (status >= 200 && status < 300);
+
+ switch (contract) {
+ case CONTRACT.MAY_SERVE_STALE: {
+ if (!is2xx) {
+ // Refusing is allowed only when nothing was ever remembered; the
+ // caller distinguishes those two cases by warming the endpoint first.
+ return REFUSAL_STATUSES.has(status)
+ ? { ok: true, reason: `refused with a typed ${status} (nothing remembered yet)` }
+ : { ok: false, reason: `answered ${status}, which is neither stale data nor a typed refusal` };
+ }
+ if (!markers.length) {
+ return {
+ ok: false,
+ reason: 'served a 200 during a total upstream outage without marking it stale, so a remembered value is indistinguishable from a live one',
+ };
+ }
+ return { ok: true, reason: `served remembered data, marked by ${markers.join(', ')}` };
+ }
+
+ case CONTRACT.MUST_REFUSE: {
+ if (is2xx) {
+ return {
+ ok: false,
+ reason: 'answered 2xx with every upstream down, which states as fact something it could not check',
+ };
+ }
+ return REFUSAL_STATUSES.has(status)
+ ? { ok: true, reason: `refused with a typed ${status}` }
+ : { ok: false, reason: `answered ${status}, which is not a typed "try again" refusal` };
+ }
+
+ case CONTRACT.MUST_NOT_SERVE_STALE: {
+ if (is2xx && markers.length) {
+ return {
+ ok: false,
+ reason: `served remembered data (${markers.join(', ')}) on a path where acting on a stale answer costs money or grants access`,
+ };
+ }
+ if (is2xx) {
+ return {
+ ok: false,
+ reason: 'answered 2xx with every upstream down, so the answer came from somewhere other than a live read',
+ };
+ }
+ return REFUSAL_STATUSES.has(status)
+ ? { ok: true, reason: `refused with a typed ${status} rather than reusing a remembered answer` }
+ : { ok: false, reason: `answered ${status}, which is not a typed refusal` };
+ }
+
+ default:
+ return { ok: false, reason: `unknown contract "${contract}"` };
+ }
+}
+
+/**
+ * A fetch replacement that fails the way a dead upstream fails, for injecting a
+ * total outage. `mode` picks the shape of the failure, because a refused
+ * connection and a socket that accepts and never answers are different bugs and
+ * have caught different ones here.
+ *
+ * @param {'network' | 'timeout' | 'status'} mode
+ * @param {{ status?: number }} [opts]
+ */
+export function failingFetch(mode = 'network', { status = 503 } = {}) {
+ if (mode === 'status') {
+ return async () =>
+ new Response(JSON.stringify({ error: 'upstream is down' }), {
+ status,
+ headers: { 'content-type': 'application/json' },
+ });
+ }
+ if (mode === 'timeout') {
+ return async (_url, init) =>
+ new Promise((_resolve, reject) => {
+ const signal = init?.signal;
+ const fail = () => reject(Object.assign(new Error('The operation was aborted'), { name: 'AbortError' }));
+ if (!signal) return; // no deadline: hangs, which is itself the finding
+ if (signal.aborted) return fail();
+ signal.addEventListener('abort', fail, { once: true });
+ });
+ }
+ return async () => {
+ const err = new TypeError('fetch failed');
+ err.cause = { code: 'ECONNREFUSED' };
+ throw err;
+ };
+}
diff --git a/src/shared/failover-fetch.js b/src/shared/failover-fetch.js
index 335fcbc6e..a15308fca 100644
--- a/src/shared/failover-fetch.js
+++ b/src/shared/failover-fetch.js
@@ -26,6 +26,50 @@
const _cooldowns = new Map(); // provider name -> epoch ms until which it is skipped
+// Brownout hooks. This module is isomorphic (the browser bundle imports it), and
+// provenance/chaos are node:async_hooks, so they are loaded lazily behind an
+// isServer() gate exactly as api/_lib/solana/connection.js loads the shared
+// cache. In a browser both resolve to no-ops, which is correct: there is no
+// request to attribute a source to, and nothing should be able to inject a
+// fault from the client side.
+let _brownoutPromise;
+// Resolved module, once the import lands. Recording has to be SYNCHRONOUS in the
+// hot path: the response header is written when the handler ends, and an await
+// here would let a record arrive after its own response had already gone out.
+// That is not a lost log line, it is a trace that omits the rung that answered.
+let _brownoutMod = null;
+function brownout() {
+ if (_brownoutPromise !== undefined) return _brownoutPromise;
+ _brownoutPromise = null;
+ if (typeof window === 'undefined') {
+ _brownoutPromise = import('../../api/_lib/brownout/index.js')
+ .then((m) => {
+ _brownoutMod = m;
+ return m;
+ })
+ .catch(() => null);
+ }
+ return _brownoutPromise;
+}
+// Warm the import at module load on the server, so by the time a request runs
+// the module is in hand and every record is written synchronously.
+brownout();
+
+function noteSource(rec) {
+ if (_brownoutMod) {
+ _brownoutMod.recordSource?.(rec);
+ return;
+ }
+ // Only on the very first call of a cold process, before the import lands.
+ const pending = brownout();
+ if (pending) void pending.then((m) => m?.recordSource?.(rec));
+}
+async function faultForProvider(name) {
+ if (_brownoutMod) return _brownoutMod.faultFor?.(name) ?? null;
+ const mod = await brownout();
+ return mod?.faultFor?.(name) ?? null;
+}
+
/**
* @typedef {object} Provider
* @property {string} name Stable id; keys the cooldown map.
@@ -55,24 +99,51 @@ export async function fetchFirst(providers, { timeoutMs = 4000, cooldownMs = 60_
let lastErr;
for (const p of order) {
+ const startedAt = Date.now();
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), timeoutMs);
// A caller-supplied signal (stale-search abort) composes with the
// per-attempt timeout rather than being replaced by it.
const signal = p.init?.signal ? AbortSignal.any([ctrl.signal, p.init.signal]) : ctrl.signal;
try {
- const res = await fetch(p.url, {
- headers: { accept: 'application/json' },
- ...p.init,
- signal,
- });
+ // A declared fault stands in for the real one, in the same position the
+ // real failure would occupy, so the ladder below fails over identically.
+ const fault = await faultForProvider(p.name);
+ let res;
+ if (fault) {
+ const mod = _brownoutMod || (await brownout());
+ const injected = await mod.applyFault(fault, p.url);
+ if (injected) res = injected;
+ }
+ if (!res) {
+ res = await fetch(p.url, {
+ headers: { accept: 'application/json' },
+ ...p.init,
+ signal,
+ });
+ }
if (!res.ok) throw new Error(`http_${res.status}`);
const value = await (p.parse ? p.parse(res) : res.json());
- if (value != null) return { value, source: p.name };
+ if (value != null) {
+ // The rung that answered is `live`; the ones before it are recorded
+ // in the catch below, so a trace shows the whole walk, not just the
+ // winner. That is the difference between "we have a ladder" and
+ // "here is the ladder working".
+ noteSource({ name: p.name, outcome: 'ok', ms: Date.now() - startedAt, tier: 'live' });
+ return { value, source: p.name };
+ }
// Miss: provider is healthy but has no data for this query.
+ noteSource({ name: p.name, outcome: 'skip', ms: Date.now() - startedAt, tier: 'live', detail: 'no_data' });
lastErr = new Error(`${p.name}: no_data`);
} catch (err) {
lastErr = err instanceof Error ? err : new Error(String(err));
+ noteSource({
+ name: p.name,
+ outcome: 'fail',
+ ms: Date.now() - startedAt,
+ tier: 'live',
+ detail: /http_(\d{3})/.exec(lastErr.message)?.[1] || (lastErr.name === 'TimeoutError' ? 'timeout' : 'error'),
+ });
// Caller abandoned the request (stale search) — stop the whole chain
// and don't penalise the provider for it.
if (p.init?.signal?.aborted) {