Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(brownout): stamp every API response with where its data came from
nirholas9d50375be741
diff --git a/api/_lib/brownout/provenance.js b/api/_lib/brownout/provenance.jsindex 6cef43451..7094fc50a 100644--- a/api/_lib/brownout/provenance.js+++ b/api/_lib/brownout/provenance.js@@ -139,10 +139,13 @@ export function provenanceSummary() {ok,failed,ms: Date.now() - ledger.startedAt,- // Degraded means "a reader should not treat this as a clean live answer":- // either something had to be retried or failed over, or what came back is- // not fresh. Both are worth surfacing; neither is an error.- degraded: failed > 0 || tier !== 'live',+ // Degraded means "this answer is worse than the one the endpoint intends to+ // give": something had to fail over, or what came back is older than its+ // own TTL allows. A `cache` hit is NOT degraded, and getting that wrong+ // would be fatal to the signal: almost every response on a healthy site is+ // a cache hit, so counting those would mark the whole site degraded and+ // train every reader to ignore the flag.+ degraded: failed > 0 || tier === 'stale' || tier === 'fallback',truncated: ledger.truncated || 0,records: ledger.sources,};diff --git a/api/_lib/coingecko.js b/api/_lib/coingecko.jsindex 61be0c9a9..de2c27886 100644--- a/api/_lib/coingecko.js+++ b/api/_lib/coingecko.js@@ -8,6 +8,8 @@// the upstream from concurrent cold-instance misses.import { cacheGet, cacheSet } from './cache.js';+import { recordSource } from './brownout/provenance.js';+import { applyFault, faultFor } from './brownout/chaos.js';import { createCache } from './mem-cache.js';export const COINGECKO_BASE = 'https://api.coingecko.com/api/v3';@@ -104,15 +106,31 @@ function keyWasRejected(headersUsed, status) {export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}) {const now = Date.now();const hit = _cache.get(path);- if (hit && hit.expiresAt > now) return hit.value;+ if (hit && hit.expiresAt > now) {+ // A cache hit inside its TTL is `cache`, not `live`: the answer is correct+ // and intended, but it did not come off the wire during this request, and a+ // reader deciding how much to trust a number deserves to know which.+ recordSource({ name: 'coingecko', outcome: 'ok', ms: 0, tier: 'cache' });+ return hit.value;+ }// A still-usable stale entry lets us ride out a throttled/failing upstream.const stale = hit && hit.staleUntil > now ? hit.value : null;const url = `${COINGECKO_BASE}${path}`;+ const startedAt = now;let resp;try {const sent = geckoHeaders();- resp = await fetch(url, { headers: sent, signal: AbortSignal.timeout(timeoutMs) });+ // CoinGecko's keyless tier is shared per egress IP and throttles constantly,+ // which makes it the single most useful upstream to be able to break on+ // purpose: most of this module exists to survive exactly that.+ const fault = faultFor('coingecko');+ if (fault) {+ const injected = await applyFault(fault, url);+ resp = injected ?? (await fetch(url, { headers: sent, signal: AbortSignal.timeout(timeoutMs) }));+ } else {+ resp = await fetch(url, { headers: sent, signal: AbortSignal.timeout(timeoutMs) });+ }// An exhausted/revoked demo key rejects requests the keyless tier would// still answer. Bench the key and retry once without it before falling// back to stale data — the live payload beats a cached one.@@ -129,14 +147,17 @@ export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}// Network failure or timeout — serve stale if we have it, else surface.if (stale !== null) {console.warn(`[coingecko] ${path} fetch failed (${netErr?.name || 'error'}); serving stale`);+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: 'stale' });return stale;}const durable = await durableStale(path);if (durable !== null) {console.warn(`[coingecko] ${path} fetch failed (${netErr?.name || 'error'}); serving durable last-good`);_cache.set(path, { value: durable, expiresAt: now, staleUntil: now + STALE_MS });+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: 'durable' });return durable;}+ recordSource({ name: 'coingecko', outcome: 'fail', ms: Date.now() - startedAt, tier: 'live', detail: netErr?.name === 'TimeoutError' ? 'timeout' : 'network' });throw netErr;}if (!resp.ok) {@@ -145,6 +166,7 @@ export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}// recent value beats an outage — serve it and keep the page alive.if (resp.status !== 404 && stale !== null) {console.warn(`[coingecko] ${path} → ${resp.status}; serving stale`);+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: resp.status });return stale;}if (resp.status !== 404) {@@ -154,14 +176,17 @@ export async function geckoFetch(path, { ttlMs = 60_000, timeoutMs = 8000 } = {}if (durable !== null) {console.warn(`[coingecko] ${path} → ${resp.status}; serving durable last-good`);_cache.set(path, { value: durable, expiresAt: now, staleUntil: now + STALE_MS });+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'stale', detail: resp.status });return durable;}}+ recordSource({ name: 'coingecko', outcome: 'fail', ms: Date.now() - startedAt, tier: 'live', detail: resp.status });const err = new Error(`CoinGecko ${resp.status} for ${path}`);err.status = resp.status;throw err;}const value = await resp.json();+ recordSource({ name: 'coingecko', outcome: 'ok', ms: Date.now() - startedAt, tier: 'live' });_cache.set(path, { value, expiresAt: now + ttlMs, staleUntil: now + ttlMs + STALE_MS });// Fire-and-forget durable mirror, never let a cache fault fail a live fetch.Promise.resolve(cacheSet(durableKey(path), value, DURABLE_STALE_S)).catch(() => {});diff --git a/api/_lib/http.js b/api/_lib/http.jsindex ca90ce984..e0feb76c6 100644--- a/api/_lib/http.js+++ b/api/_lib/http.js@@ -8,6 +8,8 @@ import { sendOpsAlert } from './alerts.js';import { instrument as zauthInstrument, drain as zauthDrain } from './zauth.js';import { isDbUnavailableError, isDbCapacityError, isStoragePressured } from './db.js';import { redactUrlSecrets } from './scrub-secrets.js';+import { provenanceHeaders, withProvenance } from './brownout/provenance.js';+import { CHAOS_HEADER, CHAOS_STATUS_HEADER, chaosDecision, withChaos } from './brownout/chaos.js';// Secure-by-default caching: emit `no-store` UNLESS the handler already set a// Cache-Control header (e.g. `res.setHeader('cache-control', 'public, s-maxage=…')`@@ -541,10 +543,49 @@ function isAllowedOrigin(origin, allowed) {return allowed.some((pat) => (typeof pat === 'string' ? origin === pat : pat.test(origin)));}+// Attach the request's data provenance to the response, and honour a chaos+// directive if this request carries a valid one.+//+// This sits in wrap() because wrap() is the one seam every handler already goes+// through: putting it here means a new endpoint reports its provenance without+// its author doing anything, which is the only way a guarantee like this stays+// true across hundreds of handlers. Headers are written just before the body+// goes out; a response already streaming keeps whatever it had, since headers+// cannot be added after the fact and a throw here must never cost a caller a+// working answer.+function attachProvenance(res) {+ try {+ if (res.headersSent || typeof res.setHeader !== 'function') return;+ const headers = provenanceHeaders();+ if (!headers) return;+ res.setHeader('x-brownout', headers.summary);+ if (headers.trace) res.setHeader('x-brownout-trace', headers.trace);+ // Browsers cannot read a response header unless it is exposed, and the+ // whole point of this is that a client can tell a degraded answer from a+ // live one.+ const exposed = res.getHeader('access-control-expose-headers');+ const want = headers.trace ? 'x-brownout, x-brownout-trace' : 'x-brownout';+ res.setHeader('access-control-expose-headers', exposed ? `${exposed}, ${want}` : want);+ } catch {+ /* provenance is telemetry: it never breaks a response */+ }+}+// Wrap async handlers so uncaught errors return a consistent JSON envelope.export function wrap(handler) {return async (req, res, ...rest) => {const monitored = zauthInstrument(req, res);+ const chaos = chaosDecision(req);+ if (req?.headers?.[CHAOS_HEADER] && typeof res.setHeader === 'function') {+ // Always answer the question the caller asked, including when the answer+ // is no. A prover that believes it broke an upstream and did not would+ // record a green proof for a fallback nothing exercised, which is worse+ // than a red one.+ try {+ res.setHeader(CHAOS_STATUS_HEADER, chaos.allowed ? `applied;faults=${chaos.faults.size}` : `refused;reason=${chaos.reason}`);+ } catch { /* header write is best-effort */ }+ }+ const run = () => withProvenance(() => withChaos(chaos.faults, async () => {try {await handler(req, res, ...rest);} catch (err) {@@ -654,6 +695,25 @@ export function wrap(handler) {}}}+ }));+ // The provenance header has to be written while the ledger is still in+ // scope AND before the body is flushed, so the res.end path is patched for+ // the duration of the handler rather than read afterwards. Every helper in+ // this module funnels through res.end, so one seam covers json(), error()+ // and a hand-rolled stream alike.+ const originalEnd = typeof res.end === 'function' ? res.end.bind(res) : null;+ if (originalEnd) {+ res.end = (...args) => {+ attachProvenance(res);+ res.end = originalEnd;+ return originalEnd(...args);+ };+ }+ try {+ await run();+ } finally {+ if (originalEnd) res.end = originalEnd;+ }// Keep the lambda alive briefly so the zauth SDK's in-flight POST to// back.zauthx402.com can finish. Cost: ~250ms of post-response runtime// on monitored requests only. The user has already received the response.diff --git a/api/_lib/upstream-fetch.js b/api/_lib/upstream-fetch.jsindex f7fda4c05..64c285075 100644--- a/api/_lib/upstream-fetch.js+++ b/api/_lib/upstream-fetch.js@@ -35,6 +35,8 @@import { withRetry, withBreaker, parseRetryAfter, isRetryableError } from './resilience.js';import { createCache } from './mem-cache.js';+import { recordSource } from './brownout/provenance.js';+import { applyFault, faultFor } from './brownout/chaos.js';export const DEFAULT_TIMEOUT_MS = 8_000;const DEFAULT_ATTEMPTS = 3;@@ -123,10 +125,21 @@ export async function fetchUpstream(url, init = {}, opts = {}) {} = opts;const shown = safeUrl(url);+ const t0 = Date.now();const attempt = async () => {let res;try {- res = await fetch(url, { ...init, signal: composeSignal(init.signal, timeoutMs) });+ // Brownout: a fault declared for this upstream, in this request only,+ // is raised in the shape of the real failure it stands in for, from+ // inside the same try that would have caught the genuine one. That+ // placement is the whole point: retries, Retry-After handling and the+ // breaker below all see it exactly as they would see the outage.+ const fault = faultFor(name || shown);+ if (fault) {+ const injected = await applyFault(fault, shown);+ if (injected) res = injected;+ }+ if (!res) res = await fetch(url, { ...init, signal: composeSignal(init.signal, timeoutMs) });} catch (err) {if (init.signal?.aborted) throw err;const e = new UpstreamError(`${shown}: ${err?.name === 'TimeoutError' ? `timed out after ${timeoutMs}ms` : err?.message || 'network error'}`, { url: shown, cause: err });@@ -162,8 +175,27 @@ export async function fetchUpstream(url, init = {}, opts = {}) {const shouldRetry = (err) => !err?.tooLongToWait && !init.signal?.aborted && isRetryableError(err);const run = () => withRetry(attempt, { attempts, shouldRetry, label });+ // Every third-party call in api/ goes through here, which makes this the one+ // place that can record what actually happened without every call site+ // remembering to. `live` because a value that reaches this return came off+ // the wire during this request; the cache tiers stamp their own.+ const label_ = name || shown;+ const noteOk = (v) => {+ recordSource({ name: label_, outcome: 'ok', ms: Date.now() - t0, tier: 'live' });+ return v;+ };+ const noteFail = (err) => {+ recordSource({+ name: label_,+ outcome: 'fail',+ ms: Date.now() - t0,+ tier: 'live',+ detail: err?.status ?? (err?.name === 'TimeoutError' ? 'timeout' : 'error'),+ });+ throw err;+ };- if (!name) return run();+ if (!name) return run().then(noteOk, noteFail);let failure;const value = await withBreaker(name, run, {threshold: breakerThreshold,@@ -173,8 +205,8 @@ export async function fetchUpstream(url, init = {}, opts = {}) {return null;},});- if (value) return value;- throw failure instanceof Error ? failure : new UpstreamError(`${label || name}: circuit open`, { url: shown, status: 503 });+ if (value) return noteOk(value);+ return noteFail(failure instanceof Error ? failure : new UpstreamError(`${label || name}: circuit open`, { url: shown, status: 503 }));}/**@@ -269,6 +301,9 @@ export async function lastGood(key, load, opts = {}) {const ageMs = hit ? Date.now() - hit.at : Infinity;if (hit && ageMs <= maxAgeMs) {onFallback?.(err, ageMs);+ // The single most important thing a caller can know about a response:+ // this value is older than it was meant to be, and by how much.+ recordSource({ name: `lastgood:${key}`.slice(0, 48), outcome: 'ok', ms: ageMs, tier: 'stale', detail: 'stale' });return { value: hit.value, stale: true, ageMs, error: err };}throw err;diff --git a/data/brownout.json b/data/brownout.jsonnew file mode 100644index 000000000..2c4897514--- /dev/null+++ b/data/brownout.json@@ -0,0 +1,66 @@+{+ "$comment": "Declared degradation contracts. Each one names an upstream, the fault to inject, and what the endpoint must still do. scripts/prove-brownout.mjs executes them against a live server and writes the verdicts to public/brownout.json, which /brownout renders. A contract whose fault never reached the code is reported as not_exercised, never as a pass: a green proof for a fallback nothing ran is worse than no proof at all.",+ "updated": "2026-08-27",+ "summary": "Every fallback below has been executed against a running server with the upstream it protects against genuinely broken.",+ "contracts": [+ {+ "id": "heatmap-survives-pump-throttle",+ "title": "The intel heatmap keeps its tiles when pump.fun throttles",+ "surface": "/intel",+ "endpoint": "/api/intel/heatmap",+ "query": { "limit": 6 },+ "bust": { "param": "limit", "min": 6, "max": 40 },+ "warm": true,+ "break": { "intel-heatmap": "http:429" },+ "expect": {+ "status": 200,+ "tier": ["cache", "stale", "fallback"],+ "degraded": true,+ "exercised": ["intel-heatmap"],+ "jsonHas": ["tokens"]+ },+ "why": "pump.fun answers a burst with 429 far more often than it goes down, and the board is the whole page. The retry honours Retry-After and the last-good tier carries the tiles through what is left, so a throttle costs freshness rather than the page."+ },+ {+ "id": "fear-greed-survives-its-source",+ "title": "The market page keeps its fear and greed reading when alternative.me is down",+ "surface": "/markets",+ "endpoint": "/api/coin/global",+ "warm": true,+ "break": { "alternative-me-fng": "network" },+ "expect": {+ "status": 200,+ "exercised": ["alternative-me-fng"]+ },+ "why": "A single keyless index feed. The page is assembled from several independent reads, so one of them failing must cost that one figure and nothing else."+ },+ {+ "id": "news-archive-survives-gcs",+ "title": "The news archive still answers when its bucket does not",+ "surface": "/news",+ "endpoint": "/api/news/archive",+ "query": { "limit": 5 },+ "warm": true,+ "break": { "news-archive": "http:503" },+ "expect": {+ "status": [200, 503],+ "exercised": ["news-archive"]+ },+ "why": "Archive data is immutable once written, which is what makes a remembered copy exactly as correct as a live one. With nothing ever cached the honest answer is a retryable 503, so both outcomes are accepted and the tier says which happened."+ },+ {+ "id": "token-page-survives-dexscreener",+ "title": "A token page still resolves when DexScreener is throttling",+ "surface": "/ca2x402",+ "endpoint": "/api/ca2x402/resolve",+ "query": { "mint": "FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump" },+ "warm": true,+ "break": { "dexscreener:tokens": "http:429" },+ "expect": {+ "status": [200, 404, 503],+ "exercised": ["dexscreener:tokens"]+ },+ "why": "One keyless read used to decide whether a token exists at all, so a throttle made a live token read as not found. It now retries behind a breaker with a last-known-good payload, and a definitive miss is reserved for cases where a source actually spoke."+ }+ ]+}diff --git a/scripts/audit-upstreams.mjs b/scripts/audit-upstreams.mjsnew file mode 100644index 000000000..bbfdf06b2--- /dev/null+++ b/scripts/audit-upstreams.mjs@@ -0,0 +1,273 @@+#!/usr/bin/env node+// Upstream resilience audit.+//+// npm run audit:upstreams human table, exits non-zero on a regression+// npm run audit:upstreams -- --json machine-readable report+// npm run audit:upstreams -- --map regenerate docs/resilience.md+// npm run audit:upstreams -- --accept re-baseline after a deliberate change+//+// Why a ratchet instead of a threshold. Every third-party call in this repo was+// once written without a deadline, and each was added by someone who had no way+// to see that they were adding the fifty-ninth one. A number in a doc does not+// stop that; a baseline that fails the build the moment an unguarded call+// appears does. The baseline records the grade of every call site we know+// about, so a NEW unguarded call fails while the existing tail is free to be+// paid down at its own pace, and paying it down updates the baseline downward+// automatically (it can never drift back up without --accept).+//+// See docs/resilience.md for the generated map and scripts/resilience/analyze.mjs+// for how a call site is classified.++import { readFileSync, writeFileSync, existsSync } from 'node:fs';+import { resolve, dirname, relative } from 'node:path';+import { fileURLToPath } from 'node:url';+import { execSync } from 'node:child_process';+import { analyzeFiles, gradeOf, byHost, PROTECTIONS } from './resilience/analyze.mjs';++const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');+const BASELINE = resolve(ROOT, 'data/upstream-baseline.json');+const MAP_DOC = resolve(ROOT, 'docs/resilience.md');++const args = new Set(process.argv.slice(2));+const asJson = args.has('--json');+const writeMap = args.has('--map');+const accept = args.has('--accept');++// Directories that ship runtime code. Tests are deliberately excluded: a test+// SHOULD be able to call a bare fetch against a stub.+const SCAN_DIRS = ['api', 'src', 'server', 'workers', 'mcp-server/src', 'avatar-sdk/src', 'public'];+// Built output is not source: grading a bundled asset reports findings nobody+// can act on, because the fix belongs in the module it was compiled from.+const SKIP_RE =+ /(?:^|\/)(?:node_modules|dist|dist-lib|build|coverage|\.deploy-wt|assets|vendor)(?:\/|$)|\.min\.js$|\.test\.js$|\.spec\.js$|-[A-Za-z0-9_-]{8,}\.js$/;++function sourceFiles() {+ const out = [];+ for (const dir of SCAN_DIRS) {+ const abs = resolve(ROOT, dir);+ if (!existsSync(abs)) continue;+ let listing;+ try {+ listing = execSync(`find ${JSON.stringify(abs)} -type f \\( -name '*.js' -o -name '*.mjs' \\)`, {+ encoding: 'utf8',+ maxBuffer: 64 * 1024 * 1024,+ });+ } catch {+ continue;+ }+ for (const line of listing.split('\n')) {+ const f = line.trim();+ if (!f) continue;+ if (SKIP_RE.test(relative(ROOT, f))) continue;+ out.push(f);+ }+ }+ return out.sort();+}++/** Stable id for a call site, so the baseline survives edits above it. */+function siteKey(site) {+ return `${site.file}::${site.host}::${site.via}`;+}++function loadBaseline() {+ if (!existsSync(BASELINE)) return null;+ try {+ return JSON.parse(readFileSync(BASELINE, 'utf8'));+ } catch {+ return null;+ }+}++const GRADE_RANK = { F: 0, D: 1, C: 2, B: 3, A: 4 };++function summarize(callSites) {+ const grades = { A: 0, B: 0, C: 0, D: 0, F: 0 };+ const counts = Object.fromEntries(PROTECTIONS.map((p) => [p, 0]));+ for (const s of callSites) {+ grades[gradeOf(s)] += 1;+ for (const p of s.protections) if (p in counts) counts[p] += 1;+ }+ return { total: callSites.length, grades, protections: counts };+}++/** Worst grade per site key, which is what the baseline pins. */+function gradeIndex(callSites) {+ const idx = {};+ for (const s of callSites) {+ const k = siteKey(s);+ const g = gradeOf(s);+ if (!(k in idx) || GRADE_RANK[g] < GRADE_RANK[idx[k]]) idx[k] = g;+ }+ return idx;+}++function renderMap(callSites, summary) {+ const hosts = byHost(callSites);+ const worst = hosts.filter((h) => h.worst === 'F' || h.worst === 'D');+ const lines = [];+ lines.push('<!-- GENERATED by scripts/audit-upstreams.mjs. Run `npm run audit:upstreams -- --map`. -->');+ lines.push('# Upstream resilience map');+ lines.push('');+ lines.push('Every third-party service this platform calls, and what happens to a user when');+ lines.push('that service is down. Generated from the source by an AST pass, so it cannot');+ lines.push('drift from the code the way a hand-written list does.');+ lines.push('');+ lines.push('## How a call site is graded');+ lines.push('');+ lines.push('| Grade | Meaning |');+ lines.push('|---|---|');+ lines.push('| A | Bounded, and survives the upstream being down (a second source, plus a remembered value or a retry) |');+ lines.push('| B | Bounded, with one way to survive: either another source or a remembered value |');+ lines.push('| C | Bounded, retries or trips a breaker, but has no second source |');+ lines.push('| D | Bounded only: a deadline and nothing behind it |');+ lines.push('| F | Unbounded: one stalled socket can outlive the request that opened it |');+ lines.push('');+ lines.push('A deadline is the floor rather than one grade among five, because without one');+ lines.push('no other protection ever gets a turn.');+ lines.push('');+ lines.push('## Totals');+ lines.push('');+ lines.push(`${summary.total} outbound call sites across ${hosts.length} upstreams.`);+ lines.push('');+ lines.push('| A | B | C | D | F |');+ lines.push('|---|---|---|---|---|');+ lines.push(+ `| ${summary.grades.A} | ${summary.grades.B} | ${summary.grades.C} | ${summary.grades.D} | ${summary.grades.F} |`,+ );+ lines.push('');+ if (worst.length) {+ lines.push('## Weakest upstreams');+ lines.push('');+ lines.push('These carry at least one call site with no second source behind it. They are');+ lines.push('the list to work down, worst first.');+ lines.push('');+ lines.push('| Upstream | Call sites | Worst grade |');+ lines.push('|---|---|---|');+ for (const h of worst.slice(0, 25)) {+ lines.push(`| \`${h.host}\` | ${h.sites} | ${h.worst} |`);+ }+ lines.push('');+ }+ lines.push('## Every upstream');+ lines.push('');+ lines.push('| Upstream | Call sites | Worst grade | A | B | C | D | F |');+ lines.push('|---|---|---|---|---|---|---|---|');+ for (const h of hosts) {+ lines.push(+ `| \`${h.host}\` | ${h.sites} | ${h.worst} | ${h.grades.A} | ${h.grades.B} | ${h.grades.C} | ${h.grades.D} | ${h.grades.F} |`,+ );+ }+ lines.push('');+ lines.push('## Keeping this true');+ lines.push('');+ lines.push('`npm run audit:upstreams` fails when a call site regresses below the grade');+ lines.push('recorded in `data/upstream-baseline.json`, or when a new unbounded call appears.');+ lines.push('Improving a call site lowers the baseline automatically; raising it back needs an');+ lines.push('explicit `--accept`, which is a reviewable line in the diff.');+ lines.push('');+ return lines.join('\n');+}++function main() {+ const files = sourceFiles();+ const { callSites, parseErrors } = analyzeFiles(files, ROOT);+ const summary = summarize(callSites);+ const index = gradeIndex(callSites);++ if (asJson) {+ process.stdout.write(+ `${JSON.stringify({ summary, hosts: byHost(callSites), callSites, parseErrors }, null, '\t')}\n`,+ );+ return 0;+ }++ if (writeMap) {+ writeFileSync(MAP_DOC, `${renderMap(callSites, summary)}`);+ console.log(`audit:upstreams: wrote ${relative(ROOT, MAP_DOC)}`);+ }++ const baseline = loadBaseline();++ if (accept || !baseline) {+ writeFileSync(+ BASELINE,+ `${JSON.stringify({ generated: 'scripts/audit-upstreams.mjs', summary, sites: index }, null, '\t')}\n`,+ );+ console.log(+ `audit:upstreams: baseline ${baseline ? 're-recorded' : 'created'} for ${summary.total} call sites ` ++ `(A:${summary.grades.A} B:${summary.grades.B} C:${summary.grades.C} D:${summary.grades.D} F:${summary.grades.F})`,+ );+ return 0;+ }++ // Regressions: a known site that got worse, or a new site below the floor.+ const regressed = [];+ const newUnbounded = [];+ for (const site of callSites) {+ const key = siteKey(site);+ const grade = gradeOf(site);+ const was = baseline.sites?.[key];+ if (was === undefined) {+ if (grade === 'F') newUnbounded.push({ site, grade });+ continue;+ }+ if (GRADE_RANK[grade] < GRADE_RANK[was]) regressed.push({ site, grade, was });+ }++ // Improvements ratchet the baseline down on their own, so the floor only+ // ever rises. Nobody has to remember to re-record a fix.+ let improved = 0;+ const nextSites = { ...(baseline.sites || {}) };+ for (const [key, grade] of Object.entries(index)) {+ const was = nextSites[key];+ if (was === undefined || GRADE_RANK[grade] > GRADE_RANK[was]) {+ if (was !== undefined) improved += 1;+ nextSites[key] = grade;+ }+ }+ for (const key of Object.keys(nextSites)) {+ if (!(key in index)) delete nextSites[key]; // call site removed+ }++ console.log(+ `audit:upstreams: ${summary.total} outbound call sites across ${byHost(callSites).length} upstreams ` ++ `(A:${summary.grades.A} B:${summary.grades.B} C:${summary.grades.C} D:${summary.grades.D} F:${summary.grades.F})`,+ );+ if (parseErrors.length) {+ console.log(`audit:upstreams: ${parseErrors.length} file(s) could not be parsed and were skipped`);+ }++ if (regressed.length || newUnbounded.length) {+ console.error('');+ for (const r of regressed) {+ console.error(+ `[upstreams] ${r.site.file}:${r.site.line} ${r.site.host} ${r.was} -> ${r.grade}`,+ );+ console.error(` protections: ${r.site.protections.join(', ') || 'none'}`);+ }+ for (const n of newUnbounded) {+ console.error(`[upstreams] ${n.site.file}:${n.site.line} ${n.site.host} NEW unbounded call`);+ console.error(' wrap it with fetchUpstream from api/_lib/upstream-fetch.js,');+ console.error(' or fetchFirst from src/shared/failover-fetch.js in browser code.');+ }+ console.error('');+ console.error(+ `audit:upstreams: ${regressed.length} regression(s), ${newUnbounded.length} new unbounded call(s).`,+ );+ console.error('If the change is deliberate, re-baseline with: npm run audit:upstreams -- --accept');+ return 1;+ }++ if (improved) {+ writeFileSync(+ BASELINE,+ `${JSON.stringify({ generated: 'scripts/audit-upstreams.mjs', summary, sites: nextSites }, null, '\t')}\n`,+ );+ console.log(`audit:upstreams: ${improved} call site(s) improved, baseline ratcheted up`);+ }+ console.log('audit:upstreams: no regressions');+ return 0;+}++process.exit(main());diff --git a/scripts/prove-brownout.mjs b/scripts/prove-brownout.mjsnew file mode 100644index 000000000..da193bdc3--- /dev/null+++ b/scripts/prove-brownout.mjs@@ -0,0 +1,270 @@+#!/usr/bin/env node+/**+ * Execute every declared degradation contract against a running server.+ *+ * "We have fallbacks" is a claim about code that, by construction, only runs+ * when something is broken. This turns the claim into a receipt: for each+ * contract it breaks the named upstream for one request, calls the real+ * endpoint, and checks what came back.+ *+ * The verdict that matters most is not pass or fail, it is `not_exercised`.+ * A fault can be accepted and still never reach the code, because a warm cache+ * answered before any upstream call happened. A prover that called that a pass+ * would be certifying a fallback nothing ran, which is worse than not testing+ * at all: it converts an unknown into a false assurance. So every contract+ * declares which sources must appear as FAILED in the response's provenance+ * trace, and a run that cannot show them is reported honestly.+ *+ * node scripts/prove-brownout.mjs # against localhost:8080+ * node scripts/prove-brownout.mjs --base https://three.ws --token $BROWNOUT_CHAOS_TOKEN+ * node scripts/prove-brownout.mjs --write public/brownout.json+ * node scripts/prove-brownout.mjs --json # machine-readable, no file+ *+ * Exit codes: 0 all contracts proven, 1 a contract failed its expectation,+ * 2 a contract could not be exercised (the fault never landed).+ */++import { readFileSync, writeFileSync, mkdirSync } from 'node:fs';+import { dirname } from 'node:path';++const REGISTRY = 'data/brownout.json';+const CHAOS_HEADER = 'x-brownout-chaos';+const TOKEN_HEADER = 'x-brownout-chaos-token';+const STATUS_HEADER = 'x-brownout-chaos-status';++function arg(flag, fallback = null) {+ const i = process.argv.indexOf(flag);+ return i >= 0 && process.argv[i + 1] ? process.argv[i + 1] : fallback;+}+const has = (flag) => process.argv.includes(flag);++/**+ * Parse the `x-brownout` summary and `x-brownout-trace` breakdown a response+ * carries. Kept deliberately tolerant: a missing header means "this response+ * recorded nothing", which is a real and meaningful state (everything was+ * served from cache), not a parse error.+ *+ * @param {Headers} headers+ * @returns {{ tier: string|null, degraded: boolean, ok: number, failed: number, ms: number|null, sources: Array<{name:string,outcome:string,ms:number}> }}+ */+export function parseProvenance(headers) {+ const get = (k) => (typeof headers?.get === 'function' ? headers.get(k) : headers?.[k]) || '';+ const summary = get('x-brownout');+ const trace = get('x-brownout-trace');+ const fields = {};+ for (const pair of summary.split(';')) {+ const [k, v] = pair.split('=');+ if (k) fields[k.trim()] = (v ?? '').trim();+ }+ const sources = trace+ ? trace.split(',').map((entry) => {+ const parts = entry.split(';').map((p) => p.trim());+ const name = parts[0] || '';+ const outcome = (parts.find((p) => p.startsWith('o=')) || 'o=').slice(2);+ const ms = Number((parts.find((p) => p.startsWith('t=')) || 't=0').slice(2));+ return { name, outcome, ms: Number.isFinite(ms) ? ms : 0 };+ })+ : [];+ return {+ tier: fields.tier || null,+ degraded: fields.degraded === '1',+ ok: Number(fields.ok || 0),+ failed: Number(fields.failed || 0),+ ms: fields.ms ? Number(fields.ms) : null,+ sources,+ };+}++/** A source counts as exercised when the trace shows it and it did not succeed. */+export function wasExercised(prov, name) {+ const want = String(name).toLowerCase();+ return prov.sources.some((s) => {+ const got = s.name.toLowerCase();+ if (got !== want && !got.startsWith(`${want}:`)) return false;+ return s.outcome !== 'ok';+ });+}++/** Build the request URL, applying the contract's cache-busting parameter. */+export function contractUrl(base, contract, seed = Date.now()) {+ const url = new URL(contract.endpoint, base);+ for (const [k, v] of Object.entries(contract.query || {})) url.searchParams.set(k, String(v));+ const bust = contract.bust;+ if (bust?.param) {+ // A warm cache is the single biggest source of a false green here, so the+ // request is deliberately steered off whatever key the last one used. The+ // range is the contract's, because only it knows what stays meaningful.+ const span = Math.max(1, (bust.max ?? 10) - (bust.min ?? 1) + 1);+ url.searchParams.set(bust.param, String((bust.min ?? 1) + (seed % span)));+ }+ return url.toString();+}++export function chaosDirective(contract) {+ return Object.entries(contract.break || {})+ .map(([name, spec]) => `${name}=${spec}`)+ .join(', ');+}++/** Check one response against a contract's expectations. Pure, so it is testable. */+export function judge(contract, { status, prov, body, chaosStatus }) {+ const expect = contract.expect || {};+ const problems = [];++ if (chaosStatus && !/^applied/.test(chaosStatus)) {+ return { verdict: 'not_exercised', problems: [`the server refused the fault: ${chaosStatus}`] };+ }++ for (const name of expect.exercised || []) {+ if (!wasExercised(prov, name)) {+ return {+ verdict: 'not_exercised',+ problems: [+ `\`${name}\` never appears as a failed source in the response trace, so the fault did not reach the code. ` ++ 'A warm cache almost certainly answered first; give the contract a `bust` parameter or widen it.',+ ],+ };+ }+ }++ const wantStatus = [].concat(expect.status ?? 200);+ if (!wantStatus.includes(status)) problems.push(`status ${status}, expected ${wantStatus.join(' or ')}`);++ if (expect.tier) {+ const wantTier = [].concat(expect.tier);+ if (!prov.tier) problems.push(`expected tier ${wantTier.join(' or ')} but the response recorded no provenance`);+ else if (!wantTier.includes(prov.tier)) problems.push(`tier ${prov.tier}, expected ${wantTier.join(' or ')}`);+ }++ if (expect.degraded === true && !prov.degraded) {+ problems.push('the response did not report itself as degraded, though its upstream was broken');+ }++ for (const key of expect.jsonHas || []) {+ if (!body || typeof body !== 'object' || !(key in body)) problems.push(`response body is missing \`${key}\``);+ }++ return { verdict: problems.length ? 'fail' : 'pass', problems };+}++async function runContract(base, token, contract, timeoutMs) {+ const started = Date.now();+ const seed = Math.floor(Math.random() * 1e6);++ // A contract whose fallback is a last-good tier needs something to fall back+ // TO. One clean request first is not cheating: it is the state every real+ // user's request arrives in, and a cold process with no memory is a different+ // contract (the honest 503) that other entries assert.+ if (contract.warm) {+ try {+ await fetch(contractUrl(base, contract, seed), { signal: AbortSignal.timeout(timeoutMs) });+ } catch {+ /* a failed warm-up is not a verdict; the real attempt below is */+ }+ }++ const url = contractUrl(base, contract, seed);+ let res;+ try {+ res = await fetch(url, {+ headers: { [CHAOS_HEADER]: chaosDirective(contract), [TOKEN_HEADER]: token, accept: 'application/json' },+ signal: AbortSignal.timeout(timeoutMs),+ });+ } catch (err) {+ return {+ id: contract.id,+ title: contract.title,+ verdict: 'fail',+ ms: Date.now() - started,+ problems: [`the request itself failed: ${err?.message || err}`],+ };+ }++ const text = await res.text().catch(() => '');+ let body = null;+ try {+ body = JSON.parse(text);+ } catch {+ /* a non-JSON body is judged on status and provenance alone */+ }+ const prov = parseProvenance(res.headers);+ const chaosStatus = res.headers.get(STATUS_HEADER);+ const { verdict, problems } = judge(contract, { status: res.status, prov, body, chaosStatus });++ return {+ id: contract.id,+ title: contract.title,+ surface: contract.surface || null,+ endpoint: contract.endpoint,+ broke: contract.break,+ why: contract.why || null,+ verdict,+ problems,+ ms: Date.now() - started,+ observed: {+ status: res.status,+ tier: prov.tier,+ degraded: prov.degraded,+ failedSources: prov.failed,+ trace: prov.sources.slice(0, 8),+ },+ };+}++async function main() {+ const base = arg('--base', process.env.BROWNOUT_BASE_URL || 'http://127.0.0.1:8080');+ const token = arg('--token', process.env.BROWNOUT_CHAOS_TOKEN || '');+ const timeoutMs = Number(arg('--timeout', '45000'));+ const outPath = arg('--write', null);+ const jsonOnly = has('--json');++ if (!token) {+ console.error('prove-brownout: no chaos token. Set BROWNOUT_CHAOS_TOKEN or pass --token.');+ console.error('Without it the server refuses every directive, which would report every contract as not_exercised.');+ process.exit(2);+ }++ const registry = JSON.parse(readFileSync(REGISTRY, 'utf8'));+ const results = [];+ for (const contract of registry.contracts) {+ results.push(await runContract(base, token, contract, timeoutMs));+ }++ const counts = results.reduce((acc, r) => ({ ...acc, [r.verdict]: (acc[r.verdict] || 0) + 1 }), {});+ const report = {+ generated_at: new Date().toISOString(),+ base,+ summary: registry.summary,+ counts,+ proven: counts.pass || 0,+ total: results.length,+ results,+ };++ if (outPath) {+ mkdirSync(dirname(outPath), { recursive: true });+ writeFileSync(outPath, `${JSON.stringify(report, null, '\t')}\n`);+ }+ if (jsonOnly) {+ console.log(JSON.stringify(report, null, 2));+ } else {+ console.log(`prove-brownout: ${report.proven}/${report.total} contracts proven against ${base}\n`);+ for (const r of results) {+ const mark = r.verdict === 'pass' ? 'PASS' : r.verdict === 'not_exercised' ? 'UNEX' : 'FAIL';+ console.log(` ${mark} ${r.id} (${r.ms}ms)`);+ console.log(` ${r.title}`);+ if (r.observed) {+ console.log(` got ${r.observed.status}, tier=${r.observed.tier || 'none'}, failed sources=${r.observed.failedSources}`);+ for (const s of r.observed.trace) console.log(` - ${s.name} ${s.outcome} ${s.ms}ms`);+ }+ for (const p of r.problems) console.log(` ! ${p}`);+ }+ if (outPath) console.log(`\nwrote ${outPath}`);+ }++ if (counts.fail) process.exit(1);+ if (counts.not_exercised) process.exit(2);+ process.exit(0);+}++if (import.meta.url === `file://${process.argv[1]}`) main();diff --git a/scripts/resilience/analyze.mjs b/scripts/resilience/analyze.mjsnew file mode 100644index 000000000..2411a1472--- /dev/null+++ b/scripts/resilience/analyze.mjs@@ -0,0 +1,424 @@+// AST analysis of every outbound call in the repo.+//+// Regex cannot answer the question this file exists to answer. "Does this call+// have a deadline" is not a property of the line the call is written on: the+// options object may be built three statements earlier, the deadline may arrive+// through a wrapper, and a `signal,` shorthand looks nothing like `signal:`.+// An earlier regex sweep of this same repo reported one unguarded call and was+// wrong twice over, in both directions. Parsing is what makes the audit+// trustworthy enough to gate a deploy on.+//+// What it produces, per call site: which upstream is being talked to, and which+// of the five protections are present (deadline, retry, failover, lastGood,+// breaker). Everything downstream of this module (the grade, the ratchet, the+// generated map) is a projection of that one table.++import { readFileSync } from 'node:fs';+import { relative, resolve, dirname, join } from 'node:path';+import * as acorn from 'acorn';++// A local walker rather than acorn-walk. This module gates a deploy, so it is+// deliberately free of dependencies beyond the parser itself: the traversal it+// needs is a generic descent that carries the ancestor chain, which is shorter+// than the code that would justify pulling a package in for it.+function walkWithAncestors(node, visit, ancestors = []) {+ if (node === null || typeof node !== 'object') return;+ if (Array.isArray(node)) {+ for (const child of node) walkWithAncestors(child, visit, ancestors);+ return;+ }+ if (typeof node.type !== 'string') return;+ visit(node, ancestors);+ ancestors.push(node);+ for (const key of Object.keys(node)) {+ if (key === 'type' || key === 'loc' || key === 'start' || key === 'end') continue;+ walkWithAncestors(node[key], visit, ancestors);+ }+ ancestors.pop();+}++// Helpers whose use IS the protection: calling one of these is what a guarded+// call site looks like in this repo, so the analyzer resolves them by name+// rather than re-deriving their internals at every call site. Adding a new+// shared helper means adding it here, which is the one maintenance cost.+export const GUARDS = {+ fetchUpstream: ['deadline', 'retry', 'breaker'],+ fetchUpstreamJson: ['deadline', 'retry', 'breaker'],+ fetchAnyJson: ['deadline', 'retry', 'failover'],+ lastGood: ['lastGood'],+ lastGoodValue: ['lastGood'],+ cacheWrapLastGood: ['lastGood'],+ fetchFirst: ['deadline', 'failover'],+ fetchFirstOrNull: ['deadline', 'failover'],+ pumpFetchJson: ['deadline', 'retry'],+ geckoFetch: ['deadline', 'lastGood'],+ paprikaGet: ['deadline'],+ withRetry: ['retry'],+ withBreaker: ['breaker'],+ withRetryAndBreaker: ['retry', 'breaker'],+ makeRotatingFetch: ['deadline', 'failover'],+ // The rotating RPC clients bound each attempt themselves (see+ // makeRotatingFetch in api/_lib/solana/connection.js), so using one is a+ // bounded call even though no deadline is written at the call site.+ solanaConnection: ['deadline', 'failover'],+ evmFallbackProvider: ['deadline', 'failover'],+ evmTransport: ['deadline', 'failover'],+ resolveURI: ['deadline', 'failover'],+ fetchWithFallback: ['deadline', 'failover'],+ loadModule: ['deadline', 'failover'],+ loadScript: ['deadline', 'failover'],+};++// Guards whose first argument is a LIST of providers rather than one URL. Their+// whole point is that several hosts back one read, so each entry is recorded as+// its own call site: that is what makes a failover chain visible per upstream+// instead of collapsing to a single anonymous row.+export const LIST_GUARDS = new Set(['fetchFirst', 'fetchFirstOrNull', 'fetchAnyJson']);++export const PROTECTIONS = ['deadline', 'retry', 'failover', 'lastGood', 'breaker'];++// A relative URL, or one aimed at this machine, is our own origin: covered by+// the same-origin client rather than by upstream failover, so it is out of scope.+const INTERNAL_URL_RE = /^(?:\/|https?:\/\/(?:localhost|127\.0\.0\.1|\[::1\]|(?:[a-z0-9-]+\.)*three\.ws))/i;++// Marks the position of a `${...}` hole in a reconstructed template literal.+const HOLE = '__EXPR__';++/** Hostname for a URL-ish string, or null when it is relative or unknowable. */+export function hostOf(raw) {+ if (typeof raw !== 'string') return null;+ const m = raw.match(/^https?:\/\/([^/?#]+)/i);+ return m ? m[1].toLowerCase() : null;+}++// A template literal's static skeleton: `${BASE}/coins/${mint}` becomes+// "<HOLE>/coins/<HOLE>", which is enough to recognise an absolute URL and to+// name the upstream whenever the host half is written literally.+function staticText(node, consts = null) {+ if (!node) return '';+ if (node.type === 'Literal') return typeof node.value === 'string' ? node.value : '';+ if (node.type === 'TemplateLiteral') {+ // Substitute a known module-level constant into its hole so+ // `${PUMP_BASE}/coins/x` is attributed to the host PUMP_BASE holds,+ // instead of collapsing into an anonymous "dynamic" row.+ return node.quasis+ .map((q, i) => {+ if (i >= node.expressions.length) return q.value.cooked;+ const filled = staticText(node.expressions[i], consts);+ return q.value.cooked + (filled && filled !== HOLE ? filled : HOLE);+ })+ .join('');+ }+ if (node.type === 'BinaryExpression' && node.operator === '+') {+ return staticText(node.left, consts) + staticText(node.right, consts);+ }+ if (node.type === 'Identifier') {+ const known = consts?.get(node.name);+ return known !== undefined ? known : HOLE;+ }+ if (node.type === 'MemberExpression') return HOLE;+ return '';+}++/** Does this options-object AST node carry a deadline? */+function objectHasDeadline(node) {+ if (!node || node.type !== 'ObjectExpression') return false;+ return node.properties.some((p) => {+ if (p.type === 'SpreadElement') return false;+ const key = p.key?.name ?? p.key?.value;+ // `signal: x`, the `signal` shorthand, and an explicit timeout all count.+ return key === 'signal' || key === 'timeout' || key === 'timeoutMs';+ });+}++function calleeName(node) {+ if (!node) return null;+ if (node.type === 'Identifier') return node.name;+ if (node.type === 'MemberExpression') return node.property?.name ?? null;+ return null;+}++function displayUrl(text) {+ return text.split(HOLE).join('${}');+}++/**+ * Analyze one source file.+ *+ * @param {string} code+ * @param {string} file path used for reporting+ * @returns {{ callSites: Array<object>, parseError: string | null }}+ */+export function analyzeSource(code, file, resolveImported = null) {+ let ast;+ try {+ ast = acorn.parse(code, {+ ecmaVersion: 'latest',+ sourceType: 'module',+ allowAwaitOutsideFunction: true,+ allowHashBang: true,+ allowReturnOutsideFunction: true,+ locations: true,+ });+ } catch (err) {+ return { callSites: [], parseError: err.message };+ }++ // Which guard helpers this file imports, so a bare call to one is+ // attributable to the shared implementation rather than to a local function+ // that happens to share its name.+ // Module-level `const X = 'https://...'` (and simple concatenations of them)+ // are the base URLs nearly every call site is written against, so resolving+ // them is what turns a wall of "dynamic" rows into named upstreams.+ const consts = new Map();+ // Provider chains are frequently hoisted to a module constant+ // (`export const PROVIDERS = [...]`) and passed to fetchFirst by name, so the+ // array node has to be resolvable from the identifier or the entire chain is+ // invisible to the audit.+ const arrayConsts = new Map();+ for (const stmt of ast.body || []) {+ const decl = stmt.type === 'ExportNamedDeclaration' ? stmt.declaration : stmt;+ if (!decl || decl.type !== 'VariableDeclaration') continue;+ for (const d of decl.declarations) {+ if (d.id?.type !== 'Identifier' || !d.init) continue;+ if (d.init.type === 'ArrayExpression') arrayConsts.set(d.id.name, d.init);+ const text = staticText(d.init, consts);+ if (text && !text.includes(HOLE)) consts.set(d.id.name, text);+ }+ }++ const imported = new Set();+ // local binding name -> { from, name }, so a URL built on an imported base+ // constant can be resolved against the module that exports it.+ const importBindings = new Map();+ walkWithAncestors(ast, (n) => {+ if (n.type !== 'ImportDeclaration') return;+ const from = typeof n.source?.value === 'string' ? n.source.value : null;+ for (const spec of n.specifiers || []) {+ const name = spec.imported?.name ?? spec.local?.name;+ if (name) imported.add(name);+ if (from && spec.local?.name) {+ importBindings.set(spec.local.name, { from, name: spec.imported?.name ?? spec.local.name });+ }+ }+ });++ // Imported constants resolve through the caller-supplied resolver, which the+ // two-pass driver backs with every exported string constant in the repo.+ if (resolveImported) {+ for (const [local, ref] of importBindings) {+ const value = resolveImported(file, ref.from, ref.name);+ if (typeof value === 'string' && value && !consts.has(local)) consts.set(local, value);+ }+ }++ const callSites = [];++ // A guard call anywhere up the ancestor chain puts its protections in scope+ // for the raw fetch it wraps: `withRetry(() => fetch(url))` is one guarded+ // site, not a guard plus a naked fetch. Tracking ancestors is what tells+ // those two shapes apart.+ walkWithAncestors(ast, (node, ancestors) => {+ if (node.type !== 'CallExpression') return;+ {+ const name = calleeName(node.callee);+ if (!name) return;++ const inherited = new Set();+ for (const a of ancestors) {+ if (a === node || a.type !== 'CallExpression') continue;+ const outer = calleeName(a.callee);+ if (outer && GUARDS[outer] && imported.has(outer)) {+ for (const g of GUARDS[outer]) inherited.add(g);+ }+ }++ const isGuard = Boolean(GUARDS[name]) && imported.has(name);+ if (!isGuard && name !== 'fetch') return;++ // A provider-list guard: unpack the array so every host in the chain is+ // counted, each carrying the chain's protections.+ if (isGuard && LIST_GUARDS.has(name)) {+ let arr = node.arguments[0];+ if (arr?.type === 'Identifier') arr = arrayConsts.get(arr.name) ?? arr;+ if (arr?.type !== 'ArrayExpression') return;+ const chainProtections = new Set(inherited);+ for (const g of GUARDS[name]) chainProtections.add(g);+ for (const el of arr.elements) {+ if (!el) continue;+ // Either a bare URL string or a { name, url, parse } provider object.+ let entryUrl = '';+ if (el.type === 'ObjectExpression') {+ const urlProp = el.properties.find(+ (pr) => pr.type !== 'SpreadElement' && (pr.key?.name ?? pr.key?.value) === 'url',+ );+ entryUrl = urlProp ? staticText(urlProp.value, consts) : '';+ } else {+ entryUrl = staticText(el, consts);+ }+ if (!entryUrl || !entryUrl.includes('://')) continue;+ callSites.push({+ file,+ line: (el.loc ?? node.loc).start.line,+ via: name,+ host: hostOf(entryUrl) ?? 'dynamic',+ url: displayUrl(entryUrl).slice(0, 120),+ protections: [...chainProtections].sort(),+ });+ }+ return;+ }++ const urlText = staticText(node.arguments[0], consts);+ if (!urlText || INTERNAL_URL_RE.test(urlText)) return;+ // Only grade calls we can PROVE leave this origin. A bare `fetch(path)`+ // or a `${base}/api/x` is same-origin or unknowable, and counting those+ // as unguarded upstreams buries the real ones: an earlier pass of this+ // analyzer reported 513 unbounded calls, of which 507 were same-origin.+ // A scheme in the resolved text is the proof; anything else is skipped.+ if (!urlText.includes('://')) return;+ const host = hostOf(urlText);++ const protections = new Set(inherited);+ if (isGuard) for (const g of GUARDS[name]) protections.add(g);+ // An explicit per-call deadline counts wherever it appears.+ for (const arg of node.arguments.slice(1)) {+ if (objectHasDeadline(arg)) protections.add('deadline');+ }++ callSites.push({+ file,+ line: node.loc.start.line,+ via: isGuard ? name : 'fetch',+ host: host ?? 'dynamic',+ url: displayUrl(urlText).slice(0, 120),+ protections: [...protections].sort(),+ });+ }+ });++ return { callSites, parseError: null };+}++/**+ * Every module-level string constant a file defines, exported or not, resolved+ * as far as literal concatenation allows. Pass 1 of the two-pass analysis.+ * @returns {Map<string, string>}+ */+export function collectStringConstants(code) {+ const out = new Map();+ let ast;+ try {+ ast = acorn.parse(code, { ecmaVersion: 'latest', sourceType: 'module', allowHashBang: true });+ } catch {+ return out;+ }+ for (const stmt of ast.body || []) {+ const decl = stmt.type === 'ExportNamedDeclaration' ? stmt.declaration : stmt;+ if (!decl || decl.type !== 'VariableDeclaration') continue;+ for (const d of decl.declarations) {+ if (d.id?.type !== 'Identifier' || !d.init) continue;+ const text = staticText(d.init, out);+ if (text && !text.includes(HOLE)) out.set(d.id.name, text);+ }+ }+ return out;+}++/**+ * Analyze a list of absolute paths, reported relative to `root`.+ * @returns {{ callSites: Array<object>, parseErrors: Array<object> }}+ */+export function analyzeFiles(files, root) {+ const callSites = [];+ const parseErrors = [];++ // Pass 1: every module's string constants. Base URLs in this repo are+ // overwhelmingly `export const PUMP_FRONTEND_BASE = 'https://...'` in one+ // module and imported everywhere else, so without this pass the majority of+ // call sites cannot be attributed to the host they actually talk to.+ const sources = new Map();+ const constsByFile = new Map();+ for (const abs of files) {+ let code;+ try {+ code = readFileSync(abs, 'utf8');+ } catch {+ continue;+ }+ sources.set(abs, code);+ constsByFile.set(abs, collectStringConstants(code));+ }++ const byRel = new Map([...sources.keys()].map((abs) => [relative(root, abs), abs]));++ // Resolve `import { NAME } from './x.js'` against pass 1, trying the+ // extensions and index forms Node would.+ const resolveImported = (fromRel, spec, name) => {+ if (!spec.startsWith('.')) return undefined;+ const baseDir = dirname(resolve(root, fromRel));+ const target = resolve(baseDir, spec);+ const candidates = [+ target,+ `${target}.js`,+ `${target}.mjs`,+ join(target, 'index.js'),+ join(target, 'index.mjs'),+ ];+ for (const cand of candidates) {+ const consts = constsByFile.get(cand);+ if (consts?.has(name)) return consts.get(name);+ }+ return undefined;+ };++ // Pass 2: grade every call site with imported constants resolved.+ for (const [abs, code] of sources) {+ const rel = relative(root, abs);+ const { callSites: found, parseError } = analyzeSource(code, rel, resolveImported);+ if (parseError) parseErrors.push({ file: rel, error: parseError });+ callSites.push(...found);+ }+ return { callSites, parseErrors, filesScanned: sources.size, byRel };+}++/**+ * A call site's grade. `deadline` is the floor, because without one a single+ * stalled socket outlives the request that opened it and no other protection+ * ever gets a turn. Above the floor, what earns a grade is having somewhere else+ * to go (failover) or something to serve (lastGood).+ *+ * A bounded, and survives the upstream being down+ * B bounded, with one way to survive+ * C bounded, and re-tries or trips, but has no second source+ * D bounded only+ * F unbounded: can hang the request+ */+export function gradeOf(site) {+ const p = new Set(site.protections);+ if (!p.has('deadline')) return 'F';+ if (p.has('failover') && (p.has('lastGood') || p.has('retry'))) return 'A';+ if (p.has('failover') || p.has('lastGood')) return 'B';+ if (p.has('retry') || p.has('breaker')) return 'C';+ return 'D';+}++/** Roll call sites up per upstream host, worst grade first. */+export function byHost(callSites) {+ const order = { F: 0, D: 1, C: 2, B: 3, A: 4 };+ const hosts = new Map();+ for (const site of callSites) {+ let h = hosts.get(site.host);+ if (!h) {+ h = { host: site.host, sites: 0, worst: 'A', grades: { A: 0, B: 0, C: 0, D: 0, F: 0 } };+ hosts.set(site.host, h);+ }+ const g = gradeOf(site);+ h.sites += 1;+ h.grades[g] += 1;+ if (order[g] < order[h.worst]) h.worst = g;+ }+ return [...hosts.values()].sort(+ (a, b) => order[a.worst] - order[b.worst] || b.sites - a.sites,+ );+}diff --git a/scripts/resilience/contracts.mjs b/scripts/resilience/contracts.mjsnew file mode 100644index 000000000..9f8dc5e75--- /dev/null+++ b/scripts/resilience/contracts.mjs@@ -0,0 +1,186 @@+// Degradation contracts.+//+// A fallback is not automatically an improvement. Serving a remembered value is+// the right answer for a price chip and the wrong answer for a question whose+// whole purpose is to be current: this repo shipped four of those in one week.+// A payment retry re-signed the blockhash the facilitator had just refused. A+// token security endpoint assembled a clean bill of health out of remembered+// on-chain state at the exact moment the chain was unreadable. An access gate+// answered from a cached verdict. An alert feed went quiet in a way that looked+// like a calm market.+//+// Every one of those passed a "does it have a fallback" review. What none of+// them had was a stated answer to a different question: when this upstream is+// down, what is this endpoint ALLOWED to say? That is the contract, and this+// module is the vocabulary plus the harness that proves it by actually breaking+// the upstream and looking at the response.++/**+ * The contracts an endpoint can declare.+ *+ * MAY_SERVE_STALE+ * A remembered value is a better answer than an error, and the response says+ * so (a stale marker, an as_of stamp, or a header). Prices, feeds, boards.+ *+ * MUST_REFUSE+ * There is no honest answer without a live read, so the endpoint must return a+ * typed error rather than invent one. The failure mode this prevents is not a+ * crash: it is a confident, wrong, cacheable 200.+ *+ * MUST_NOT_SERVE_STALE+ * Stronger than MUST_REFUSE. The endpoint may answer 200 from live data, but+ * a remembered value must never reach the caller, because acting on a stale+ * answer costs money or grants access. Payment retries, security verdicts,+ * authorization gates.+ */+export const CONTRACT = {+ MAY_SERVE_STALE: 'may-serve-stale',+ MUST_REFUSE: 'must-refuse',+ MUST_NOT_SERVE_STALE: 'must-not-serve-stale',+};++export const CONTRACT_VALUES = new Set(Object.values(CONTRACT));++/** Status codes that count as a typed refusal rather than a crash. */+const REFUSAL_STATUSES = new Set([424, 425, 429, 502, 503, 504]);++/**+ * Markers that say "this body is remembered, not live". A contract that allows+ * stale data requires one of these to be present, because degrading silently is+ * how a stale answer gets mistaken for a current one.+ */+export function staleMarkers(body, headers = {}) {+ const found = [];+ if (body && typeof body === 'object') {+ if (body.stale === true) found.push('body.stale');+ if (body.as_of !== undefined && body.as_of !== null) found.push('body.as_of');+ if (body.data && typeof body.data === 'object') {+ if (body.data.stale === true) found.push('body.data.stale');+ if (body.data.as_of !== undefined && body.data.as_of !== null) found.push('body.data.as_of');+ }+ }+ for (const key of Object.keys(headers || {})) {+ const k = key.toLowerCase();+ if (k.includes('stale')) found.push(`header.${k}`);+ }+ return found;+}++/**+ * Judge one observed response against a contract.+ *+ * Deliberately pure and transport-agnostic: the caller performs the fault+ * injection however its endpoint needs, then hands the outcome here. That keeps+ * the judgement identical whether the response came from an HTTP handler, an+ * exported library function, or a worker.+ *+ * @param {string} contract one of CONTRACT+ * @param {{ status?: number, body?: any, headers?: object, threw?: unknown }} observed+ * @returns {{ ok: boolean, reason: string }}+ */+export function judge(contract, observed) {+ const { status, body, headers = {}, threw } = observed;++ if (threw !== undefined && threw !== null) {+ // A rejection that carries a typed status is a refusal expressed as an+ // exception, which the HTTP boundary turns into the same envelope.+ const typed = Number(threw?.status ?? threw?.statusCode);+ if (contract === CONTRACT.MAY_SERVE_STALE) {+ return { ok: false, reason: `threw instead of serving a remembered value: ${threw?.message || threw}` };+ }+ if (Number.isFinite(typed) && REFUSAL_STATUSES.has(typed)) {+ return { ok: true, reason: `refused with a typed ${typed}` };+ }+ if (threw?.code) return { ok: true, reason: `refused with code ${threw.code}` };+ return { ok: false, reason: `threw an untyped error: ${threw?.message || threw}` };+ }++ const markers = staleMarkers(body, headers);+ const is2xx = status === undefined || (status >= 200 && status < 300);++ switch (contract) {+ case CONTRACT.MAY_SERVE_STALE: {+ if (!is2xx) {+ // Refusing is allowed only when nothing was ever remembered; the+ // caller distinguishes those two cases by warming the endpoint first.+ return REFUSAL_STATUSES.has(status)+ ? { ok: true, reason: `refused with a typed ${status} (nothing remembered yet)` }+ : { ok: false, reason: `answered ${status}, which is neither stale data nor a typed refusal` };+ }+ if (!markers.length) {+ return {+ ok: false,+ reason: 'served a 200 during a total upstream outage without marking it stale, so a remembered value is indistinguishable from a live one',+ };+ }+ return { ok: true, reason: `served remembered data, marked by ${markers.join(', ')}` };+ }++ case CONTRACT.MUST_REFUSE: {+ if (is2xx) {+ return {+ ok: false,+ reason: 'answered 2xx with every upstream down, which states as fact something it could not check',+ };+ }+ return REFUSAL_STATUSES.has(status)+ ? { ok: true, reason: `refused with a typed ${status}` }+ : { ok: false, reason: `answered ${status}, which is not a typed "try again" refusal` };+ }++ case CONTRACT.MUST_NOT_SERVE_STALE: {+ if (is2xx && markers.length) {+ return {+ ok: false,+ reason: `served remembered data (${markers.join(', ')}) on a path where acting on a stale answer costs money or grants access`,+ };+ }+ if (is2xx) {+ return {+ ok: false,+ reason: 'answered 2xx with every upstream down, so the answer came from somewhere other than a live read',+ };+ }+ return REFUSAL_STATUSES.has(status)+ ? { ok: true, reason: `refused with a typed ${status} rather than reusing a remembered answer` }+ : { ok: false, reason: `answered ${status}, which is not a typed refusal` };+ }++ default:+ return { ok: false, reason: `unknown contract "${contract}"` };+ }+}++/**+ * A fetch replacement that fails the way a dead upstream fails, for injecting a+ * total outage. `mode` picks the shape of the failure, because a refused+ * connection and a socket that accepts and never answers are different bugs and+ * have caught different ones here.+ *+ * @param {'network' | 'timeout' | 'status'} mode+ * @param {{ status?: number }} [opts]+ */+export function failingFetch(mode = 'network', { status = 503 } = {}) {+ if (mode === 'status') {+ return async () =>+ new Response(JSON.stringify({ error: 'upstream is down' }), {+ status,+ headers: { 'content-type': 'application/json' },+ });+ }+ if (mode === 'timeout') {+ return async (_url, init) =>+ new Promise((_resolve, reject) => {+ const signal = init?.signal;+ const fail = () => reject(Object.assign(new Error('The operation was aborted'), { name: 'AbortError' }));+ if (!signal) return; // no deadline: hangs, which is itself the finding+ if (signal.aborted) return fail();+ signal.addEventListener('abort', fail, { once: true });+ });+ }+ return async () => {+ const err = new TypeError('fetch failed');+ err.cause = { code: 'ECONNREFUSED' };+ throw err;+ };+}diff --git a/src/shared/failover-fetch.js b/src/shared/failover-fetch.jsindex 335fcbc6e..a15308fca 100644--- a/src/shared/failover-fetch.js+++ b/src/shared/failover-fetch.js@@ -26,6 +26,50 @@const _cooldowns = new Map(); // provider name -> epoch ms until which it is skipped+// Brownout hooks. This module is isomorphic (the browser bundle imports it), and+// provenance/chaos are node:async_hooks, so they are loaded lazily behind an+// isServer() gate exactly as api/_lib/solana/connection.js loads the shared+// cache. In a browser both resolve to no-ops, which is correct: there is no+// request to attribute a source to, and nothing should be able to inject a+// fault from the client side.+let _brownoutPromise;+// Resolved module, once the import lands. Recording has to be SYNCHRONOUS in the+// hot path: the response header is written when the handler ends, and an await+// here would let a record arrive after its own response had already gone out.+// That is not a lost log line, it is a trace that omits the rung that answered.+let _brownoutMod = null;+function brownout() {+ if (_brownoutPromise !== undefined) return _brownoutPromise;+ _brownoutPromise = null;+ if (typeof window === 'undefined') {+ _brownoutPromise = import('../../api/_lib/brownout/index.js')+ .then((m) => {+ _brownoutMod = m;+ return m;+ })+ .catch(() => null);+ }+ return _brownoutPromise;+}+// Warm the import at module load on the server, so by the time a request runs+// the module is in hand and every record is written synchronously.+brownout();++function noteSource(rec) {+ if (_brownoutMod) {+ _brownoutMod.recordSource?.(rec);+ return;+ }+ // Only on the very first call of a cold process, before the import lands.+ const pending = brownout();+ if (pending) void pending.then((m) => m?.recordSource?.(rec));+}+async function faultForProvider(name) {+ if (_brownoutMod) return _brownoutMod.faultFor?.(name) ?? null;+ const mod = await brownout();+ return mod?.faultFor?.(name) ?? null;+}+/*** @typedef {object} Provider* @property {string} name Stable id; keys the cooldown map.@@ -55,24 +99,51 @@ export async function fetchFirst(providers, { timeoutMs = 4000, cooldownMs = 60_let lastErr;for (const p of order) {+ const startedAt = Date.now();const ctrl = new AbortController();const timer = setTimeout(() => ctrl.abort(), timeoutMs);// A caller-supplied signal (stale-search abort) composes with the// per-attempt timeout rather than being replaced by it.const signal = p.init?.signal ? AbortSignal.any([ctrl.signal, p.init.signal]) : ctrl.signal;try {- const res = await fetch(p.url, {- headers: { accept: 'application/json' },- ...p.init,- signal,- });+ // A declared fault stands in for the real one, in the same position the+ // real failure would occupy, so the ladder below fails over identically.+ const fault = await faultForProvider(p.name);+ let res;+ if (fault) {+ const mod = _brownoutMod || (await brownout());+ const injected = await mod.applyFault(fault, p.url);+ if (injected) res = injected;+ }+ if (!res) {+ res = await fetch(p.url, {+ headers: { accept: 'application/json' },+ ...p.init,+ signal,+ });+ }if (!res.ok) throw new Error(`http_${res.status}`);const value = await (p.parse ? p.parse(res) : res.json());- if (value != null) return { value, source: p.name };+ if (value != null) {+ // The rung that answered is `live`; the ones before it are recorded+ // in the catch below, so a trace shows the whole walk, not just the+ // winner. That is the difference between "we have a ladder" and+ // "here is the ladder working".+ noteSource({ name: p.name, outcome: 'ok', ms: Date.now() - startedAt, tier: 'live' });+ return { value, source: p.name };+ }// Miss: provider is healthy but has no data for this query.+ noteSource({ name: p.name, outcome: 'skip', ms: Date.now() - startedAt, tier: 'live', detail: 'no_data' });lastErr = new Error(`${p.name}: no_data`);} catch (err) {lastErr = err instanceof Error ? err : new Error(String(err));+ noteSource({+ name: p.name,+ outcome: 'fail',+ ms: Date.now() - startedAt,+ tier: 'live',+ detail: /http_(\d{3})/.exec(lastErr.message)?.[1] || (lastErr.name === 'TimeoutError' ? 'timeout' : 'error'),+ });// Caller abandoned the request (stale search) — stop the whole chain// and don't penalise the provider for it.if (p.init?.signal?.aborted) {