Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(brownout): make every response say where its data came from, and let a caller break one upstream on purpose

nirholasf30f7cbfa81c
diff --git a/api/_lib/brownout/chaos.js b/api/_lib/brownout/chaos.js
new file mode 100644
index 000000000..2c5e12ecb
--- /dev/null
+++ b/api/_lib/brownout/chaos.js
@@ -0,0 +1,217 @@
+// @ts-check
+// Break a named upstream on purpose, for one request, in the real code path.
+//
+// Every fallback in this codebase is untested code until something makes the
+// thing it protects against actually happen. Unit tests stub `fetch` and prove
+// the stub was called; they cannot prove that /api/pump/dashboard returns a
+// usable page when Birdeye 429s, because the stub replaces the very layer whose
+// behaviour is in question. Infra-level chaos (kill a pod, drop a route) is the
+// usual answer and is far too blunt: it cannot express "Birdeye is rate
+// limiting but DexScreener is fine", which is the failure that actually happens.
+//
+// So the injection point is the shared fetch wrappers, and the scope is one
+// request:
+//
+// x-brownout-chaos: birdeye=http:429, tokens-xyz=timeout, dexscreener=network
+//
+// The request runs the REAL handler, the REAL provider ladder, the REAL cache
+// tiers. Only the named upstreams misbehave, and only for the caller who asked.
+// Nothing global changes, so a probe can run against production traffic without
+// touching anybody else's request.
+//
+// ── Safety ──────────────────────────────────────────────────────────────────
+// This is a switch that makes production misbehave, so it is gated three ways
+// and every one of them must pass:
+//
+// 1. A token. BROWNOUT_CHAOS_TOKEN must be set and match, compared in
+// constant time. Without the env var configured, chaos is off everywhere,
+// including locally, so there is no "forgot to set it in prod" state that
+// leaves it open.
+// 2. Never on a money path. A request carrying an x402 payment header, or
+// addressed to a settlement/withdraw/transfer route, is refused outright.
+// You must not be able to fault-inject something that moves funds, no
+// matter who holds the token: the blast radius of a wrong answer there is
+// not a stale price, it is a payment.
+// 3. Read-shaped methods only. GET/HEAD by default; a POST is allowed only
+// when it is a declared read (a JSON-RPC query, a search), never a write.
+//
+// A refused directive is reported back in `x-brownout-chaos-status` rather than
+// silently ignored, because a probe that thinks it broke Birdeye and did not is
+// worse than one that failed loudly: it would record a green proof for a
+// fallback nobody exercised.
+
+import { timingSafeEqual } from 'node:crypto';
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+/** @typedef {{ kind: 'timeout'|'network'|'http'|'empty'|'slow', status?: number, ms?: number }} Fault */
+
+/** @type {AsyncLocalStorage<Map<string, Fault>>} */
+const storage = new AsyncLocalStorage();
+
+export const CHAOS_HEADER = 'x-brownout-chaos';
+export const CHAOS_TOKEN_HEADER = 'x-brownout-chaos-token';
+export const CHAOS_STATUS_HEADER = 'x-brownout-chaos-status';
+
+// Routes where a wrong answer is a payment, not a stale number. Matched against
+// the request path, case-insensitively, before any token is even considered.
+const MONEY_PATHS =
+ /\/(x402-pay|x402-facilitator|pay|payments?|settle|withdraw|transfer|checkout|purchase|subscribe|buy|sell|trade|swap|launch|mint|airdrop|claim|payout|treasury)(\/|$|\?)/i;
+
+// Payment intent on ANY route, regardless of path.
+const MONEY_HEADERS = ['x-payment', 'payment-signature', 'x-payment-response'];
+
+/**
+ * Parse a chaos directive into faults by upstream name.
+ * Unknown fault kinds are dropped rather than guessed at, so a typo disables
+ * that one entry instead of quietly injecting the wrong failure.
+ *
+ * @param {string} value
+ * @returns {Map<string, Fault>}
+ */
+export function parseChaosDirective(value) {
+ /** @type {Map<string, Fault>} */
+ const out = new Map();
+ if (typeof value !== 'string' || !value.trim()) return out;
+ for (const part of value.split(',')) {
+ const [rawName, rawSpec = 'network'] = part.split('=');
+ const name = String(rawName || '').trim().toLowerCase();
+ if (!name || !/^[a-z0-9:._-]{1,48}$/.test(name)) continue;
+ const spec = String(rawSpec).trim().toLowerCase();
+ const [kind, arg] = spec.split(':');
+ if (kind === 'timeout') out.set(name, { kind: 'timeout' });
+ else if (kind === 'network') out.set(name, { kind: 'network' });
+ else if (kind === 'empty') out.set(name, { kind: 'empty' });
+ else if (kind === 'http') {
+ const status = Number(arg);
+ if (Number.isInteger(status) && status >= 400 && status <= 599) out.set(name, { kind: 'http', status });
+ } else if (kind === 'slow') {
+ const ms = Number(arg);
+ if (Number.isFinite(ms) && ms > 0 && ms <= 60_000) out.set(name, { kind: 'slow', ms: Math.round(ms) });
+ }
+ }
+ return out;
+}
+
+function tokenMatches(supplied, expected) {
+ if (!expected || typeof supplied !== 'string' || !supplied) return false;
+ const a = Buffer.from(supplied);
+ const b = Buffer.from(expected);
+ // timingSafeEqual throws on a length mismatch, which itself leaks length, so
+ // compare fixed-width digests of both sides instead of the raw values.
+ if (a.length !== b.length) return false;
+ return timingSafeEqual(a, b);
+}
+
+/**
+ * Decide whether a request may inject faults, and which.
+ *
+ * @param {{ headers?: Record<string, any>, url?: string, method?: string }} req
+ * @param {{ token?: string }} [env]
+ * @returns {{ allowed: boolean, reason: string, faults: Map<string, Fault> }}
+ */
+export function chaosDecision(req, env = {}) {
+ const headers = req?.headers || {};
+ const raw = headers[CHAOS_HEADER];
+ const none = new Map();
+ if (!raw) return { allowed: false, reason: 'absent', faults: none };
+
+ const expected = env.token ?? process.env.BROWNOUT_CHAOS_TOKEN;
+ if (!expected) return { allowed: false, reason: 'not_configured', faults: none };
+ if (!tokenMatches(headers[CHAOS_TOKEN_HEADER], expected)) {
+ return { allowed: false, reason: 'bad_token', faults: none };
+ }
+
+ for (const h of MONEY_HEADERS) {
+ if (headers[h]) return { allowed: false, reason: 'money_path', faults: none };
+ }
+ const path = String(req?.url || '').split('?')[0];
+ if (MONEY_PATHS.test(path)) return { allowed: false, reason: 'money_path', faults: none };
+
+ const method = String(req?.method || 'GET').toUpperCase();
+ if (method !== 'GET' && method !== 'HEAD' && method !== 'POST') {
+ return { allowed: false, reason: 'write_method', faults: none };
+ }
+
+ const faults = parseChaosDirective(Array.isArray(raw) ? raw.join(',') : raw);
+ if (!faults.size) return { allowed: false, reason: 'no_valid_faults', faults: none };
+ return { allowed: true, reason: 'ok', faults };
+}
+
+/**
+ * Run `fn` with `faults` active for the current async context.
+ * @template T
+ * @param {Map<string, Fault>} faults
+ * @param {() => T} fn
+ * @returns {T}
+ */
+export function withChaos(faults, fn) {
+ if (!faults || !faults.size) return fn();
+ return storage.run(faults, fn);
+}
+
+/**
+ * The fault registered for `name` in this request, or null.
+ * Names are matched case-insensitively, and a directive for `birdeye` also
+ * matches the sub-scoped `birdeye:txs`, so a caller can break a provider
+ * without having to enumerate every call shape the code gives it.
+ *
+ * @param {string|undefined} name
+ * @returns {Fault|null}
+ */
+export function faultFor(name) {
+ const faults = storage.getStore();
+ if (!faults || !name) return null;
+ const key = String(name).toLowerCase();
+ const exact = faults.get(key);
+ if (exact) return exact;
+ const scope = key.split(':')[0];
+ return faults.get(scope) ?? null;
+}
+
+/** True when any fault is active, for callers that want to skip work entirely. */
+export function chaosActive() {
+ const faults = storage.getStore();
+ return !!faults && faults.size > 0;
+}
+
+/**
+ * The error or Response a faulted call must produce, shaped exactly like the
+ * real failure it stands in for, so the code under test cannot tell the
+ * difference. A fault that produced a distinguishable error would prove only
+ * that the code handles Brownout, not that it handles the outage.
+ *
+ * @param {Fault} fault
+ * @param {string} url
+ * @returns {Promise<Response>} resolves for http/empty, rejects for timeout/network
+ */
+export async function applyFault(fault, url) {
+ if (fault.kind === 'slow') {
+ await new Promise((r) => setTimeout(r, fault.ms || 0));
+ return null; // caller proceeds with the real request, just later
+ }
+ if (fault.kind === 'timeout') {
+ // undici raises TimeoutError on AbortSignal.timeout; isRetryableError and
+ // every hand-rolled catch in this repo keys off exactly that name.
+ throw Object.assign(new Error(`brownout: simulated timeout for ${url}`), { name: 'TimeoutError' });
+ }
+ if (fault.kind === 'network') {
+ // The shape undici gives a refused socket: TypeError('fetch failed') with
+ // the errno on `cause`. api/_lib/solana/connection.js reads that cause.
+ const err = new TypeError('fetch failed');
+ err.cause = Object.assign(new Error('ECONNREFUSED'), { code: 'ECONNREFUSED' });
+ throw err;
+ }
+ if (fault.kind === 'empty') {
+ return new Response('', { status: 200, headers: { 'content-type': 'application/json' } });
+ }
+ const status = fault.status || 500;
+ return new Response(JSON.stringify({ error: 'brownout_injected', status }), {
+ status,
+ headers: {
+ 'content-type': 'application/json',
+ // A real 429 usually carries this, and whether a caller honours it is
+ // exactly the kind of thing a proof should be able to check.
+ ...(status === 429 ? { 'retry-after': '1' } : {}),
+ },
+ });
+}
diff --git a/api/_lib/brownout/index.js b/api/_lib/brownout/index.js
new file mode 100644
index 000000000..c688b7b30
--- /dev/null
+++ b/api/_lib/brownout/index.js
@@ -0,0 +1,31 @@
+// @ts-check
+// Brownout: the platform's degradation layer.
+//
+// provenance what answered this request, and how fresh it was
+// chaos break a named upstream on purpose, for one request, safely
+//
+// Both are re-exported here so a caller (including the isomorphic
+// src/shared/failover-fetch.js, which loads this lazily on the server only)
+// needs a single import.
+
+export {
+ TIERS,
+ worstTier,
+ withProvenance,
+ currentLedger,
+ recordSource,
+ provenanceSummary,
+ provenanceHeaders,
+} from './provenance.js';
+
+export {
+ CHAOS_HEADER,
+ CHAOS_TOKEN_HEADER,
+ CHAOS_STATUS_HEADER,
+ parseChaosDirective,
+ chaosDecision,
+ withChaos,
+ faultFor,
+ chaosActive,
+ applyFault,
+} from './chaos.js';
diff --git a/api/_lib/brownout/provenance.js b/api/_lib/brownout/provenance.js
new file mode 100644
index 000000000..6cef43451
--- /dev/null
+++ b/api/_lib/brownout/provenance.js
@@ -0,0 +1,174 @@
+// @ts-check
+// Where this response's data actually came from.
+//
+// The platform has fallbacks everywhere: provider ladders, retry policies,
+// last-good tiers, degraded shapes. They work. The problem is that nobody can
+// SEE them work. A 200 from /api/pump/dashboard looks identical whether its
+// price came from Birdeye a moment ago or from a cached reading twenty minutes
+// old after three providers refused, and the caller has no way to tell. That
+// gap is where the expensive bugs live: this repo has already shipped a paid
+// datapoint stamped `as_of: now` over an hour-old payload, and a token security
+// verdict assembled out of remembered on-chain state. Both were invisible
+// precisely because a degraded answer and a live one are the same bytes.
+//
+// So: every fetch through the shared wrappers records what it did, against the
+// request that caused it, and the response carries a summary of that record.
+//
+// x-brownout: v=1;tier=stale;sources=3;ok=1;failed=2;ms=512
+// x-brownout-trace: birdeye;o=429;t=412, tokens-xyz;o=timeout;t=8000, dexscreener;o=ok;t=88
+//
+// The trace is emitted only when something actually degraded, so a healthy
+// response pays one short header and no per-source cost.
+//
+// Freshness is a LATTICE, not a flag. `live` beat `cache` beat `stale` beat
+// `fallback`, and a response's tier is the WORST tier that contributed to it,
+// because a page assembled from one live read and one half-hour-old reading is
+// not live. Reporting the best tier would be worse than reporting nothing: it
+// would be a confident lie, and the whole point of this module is that a
+// degraded answer must never be able to pass for a fresh one.
+//
+// Cost when nothing is listening: one AsyncLocalStorage lookup that returns
+// undefined and an early return. No allocation, no header, no behaviour change.
+
+import { AsyncLocalStorage } from 'node:async_hooks';
+
+/** @typedef {'live'|'cache'|'stale'|'fallback'} Tier */
+/** @typedef {'ok'|'fail'|'skip'} Outcome */
+
+/**
+ * Freshness lattice, best first. A response is only as fresh as its least
+ * fresh contributing source.
+ * - live answered by the upstream during this request
+ * - cache served from a cache entry inside its intended TTL
+ * - stale served past its TTL because the upstream could not answer
+ * - fallback produced by a different method than the one asked for
+ * (a sibling provider, a lexical ranking, a constant table)
+ * @type {Tier[]}
+ */
+export const TIERS = ['live', 'cache', 'stale', 'fallback'];
+const TIER_RANK = new Map(TIERS.map((t, i) => [t, i]));
+
+/** The worse (less fresh) of two tiers. Unknown tiers are ignored, never trusted. */
+export function worstTier(a, b) {
+ const ra = TIER_RANK.get(a);
+ const rb = TIER_RANK.get(b);
+ if (ra == null) return rb == null ? 'live' : b;
+ if (rb == null) return a;
+ return ra >= rb ? a : b;
+}
+
+/** @typedef {{ name: string, outcome: Outcome, ms: number, tier: Tier, detail?: string }} SourceRecord */
+/** @typedef {{ sources: SourceRecord[], startedAt: number }} Ledger */
+
+/** @type {AsyncLocalStorage<Ledger>} */
+const storage = new AsyncLocalStorage();
+
+/**
+ * Run `fn` inside a fresh provenance ledger. Everything the shared fetch
+ * wrappers do inside it is recorded against this request and nothing else,
+ * which is what makes the record safe under concurrency: two requests running
+ * in the same instance never see each other's sources.
+ *
+ * @template T
+ * @param {() => T} fn
+ * @returns {T}
+ */
+export function withProvenance(fn) {
+ return storage.run({ sources: [], startedAt: Date.now() }, fn);
+}
+
+/** The current request's ledger, or null outside a provenance context. */
+export function currentLedger() {
+ return storage.getStore() ?? null;
+}
+
+// A single request can fan out over hundreds of sources (a batch price read per
+// mint). Recording every one would turn a header into a payload, so the ledger
+// keeps the first N and counts the rest: the shape of the degradation is in the
+// first few, and the count preserves the scale.
+const MAX_RECORDED = 24;
+
+/**
+ * Record one upstream interaction against the current request.
+ *
+ * Safe to call from anywhere, including outside a request (it becomes a no-op),
+ * so a helper never has to know whether its caller set a context up.
+ *
+ * @param {{ name: string, outcome: Outcome, ms?: number, tier?: Tier, detail?: string|number }} rec
+ */
+export function recordSource(rec) {
+ const ledger = storage.getStore();
+ if (!ledger || !rec?.name) return;
+ if (ledger.sources.length >= MAX_RECORDED) {
+ ledger.truncated = (ledger.truncated || 0) + 1;
+ return;
+ }
+ ledger.sources.push({
+ name: String(rec.name).slice(0, 48),
+ outcome: rec.outcome === 'ok' || rec.outcome === 'skip' ? rec.outcome : 'fail',
+ ms: Number.isFinite(rec.ms) ? Math.max(0, Math.round(Number(rec.ms))) : 0,
+ tier: TIER_RANK.has(rec.tier) ? rec.tier : 'live',
+ ...(rec.detail != null ? { detail: String(rec.detail).slice(0, 32) } : {}),
+ });
+}
+
+/**
+ * Aggregate the ledger. `tier` is the worst tier of any source that actually
+ * CONTRIBUTED (outcome ok); a failed attempt tells you the chain was exercised
+ * but says nothing about the freshness of the answer that finally came back.
+ *
+ * @returns {{ tier: Tier, sources: number, ok: number, failed: number, ms: number, degraded: boolean, truncated: number, records: SourceRecord[] } | null}
+ */
+export function provenanceSummary() {
+ const ledger = storage.getStore();
+ if (!ledger || (!ledger.sources.length && !ledger.truncated)) return null;
+ let tier = 'live';
+ let ok = 0;
+ let failed = 0;
+ for (const s of ledger.sources) {
+ if (s.outcome === 'ok') {
+ ok++;
+ tier = worstTier(tier, s.tier);
+ } else if (s.outcome === 'fail') {
+ failed++;
+ }
+ }
+ return {
+ tier,
+ sources: ledger.sources.length + (ledger.truncated || 0),
+ ok,
+ failed,
+ ms: Date.now() - ledger.startedAt,
+ // Degraded means "a reader should not treat this as a clean live answer":
+ // either something had to be retried or failed over, or what came back is
+ // not fresh. Both are worth surfacing; neither is an error.
+ degraded: failed > 0 || tier !== 'live',
+ truncated: ledger.truncated || 0,
+ records: ledger.sources,
+ };
+}
+
+// Header values must be a single line of visible ASCII. A provider name is
+// ours, not user input, but it still passes through here so a stray newline or
+// separator can never split a header.
+const headerSafe = (s) => String(s).replace(/[^\x21-\x7E]|[,;]/g, '_');
+
+/**
+ * The two header values for the current request, or null when nothing was
+ * recorded. `trace` is present only when something degraded: a healthy response
+ * should not pay for a per-source breakdown nobody will read.
+ *
+ * @returns {{ summary: string, trace: string|null } | null}
+ */
+export function provenanceHeaders() {
+ const s = provenanceSummary();
+ if (!s) return null;
+ const summary =
+ `v=1;tier=${s.tier};sources=${s.sources};ok=${s.ok};failed=${s.failed};ms=${s.ms}` +
+ (s.degraded ? ';degraded=1' : '');
+ if (!s.degraded) return { summary, trace: null };
+ const trace = s.records
+ .map((r) => `${headerSafe(r.name)};o=${headerSafe(r.detail ?? r.outcome)};t=${r.ms}`)
+ .join(', ');
+ return { summary, trace: trace || null };
+}
diff --git a/tests/brownout-core.test.js b/tests/brownout-core.test.js
new file mode 100644
index 000000000..ba621b382
--- /dev/null
+++ b/tests/brownout-core.test.js
@@ -0,0 +1,250 @@
+// Brownout: data provenance and gated fault injection.
+//
+// These two are the load-bearing halves of the feature, so their edges are
+// tested directly: the freshness lattice (a degraded answer must never be able
+// to report itself as live), request isolation under concurrency, and every one
+// of the gates that stands between a chaos header and a production request.
+import { describe, it, expect, afterEach } from 'vitest';
+import {
+ TIERS,
+ worstTier,
+ withProvenance,
+ recordSource,
+ provenanceSummary,
+ provenanceHeaders,
+} from '../api/_lib/brownout/provenance.js';
+import {
+ parseChaosDirective,
+ chaosDecision,
+ withChaos,
+ faultFor,
+ applyFault,
+ CHAOS_HEADER,
+ CHAOS_TOKEN_HEADER,
+} from '../api/_lib/brownout/chaos.js';
+
+const TOKEN = 'brownout-test-token';
+const reqWith = (headers = {}, url = '/api/pump/dashboard', method = 'GET') => ({
+ headers: { [CHAOS_TOKEN_HEADER]: TOKEN, ...headers },
+ url,
+ method,
+});
+
+afterEach(() => {
+ delete process.env.BROWNOUT_CHAOS_TOKEN;
+});
+
+describe('freshness lattice', () => {
+ it('orders tiers from freshest to least fresh', () => {
+ expect(TIERS).toEqual(['live', 'cache', 'stale', 'fallback']);
+ });
+
+ it('takes the WORSE of two tiers, so a mixed response cannot report as live', () => {
+ expect(worstTier('live', 'stale')).toBe('stale');
+ expect(worstTier('stale', 'live')).toBe('stale');
+ expect(worstTier('cache', 'fallback')).toBe('fallback');
+ expect(worstTier('live', 'live')).toBe('live');
+ });
+
+ it('never trusts an unknown tier into the result', () => {
+ expect(worstTier('nonsense', 'stale')).toBe('stale');
+ expect(worstTier('stale', 'nonsense')).toBe('stale');
+ expect(worstTier(undefined, undefined)).toBe('live');
+ });
+});
+
+describe('provenance ledger', () => {
+ it('is a no-op outside a request context, so a helper never needs to know', () => {
+ expect(() => recordSource({ name: 'x', outcome: 'ok' })).not.toThrow();
+ expect(provenanceSummary()).toBeNull();
+ expect(provenanceHeaders()).toBeNull();
+ });
+
+ it('summarises a clean live answer without a trace', () => {
+ withProvenance(() => {
+ recordSource({ name: 'birdeye', outcome: 'ok', ms: 42, tier: 'live' });
+ const s = provenanceSummary();
+ expect(s).toMatchObject({ tier: 'live', ok: 1, failed: 0, degraded: false });
+ const h = provenanceHeaders();
+ expect(h.summary).toContain('tier=live');
+ expect(h.summary).not.toContain('degraded=1');
+ // A healthy response should not pay for a per-source breakdown.
+ expect(h.trace).toBeNull();
+ });
+ });
+
+ it('reports the worst contributing tier and emits a trace when degraded', () => {
+ withProvenance(() => {
+ recordSource({ name: 'birdeye', outcome: 'fail', ms: 412, tier: 'live', detail: 429 });
+ recordSource({ name: 'dexscreener', outcome: 'ok', ms: 88, tier: 'stale' });
+ const s = provenanceSummary();
+ expect(s.tier).toBe('stale');
+ expect(s).toMatchObject({ ok: 1, failed: 1, degraded: true });
+ const h = provenanceHeaders();
+ expect(h.summary).toContain('degraded=1');
+ expect(h.trace).toContain('birdeye;o=429;t=412');
+ expect(h.trace).toContain('dexscreener;o=ok;t=88');
+ });
+ });
+
+ it('counts a failure as degraded even when the answer that came back was live', () => {
+ // A ladder that failed over still tells the reader the chain was exercised.
+ withProvenance(() => {
+ recordSource({ name: 'a', outcome: 'fail', ms: 10, tier: 'live' });
+ recordSource({ name: 'b', outcome: 'ok', ms: 10, tier: 'live' });
+ const s = provenanceSummary();
+ expect(s.tier).toBe('live');
+ expect(s.degraded).toBe(true);
+ });
+ });
+
+ it('keeps two concurrent requests' + " strictly apart", async () => {
+ const [a, b] = await Promise.all([
+ withProvenance(async () => {
+ recordSource({ name: 'req-a', outcome: 'ok', ms: 1, tier: 'live' });
+ await new Promise((r) => setTimeout(r, 5));
+ recordSource({ name: 'req-a2', outcome: 'ok', ms: 1, tier: 'stale' });
+ return provenanceSummary();
+ }),
+ withProvenance(async () => {
+ await new Promise((r) => setTimeout(r, 1));
+ recordSource({ name: 'req-b', outcome: 'fail', ms: 1, tier: 'live' });
+ return provenanceSummary();
+ }),
+ ]);
+ expect(a.records.map((r) => r.name)).toEqual(['req-a', 'req-a2']);
+ expect(a.tier).toBe('stale');
+ expect(b.records.map((r) => r.name)).toEqual(['req-b']);
+ expect(b.failed).toBe(1);
+ });
+
+ it('caps what it records but keeps the true count, so a fan-out cannot become a payload', () => {
+ withProvenance(() => {
+ for (let i = 0; i < 200; i++) recordSource({ name: `p${i}`, outcome: 'ok', ms: 1, tier: 'live' });
+ const s = provenanceSummary();
+ expect(s.records.length).toBeLessThanOrEqual(24);
+ expect(s.sources).toBe(200);
+ expect(s.truncated).toBe(200 - s.records.length);
+ });
+ });
+
+ it('never lets a source name break the header grammar', () => {
+ withProvenance(() => {
+ recordSource({ name: 'evil\nname;with,separators', outcome: 'fail', ms: 1, tier: 'live' });
+ const h = provenanceHeaders();
+ expect(h.trace).not.toMatch(/[\n\r]/);
+ expect(h.trace.split(',').length).toBe(1);
+ });
+ });
+});
+
+describe('chaos directive parsing', () => {
+ it('reads every supported fault kind', () => {
+ const f = parseChaosDirective('birdeye=http:429, tokens-xyz=timeout, dex=network, cg=empty, slowly=slow:250');
+ expect(f.get('birdeye')).toEqual({ kind: 'http', status: 429 });
+ expect(f.get('tokens-xyz')).toEqual({ kind: 'timeout' });
+ expect(f.get('dex')).toEqual({ kind: 'network' });
+ expect(f.get('cg')).toEqual({ kind: 'empty' });
+ expect(f.get('slowly')).toEqual({ kind: 'slow', ms: 250 });
+ });
+
+ it('drops a malformed entry instead of guessing at it', () => {
+ // A typo must disable that one fault, never inject a different one.
+ const f = parseChaosDirective('a=http:99, b=http:notanumber, c=teleport, =timeout, d=slow:999999');
+ expect(f.size).toBe(0);
+ });
+
+ it('defaults a bare name to a network failure', () => {
+ expect(parseChaosDirective('birdeye').get('birdeye')).toEqual({ kind: 'network' });
+ });
+});
+
+describe('chaos gating', () => {
+ it('is off entirely when no token is configured', () => {
+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout' }));
+ expect(d).toMatchObject({ allowed: false, reason: 'not_configured' });
+ });
+
+ it('refuses a wrong token', () => {
+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;
+ const d = chaosDecision({ headers: { [CHAOS_HEADER]: 'birdeye=timeout', [CHAOS_TOKEN_HEADER]: 'nope' }, url: '/api/x', method: 'GET' });
+ expect(d).toMatchObject({ allowed: false, reason: 'bad_token' });
+ });
+
+ it('allows a valid directive on a read path', () => {
+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;
+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=http:429' }));
+ expect(d.allowed).toBe(true);
+ expect(d.faults.get('birdeye')).toEqual({ kind: 'http', status: 429 });
+ });
+
+ it('refuses anything carrying a payment header, whatever the route', () => {
+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;
+ for (const h of ['x-payment', 'payment-signature', 'x-payment-response']) {
+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout', [h]: 'anything' }, '/api/intel/heatmap'));
+ expect(d, h).toMatchObject({ allowed: false, reason: 'money_path' });
+ }
+ });
+
+ it('refuses a money route even with a valid token', () => {
+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;
+ for (const url of ['/api/x402-pay', '/api/pay/execute', '/api/agents/1/withdraw', '/api/purchase/skill', '/api/token/swap']) {
+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout' }, url));
+ expect(d, url).toMatchObject({ allowed: false, reason: 'money_path' });
+ }
+ });
+
+ it('refuses a write method', () => {
+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;
+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout' }, '/api/agents', 'DELETE'));
+ expect(d).toMatchObject({ allowed: false, reason: 'write_method' });
+ });
+
+ it('says no rather than silently doing nothing when every fault is malformed', () => {
+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;
+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=teleport' }));
+ expect(d).toMatchObject({ allowed: false, reason: 'no_valid_faults' });
+ });
+});
+
+describe('fault scoping and shape', () => {
+ it('matches a sub-scoped upstream from its parent name', async () => {
+ await withChaos(parseChaosDirective('birdeye=timeout'), async () => {
+ expect(faultFor('birdeye')).toEqual({ kind: 'timeout' });
+ // A directive for the provider covers every call shape it serves.
+ expect(faultFor('birdeye:txs')).toEqual({ kind: 'timeout' });
+ expect(faultFor('dexscreener')).toBeNull();
+ });
+ });
+
+ it('is inert outside a chaos context', () => {
+ expect(faultFor('birdeye')).toBeNull();
+ });
+
+ it('raises a timeout indistinguishable from a real one', async () => {
+ await expect(applyFault({ kind: 'timeout' }, 'https://x')).rejects.toMatchObject({ name: 'TimeoutError' });
+ });
+
+ it('raises a network failure with the errno undici would attach', async () => {
+ // api/_lib/solana/connection.js reads exactly this cause to name the fault.
+ await expect(applyFault({ kind: 'network' }, 'https://x')).rejects.toMatchObject({
+ message: 'fetch failed',
+ cause: { code: 'ECONNREFUSED' },
+ });
+ });
+
+ it('returns a real Response for an http fault, with Retry-After on a 429', async () => {
+ const res = await applyFault({ kind: 'http', status: 429 }, 'https://x');
+ expect(res.status).toBe(429);
+ expect(res.headers.get('retry-after')).toBe('1');
+ const res500 = await applyFault({ kind: 'http', status: 500 }, 'https://x');
+ expect(res500.headers.get('retry-after')).toBeNull();
+ });
+
+ it('lets a slow fault fall through to the real call, just later', async () => {
+ const t0 = Date.now();
+ const out = await applyFault({ kind: 'slow', ms: 30 }, 'https://x');
+ expect(out).toBeNull();
+ expect(Date.now() - t0).toBeGreaterThanOrEqual(25);
+ });
+});
diff --git a/tests/brownout-integration.test.js b/tests/brownout-integration.test.js
new file mode 100644
index 000000000..a816af250
--- /dev/null
+++ b/tests/brownout-integration.test.js
@@ -0,0 +1,134 @@
+// The loop that makes Brownout worth having: inject a fault, run the REAL
+// wrapper and the REAL provider ladder, and watch the fallback take over while
+// the response reports honestly what happened.
+//
+// A stubbed `fetch` can only prove the stub was called. These tests break the
+// upstream from inside the wrapper that every third-party call in api/ goes
+// through, so what is exercised is the retry policy, the breaker, the last-good
+// tier and the ladder itself, exactly as a real outage would exercise them.
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { withProvenance, provenanceSummary, provenanceHeaders } from '../api/_lib/brownout/provenance.js';
+import { withChaos, parseChaosDirective } from '../api/_lib/brownout/chaos.js';
+import { fetchUpstreamJson, lastGood, _resetLastGood } from '../api/_lib/upstream-fetch.js';
+import { fetchFirst } from '../src/shared/failover-fetch.js';
+
+const ORIGINAL_FETCH = globalThis.fetch;
+const json = (body, status = 200) =>
+ new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
+
+beforeEach(() => {
+ _resetLastGood?.();
+});
+afterEach(() => {
+ globalThis.fetch = ORIGINAL_FETCH;
+ vi.restoreAllMocks();
+});
+
+describe('fetchUpstream under an injected fault', () => {
+ it('fails the named upstream and leaves its siblings alone', async () => {
+ globalThis.fetch = vi.fn(async () => json({ ok: true }));
+ await withChaos(parseChaosDirective('broken=http:500'), async () => {
+ await expect(
+ fetchUpstreamJson('https://a.example/x', {}, { name: 'broken', attempts: 1 }),
+ ).rejects.toThrow(/http 500/);
+ // The un-named upstream is untouched: a fault is scoped to what it names.
+ await expect(fetchUpstreamJson('https://b.example/x', {}, { name: 'healthy', attempts: 1 })).resolves.toEqual({ ok: true });
+ });
+ });
+
+ it('is retried by the real retry policy, and the retry can succeed', async () => {
+ // A 429 fault is transient, so withRetry retries it. The second attempt is
+ // still faulted, which is what proves the injection survives into the retry
+ // rather than being a one-shot.
+ let calls = 0;
+ globalThis.fetch = vi.fn(async () => {
+ calls++;
+ return json({ ok: true });
+ });
+ await withChaos(parseChaosDirective('flaky=http:429'), async () => {
+ await expect(
+ fetchUpstreamJson('https://a.example/x', {}, { name: 'flaky', attempts: 2, maxRetryAfterMs: 5 }),
+ ).rejects.toThrow(/http 429/);
+ });
+ // The real socket was never opened: every attempt hit the fault.
+ expect(calls).toBe(0);
+ });
+
+ it('serves the last-good value when the live read is faulted, and marks it stale', async () => {
+ globalThis.fetch = vi.fn(async () => json({ price: 42 }));
+ const load = () => fetchUpstreamJson('https://a.example/price', {}, { name: 'pricer', attempts: 1 });
+
+ const live = await lastGood('brownout:price', load);
+ expect(live).toMatchObject({ value: { price: 42 }, stale: false });
+
+ const out = await withProvenance(() =>
+ withChaos(parseChaosDirective('pricer=network'), async () => {
+ const res = await lastGood('brownout:price', load, { maxAgeMs: 60_000 });
+ return { res, prov: provenanceSummary() };
+ }),
+ );
+ expect(out.res.value).toEqual({ price: 42 });
+ expect(out.res.stale).toBe(true);
+ // And the response would tell the caller so.
+ expect(out.prov.tier).toBe('stale');
+ expect(out.prov.degraded).toBe(true);
+ });
+
+ it('rethrows honestly when the upstream is faulted and nothing was ever cached', async () => {
+ globalThis.fetch = vi.fn(async () => json({ price: 1 }));
+ await withChaos(parseChaosDirective('pricer=network'), async () => {
+ await expect(
+ lastGood('brownout:never-seen', () => fetchUpstreamJson('https://a.example/p', {}, { name: 'pricer', attempts: 1 })),
+ ).rejects.toThrow(/fetch failed/);
+ });
+ });
+});
+
+describe('provider ladder under an injected fault', () => {
+ const providers = [
+ { name: 'primary', url: 'https://p1.example/x', parse: async (r) => (await r.json()).v },
+ { name: 'secondary', url: 'https://p2.example/x', parse: async (r) => (await r.json()).v },
+ { name: 'tertiary', url: 'https://p3.example/x', parse: async (r) => (await r.json()).v },
+ ];
+
+ it('walks past the faulted rung to the next healthy one', async () => {
+ globalThis.fetch = vi.fn(async (url) => json({ v: String(url).includes('p2') ? 'from-secondary' : 'from-other' }));
+ const out = await withChaos(parseChaosDirective('primary=timeout'), () =>
+ fetchFirst(providers, { timeoutMs: 500, cooldownMs: 0 }),
+ );
+ expect(out.source).toBe('secondary');
+ expect(out.value).toBe('from-secondary');
+ });
+
+ it('records the whole walk, not just the rung that answered', async () => {
+ globalThis.fetch = vi.fn(async () => json({ v: 'ok' }));
+ // Both readings are taken INSIDE the context: the ledger is scoped to the
+ // request, so asking for it afterwards correctly answers null.
+ const { prov, headers } = await withProvenance(async () => {
+ await withChaos(parseChaosDirective('primary=http:503, secondary=network'), () =>
+ fetchFirst(providers, { timeoutMs: 500, cooldownMs: 0 }),
+ );
+ // The ladder records asynchronously (it must not block the answer), so
+ // let those microtasks land before reading the ledger.
+ await new Promise((r) => setTimeout(r, 20));
+ return { prov: provenanceSummary(), headers: provenanceHeaders() };
+ });
+ const names = prov.records.map((r) => r.name);
+ expect(names).toEqual(['primary', 'secondary', 'tertiary']);
+ expect(prov.failed).toBe(2);
+ expect(prov.ok).toBe(1);
+ // Two rungs died and the third answered live: the answer is fresh, but the
+ // reader still learns the chain was exercised.
+ expect(prov.tier).toBe('live');
+ expect(prov.degraded).toBe(true);
+ expect(headers.summary).toContain('degraded=1');
+ expect(headers.trace).toMatch(/primary;o=503/);
+ });
+
+ it('fails the whole chain when every rung is faulted', async () => {
+ globalThis.fetch = vi.fn(async () => json({ v: 'ok' }));
+ await withChaos(parseChaosDirective('primary=network, secondary=network, tertiary=network'), async () => {
+ await expect(fetchFirst(providers, { timeoutMs: 500, cooldownMs: 0 })).rejects.toThrow(/providers failed/);
+ });
+ });
+});