Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(brownout): make every response say where its data came from, and let a caller break one upstream on purpose
nirholasf30f7cbfa81c
diff --git a/api/_lib/brownout/chaos.js b/api/_lib/brownout/chaos.jsnew file mode 100644index 000000000..2c5e12ecb--- /dev/null+++ b/api/_lib/brownout/chaos.js@@ -0,0 +1,217 @@+// @ts-check+// Break a named upstream on purpose, for one request, in the real code path.+//+// Every fallback in this codebase is untested code until something makes the+// thing it protects against actually happen. Unit tests stub `fetch` and prove+// the stub was called; they cannot prove that /api/pump/dashboard returns a+// usable page when Birdeye 429s, because the stub replaces the very layer whose+// behaviour is in question. Infra-level chaos (kill a pod, drop a route) is the+// usual answer and is far too blunt: it cannot express "Birdeye is rate+// limiting but DexScreener is fine", which is the failure that actually happens.+//+// So the injection point is the shared fetch wrappers, and the scope is one+// request:+//+// x-brownout-chaos: birdeye=http:429, tokens-xyz=timeout, dexscreener=network+//+// The request runs the REAL handler, the REAL provider ladder, the REAL cache+// tiers. Only the named upstreams misbehave, and only for the caller who asked.+// Nothing global changes, so a probe can run against production traffic without+// touching anybody else's request.+//+// ── Safety ──────────────────────────────────────────────────────────────────+// This is a switch that makes production misbehave, so it is gated three ways+// and every one of them must pass:+//+// 1. A token. BROWNOUT_CHAOS_TOKEN must be set and match, compared in+// constant time. Without the env var configured, chaos is off everywhere,+// including locally, so there is no "forgot to set it in prod" state that+// leaves it open.+// 2. Never on a money path. A request carrying an x402 payment header, or+// addressed to a settlement/withdraw/transfer route, is refused outright.+// You must not be able to fault-inject something that moves funds, no+// matter who holds the token: the blast radius of a wrong answer there is+// not a stale price, it is a payment.+// 3. Read-shaped methods only. GET/HEAD by default; a POST is allowed only+// when it is a declared read (a JSON-RPC query, a search), never a write.+//+// A refused directive is reported back in `x-brownout-chaos-status` rather than+// silently ignored, because a probe that thinks it broke Birdeye and did not is+// worse than one that failed loudly: it would record a green proof for a+// fallback nobody exercised.++import { timingSafeEqual } from 'node:crypto';+import { AsyncLocalStorage } from 'node:async_hooks';++/** @typedef {{ kind: 'timeout'|'network'|'http'|'empty'|'slow', status?: number, ms?: number }} Fault */++/** @type {AsyncLocalStorage<Map<string, Fault>>} */+const storage = new AsyncLocalStorage();++export const CHAOS_HEADER = 'x-brownout-chaos';+export const CHAOS_TOKEN_HEADER = 'x-brownout-chaos-token';+export const CHAOS_STATUS_HEADER = 'x-brownout-chaos-status';++// Routes where a wrong answer is a payment, not a stale number. Matched against+// the request path, case-insensitively, before any token is even considered.+const MONEY_PATHS =+ /\/(x402-pay|x402-facilitator|pay|payments?|settle|withdraw|transfer|checkout|purchase|subscribe|buy|sell|trade|swap|launch|mint|airdrop|claim|payout|treasury)(\/|$|\?)/i;++// Payment intent on ANY route, regardless of path.+const MONEY_HEADERS = ['x-payment', 'payment-signature', 'x-payment-response'];++/**+ * Parse a chaos directive into faults by upstream name.+ * Unknown fault kinds are dropped rather than guessed at, so a typo disables+ * that one entry instead of quietly injecting the wrong failure.+ *+ * @param {string} value+ * @returns {Map<string, Fault>}+ */+export function parseChaosDirective(value) {+ /** @type {Map<string, Fault>} */+ const out = new Map();+ if (typeof value !== 'string' || !value.trim()) return out;+ for (const part of value.split(',')) {+ const [rawName, rawSpec = 'network'] = part.split('=');+ const name = String(rawName || '').trim().toLowerCase();+ if (!name || !/^[a-z0-9:._-]{1,48}$/.test(name)) continue;+ const spec = String(rawSpec).trim().toLowerCase();+ const [kind, arg] = spec.split(':');+ if (kind === 'timeout') out.set(name, { kind: 'timeout' });+ else if (kind === 'network') out.set(name, { kind: 'network' });+ else if (kind === 'empty') out.set(name, { kind: 'empty' });+ else if (kind === 'http') {+ const status = Number(arg);+ if (Number.isInteger(status) && status >= 400 && status <= 599) out.set(name, { kind: 'http', status });+ } else if (kind === 'slow') {+ const ms = Number(arg);+ if (Number.isFinite(ms) && ms > 0 && ms <= 60_000) out.set(name, { kind: 'slow', ms: Math.round(ms) });+ }+ }+ return out;+}++function tokenMatches(supplied, expected) {+ if (!expected || typeof supplied !== 'string' || !supplied) return false;+ const a = Buffer.from(supplied);+ const b = Buffer.from(expected);+ // timingSafeEqual throws on a length mismatch, which itself leaks length, so+ // compare fixed-width digests of both sides instead of the raw values.+ if (a.length !== b.length) return false;+ return timingSafeEqual(a, b);+}++/**+ * Decide whether a request may inject faults, and which.+ *+ * @param {{ headers?: Record<string, any>, url?: string, method?: string }} req+ * @param {{ token?: string }} [env]+ * @returns {{ allowed: boolean, reason: string, faults: Map<string, Fault> }}+ */+export function chaosDecision(req, env = {}) {+ const headers = req?.headers || {};+ const raw = headers[CHAOS_HEADER];+ const none = new Map();+ if (!raw) return { allowed: false, reason: 'absent', faults: none };++ const expected = env.token ?? process.env.BROWNOUT_CHAOS_TOKEN;+ if (!expected) return { allowed: false, reason: 'not_configured', faults: none };+ if (!tokenMatches(headers[CHAOS_TOKEN_HEADER], expected)) {+ return { allowed: false, reason: 'bad_token', faults: none };+ }++ for (const h of MONEY_HEADERS) {+ if (headers[h]) return { allowed: false, reason: 'money_path', faults: none };+ }+ const path = String(req?.url || '').split('?')[0];+ if (MONEY_PATHS.test(path)) return { allowed: false, reason: 'money_path', faults: none };++ const method = String(req?.method || 'GET').toUpperCase();+ if (method !== 'GET' && method !== 'HEAD' && method !== 'POST') {+ return { allowed: false, reason: 'write_method', faults: none };+ }++ const faults = parseChaosDirective(Array.isArray(raw) ? raw.join(',') : raw);+ if (!faults.size) return { allowed: false, reason: 'no_valid_faults', faults: none };+ return { allowed: true, reason: 'ok', faults };+}++/**+ * Run `fn` with `faults` active for the current async context.+ * @template T+ * @param {Map<string, Fault>} faults+ * @param {() => T} fn+ * @returns {T}+ */+export function withChaos(faults, fn) {+ if (!faults || !faults.size) return fn();+ return storage.run(faults, fn);+}++/**+ * The fault registered for `name` in this request, or null.+ * Names are matched case-insensitively, and a directive for `birdeye` also+ * matches the sub-scoped `birdeye:txs`, so a caller can break a provider+ * without having to enumerate every call shape the code gives it.+ *+ * @param {string|undefined} name+ * @returns {Fault|null}+ */+export function faultFor(name) {+ const faults = storage.getStore();+ if (!faults || !name) return null;+ const key = String(name).toLowerCase();+ const exact = faults.get(key);+ if (exact) return exact;+ const scope = key.split(':')[0];+ return faults.get(scope) ?? null;+}++/** True when any fault is active, for callers that want to skip work entirely. */+export function chaosActive() {+ const faults = storage.getStore();+ return !!faults && faults.size > 0;+}++/**+ * The error or Response a faulted call must produce, shaped exactly like the+ * real failure it stands in for, so the code under test cannot tell the+ * difference. A fault that produced a distinguishable error would prove only+ * that the code handles Brownout, not that it handles the outage.+ *+ * @param {Fault} fault+ * @param {string} url+ * @returns {Promise<Response>} resolves for http/empty, rejects for timeout/network+ */+export async function applyFault(fault, url) {+ if (fault.kind === 'slow') {+ await new Promise((r) => setTimeout(r, fault.ms || 0));+ return null; // caller proceeds with the real request, just later+ }+ if (fault.kind === 'timeout') {+ // undici raises TimeoutError on AbortSignal.timeout; isRetryableError and+ // every hand-rolled catch in this repo keys off exactly that name.+ throw Object.assign(new Error(`brownout: simulated timeout for ${url}`), { name: 'TimeoutError' });+ }+ if (fault.kind === 'network') {+ // The shape undici gives a refused socket: TypeError('fetch failed') with+ // the errno on `cause`. api/_lib/solana/connection.js reads that cause.+ const err = new TypeError('fetch failed');+ err.cause = Object.assign(new Error('ECONNREFUSED'), { code: 'ECONNREFUSED' });+ throw err;+ }+ if (fault.kind === 'empty') {+ return new Response('', { status: 200, headers: { 'content-type': 'application/json' } });+ }+ const status = fault.status || 500;+ return new Response(JSON.stringify({ error: 'brownout_injected', status }), {+ status,+ headers: {+ 'content-type': 'application/json',+ // A real 429 usually carries this, and whether a caller honours it is+ // exactly the kind of thing a proof should be able to check.+ ...(status === 429 ? { 'retry-after': '1' } : {}),+ },+ });+}diff --git a/api/_lib/brownout/index.js b/api/_lib/brownout/index.jsnew file mode 100644index 000000000..c688b7b30--- /dev/null+++ b/api/_lib/brownout/index.js@@ -0,0 +1,31 @@+// @ts-check+// Brownout: the platform's degradation layer.+//+// provenance what answered this request, and how fresh it was+// chaos break a named upstream on purpose, for one request, safely+//+// Both are re-exported here so a caller (including the isomorphic+// src/shared/failover-fetch.js, which loads this lazily on the server only)+// needs a single import.++export {+ TIERS,+ worstTier,+ withProvenance,+ currentLedger,+ recordSource,+ provenanceSummary,+ provenanceHeaders,+} from './provenance.js';++export {+ CHAOS_HEADER,+ CHAOS_TOKEN_HEADER,+ CHAOS_STATUS_HEADER,+ parseChaosDirective,+ chaosDecision,+ withChaos,+ faultFor,+ chaosActive,+ applyFault,+} from './chaos.js';diff --git a/api/_lib/brownout/provenance.js b/api/_lib/brownout/provenance.jsnew file mode 100644index 000000000..6cef43451--- /dev/null+++ b/api/_lib/brownout/provenance.js@@ -0,0 +1,174 @@+// @ts-check+// Where this response's data actually came from.+//+// The platform has fallbacks everywhere: provider ladders, retry policies,+// last-good tiers, degraded shapes. They work. The problem is that nobody can+// SEE them work. A 200 from /api/pump/dashboard looks identical whether its+// price came from Birdeye a moment ago or from a cached reading twenty minutes+// old after three providers refused, and the caller has no way to tell. That+// gap is where the expensive bugs live: this repo has already shipped a paid+// datapoint stamped `as_of: now` over an hour-old payload, and a token security+// verdict assembled out of remembered on-chain state. Both were invisible+// precisely because a degraded answer and a live one are the same bytes.+//+// So: every fetch through the shared wrappers records what it did, against the+// request that caused it, and the response carries a summary of that record.+//+// x-brownout: v=1;tier=stale;sources=3;ok=1;failed=2;ms=512+// x-brownout-trace: birdeye;o=429;t=412, tokens-xyz;o=timeout;t=8000, dexscreener;o=ok;t=88+//+// The trace is emitted only when something actually degraded, so a healthy+// response pays one short header and no per-source cost.+//+// Freshness is a LATTICE, not a flag. `live` beat `cache` beat `stale` beat+// `fallback`, and a response's tier is the WORST tier that contributed to it,+// because a page assembled from one live read and one half-hour-old reading is+// not live. Reporting the best tier would be worse than reporting nothing: it+// would be a confident lie, and the whole point of this module is that a+// degraded answer must never be able to pass for a fresh one.+//+// Cost when nothing is listening: one AsyncLocalStorage lookup that returns+// undefined and an early return. No allocation, no header, no behaviour change.++import { AsyncLocalStorage } from 'node:async_hooks';++/** @typedef {'live'|'cache'|'stale'|'fallback'} Tier */+/** @typedef {'ok'|'fail'|'skip'} Outcome */++/**+ * Freshness lattice, best first. A response is only as fresh as its least+ * fresh contributing source.+ * - live answered by the upstream during this request+ * - cache served from a cache entry inside its intended TTL+ * - stale served past its TTL because the upstream could not answer+ * - fallback produced by a different method than the one asked for+ * (a sibling provider, a lexical ranking, a constant table)+ * @type {Tier[]}+ */+export const TIERS = ['live', 'cache', 'stale', 'fallback'];+const TIER_RANK = new Map(TIERS.map((t, i) => [t, i]));++/** The worse (less fresh) of two tiers. Unknown tiers are ignored, never trusted. */+export function worstTier(a, b) {+ const ra = TIER_RANK.get(a);+ const rb = TIER_RANK.get(b);+ if (ra == null) return rb == null ? 'live' : b;+ if (rb == null) return a;+ return ra >= rb ? a : b;+}++/** @typedef {{ name: string, outcome: Outcome, ms: number, tier: Tier, detail?: string }} SourceRecord */+/** @typedef {{ sources: SourceRecord[], startedAt: number }} Ledger */++/** @type {AsyncLocalStorage<Ledger>} */+const storage = new AsyncLocalStorage();++/**+ * Run `fn` inside a fresh provenance ledger. Everything the shared fetch+ * wrappers do inside it is recorded against this request and nothing else,+ * which is what makes the record safe under concurrency: two requests running+ * in the same instance never see each other's sources.+ *+ * @template T+ * @param {() => T} fn+ * @returns {T}+ */+export function withProvenance(fn) {+ return storage.run({ sources: [], startedAt: Date.now() }, fn);+}++/** The current request's ledger, or null outside a provenance context. */+export function currentLedger() {+ return storage.getStore() ?? null;+}++// A single request can fan out over hundreds of sources (a batch price read per+// mint). Recording every one would turn a header into a payload, so the ledger+// keeps the first N and counts the rest: the shape of the degradation is in the+// first few, and the count preserves the scale.+const MAX_RECORDED = 24;++/**+ * Record one upstream interaction against the current request.+ *+ * Safe to call from anywhere, including outside a request (it becomes a no-op),+ * so a helper never has to know whether its caller set a context up.+ *+ * @param {{ name: string, outcome: Outcome, ms?: number, tier?: Tier, detail?: string|number }} rec+ */+export function recordSource(rec) {+ const ledger = storage.getStore();+ if (!ledger || !rec?.name) return;+ if (ledger.sources.length >= MAX_RECORDED) {+ ledger.truncated = (ledger.truncated || 0) + 1;+ return;+ }+ ledger.sources.push({+ name: String(rec.name).slice(0, 48),+ outcome: rec.outcome === 'ok' || rec.outcome === 'skip' ? rec.outcome : 'fail',+ ms: Number.isFinite(rec.ms) ? Math.max(0, Math.round(Number(rec.ms))) : 0,+ tier: TIER_RANK.has(rec.tier) ? rec.tier : 'live',+ ...(rec.detail != null ? { detail: String(rec.detail).slice(0, 32) } : {}),+ });+}++/**+ * Aggregate the ledger. `tier` is the worst tier of any source that actually+ * CONTRIBUTED (outcome ok); a failed attempt tells you the chain was exercised+ * but says nothing about the freshness of the answer that finally came back.+ *+ * @returns {{ tier: Tier, sources: number, ok: number, failed: number, ms: number, degraded: boolean, truncated: number, records: SourceRecord[] } | null}+ */+export function provenanceSummary() {+ const ledger = storage.getStore();+ if (!ledger || (!ledger.sources.length && !ledger.truncated)) return null;+ let tier = 'live';+ let ok = 0;+ let failed = 0;+ for (const s of ledger.sources) {+ if (s.outcome === 'ok') {+ ok++;+ tier = worstTier(tier, s.tier);+ } else if (s.outcome === 'fail') {+ failed++;+ }+ }+ return {+ tier,+ sources: ledger.sources.length + (ledger.truncated || 0),+ ok,+ failed,+ ms: Date.now() - ledger.startedAt,+ // Degraded means "a reader should not treat this as a clean live answer":+ // either something had to be retried or failed over, or what came back is+ // not fresh. Both are worth surfacing; neither is an error.+ degraded: failed > 0 || tier !== 'live',+ truncated: ledger.truncated || 0,+ records: ledger.sources,+ };+}++// Header values must be a single line of visible ASCII. A provider name is+// ours, not user input, but it still passes through here so a stray newline or+// separator can never split a header.+const headerSafe = (s) => String(s).replace(/[^\x21-\x7E]|[,;]/g, '_');++/**+ * The two header values for the current request, or null when nothing was+ * recorded. `trace` is present only when something degraded: a healthy response+ * should not pay for a per-source breakdown nobody will read.+ *+ * @returns {{ summary: string, trace: string|null } | null}+ */+export function provenanceHeaders() {+ const s = provenanceSummary();+ if (!s) return null;+ const summary =+ `v=1;tier=${s.tier};sources=${s.sources};ok=${s.ok};failed=${s.failed};ms=${s.ms}` ++ (s.degraded ? ';degraded=1' : '');+ if (!s.degraded) return { summary, trace: null };+ const trace = s.records+ .map((r) => `${headerSafe(r.name)};o=${headerSafe(r.detail ?? r.outcome)};t=${r.ms}`)+ .join(', ');+ return { summary, trace: trace || null };+}diff --git a/tests/brownout-core.test.js b/tests/brownout-core.test.jsnew file mode 100644index 000000000..ba621b382--- /dev/null+++ b/tests/brownout-core.test.js@@ -0,0 +1,250 @@+// Brownout: data provenance and gated fault injection.+//+// These two are the load-bearing halves of the feature, so their edges are+// tested directly: the freshness lattice (a degraded answer must never be able+// to report itself as live), request isolation under concurrency, and every one+// of the gates that stands between a chaos header and a production request.+import { describe, it, expect, afterEach } from 'vitest';+import {+ TIERS,+ worstTier,+ withProvenance,+ recordSource,+ provenanceSummary,+ provenanceHeaders,+} from '../api/_lib/brownout/provenance.js';+import {+ parseChaosDirective,+ chaosDecision,+ withChaos,+ faultFor,+ applyFault,+ CHAOS_HEADER,+ CHAOS_TOKEN_HEADER,+} from '../api/_lib/brownout/chaos.js';++const TOKEN = 'brownout-test-token';+const reqWith = (headers = {}, url = '/api/pump/dashboard', method = 'GET') => ({+ headers: { [CHAOS_TOKEN_HEADER]: TOKEN, ...headers },+ url,+ method,+});++afterEach(() => {+ delete process.env.BROWNOUT_CHAOS_TOKEN;+});++describe('freshness lattice', () => {+ it('orders tiers from freshest to least fresh', () => {+ expect(TIERS).toEqual(['live', 'cache', 'stale', 'fallback']);+ });++ it('takes the WORSE of two tiers, so a mixed response cannot report as live', () => {+ expect(worstTier('live', 'stale')).toBe('stale');+ expect(worstTier('stale', 'live')).toBe('stale');+ expect(worstTier('cache', 'fallback')).toBe('fallback');+ expect(worstTier('live', 'live')).toBe('live');+ });++ it('never trusts an unknown tier into the result', () => {+ expect(worstTier('nonsense', 'stale')).toBe('stale');+ expect(worstTier('stale', 'nonsense')).toBe('stale');+ expect(worstTier(undefined, undefined)).toBe('live');+ });+});++describe('provenance ledger', () => {+ it('is a no-op outside a request context, so a helper never needs to know', () => {+ expect(() => recordSource({ name: 'x', outcome: 'ok' })).not.toThrow();+ expect(provenanceSummary()).toBeNull();+ expect(provenanceHeaders()).toBeNull();+ });++ it('summarises a clean live answer without a trace', () => {+ withProvenance(() => {+ recordSource({ name: 'birdeye', outcome: 'ok', ms: 42, tier: 'live' });+ const s = provenanceSummary();+ expect(s).toMatchObject({ tier: 'live', ok: 1, failed: 0, degraded: false });+ const h = provenanceHeaders();+ expect(h.summary).toContain('tier=live');+ expect(h.summary).not.toContain('degraded=1');+ // A healthy response should not pay for a per-source breakdown.+ expect(h.trace).toBeNull();+ });+ });++ it('reports the worst contributing tier and emits a trace when degraded', () => {+ withProvenance(() => {+ recordSource({ name: 'birdeye', outcome: 'fail', ms: 412, tier: 'live', detail: 429 });+ recordSource({ name: 'dexscreener', outcome: 'ok', ms: 88, tier: 'stale' });+ const s = provenanceSummary();+ expect(s.tier).toBe('stale');+ expect(s).toMatchObject({ ok: 1, failed: 1, degraded: true });+ const h = provenanceHeaders();+ expect(h.summary).toContain('degraded=1');+ expect(h.trace).toContain('birdeye;o=429;t=412');+ expect(h.trace).toContain('dexscreener;o=ok;t=88');+ });+ });++ it('counts a failure as degraded even when the answer that came back was live', () => {+ // A ladder that failed over still tells the reader the chain was exercised.+ withProvenance(() => {+ recordSource({ name: 'a', outcome: 'fail', ms: 10, tier: 'live' });+ recordSource({ name: 'b', outcome: 'ok', ms: 10, tier: 'live' });+ const s = provenanceSummary();+ expect(s.tier).toBe('live');+ expect(s.degraded).toBe(true);+ });+ });++ it('keeps two concurrent requests' + " strictly apart", async () => {+ const [a, b] = await Promise.all([+ withProvenance(async () => {+ recordSource({ name: 'req-a', outcome: 'ok', ms: 1, tier: 'live' });+ await new Promise((r) => setTimeout(r, 5));+ recordSource({ name: 'req-a2', outcome: 'ok', ms: 1, tier: 'stale' });+ return provenanceSummary();+ }),+ withProvenance(async () => {+ await new Promise((r) => setTimeout(r, 1));+ recordSource({ name: 'req-b', outcome: 'fail', ms: 1, tier: 'live' });+ return provenanceSummary();+ }),+ ]);+ expect(a.records.map((r) => r.name)).toEqual(['req-a', 'req-a2']);+ expect(a.tier).toBe('stale');+ expect(b.records.map((r) => r.name)).toEqual(['req-b']);+ expect(b.failed).toBe(1);+ });++ it('caps what it records but keeps the true count, so a fan-out cannot become a payload', () => {+ withProvenance(() => {+ for (let i = 0; i < 200; i++) recordSource({ name: `p${i}`, outcome: 'ok', ms: 1, tier: 'live' });+ const s = provenanceSummary();+ expect(s.records.length).toBeLessThanOrEqual(24);+ expect(s.sources).toBe(200);+ expect(s.truncated).toBe(200 - s.records.length);+ });+ });++ it('never lets a source name break the header grammar', () => {+ withProvenance(() => {+ recordSource({ name: 'evil\nname;with,separators', outcome: 'fail', ms: 1, tier: 'live' });+ const h = provenanceHeaders();+ expect(h.trace).not.toMatch(/[\n\r]/);+ expect(h.trace.split(',').length).toBe(1);+ });+ });+});++describe('chaos directive parsing', () => {+ it('reads every supported fault kind', () => {+ const f = parseChaosDirective('birdeye=http:429, tokens-xyz=timeout, dex=network, cg=empty, slowly=slow:250');+ expect(f.get('birdeye')).toEqual({ kind: 'http', status: 429 });+ expect(f.get('tokens-xyz')).toEqual({ kind: 'timeout' });+ expect(f.get('dex')).toEqual({ kind: 'network' });+ expect(f.get('cg')).toEqual({ kind: 'empty' });+ expect(f.get('slowly')).toEqual({ kind: 'slow', ms: 250 });+ });++ it('drops a malformed entry instead of guessing at it', () => {+ // A typo must disable that one fault, never inject a different one.+ const f = parseChaosDirective('a=http:99, b=http:notanumber, c=teleport, =timeout, d=slow:999999');+ expect(f.size).toBe(0);+ });++ it('defaults a bare name to a network failure', () => {+ expect(parseChaosDirective('birdeye').get('birdeye')).toEqual({ kind: 'network' });+ });+});++describe('chaos gating', () => {+ it('is off entirely when no token is configured', () => {+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout' }));+ expect(d).toMatchObject({ allowed: false, reason: 'not_configured' });+ });++ it('refuses a wrong token', () => {+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;+ const d = chaosDecision({ headers: { [CHAOS_HEADER]: 'birdeye=timeout', [CHAOS_TOKEN_HEADER]: 'nope' }, url: '/api/x', method: 'GET' });+ expect(d).toMatchObject({ allowed: false, reason: 'bad_token' });+ });++ it('allows a valid directive on a read path', () => {+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=http:429' }));+ expect(d.allowed).toBe(true);+ expect(d.faults.get('birdeye')).toEqual({ kind: 'http', status: 429 });+ });++ it('refuses anything carrying a payment header, whatever the route', () => {+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;+ for (const h of ['x-payment', 'payment-signature', 'x-payment-response']) {+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout', [h]: 'anything' }, '/api/intel/heatmap'));+ expect(d, h).toMatchObject({ allowed: false, reason: 'money_path' });+ }+ });++ it('refuses a money route even with a valid token', () => {+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;+ for (const url of ['/api/x402-pay', '/api/pay/execute', '/api/agents/1/withdraw', '/api/purchase/skill', '/api/token/swap']) {+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout' }, url));+ expect(d, url).toMatchObject({ allowed: false, reason: 'money_path' });+ }+ });++ it('refuses a write method', () => {+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=timeout' }, '/api/agents', 'DELETE'));+ expect(d).toMatchObject({ allowed: false, reason: 'write_method' });+ });++ it('says no rather than silently doing nothing when every fault is malformed', () => {+ process.env.BROWNOUT_CHAOS_TOKEN = TOKEN;+ const d = chaosDecision(reqWith({ [CHAOS_HEADER]: 'birdeye=teleport' }));+ expect(d).toMatchObject({ allowed: false, reason: 'no_valid_faults' });+ });+});++describe('fault scoping and shape', () => {+ it('matches a sub-scoped upstream from its parent name', async () => {+ await withChaos(parseChaosDirective('birdeye=timeout'), async () => {+ expect(faultFor('birdeye')).toEqual({ kind: 'timeout' });+ // A directive for the provider covers every call shape it serves.+ expect(faultFor('birdeye:txs')).toEqual({ kind: 'timeout' });+ expect(faultFor('dexscreener')).toBeNull();+ });+ });++ it('is inert outside a chaos context', () => {+ expect(faultFor('birdeye')).toBeNull();+ });++ it('raises a timeout indistinguishable from a real one', async () => {+ await expect(applyFault({ kind: 'timeout' }, 'https://x')).rejects.toMatchObject({ name: 'TimeoutError' });+ });++ it('raises a network failure with the errno undici would attach', async () => {+ // api/_lib/solana/connection.js reads exactly this cause to name the fault.+ await expect(applyFault({ kind: 'network' }, 'https://x')).rejects.toMatchObject({+ message: 'fetch failed',+ cause: { code: 'ECONNREFUSED' },+ });+ });++ it('returns a real Response for an http fault, with Retry-After on a 429', async () => {+ const res = await applyFault({ kind: 'http', status: 429 }, 'https://x');+ expect(res.status).toBe(429);+ expect(res.headers.get('retry-after')).toBe('1');+ const res500 = await applyFault({ kind: 'http', status: 500 }, 'https://x');+ expect(res500.headers.get('retry-after')).toBeNull();+ });++ it('lets a slow fault fall through to the real call, just later', async () => {+ const t0 = Date.now();+ const out = await applyFault({ kind: 'slow', ms: 30 }, 'https://x');+ expect(out).toBeNull();+ expect(Date.now() - t0).toBeGreaterThanOrEqual(25);+ });+});diff --git a/tests/brownout-integration.test.js b/tests/brownout-integration.test.jsnew file mode 100644index 000000000..a816af250--- /dev/null+++ b/tests/brownout-integration.test.js@@ -0,0 +1,134 @@+// The loop that makes Brownout worth having: inject a fault, run the REAL+// wrapper and the REAL provider ladder, and watch the fallback take over while+// the response reports honestly what happened.+//+// A stubbed `fetch` can only prove the stub was called. These tests break the+// upstream from inside the wrapper that every third-party call in api/ goes+// through, so what is exercised is the retry policy, the breaker, the last-good+// tier and the ladder itself, exactly as a real outage would exercise them.+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import { withProvenance, provenanceSummary, provenanceHeaders } from '../api/_lib/brownout/provenance.js';+import { withChaos, parseChaosDirective } from '../api/_lib/brownout/chaos.js';+import { fetchUpstreamJson, lastGood, _resetLastGood } from '../api/_lib/upstream-fetch.js';+import { fetchFirst } from '../src/shared/failover-fetch.js';++const ORIGINAL_FETCH = globalThis.fetch;+const json = (body, status = 200) =>+ new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });++beforeEach(() => {+ _resetLastGood?.();+});+afterEach(() => {+ globalThis.fetch = ORIGINAL_FETCH;+ vi.restoreAllMocks();+});++describe('fetchUpstream under an injected fault', () => {+ it('fails the named upstream and leaves its siblings alone', async () => {+ globalThis.fetch = vi.fn(async () => json({ ok: true }));+ await withChaos(parseChaosDirective('broken=http:500'), async () => {+ await expect(+ fetchUpstreamJson('https://a.example/x', {}, { name: 'broken', attempts: 1 }),+ ).rejects.toThrow(/http 500/);+ // The un-named upstream is untouched: a fault is scoped to what it names.+ await expect(fetchUpstreamJson('https://b.example/x', {}, { name: 'healthy', attempts: 1 })).resolves.toEqual({ ok: true });+ });+ });++ it('is retried by the real retry policy, and the retry can succeed', async () => {+ // A 429 fault is transient, so withRetry retries it. The second attempt is+ // still faulted, which is what proves the injection survives into the retry+ // rather than being a one-shot.+ let calls = 0;+ globalThis.fetch = vi.fn(async () => {+ calls++;+ return json({ ok: true });+ });+ await withChaos(parseChaosDirective('flaky=http:429'), async () => {+ await expect(+ fetchUpstreamJson('https://a.example/x', {}, { name: 'flaky', attempts: 2, maxRetryAfterMs: 5 }),+ ).rejects.toThrow(/http 429/);+ });+ // The real socket was never opened: every attempt hit the fault.+ expect(calls).toBe(0);+ });++ it('serves the last-good value when the live read is faulted, and marks it stale', async () => {+ globalThis.fetch = vi.fn(async () => json({ price: 42 }));+ const load = () => fetchUpstreamJson('https://a.example/price', {}, { name: 'pricer', attempts: 1 });++ const live = await lastGood('brownout:price', load);+ expect(live).toMatchObject({ value: { price: 42 }, stale: false });++ const out = await withProvenance(() =>+ withChaos(parseChaosDirective('pricer=network'), async () => {+ const res = await lastGood('brownout:price', load, { maxAgeMs: 60_000 });+ return { res, prov: provenanceSummary() };+ }),+ );+ expect(out.res.value).toEqual({ price: 42 });+ expect(out.res.stale).toBe(true);+ // And the response would tell the caller so.+ expect(out.prov.tier).toBe('stale');+ expect(out.prov.degraded).toBe(true);+ });++ it('rethrows honestly when the upstream is faulted and nothing was ever cached', async () => {+ globalThis.fetch = vi.fn(async () => json({ price: 1 }));+ await withChaos(parseChaosDirective('pricer=network'), async () => {+ await expect(+ lastGood('brownout:never-seen', () => fetchUpstreamJson('https://a.example/p', {}, { name: 'pricer', attempts: 1 })),+ ).rejects.toThrow(/fetch failed/);+ });+ });+});++describe('provider ladder under an injected fault', () => {+ const providers = [+ { name: 'primary', url: 'https://p1.example/x', parse: async (r) => (await r.json()).v },+ { name: 'secondary', url: 'https://p2.example/x', parse: async (r) => (await r.json()).v },+ { name: 'tertiary', url: 'https://p3.example/x', parse: async (r) => (await r.json()).v },+ ];++ it('walks past the faulted rung to the next healthy one', async () => {+ globalThis.fetch = vi.fn(async (url) => json({ v: String(url).includes('p2') ? 'from-secondary' : 'from-other' }));+ const out = await withChaos(parseChaosDirective('primary=timeout'), () =>+ fetchFirst(providers, { timeoutMs: 500, cooldownMs: 0 }),+ );+ expect(out.source).toBe('secondary');+ expect(out.value).toBe('from-secondary');+ });++ it('records the whole walk, not just the rung that answered', async () => {+ globalThis.fetch = vi.fn(async () => json({ v: 'ok' }));+ // Both readings are taken INSIDE the context: the ledger is scoped to the+ // request, so asking for it afterwards correctly answers null.+ const { prov, headers } = await withProvenance(async () => {+ await withChaos(parseChaosDirective('primary=http:503, secondary=network'), () =>+ fetchFirst(providers, { timeoutMs: 500, cooldownMs: 0 }),+ );+ // The ladder records asynchronously (it must not block the answer), so+ // let those microtasks land before reading the ledger.+ await new Promise((r) => setTimeout(r, 20));+ return { prov: provenanceSummary(), headers: provenanceHeaders() };+ });+ const names = prov.records.map((r) => r.name);+ expect(names).toEqual(['primary', 'secondary', 'tertiary']);+ expect(prov.failed).toBe(2);+ expect(prov.ok).toBe(1);+ // Two rungs died and the third answered live: the answer is fresh, but the+ // reader still learns the chain was exercised.+ expect(prov.tier).toBe('live');+ expect(prov.degraded).toBe(true);+ expect(headers.summary).toContain('degraded=1');+ expect(headers.trace).toMatch(/primary;o=503/);+ });++ it('fails the whole chain when every rung is faulted', async () => {+ globalThis.fetch = vi.fn(async () => json({ v: 'ok' }));+ await withChaos(parseChaosDirective('primary=network, secondary=network, tertiary=network'), async () => {+ await expect(fetchFirst(providers, { timeoutMs: 500, cooldownMs: 0 })).rejects.toThrow(/providers failed/);+ });+ });+});