Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(resilience): one-line upstream fetch with timeout, Retry-After retry, breaker, host fallback and last-good cache for every third-party call

nirholas003c9fa0863b
diff --git a/api/_lib/upstream-fetch.js b/api/_lib/upstream-fetch.js
new file mode 100644
index 000000000..efd4b3dd9
--- /dev/null
+++ b/api/_lib/upstream-fetch.js
@@ -0,0 +1,257 @@
+// @ts-check
+// One fetch for every third-party upstream call from api/**.
+//
+// The audit of 2026-08-27 found ~60 handler and helper modules calling a raw
+// `fetch(url)` against an external host with no timeout, no retry, no
+// Retry-After handling and no alternative source. Each of those is a place
+// where one slow or rate-limited provider turns into a hung invocation or a
+// 500 for the user. The building blocks already existed (cockatiel retries and
+// breakers in resilience.js, the ordered-provider chain in
+// src/shared/failover-fetch.js, the last-good cache in mem-cache.js); what was
+// missing was one call that composed them so a handler could adopt them in a
+// single line. That is this module.
+//
+// fetchUpstream(url, init, opts) Response, bounded and retried
+// fetchUpstreamJson(url, init, opts) parsed body, same guarantees
+// fetchAnyJson([url, ...], init, opts) first host that answers, in order
+// lastGood(key, load, opts) serve the previous value when load fails
+//
+// Guarantees, in order of what they protect:
+// - A per-attempt timeout (default 8s) composed with any caller signal, so a
+// stalled socket can never outlive the request that opened it.
+// - Retries only on transient failures: network errno, aborts, 408/425/429 and
+// 5xx. A 4xx that is not a rate limit is returned to the caller unchanged so a
+// bad request is never retried into a quota.
+// - Retry-After is honoured up to `maxRetryAfterMs`; beyond that the call gives
+// up immediately rather than sleeping through the whole invocation budget.
+// - One consecutive-failure breaker per logical upstream (`opts.name`), so an
+// outage stops costing every request its full timeout after a few failures.
+// - Non-2xx surfaces as an Error carrying `status`, `url` and a body excerpt so
+// isRetryableError (resilience.js) and callers can branch on it.
+//
+// What this module deliberately does not do: SSRF validation of caller-supplied
+// URLs (use fetch-model.js / ssrf.js for those) and Solana/EVM RPC rotation,
+// which have method-aware chains in solana/connection.js and evm/rpc.js.
+
+import { withRetry, withBreaker, parseRetryAfter, isRetryableError } from './resilience.js';
+import { createCache } from './mem-cache.js';
+
+export const DEFAULT_TIMEOUT_MS = 8_000;
+const DEFAULT_ATTEMPTS = 3;
+const DEFAULT_MAX_RETRY_AFTER_MS = 5_000;
+const BODY_EXCERPT = 200;
+
+export class UpstreamError extends Error {
+ /**
+ * @param {string} message
+ * @param {{ status?: number, url?: string, body?: string, retryAfterMs?: number | null, cause?: unknown }} [info]
+ */
+ constructor(message, info = {}) {
+ super(message);
+ this.name = 'UpstreamError';
+ this.status = info.status;
+ this.url = info.url;
+ this.body = info.body;
+ this.retryAfterMs = info.retryAfterMs ?? null;
+ if (info.cause !== undefined) this.cause = info.cause;
+ }
+}
+
+/** Strip query-string credentials before a URL reaches an error message or log. */
+export function safeUrl(url) {
+ try {
+ const u = new URL(String(url));
+ for (const k of [...u.searchParams.keys()]) {
+ if (/key|token|secret|auth|sig/i.test(k)) u.searchParams.set(k, 'REDACTED');
+ }
+ return u.toString();
+ } catch {
+ return String(url);
+ }
+}
+
+function composeSignal(callerSignal, timeoutMs) {
+ const timeout = AbortSignal.timeout(timeoutMs);
+ if (!callerSignal) return timeout;
+ return AbortSignal.any([callerSignal, timeout]);
+}
+
+/**
+ * @typedef {object} UpstreamOptions
+ * @property {number} [timeoutMs] per-attempt budget (default 8000)
+ * @property {number} [attempts] total attempts including the first (default 3)
+ * @property {number} [maxRetryAfterMs] longest Retry-After we will sleep for (default 5000)
+ * @property {string} [name] logical upstream id; enables the shared breaker
+ * @property {number} [breakerThreshold] consecutive failures before the breaker opens (default 5)
+ * @property {number} [halfOpenAfterMs] breaker cooldown (default 30000)
+ * @property {(res: Response) => boolean} [okWhen] override "success" (default res.ok)
+ * @property {string} [label] prefix for thrown error messages
+ */
+
+/**
+ * Fetch an external URL with a timeout, transient-failure retries and an
+ * optional breaker. Resolves with the Response only when `okWhen(res)` holds
+ * (default `res.ok`); every other outcome rejects with an UpstreamError so no
+ * caller has to remember to check `res.ok`.
+ *
+ * @param {string} url
+ * @param {RequestInit} [init]
+ * @param {UpstreamOptions} [opts]
+ * @returns {Promise<Response>}
+ */
+export async function fetchUpstream(url, init = {}, opts = {}) {
+ const {
+ timeoutMs = DEFAULT_TIMEOUT_MS,
+ attempts = DEFAULT_ATTEMPTS,
+ maxRetryAfterMs = DEFAULT_MAX_RETRY_AFTER_MS,
+ name,
+ breakerThreshold = 5,
+ halfOpenAfterMs = 30_000,
+ okWhen = (res) => res.ok,
+ label,
+ } = opts;
+ const shown = safeUrl(url);
+
+ const attempt = async () => {
+ let res;
+ try {
+ res = await fetch(url, { ...init, signal: composeSignal(init.signal, timeoutMs) });
+ } catch (err) {
+ if (init.signal?.aborted) throw err;
+ const e = new UpstreamError(`${shown}: ${err?.name === 'TimeoutError' ? `timed out after ${timeoutMs}ms` : err?.message || 'network error'}`, { url: shown, cause: err });
+ // Mark as transient so withRetry keeps going; isRetryableError recognises the name.
+ e.name = err?.name === 'TimeoutError' ? 'TimeoutError' : 'UpstreamError';
+ if (e.name === 'UpstreamError') e.status = 503;
+ throw e;
+ }
+ if (okWhen(res)) return res;
+ const body = await res.text().catch(() => '');
+ const retryAfterMs = parseRetryAfter(res.headers.get('retry-after'));
+ const e = new UpstreamError(`${shown}: http ${res.status}${body ? ` ${body.slice(0, BODY_EXCERPT)}` : ''}`, {
+ status: res.status,
+ url: shown,
+ body: body.slice(0, BODY_EXCERPT),
+ retryAfterMs,
+ });
+ if (retryAfterMs != null && retryAfterMs > maxRetryAfterMs) {
+ // The upstream asked for a wait longer than we will spend: fail now so the
+ // caller can move to its next source instead of burning the invocation.
+ e.tooLongToWait = true;
+ } else if (retryAfterMs != null && retryAfterMs > 0 && isRetryableError(e)) {
+ await new Promise((r) => setTimeout(r, retryAfterMs));
+ }
+ throw e;
+ };
+
+ const shouldRetry = (err) => !err?.tooLongToWait && !init.signal?.aborted && isRetryableError(err);
+ const run = () => withRetry(attempt, { attempts, shouldRetry, label });
+
+ if (!name) return run();
+ let failure;
+ const value = await withBreaker(name, run, {
+ threshold: breakerThreshold,
+ halfOpenAfterMs,
+ fallback: (err) => {
+ failure = err;
+ return null;
+ },
+ });
+ if (value) return value;
+ throw failure instanceof Error ? failure : new UpstreamError(`${label || name}: circuit open`, { url: shown, status: 503 });
+}
+
+/**
+ * fetchUpstream + JSON parse. A body that is not JSON rejects with an
+ * UpstreamError (status 502) rather than a bare SyntaxError, so a provider that
+ * starts returning an HTML error page is classified as an upstream fault.
+ *
+ * @template T
+ * @param {string} url
+ * @param {RequestInit} [init]
+ * @param {UpstreamOptions} [opts]
+ * @returns {Promise<T>}
+ */
+export async function fetchUpstreamJson(url, init = {}, opts = {}) {
+ const res = await fetchUpstream(url, { headers: { accept: 'application/json' }, ...init }, opts);
+ const text = await res.text();
+ try {
+ return JSON.parse(text);
+ } catch {
+ throw new UpstreamError(`${safeUrl(url)}: non-JSON body ${text.slice(0, BODY_EXCERPT)}`, { status: 502, url: safeUrl(url), body: text.slice(0, BODY_EXCERPT) });
+ }
+}
+
+/**
+ * Try several equivalent hosts in order and return the first parsed JSON body.
+ * Each host gets its own timeout and (short) retry; a host that fails is skipped
+ * for the rest of this call, and, when `opts.name` is set, tracked by its own
+ * breaker (`${name}:${index}`) so a dead mirror is skipped on later calls too.
+ * `opts.accept(value, url)` may reject a 2xx body that carries no usable data
+ * (return false) so the chain moves on without penalising the host.
+ *
+ * @template T
+ * @param {string[]} urls
+ * @param {RequestInit} [init]
+ * @param {UpstreamOptions & { accept?: (value: any, url: string) => boolean }} [opts]
+ * @returns {Promise<{ value: T, url: string, index: number }>}
+ */
+export async function fetchAnyJson(urls, init = {}, opts = {}) {
+ const { accept, name, attempts = 2, ...rest } = opts;
+ const list = urls.filter(Boolean);
+ if (!list.length) throw new UpstreamError(`${opts.label || 'fetchAnyJson'}: no urls configured`, { status: 503 });
+ let lastErr;
+ for (let i = 0; i < list.length; i++) {
+ const url = list[i];
+ try {
+ const value = await fetchUpstreamJson(url, init, { ...rest, attempts, name: name ? `${name}:${i}` : undefined });
+ if (accept && !accept(value, url)) {
+ lastErr = new UpstreamError(`${safeUrl(url)}: no usable data`, { url: safeUrl(url), status: 404 });
+ continue;
+ }
+ return { value, url, index: i };
+ } catch (err) {
+ lastErr = err;
+ if (init.signal?.aborted) break;
+ }
+ }
+ throw new UpstreamError(`${opts.label || 'fetchAnyJson'}: all ${list.length} sources failed (${lastErr?.message || 'unknown'})`, { status: 503, cause: lastErr });
+}
+
+// Process-wide last-known-good store. Keys are caller-chosen; values live for
+// `maxAgeMs` after their last successful load. Small on purpose: this is the
+// "keep the page alive through a blip" layer, not a cache.
+const lastGoodStore = createCache({ max: 2_000, ttlMs: 6 * 60 * 60_000 });
+
+/**
+ * Run `load()`; on success remember the value under `key`; on failure return
+ * the remembered value if it is younger than `maxAgeMs`, else rethrow. The
+ * returned object says whether the value is stale so a handler can label it
+ * (`x-data-stale: 1`, a "last updated" caption) instead of passing it off as live.
+ *
+ * @template T
+ * @param {string} key
+ * @param {() => Promise<T>} load
+ * @param {{ maxAgeMs?: number, onFallback?: (err: unknown, ageMs: number) => void }} [opts]
+ * @returns {Promise<{ value: T, stale: boolean, ageMs: number, error?: unknown }>}
+ */
+export async function lastGood(key, load, opts = {}) {
+ const { maxAgeMs = 15 * 60_000, onFallback } = opts;
+ try {
+ const value = await load();
+ if (value !== undefined && value !== null) lastGoodStore.set(key, { value, at: Date.now() });
+ return { value, stale: false, ageMs: 0 };
+ } catch (err) {
+ const hit = lastGoodStore.get(key);
+ const ageMs = hit ? Date.now() - hit.at : Infinity;
+ if (hit && ageMs <= maxAgeMs) {
+ onFallback?.(err, ageMs);
+ return { value: hit.value, stale: true, ageMs, error: err };
+ }
+ throw err;
+ }
+}
+
+/** Test hook: forget every remembered value. */
+export function _resetLastGood() {
+ lastGoodStore.clear();
+}
diff --git a/tests/upstream-fetch.test.js b/tests/upstream-fetch.test.js
new file mode 100644
index 000000000..8149b344c
--- /dev/null
+++ b/tests/upstream-fetch.test.js
@@ -0,0 +1,133 @@
+// The one-line resilient fetch every api/** third-party call adopts:
+// timeout, transient retry, Retry-After, ordered host fallback, last-good.
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import {
+ fetchUpstream,
+ fetchUpstreamJson,
+ fetchAnyJson,
+ lastGood,
+ safeUrl,
+ UpstreamError,
+ _resetLastGood,
+} from '../api/_lib/upstream-fetch.js';
+import { _resetBreakers } from '../api/_lib/resilience.js';
+
+const json = (body, init = {}) =>
+ new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init });
+
+let origFetch;
+beforeEach(() => {
+ origFetch = global.fetch;
+ _resetBreakers();
+ _resetLastGood();
+});
+afterEach(() => {
+ global.fetch = origFetch;
+});
+
+describe('fetchUpstream', () => {
+ it('returns the response on 2xx without retrying', async () => {
+ global.fetch = vi.fn(async () => json({ ok: 1 }));
+ const res = await fetchUpstream('https://a.example/x');
+ expect(res.status).toBe(200);
+ expect(global.fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('passes a timeout signal to fetch', async () => {
+ global.fetch = vi.fn(async (_u, init) => {
+ expect(init.signal).toBeInstanceOf(AbortSignal);
+ return json({});
+ });
+ await fetchUpstream('https://a.example/x', {}, { timeoutMs: 50 });
+ });
+
+ it('retries a 503 and then succeeds', async () => {
+ let n = 0;
+ global.fetch = vi.fn(async () => (++n < 3 ? new Response('down', { status: 503 }) : json({ n })));
+ const data = await fetchUpstreamJson('https://a.example/x', {}, { attempts: 3 });
+ expect(data).toEqual({ n: 3 });
+ expect(global.fetch).toHaveBeenCalledTimes(3);
+ });
+
+ it('does not retry a 404 and surfaces status + body', async () => {
+ global.fetch = vi.fn(async () => new Response('nope', { status: 404 }));
+ await expect(fetchUpstream('https://a.example/x')).rejects.toMatchObject({
+ name: 'UpstreamError',
+ status: 404,
+ body: 'nope',
+ });
+ expect(global.fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('gives up immediately on a Retry-After longer than it will wait', async () => {
+ global.fetch = vi.fn(async () => new Response('slow down', { status: 429, headers: { 'retry-after': '120' } }));
+ await expect(fetchUpstream('https://a.example/x', {}, { attempts: 3 })).rejects.toMatchObject({ status: 429, retryAfterMs: 120_000 });
+ expect(global.fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('honours a short Retry-After before retrying', async () => {
+ let n = 0;
+ global.fetch = vi.fn(async () => (++n === 1 ? new Response('', { status: 429, headers: { 'retry-after': '0' } }) : json({ n })));
+ await expect(fetchUpstreamJson('https://a.example/x', {}, { attempts: 2 })).resolves.toEqual({ n: 2 });
+ });
+
+ it('classifies a non-JSON 2xx body as a 502 upstream fault', async () => {
+ global.fetch = vi.fn(async () => new Response('<html>cloudflare</html>', { status: 200 }));
+ await expect(fetchUpstreamJson('https://a.example/x', {}, { attempts: 1 })).rejects.toMatchObject({ status: 502 });
+ });
+
+ it('opens the breaker after consecutive failures and stops calling fetch', async () => {
+ global.fetch = vi.fn(async () => new Response('', { status: 500 }));
+ const opts = { name: 'test:breaker', attempts: 1, breakerThreshold: 2 };
+ await expect(fetchUpstream('https://a.example/x', {}, opts)).rejects.toBeInstanceOf(UpstreamError);
+ await expect(fetchUpstream('https://a.example/x', {}, opts)).rejects.toBeInstanceOf(UpstreamError);
+ const calls = global.fetch.mock.calls.length;
+ await expect(fetchUpstream('https://a.example/x', {}, opts)).rejects.toBeInstanceOf(Error);
+ expect(global.fetch.mock.calls.length).toBe(calls);
+ });
+});
+
+describe('fetchAnyJson', () => {
+ it('falls through to the next host and reports which one answered', async () => {
+ global.fetch = vi.fn(async (url) => (String(url).includes('dead') ? new Response('', { status: 502 }) : json({ from: 'b' })));
+ const r = await fetchAnyJson(['https://dead.example/', 'https://b.example/'], {}, { attempts: 1 });
+ expect(r.value).toEqual({ from: 'b' });
+ expect(r.index).toBe(1);
+ });
+
+ it('skips a healthy host whose body carries no usable data', async () => {
+ global.fetch = vi.fn(async (url) => json(String(url).includes('a.') ? {} : { price: 1 }));
+ const r = await fetchAnyJson(['https://a.example/', 'https://b.example/'], {}, { accept: (v) => v.price != null });
+ expect(r.value.price).toBe(1);
+ });
+
+ it('throws a 503 when every host fails', async () => {
+ global.fetch = vi.fn(async () => new Response('', { status: 500 }));
+ await expect(fetchAnyJson(['https://a.example/', 'https://b.example/'], {}, { attempts: 1 })).rejects.toMatchObject({ status: 503 });
+ });
+});
+
+describe('lastGood', () => {
+ it('serves the previous value, flagged stale, when the loader fails', async () => {
+ let fail = false;
+ const load = async () => {
+ if (fail) throw new Error('boom');
+ return { v: 1 };
+ };
+ expect(await lastGood('k', load)).toMatchObject({ value: { v: 1 }, stale: false });
+ fail = true;
+ const r = await lastGood('k', load);
+ expect(r.stale).toBe(true);
+ expect(r.value).toEqual({ v: 1 });
+ });
+
+ it('rethrows when nothing was ever loaded', async () => {
+ await expect(lastGood('empty', async () => { throw new Error('boom'); })).rejects.toThrow('boom');
+ });
+});
+
+describe('safeUrl', () => {
+ it('redacts credential-looking query params', () => {
+ expect(safeUrl('https://api.helius.xyz/v0/tx?api-key=abc&limit=1')).toBe('https://api.helius.xyz/v0/tx?api-key=REDACTED&limit=1');
+ });
+});