Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(resilience): one-line upstream fetch with timeout, Retry-After retry, breaker, host fallback and last-good cache for every third-party call
nirholas003c9fa0863b
diff --git a/api/_lib/upstream-fetch.js b/api/_lib/upstream-fetch.jsnew file mode 100644index 000000000..efd4b3dd9--- /dev/null+++ b/api/_lib/upstream-fetch.js@@ -0,0 +1,257 @@+// @ts-check+// One fetch for every third-party upstream call from api/**.+//+// The audit of 2026-08-27 found ~60 handler and helper modules calling a raw+// `fetch(url)` against an external host with no timeout, no retry, no+// Retry-After handling and no alternative source. Each of those is a place+// where one slow or rate-limited provider turns into a hung invocation or a+// 500 for the user. The building blocks already existed (cockatiel retries and+// breakers in resilience.js, the ordered-provider chain in+// src/shared/failover-fetch.js, the last-good cache in mem-cache.js); what was+// missing was one call that composed them so a handler could adopt them in a+// single line. That is this module.+//+// fetchUpstream(url, init, opts) Response, bounded and retried+// fetchUpstreamJson(url, init, opts) parsed body, same guarantees+// fetchAnyJson([url, ...], init, opts) first host that answers, in order+// lastGood(key, load, opts) serve the previous value when load fails+//+// Guarantees, in order of what they protect:+// - A per-attempt timeout (default 8s) composed with any caller signal, so a+// stalled socket can never outlive the request that opened it.+// - Retries only on transient failures: network errno, aborts, 408/425/429 and+// 5xx. A 4xx that is not a rate limit is returned to the caller unchanged so a+// bad request is never retried into a quota.+// - Retry-After is honoured up to `maxRetryAfterMs`; beyond that the call gives+// up immediately rather than sleeping through the whole invocation budget.+// - One consecutive-failure breaker per logical upstream (`opts.name`), so an+// outage stops costing every request its full timeout after a few failures.+// - Non-2xx surfaces as an Error carrying `status`, `url` and a body excerpt so+// isRetryableError (resilience.js) and callers can branch on it.+//+// What this module deliberately does not do: SSRF validation of caller-supplied+// URLs (use fetch-model.js / ssrf.js for those) and Solana/EVM RPC rotation,+// which have method-aware chains in solana/connection.js and evm/rpc.js.++import { withRetry, withBreaker, parseRetryAfter, isRetryableError } from './resilience.js';+import { createCache } from './mem-cache.js';++export const DEFAULT_TIMEOUT_MS = 8_000;+const DEFAULT_ATTEMPTS = 3;+const DEFAULT_MAX_RETRY_AFTER_MS = 5_000;+const BODY_EXCERPT = 200;++export class UpstreamError extends Error {+ /**+ * @param {string} message+ * @param {{ status?: number, url?: string, body?: string, retryAfterMs?: number | null, cause?: unknown }} [info]+ */+ constructor(message, info = {}) {+ super(message);+ this.name = 'UpstreamError';+ this.status = info.status;+ this.url = info.url;+ this.body = info.body;+ this.retryAfterMs = info.retryAfterMs ?? null;+ if (info.cause !== undefined) this.cause = info.cause;+ }+}++/** Strip query-string credentials before a URL reaches an error message or log. */+export function safeUrl(url) {+ try {+ const u = new URL(String(url));+ for (const k of [...u.searchParams.keys()]) {+ if (/key|token|secret|auth|sig/i.test(k)) u.searchParams.set(k, 'REDACTED');+ }+ return u.toString();+ } catch {+ return String(url);+ }+}++function composeSignal(callerSignal, timeoutMs) {+ const timeout = AbortSignal.timeout(timeoutMs);+ if (!callerSignal) return timeout;+ return AbortSignal.any([callerSignal, timeout]);+}++/**+ * @typedef {object} UpstreamOptions+ * @property {number} [timeoutMs] per-attempt budget (default 8000)+ * @property {number} [attempts] total attempts including the first (default 3)+ * @property {number} [maxRetryAfterMs] longest Retry-After we will sleep for (default 5000)+ * @property {string} [name] logical upstream id; enables the shared breaker+ * @property {number} [breakerThreshold] consecutive failures before the breaker opens (default 5)+ * @property {number} [halfOpenAfterMs] breaker cooldown (default 30000)+ * @property {(res: Response) => boolean} [okWhen] override "success" (default res.ok)+ * @property {string} [label] prefix for thrown error messages+ */++/**+ * Fetch an external URL with a timeout, transient-failure retries and an+ * optional breaker. Resolves with the Response only when `okWhen(res)` holds+ * (default `res.ok`); every other outcome rejects with an UpstreamError so no+ * caller has to remember to check `res.ok`.+ *+ * @param {string} url+ * @param {RequestInit} [init]+ * @param {UpstreamOptions} [opts]+ * @returns {Promise<Response>}+ */+export async function fetchUpstream(url, init = {}, opts = {}) {+ const {+ timeoutMs = DEFAULT_TIMEOUT_MS,+ attempts = DEFAULT_ATTEMPTS,+ maxRetryAfterMs = DEFAULT_MAX_RETRY_AFTER_MS,+ name,+ breakerThreshold = 5,+ halfOpenAfterMs = 30_000,+ okWhen = (res) => res.ok,+ label,+ } = opts;+ const shown = safeUrl(url);++ const attempt = async () => {+ let res;+ try {+ res = await fetch(url, { ...init, signal: composeSignal(init.signal, timeoutMs) });+ } catch (err) {+ if (init.signal?.aborted) throw err;+ const e = new UpstreamError(`${shown}: ${err?.name === 'TimeoutError' ? `timed out after ${timeoutMs}ms` : err?.message || 'network error'}`, { url: shown, cause: err });+ // Mark as transient so withRetry keeps going; isRetryableError recognises the name.+ e.name = err?.name === 'TimeoutError' ? 'TimeoutError' : 'UpstreamError';+ if (e.name === 'UpstreamError') e.status = 503;+ throw e;+ }+ if (okWhen(res)) return res;+ const body = await res.text().catch(() => '');+ const retryAfterMs = parseRetryAfter(res.headers.get('retry-after'));+ const e = new UpstreamError(`${shown}: http ${res.status}${body ? ` ${body.slice(0, BODY_EXCERPT)}` : ''}`, {+ status: res.status,+ url: shown,+ body: body.slice(0, BODY_EXCERPT),+ retryAfterMs,+ });+ if (retryAfterMs != null && retryAfterMs > maxRetryAfterMs) {+ // The upstream asked for a wait longer than we will spend: fail now so the+ // caller can move to its next source instead of burning the invocation.+ e.tooLongToWait = true;+ } else if (retryAfterMs != null && retryAfterMs > 0 && isRetryableError(e)) {+ await new Promise((r) => setTimeout(r, retryAfterMs));+ }+ throw e;+ };++ const shouldRetry = (err) => !err?.tooLongToWait && !init.signal?.aborted && isRetryableError(err);+ const run = () => withRetry(attempt, { attempts, shouldRetry, label });++ if (!name) return run();+ let failure;+ const value = await withBreaker(name, run, {+ threshold: breakerThreshold,+ halfOpenAfterMs,+ fallback: (err) => {+ failure = err;+ return null;+ },+ });+ if (value) return value;+ throw failure instanceof Error ? failure : new UpstreamError(`${label || name}: circuit open`, { url: shown, status: 503 });+}++/**+ * fetchUpstream + JSON parse. A body that is not JSON rejects with an+ * UpstreamError (status 502) rather than a bare SyntaxError, so a provider that+ * starts returning an HTML error page is classified as an upstream fault.+ *+ * @template T+ * @param {string} url+ * @param {RequestInit} [init]+ * @param {UpstreamOptions} [opts]+ * @returns {Promise<T>}+ */+export async function fetchUpstreamJson(url, init = {}, opts = {}) {+ const res = await fetchUpstream(url, { headers: { accept: 'application/json' }, ...init }, opts);+ const text = await res.text();+ try {+ return JSON.parse(text);+ } catch {+ throw new UpstreamError(`${safeUrl(url)}: non-JSON body ${text.slice(0, BODY_EXCERPT)}`, { status: 502, url: safeUrl(url), body: text.slice(0, BODY_EXCERPT) });+ }+}++/**+ * Try several equivalent hosts in order and return the first parsed JSON body.+ * Each host gets its own timeout and (short) retry; a host that fails is skipped+ * for the rest of this call, and, when `opts.name` is set, tracked by its own+ * breaker (`${name}:${index}`) so a dead mirror is skipped on later calls too.+ * `opts.accept(value, url)` may reject a 2xx body that carries no usable data+ * (return false) so the chain moves on without penalising the host.+ *+ * @template T+ * @param {string[]} urls+ * @param {RequestInit} [init]+ * @param {UpstreamOptions & { accept?: (value: any, url: string) => boolean }} [opts]+ * @returns {Promise<{ value: T, url: string, index: number }>}+ */+export async function fetchAnyJson(urls, init = {}, opts = {}) {+ const { accept, name, attempts = 2, ...rest } = opts;+ const list = urls.filter(Boolean);+ if (!list.length) throw new UpstreamError(`${opts.label || 'fetchAnyJson'}: no urls configured`, { status: 503 });+ let lastErr;+ for (let i = 0; i < list.length; i++) {+ const url = list[i];+ try {+ const value = await fetchUpstreamJson(url, init, { ...rest, attempts, name: name ? `${name}:${i}` : undefined });+ if (accept && !accept(value, url)) {+ lastErr = new UpstreamError(`${safeUrl(url)}: no usable data`, { url: safeUrl(url), status: 404 });+ continue;+ }+ return { value, url, index: i };+ } catch (err) {+ lastErr = err;+ if (init.signal?.aborted) break;+ }+ }+ throw new UpstreamError(`${opts.label || 'fetchAnyJson'}: all ${list.length} sources failed (${lastErr?.message || 'unknown'})`, { status: 503, cause: lastErr });+}++// Process-wide last-known-good store. Keys are caller-chosen; values live for+// `maxAgeMs` after their last successful load. Small on purpose: this is the+// "keep the page alive through a blip" layer, not a cache.+const lastGoodStore = createCache({ max: 2_000, ttlMs: 6 * 60 * 60_000 });++/**+ * Run `load()`; on success remember the value under `key`; on failure return+ * the remembered value if it is younger than `maxAgeMs`, else rethrow. The+ * returned object says whether the value is stale so a handler can label it+ * (`x-data-stale: 1`, a "last updated" caption) instead of passing it off as live.+ *+ * @template T+ * @param {string} key+ * @param {() => Promise<T>} load+ * @param {{ maxAgeMs?: number, onFallback?: (err: unknown, ageMs: number) => void }} [opts]+ * @returns {Promise<{ value: T, stale: boolean, ageMs: number, error?: unknown }>}+ */+export async function lastGood(key, load, opts = {}) {+ const { maxAgeMs = 15 * 60_000, onFallback } = opts;+ try {+ const value = await load();+ if (value !== undefined && value !== null) lastGoodStore.set(key, { value, at: Date.now() });+ return { value, stale: false, ageMs: 0 };+ } catch (err) {+ const hit = lastGoodStore.get(key);+ const ageMs = hit ? Date.now() - hit.at : Infinity;+ if (hit && ageMs <= maxAgeMs) {+ onFallback?.(err, ageMs);+ return { value: hit.value, stale: true, ageMs, error: err };+ }+ throw err;+ }+}++/** Test hook: forget every remembered value. */+export function _resetLastGood() {+ lastGoodStore.clear();+}diff --git a/tests/upstream-fetch.test.js b/tests/upstream-fetch.test.jsnew file mode 100644index 000000000..8149b344c--- /dev/null+++ b/tests/upstream-fetch.test.js@@ -0,0 +1,133 @@+// The one-line resilient fetch every api/** third-party call adopts:+// timeout, transient retry, Retry-After, ordered host fallback, last-good.+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import {+ fetchUpstream,+ fetchUpstreamJson,+ fetchAnyJson,+ lastGood,+ safeUrl,+ UpstreamError,+ _resetLastGood,+} from '../api/_lib/upstream-fetch.js';+import { _resetBreakers } from '../api/_lib/resilience.js';++const json = (body, init = {}) =>+ new Response(JSON.stringify(body), { status: 200, headers: { 'content-type': 'application/json' }, ...init });++let origFetch;+beforeEach(() => {+ origFetch = global.fetch;+ _resetBreakers();+ _resetLastGood();+});+afterEach(() => {+ global.fetch = origFetch;+});++describe('fetchUpstream', () => {+ it('returns the response on 2xx without retrying', async () => {+ global.fetch = vi.fn(async () => json({ ok: 1 }));+ const res = await fetchUpstream('https://a.example/x');+ expect(res.status).toBe(200);+ expect(global.fetch).toHaveBeenCalledTimes(1);+ });++ it('passes a timeout signal to fetch', async () => {+ global.fetch = vi.fn(async (_u, init) => {+ expect(init.signal).toBeInstanceOf(AbortSignal);+ return json({});+ });+ await fetchUpstream('https://a.example/x', {}, { timeoutMs: 50 });+ });++ it('retries a 503 and then succeeds', async () => {+ let n = 0;+ global.fetch = vi.fn(async () => (++n < 3 ? new Response('down', { status: 503 }) : json({ n })));+ const data = await fetchUpstreamJson('https://a.example/x', {}, { attempts: 3 });+ expect(data).toEqual({ n: 3 });+ expect(global.fetch).toHaveBeenCalledTimes(3);+ });++ it('does not retry a 404 and surfaces status + body', async () => {+ global.fetch = vi.fn(async () => new Response('nope', { status: 404 }));+ await expect(fetchUpstream('https://a.example/x')).rejects.toMatchObject({+ name: 'UpstreamError',+ status: 404,+ body: 'nope',+ });+ expect(global.fetch).toHaveBeenCalledTimes(1);+ });++ it('gives up immediately on a Retry-After longer than it will wait', async () => {+ global.fetch = vi.fn(async () => new Response('slow down', { status: 429, headers: { 'retry-after': '120' } }));+ await expect(fetchUpstream('https://a.example/x', {}, { attempts: 3 })).rejects.toMatchObject({ status: 429, retryAfterMs: 120_000 });+ expect(global.fetch).toHaveBeenCalledTimes(1);+ });++ it('honours a short Retry-After before retrying', async () => {+ let n = 0;+ global.fetch = vi.fn(async () => (++n === 1 ? new Response('', { status: 429, headers: { 'retry-after': '0' } }) : json({ n })));+ await expect(fetchUpstreamJson('https://a.example/x', {}, { attempts: 2 })).resolves.toEqual({ n: 2 });+ });++ it('classifies a non-JSON 2xx body as a 502 upstream fault', async () => {+ global.fetch = vi.fn(async () => new Response('<html>cloudflare</html>', { status: 200 }));+ await expect(fetchUpstreamJson('https://a.example/x', {}, { attempts: 1 })).rejects.toMatchObject({ status: 502 });+ });++ it('opens the breaker after consecutive failures and stops calling fetch', async () => {+ global.fetch = vi.fn(async () => new Response('', { status: 500 }));+ const opts = { name: 'test:breaker', attempts: 1, breakerThreshold: 2 };+ await expect(fetchUpstream('https://a.example/x', {}, opts)).rejects.toBeInstanceOf(UpstreamError);+ await expect(fetchUpstream('https://a.example/x', {}, opts)).rejects.toBeInstanceOf(UpstreamError);+ const calls = global.fetch.mock.calls.length;+ await expect(fetchUpstream('https://a.example/x', {}, opts)).rejects.toBeInstanceOf(Error);+ expect(global.fetch.mock.calls.length).toBe(calls);+ });+});++describe('fetchAnyJson', () => {+ it('falls through to the next host and reports which one answered', async () => {+ global.fetch = vi.fn(async (url) => (String(url).includes('dead') ? new Response('', { status: 502 }) : json({ from: 'b' })));+ const r = await fetchAnyJson(['https://dead.example/', 'https://b.example/'], {}, { attempts: 1 });+ expect(r.value).toEqual({ from: 'b' });+ expect(r.index).toBe(1);+ });++ it('skips a healthy host whose body carries no usable data', async () => {+ global.fetch = vi.fn(async (url) => json(String(url).includes('a.') ? {} : { price: 1 }));+ const r = await fetchAnyJson(['https://a.example/', 'https://b.example/'], {}, { accept: (v) => v.price != null });+ expect(r.value.price).toBe(1);+ });++ it('throws a 503 when every host fails', async () => {+ global.fetch = vi.fn(async () => new Response('', { status: 500 }));+ await expect(fetchAnyJson(['https://a.example/', 'https://b.example/'], {}, { attempts: 1 })).rejects.toMatchObject({ status: 503 });+ });+});++describe('lastGood', () => {+ it('serves the previous value, flagged stale, when the loader fails', async () => {+ let fail = false;+ const load = async () => {+ if (fail) throw new Error('boom');+ return { v: 1 };+ };+ expect(await lastGood('k', load)).toMatchObject({ value: { v: 1 }, stale: false });+ fail = true;+ const r = await lastGood('k', load);+ expect(r.stale).toBe(true);+ expect(r.value).toEqual({ v: 1 });+ });++ it('rethrows when nothing was ever loaded', async () => {+ await expect(lastGood('empty', async () => { throw new Error('boom'); })).rejects.toThrow('boom');+ });+});++describe('safeUrl', () => {+ it('redacts credential-looking query params', () => {+ expect(safeUrl('https://api.helius.xyz/v0/tx?api-key=abc&limit=1')).toBe('https://api.helius.xyz/v0/tx?api-key=REDACTED&limit=1');+ });+});