Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(resilience): serve last-good data through upstream blips and put a deadline on every browser API call

nirholasb0539e498dfb
diff --git a/api/_lib/cache.js b/api/_lib/cache.js
index 078be9874..d275af7d2 100644
--- a/api/_lib/cache.js
+++ b/api/_lib/cache.js
@@ -557,3 +557,38 @@ export async function cacheWrap(key, ttlSeconds, fn) {
}
return value;
}
+
+/**
+ * `cacheWrap` with a last-known-good tier in the shared store. A fresh value is
+ * served for `ttlSeconds`; every fresh value is also mirrored under `<key>:lkg`
+ * for `staleTtlSeconds` (default 24h). When `fn()` throws and a mirror exists,
+ * the mirror is returned instead of the rejection, so a throttled or dead
+ * upstream degrades to minutes-stale data rather than a 5xx. The result is
+ * unwrapped; pass `{ withMeta: true }` to get `{ value, stale }`.
+ *
+ * @template T
+ * @param {string} key
+ * @param {number} ttlSeconds
+ * @param {() => Promise<T>} fn
+ * @param {{ staleTtlSeconds?: number, withMeta?: boolean }} [opts]
+ * @returns {Promise<T | { value: T, stale: boolean }>}
+ */
+export async function cacheWrapLastGood(key, ttlSeconds, fn, opts = {}) {
+ const { staleTtlSeconds = 24 * 3600, withMeta = false } = opts;
+ const lkgKey = `${key}:lkg`;
+ const hit = await cacheGet(key);
+ if (hit !== null && hit !== undefined) return withMeta ? { value: hit, stale: false } : hit;
+ try {
+ const value = await fn();
+ if (value !== null && value !== undefined) {
+ cacheSet(key, value, ttlSeconds).catch(() => {});
+ cacheSet(lkgKey, value, staleTtlSeconds).catch(() => {});
+ }
+ return withMeta ? { value, stale: false } : value;
+ } catch (err) {
+ const lkg = await cacheGet(lkgKey);
+ if (lkg === null || lkg === undefined) throw err;
+ console.warn(`[cache] "${key}" refresh failed (${err?.message || err}); serving last-good`);
+ return withMeta ? { value: lkg, stale: true } : lkg;
+ }
+}
diff --git a/api/_lib/mem-cache.js b/api/_lib/mem-cache.js
index 039176475..eed878fc7 100644
--- a/api/_lib/mem-cache.js
+++ b/api/_lib/mem-cache.js
@@ -50,13 +50,23 @@ export function createCache({ max = 512, ttlMs, updateAgeOnGet = false } = {}) {
* the same key share one `load()` call instead of stampeding the upstream.
* Four separate inflight-dedupe Maps existed across api/ before this.
*
+ * Stale-on-error: every value that `load()` produces is also mirrored into a
+ * per-cache last-known-good tier that outlives the entry's TTL (default 30
+ * minutes, `staleMs`). When a later `load()` throws, the caller gets that
+ * last-good value instead of the rejection. Before this, eleven DefiLlama
+ * handlers and the Fear & Greed endpoint went straight from "upstream blipped"
+ * to a 502 while a sixty-second-old payload sat in the same process. Pass
+ * `staleMs: 0` to opt out where an error must surface (a write, a probe).
+ *
* @template V
* @param {LRUCache<string, V>} cache
* @param {string} key
* @param {() => Promise<V>} load
+ * @param {{ staleMs?: number, onStale?: (err: unknown) => void }} [opts]
* @returns {Promise<V>}
*/
-export function cached(cache, key, load) {
+export function cached(cache, key, load, opts = {}) {
+ const { staleMs = DEFAULT_STALE_MS, onStale } = opts;
const hit = cache.get(key);
if (hit !== undefined) return Promise.resolve(hit);
@@ -71,8 +81,17 @@ export function cached(cache, key, load) {
const p = (async () => {
try {
const value = await load();
- if (value !== undefined) cache.set(key, value);
+ if (value !== undefined) {
+ cache.set(key, value);
+ if (staleMs > 0) lastGoodTier(cache, staleMs).set(key, value, { ttl: staleMs });
+ }
return value;
+ } catch (err) {
+ const stale = staleMs > 0 ? lastGoodTier(cache, staleMs).get(key) : undefined;
+ if (stale === undefined) throw err;
+ if (onStale) onStale(err);
+ else warnStale(key, err);
+ return stale;
} finally {
inflight.delete(key);
}
@@ -81,5 +100,35 @@ export function cached(cache, key, load) {
return p;
}
-// Per-cache in-flight maps, keyed weakly so a discarded cache is collectable.
+/** True when `cached()` has a last-good copy of `key` it could serve on error. */
+export function hasLastGood(cache, key) {
+ const tier = _lastGood.get(cache);
+ return tier ? tier.get(key) !== undefined : false;
+}
+
+const DEFAULT_STALE_MS = 30 * 60_000;
+
+// One last-good tier per cache, sized like its parent so it cannot outgrow it.
+function lastGoodTier(cache, staleMs) {
+ let tier = _lastGood.get(cache);
+ if (!tier) {
+ tier = new LRUCache({ max: cache.max || 512, ttl: staleMs, perf: { now: () => Date.now() }, ttlResolution: 0 });
+ _lastGood.set(cache, tier);
+ }
+ return tier;
+}
+
+// One warning per key per minute: an outage should be visible in the logs,
+// not fill them.
+const _warnedAt = new Map();
+function warnStale(key, err) {
+ const now = Date.now();
+ if ((_warnedAt.get(key) || 0) > now - 60_000) return;
+ _warnedAt.set(key, now);
+ console.warn(`[mem-cache] load failed for "${key}" (${err?.message || err}); serving last-good`);
+}
+
+// Per-cache in-flight maps and last-good tiers, keyed weakly so a discarded
+// cache is collectable.
const _inflight = new WeakMap();
+const _lastGood = new WeakMap();
diff --git a/data/changelog.json b/data/changelog.json
index a8ec0f26a..5a58ce158 100644
--- a/data/changelog.json
+++ b/data/changelog.json
@@ -1,6 +1,16 @@
{
"$comment": "Curated changelog entries for three.ws, the editorial layer of the public changelog. New PAGE launches are tracked automatically via the `added` field in data/pages.json; this file is for everything else holders care about: improvements, fixes, SDK releases, trading/payments capabilities, security work. scripts/build-page-index.mjs merges both sources into CHANGELOG.md, public/changelog.json, and public/changelog.xml. Newest first. Allowed tags: feature, improvement, fix, sdk, infra, docs, security. `link` is optional and must be a live three.ws path.",
"entries": [
+ {
+ "date": "2026-08-27",
+ "title": "Every outside data source now has a backup",
+ "summary": "Prices, DeFi stats, token data, pump.fun feeds, RPC reads and 3D generation providers all had single points of failure: one rate limit or outage upstream turned into a blank section or an error page. Every one of those reads now has a deadline, retries only on transient faults, a second and third source where one exists, and keeps serving the last good value (marked as stale) when everything upstream is down. In the browser, API calls time out instead of spinning forever, script and model loads try mirror CDNs, and payment flows report a failed confirmation with the transaction signature instead of showing nothing.",
+ "tags": [
+ "improvement",
+ "fix",
+ "infra"
+ ]
+ },
{
"date": "2026-08-27",
"title": "Every tap target on the marketplace and your agents pages now fits a thumb",
diff --git a/docs/shared-utilities.md b/docs/shared-utilities.md
index ee173f19e..b7f08103a 100644
--- a/docs/shared-utilities.md
+++ b/docs/shared-utilities.md
@@ -124,6 +124,42 @@ a 400 or a `TypeError` is not retried. Pass `shouldRetry` to override.
hand-rolled `sleep(base * 2 ** attempt)` loops had no jitter at all, so a shared
upstream blip re-synchronized every caller into the same retry wave.
+### `api/_lib/upstream-fetch.js` — the one fetch for third-party calls
+
+```js
+import { fetchUpstream, fetchUpstreamJson, fetchAnyJson, lastGood } from './_lib/upstream-fetch.js';
+
+// Deadline (8s default), transient-only retries, Retry-After honoured, breaker.
+const body = await fetchUpstreamJson(url, {}, { timeoutMs: 6000, name: 'dexscreener' });
+
+// Equivalent mirrors, first one that answers wins; a dead mirror is benched.
+const { value } = await fetchAnyJson(['https://api.drand.sh/public/latest', 'https://api2.drand.sh/public/latest']);
+
+// Keep the page alive through a blip: serve the previous value, flagged stale.
+const { value: rows, stale } = await lastGood('llama:chains', () => fetchUpstreamJson(LLAMA));
+```
+
+A non-2xx response rejects with an `UpstreamError` carrying `status`, `url` and
+a body excerpt, so `isRetryableError` and handlers can branch on it. Solana and
+EVM RPC do not go through here: `api/_lib/solana/connection.js` and
+`api/_lib/evm/rpc.js` have method-aware rotation. Ordered lists of *different*
+providers (with per-provider parsers) use `fetchFirst` from
+`src/shared/failover-fetch.js` instead.
+
+### `api/_lib/cache.js` — `cacheWrapLastGood`
+
+```js
+import { cacheWrapLastGood } from './_lib/cache.js';
+
+// Fresh for 5 minutes; on a thrown refresh, the fleet-wide last-good copy (24h).
+const board = await cacheWrapLastGood('pump:board', 300, () => fetchBoard());
+const { value, stale } = await cacheWrapLastGood(key, 300, load, { withMeta: true });
+```
+
+Unlike `lastGood` above, the mirror lives in the shared store (Upstash), so a
+freshly started instance can ride out an outage on a value another instance
+fetched.
+
### `api/_lib/mem-cache.js` — bounded in-process cache
Backed by [`lru-cache`](https://www.npmjs.com/package/lru-cache).
@@ -134,6 +170,8 @@ import { createCache, cached } from './_lib/mem-cache.js';
const cache = createCache({ max: 512, ttlMs: 60_000 });
// Read-through with single-flight: concurrent misses share one load().
+// When load() throws, the last value it returned for this key (kept 30 min,
+// `staleMs`) is served instead; pass `staleMs: 0` where the error must surface.
const price = await cached(cache, mint, () => fetchPrice(mint));
// Per-entry TTL when lifetimes vary by key:
diff --git a/src/api.js b/src/api.js
index b38eec401..3bc5f5561 100644
--- a/src/api.js
+++ b/src/api.js
@@ -6,15 +6,40 @@
// • transient 5xx retry on safe methods
// • 401 → /login?next=… redirect
//
-// Every page that talks to /api MUST import apiFetch from here. The ESLint
-// rule no-restricted-syntax in .eslintrc blocks raw `fetch(`${API_BASE}…`)` to
-// prevent the kind of drift that left agent-edit.js without CSRF and silently
-// 403'ing every save until a user complained.
+// • a per-attempt deadline (timeoutMs: 20s reads, 90s mutations) so a hung edge worker
+// never leaves a spinner up forever
+//
+// Every page that talks to /api should import apiFetch from here, so it gets
+// CSRF, retry, the deadline and the 401 redirect for free. Raw `fetch('/api…')`
+// calls get none of that; the drift that left agent-edit.js without CSRF and
+// silently 403'ing every save is exactly what this module exists to prevent.
import { trackError, track, ANALYTICS_EVENTS } from './analytics.js';
const SAFE_METHODS = new Set(['GET', 'HEAD', 'OPTIONS']);
const TRANSIENT_STATUSES = new Set([502, 503, 504]);
+// Reads get a tight deadline; mutations (uploads, job submits) get a long one
+// so a slow link never fails a 50 MB avatar upload that is still making progress.
+const DEFAULT_TIMEOUT_MS = 20_000;
+const DEFAULT_MUTATION_TIMEOUT_MS = 90_000;
+
+// Per-attempt deadline composed with whatever signal the caller passed, so a
+// caller's own abort (stale search, navigation) still wins immediately.
+function attemptSignal(callerSignal, timeoutMs) {
+ if (!(timeoutMs > 0) || typeof AbortSignal === 'undefined' || typeof AbortSignal.timeout !== 'function') {
+ return callerSignal;
+ }
+ const timeout = AbortSignal.timeout(timeoutMs);
+ if (!callerSignal) return timeout;
+ if (typeof AbortSignal.any === 'function') return AbortSignal.any([callerSignal, timeout]);
+ const ctrl = new AbortController();
+ const forward = (sig) => () => { if (!ctrl.signal.aborted) ctrl.abort(sig.reason); };
+ for (const sig of [callerSignal, timeout]) {
+ if (sig.aborted) { ctrl.abort(sig.reason); break; }
+ sig.addEventListener('abort', forward(sig), { once: true });
+ }
+ return ctrl.signal;
+}
// Reduce an API path to a low-cardinality route for analytics: drop the query
// string (may carry tokens) and collapse UUIDs / long ids to ':id' so the
@@ -99,10 +124,12 @@ function redirectToLogin() {
// Drop-in fetch replacement. Pass allowAnonymous:true on endpoints where a
// 401 is a legitimate answer the caller wants to inspect itself (e.g.
-// /api/auth/me on first paint).
+// /api/auth/me on first paint). Pass timeoutMs to change the per-attempt
+// deadline (0 disables it, for long-running job submits that stream).
export async function apiFetch(path, options = {}) {
const { allowAnonymous = false, ...init } = options;
const method = (init.method || 'GET').toUpperCase();
+ const timeoutMs = options.timeoutMs ?? (SAFE_METHODS.has(method) ? DEFAULT_TIMEOUT_MS : DEFAULT_MUTATION_TIMEOUT_MS);
const canRetry = SAFE_METHODS.has(method);
const headers = new Headers(init.headers || {});
@@ -124,6 +151,7 @@ export async function apiFetch(path, options = {}) {
credentials: 'include',
...init,
headers,
+ signal: attemptSignal(init.signal, timeoutMs),
});
let res;
@@ -138,6 +166,10 @@ export async function apiFetch(path, options = {}) {
} catch (networkErr) {
lastErr = networkErr;
res = undefined;
+ // The caller gave up (its own signal fired): stop, do not retry on
+ // its behalf. A deadline we imposed is transient and retries like a
+ // dropped connection.
+ if (init.signal?.aborted) break;
continue;
}
if (canRetry && TRANSIENT_STATUSES.has(res.status)) continue;
diff --git a/tests/mem-cache-stale.test.js b/tests/mem-cache-stale.test.js
new file mode 100644
index 000000000..78c380692
--- /dev/null
+++ b/tests/mem-cache-stale.test.js
@@ -0,0 +1,26 @@
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { createCache, cached, hasLastGood } from '../api/_lib/mem-cache.js';
+
+const json = (body, status = 200, headers = {}) =>
+ new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json', ...headers } });
+
+describe('cached: stale-on-error', () => {
+ it('serves the last-good value when a refresh throws, after the TTL expired', async () => {
+ const cache = createCache({ max: 4, ttlMs: 5 });
+ expect(await cached(cache, 'k', async () => ({ v: 1 }))).toEqual({ v: 1 });
+ expect(hasLastGood(cache, 'k')).toBe(true);
+ await new Promise((r) => setTimeout(r, 15));
+ expect(cache.get('k')).toBeUndefined();
+ const onStale = vi.fn();
+ expect(await cached(cache, 'k', async () => { throw new Error('upstream 503'); }, { onStale })).toEqual({ v: 1 });
+ expect(onStale).toHaveBeenCalledTimes(1);
+ });
+
+ it('still rejects when there is nothing last-good, or when opted out', async () => {
+ const cache = createCache({ max: 4, ttlMs: 5 });
+ await expect(cached(cache, 'x', async () => { throw new Error('cold'); })).rejects.toThrow('cold');
+ await cached(cache, 'y', async () => 'good');
+ await new Promise((r) => setTimeout(r, 15));
+ await expect(cached(cache, 'y', async () => { throw new Error('opted out'); }, { staleMs: 0 })).rejects.toThrow('opted out');
+ });
+});