Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(resilience): serve last-good reads through RPC and upstream outages, race CDN module loads across three hosts, and type every /api JSON failure

nirholas535a91d81bb3
diff --git a/api/_lib/agora-human.js b/api/_lib/agora-human.js
index fe065b4b1..a1ffbe9a1 100644
--- a/api/_lib/agora-human.js
+++ b/api/_lib/agora-human.js
@@ -515,10 +515,15 @@ const MEMO_PROGRAM_ID = 'MemoSq4gqABAXKb96qnH8TysNcWxMyWCqXgDLGmfcHr';
* agora_vouches edge and the 'vouched' activity row cite. Returns the tx sig.
*/
export async function sendOnchainAttestation({ cluster, signer, memo }) {
- const { Connection, Transaction, TransactionInstruction, PublicKey } =
- await import('@solana/web3.js');
- const rpc = pickRpc(cluster) || (cluster === 'devnet' ? 'https://api.devnet.solana.com' : 'https://api.mainnet-beta.solana.com');
- const conn = new Connection(rpc, 'confirmed');
+ const { Transaction, TransactionInstruction, PublicKey } = await import('@solana/web3.js');
+ const { solanaConnection } = await import('./solana/connection.js');
+ // The cluster's configured RPC (AGENC_RPC_URL / SOLANA_RPC_URL*) stays primary;
+ // the shared failover chain for that network rotates in behind it.
+ const conn = solanaConnection({
+ url: pickRpc(cluster) || null,
+ network: cluster === 'devnet' ? 'devnet' : 'mainnet',
+ commitment: 'confirmed',
+ });
if (cluster === 'devnet') await ensureDevnetBalance(conn, signer, 5_000_000);
const ix = new TransactionInstruction({
keys: [{ pubkey: signer.publicKey, isSigner: true, isWritable: false }],
diff --git a/api/_lib/alert-delivery.js b/api/_lib/alert-delivery.js
index 1c37d8d2a..c1bbba32d 100644
--- a/api/_lib/alert-delivery.js
+++ b/api/_lib/alert-delivery.js
@@ -16,6 +16,7 @@ import { hmacSha256, randomToken } from './crypto.js';
import { validatePublicUrl, resolvePublicHost, pinnedAgent, SsrfError } from './ssrf.js';
import { formatAlertSummary } from './pump-alert-eval.js';
+import { fetchUpstream } from './upstream-fetch.js';
const WEBHOOK_TIMEOUT_MS = 8_000;
const TELEGRAM_TIMEOUT_MS = 5_000;
@@ -92,7 +93,7 @@ async function deliverWebhook(rule, payload) {
const sig = await hmacSha256(rule.webhook_secret, `${eventId}.${timestamp}.${body}`);
headers['webhook-signature'] = `v1,${sig}`;
}
- const res = await fetch(target, {
+ const res = await fetchUpstream(target, {
method: 'POST',
redirect: 'manual',
// @ts-ignore — undici dispatcher option, pins the resolved address.
@@ -100,7 +101,7 @@ async function deliverWebhook(rule, payload) {
headers,
body,
signal: controller.signal,
- });
+ }, { timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (res.status >= 300 && res.status < 400) {
return { attempted: true, ok: false, detail: `redirect_not_followed:${res.status}` };
}
@@ -125,7 +126,7 @@ async function deliverTelegram(rule, payload) {
const controller = new AbortController();
const timer = setTimeout(() => controller.abort(), TELEGRAM_TIMEOUT_MS);
try {
- const res = await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+ const res = await fetchUpstream(`https://api.telegram.org/bot${token}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -134,7 +135,7 @@ async function deliverTelegram(rule, payload) {
disable_web_page_preview: true,
}),
signal: controller.signal,
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const data = await res.json().catch(() => ({}));
if (!res.ok || data?.ok === false) {
return { attempted: true, ok: false, detail: data?.description || `http_${res.status}` };
diff --git a/api/_lib/autopilot.js b/api/_lib/autopilot.js
index 1aa6e9cec..0aceb5aed 100644
--- a/api/_lib/autopilot.js
+++ b/api/_lib/autopilot.js
@@ -600,9 +600,11 @@ export async function dailySolSpent(agentId) {
/** Live native SOL balance (in SOL) for an address. Best-effort RPC read. */
async function agentSolBalance(address) {
- const { Connection, PublicKey } = await import('@solana/web3.js');
- const url = process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com';
- const conn = new Connection(url, 'confirmed');
+ const { PublicKey } = await import('@solana/web3.js');
+ const { solanaConnection } = await import('./solana/connection.js');
+ // The operator's SOLANA_RPC_URL stays primary; the shared keyed/public lanes
+ // rotate in behind it so one provider's outage never blanks a balance check.
+ const conn = solanaConnection({ url: process.env.SOLANA_RPC_URL || null, network: 'mainnet', commitment: 'confirmed' });
const lamports = await conn.getBalance(new PublicKey(address), 'confirmed');
return lamports / LAMPORTS_PER_SOL;
}
diff --git a/api/_lib/avatar-capabilities.js b/api/_lib/avatar-capabilities.js
index 1a9da6dc8..b6cab5d68 100644
--- a/api/_lib/avatar-capabilities.js
+++ b/api/_lib/avatar-capabilities.js
@@ -29,6 +29,7 @@ import { inspectGlb, glbJsonChunkEnd } from './glb-inspect.js';
import { conformanceFromNames, ARKIT_52 } from '../../src/runtime/arkit52.js';
import { canonicalizeBoneName, CANONICAL_BONES } from '../../src/glb-canonicalize.js';
import { MIN_COVERAGE } from '../../src/animation-retarget.js';
+import { fetchUpstream } from './upstream-fetch.js';
// 256 KB covers the JSON chunk of essentially every real avatar in one read.
// Same first-pass budget as rig-inspect.js, for the same reason.
@@ -72,7 +73,7 @@ const RIG_SIGNATURES = Object.freeze([
async function readGlbPrefix(storageKey, length) {
if (/^https?:\/\//i.test(storageKey)) {
- const r = await fetch(storageKey, { headers: { Range: `bytes=0-${length - 1}` } });
+ const r = await fetchUpstream(storageKey, { headers: { Range: `bytes=0-${length - 1}` } }, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!r.ok) throw new Error(`http ${r.status}`);
return Buffer.from(await r.arrayBuffer());
}
diff --git a/api/_lib/aws-marketplace.js b/api/_lib/aws-marketplace.js
index d3b6a7861..00e689b3b 100644
--- a/api/_lib/aws-marketplace.js
+++ b/api/_lib/aws-marketplace.js
@@ -29,6 +29,7 @@ import {
} from '@aws-sdk/client-marketplace-entitlement-service';
import { createVerify, createPublicKey, timingSafeEqual } from 'node:crypto';
import { env } from './env.js';
+import { fetchUpstream } from './upstream-fetch.js';
function credentials() {
return {
@@ -248,7 +249,8 @@ async function fetchCert(url) {
// Only trust certs hosted on *.amazonaws.com over HTTPS.
assertAwsHttpsUrl(url, 'SNS signing cert URL');
- const res = await fetch(url);
+ // The PEM is cached per URL above, so this is paid once per cert rotation.
+ const res = await fetchUpstream(url, {}, { name: 'aws-sns-cert', timeoutMs: 8_000, attempts: 3, okWhen: () => true });
if (!res.ok) throw new Error(`Failed to fetch SNS cert: ${res.status}`);
const pem = await res.text();
certCache.set(url, pem);
diff --git a/api/_lib/bake-garments.js b/api/_lib/bake-garments.js
index 16280e1b1..4e1859cd8 100644
--- a/api/_lib/bake-garments.js
+++ b/api/_lib/bake-garments.js
@@ -30,6 +30,7 @@ import {
} from '../../src/garment-taxonomy.js';
import { sanitizeCatalog } from '../../src/garment-catalog.js';
+import { fetchUpstream } from './upstream-fetch.js';
export const GARMENT_CATALOG_URL =
'https://storage.googleapis.com/three-ws-garments/garments/catalog.json';
@@ -40,7 +41,7 @@ const CATALOG_TTL_MS = 5 * 60_000;
async function loadServerCatalog() {
if (_catalog && _catalog.expiresAt > Date.now()) return _catalog.garments;
- const res = await fetch(GARMENT_CATALOG_URL, { headers: { accept: 'application/json' } });
+ const res = await fetchUpstream(GARMENT_CATALOG_URL, { headers: { accept: 'application/json' } }, { name: 'gcs-garments', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
if (!res.ok) throw new Error(`garment catalog fetch failed: ${res.status}`);
const { garments } = sanitizeCatalog(await res.json());
_catalog = { garments, expiresAt: Date.now() + CATALOG_TTL_MS };
@@ -393,7 +394,7 @@ export async function applyGarments(io, doc, garmentRefs, mergeDocumentsFn, opts
}
async function fetchGarmentBytes(manifest) {
- const res = await fetch(manifest.model.uri);
+ const res = await fetchUpstream(manifest.model.uri, {}, { timeoutMs: 60_000, attempts: 2, okWhen: () => true });
if (!res.ok) throw new Error(`garment download failed: ${res.status}`);
const bytes = await res.arrayBuffer();
const hex = createHash('sha256').update(Buffer.from(bytes)).digest('hex');
diff --git a/api/_lib/bake.js b/api/_lib/bake.js
index ff80f76ed..e4cc5ee83 100644
--- a/api/_lib/bake.js
+++ b/api/_lib/bake.js
@@ -33,6 +33,7 @@ import { createHash } from 'node:crypto';
import { isValidPresetId } from './accessories.js';
import { env } from './env.js';
import { getObjectBuffer, putObject } from './r2.js';
+import { fetchUpstream } from './upstream-fetch.js';
// ── Public API ─────────────────────────────────────────────────────────────
@@ -496,7 +497,7 @@ async function loadAccessoryGlb(glbUrl) {
// On Vercel functions, public/ assets may not be bundled with the function.
// Fall back to fetching from the deployed site over HTTPS.
const base = env.APP_ORIGIN || 'https://three.ws';
- const r = await fetch(base + glbUrl);
+ const r = await fetchUpstream(base + glbUrl, {}, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!r.ok) {
throw new Error(`accessory fetch failed: ${base + glbUrl} → ${r.status}`);
}
diff --git a/api/_lib/birdeye.js b/api/_lib/birdeye.js
index 54e423c12..d03b45667 100644
--- a/api/_lib/birdeye.js
+++ b/api/_lib/birdeye.js
@@ -1,18 +1,54 @@
// Shared Birdeye OHLCV fetch. Keeps BIRDEYE_API_KEY server-side and returns
// parsed candles [{ t, o, h, l, c, v }] (t = unix seconds), ascending by time.
// Throws an Error tagged with { status } so callers map to the right HTTP code.
-// Never fabricates candles — upstream failures surface verbatim.
+// Never fabricates candles: upstream failures surface verbatim, and only when
+// every rung failed and nothing is cached.
+//
+// Rungs, in order:
+// 1. Birdeye token OHLCV (keyed)
+// 2. GeckoTerminal: top pool for the mint, then that pool's candles
+// (api/_lib/market/ohlcv.js, which itself carries Birdeye-pair and CEX
+// rungs behind GeckoTerminal)
+// 3. The last good answer for this mint + interval + window, from the shared
+// cache, when both live sources are down.
+
+import { cacheWrapLastGood } from './cache.js';
+import { fetchUpstream } from './upstream-fetch.js';
+import { topPoolForToken, fetchOhlcv, barSeconds } from './market/ohlcv.js';
const BIRDEYE_OHLCV_URL = 'https://public-api.birdeye.so/defi/ohlcv';
+// Fresh window for the shared cache: a chart poll inside it reuses the candles
+// verbatim. Stale window: how long a last-good series is servable in an outage.
+const FRESH_TTL_S = 15;
+const STALE_TTL_S = 6 * 3600;
+const GECKO_MAX_BARS = 1000;
+
export function birdeyeConfigured() {
return !!process.env.BIRDEYE_API_KEY;
}
-export async function fetchBirdeyeOhlcv({ mint, interval, from, to }) {
+// Birdeye `type` -> the closest GeckoTerminal timeframe/aggregate pair. Birdeye
+// offers finer steps than GeckoTerminal (3m, 30m, 2H...); those map to the
+// nearest cadence GeckoTerminal serves, and the caller's [from, to] window is
+// still honoured by filtering.
+export function geckoFrameFor(interval) {
+ const map = {
+ '1m': ['minute', 1], '3m': ['minute', 5], '5m': ['minute', 5], '15m': ['minute', 15], '30m': ['minute', 15],
+ '1H': ['hour', 1], '2H': ['hour', 1], '4H': ['hour', 4], '6H': ['hour', 4], '8H': ['hour', 4], '12H': ['hour', 12],
+ '1D': ['day', 1], '3D': ['day', 1], '1W': ['day', 1], '1M': ['day', 1],
+ };
+ const [timeframe, aggregate] = map[String(interval)] || ['hour', 1];
+ return { timeframe, aggregate };
+}
+
+function tag(message, status) {
+ return Object.assign(new Error(message), { status });
+}
+
+async function fromBirdeye({ mint, interval, from, to }) {
const apiKey = process.env.BIRDEYE_API_KEY;
- if (!apiKey)
- throw Object.assign(new Error('On-chain data provider is not configured'), { status: 503 });
+ if (!apiKey) throw tag('On-chain data provider is not configured', 503);
const url =
`${BIRDEYE_OHLCV_URL}?address=${encodeURIComponent(mint)}` +
@@ -20,25 +56,18 @@ export async function fetchBirdeyeOhlcv({ mint, interval, from, to }) {
let upstream;
try {
- upstream = await fetch(url, {
- headers: { 'X-API-KEY': apiKey, 'x-chain': 'solana', accept: 'application/json' },
- signal: AbortSignal.timeout(10_000),
- });
+ upstream = await fetchUpstream(
+ url,
+ { headers: { 'X-API-KEY': apiKey, 'x-chain': 'solana', accept: 'application/json' } },
+ { timeoutMs: 10_000, attempts: 2, name: 'birdeye:ohlcv', label: 'Birdeye OHLCV' },
+ );
} catch (e) {
- throw Object.assign(new Error(`Birdeye unreachable: ${e.message}`), { status: 502 });
- }
-
- if (!upstream.ok) {
- const body = await upstream.text().catch(() => '');
- throw Object.assign(new Error(`Birdeye ${upstream.status}: ${body.slice(0, 200)}`), {
- status: 502,
- });
+ throw tag(`Birdeye ${e?.status ? `${e.status}: ${String(e.body || '').slice(0, 200)}` : `unreachable: ${e?.message || e}`}`, 502);
}
const payload = await upstream.json().catch(() => null);
const items = payload?.data?.items;
- if (!Array.isArray(items))
- throw Object.assign(new Error('Birdeye returned an unexpected payload'), { status: 502 });
+ if (!Array.isArray(items)) throw tag('Birdeye returned an unexpected payload', 502);
return items
.map((it) => ({
@@ -51,3 +80,53 @@ export async function fetchBirdeyeOhlcv({ mint, interval, from, to }) {
}))
.filter((d) => Number.isFinite(d.t) && Number.isFinite(d.c));
}
+
+async function fromGeckoTerminal({ mint, interval, from, to }) {
+ const { timeframe, aggregate } = geckoFrameFor(interval);
+ const bars = Math.ceil(Math.max(0, to - from) / barSeconds(timeframe, aggregate)) + 2;
+ const pool = await topPoolForToken(mint, 'solana');
+ const { candles } = await fetchOhlcv({
+ pool,
+ network: 'solana',
+ timeframe,
+ aggregate,
+ limit: Math.min(GECKO_MAX_BARS, Math.max(2, bars)),
+ });
+ const inWindow = candles.filter((d) => d.t >= from && d.t <= to);
+ if (!inWindow.length) throw tag('GeckoTerminal has no candles in this window', 502);
+ return inWindow;
+}
+
+async function fetchLive(args) {
+ let firstErr = null;
+ try {
+ return await fromBirdeye(args);
+ } catch (e) {
+ firstErr = e;
+ }
+ try {
+ return await fromGeckoTerminal(args);
+ } catch (e) {
+ // A pool GeckoTerminal never indexed is not an outage; keep Birdeye's
+ // verdict (503 unconfigured / 502 upstream) as the one the route maps.
+ if (firstErr.status === 503 && e?.status !== 404) firstErr = tag(`GeckoTerminal: ${e?.message || e}`, 502);
+ }
+ throw firstErr;
+}
+
+/**
+ * Candles for a mint over [from, to] at a Birdeye interval ('1m', '15m', '1H',
+ * '1D', ...). Birdeye first, GeckoTerminal second, last good series third.
+ *
+ * @param {{ mint: string, interval: string, from: number, to: number }} args
+ * @returns {Promise<Array<{ t: number, o: number, h: number, l: number, c: number, v: number }>>}
+ */
+export async function fetchBirdeyeOhlcv({ mint, interval, from, to }) {
+ const span = Math.max(0, Number(to) - Number(from));
+ return cacheWrapLastGood(
+ `birdeye:ohlcv:v1:${mint}:${interval}:${span}`,
+ FRESH_TTL_S,
+ () => fetchLive({ mint, interval, from, to }),
+ { staleTtlSeconds: STALE_TTL_S },
+ );
+}
diff --git a/api/_lib/brain-anchor.js b/api/_lib/brain-anchor.js
index 5c4a91522..48843f569 100644
--- a/api/_lib/brain-anchor.js
+++ b/api/_lib/brain-anchor.js
@@ -22,7 +22,7 @@ import { Contract, Wallet, keccak256, toUtf8Bytes } from 'ethers';
import { env } from './env.js';
import { sql } from './db.js';
import { CHAIN_BY_ID, VALIDATION_REGISTRY_ABI, validationRegistryFor } from './erc8004-chains.js';
-import { evmRpcEndpoints } from './evm/rpc.js';
+import { evmFallbackProvider } from './evm/rpc.js';
import { explorerLink } from './onchain.js';
import { putObject, publicUrl } from './r2.js';
import { computeBrainHash } from './brain-bundle.js';
@@ -207,11 +207,9 @@ export async function anchorBrain({ agentId, anchoredAt, publicOnly = false }) {
}
const proofHash = keccak256(toUtf8Bytes(JSON.stringify(passport)));
- // Provider + validator wallet + registry.
- const { JsonRpcProvider, Network } = await import('ethers');
- const network = Network.from(chainId);
- const endpoints = evmRpcEndpoints(chainId);
- const provider = new JsonRpcProvider(endpoints[0], network, { staticNetwork: network });
+ // Provider + validator wallet + registry. Quorum-1 failover across the
+ // chain's endpoints, so a single dead RPC never blocks an anchor.
+ const provider = await evmFallbackProvider(chainId);
const wallet = new Wallet(pk, provider);
const registry = new Contract(registryAddr, VALIDATION_REGISTRY_ABI, wallet);
diff --git a/api/_lib/changelog-push.js b/api/_lib/changelog-push.js
index 27cc73dd7..c22d9527d 100644
--- a/api/_lib/changelog-push.js
+++ b/api/_lib/changelog-push.js
@@ -27,6 +27,7 @@ import { readFileSync } from 'node:fs';
import { join } from 'node:path';
import { sql } from './db.js';
+import { fetchUpstream } from './upstream-fetch.js';
const CUTOFF_DAYS = 3;
const TELEGRAM_LIMIT = 8; // per run; Bot API allows ~20 msg/min per chat
const TELEGRAM_PACE_MS = 3500;
@@ -146,7 +147,7 @@ export function formatTelegramMessage(e) {
}
async function sendTelegram(botToken, chatId, text) {
- const res = await fetch(`https://api.telegram.org/bot${botToken}/sendMessage`, {
+ const res = await fetchUpstream(`https://api.telegram.org/bot${botToken}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -156,7 +157,7 @@ async function sendTelegram(botToken, chatId, text) {
link_preview_options: { is_disabled: false, prefer_small_media: true },
}),
signal: AbortSignal.timeout(TELEGRAM_TIMEOUT_MS),
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const body = await res.json().catch(() => ({}));
if (!res.ok || !body.ok) {
throw new Error(`Telegram sendMessage failed (${res.status}): ${body.description || 'unknown error'}`);
diff --git a/api/_lib/coin/randomness.js b/api/_lib/coin/randomness.js
index ea2b8da60..1f1704805 100644
--- a/api/_lib/coin/randomness.js
+++ b/api/_lib/coin/randomness.js
@@ -20,8 +20,9 @@
// audit after the fact even if we didn't verify in-process.
import { sha256 } from '@noble/hashes/sha256';
+import { fetchUpstream } from '../upstream-fetch.js';
-const QUICKNET_CHAIN_HASH = '52db9ba70e0cc0f6eaf7803dd07447a1f5477735fd3f661792ba94600c84e971';
+export const QUICKNET_CHAIN_HASH = '52db9ba70e0cc0f6eaf7803dd07447a1f5477735fd3f661792ba94600c84e971';
const QUICKNET_GENESIS = 1692803367;
const QUICKNET_PERIOD = 3;
const QUICKNET_PUBLIC_KEY_HEX =
@@ -29,7 +30,15 @@ const QUICKNET_PUBLIC_KEY_HEX =
'8c4b450b6a0a6c3ac6a5776a2d1064510d1fec758c921cc22b0e17e63aaf4bcb' +
'5ed66304de9cf809bd274ca73bab4af5a6e9c76a4bc09e76eae8991ef5ece45a';
-const DRAND_BASE = `https://api.drand.sh/${QUICKNET_CHAIN_HASH}/public`;
+// Public drand relays, all serving quicknet. api.drand.sh is the League of
+// Entropy's primary; the others are independent mirrors of the same beacon.
+export const DRAND_RELAYS = [
+ 'https://api.drand.sh',
+ 'https://api2.drand.sh',
+ 'https://api3.drand.sh',
+ 'https://drand.cloudflare.com',
+];
+const DRAND_BASE = `${DRAND_RELAYS[0]}/${QUICKNET_CHAIN_HASH}/public`;
/**
* Compute the Drand round number that will be live at or after the given unix
@@ -79,15 +88,34 @@ export function drandRoundMessage(round) {
* @returns {Promise<{ round: number, randomness: string, signature: string }>}
*/
export async function fetchDrandRound(round) {
- const url = `${DRAND_BASE}/${round}`;
- const resp = await fetch(url, { headers: { accept: 'application/json' } });
- if (resp.status === 404) {
- throw new Error(`drand_round_unavailable:${round}`);
+ // Every relay serves the same signed beacon, so any one of them is
+ // authoritative; the signature check below is what we trust, not the host.
+ // A 404 is a real answer (the round has not been published yet) and must
+ // not be retried on the next relay, so it short-circuits the chain.
+ let data = null;
+ let lastStatus = 0;
+ for (const base of DRAND_RELAYS) {
+ let resp;
+ try {
+ resp = await fetchUpstream(`${base}/${QUICKNET_CHAIN_HASH}/public/${round}`, { headers: { accept: 'application/json' } }, {
+ name: `drand:${base}`,
+ timeoutMs: 5_000,
+ attempts: 2,
+ okWhen: (r) => r.ok || r.status === 404,
+ });
+ } catch (err) {
+ lastStatus = err?.status || 0;
+ continue;
+ }
+ if (resp.status === 404) {
+ throw new Error(`drand_round_unavailable:${round}`);
+ }
+ data = await resp.json().catch(() => null);
+ if (data) break;
}
- if (!resp.ok) {
- throw new Error(`drand_fetch_failed:${resp.status}`);
+ if (!data) {
+ throw new Error(`drand_fetch_failed:${lastStatus || 'unreachable'}`);
}
- const data = await resp.json();
if (!data || typeof data.round !== 'number' || !data.randomness || !data.signature) {
throw new Error('drand_response_malformed');
}
diff --git a/api/_lib/commit-feed-push.js b/api/_lib/commit-feed-push.js
index fb208dbd7..3da5b4420 100644
--- a/api/_lib/commit-feed-push.js
+++ b/api/_lib/commit-feed-push.js
@@ -34,6 +34,7 @@
import { sql } from './db.js';
+import { fetchUpstream } from './upstream-fetch.js';
const REPO = 'nirholas/three.ws';
const BASE = 'https://three.ws';
const TELEGRAM_LIMIT = 15; // per run; Bot API allows ~20 msg/min per chat
@@ -256,7 +257,7 @@ export function formatTelegramMessage(commit) {
}
async function sendTelegram(botToken, chatId, text, previewUrl) {
- const res = await fetch(`https://api.telegram.org/bot${botToken}/sendMessage`, {
+ const res = await fetchUpstream(`https://api.telegram.org/bot${botToken}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -268,7 +269,7 @@ async function sendTelegram(botToken, chatId, text, previewUrl) {
link_preview_options: { is_disabled: false, url: previewUrl, show_above_text: false },
}),
signal: AbortSignal.timeout(TELEGRAM_TIMEOUT_MS),
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const body = await res.json().catch(() => ({}));
if (!res.ok || !body.ok) {
throw new Error(`Telegram sendMessage failed (${res.status}): ${body.description || 'unknown error'}`);
diff --git a/api/_lib/crypto-token-holders.js b/api/_lib/crypto-token-holders.js
index 3e2e9bc5c..591dbf4ee 100644
--- a/api/_lib/crypto-token-holders.js
+++ b/api/_lib/crypto-token-holders.js
@@ -20,6 +20,7 @@
import { parseMintAccount } from '../v1/token/security.js';
import { solanaRpcEndpoints, makeRotatingFetch } from './solana/connection.js';
+import { withDeadline } from './rpc-degrade.js';
// Budget for ONE JSON-RPC read across the WHOLE failover chain, not per
// endpoint. makeRotatingFetch bounds each attempt itself (10s) and composes the
@@ -30,6 +31,13 @@ import { solanaRpcEndpoints, makeRotatingFetch } from './solana/connection.js';
// to cover a couple of real attempts, matching the wallet-activity lane.
const RPC_TIMEOUT_MS = 25_000;
+// Per-call wall clock for the keyless fan-out (largest accounts, then owner
+// resolution). Each read is bounded by RPC_TIMEOUT_MS, but two of them back to
+// back after a slow Helius walk could still outlive the gateway; this cap makes
+// the caller's upstream_down (and its last-good tier) fire while the response
+// can still be delivered.
+const KEYLESS_CALL_DEADLINE_MS = 20_000;
+
// Helius DAS getTokenAccounts pages at up to 1000 accounts; cap the walk so a
// mega-token (millions of accounts) can't burn the invocation. Within the cap
// the holder count is exact; beyond it we report null + a note, never a guess.
@@ -282,7 +290,7 @@ export async function composeTokenHolders({ address, limit = DEFAULT_LIMIT }, de
let largest = null;
let largestAnswered = false;
try {
- const json = await deps.fetchLargestAccounts(address);
+ const json = await withDeadline(deps.fetchLargestAccounts(address), KEYLESS_CALL_DEADLINE_MS, 'getTokenLargestAccounts');
if (Array.isArray(json?.result?.value)) {
largestAnswered = true;
largest = json.result.value;
@@ -292,7 +300,7 @@ export async function composeTokenHolders({ address, limit = DEFAULT_LIMIT }, de
if (largest?.length) {
let owners = new Map();
try {
- owners = await deps.fetchAccountOwners(largest.map((a) => a.address));
+ owners = await withDeadline(deps.fetchAccountOwners(largest.map((a) => a.address)), KEYLESS_CALL_DEADLINE_MS, 'getMultipleAccounts');
} catch { /* owner resolution is enrichment — top-N stands on account addresses */ }
const accounts = largest.map((a) => ({
address: a.address,
diff --git a/api/_lib/elevenlabs.js b/api/_lib/elevenlabs.js
index 9a5a7c0d1..506b803a7 100644
--- a/api/_lib/elevenlabs.js
+++ b/api/_lib/elevenlabs.js
@@ -7,6 +7,7 @@
// each reimplement them. Lazy: nothing here touches the network at import time.
import { env } from './env.js';
+import { fetchUpstream } from './upstream-fetch.js';
export const ELEVEN_BASE = 'https://api.elevenlabs.io/v1';
@@ -121,7 +122,7 @@ export async function listVoices({ force = false, apiKey: keyOverride = null } =
let resp;
try {
- resp = await fetch(`${ELEVEN_BASE}/voices`, { headers: { 'xi-api-key': apiKey } });
+ resp = await fetchUpstream(`${ELEVEN_BASE}/voices`, { headers: { 'xi-api-key': apiKey } }, { name: 'elevenlabs', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
} catch (e) {
throw upstreamError('Could not reach ElevenLabs', 502, { cause: e });
}
@@ -207,10 +208,10 @@ export async function deleteVoice(voiceId, { apiKey: keyOverride = null } = {})
const apiKey = keyOverride || elevenApiKey();
if (!apiKey || !voiceId) return;
try {
- await fetch(`${ELEVEN_BASE}/voices/${encodeURIComponent(voiceId)}`, {
+ await fetchUpstream(`${ELEVEN_BASE}/voices/${encodeURIComponent(voiceId)}`, {
method: 'DELETE',
headers: { 'xi-api-key': apiKey },
- });
+ }, { name: 'elevenlabs', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
invalidateVoiceCache();
} catch (e) {
console.warn('[elevenlabs] deleteVoice failed', voiceId, e?.message);
diff --git a/api/_lib/execution-engine.js b/api/_lib/execution-engine.js
index 18cb47888..77ab3ef60 100644
--- a/api/_lib/execution-engine.js
+++ b/api/_lib/execution-engine.js
@@ -46,6 +46,7 @@ import {
VersionedTransaction,
} from '@solana/web3.js';
import { pollConfirmation } from './solana/confirm.js';
+import { fetchUpstream } from './upstream-fetch.js';
// Jito's published mainnet tip accounts. A bundle's tip must transfer SOL to one
// of these — Jito only accepts a bundle that pays a tip to a known tip account.
@@ -63,6 +64,26 @@ const JITO_TIP_ACCOUNTS = [
];
const DEFAULT_JITO_URL = 'https://mainnet.block-engine.jito.wtf';
+// Jito's public regional block engines. Every region serves the same bundle API
+// at the same path, so the configured (or default) engine leads and the others
+// are the alternates a network error or 5xx on the send fails over to. A bundle
+// is only ever re-sent when the first engine gave NO answer about it: a JSON-RPC
+// rejection or a 4xx is an answer and is never retried on another host.
+const JITO_REGION_URLS = [
+ DEFAULT_JITO_URL,
+ 'https://ny.mainnet.block-engine.jito.wtf',
+ 'https://amsterdam.mainnet.block-engine.jito.wtf',
+ 'https://frankfurt.mainnet.block-engine.jito.wtf',
+ 'https://tokyo.mainnet.block-engine.jito.wtf',
+ 'https://slc.mainnet.block-engine.jito.wtf',
+];
+// Deadlines. The fee + tip-floor reads sit on the hot snipe path and have safe
+// fallbacks (the connection's own fee sample, the mode floor), so they get a
+// short budget; the bundle send is the one that matters and gets 10s.
+const HELIUS_FEE_TIMEOUT_MS = 4_000;
+const JITO_TIP_FLOOR_TIMEOUT_MS = 4_000;
+const JITO_SEND_TIMEOUT_MS = 10_000;
+const JITO_STATUS_TIMEOUT_MS = 4_000;
const LAMPORTS_PER_SOL = 1_000_000_000;
// ── adaptive tip sizing (lamports) ────────────────────────────────────────────
@@ -99,6 +120,12 @@ function jitoBlockEngineUrl() {
return u.replace(/\/+$/, '');
}
+/** The configured engine first, then every other public region, deduplicated. */
+function jitoBlockEngineUrls() {
+ const primary = jitoBlockEngineUrl();
+ return [primary, ...JITO_REGION_URLS.filter((u) => u !== primary)];
+}
+
/** A Jito bundle route is only attempted on mainnet with a non-'off' tip mode. */
function jitoEligible(network, tipMode) {
return network !== 'devnet' && (tipMode === 'economy' || tipMode === 'turbo');
@@ -152,6 +179,7 @@ async function heliusPriorityFee(heliusKey, network) {
// is the right anchor.
params: [{ options: { recommended: true } }],
}),
+ signal: AbortSignal.timeout(HELIUS_FEE_TIMEOUT_MS),
});
if (!resp.ok) throw new Error(`helius pf ${resp.status}`);
const body = await resp.json();
@@ -202,17 +230,19 @@ async function estimateComputeUnits(connection, payer, instructions) {
}
// ── Jito Block Engine REST ────────────────────────────────────────────────────
-async function jitoRpc(method, params) {
- const url = `${jitoBlockEngineUrl()}/api/v1/bundles`;
+async function jitoRpc(method, params, { baseUrl = jitoBlockEngineUrl(), timeoutMs = JITO_STATUS_TIMEOUT_MS } = {}) {
+ const url = `${baseUrl}/api/v1/bundles`;
const resp = await fetch(url, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ jsonrpc: '2.0', id: 1, method, params }),
+ signal: AbortSignal.timeout(timeoutMs),
});
if (!resp.ok) {
const text = await resp.text().catch(() => '');
const err = new Error(`jito ${method} ${resp.status}: ${text.slice(0, 160)}`);
err.code = 'JITO_HTTP';
+ err.status = resp.status;
throw err;
}
const body = await resp.json();
@@ -224,28 +254,62 @@ async function jitoRpc(method, params) {
return body.result;
}
+// A failure that says nothing about the bundle itself: the engine never
+// answered (network error, deadline) or answered with a 5xx. Only these move
+// the send to another region; a JSON-RPC rejection or a 4xx is final.
+function isJitoFailoverError(err) {
+ if (err?.code === 'JITO_RPC') return false;
+ if (err?.code === 'JITO_HTTP') return Number(err.status) >= 500;
+ return true;
+}
+
+// sendBundle against the primary engine, then exactly one alternate region when
+// the primary gave no answer. Returns the engine that accepted the bundle so the
+// status polls ask the same one.
+async function jitoSendBundle(params) {
+ const urls = jitoBlockEngineUrls().slice(0, 2);
+ let lastErr;
+ for (const baseUrl of urls) {
+ try {
+ const bundleId = await jitoRpc('sendBundle', params, { baseUrl, timeoutMs: JITO_SEND_TIMEOUT_MS });
+ return { bundleId, baseUrl };
+ } catch (err) {
+ lastErr = err;
+ if (!isJitoFailoverError(err)) throw err;
+ }
+ }
+ throw lastErr;
+}
+
// Live Jito tip-floor (lamports). Reads the tip-floor endpoint and uses the 50th
// percentile of recent landed-tips as the baseline. Cached briefly. Returns null
-// when the endpoint is unreachable — the caller then uses the mode floor.
+// when no engine answers; the caller then uses the mode floor. The primary
+// engine and one alternate region are asked, each on a short deadline.
async function fetchJitoTipFloorLamports() {
const url = jitoBlockEngineUrl();
const cached = _tipFloorCache.get(url);
if (cached && Date.now() - cached.at < TIP_FLOOR_TTL_MS) return cached.lamports;
- try {
- const resp = await fetch(`${url}/api/v1/bundles/tip_floor`, { method: 'GET' });
- if (!resp.ok) return null;
- const body = await resp.json();
- const row = Array.isArray(body) ? body[0] : body;
- // The endpoint reports tips in SOL; prefer the 50th percentile, fall back to
- // the EMA of landed tips.
- const sol = row?.landed_tips_50th_percentile ?? row?.ema_landed_tips_50th_percentile ?? null;
- if (sol == null || !Number.isFinite(Number(sol))) return null;
- const lamports = BigInt(Math.max(0, Math.round(Number(sol) * LAMPORTS_PER_SOL)));
- _tipFloorCache.set(url, { at: Date.now(), lamports });
- return lamports;
- } catch {
- return null;
+ for (const baseUrl of jitoBlockEngineUrls().slice(0, 2)) {
+ try {
+ const resp = await fetch(`${baseUrl}/api/v1/bundles/tip_floor`, {
+ method: 'GET',
+ signal: AbortSignal.timeout(JITO_TIP_FLOOR_TIMEOUT_MS),
+ });
+ if (!resp.ok) continue;
+ const body = await resp.json();
+ const row = Array.isArray(body) ? body[0] : body;
+ // The endpoint reports tips in SOL; prefer the 50th percentile, fall back to
+ // the EMA of landed tips.
+ const sol = row?.landed_tips_50th_percentile ?? row?.ema_landed_tips_50th_percentile ?? null;
+ if (sol == null || !Number.isFinite(Number(sol))) continue;
+ const lamports = BigInt(Math.max(0, Math.round(Number(sol) * LAMPORTS_PER_SOL)));
+ _tipFloorCache.set(url, { at: Date.now(), lamports });
+ return lamports;
+ } catch {
+ continue;
+ }
}
+ return null;
}
// Resolve the tip for this attempt: max(mode floor, live tip-floor) scaled by the
@@ -264,7 +328,7 @@ function resolveTipLamports(tipMode, tipFloorLamports, attempt) {
// without an on-chain landing (the caller falls back to the protected route).
async function sendJitoBundle(signedTx, signature, connection) {
const b64 = Buffer.from(signedTx.serialize()).toString('base64');
- const bundleId = await jitoRpc('sendBundle', [[b64], { encoding: 'base64' }]);
+ const { bundleId, baseUrl } = await jitoSendBundle([[b64], { encoding: 'base64' }]);
const deadline = Date.now() + BUNDLE_POLL_TIMEOUT_MS;
while (Date.now() < deadline) {
@@ -283,7 +347,7 @@ async function sendJitoBundle(signedTx, signature, connection) {
}
// Cross-check Jito's bundle status: a 'Failed'/'Dropped' verdict ends the wait
// early so we fall back without burning the full timeout.
- const jitoStatus = await jitoRpc('getBundleStatuses', [[bundleId]]).catch(() => null);
+ const jitoStatus = await jitoRpc('getBundleStatuses', [[bundleId]], { baseUrl }).catch(() => null);
const st = jitoStatus?.value?.[0]?.confirmation_status || jitoStatus?.value?.[0]?.status;
if (st === 'failed' || st === 'Failed' || st === 'dropped' || st === 'Dropped') {
return { landed: false, slot: null, bundleId };
diff --git a/api/_lib/face-embed.js b/api/_lib/face-embed.js
index d27efb652..a8ab8692c 100644
--- a/api/_lib/face-embed.js
+++ b/api/_lib/face-embed.js
@@ -41,6 +41,7 @@ import { createHash } from 'node:crypto';
import { mkdir, readFile, writeFile } from 'node:fs/promises';
import os from 'node:os';
import path from 'node:path';
+import { fetchUpstream } from './upstream-fetch.js';
// Pinned to one opencv_zoo commit, not to `main`: a moving ref means a model
// swap could land in a cron run nobody triggered, and every stored score before
@@ -109,7 +110,7 @@ async function modelBytes(spec) {
const cached = await readFile(file).catch(() => null);
if (cached && createHash('sha256').update(cached).digest('hex') === spec.sha256) return cached;
- const res = await fetch(spec.url);
+ const res = await fetchUpstream(spec.url, {}, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!res.ok) throw new Error(`face model ${spec.id} download failed: HTTP ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
const got = createHash('sha256').update(buf).digest('hex');
diff --git a/api/_lib/forge-store.js b/api/_lib/forge-store.js
index 10d07bce1..a06000890 100644
--- a/api/_lib/forge-store.js
+++ b/api/_lib/forge-store.js
@@ -25,6 +25,7 @@ import { compressGlb } from './glb-compress.js';
import { classifyModelCategory } from './forge-classify.js';
import { cleanupGlb } from './glb-cleanup.js';
import { derivePbrChannels } from './glb-pbr-derive.js';
+import { fetchUpstream } from './upstream-fetch.js';
// Stable, non-secret salt so a leaked DB row can't be trivially reversed to the
// raw browser-local id. The id is anonymous to begin with; this is hygiene, not
@@ -264,7 +265,9 @@ export async function findByJob({ replicateJobId, clientKey }) {
// a generation. A hard 404/410 means the file is already gone, so retrying is
// pointless: fail fast on those and let the caller fall back to the provider URL.
const COPY_MAX_ATTEMPTS = 3;
-const COPY_RETRY_BASE_MS = 400;
+// A provider delivery blob can be tens of MB; the deadline covers the whole
+// body read, so it is sized for the 64 MB ceiling on a slow origin.
+const COPY_TIMEOUT_MS = 120_000;
// Score, clean, and (optionally) compress a freshly-downloaded GLB before it
// lands in the bucket. Every step is pure, local, and best-effort: a failure
@@ -341,53 +344,34 @@ function mediaTypeOr(header, fallback) {
}
async function copyToBucket({ sourceUrl, key, fallbackContentType, maxBytes, computeQuality = false, compress = null, cleanup = false, forceContentType = null }) {
- let lastErr;
- for (let attempt = 1; attempt <= COPY_MAX_ATTEMPTS; attempt++) {
- try {
- const resp = await fetch(sourceUrl);
- if (!resp.ok) {
- const err = new Error(`fetch ${sourceUrl}: ${resp.status}`);
- err.status = resp.status;
- // 404/410 = the ephemeral asset has already expired; no retry can recover it.
- if (resp.status === 404 || resp.status === 410) throw err;
- // 408/425/429/5xx are transient — retry within the loop.
- if (attempt < COPY_MAX_ATTEMPTS) { lastErr = err; }
- else throw err;
- } else {
- let buf = Buffer.from(await resp.arrayBuffer());
- if (buf.length > maxBytes) throw new Error(`asset too large: ${buf.length} bytes`);
- // forceContentType wins over the upstream header: providers (Replicate
- // et al.) often serve ephemeral output blobs as `application/octet-stream`
- // or another generic type regardless of the actual bytes. Trusting that
- // verbatim into R2's stored Content-Type is what made every homepage
- // forge thumbnail fail Chrome's Opaque Response Blocking — the browser
- // won't render a cross-origin <img> whose declared type isn't an image
- // type. Callers that know the asset kind (e.g. the preview image, whose
- // extension is already decided by imageExtFor) should force it.
- const contentType = forceContentType || mediaTypeOr(resp.headers.get('content-type'), fallbackContentType);
- let quality = null;
- let compression = null;
- let cleaned = null;
- if (computeQuality || compress || cleanup) {
- const scored = await scoreAndCompress(buf, { computeQuality, compress, cleanup });
- buf = scored.buf;
- quality = scored.quality;
- compression = scored.compression;
- cleaned = scored.cleaned;
- }
- await putObject({ key, body: buf, contentType, metadata: { source: 'forge' } });
- return { bytes: buf.length, publicUrl: publicUrl(key), quality, compression, cleaned };
- }
- } catch (err) {
- // Permanent (404/410) or too-large: surface immediately. Network errors
- // (fetch throw) are transient and retried until attempts are exhausted.
- if (err?.status === 404 || err?.status === 410 || /asset too large/.test(err?.message || '')) throw err;
- lastErr = err;
- if (attempt >= COPY_MAX_ATTEMPTS) throw err;
- }
- await new Promise((r) => setTimeout(r, COPY_RETRY_BASE_MS * attempt));
+ // fetchUpstream retries network errors and 408/425/429/5xx with jittered
+ // backoff and gives up immediately on 404/410 (the ephemeral asset has
+ // already expired; no retry can recover it), rejecting with an error that
+ // carries `status` so callers keep classifying it the same way.
+ const resp = await fetchUpstream(sourceUrl, {}, { timeoutMs: COPY_TIMEOUT_MS, attempts: COPY_MAX_ATTEMPTS, label: 'forge asset copy' });
+ let buf = Buffer.from(await resp.arrayBuffer());
+ if (buf.length > maxBytes) throw new Error(`asset too large: ${buf.length} bytes`);
+ // forceContentType wins over the upstream header: providers (Replicate
+ // et al.) often serve ephemeral output blobs as `application/octet-stream`
+ // or another generic type regardless of the actual bytes. Trusting that
+ // verbatim into R2's stored Content-Type is what made every homepage
+ // forge thumbnail fail Chrome's Opaque Response Blocking — the browser
+ // won't render a cross-origin <img> whose declared type isn't an image
+ // type. Callers that know the asset kind (e.g. the preview image, whose
+ // extension is already decided by imageExtFor) should force it.
+ const contentType = forceContentType || mediaTypeOr(resp.headers.get('content-type'), fallbackContentType);
+ let quality = null;
+ let compression = null;
+ let cleaned = null;
+ if (computeQuality || compress || cleanup) {
+ const scored = await scoreAndCompress(buf, { computeQuality, compress, cleanup });
+ buf = scored.buf;
+ quality = scored.quality;
+ compression = scored.compression;
+ cleaned = scored.cleaned;
}
- throw lastErr;
+ await putObject({ key, body: buf, contentType, metadata: { source: 'forge' } });
+ return { bytes: buf.length, publicUrl: publicUrl(key), quality, compression, cleaned };
}
function imageExtFor(url) {
diff --git a/api/_lib/gcp-auth.js b/api/_lib/gcp-auth.js
index da23259ff..caefa9575 100644
--- a/api/_lib/gcp-auth.js
+++ b/api/_lib/gcp-auth.js
@@ -15,43 +15,93 @@
//
// Never log the returned token or any service-account material.
+import { fetchUpstream } from './upstream-fetch.js';
+
function readEnv(name) {
if (typeof process !== 'undefined' && process.env?.[name]) return process.env[name];
return null;
}
-// Tokens are cached in-process for (expiry - 60s) to avoid hammering the token
-// endpoint on every request. The cloud-platform scope is broad enough to cover
-// both Imagen (:predict) and Claude (:rawPredict / :streamRawPredict).
-const _tokenCache = { token: null, expiresAt: 0 };
+// Tokens are cached in-process so the token endpoint is not hit on every
+// request. A refresh is attempted REFRESH_AHEAD_S before the token expires; if
+// that refresh fails (the metadata server or oauth2.googleapis.com blipping)
+// the cached token keeps being served while it still has more than
+// HARD_MARGIN_S of validity left, so a token-endpoint outage never takes every
+// Vertex call down for the four minutes a perfectly good token remains usable.
+// The cloud-platform scope is broad enough to cover both Imagen (:predict) and
+// Claude (:rawPredict / :streamRawPredict).
+const REFRESH_AHEAD_S = 300;
+const HARD_MARGIN_S = 60;
+const _tokenCache = { token: null, refreshAt: 0, expiresAt: 0 };
+
+function rememberToken(token, expiresInSeconds, now = Date.now()) {
+ const ttlMs = Number(expiresInSeconds) * 1000;
+ _tokenCache.token = token;
+ _tokenCache.expiresAt = now + ttlMs - HARD_MARGIN_S * 1000;
+ _tokenCache.refreshAt = Math.min(_tokenCache.expiresAt, now + ttlMs - REFRESH_AHEAD_S * 1000);
+ return token;
+}
+
+/** Test hook: forget the cached token. */
+export function _resetGcpTokenCache() {
+ _tokenCache.token = null;
+ _tokenCache.refreshAt = 0;
+ _tokenCache.expiresAt = 0;
+}
// Obtain a Google OAuth 2.0 access token, cached until shortly before expiry.
// Throws an error with code:'unconfigured' when no credentials are available so
// callers can branch to a fallback provider instead of hard-failing.
export async function getGcpAccessToken() {
const now = Date.now();
- if (_tokenCache.token && now < _tokenCache.expiresAt) {
+ if (_tokenCache.token && now < _tokenCache.refreshAt) {
return _tokenCache.token;
}
+ try {
+ return await _mintToken(now);
+ } catch (err) {
+ if (err?.code === 'unconfigured') throw err;
+ if (_tokenCache.token && Date.now() < _tokenCache.expiresAt) {
+ const leftS = Math.round((_tokenCache.expiresAt - Date.now()) / 1000);
+ console.warn(`[gcp-auth] token refresh failed (${err?.message || err}); serving the cached token for up to ${leftS}s more`);
+ return _tokenCache.token;
+ }
+ throw err;
+ }
+}
+async function _mintToken(now) {
const saJson = readEnv('GCP_SERVICE_ACCOUNT_JSON');
if (saJson && saJson.trim() && saJson.trim() !== '""') {
return _tokenFromServiceAccount(parseServiceAccount(saJson));
}
// Fall back to the metadata server (Cloud Run / GCE).
- const metaRes = await fetch(
+ // The metadata server answers in milliseconds on GCP and does not exist
+ // anywhere else: a short timeout keeps a local/dev boot from hanging on DNS.
+ // Three bounded attempts ride out the momentary 5xx it returns under a
+ // credential rotation; a host where it does not exist fails fast on DNS.
+ let metaErr = null;
+ const metaRes = await fetchUpstream(
'http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/token',
{ headers: { 'Metadata-Flavor': 'Google' } },
- ).catch(() => null);
+ { name: 'gcp-metadata-token', timeoutMs: 3_000, attempts: 3 },
+ ).catch((err) => {
+ metaErr = err;
+ return null;
+ });
if (metaRes?.ok) {
const data = await metaRes.json();
- _tokenCache.token = data.access_token;
- _tokenCache.expiresAt = now + (data.expires_in - 60) * 1000;
- return _tokenCache.token;
+ return rememberToken(data.access_token, data.expires_in, now);
}
+ // A metadata server that answered before (a token is cached) but failed now
+ // is an outage, not a misconfiguration: surface it so the stale-token grace
+ // applies instead of the unconfigured branch.
+ if (_tokenCache.token && metaErr) {
+ throw Object.assign(new Error(`GCP metadata token refresh failed: ${metaErr.message}`), { code: 'metadata_unavailable' });
+ }
throw Object.assign(
new Error('No GCP credentials found. Set GCP_SERVICE_ACCOUNT_JSON or run on GCE/Cloud Run.'),
{ code: 'unconfigured' },
@@ -169,13 +219,23 @@ async function _tokenFromServiceAccount(sa) {
const sig = Buffer.from(sigBuffer).toString('base64url');
const jwt = `${signingInput}.${sig}`;
- const tokenRes = await fetch('https://oauth2.googleapis.com/token', {
+ // 8s per attempt, three attempts with jittered backoff on 429/5xx/network.
+ // A 4xx is a definitive answer about the assertion (bad key, disabled
+ // account) and is surfaced at once rather than retried into a quota.
+ const tokenRes = await fetchUpstream('https://oauth2.googleapis.com/token', {
method: 'POST',
headers: { 'content-type': 'application/x-www-form-urlencoded' },
body: new URLSearchParams({
grant_type: 'urn:ietf:params:oauth:grant-type:jwt-bearer',
assertion: jwt,
}),
+ }, {
+ name: 'google-oauth',
+ timeoutMs: 8_000,
+ attempts: 3,
+ okWhen: (res) => res.ok || (res.status < 500 && res.status !== 429),
+ }).catch((err) => {
+ throw new Error(`GCP token exchange failed: ${err?.body || err?.message || err}`);
});
if (!tokenRes.ok) {
@@ -184,9 +244,7 @@ async function _tokenFromServiceAccount(sa) {
}
const data = await tokenRes.json();
- _tokenCache.token = data.access_token;
- _tokenCache.expiresAt = Date.now() + (data.expires_in - 60) * 1000;
- return _tokenCache.token;
+ return rememberToken(data.access_token, data.expires_in);
}
// True when a service-account credential is present in the environment. Cloud
diff --git a/api/_lib/github-api.js b/api/_lib/github-api.js
index 64dccad3f..c32f97714 100644
--- a/api/_lib/github-api.js
+++ b/api/_lib/github-api.js
@@ -4,6 +4,8 @@
// Upstream failures are tagged with `status` so wrap() reports them as 502
// upstream errors rather than as our own 500.
+import { fetchUpstream } from './upstream-fetch.js';
+
const API = 'https://api.github.com';
const UA = 'three.ws/1.0';
const TIMEOUT_MS = 15_000;
@@ -22,10 +24,10 @@ function upstream(message, status = 502) {
}
async function ghJson(path, token) {
- const res = await fetch(`${API}${path}`, {
+ const res = await fetchUpstream(`${API}${path}`, {
headers: headers(token),
signal: AbortSignal.timeout(TIMEOUT_MS),
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (res.status === 401) throw upstream('GitHub rejected the stored token', 401);
if (!res.ok) throw upstream(`GitHub ${path} failed: ${res.status}`);
return res.json();
@@ -44,10 +46,10 @@ export function fetchProfile(token) {
* an upstream outage to report.
*/
export async function verifyToken(token) {
- const res = await fetch(`${API}/user`, {
+ const res = await fetchUpstream(`${API}/user`, {
headers: headers(token),
signal: AbortSignal.timeout(TIMEOUT_MS),
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (res.status === 401 || res.status === 403) return { valid: false, status: res.status };
if (!res.ok) throw upstream(`GitHub token check failed: ${res.status}`);
const profile = await res.json();
@@ -84,12 +86,12 @@ const PINNED_QUERY = `query($login:String!){
* empty pin list rather than failing the whole catalog.
*/
export async function fetchPinnedRepos(token, login) {
- const res = await fetch(`${API}/graphql`, {
+ const res = await fetchUpstream(`${API}/graphql`, {
method: 'POST',
headers: { ...headers(token), 'content-type': 'application/json' },
body: JSON.stringify({ query: PINNED_QUERY, variables: { login } }),
signal: AbortSignal.timeout(TIMEOUT_MS),
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (!res.ok) return [];
const body = await res.json().catch(() => null);
const nodes = body?.data?.user?.pinnedItems?.nodes;
@@ -113,10 +115,10 @@ export async function fetchPinnedRepos(token, login) {
* that do have one.
*/
export async function fetchReadme(token, repoKey) {
- const res = await fetch(`${API}/repos/${repoKey}/readme`, {
+ const res = await fetchUpstream(`${API}/repos/${repoKey}/readme`, {
headers: headers(token, 'application/vnd.github.raw'),
signal: AbortSignal.timeout(TIMEOUT_MS),
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (res.status === 404) return null;
if (!res.ok) throw upstream(`GitHub README fetch failed for ${repoKey}: ${res.status}`);
return res.text();
@@ -125,7 +127,7 @@ export async function fetchReadme(token, repoKey) {
/** Best-effort revocation of our OAuth grant on GitHub's side. */
export async function revokeGrant(token, clientId, clientSecret) {
const creds = Buffer.from(`${clientId}:${clientSecret}`).toString('base64');
- const res = await fetch(`${API}/applications/${clientId}/grant`, {
+ const res = await fetchUpstream(`${API}/applications/${clientId}/grant`, {
method: 'DELETE',
headers: {
authorization: `Basic ${creds}`,
@@ -135,6 +137,6 @@ export async function revokeGrant(token, clientId, clientSecret) {
},
body: JSON.stringify({ access_token: token }),
signal: AbortSignal.timeout(TIMEOUT_MS),
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 1, okWhen: () => true });
return res.status === 204;
}
diff --git a/api/_lib/ipfs-gateways.js b/api/_lib/ipfs-gateways.js
new file mode 100644
index 000000000..dcc72f4c6
--- /dev/null
+++ b/api/_lib/ipfs-gateways.js
@@ -0,0 +1,129 @@
+// Multi-gateway reads for content-addressed URIs (ipfs://, ar://).
+//
+// Every server-side site that resolved `ipfs://` or `ar://` to ONE public
+// gateway (ipfs.io, arweave.net) went dark with that gateway: an ERC-8004
+// registration lost its name and avatar, a token's off-chain metadata came
+// back null, an on-chain agent card failed to load. Content addressing means
+// any gateway serves identical bytes, so a read should walk an ordered list
+// and take the first answer.
+//
+// The IPFS list is the one api/_lib/ipfs-pin.js verifies manifests against
+// (imported, not copied) plus 4everland, the extra mirror api/img.js races.
+// cloudflare-ipfs.com / cf-ipfs.com (retired 2024) and flk-ipfs.xyz (refusing
+// connections) are deliberately absent: a dead host in a ladder burns the
+// timeout budget and reports a network error as a miss.
+
+import { IPFS_READ_GATEWAYS } from './ipfs-pin.js';
+import { fetchFirst } from '../../src/shared/failover-fetch.js';
+
+/** Ordered IPFS gateway prefixes (each ends in `/ipfs/`). */
+export const IPFS_GATEWAYS = [...IPFS_READ_GATEWAYS, 'https://4everland.io/ipfs/'];
+
+/** Ordered Arweave gateway prefixes (each ends in `/`). */
+export const ARWEAVE_GATEWAYS = [
+ 'https://arweave.net/',
+ 'https://ar-io.net/',
+ 'https://arweave.dev/',
+];
+
+const DEFAULT_TIMEOUT_MS = 8_000;
+
+/**
+ * Pull the `<cid>/<path?>` portion out of an `ipfs://` URI or any
+ * `https://<gateway>/ipfs/<cid>` URL so it can be re-pointed at every gateway.
+ * @param {string} uri
+ * @returns {string|null}
+ */
+export function ipfsPath(uri) {
+ if (uri.startsWith('ipfs://')) return uri.slice('ipfs://'.length).replace(/^ipfs\//, '') || null;
+ const m = /^https?:\/\/[^/]+\/ipfs\/(.+)$/.exec(uri);
+ return m ? m[1] : null;
+}
+
+/**
+ * Ordered https URLs to try for one logical URI. Plain https URLs that are not
+ * gateway-addressed come back as a single-element list; anything else (empty,
+ * unsupported scheme) is an empty list.
+ *
+ * @param {string} uri
+ * @returns {string[]}
+ */
+export function gatewayCandidates(uri) {
+ const raw = typeof uri === 'string' ? uri.trim() : '';
+ if (!raw) return [];
+ const cidPath = ipfsPath(raw);
+ if (cidPath) return IPFS_GATEWAYS.map((g) => g + cidPath);
+ if (raw.startsWith('ar://')) {
+ const id = raw.slice('ar://'.length);
+ return id ? ARWEAVE_GATEWAYS.map((g) => g + id) : [];
+ }
+ if (/^https?:\/\//.test(raw)) return [raw];
+ return [];
+}
+
+/**
+ * Walk the candidates with a caller-supplied fetcher (an SSRF-pinned fetch,
+ * for example) and return the first result the caller accepts. `attempt`
+ * receives one URL and must resolve to a value or throw; a null/undefined
+ * result is treated as a miss and the walk continues.
+ *
+ * @template T
+ * @param {string} uri
+ * @param {(url: string) => Promise<T>} attempt
+ * @param {{ label?: string }} [opts]
+ * @returns {Promise<{ value: T, url: string }>}
+ */
+export async function walkGateways(uri, attempt, { label = 'gateway' } = {}) {
+ const urls = gatewayCandidates(uri);
+ if (!urls.length) {
+ throw Object.assign(new Error(`${label}: unsupported uri`), { code: 'unsupported_uri' });
+ }
+ const failures = [];
+ for (const url of urls) {
+ try {
+ const value = await attempt(url);
+ if (value !== null && value !== undefined) return { value, url };
+ failures.push(`${url}: miss`);
+ } catch (err) {
+ failures.push(`${url}: ${err?.message || err}`);
+ }
+ }
+ throw Object.assign(new Error(`${label}: no gateway served ${uri} (${failures.join('; ')})`), {
+ code: 'gateway_unreachable',
+ status: 502,
+ });
+}
+
+/**
+ * Fetch a URI across every gateway in order (fetchFirst: per-attempt timeout,
+ * per-host cooldown after a failure) and return the response body.
+ *
+ * @param {string} uri
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs]
+ * @param {number} [opts.maxBytes=2097152] reject bodies larger than this
+ * @param {'json'|'text'} [opts.as='json']
+ * @param {Record<string,string>} [opts.headers]
+ * @returns {Promise<{ value: any, url: string }>}
+ */
+export async function fetchFromGateways(uri, { timeoutMs = DEFAULT_TIMEOUT_MS, maxBytes = 2 * 1024 * 1024, as = 'json', headers } = {}) {
+ const urls = gatewayCandidates(uri);
+ if (!urls.length) {
+ throw Object.assign(new Error(`gateway: unsupported uri ${uri}`), { code: 'unsupported_uri' });
+ }
+ const parse = async (res) => {
+ if (Number(res.headers.get('content-length') || 0) > maxBytes) throw new Error('too_large');
+ const text = await res.text();
+ if (text.length > maxBytes) throw new Error('too_large');
+ if (as === 'text') return text;
+ return JSON.parse(text);
+ };
+ const providers = urls.map((url) => ({
+ name: new URL(url).host,
+ url,
+ init: { redirect: 'follow', headers: { accept: 'application/json, */*', ...(headers || {}) } },
+ parse,
+ }));
+ const { value, source } = await fetchFirst(providers, { timeoutMs, label: 'ipfs-gateways' });
+ return { value, url: urls.find((u) => new URL(u).host === source) || urls[0] };
+}
diff --git a/api/_lib/ipfs-pin.js b/api/_lib/ipfs-pin.js
index 7625edb21..c7a9fb89b 100644
--- a/api/_lib/ipfs-pin.js
+++ b/api/_lib/ipfs-pin.js
@@ -12,6 +12,8 @@
* fall back to R2 HTTPS hosting, a valid metadata URI but not a CID.
*/
+import { fetchUpstream } from './upstream-fetch.js';
+
const PINATA_FILE_ENDPOINT = 'https://api.pinata.cloud/pinning/pinFileToIPFS';
const WEB3_STORAGE_ENDPOINT = 'https://api.web3.storage/upload';
@@ -104,18 +106,22 @@ export async function fetchFromGateways(cid, { maxBytes = 1024 * 1024, gateways,
}
// Bounded so a hung provider cannot hold an interactive request (persona save
-// pins inline) open until the platform's own request timeout fires.
-const PIN_TIMEOUT_MS = 25000;
+// pins inline) open until the platform's own request timeout fires. A pin is
+// content-addressed, so a second attempt after a network failure is safe: the
+// same bytes yield the same CID. Each provider carries its own breaker so a
+// dead host fails fast and the chain moves on instead of paying the deadline
+// on every request during its outage.
+const PIN_TIMEOUT_MS = 30_000;
+const PIN_ATTEMPTS = 2;
async function pinViaPinata(buf, filename) {
const form = new FormData();
form.append('file', new Blob([buf]), filename);
- const resp = await fetch(PINATA_FILE_ENDPOINT, {
+ const resp = await fetchUpstream(PINATA_FILE_ENDPOINT, {
method: 'POST',
headers: { Authorization: `Bearer ${process.env.PINATA_JWT}` },
body: form,
- signal: AbortSignal.timeout(PIN_TIMEOUT_MS),
- });
+ }, { name: 'ipfs:pinata', timeoutMs: PIN_TIMEOUT_MS, attempts: PIN_ATTEMPTS, okWhen: () => true });
if (!resp.ok) {
const detail = await resp.text().catch(() => '');
throw Object.assign(new Error(`Pinata error ${resp.status}`), { status: 502, detail });
@@ -125,12 +131,11 @@ async function pinViaPinata(buf, filename) {
}
async function pinViaWeb3Storage(buf, filename) {
- const resp = await fetch(WEB3_STORAGE_ENDPOINT, {
+ const resp = await fetchUpstream(WEB3_STORAGE_ENDPOINT, {
method: 'POST',
headers: { Authorization: `Bearer ${process.env.WEB3_STORAGE_TOKEN}`, 'X-NAME': filename },
body: buf,
- signal: AbortSignal.timeout(PIN_TIMEOUT_MS),
- });
+ }, { name: 'ipfs:web3.storage', timeoutMs: PIN_TIMEOUT_MS, attempts: PIN_ATTEMPTS, okWhen: () => true });
if (!resp.ok) {
const detail = await resp.text().catch(() => '');
throw Object.assign(new Error(`Web3.Storage error ${resp.status}`), { status: 502, detail });
@@ -145,7 +150,11 @@ export function ipfsPinningConfigured() {
}
/**
- * Pin a buffer to IPFS via the configured provider.
+ * Pin a buffer to IPFS through every configured provider in order (Pinata,
+ * then web3.storage) until one answers with a CID. A provider that is down,
+ * rate limited, or past its deadline is logged and the next rung is tried; the
+ * error thrown when the whole chain fails carries the last rung's status and
+ * detail so the caller reports the true cause.
*
* @param {Buffer} buf
* @param {string} filename
@@ -153,13 +162,30 @@ export function ipfsPinningConfigured() {
* the pinned CID + gateway URI, or null when no provider is configured.
*/
export async function pinToIPFS(buf, filename) {
- if (process.env.PINATA_JWT) {
- const { cid, provider } = await pinViaPinata(buf, filename);
- return { cid, uri: ipfsGatewayUrl(cid), provider };
- }
- if (process.env.WEB3_STORAGE_TOKEN) {
- const { cid, provider } = await pinViaWeb3Storage(buf, filename);
- return { cid, uri: ipfsGatewayUrl(cid), provider };
+ const rungs = [];
+ if (process.env.PINATA_JWT) rungs.push(pinViaPinata);
+ if (process.env.WEB3_STORAGE_TOKEN) rungs.push(pinViaWeb3Storage);
+ if (!rungs.length) return null;
+ let lastErr;
+ for (const rung of rungs) {
+ try {
+ const { cid, provider } = await rung(buf, filename);
+ return { cid, uri: ipfsGatewayUrl(cid), provider };
+ } catch (err) {
+ lastErr = err;
+ if (rungs.length > 1) console.warn(`[ipfs-pin] ${filename}: ${err?.message || err}; trying next provider`);
+ }
}
- return null;
+ throw lastErr;
+}
+
+/**
+ * Pin a JSON document (metadata, manifests) through the same provider chain.
+ *
+ * @param {object} doc
+ * @param {string} filename
+ * @returns {Promise<{cid: string, uri: string, provider: string} | null>}
+ */
+export function pinJsonToIPFS(doc, filename) {
+ return pinToIPFS(Buffer.from(JSON.stringify(doc), 'utf-8'), filename);
}
diff --git a/api/_lib/irl-bake.js b/api/_lib/irl-bake.js
index a26c971e4..ab527ef94 100644
--- a/api/_lib/irl-bake.js
+++ b/api/_lib/irl-bake.js
@@ -19,6 +19,7 @@
import { bakeAppearance, appearanceHash, isBakeable } from './bake.js';
import { putObject, publicUrl } from './r2.js';
import { env } from './env.js';
+import { fetchUpstream } from './upstream-fetch.js';
export { isBakeable };
@@ -35,7 +36,7 @@ async function fetchBaseGlb(baseUrl) {
const abs = /^https?:\/\//i.test(baseUrl)
? baseUrl
: `${env.APP_ORIGIN}${baseUrl.startsWith('/') ? '' : '/'}${baseUrl}`;
- const r = await fetch(abs);
+ const r = await fetchUpstream(abs, {}, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!r.ok) throw new Error(`base GLB fetch failed: ${abs} → ${r.status}`);
const declared = Number(r.headers.get('content-length') || 0);
if (declared && declared > MAX_BASE_GLB_BYTES) {
diff --git a/api/_lib/likeness-score.js b/api/_lib/likeness-score.js
index 63b0bd18d..d34396f6c 100644
--- a/api/_lib/likeness-score.js
+++ b/api/_lib/likeness-score.js
@@ -24,6 +24,7 @@
// different findings and a table that conflates them is worse than no table.
import { renderAvatarScene, SCENE_PRESETS } from './avatar-render.js';
+import { fetchUpstream } from './upstream-fetch.js';
import {
SCORER_VERSION,
SFACE_SAME_IDENTITY_COSINE,
@@ -79,7 +80,7 @@ export function cosineToScore5(cosine) {
}
async function fetchImage(url, { maxBytes = 12 * 1024 * 1024 } = {}) {
- const res = await fetch(url);
+ const res = await fetchUpstream(url, {}, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!res.ok) throw new Error(`capture fetch ${res.status}`);
const buf = Buffer.from(await res.arrayBuffer());
if (buf.length > maxBytes) throw new Error(`capture too large: ${buf.length} bytes`);
diff --git a/api/_lib/market/ohlcv.js b/api/_lib/market/ohlcv.js
index b8f00da89..3195d2d21 100644
--- a/api/_lib/market/ohlcv.js
+++ b/api/_lib/market/ohlcv.js
@@ -1,40 +1,63 @@
// Keyless on-chain OHLCV source for the Granite Oracle, backed by the public
// GeckoTerminal API (https://www.geckoterminal.com/dex-api). No API key, real
-// candles, deep history (up to 1000 points) — enough to fill the ≥512-point
+// candles, deep history (up to 1000 points): enough to fill the >=512-point
// context window the Granite TimeSeries forecaster requires.
//
-// Never fabricates candles: any upstream failure throws with the real status so
-// the caller surfaces the true cause.
+// GeckoTerminal is the first rung, never the only one. Every exported read has
+// a second source and a last-good tier behind it:
+// - fetchOhlcv: GeckoTerminal -> Birdeye pair OHLCV (keyed, same pool
+// address) -> CEX klines for majors (close-only, flagged)
+// - topPoolForToken: GeckoTerminal -> DexScreener deepest pair -> Birdeye markets
+// - trendingPools: GeckoTerminal, then the last good board
+// A value that was served live once is remembered in the shared cache
+// (cacheWrapLastGood) so a GeckoTerminal outage answers with the previous
+// candles rather than a 502. Nothing here fabricates a candle: when every rung
+// fails and nothing is cached, the real upstream status is thrown.
import { createCache } from '../mem-cache.js';
+import { cacheWrapLastGood } from '../cache.js';
+import { fetchUpstream } from '../upstream-fetch.js';
+import { fetchFirstOrNull } from '../../../src/shared/failover-fetch.js';
+import { cexCandleProviders } from '../cex-public.js';
const BASE = 'https://api.geckoterminal.com/api/v2';
+const BIRDEYE_BASE = 'https://public-api.birdeye.so';
+const DEXSCREENER_TOKENS = 'https://api.dexscreener.com/latest/dex/tokens';
const UA = 'three.ws-granite-oracle/1.0';
-// Small in-memory cache — GeckoTerminal's free tier is ~30 req/min and candle
+// Small in-memory cache: GeckoTerminal's free tier is ~30 req/min and candle
// data only changes once per bar, so a short TTL keeps us well under the cap.
-// True LRU bounding: the previous Map was trimmed with
-// `delete(keys().next().value)`, evicting the oldest INSERTED url rather than
-// the least recently used one.
-const cache = createCache({ max: 256 }); // url → { value, expiresAt }
+const cache = createCache({ max: 256 }); // url -> { value, expiresAt }
const TTL_MS = 20_000;
-// Bounded exponential backoff with jitter for transient upstream throttling.
-// GeckoTerminal's free tier (~30 req/min) bursts into 429s and occasional 5xx;
-// a couple of short, jittered waits clear most of them. Bounded so the total
-// added latency (≈0.4s + ≈0.8s worst case, ~1.8s with jitter) never approaches
-// the function time budget. 404 and other client errors are NOT retried.
+// Shared-cache windows for the last-good tier. The fresh TTL mirrors the
+// in-memory one; the stale window is how long an outage can be ridden out.
+const LKG_FRESH_S = 20;
+const LKG_STALE_S = 24 * 3600;
+const POOL_FRESH_S = 300;
+const TRENDING_FRESH_S = 60;
+
+// Bounded retries for transient upstream throttling. GeckoTerminal's free tier
+// bursts into 429s and occasional 5xx; the shared upstream fetch applies
+// jittered backoff and honours Retry-After. 404 and other client errors are
+// NOT retried.
const MAX_ATTEMPTS = 3;
-const BACKOFF_BASE_MS = 400;
-const BACKOFF_CAP_MS = 1200;
-const RETRYABLE = new Set([429, 502, 503, 504]);
-const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
-// attempt is 0-based: delay grows 400ms, 800ms, … capped, with ±50% jitter so a
-// fleet of lambdas throttled together don't all retry on the same beat.
-function backoffMs(attempt) {
- const exp = Math.min(BACKOFF_CAP_MS, BACKOFF_BASE_MS * 2 ** attempt);
- return Math.round(exp * (0.5 + Math.random() * 0.5));
+// DexScreener chain slug for a GeckoTerminal network id. The two agree for the
+// networks this oracle serves; anything else maps to itself.
+const DEX_CHAIN_BY_NETWORK = { solana: 'solana', eth: 'ethereum', base: 'base', bsc: 'bsc' };
+
+const num = (v) => {
+ const n = Number(v);
+ return Number.isFinite(n) ? n : null;
+};
+
+// Normalise an upstream failure to the status ladder callers branch on: 404
+// (unknown token/pool) and 429 (throttled) pass through, everything else is a
+// 502 so the route answers "upstream trouble", never a generic 500.
+function tagged(message, status) {
+ const s = status === 404 ? 404 : status === 429 ? 429 : status === 400 ? 400 : 502;
+ return Object.assign(new Error(message), { status: s });
}
async function gecko(path) {
@@ -43,50 +66,32 @@ async function gecko(path) {
const hit = cache.get(url);
if (hit && hit.expiresAt > now) return hit.value;
- // Retry transient throttles (429/5xx) with bounded jittered backoff before
- // giving up. Preserves the real status (esp. 429) on the thrown error so the
- // caller maps it to a retryable response instead of a generic 502/500.
let res;
- let lastStatus = 502;
- for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {
- try {
- res = await fetch(url, {
- headers: { accept: 'application/json', 'user-agent': UA },
- signal: AbortSignal.timeout(12_000),
- });
- } catch (e) {
- throw Object.assign(new Error(`GeckoTerminal unreachable: ${e.message}`), {
- status: 502,
- });
- }
- if (res.ok) break;
- lastStatus = res.status;
- const text = await res.text();
- if (RETRYABLE.has(res.status) && attempt < MAX_ATTEMPTS - 1) {
- await sleep(backoffMs(attempt));
- continue; // retry with backoff
+ try {
+ res = await fetchUpstream(
+ url,
+ { headers: { accept: 'application/json', 'user-agent': UA } },
+ { timeoutMs: 12_000, attempts: MAX_ATTEMPTS, label: 'GeckoTerminal' },
+ );
+ } catch (e) {
+ const status = e?.status;
+ if (typeof status === 'number' && status !== 503) {
+ throw tagged(`GeckoTerminal ${status}: ${String(e.body || e.message || '').slice(0, 160)}`, status);
}
- throw Object.assign(new Error(`GeckoTerminal ${res.status}: ${text.slice(0, 160)}`), {
- status: res.status === 404 ? 404 : res.status === 429 ? 429 : 502,
- });
- }
- if (!res || !res.ok) {
- throw Object.assign(new Error(`GeckoTerminal ${lastStatus}`), {
- status: lastStatus === 404 ? 404 : lastStatus === 429 ? 429 : 502,
- });
+ throw tagged(`GeckoTerminal unreachable: ${e?.message || e}`, 502);
}
const text = await res.text();
let json;
try {
json = JSON.parse(text);
} catch {
- throw Object.assign(new Error('GeckoTerminal returned non-JSON'), { status: 502 });
+ throw tagged('GeckoTerminal returned non-JSON', 502);
}
cache.set(url, { value: json, expiresAt: now + TTL_MS });
return json;
}
-// Strip GeckoTerminal's "network_" id prefix (e.g. "solana_<addr>" → "<addr>").
+// Strip GeckoTerminal's "network_" id prefix (e.g. "solana_<addr>" -> "<addr>").
function bareId(id) {
if (typeof id !== 'string') return '';
const i = id.indexOf('_');
@@ -94,7 +99,7 @@ function bareId(id) {
}
// pandas-style cadence string the forecast API expects, from a GeckoTerminal
-// timeframe + aggregate (e.g. hour/1 → "1h", minute/15 → "15min", day/1 → "1D").
+// timeframe + aggregate (e.g. hour/1 -> "1h", minute/15 -> "15min", day/1 -> "1D").
export function freqFor(timeframe, aggregate = 1) {
const n = Math.max(1, Number(aggregate) || 1);
if (timeframe === 'minute') return `${n}min`;
@@ -102,17 +107,99 @@ export function freqFor(timeframe, aggregate = 1) {
return `${n}h`; // hour
}
-// Top (most-liquid) pool for a Solana token mint. Returns the pool address.
+// Seconds per bar for a timeframe + aggregate.
+export function barSeconds(timeframe, aggregate = 1) {
+ const n = Math.max(1, Number(aggregate) || 1);
+ if (timeframe === 'minute') return 60 * n;
+ if (timeframe === 'day') return 86_400 * n;
+ return 3_600 * n;
+}
+
+// Birdeye OHLCV `type` for a timeframe + aggregate, or null when Birdeye has no
+// matching bar (the rung is skipped rather than answering with the wrong cadence).
+export function birdeyeTypeFor(timeframe, aggregate = 1) {
+ const n = Math.max(1, Number(aggregate) || 1);
+ const table = {
+ minute: { 1: '1m', 3: '3m', 5: '5m', 15: '15m', 30: '30m' },
+ hour: { 1: '1H', 2: '2H', 4: '4H', 6: '6H', 8: '8H', 12: '12H' },
+ day: { 1: '1D', 3: '3D', 7: '1W' },
+ };
+ return table[timeframe]?.[n] ?? null;
+}
+
+// Chart window (days) a CEX kline request must cover to hold `limit` bars of
+// this cadence, snapped to the windows cex-public.js can serve in one call.
+function cexDaysFor(timeframe, aggregate, limit) {
+ const spanDays = (barSeconds(timeframe, aggregate) * Math.max(1, limit)) / 86_400;
+ for (const d of [1, 7, 30, 90, 365]) if (spanDays <= d) return d;
+ return 365;
+}
+
+// Rung 2 for topPoolForToken: DexScreener's deepest pair for the token. Pair
+// addresses are the on-chain AMM pool accounts, the same ids GeckoTerminal uses.
+function dexscreenerPoolProvider(mint, network) {
+ const chain = DEX_CHAIN_BY_NETWORK[network] || network;
+ return {
+ name: 'dexscreener-pool',
+ url: `${DEXSCREENER_TOKENS}/${encodeURIComponent(mint)}`,
+ parse: async (r) => {
+ const data = await r.json();
+ const pairs = (Array.isArray(data?.pairs) ? data.pairs : []).filter(
+ (p) => p?.pairAddress && (p.chainId === chain || !p.chainId),
+ );
+ if (!pairs.length) return null;
+ pairs.sort((a, b) => (b.liquidity?.usd ?? 0) - (a.liquidity?.usd ?? 0));
+ return pairs[0].pairAddress;
+ },
+ };
+}
+
+// Rung 3 for topPoolForToken: Birdeye's market list for the token (keyed).
+function birdeyePoolProvider(mint, network) {
+ const key = process.env.BIRDEYE_API_KEY;
+ if (!key || network !== 'solana') return null;
+ return {
+ name: 'birdeye-markets',
+ url: `${BIRDEYE_BASE}/defi/v2/markets?address=${encodeURIComponent(mint)}&sort_by=liquidity&sort_type=desc&offset=0&limit=1`,
+ init: { headers: { 'X-API-KEY': key, 'x-chain': 'solana', accept: 'application/json' } },
+ parse: async (r) => {
+ const data = await r.json();
+ const item = data?.data?.items?.[0];
+ return item?.address || null;
+ },
+ };
+}
+
+async function topPoolLive(mint, network) {
+ let geckoErr = null;
+ try {
+ const json = await gecko(`/networks/${network}/tokens/${mint}/pools?page=1`);
+ const pool = json?.data?.[0];
+ if (pool) return pool.attributes?.address || bareId(pool.id);
+ geckoErr = tagged(`no pools found for token ${mint}`, 404);
+ } catch (e) {
+ geckoErr = e;
+ }
+ // GeckoTerminal does not know the token, or is down: ask the keyless
+ // DexScreener index, then Birdeye. A miss everywhere keeps GeckoTerminal's
+ // verdict (404 = genuinely no market) so the route's empty state still fires.
+ const providers = [dexscreenerPoolProvider(mint, network), birdeyePoolProvider(mint, network)].filter(Boolean);
+ const pool = await fetchFirstOrNull(providers, { timeoutMs: 6000, label: `top-pool:${mint}` });
+ if (pool) return pool;
+ throw geckoErr;
+}
+
+// Top (most-liquid) pool for a token mint. Returns the pool address.
export async function topPoolForToken(mint, network = 'solana') {
- const json = await gecko(`/networks/${network}/tokens/${mint}/pools?page=1`);
- const pool = json?.data?.[0];
- if (!pool) throw Object.assign(new Error(`no pools found for token ${mint}`), { status: 404 });
- return pool.attributes?.address || bareId(pool.id);
+ return cacheWrapLastGood(
+ `ohlcv:toppool:v1:${network}:${mint}`,
+ POOL_FRESH_S,
+ () => topPoolLive(mint, network),
+ { staleTtlSeconds: LKG_STALE_S },
+ );
}
-// Trending Solana pools, normalised for the on-screen picker.
-export async function trendingPools(network = 'solana', limit = 8) {
- const json = await gecko(`/networks/${network}/trending_pools?page=1`);
+function normalizeTrending(json, limit) {
return (json?.data || []).slice(0, limit).map((p) => {
const a = p.attributes || {};
return {
@@ -128,44 +215,136 @@ export async function trendingPools(network = 'solana', limit = 8) {
});
}
-// Fetch OHLCV candles for a pool, returned oldest → newest (chronological), plus
-// the base/quote token metadata GeckoTerminal includes in the same response.
-// timeframe: 'minute' | 'hour' | 'day' aggregate: bars per candle limit: ≤1000
-export async function fetchOhlcv({
- pool,
- network = 'solana',
- timeframe = 'hour',
- aggregate = 1,
- limit = 1000,
-}) {
- if (!pool) throw Object.assign(new Error('pool is required'), { status: 400 });
+// Trending pools, normalised for the on-screen picker. The board turns over
+// slowly, so the last good one is served through a GeckoTerminal outage.
+export async function trendingPools(network = 'solana', limit = 8) {
+ return cacheWrapLastGood(
+ `ohlcv:trending:v1:${network}:${limit}`,
+ TRENDING_FRESH_S,
+ async () => normalizeTrending(await gecko(`/networks/${network}/trending_pools?page=1`), limit),
+ { staleTtlSeconds: LKG_STALE_S },
+ );
+}
+
+const tokenOf = (m) => (m ? { name: m.name, symbol: m.symbol, address: m.address } : null);
+
+function sortCandles(rows) {
+ return rows
+ .filter((d) => Number.isFinite(d.t) && Number.isFinite(d.c) && d.c > 0)
+ .sort((a, b) => a.t - b.t);
+}
+
+async function geckoCandles({ pool, network, timeframe, aggregate, limit }) {
const json = await gecko(
`/networks/${network}/pools/${pool}/ohlcv/${timeframe}?aggregate=${aggregate}&limit=${limit}&currency=usd`,
);
const list = json?.data?.attributes?.ohlcv_list;
- if (!Array.isArray(list)) {
- throw Object.assign(new Error('GeckoTerminal returned no candles'), { status: 502 });
- }
+ if (!Array.isArray(list)) throw tagged('GeckoTerminal returned no candles', 502);
// Upstream is newest-first; reverse to chronological and coerce to numbers.
- const candles = list
- .map((row) => ({
+ const candles = sortCandles(
+ list.map((row) => ({
t: Number(row[0]),
o: Number(row[1]),
h: Number(row[2]),
l: Number(row[3]),
c: Number(row[4]),
v: Number(row[5] ?? 0),
- }))
- .filter((d) => Number.isFinite(d.t) && Number.isFinite(d.c) && d.c > 0)
- .sort((a, b) => a.t - b.t);
-
+ })),
+ );
const meta = json?.meta || {};
- const tokenOf = (m) => (m ? { name: m.name, symbol: m.symbol, address: m.address } : null);
+ return { candles, base: tokenOf(meta.base), quote: tokenOf(meta.quote), source: 'geckoterminal' };
+}
+// Birdeye's pair OHLCV takes the same pool address GeckoTerminal does, so it
+// is a like-for-like second source for any Solana pool once the key is set.
+async function birdeyePairCandles({ pool, network, timeframe, aggregate, limit }) {
+ const key = process.env.BIRDEYE_API_KEY;
+ const type = birdeyeTypeFor(timeframe, aggregate);
+ if (!key || network !== 'solana' || !type) return null;
+ const to = Math.floor(Date.now() / 1000);
+ const from = to - barSeconds(timeframe, aggregate) * limit;
+ const url =
+ `${BIRDEYE_BASE}/defi/ohlcv/pair?address=${encodeURIComponent(pool)}` +
+ `&type=${type}&time_from=${from}&time_to=${to}`;
+ const res = await fetchUpstream(
+ url,
+ { headers: { 'X-API-KEY': key, 'x-chain': 'solana', accept: 'application/json' } },
+ { timeoutMs: 10_000, attempts: 2, name: 'birdeye:ohlcv-pair', label: 'Birdeye pair OHLCV' },
+ );
+ const payload = await res.json().catch(() => null);
+ const items = payload?.data?.items;
+ if (!Array.isArray(items) || !items.length) return null;
+ const candles = sortCandles(
+ items.map((it) => ({
+ t: Number(it.unixTime),
+ o: Number(it.o),
+ h: Number(it.h),
+ l: Number(it.l),
+ c: Number(it.c),
+ v: Number(it.v ?? 0),
+ })),
+ );
+ return candles.length ? { candles, base: null, quote: null, source: 'birdeye' } : null;
+}
+
+// Last rung, majors only: exchange klines for the pool's base asset. The
+// shared CEX layer yields a close series, so each bar is flagged close-only
+// (open/high/low equal the close) rather than passing invented wicks off as
+// real ones. Forecasters that consume `c` are unaffected; renderers can hide
+// the wick.
+async function cexCandles({ baseSymbol, timeframe, aggregate, limit }) {
+ if (!baseSymbol) return null;
+ const days = cexDaysFor(timeframe, aggregate, limit);
+ const providers = cexCandleProviders(String(baseSymbol).toUpperCase(), days);
+ if (!providers.length) return null;
+ const rows = await fetchFirstOrNull(providers, { timeoutMs: 6000, label: `cex-klines:${baseSymbol}` });
+ if (!Array.isArray(rows) || !rows.length) return null;
+ const candles = sortCandles(
+ rows.map(([tsMs, close]) => ({ t: Math.floor(tsMs / 1000), o: close, h: close, l: close, c: close, v: 0 })),
+ );
+ return candles.length ? { candles, base: { symbol: String(baseSymbol).toUpperCase() }, quote: { symbol: 'USD' }, source: 'cex', closeOnly: true } : null;
+}
+
+async function fetchOhlcvLive(args) {
+ let firstErr = null;
+ try {
+ return await geckoCandles(args);
+ } catch (e) {
+ firstErr = e;
+ }
+ for (const rung of [birdeyePairCandles, cexCandles]) {
+ try {
+ const out = await rung(args);
+ if (out) return out;
+ } catch (e) {
+ if (!firstErr) firstErr = e;
+ }
+ }
+ throw firstErr || tagged('no OHLCV source answered', 502);
+}
+
+// Fetch OHLCV candles for a pool, returned oldest -> newest (chronological), plus
+// the base/quote token metadata GeckoTerminal includes in the same response.
+// timeframe: 'minute' | 'hour' | 'day' aggregate: bars per candle limit: <=1000
+// baseSymbol: optional base-asset ticker (e.g. 'SOL') that unlocks the CEX rung
+// Result carries `source` ('geckoterminal' | 'birdeye' | 'cex') and, for the
+// CEX rung, `closeOnly: true`.
+export async function fetchOhlcv({
+ pool,
+ network = 'solana',
+ timeframe = 'hour',
+ aggregate = 1,
+ limit = 1000,
+ baseSymbol = null,
+}) {
+ if (!pool) throw tagged('pool is required', 400);
+ const args = { pool, network, timeframe, aggregate, limit, baseSymbol };
+ const key = `ohlcv:candles:v1:${network}:${pool}:${timeframe}:${aggregate}:${limit}`;
+ const out = await cacheWrapLastGood(key, LKG_FRESH_S, () => fetchOhlcvLive(args), {
+ staleTtlSeconds: LKG_STALE_S,
+ });
return {
- candles,
- base: tokenOf(meta.base),
- quote: tokenOf(meta.quote),
+ ...out,
freq: freqFor(timeframe, aggregate),
timeframe,
aggregate,
diff --git a/api/_lib/oracle/alerts.js b/api/_lib/oracle/alerts.js
index d019364ae..e145be6c3 100644
--- a/api/_lib/oracle/alerts.js
+++ b/api/_lib/oracle/alerts.js
@@ -25,6 +25,7 @@
import { sql } from '../db.js';
import { gmgnTokenUrl, axiomTokenUrl } from '../../../src/shared/trading-terminals.js';
+import { fetchUpstream } from '../upstream-fetch.js';
const ALERT_TIMEOUT_MS = 4000;
// Minimum tier to alert on. 'prime' only = exclusive. 'strong' = more volume.
const MIN_ALERT_TIER = process.env.ORACLE_ALERT_MIN_TIER || 'strong';
@@ -96,7 +97,7 @@ async function send(text) {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), ALERT_TIMEOUT_MS);
try {
- await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+ await fetchUpstream(`https://api.telegram.org/bot${token}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -106,7 +107,7 @@ async function send(text) {
disable_web_page_preview: true,
}),
signal: ctrl.signal,
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
} catch { /* fire-and-forget */ } finally {
clearTimeout(timer);
}
@@ -379,7 +380,7 @@ async function sendTo(chatId, text) {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), ALERT_TIMEOUT_MS);
try {
- await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+ await fetchUpstream(`https://api.telegram.org/bot${token}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -389,7 +390,7 @@ async function sendTo(chatId, text) {
disable_web_page_preview: true,
}),
signal: ctrl.signal,
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
} catch { /* fire-and-forget */ } finally {
clearTimeout(timer);
}
diff --git a/api/_lib/persona-store.js b/api/_lib/persona-store.js
index a1de40826..8d8f223e5 100644
--- a/api/_lib/persona-store.js
+++ b/api/_lib/persona-store.js
@@ -28,6 +28,7 @@ import path from 'node:path';
import { sql } from './db.js';
import { databaseConfigured } from './env.js';
import { putObject, copyObject, publicUrl } from './r2.js';
+import { fetchUpstream } from './upstream-fetch.js';
const dbConfigured = () => databaseConfigured();
const r2Configured = () => !!(process.env.S3_BUCKET && process.env.S3_PUBLIC_DOMAIN);
@@ -124,7 +125,7 @@ export async function persistPersonaGlb(sourceUrl, personaId, buffer = null) {
await putObject({ key, body: buffer, contentType: 'model/gltf-binary', metadata: { kind: 'persona', persona_id: personaId } });
} else if (/^https?:\/\//i.test(sourceUrl)) {
// Fetch + put (copyObject only works for keys already in our bucket).
- const resp = await fetch(sourceUrl);
+ const resp = await fetchUpstream(sourceUrl, {}, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!resp.ok) throw new Error(`fetch ${resp.status}`);
const body = Buffer.from(await resp.arrayBuffer());
await putObject({ key, body, contentType: 'model/gltf-binary', metadata: { kind: 'persona', persona_id: personaId } });
diff --git a/api/_lib/persona-wallet.js b/api/_lib/persona-wallet.js
index ea58734ef..d10bdc154 100644
--- a/api/_lib/persona-wallet.js
+++ b/api/_lib/persona-wallet.js
@@ -42,6 +42,7 @@ import { valuateHoldings } from './portfolio.js';
import { solUsdPrice, explorerAccountUrl, explorerTxUrl } from './avatar-wallet.js';
import { solanaReputation } from '../_mcp/tools/solana.js';
import { checkPersonaSpend, recordPersonaSpend, defaultSessionId, PERSONA_SPEND_CAPS } from './persona-spend-ledger.js';
+import { fetchUpstreamJson } from './upstream-fetch.js';
const HMAC_DOMAIN = 'three.ws:persona-wallet:v1:';
const SNS_API = 'https://sns-api.bonfida.com';
@@ -170,13 +171,10 @@ export async function getPersonaHoldings(address, network = 'mainnet') {
*/
export async function getPersonaNameplate(address) {
try {
- const r = await fetch(`${SNS_API}/v2/user/fav-domains/${address}`);
- if (r.ok) {
- const body = await r.json();
- const fav = body?.[address] || body?.data?.[address] || null;
- const name = typeof fav === 'string' ? fav : fav?.domain || fav?.name || null;
- if (name) return { name: name.endsWith('.sol') ? name : `${name}.sol`, verified: true, source: 'sns' };
- }
+ const body = await fetchUpstreamJson(`${SNS_API}/v2/user/fav-domains/${address}`, {}, { name: 'bonfida-sns', timeoutMs: 5_000, attempts: 2 });
+ const fav = body?.[address] || body?.data?.[address] || null;
+ const name = typeof fav === 'string' ? fav : fav?.domain || fav?.name || null;
+ if (name) return { name: name.endsWith('.sol') ? name : `${name}.sol`, verified: true, source: 'sns' };
} catch { /* best effort */ }
return { name: null, verified: false, source: 'sns' };
}
diff --git a/api/_lib/privy.js b/api/_lib/privy.js
index cf02bfab0..6b2e17fb2 100644
--- a/api/_lib/privy.js
+++ b/api/_lib/privy.js
@@ -11,6 +11,7 @@
import { env } from './env.js';
+import { fetchUpstream } from './upstream-fetch.js';
const PRIVY_API_BASE = 'https://auth.privy.io/api/v1';
function authHeaders() {
@@ -33,7 +34,7 @@ function authHeaders() {
export async function fetchPrivyUser(did) {
const headers = authHeaders();
if (!headers || !did) return null;
- const res = await fetch(`${PRIVY_API_BASE}/users/${encodeURIComponent(did)}`, { headers });
+ const res = await fetchUpstream(`${PRIVY_API_BASE}/users/${encodeURIComponent(did)}`, { headers }, { name: 'privy', timeoutMs: 8_000, attempts: 2, okWhen: () => true });
if (!res.ok) return null;
return res.json();
}
diff --git a/api/_lib/pump-claims-push.js b/api/_lib/pump-claims-push.js
index 34d39b37c..50a209f98 100644
--- a/api/_lib/pump-claims-push.js
+++ b/api/_lib/pump-claims-push.js
@@ -32,6 +32,7 @@
import { sql } from './db.js';
import { scanFirstClaims } from './pump-claims.js';
+import { fetchUpstream } from './upstream-fetch.js';
const TELEGRAM_LIMIT = 10; // per run; Bot API allows ~20 msg/min per chat
const TELEGRAM_PACE_MS = 3500;
// How far back each tick scans. The cron runs every 5 minutes, so a 30-minute
@@ -174,7 +175,7 @@ export function formatTelegramMessage(claim) {
}
async function sendTelegram(botToken, chatId, text) {
- const res = await fetch(`https://api.telegram.org/bot${botToken}/sendMessage`, {
+ const res = await fetchUpstream(`https://api.telegram.org/bot${botToken}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -186,7 +187,7 @@ async function sendTelegram(botToken, chatId, text) {
link_preview_options: { is_disabled: true },
}),
signal: AbortSignal.timeout(TELEGRAM_TIMEOUT_MS),
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const body = await res.json().catch(() => ({}));
if (!res.ok || !body.ok) {
throw new Error(
diff --git a/api/_lib/pump-curve-view.js b/api/_lib/pump-curve-view.js
index 576fa4b12..c506e5752 100644
--- a/api/_lib/pump-curve-view.js
+++ b/api/_lib/pump-curve-view.js
@@ -14,6 +14,7 @@
import { rpcFallbackFromEnv, getBondingCurveState, getTokenPrice, getGraduationProgress } from './solana/index.js';
import { createCache } from './mem-cache.js';
+import { cacheGet, cacheSet } from './cache.js';
import { hasThreeWsMark } from '../../src/solana/vanity/brand.js';
// Mints that can never carry a pump.fun bonding curve. These are coin-agnostic
@@ -120,15 +121,33 @@ async function isRegisteredPlatformLaunch(mint, network) {
// entry must remain readable to serve degraded data.
const _lastGood = createCache({ max: 256 }); // `${network}:${mint}` → { body, at }
const STALE_MAX_MS = 10 * 60_000;
+// The shared tier outlives a single instance: a cold Cloud Run instance used to
+// hold no memory copy at all, so the first read during an RPC outage answered
+// 502 no matter how recently a warm sibling had served the same mint.
+const SHARED_LKG_TTL_S = 24 * 3600;
+
+function lkgKey(network, mint) {
+ return `pump:curve:lkg:${network}:${mint}`;
+}
function rememberGood(network, mint, body) {
- _lastGood.set(`${network}:${mint}`, { body, at: Date.now() });
+ const entry = { body, at: Date.now() };
+ _lastGood.set(`${network}:${mint}`, entry);
+ // Fire-and-forget: cacheSet never rejects, and the response is already in hand.
+ cacheSet(lkgKey(network, mint), entry, SHARED_LKG_TTL_S).catch(() => {});
}
-function recallGood(network, mint) {
+async function recallGood(network, mint) {
const hit = _lastGood.get(`${network}:${mint}`);
- if (!hit || Date.now() - hit.at > STALE_MAX_MS) return null;
- return hit;
+ if (hit && Date.now() - hit.at <= STALE_MAX_MS) return hit;
+ let shared = null;
+ try {
+ shared = await cacheGet(lkgKey(network, mint));
+ } catch {
+ shared = null;
+ }
+ if (shared && shared.body && typeof shared.at === 'number') return shared;
+ return hit || null;
}
async function jupiterPriceFallback(mint) {
@@ -205,12 +224,12 @@ export async function getCurveView({ mint, network = 'mainnet' }) {
// for this mint marked stale, else a well-formed 502 the clients' existing
// error states already render.
console.error('[pump-curve-view] all RPC lanes failed', err?.message);
- const stale = recallGood(network, mint);
+ const stale = await recallGood(network, mint);
if (stale) {
return {
httpStatus: 200,
cacheControl: 'public, max-age=10, s-maxage=10',
- body: { ...stale.body, stale: true, as_of: Math.floor(stale.at / 1000) },
+ body: { ...stale.body, stale: true, as_of: new Date(stale.at).toISOString(), as_of_unix: Math.floor(stale.at / 1000) },
};
}
return {
diff --git a/api/_lib/pump-launch.js b/api/_lib/pump-launch.js
index 1c3b38e9d..3931dace4 100644
--- a/api/_lib/pump-launch.js
+++ b/api/_lib/pump-launch.js
@@ -28,7 +28,13 @@ import {
} from './vanity-inventory-store.js';
import { openSecret } from './vanity-vault.js';
+import { fetchUpstream } from './upstream-fetch.js';
+import { pinToIPFS, pinJsonToIPFS, ipfsPinningConfigured } from './ipfs-pin.js';
+
const PUMP_IPFS_ENDPOINT = 'https://pump.fun/api/ipfs';
+// pump.fun's pinning endpoint is a third party we cannot fail over inside; a
+// stalled socket there used to hold the launch until Cloud Run killed it.
+const PUMP_IPFS_TIMEOUT_MS = 30_000;
// Image guardrails for the upload path. pump.fun renders square images up to a
// few hundred KB; we cap fetch size so a hostile imageUrl can't exhaust the
@@ -148,28 +154,95 @@ export async function uploadPumpMetadata({
form.append('website', website);
form.append('showName', showName ? 'true' : 'false');
- let pinRes;
+ let pinRes = null;
+ let pumpFailure = null;
try {
- pinRes = await fetch(PUMP_IPFS_ENDPOINT, { method: 'POST', body: form });
+ pinRes = await fetchUpstream(
+ PUMP_IPFS_ENDPOINT,
+ { method: 'POST', body: form },
+ { name: 'pump-ipfs', timeoutMs: PUMP_IPFS_TIMEOUT_MS, attempts: 2, okWhen: () => true },
+ );
} catch (err) {
- throw launchError(502, 'ipfs_upload_failed', `pump.fun IPFS upload failed: ${err.message}`);
+ pumpFailure = `pump.fun IPFS upload failed: ${err.message}`;
}
- if (!pinRes.ok) {
+ if (pinRes && !pinRes.ok) {
const detail = await pinRes.text().catch(() => '');
- throw launchError(
- 502,
- 'ipfs_upload_failed',
- `pump.fun IPFS upload returned ${pinRes.status}${detail ? `: ${detail.slice(0, 200)}` : ''}`,
- );
+ pumpFailure = `pump.fun IPFS upload returned ${pinRes.status}${detail ? `: ${detail.slice(0, 200)}` : ''}`;
+ }
+ let out = null;
+ if (!pumpFailure) {
+ out = await pinRes.json().catch(() => null);
+ if (!(out?.metadataUri || out?.metadata_uri || out?.uri)) {
+ pumpFailure = 'pump.fun IPFS response had no metadataUri';
+ }
}
- const out = await pinRes.json().catch(() => null);
- const metadataUri = out?.metadataUri || out?.metadata_uri || out?.uri;
- if (!metadataUri) {
- throw launchError(502, 'ipfs_upload_failed', 'pump.fun IPFS response had no metadataUri');
+ if (pumpFailure) {
+ return pinMetadataViaProviderChain({
+ bytes,
+ mime,
+ ext,
+ name,
+ symbol,
+ description,
+ twitter,
+ telegram,
+ website,
+ showName,
+ pumpFailure,
+ });
}
+ const metadataUri = out.metadataUri || out.metadata_uri || out.uri;
return { metadataUri, image: out?.metadata?.image || out?.image || null };
}
+/**
+ * Fallback for uploadPumpMetadata when pump.fun's own IPFS endpoint is down:
+ * pin the image and then a pump.fun-shaped metadata document through the
+ * platform's provider chain (api/_lib/ipfs-pin.js: Pinata, then web3.storage).
+ * The returned gateway URI is consumed exactly like pump's `metadataUri`: it
+ * becomes the on-chain `uri` of the create instruction, and wallets resolve
+ * it to the same {name, symbol, description, image, ...} document pump's
+ * frontend writes.
+ */
+async function pinMetadataViaProviderChain({
+ bytes,
+ mime,
+ ext,
+ name,
+ symbol,
+ description,
+ twitter,
+ telegram,
+ website,
+ showName,
+ pumpFailure,
+}) {
+ if (!ipfsPinningConfigured()) {
+ throw launchError(502, 'ipfs_upload_failed', `${pumpFailure}; no fallback IPFS provider is configured`);
+ }
+ console.warn(`[pump-launch] ${pumpFailure}; pinning metadata through the provider chain instead`);
+ let image;
+ let metadata;
+ try {
+ image = await pinToIPFS(bytes, `image.${ext}`);
+ const doc = {
+ name,
+ symbol,
+ description,
+ image: image.uri,
+ showName: Boolean(showName),
+ createdOn: 'https://pump.fun',
+ };
+ if (twitter) doc.twitter = twitter;
+ if (telegram) doc.telegram = telegram;
+ if (website) doc.website = website;
+ metadata = await pinJsonToIPFS(doc, 'metadata.json');
+ } catch (err) {
+ throw launchError(502, 'ipfs_upload_failed', `${pumpFailure}; fallback IPFS pin failed: ${err.message}`);
+ }
+ return { metadataUri: metadata.uri, image: image.uri, mime, provider: metadata.provider };
+}
+
/**
* Create a pump.fun token (bonding-curve only — no dev buy) signed + paid by
* the launcher keypair. Optionally grinds a vanity mint address.
diff --git a/api/_lib/pump.js b/api/_lib/pump.js
index 1851c236b..0c98463cd 100644
--- a/api/_lib/pump.js
+++ b/api/_lib/pump.js
@@ -21,6 +21,7 @@ import { PublicKey } from '@solana/web3.js';
import { solanaConnection } from './solana/connection.js';
import { rpcFallbackFromEnv } from './solana/rpc-fallback.js';
import { PUMP_PROGRAM_ID, PUMP_AGENT_PAYMENTS_PROGRAM_ID } from './solana/programs.js';
+import { isRpcOutageError, rpcUnavailableError } from './rpc-degrade.js';
// @solana/web3.js calls console.error() on every 429 retry attempt. Those
// retries succeed (callers see a resolved value, not a thrown error), but
@@ -293,11 +294,18 @@ export async function getAmmPoolState({ network = 'mainnet', mint, quoteMint = n
throw e;
}
- const [baseAccInfo, quoteAccInfo, baseMintInfo] = await Promise.all([
- connection.getAccountInfo(pool.poolBaseTokenAccount),
- connection.getAccountInfo(pool.poolQuoteTokenAccount),
- connection.getAccountInfo(pool.baseMint),
- ]);
+ let baseAccInfo, quoteAccInfo, baseMintInfo;
+ try {
+ [baseAccInfo, quoteAccInfo, baseMintInfo] = await Promise.all([
+ connection.getAccountInfo(pool.poolBaseTokenAccount),
+ connection.getAccountInfo(pool.poolQuoteTokenAccount),
+ connection.getAccountInfo(pool.baseMint),
+ ]);
+ } catch (err) {
+ // A dead RPC lane is retryable and says nothing about the pool.
+ if (isRpcOutageError(err)) throw rpcUnavailableError('pool reserves could not be read: Solana RPC is temporarily unavailable', err);
+ throw err;
+ }
if (!baseAccInfo || !quoteAccInfo || !baseMintInfo) {
const e = new Error('pool token accounts unavailable');
e.status = 502;
@@ -333,12 +341,18 @@ export async function getAmmPoolState({ network = 'mainnet', mint, quoteMint = n
const effectiveQuoteReserve = quoteReserve.add(virtualQuoteReserves);
const offline = new PumpAmmSdk();
- const [globalConfigInfo, feeConfigInfo] = await Promise.all([
- connection.getAccountInfo((await import('@pump-fun/pump-swap-sdk')).GLOBAL_CONFIG_PDA),
- connection.getAccountInfo(
- (await import('@pump-fun/pump-swap-sdk')).PUMP_AMM_FEE_CONFIG_PDA,
- ),
- ]);
+ let globalConfigInfo, feeConfigInfo;
+ try {
+ [globalConfigInfo, feeConfigInfo] = await Promise.all([
+ connection.getAccountInfo((await import('@pump-fun/pump-swap-sdk')).GLOBAL_CONFIG_PDA),
+ connection.getAccountInfo(
+ (await import('@pump-fun/pump-swap-sdk')).PUMP_AMM_FEE_CONFIG_PDA,
+ ),
+ ]);
+ } catch (err) {
+ if (isRpcOutageError(err)) throw rpcUnavailableError('pool fee config could not be read: Solana RPC is temporarily unavailable', err);
+ throw err;
+ }
const globalConfig = globalConfigInfo ? offline.decodeGlobalConfig(globalConfigInfo) : null;
const feeConfig = feeConfigInfo ? offline.decodeFeeConfig(feeConfigInfo) : null;
@@ -366,7 +380,13 @@ export async function buildUnsignedTxBase64({ network, payer, instructions }) {
import('@solana/web3.js'),
]);
const connection = getConnection({ network });
- const { blockhash } = await connection.getLatestBlockhash('confirmed');
+ let blockhash;
+ try {
+ ({ blockhash } = await connection.getLatestBlockhash('confirmed'));
+ } catch (err) {
+ if (isRpcOutageError(err)) throw rpcUnavailableError('could not fetch a recent blockhash: Solana RPC is temporarily unavailable', err);
+ throw err;
+ }
const payerPk = payer instanceof PK ? payer : new PK(payer);
const msg = new TransactionMessage({
payerKey: payerPk,
diff --git a/api/_lib/pumpfun-mcp.js b/api/_lib/pumpfun-mcp.js
index 0f6b78324..f54d0c90b 100644
--- a/api/_lib/pumpfun-mcp.js
+++ b/api/_lib/pumpfun-mcp.js
@@ -18,6 +18,7 @@
import { env } from './env.js';
import { getRedis } from './redis.js';
+import { fetchUpstream } from './upstream-fetch.js';
const LIST_KEY = process.env.GRADUATIONS_LIST_KEY || 'pf:graduations';
@@ -81,11 +82,16 @@ async function jsonrpc(toolName, args) {
const token = process.env.PUMPFUN_BOT_TOKEN;
const headers = { 'content-type': 'application/json' };
if (token) headers.authorization = `Bearer ${token}`;
- const res = await fetch(url, {
- method: 'POST',
- headers,
- body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: toolName, arguments: args } }),
- });
+ let res;
+ try {
+ res = await fetchUpstream(url, {
+ method: 'POST',
+ headers,
+ body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'tools/call', params: { name: toolName, arguments: args } }),
+ }, { name: 'pumpfun-bot', timeoutMs: 15_000, attempts: 1, okWhen: () => true });
+ } catch (err) {
+ return { ok: false, error: err?.message || 'bot unreachable' };
+ }
if (!res.ok) return { ok: false, error: `HTTP ${res.status}` };
const j = await res.json();
if (j.error) return { ok: false, error: j.error.message || JSON.stringify(j.error) };
diff --git a/api/_lib/rate-limit.js b/api/_lib/rate-limit.js
index 66a8d298f..bc6348464 100644
--- a/api/_lib/rate-limit.js
+++ b/api/_lib/rate-limit.js
@@ -1382,6 +1382,12 @@ export const limits = {
solanaRpcIp: (ip) => getLimiter('solana-rpc:ip', { limit: 120, window: '1 m' }).limit(ip),
solanaRpcGlobal: () =>
getLimiter('solana-rpc:global', { limit: 12000, window: '1 h' }).limit('global'),
+ // Browser EVM JSON-RPC proxy (api/evm-rpc). Same shape as the Solana proxy:
+ // the chain may front a keyed Alchemy upstream, so cap per-IP burst and add a
+ // global hourly ceiling as the hard cost cap.
+ evmRpcIp: (ip) => getLimiter('evm-rpc:ip', { limit: 120, window: '1 m' }).limit(ip),
+ evmRpcGlobal: () =>
+ getLimiter('evm-rpc:global', { limit: 12000, window: '1 h' }).limit('global'),
// Helius DAS / enhanced-API endpoints (nft/resolve getAsset, tx/explain
// enhanced-tx, live holder cohorts getTokenAccounts). DAS is billed at a far
// higher credit multiplier than plain RPC, and these are public. Per-endpoint
diff --git a/api/_lib/resilience.js b/api/_lib/resilience.js
index e0eb7fe73..439e5cf19 100644
--- a/api/_lib/resilience.js
+++ b/api/_lib/resilience.js
@@ -176,3 +176,4 @@ export function withRetryAndBreaker(name, fn, opts = {}) {
export function _resetBreakers() {
_breakers.clear();
}
+
diff --git a/api/_lib/rig-inspect.js b/api/_lib/rig-inspect.js
index 1b4a16189..601935f29 100644
--- a/api/_lib/rig-inspect.js
+++ b/api/_lib/rig-inspect.js
@@ -11,6 +11,7 @@
import { getObjectRange } from './r2.js';
import { inspectGlb, glbJsonChunkEnd } from './glb-inspect.js';
+import { fetchUpstream } from './upstream-fetch.js';
// 256 KB covers the JSON chunk of essentially every real avatar in one read.
const INITIAL_PREFIX = 256 * 1024;
@@ -22,7 +23,7 @@ async function readGlbPrefix(storageKey, length) {
if (/^https?:\/\//i.test(storageKey)) {
// Externally-hosted / first-party absolute URLs (e.g. the built-in sample
// avatars at three.ws/avatars/*.glb) aren't bucket objects — range-fetch.
- const r = await fetch(storageKey, { headers: { Range: `bytes=0-${length - 1}` } });
+ const r = await fetchUpstream(storageKey, { headers: { Range: `bytes=0-${length - 1}` } }, { timeoutMs: 60_000, attempts: 3, okWhen: () => true });
if (!r.ok) throw new Error(`http ${r.status}`);
return Buffer.from(await r.arrayBuffer());
}
diff --git a/api/_lib/robinhood.js b/api/_lib/robinhood.js
index 7d64c9f09..4b6c848e4 100644
--- a/api/_lib/robinhood.js
+++ b/api/_lib/robinhood.js
@@ -16,13 +16,15 @@
// • memecoin screener — CoinGecko categories + DexScreener
// • launchpad activity — on-chain eth_getLogs (NOXA / Odyssey factories)
//
-// Failover: public RPC → Alchemy (when ROBINHOOD_ALCHEMY_KEY is set). All
+// Failover: Alchemy (when ROBINHOOD_ALCHEMY_KEY is set) → RPC_URL_4663 override
+// → the public sequencer RPC, as one viem fallback transport with per-request
+// retries, so no single vendor endpoint is the board's only lane. All
// upstreams are wrapped in short-TTL caches so the board never fans out 95 RPC
// calls per request (the Chainlink snapshot is one multicall behind a 20s TTL).
import { readFileSync } from 'node:fs';
import path from 'node:path';
-import { createPublicClient, http } from 'viem';
+import { createPublicClient, fallback, http } from 'viem';
import { cacheWrap } from './cache.js';
// ── Chain definitions (viem 2.52 predates the official robinhood chain defs) ──
@@ -63,17 +65,31 @@ export const USDG_ADDRESS = '0x5fc5360D0400a0Fd4f2af552ADD042D716F1d168';
export const FEED_DECIMALS = 8;
export const STOCK_TOKEN_DECIMALS = 18;
-// Resolve the best RPC URL: Alchemy when a key is present (recommended paid
-// RPC), else the public sequencer RPC. Same env-at-module-load contract every
-// other provider on the platform uses.
-function rpcUrl(testnet) {
+// Priority-ordered RPC URLs: Alchemy when a key is present (recommended paid
+// RPC), then an explicit RPC_URL_<chainId> override, then the public sequencer
+// RPC. The keyed vendor stays primary; the public node is the safety net behind
+// it rather than the only lane. Same env-at-module-load contract every other
+// provider on the platform uses.
+export function rpcUrls(testnet) {
+ const chain = testnet ? HOOD_TESTNET : HOOD_MAINNET;
const key = process.env.ROBINHOOD_ALCHEMY_KEY;
- if (key) {
- return testnet
+ const urls = [
+ key ? (testnet
? `https://robinhood-testnet.g.alchemy.com/v2/${key}`
- : `https://robinhood-mainnet.g.alchemy.com/v2/${key}`;
- }
- return (testnet ? HOOD_TESTNET : HOOD_MAINNET).rpcUrls.default.http[0];
+ : `https://robinhood-mainnet.g.alchemy.com/v2/${key}`) : null,
+ process.env[`RPC_URL_${chain.id}`] || null,
+ ...chain.rpcUrls.default.http,
+ ];
+ return urls.filter((u, i, a) => u && a.indexOf(u) === i);
+}
+
+// Per-endpoint timeout plus two backed-off retries on the same endpoint before
+// viem's fallback moves to the next one; `rank: false` keeps strict priority.
+function hoodTransport(testnet) {
+ const httpOpts = { batch: true, timeout: 12_000, retryCount: 2 };
+ const urls = rpcUrls(testnet);
+ if (urls.length === 1) return http(urls[0], httpOpts);
+ return fallback(urls.map((u) => http(u, httpOpts)), { rank: false });
}
let _clients = { mainnet: null, testnet: null };
@@ -85,7 +101,7 @@ export function publicClient(testnet = false) {
const chain = testnet ? HOOD_TESTNET : HOOD_MAINNET;
_clients[slot] = createPublicClient({
chain,
- transport: http(rpcUrl(testnet), { batch: true, timeout: 12_000 }),
+ transport: hoodTransport(testnet),
batch: { multicall: { wait: 16 } },
});
return _clients[slot];
diff --git a/api/_lib/royalty.js b/api/_lib/royalty.js
index d7917cfee..693c6183f 100644
--- a/api/_lib/royalty.js
+++ b/api/_lib/royalty.js
@@ -10,6 +10,7 @@ import { encodeFunctionData, parseAbi } from 'viem';
import { sql } from './db.js';
import { env } from './env.js';
import { EVM_USDC, toUsdcAtomics } from '../payments/_config.js';
+import { fetchUpstream } from './upstream-fetch.js';
// USDC transferFrom signature — the delegation manager calls this on the
// USDC contract on behalf of the agent's wallet. The delegation grants the
@@ -256,7 +257,8 @@ async function _redeemForGroup(group, authorUserId) {
const cronSecret = env.CRON_SECRET;
const issuer = env.ISSUER ?? 'http://localhost:3000';
- const resp = await fetch(`${issuer}/api/permissions/redeem`, {
+ // A redeem moves funds, so it is never retried: one attempt, bounded.
+ const resp = await fetchUpstream(`${issuer}/api/permissions/redeem`, {
method: 'POST',
headers: {
'content-type': 'application/json',
diff --git a/api/_lib/rpc-degrade.js b/api/_lib/rpc-degrade.js
new file mode 100644
index 000000000..1e28ae926
--- /dev/null
+++ b/api/_lib/rpc-degrade.js
@@ -0,0 +1,85 @@
+// Shared "the RPC chain is down" plumbing for public Solana read endpoints.
+//
+// During an RPC outage every read handler used to answer its own flavour of 5xx
+// (a bare 502, an unhandled throw, a hang past the gateway timeout). This module
+// gives them one vocabulary:
+// - isRpcOutageError(err) recognises transport/chain-exhaustion failures
+// - rpcUnavailableError() the typed retryable 503 those failures map to
+// - withDeadline(p, ms) a hard cap so a stalled lane fails fast enough to
+// fall back to a last-good value inside the request
+// - staleEnvelope(body, at) the `{ ...body, stale: true, as_of }` marker every
+// last-good tier serves at HTTP 200
+//
+// The actual last-good storage stays with the caller (api/_lib/cache.js keys are
+// endpoint-specific); this module only shapes the failure and the marker.
+
+// Solana RPC errors arrive as fetch failures, JSON-RPC envelopes, or the
+// rotating connection's "all N endpoints failed" summary. None carry an HTTP
+// status the resilience helper recognises, so match on the message families.
+const OUTAGE_RE =
+ /fetch failed|failed to get|failed to fetch|all .* endpoints failed|rpc_unavailable|\b429\b|too many requests|rate.?limit|max usage|\b50[234]\b|timed? ?out|aborted|ECONNRESET|ECONNREFUSED|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|socket hang up/i;
+
+export const RPC_RETRY_AFTER_S = 15;
+
+/**
+ * True when `err` describes an RPC lane being unreachable, throttled, or
+ * exhausted, as opposed to a real on-chain answer (account missing, bad input).
+ * @param {unknown} err
+ */
+export function isRpcOutageError(err) {
+ if (!err) return false;
+ if (err.code === 'rpc_unavailable' || err.code === 'rpc_error' || err.code === 'rpc_rate_limited') return true;
+ if (err.name === 'AbortError' || err.name === 'TimeoutError') return true;
+ const status = err.status ?? err.statusCode;
+ if (status === 429 || status === 502 || status === 503 || status === 504) return true;
+ const causeCode = err.cause?.code;
+ if (typeof causeCode === 'string' && /^(ECONN|ETIMEDOUT|EAI_AGAIN|ENOTFOUND|UND_ERR)/.test(causeCode)) return true;
+ return OUTAGE_RE.test(String(err.message || err));
+}
+
+/**
+ * A typed, retryable 503 the http helpers already know how to render. Callers
+ * set `Retry-After: RPC_RETRY_AFTER_S` alongside it.
+ * @param {string} [message]
+ * @param {unknown} [cause]
+ */
+export function rpcUnavailableError(message = 'Solana RPC is temporarily unavailable, retry shortly', cause) {
+ const e = new Error(message);
+ e.status = 503;
+ e.code = 'rpc_unavailable';
+ e.retryAfter = RPC_RETRY_AFTER_S;
+ if (cause !== undefined) e.cause = cause;
+ return e;
+}
+
+/**
+ * Race `promise` against a hard deadline. On timeout rejects with a typed
+ * rpc_unavailable error whose message names `label`, so the caller falls into
+ * its last-good branch instead of hanging until the gateway cuts the request.
+ * The timer is always cleared, so a resolved promise never keeps the loop alive.
+ *
+ * @template T
+ * @param {Promise<T>} promise
+ * @param {number} ms
+ * @param {string} [label]
+ * @returns {Promise<T>}
+ */
+export function withDeadline(promise, ms, label = 'rpc read') {
+ let timer;
+ const deadline = new Promise((_, reject) => {
+ timer = setTimeout(() => reject(rpcUnavailableError(`${label} exceeded ${ms}ms deadline`)), ms);
+ });
+ return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
+}
+
+/**
+ * Mark a last-good body as stale. `at` is the epoch-ms the value was captured;
+ * `as_of` is ISO so a UI can say "as of N minutes ago" without guessing units.
+ * @template {object} T
+ * @param {T} body
+ * @param {number} at
+ * @returns {T & { stale: true, as_of: string }}
+ */
+export function staleEnvelope(body, at) {
+ return { ...body, stale: true, as_of: new Date(at).toISOString() };
+}
diff --git a/api/_lib/saml.js b/api/_lib/saml.js
index 255464c45..f5a9123c8 100644
--- a/api/_lib/saml.js
+++ b/api/_lib/saml.js
@@ -18,6 +18,7 @@ import { XMLParser } from 'fast-xml-parser';
import { env } from './env.js';
import { sql } from './db.js';
import { hmacSha256, constantTimeEquals, sha256 } from './crypto.js';
+import { fetchUpstream } from './upstream-fetch.js';
// Request IDs are valid for one login round-trip; an hour is generous slack for
// a user who lands on the IdP, authenticates, and is redirected back. Must match
@@ -73,9 +74,9 @@ async function resolveIdpConfig() {
async function fetchAndParseMetadata(url) {
try {
- const res = await fetch(url, {
+ const res = await fetchUpstream(url, {
headers: { accept: 'application/samlmetadata+xml, application/xml, text/xml' },
- });
+ }, { name: 'saml-idp-metadata', timeoutMs: 10_000, attempts: 3, okWhen: () => true });
if (!res.ok) return null;
return parseIdpMetadata(await res.text());
} catch {
diff --git a/api/_lib/seed-quality.js b/api/_lib/seed-quality.js
index 70b8ca213..06f2e18cf 100644
--- a/api/_lib/seed-quality.js
+++ b/api/_lib/seed-quality.js
@@ -31,6 +31,7 @@
import { judgeOnce, JUDGE_MODEL, RENDER_BACKGROUND } from './quality-bench.js';
import { vertexGeminiAvailable, vertexGeminiChatUrl, vertexGeminiHeaders } from './vertex-gemini.js';
+import { fetchUpstream } from './upstream-fetch.js';
import { copyObject, putObject } from './r2.js';
// Stage 1 lives in its own module because it is the only part of this gate that
@@ -133,7 +134,7 @@ async function rigReadinessVertex({ png, prompt, category }) {
},
],
};
- const res = await fetch(vertexGeminiChatUrl(), { method: 'POST', headers, body: JSON.stringify(body) });
+ const res = await fetchUpstream(vertexGeminiChatUrl(), { method: 'POST', headers, body: JSON.stringify(body) }, { name: 'vertex-gemini', timeoutMs: 45_000, attempts: 1, okWhen: () => true });
const data = await res.json().catch(() => ({}));
if (!res.ok) throw new Error(`rig-readiness call ${res.status}: ${JSON.stringify(data).slice(0, 300)}`);
return normalizeRigReadiness(parseJsonReply(data?.choices?.[0]?.message?.content || ''));
diff --git a/api/_lib/siwx-server.js b/api/_lib/siwx-server.js
index 9ef149adb..809be6f84 100644
--- a/api/_lib/siwx-server.js
+++ b/api/_lib/siwx-server.js
@@ -22,8 +22,9 @@ import {
SIGN_IN_WITH_X,
} from '@x402/extensions/sign-in-with-x';
-import { createPublicClient, http } from 'viem';
+import { createPublicClient } from 'viem';
import { base } from 'viem/chains';
+import { evmTransport } from './evm/rpc.js';
import { env } from './env.js';
import { siwxStorage, normalizeAddress } from './siwx-storage.js';
@@ -34,16 +35,19 @@ export { SIGN_IN_WITH_X };
// Lazily build a viem PublicClient so verifySIWxSignature can verify
// smart-contract wallets (EIP-1271) and counterfactual wallets (EIP-6492).
-// Uses our private RPC (env.BASE_RPC_URL) to avoid leaking buyer addresses
-// to a public node. Returns undefined when BASE_RPC_URL isn't configured —
-// the upstream verifier then falls back to EOA-only verification.
+// Uses our private RPC (env.BASE_RPC_URL) first so buyer addresses are not
+// broadcast to a public node on the happy path; the shared Base endpoint chain
+// only takes over when the private node fails, so it can never be the single
+// point that blocks every smart-wallet sign-in. Returns undefined when
+// BASE_RPC_URL isn't configured; the upstream verifier then falls back to
+// EOA-only verification.
let _baseClient;
function getEvmVerifier() {
if (!env.BASE_RPC_URL) return undefined;
if (!_baseClient) {
_baseClient = createPublicClient({
chain: base,
- transport: http(env.BASE_RPC_URL),
+ transport: evmTransport(base.id, { primaryUrl: env.BASE_RPC_URL }),
});
}
return _baseClient.verifyMessage.bind(_baseClient);
diff --git a/api/_lib/solana-token-meta.js b/api/_lib/solana-token-meta.js
index a843c7db9..f0500c407 100644
--- a/api/_lib/solana-token-meta.js
+++ b/api/_lib/solana-token-meta.js
@@ -15,6 +15,7 @@ import {
getTokenMetadata,
} from '@solana/spl-token';
import { getConnection, solanaPubkey } from './pump.js';
+import { isRpcOutageError, rpcUnavailableError } from './rpc-degrade.js';
const METADATA_PROGRAM = new PublicKey('metaqbxxUerdq28cj1RbAWkYQm3ybzjb6a8bt518x1s');
const NAME_MAX = 32;
@@ -36,7 +37,15 @@ async function fetchOnchainMetadata(connection, mintPk) {
[Buffer.from('metadata'), METADATA_PROGRAM.toBuffer(), mintPk.toBuffer()],
METADATA_PROGRAM,
);
- const info = await connection.getAccountInfo(metadataPda);
+ // Metadata is decoration on top of the mint: an RPC blip here degrades the
+ // name/symbol/uri to null rather than failing the whole read.
+ let info;
+ try {
+ info = await connection.getAccountInfo(metadataPda);
+ } catch (err) {
+ console.warn(`[solana-token-meta] metadata PDA read failed for ${mintPk.toBase58()}: ${err?.message || err}`);
+ return null;
+ }
if (!info) return null;
// Layout: 1 key + 32 updateAuthority + 32 mint + name + symbol + uri.
@@ -70,7 +79,15 @@ async function fetchOnchainMetadata(connection, mintPk) {
*/
async function fetchToken2022Metadata(connection, mintPk, ownerProgram) {
if (ownerProgram !== TOKEN_2022_PROGRAM_ID.toBase58()) return null;
- const md = await getTokenMetadata(connection, mintPk, undefined, TOKEN_2022_PROGRAM_ID);
+ let md;
+ try {
+ md = await getTokenMetadata(connection, mintPk, undefined, TOKEN_2022_PROGRAM_ID);
+ } catch (err) {
+ // Same soft-null contract as the Metaplex reader: the caller still gets
+ // a token profile, just without the embedded name/symbol/uri.
+ console.warn(`[solana-token-meta] token-2022 metadata read failed for ${mintPk.toBase58()}: ${err?.message || err}`);
+ return null;
+ }
if (!md) return null;
return {
name: (md.name || '').trim(),
@@ -216,7 +233,15 @@ export async function fetchTokenMeta(mint, { network = 'mainnet', includeImage =
const connection = getConnection({ network });
- const accountInfo = await connection.getAccountInfo(pk);
+ // The mint read is the one hard dependency: without it "not found" cannot be
+ // told apart from "RPC down", so an outage answers a typed retryable 503.
+ let accountInfo;
+ try {
+ accountInfo = await connection.getAccountInfo(pk);
+ } catch (err) {
+ if (isRpcOutageError(err)) throw rpcUnavailableError(`could not read the mint on ${network}: Solana RPC is temporarily unavailable`, err);
+ throw err;
+ }
if (!accountInfo) {
const err = new Error(`mint account not found on ${network}`);
err.code = 'mint_not_found';
diff --git a/api/_lib/splits.js b/api/_lib/splits.js
index e8691ab82..cb37bad70 100644
--- a/api/_lib/splits.js
+++ b/api/_lib/splits.js
@@ -366,10 +366,11 @@ export async function createOnchainSplit({ recipients, chain, mutable = false })
}
if (!SplitsClient) return null;
- const [{ createPublicClient, http }, { privateKeyToAccount }, viemChains] = await Promise.all([
+ const [{ createPublicClient, createWalletClient }, { privateKeyToAccount }, viemChains, { evmTransport }] = await Promise.all([
import('viem'),
import('viem/accounts'),
import('viem/chains'),
+ import('./evm/rpc.js'),
]);
const chainMap = {
1: viemChains.mainnet, 8453: viemChains.base, 10: viemChains.optimism,
@@ -379,10 +380,11 @@ export async function createOnchainSplit({ recipients, chain, mutable = false })
if (!viemChain) return null;
const account = privateKeyToAccount(pk.startsWith('0x') ? pk : `0x${pk}`);
- const rpcUrl = process.env[`EVM_RPC_URL_${chainId}`] || undefined;
- const publicClient = createPublicClient({ chain: viemChain, transport: http(rpcUrl) });
- const { createWalletClient } = await import('viem');
- const walletClient = createWalletClient({ account, chain: viemChain, transport: http(rpcUrl) });
+ // The operator's EVM_RPC_URL_<chainId> stays primary; the shared keyed/public
+ // endpoints for the chain back it so one dead node cannot strand a split deploy.
+ const primaryUrl = process.env[`EVM_RPC_URL_${chainId}`] || null;
+ const publicClient = createPublicClient({ chain: viemChain, transport: evmTransport(chainId, { primaryUrl }) });
+ const walletClient = createWalletClient({ account, chain: viemChain, transport: evmTransport(chainId, { primaryUrl }) });
const client = new SplitsClient({ chainId, publicClient, walletClient }).splitV1;
diff --git a/api/_lib/three-cdn.js b/api/_lib/three-cdn.js
new file mode 100644
index 000000000..a41bf42d2
--- /dev/null
+++ b/api/_lib/three-cdn.js
@@ -0,0 +1,63 @@
+// Which CDN the headless renderers load three.js from.
+//
+// render-glb.js, avatar-render.js and render-clip.js build a page whose
+// import map points at a pinned three.js release. That page runs inside our
+// own chromium, so a CDN outage never surfaces as an error: the module import
+// hangs, the render watchdog fires, and every thumbnail and clip comes back
+// blank. Both unpkg and jsDelivr serve the identical npm tarball, so the fix
+// is to check unpkg once (bounded HEAD), remember the answer for ten minutes,
+// and hand the renderers whichever host is up.
+
+export const THREE_CDN_HOSTS = {
+ unpkg: (version) => `https://unpkg.com/three@${version}/`,
+ jsdelivr: (version) => `https://cdn.jsdelivr.net/npm/three@${version}/`,
+};
+
+const PROBE_TIMEOUT_MS = 3_000;
+const PROBE_TTL_MS = 10 * 60_000;
+
+// version -> { host, base, checkedAt }
+const probes = new Map();
+
+/** Import-map entries for a resolved base (ends in `/`). */
+export function threeImportMap(base) {
+ return {
+ three: `${base}build/three.module.js`,
+ 'three/addons/': `${base}examples/jsm/`,
+ };
+}
+
+/**
+ * Resolve the three.js CDN base for `version`: unpkg when its module answers a
+ * HEAD, jsDelivr otherwise. The probe result is cached per process for ten
+ * minutes so a render burst pays one HEAD, not one per frame.
+ *
+ * @param {string} version
+ * @param {{ fetchImpl?: typeof fetch, now?: () => number, force?: boolean }} [opts]
+ * @returns {Promise<{ host: 'unpkg'|'jsdelivr', base: string, cached: boolean }>}
+ */
+export async function resolveThreeCdn(version, { fetchImpl = globalThis.fetch, now = Date.now, force = false } = {}) {
+ const hit = probes.get(version);
+ if (hit && !force && now() - hit.checkedAt < PROBE_TTL_MS) return { host: hit.host, base: hit.base, cached: true };
+ const primary = THREE_CDN_HOSTS.unpkg(version);
+ let host = 'unpkg';
+ try {
+ const res = await fetchImpl(`${primary}build/three.module.js`, {
+ method: 'HEAD',
+ redirect: 'follow',
+ signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+ });
+ if (!res.ok) throw new Error(`http ${res.status}`);
+ } catch (err) {
+ host = 'jsdelivr';
+ console.warn(`[three-cdn] unpkg probe failed (${err?.message || err}); loading three@${version} from jsDelivr`);
+ }
+ const base = THREE_CDN_HOSTS[host](version);
+ probes.set(version, { host, base, checkedAt: now() });
+ return { host, base, cached: false };
+}
+
+/** Test hook: forget every cached probe. */
+export function _resetThreeCdnProbes() {
+ probes.clear();
+}
diff --git a/api/_lib/token-metadata.js b/api/_lib/token-metadata.js
index 0d7c0a9b8..02d76a014 100644
--- a/api/_lib/token-metadata.js
+++ b/api/_lib/token-metadata.js
@@ -18,6 +18,7 @@
import { sql, sqlValues } from './db.js';
+import { fetchUpstream } from './upstream-fetch.js';
const REFRESH_AFTER_MS = 1000 * 60 * 60 * 24 * 30; // 30 days
function heliusRpcUrl() {
@@ -98,7 +99,7 @@ async function resolveViaHelius(mints) {
const rpc = heliusRpcUrl();
if (!rpc || mints.length === 0) return [];
try {
- const r = await fetch(rpc, {
+ const r = await fetchUpstream(rpc, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -107,7 +108,7 @@ async function resolveViaHelius(mints) {
method: 'getAssetBatch',
params: { ids: mints },
}),
- });
+ }, { name: 'helius-das', timeoutMs: 8_000, attempts: 2, okWhen: () => true });
if (!r.ok) {
console.warn('[token-metadata] helius getAssetBatch failed:', r.status);
return [];
diff --git a/api/_lib/token/jupiter.js b/api/_lib/token/jupiter.js
index 4aa9cbd78..65a18e1b4 100644
--- a/api/_lib/token/jupiter.js
+++ b/api/_lib/token/jupiter.js
@@ -21,6 +21,16 @@
// upstream ignores a limit param, so trimming happens client-side).
// - /ultra/v1/shield: per-mint safety warnings.
+import { fetchUpstream } from '../upstream-fetch.js';
+import { withRetry } from '../resilience.js';
+
+// Deadlines per lane. A quote is a cheap GET that Jupiter answers in well under
+// a second, so 8s is generous; a swap build has to assemble a route and a
+// versioned transaction and legitimately takes longer under load.
+const JUP_QUOTE_TIMEOUT_MS = 8_000;
+const JUP_SWAP_TIMEOUT_MS = 15_000;
+const JUP_QUOTE_ATTEMPTS = 3;
+
const JUP_QUOTE_URL = 'https://lite-api.jup.ag/swap/v1/quote';
const JUP_SWAP_URL = 'https://lite-api.jup.ag/swap/v1/swap';
const JUP_TOKENS_URL = 'https://lite-api.jup.ag/tokens/v2';
@@ -35,8 +45,21 @@ export const JUP_TOKEN_INTERVALS = ['5m', '1h', '6h', '24h'];
* GET a Jupiter route + throw a coded error on a non-2xx / unparseable response,
* so a caller records the precise reason instead of an opaque failure.
*/
-export async function fetchJson(url, opts) {
- const r = await fetch(url, opts);
+export async function fetchJson(url, opts, { timeoutMs = JUP_QUOTE_TIMEOUT_MS, attempts = 2 } = {}) {
+ // Bounded by default (a caller signal still composes with the timeout) and
+ // retried on a transient failure. Jupiter is the only provider on the
+ // quote/swap path, so a hang here used to hang every paid swap.
+ const r = await fetchUpstream(url, opts || {}, {
+ name: 'jupiter',
+ timeoutMs,
+ attempts,
+ okWhen: (res) => res.ok || (res.status < 500 && res.status !== 429),
+ }).catch((err) => {
+ throw Object.assign(new Error(`jupiter ${err?.status || 'unreachable'}: ${err?.message || 'network error'}`), {
+ code: 'jupiter_error',
+ status: err?.status || 503,
+ });
+ });
const body = await r.json().catch(() => ({}));
if (!r.ok) {
throw Object.assign(new Error(`jupiter ${r.status}: ${JSON.stringify(body).slice(0, 200)}`), {
@@ -59,7 +82,22 @@ export async function jupiterQuote({ inputMint, outputMint, amount, slippageBps
u.searchParams.set('amount', String(amount));
u.searchParams.set('slippageBps', String(slippageBps));
u.searchParams.set('swapMode', 'ExactIn');
- return fetchJson(u.toString(), { headers: { accept: 'application/json' } });
+ // A quote is an idempotent GET: retry it through the transient statuses
+ // (429/5xx/network) so one rate-limited response never fails a paid swap.
+ return fetchJson(u.toString(), { headers: { accept: 'application/json' } }, {
+ timeoutMs: JUP_QUOTE_TIMEOUT_MS,
+ attempts: JUP_QUOTE_ATTEMPTS,
+ });
+}
+
+// A swap build is a POST, but it only returns an UNSIGNED transaction: nothing
+// is submitted on-chain by it, so repeating it is safe. It is still retried
+// exactly once and only when the failure says nothing about the request
+// itself (a network error, a timeout, a 5xx). A 4xx or a 429 is an answer and
+// is surfaced as-is. The signed transaction is broadcast by the caller, and
+// that step is never retried here.
+function isSwapBuildRetryable(err) {
+ return err?.code === 'jupiter_error' && Number(err.status) >= 500;
}
/**
@@ -75,20 +113,24 @@ export async function jupiterSwapTx({
maxPriorityLamports = 1_000_000,
priorityLevel = 'medium',
}) {
- const data = await fetchJson(JUP_SWAP_URL, {
- method: 'POST',
- headers: { 'content-type': 'application/json', accept: 'application/json' },
- body: JSON.stringify({
- quoteResponse: quote,
- userPublicKey,
- // USDC→$THREE never touches wrapped SOL; let Jupiter manage the $THREE ATA.
- wrapAndUnwrapSol,
- dynamicComputeUnitLimit: true,
- prioritizationFeeLamports: {
- priorityLevelWithMaxLamports: { maxLamports: maxPriorityLamports, priorityLevel },
- },
- }),
+ const body = JSON.stringify({
+ quoteResponse: quote,
+ userPublicKey,
+ // USDC→$THREE never touches wrapped SOL; let Jupiter manage the $THREE ATA.
+ wrapAndUnwrapSol,
+ dynamicComputeUnitLimit: true,
+ prioritizationFeeLamports: {
+ priorityLevelWithMaxLamports: { maxLamports: maxPriorityLamports, priorityLevel },
+ },
});
+ const data = await withRetry(
+ () => fetchJson(JUP_SWAP_URL, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', accept: 'application/json' },
+ body,
+ }, { timeoutMs: JUP_SWAP_TIMEOUT_MS, attempts: 1 }),
+ { attempts: 2, shouldRetry: isSwapBuildRetryable },
+ );
if (!data.swapTransaction) {
throw Object.assign(new Error('jupiter returned no swapTransaction'), { code: 'no_swap_tx' });
}
diff --git a/api/_lib/token/price.js b/api/_lib/token/price.js
index 160202f75..090a824e4 100644
--- a/api/_lib/token/price.js
+++ b/api/_lib/token/price.js
@@ -11,11 +11,12 @@ import { cacheGet, cacheSet } from '../cache.js';
import { fetchTokenMarketData } from '../market/token-market.js';
import { TOKEN_MINT, TOKEN_DECIMALS, ATOMICS_PER_TOKEN } from './config.js';
+import { fetchUpstreamJson } from '../upstream-fetch.js';
const PRICE_TTL_S = 30;
const PRICE_CACHE_KEY = `token:price:${TOKEN_MINT}`;
// Last-known-good price, held far longer than the live cache. When EVERY feed
-// (Jupiter + the Birdeye→DexScreener→GeckoTerminal chain) misses on a single
+// (Jupiter + the Birdeye→DexScreener→GeckoTerminal→DefiLlama chain) misses on a single
// request — a momentary simultaneous blip, not a real outage — falling back to a
// recent price keeps a paid quote alive instead of 503-ing the user's action.
// Bounded by STALE_PRICE_MAX_S: beyond that window a pump.fun token can have
@@ -26,14 +27,8 @@ const LAST_GOOD_TTL_S = 3_600;
const STALE_PRICE_MAX_S = 300;
async function fetchJson(url, opts = {}) {
- const r = await fetch(url, opts);
- if (!r.ok) {
- const body = await r.text().catch(() => String(r.status));
- throw Object.assign(new Error(`upstream ${r.status}: ${body.slice(0, 160)}`), {
- status: 502,
- });
- }
- return r.json();
+ // Bounded: a hung feed must throw so the market-data chain below gets its turn.
+ return fetchUpstreamJson(url, opts, { name: 'jupiter-price', timeoutMs: 6_000, attempts: 2 });
}
async function jupiterPrice(mint) {
@@ -43,6 +38,7 @@ async function jupiterPrice(mint) {
return Number.isFinite(n) && n > 0 ? n : null;
}
+
/**
* Live USD price of one whole token. Throws 503 price_unavailable when no feed
* returns a usable number — a paid action must never proceed on a guessed price.
@@ -64,7 +60,8 @@ export async function getTokenPriceUsd({ fresh = false } = {}) {
console.warn('[token] jupiter price failed:', err?.message);
}
if (!priceUsd) {
- // Jupiter down → shared market module: Birdeye → DexScreener → GeckoTerminal.
+ // Jupiter down → shared market module: Birdeye → DexScreener → GeckoTerminal
+ // → DefiLlama (coins.llama.fi, keyed `solana:<mint>`), each on its own deadline.
const md = await fetchTokenMarketData(TOKEN_MINT, { fresh }).catch((err) => {
console.warn('[token] market-data fallback failed:', err?.message);
return null;
diff --git a/api/_lib/token/quote.js b/api/_lib/token/quote.js
index dade4213c..05a000017 100644
--- a/api/_lib/token/quote.js
+++ b/api/_lib/token/quote.js
@@ -14,6 +14,7 @@
import crypto from 'node:crypto';
import { env } from '../env.js';
import { randomToken } from '../crypto.js';
+import { isRpcOutageError, rpcUnavailableError } from '../rpc-degrade.js';
import {
TOKEN_MINT,
TOKEN_DECIMALS,
@@ -83,7 +84,16 @@ export async function issueQuote({
refType = null,
refId = null,
}) {
- const priced = await quoteTokenForUsd(usd);
+ let priced;
+ try {
+ priced = await quoteTokenForUsd(usd);
+ } catch (err) {
+ // Already-typed pricing errors (price_unavailable, amount_too_*) pass
+ // through untouched. A raw RPC-chain exhaustion becomes the typed
+ // retryable 503 so no caller ever prices a payment off a guess.
+ if (err?.code || !isRpcOutageError(err)) throw err;
+ throw rpcUnavailableError('could not price the quote: Solana RPC is temporarily unavailable', err);
+ }
const legs = applySplit(priced.atomics, resolveSplitLegs(splitPolicy, { sellerWallet }));
const now = Math.floor(Date.now() / 1000);
const nonce = await randomToken(16);
diff --git a/api/_lib/treasury-autopilot.js b/api/_lib/treasury-autopilot.js
index 2f17e0438..90b9b8633 100644
--- a/api/_lib/treasury-autopilot.js
+++ b/api/_lib/treasury-autopilot.js
@@ -39,6 +39,8 @@ import {
import { THREE_MINT } from './networth-model.js';
import { llmComplete, llmConfigured } from './llm.js';
import { logAudit } from './audit.js';
+import { fetchUpstream } from './upstream-fetch.js';
+import { withRetry } from './resilience.js';
const WSOL_MINT = 'So11111111111111111111111111111111111111112';
// Rent + network-fee headroom kept above any autopilot spend so a move can never
@@ -569,14 +571,28 @@ async function swapSolToToken({ keypair, lamports, outputMint, slippageBps, netw
throw Object.assign(new Error('token swaps are mainnet-only — this wallet is on devnet'), { code: 'devnet_no_swap' });
}
const qUrl = `${JUPITER_BASE}/quote?inputMint=${WSOL_MINT}&outputMint=${outputMint}&amount=${lamports.toString()}&slippageBps=${slippageBps}&swapMode=ExactIn`;
- const qRes = await fetch(qUrl, { signal: AbortSignal.timeout(15_000) });
+ // The quote is an idempotent GET: a 429/5xx/network blip is retried with
+ // jittered backoff inside the same 15s per-attempt deadline. A 4xx (no route
+ // for this pair) is an answer and comes straight back as `no_route`, which the
+ // executor turns into an honest pause with the reason attached.
+ const qRes = await fetchUpstream(qUrl, { headers: { accept: 'application/json' } }, {
+ name: 'jupiter-treasury-quote',
+ timeoutMs: 15_000,
+ attempts: 3,
+ okWhen: (res) => res.ok || (res.status < 500 && res.status !== 429),
+ }).catch((err) => {
+ throw Object.assign(new Error(`no swap route (${err?.status || 'jupiter unreachable'}: ${err?.message || 'network error'})`), { code: 'no_route' });
+ });
if (!qRes.ok) throw Object.assign(new Error(`no swap route (${qRes.status})`), { code: 'no_route' });
const quote = await qRes.json();
if (!quote || !quote.outAmount) throw Object.assign(new Error('no swap route for this pair'), { code: 'no_route' });
- const sRes = await fetch(`${JUPITER_BASE}/swap`, {
+ // The swap build returns an UNSIGNED transaction, so repeating it is safe;
+ // it is retried once, only on a 5xx/network failure, never on a 4xx. Signing
+ // and broadcasting happen below and are never retried here.
+ const buildSwap = () => fetchUpstream(`${JUPITER_BASE}/swap`, {
method: 'POST',
- headers: { 'content-type': 'application/json' },
+ headers: { 'content-type': 'application/json', accept: 'application/json' },
body: JSON.stringify({
quoteResponse: quote,
userPublicKey: keypair.publicKey.toBase58(),
@@ -584,7 +600,17 @@ async function swapSolToToken({ keypair, lamports, outputMint, slippageBps, netw
dynamicComputeUnitLimit: true,
prioritizationFeeLamports: 'auto',
}),
- signal: AbortSignal.timeout(20_000),
+ }, {
+ name: 'jupiter-treasury-swap',
+ timeoutMs: 20_000,
+ attempts: 1,
+ okWhen: (res) => res.ok || res.status < 500,
+ });
+ const sRes = await withRetry(buildSwap, {
+ attempts: 2,
+ shouldRetry: (err) => !err?.status || Number(err.status) >= 500,
+ }).catch((err) => {
+ throw Object.assign(new Error(`swap build failed (${err?.status || 'jupiter unreachable'}: ${err?.message || 'network error'})`), { code: 'swap_failed' });
});
if (!sRes.ok) throw Object.assign(new Error(`swap build failed (${sRes.status})`), { code: 'swap_failed' });
const { swapTransaction } = await sRes.json();
diff --git a/api/_lib/upstream-fetch.js b/api/_lib/upstream-fetch.js
index efd4b3dd9..0940dd88c 100644
--- a/api/_lib/upstream-fetch.js
+++ b/api/_lib/upstream-fetch.js
@@ -251,6 +251,24 @@ export async function lastGood(key, load, opts = {}) {
}
}
+/**
+ * lastGood() for callers that only want the value: the previous good value on
+ * failure (within `maxAgeMs`), else the loader's rejection. Fits directly into
+ * a read-through cache loader: `cached(cache, key, () => lastGoodValue(key, load))`.
+ *
+ * @template T
+ * @param {string} key
+ * @param {() => Promise<T>} load
+ * @param {{ maxAgeMs?: number }} [opts]
+ * @returns {Promise<T>}
+ */
+export async function lastGoodValue(key, load, opts = {}) {
+ return (await lastGood(key, load, {
+ ...opts,
+ onFallback: (err, ageMs) => console.warn(`[upstream] ${key}: serving last good value (${Math.round(ageMs / 60_000)}m old) after ${err?.message || err}`),
+ })).value;
+}
+
/** Test hook: forget every remembered value. */
export function _resetLastGood() {
lastGoodStore.clear();
diff --git a/api/_lib/validation-attest.js b/api/_lib/validation-attest.js
index 04edff80a..500650861 100644
--- a/api/_lib/validation-attest.js
+++ b/api/_lib/validation-attest.js
@@ -26,7 +26,7 @@ import { Contract, Wallet } from 'ethers';
import { env } from './env.js';
import { CHAIN_BY_ID, VALIDATION_REGISTRY_ABI, validationRegistryFor } from './erc8004-chains.js';
-import { evmRpcEndpoints } from './evm/rpc.js';
+import { evmFallbackProvider } from './evm/rpc.js';
import { putObject, publicUrl } from './r2.js';
import { assertSafePublicUrl, SsrfBlockedError } from './ssrf-guard.js';
import { inspectModel, suggestOptimizations } from './model-inspect.js';
@@ -237,12 +237,10 @@ export async function attestValidation({ chainId, agentId, glbUrl, validatedAt,
// 1. Validate the GLB (this part never throws on an invalid model).
const { report, passed, sha256 } = await validateGlb(glbUrl, validatedAt);
- // 2. Provider + wallet. evmFallbackProvider is read-tuned; for the write we
- // use a plain JsonRpcProvider on the priority endpoint list.
- const { JsonRpcProvider, Network } = await import('ethers');
- const network = Network.from(chainId);
- const endpoints = evmRpcEndpoints(chainId);
- const provider = new JsonRpcProvider(endpoints[0], network, { staticNetwork: network });
+ // 2. Provider + wallet. The quorum-1 FallbackProvider tries the chain's
+ // endpoints in priority order for reads and broadcasts the signed write
+ // through whichever answers, so one dead RPC never blocks an attestation.
+ const provider = await evmFallbackProvider(chainId);
const wallet = new Wallet(pk, provider);
const registry = new Contract(registryAddr, VALIDATION_REGISTRY_ABI, wallet);
diff --git a/api/_lib/vanity-vault.js b/api/_lib/vanity-vault.js
index 17545d4db..46950912f 100644
--- a/api/_lib/vanity-vault.js
+++ b/api/_lib/vanity-vault.js
@@ -26,6 +26,11 @@
import { webcrypto } from 'node:crypto';
import { encryptSecret, decryptSecret } from './secret-box.js';
import { getGcpAccessToken } from './gcp-auth.js';
+import { fetchUpstream } from './upstream-fetch.js';
+
+// KMS encrypt/decrypt are idempotent (same DEK in, same wrap out), so a
+// transient failure is retried behind one breaker for the KMS host.
+const KMS_FETCH_OPTS = { name: 'gcp-kms', timeoutMs: 10_000, attempts: 3, okWhen: () => true };
const subtle = globalThis.crypto?.subtle || webcrypto.subtle;
const randomBytes = (n) => {
@@ -66,11 +71,11 @@ export function preferredScheme() {
async function kmsEncrypt(dek) {
const token = await getGcpAccessToken();
const url = `https://cloudkms.googleapis.com/v1/${kmsKeyName()}:encrypt`;
- const res = await fetch(url, {
+ const res = await fetchUpstream(url, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({ plaintext: Buffer.from(dek).toString('base64') }),
- });
+ }, KMS_FETCH_OPTS);
if (!res.ok) {
const detail = await res.text().catch(() => res.status);
throw new Error(`KMS encrypt failed (${res.status}): ${String(detail).slice(0, 300)}`);
@@ -82,11 +87,11 @@ async function kmsEncrypt(dek) {
async function kmsDecrypt(wrappedDekB64) {
const token = await getGcpAccessToken();
const url = `https://cloudkms.googleapis.com/v1/${kmsKeyName()}:decrypt`;
- const res = await fetch(url, {
+ const res = await fetchUpstream(url, {
method: 'POST',
headers: { authorization: `Bearer ${token}`, 'content-type': 'application/json' },
body: JSON.stringify({ ciphertext: wrappedDekB64 }),
- });
+ }, KMS_FETCH_OPTS);
if (!res.ok) {
const detail = await res.text().catch(() => res.status);
throw new Error(`KMS decrypt failed (${res.status}): ${String(detail).slice(0, 300)}`);
diff --git a/api/_lib/vertex-claude.js b/api/_lib/vertex-claude.js
index 588df6c95..9384e9d04 100644
--- a/api/_lib/vertex-claude.js
+++ b/api/_lib/vertex-claude.js
@@ -26,6 +26,11 @@
import { getGcpAccessToken } from './gcp-auth.js';
+// Bounds the wait for response headers only. The Response may be a stream the
+// caller pipes through for minutes, so a whole-request deadline would cut
+// live completions; a hung connection is the only thing this needs to catch.
+const VERTEX_HEADERS_TIMEOUT_MS = 45_000;
+
const DEFAULT_LOCATION = 'global';
const ANTHROPIC_VERTEX_VERSION = 'vertex-2023-10-16';
@@ -105,9 +110,16 @@ export async function vertexRequestHeaders() {
export async function vertexAnthropicMessages(body, { stream = false } = {}) {
const url = vertexMessagesUrl(body.model, { stream });
const headers = await vertexRequestHeaders();
- return fetch(url, {
- method: 'POST',
- headers,
- body: JSON.stringify(toVertexBody(body)),
- });
+ const headersCtrl = new AbortController();
+ const headersTimer = setTimeout(() => headersCtrl.abort(), VERTEX_HEADERS_TIMEOUT_MS);
+ try {
+ return await fetch(url, {
+ method: 'POST',
+ headers,
+ body: JSON.stringify(toVertexBody(body)),
+ signal: headersCtrl.signal,
+ });
+ } finally {
+ clearTimeout(headersTimer);
+ }
}
diff --git a/api/_lib/wallet-activity.js b/api/_lib/wallet-activity.js
index ac9d21186..c6d2672d7 100644
--- a/api/_lib/wallet-activity.js
+++ b/api/_lib/wallet-activity.js
@@ -19,6 +19,7 @@ import { getBalances } from './balances.js';
import { cacheWrap } from './cache.js';
import { env } from './env.js';
+import { fetchUpstream } from './upstream-fetch.js';
const DAY_MS = 86_400_000;
const ACTIVITY_TTL_S = 3600;
@@ -113,10 +114,10 @@ export async function solanaWalletActivity(address) {
async function etherscan(chainid, params) {
const key = env.ETHERSCAN_API_KEY;
const qs = new URLSearchParams({ ...params, chainid: String(chainid), apikey: key });
- const r = await fetch(`${ETHERSCAN_V2}?${qs}`, {
+ const r = await fetchUpstream(`${ETHERSCAN_V2}?${qs}`, {
headers: { accept: 'application/json' },
signal: AbortSignal.timeout(15_000),
- });
+ }, { name: 'etherscan', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
if (!r.ok) return [];
const data = await r.json().catch(() => null);
// status "0" with "No transactions found" is a real empty answer, not an error.
diff --git a/api/_lib/watsonx-forecast.js b/api/_lib/watsonx-forecast.js
index 77b9a41db..3008b5b43 100644
--- a/api/_lib/watsonx-forecast.js
+++ b/api/_lib/watsonx-forecast.js
@@ -10,6 +10,9 @@
// upstream status + message so the caller reports the true cause.
import { watsonxToken } from './watsonx.js';
+import { fetchUpstream } from './upstream-fetch.js';
+
+const INFERENCE_TIMEOUT_MS = 45_000;
// Granite TimeSeries (TinyTimeMixer) zero-shot forecasting models, keyed by the
// minimum history (context) length each requires. All forecast 96 steps ahead.
@@ -35,7 +38,7 @@ function scope(cfg) {
// real upstream status + message on failure.
async function post(cfg, path, body, version) {
const token = await watsonxToken(cfg);
- const res = await fetch(`${cfg.url}${path}?version=${version}`, {
+ const res = await fetchUpstream(`${cfg.url}${path}?version=${version}`, {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
@@ -43,7 +46,7 @@ async function post(cfg, path, body, version) {
Accept: 'application/json',
},
body: JSON.stringify({ ...body, ...scope(cfg) }),
- });
+ }, { name: 'watsonx', timeoutMs: INFERENCE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
const text = await res.text();
if (!res.ok) {
let detail = text.slice(0, 300);
diff --git a/api/_lib/watsonx.js b/api/_lib/watsonx.js
index 169a5ed1f..fc9b73e56 100644
--- a/api/_lib/watsonx.js
+++ b/api/_lib/watsonx.js
@@ -11,8 +11,15 @@
// `configured: false` so a proxy can mark the provider unavailable; any IAM or
// upstream failure throws so the proxy reports the real cause to the caller.
+import { fetchUpstream } from './upstream-fetch.js';
+
const IAM_TOKEN_URL = 'https://iam.cloud.ibm.com/identity/token';
+// The IAM exchange is idempotent (same key, same short-lived token) so it is
+// retried; an inference is not, and gets one attempt with a longer deadline.
+const IAM_TIMEOUT_MS = 10_000;
+const INFERENCE_TIMEOUT_MS = 45_000;
+
// Refresh the IAM token this many ms before its stated expiry so an in-flight
// request never races the hard expiry boundary.
const TOKEN_SKEW_MS = 5 * 60 * 1000;
@@ -60,7 +67,7 @@ export async function watsonxToken(cfg) {
if (inflight.has(cfg.apiKey)) return inflight.get(cfg.apiKey);
const p = (async () => {
- const res = await fetch(cfg.iamUrl, {
+ const res = await fetchUpstream(cfg.iamUrl, {
method: 'POST',
headers: {
'content-type': 'application/x-www-form-urlencoded',
@@ -70,7 +77,7 @@ export async function watsonxToken(cfg) {
grant_type: 'urn:ibm:params:oauth:grant-type:apikey',
apikey: cfg.apiKey,
}),
- });
+ }, { name: 'watsonx:iam', timeoutMs: IAM_TIMEOUT_MS, attempts: 3, okWhen: () => true });
const data = await res.json().catch(() => ({}));
if (!res.ok || !data.access_token) {
throw new Error(
@@ -118,11 +125,11 @@ async function jsonAuthHeaders(cfg) {
// failure so callers report the true cause (auth, quota, unsupported model).
async function watsonxPost(cfg, path, body, version) {
const headers = await jsonAuthHeaders(cfg);
- const res = await fetch(`${cfg.url}${path}?version=${version || cfg.apiVersion}`, {
+ const res = await fetchUpstream(`${cfg.url}${path}?version=${version || cfg.apiVersion}`, {
method: 'POST',
headers,
body: JSON.stringify({ ...body, ...scope(cfg) }),
- });
+ }, { name: 'watsonx', timeoutMs: INFERENCE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
const text = await res.text();
if (!res.ok) {
let detail = text.slice(0, 300);
diff --git a/api/_lib/x-post.js b/api/_lib/x-post.js
index 92c9e6e7b..caecedf6d 100644
--- a/api/_lib/x-post.js
+++ b/api/_lib/x-post.js
@@ -7,6 +7,19 @@ import { env } from './env.js';
import { encryptToken, decryptToken } from '../auth/x/[action].js';
import { X_POST_REQUIRED_SCOPES, missingScopes } from './x-scopes.js';
+import { fetchUpstream } from './upstream-fetch.js';
+
+// Every X call shares one breaker ('x:post') so a dead X API fails fast
+// instead of paying a deadline per step, and each step gets its own deadline.
+// The media flow is up to six sequential calls (INIT, APPENDs, FINALIZE,
+// STATUS polls) so it also carries one overall budget, passed as the caller
+// signal that fetchUpstream composes with the per-call timeout.
+const X_BREAKER = 'x:post';
+const X_TOKEN_TIMEOUT_MS = 10_000;
+const X_TWEET_TIMEOUT_MS = 15_000;
+const X_MEDIA_STEP_TIMEOUT_MS = 30_000;
+export const X_MEDIA_TOTAL_TIMEOUT_MS = 90_000;
+
export const FREE_MONTHLY_QUOTA = 5;
export const PRO_MONTHLY_QUOTA = 100;
export const MAX_TWEET_LEN = 280;
@@ -45,11 +58,11 @@ async function refreshIfNeeded(conn) {
const refreshToken = decryptToken(conn.refresh_token);
const creds = Buffer.from(`${env.X_OAUTH_CLIENT_ID}:${env.X_OAUTH_CLIENT_SECRET}`).toString('base64');
- const r = await fetch('https://api.twitter.com/2/oauth2/token', {
+ const r = await fetchUpstream('https://api.twitter.com/2/oauth2/token', {
method: 'POST',
headers: { 'content-type': 'application/x-www-form-urlencoded', authorization: `Basic ${creds}` },
body: new URLSearchParams({ grant_type: 'refresh_token', refresh_token: refreshToken, client_id: env.X_OAUTH_CLIENT_ID }).toString(),
- });
+ }, { name: X_BREAKER, timeoutMs: X_TOKEN_TIMEOUT_MS, attempts: 1, okWhen: () => true });
if (!r.ok) throw new XPostError('reauth_required', `X token refresh failed: ${await r.text()}`, 401);
const tok = await r.json();
const newExpiresAt = new Date(Date.now() + (tok.expires_in ?? 7200) * 1000).toISOString();
@@ -69,11 +82,11 @@ async function postOne({ accessToken, text, replyTo, mediaIds = null }) {
if (text) body.text = text;
if (replyTo) body.reply = { in_reply_to_tweet_id: replyTo };
if (mediaIds && mediaIds.length) body.media = { media_ids: mediaIds };
- const r = await fetch('https://api.twitter.com/2/tweets', {
+ const r = await fetchUpstream('https://api.twitter.com/2/tweets', {
method: 'POST',
headers: { 'content-type': 'application/json', authorization: `Bearer ${accessToken}` },
body: JSON.stringify(body),
- });
+ }, { name: X_BREAKER, timeoutMs: X_TWEET_TIMEOUT_MS, attempts: 1, okWhen: () => true });
if (!r.ok) throw new XPostError('tweet_failed', `X API error: ${(await r.text()).slice(0, 200)}`, 502);
return (await r.json()).data;
}
@@ -92,16 +105,33 @@ function mediaCategoryFor(mimeType) {
return 'tweet_image';
}
-export async function uploadMediaV2({ accessToken, buffer, mimeType }) {
+export async function uploadMediaV2({ accessToken, buffer, mimeType, deadlineMs = X_MEDIA_TOTAL_TIMEOUT_MS }) {
+ const overall = new AbortController();
+ const timer = setTimeout(() => overall.abort(), deadlineMs);
+ try {
+ return await uploadMediaSteps({ accessToken, buffer, mimeType, signal: overall.signal });
+ } catch (err) {
+ if (err instanceof XPostError) throw err;
+ if (overall.signal.aborted) {
+ throw new XPostError('media_upload_failed', `media upload exceeded its ${Math.round(deadlineMs / 1000)}s budget`, 504);
+ }
+ throw new XPostError('media_upload_failed', `media upload failed: ${err?.message || err}`, 502);
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
+async function uploadMediaSteps({ accessToken, buffer, mimeType, signal }) {
const total = buffer.length;
const auth = { authorization: `Bearer ${accessToken}` };
+ const step = { name: X_BREAKER, timeoutMs: X_MEDIA_STEP_TIMEOUT_MS, attempts: 2, okWhen: () => true };
const initForm = new FormData();
initForm.set('command', 'INIT');
initForm.set('media_type', mimeType);
initForm.set('total_bytes', String(total));
initForm.set('media_category', mediaCategoryFor(mimeType));
- const initRes = await fetch(X_MEDIA_UPLOAD_URL, { method: 'POST', headers: auth, body: initForm });
+ const initRes = await fetchUpstream(X_MEDIA_UPLOAD_URL, { method: 'POST', headers: auth, body: initForm, signal }, step);
if (!initRes.ok)
throw new XPostError('media_upload_failed', `media INIT failed: ${(await initRes.text()).slice(0, 200)}`, 502);
const initJson = await initRes.json();
@@ -116,7 +146,7 @@ export async function uploadMediaV2({ accessToken, buffer, mimeType }) {
appendForm.set('media_id', mediaId);
appendForm.set('segment_index', String(segment));
appendForm.set('media', new Blob([chunk], { type: 'application/octet-stream' }), 'chunk');
- const appendRes = await fetch(X_MEDIA_UPLOAD_URL, { method: 'POST', headers: auth, body: appendForm });
+ const appendRes = await fetchUpstream(X_MEDIA_UPLOAD_URL, { method: 'POST', headers: auth, body: appendForm, signal }, step);
if (!appendRes.ok)
throw new XPostError('media_upload_failed', `media APPEND ${segment} failed: ${(await appendRes.text()).slice(0, 200)}`, 502);
segment++;
@@ -125,7 +155,7 @@ export async function uploadMediaV2({ accessToken, buffer, mimeType }) {
const finalizeForm = new FormData();
finalizeForm.set('command', 'FINALIZE');
finalizeForm.set('media_id', mediaId);
- const finalizeRes = await fetch(X_MEDIA_UPLOAD_URL, { method: 'POST', headers: auth, body: finalizeForm });
+ const finalizeRes = await fetchUpstream(X_MEDIA_UPLOAD_URL, { method: 'POST', headers: auth, body: finalizeForm, signal }, step);
if (!finalizeRes.ok)
throw new XPostError('media_upload_failed', `media FINALIZE failed: ${(await finalizeRes.text()).slice(0, 200)}`, 502);
const finalizeJson = await finalizeRes.json();
@@ -133,9 +163,10 @@ export async function uploadMediaV2({ accessToken, buffer, mimeType }) {
// Async transcode (video): poll STATUS until the asset is ready or fails.
let tries = 0;
- while (info && (info.state === 'pending' || info.state === 'in_progress') && tries < 30) {
+ while (info && (info.state === 'pending' || info.state === 'in_progress') && tries < 30 && !signal.aborted) {
await new Promise((r) => setTimeout(r, Math.min((info.check_after_secs || 1) * 1000, 5000)));
- const statusRes = await fetch(`${X_MEDIA_UPLOAD_URL}?command=STATUS&media_id=${mediaId}`, { headers: auth });
+ if (signal.aborted) break;
+ const statusRes = await fetchUpstream(`${X_MEDIA_UPLOAD_URL}?command=STATUS&media_id=${mediaId}`, { headers: auth, signal }, step);
if (!statusRes.ok) break;
const statusJson = await statusRes.json();
info = statusJson?.data?.processing_info || statusJson?.processing_info || null;
diff --git a/api/_lib/x402-spending-price.js b/api/_lib/x402-spending-price.js
index e0497767f..a8b61d854 100644
--- a/api/_lib/x402-spending-price.js
+++ b/api/_lib/x402-spending-price.js
@@ -4,18 +4,41 @@
// Most x402 transfers today are USDC, which is already pegged 1:1 to USD
// and whose `amount` is already in micro-USD (6 decimals). For tokens
// pegged differently (e.g. USDT-on-BSC is still ~1:1 USD; SOL would not
-// be), this module exposes a lightweight price oracle that pulls from
-// Coinbase's spot API. Results are cached for 5 minutes so we don't
-// hammer the API in tight loops.
+// be), this module exposes a lightweight price oracle. It walks the shared
+// free-source chain (CoinGecko, DefiLlama, Kraken, Coinbase, Bitfinex, the
+// keyless CEX tail) for the headline assets and Coinbase spot for anything
+// else, caches the result for 5 minutes, and serves the last-known-good price
+// (up to a day old) when every source is down: a spend-cap check must not
+// fail closed on one exchange's outage. It only throws when no price for the
+// symbol has ever been observed.
//
// We deliberately avoid `import { fetch }` — Node 18+ + browsers ship
// global fetch, and this module needs to run unchanged in both.
-import { cacheGet, cacheSet } from './cache.js';
+import { cacheWrapLastGood } from './cache.js';
+import { fetchCoinPriceUsd } from './market-fallbacks.js';
+import { fetchFirst } from '../../src/shared/failover-fetch.js';
const PRICE_TTL_S = 5 * 60;
+const LAST_GOOD_TTL_S = 24 * 3600;
const PRICE_API = 'https://api.coinbase.com/v2/prices';
+// Ticker symbols x402 requirements name for non-pegged assets, mapped to the
+// CoinGecko ids the shared price chain is keyed by. Wrapped forms price as the
+// underlying. An unmapped symbol goes straight to the Coinbase spot rung.
+const COINGECKO_IDS = {
+ SOL: 'solana',
+ WSOL: 'solana',
+ ETH: 'ethereum',
+ WETH: 'ethereum',
+ BTC: 'bitcoin',
+ WBTC: 'bitcoin',
+ BNB: 'binancecoin',
+ POL: 'polygon-ecosystem-token',
+ MATIC: 'polygon-ecosystem-token',
+ AVAX: 'avalanche-2',
+};
+
// Tokens known to be 1:1 USD pegged. The asset address on the requirement
// + the symbol from `extra.name` are matched against these.
const USD_STABLECOINS = new Set([
@@ -65,43 +88,64 @@ function safeDecimals(raw) {
// symbol is hit twice in one function invocation (e.g. multi-leg settlement).
const localCache = new Map();
+/** Test seam: drop the per-process micro-cache between cases. */
+export function __resetLocalCache() {
+ localCache.clear();
+}
+
function isStablecoin(name) {
if (!name) return false;
return USD_STABLECOINS.has(String(name).trim().toLowerCase());
}
-// Pull a fresh spot price (in USD) for a symbol like 'SOL' or 'ETH'. Uses
-// Coinbase's public spot endpoint — no API key, ~30 req/sec rate limit.
-// Cached for PRICE_TTL_MS.
+function asPositive(raw) {
+ const n = Number(raw);
+ return Number.isFinite(n) && n > 0 ? n : null;
+}
+
+// One live read for a symbol: the shared multi-source chain when the symbol
+// maps to a CoinGecko id, then Coinbase spot (no key, ~30 req/sec) as the
+// direct rung for every symbol. Throws only when every source is down.
+async function loadSpotUsd(key) {
+ const id = COINGECKO_IDS[key];
+ let chainErr = null;
+ if (id) {
+ try {
+ return await fetchCoinPriceUsd(id);
+ } catch (err) {
+ chainErr = err;
+ }
+ }
+ try {
+ const { value } = await fetchFirst(
+ [{
+ name: `spending-price:coinbase:${key}`,
+ url: `${PRICE_API}/${key}-USD/spot`,
+ parse: async (r) => asPositive((await r.json())?.data?.amount),
+ }],
+ { timeoutMs: 5_000, label: `spending-price:${key}` },
+ );
+ return value;
+ } catch (err) {
+ throw new Error(`spending-price: every price source for ${key}-USD failed (${chainErr?.message ? `${chainErr.message}; ` : ''}${err.message})`);
+ }
+}
+
+// Spot price (in USD) for a symbol like 'SOL' or 'ETH': the per-process
+// micro-cache, then the shared cache, then a live read, then the last-known-good
+// value from the shared store when every live source fails. Cached for PRICE_TTL_S.
async function fetchSpotUsd(symbol) {
const key = symbol.toUpperCase();
const local = localCache.get(key);
if (local && local.expiresAt > Date.now()) return local.value;
const cacheKey = `spot-usd:${key}`;
- const shared = await cacheGet(cacheKey);
- if (typeof shared === 'number' && Number.isFinite(shared) && shared > 0) {
- localCache.set(key, { value: shared, expiresAt: Date.now() + PRICE_TTL_S * 1000 });
- return shared;
- }
- let res;
- try {
- res = await fetch(`${PRICE_API}/${key}-USD/spot`, {
- signal: AbortSignal.timeout(5_000),
- });
- } catch (err) {
- throw new Error(`spending-price: spot fetch for ${key}-USD failed: ${err.message}`);
- }
- if (!res.ok) {
- throw new Error(`spending-price: ${key}-USD returned ${res.status}`);
- }
- const json = await res.json();
- const raw = json?.data?.amount;
- const value = Number(raw);
+ const value = await cacheWrapLastGood(cacheKey, PRICE_TTL_S, () => loadSpotUsd(key), {
+ staleTtlSeconds: LAST_GOOD_TTL_S,
+ });
if (!Number.isFinite(value) || value <= 0) {
- throw new Error(`spending-price: ${key}-USD invalid response: ${raw}`);
+ throw new Error(`spending-price: ${key}-USD invalid price: ${value}`);
}
localCache.set(key, { value, expiresAt: Date.now() + PRICE_TTL_S * 1000 });
- await cacheSet(cacheKey, value, PRICE_TTL_S);
return value;
}
@@ -111,7 +155,7 @@ async function fetchSpotUsd(symbol) {
// tokens we multiply by the spot price.
//
// The function NEVER throws on a missing price for a stablecoin path —
-// that's the dominant flow. It only reaches out to Coinbase when the
+// that's the dominant flow. It only reaches out to the price chain when the
// requirement explicitly declares a non-pegged token.
export async function toMicroUsd(amount, requirement) {
const atomic = BigInt(amount);
diff --git a/api/_lib/x402/auth-hints.js b/api/_lib/x402/auth-hints.js
index a024adf41..d99298b31 100644
--- a/api/_lib/x402/auth-hints.js
+++ b/api/_lib/x402/auth-hints.js
@@ -30,8 +30,9 @@ import {
validateSIWxMessage,
verifySIWxSignature,
} from '@x402/extensions/sign-in-with-x';
-import { createPublicClient, http } from 'viem';
+import { createPublicClient } from 'viem';
import { base } from 'viem/chains';
+import { evmTransport } from '../evm/rpc.js';
import { env } from '../env.js';
import { authenticateBearer, hasScope } from '../auth.js';
@@ -213,7 +214,11 @@ let _baseClient;
function getEvmVerifier() {
if (!env.BASE_RPC_URL) return undefined;
if (!_baseClient) {
- _baseClient = createPublicClient({ chain: base, transport: http(env.BASE_RPC_URL) });
+ // BASE_RPC_URL stays primary (a private node, so buyer addresses are not
+ // broadcast to public RPCs on the happy path); the shared Base endpoints
+ // only take over when it fails, so a dead private node cannot block
+ // every smart-wallet sign-in.
+ _baseClient = createPublicClient({ chain: base, transport: evmTransport(base.id, { primaryUrl: env.BASE_RPC_URL }) });
}
return _baseClient.verifyMessage.bind(_baseClient);
}
diff --git a/api/_lib/x402/pay.js b/api/_lib/x402/pay.js
index 60139142e..50794a780 100644
--- a/api/_lib/x402/pay.js
+++ b/api/_lib/x402/pay.js
@@ -19,11 +19,12 @@ import {
import {
getAssociatedTokenAddressSync, createTransferCheckedInstruction,
createAssociatedTokenAccountIdempotentInstruction,
- TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID, getMint,
+ TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
} from '@solana/spl-token';
import { env } from '../env.js';
import { solanaConnection } from '../solana/connection.js';
+import { mintDecimals, getRecentBlockhash } from '../solana/read-guards.js';
import { assessFeeAdmission } from './wallet-fee-meter.js';
export const USDC_MINT = env.X402_ASSET_MINT_SOLANA;
@@ -298,7 +299,10 @@ async function currentBlockhash(conn, fallback, { forceFresh = false } = {}) {
const age = Date.now() - blockhashCache.fetchedAt;
if (!forceFresh && blockhashCache.value && age < BLOCKHASH_TTL_MS) return blockhashCache.value;
try {
- const { blockhash } = await conn.getLatestBlockhash('confirmed');
+ // The shared guard serves its own cached blockhash (per process and via the
+ // shared cache) inside the validity window when every RPC lane is down, so
+ // this only reaches the fallback when nothing usable exists anywhere.
+ const blockhash = await getRecentBlockhash(conn, SOLANA_RPC);
seedBlockhash(blockhash);
return blockhash;
} catch {
@@ -313,12 +317,16 @@ export async function bootstrapSolanaContext({ buyer } = {}) {
if (!USDC_MINT) throw new Error('x402 pay: X402_ASSET_MINT_SOLANA not configured');
const payer = buyer || loadSeedKeypair();
const conn = solanaConnection({ url: SOLANA_RPC, commitment: 'confirmed' });
- const [{ blockhash }, mintInfo] = await Promise.all([
- conn.getLatestBlockhash('confirmed'),
- getMint(conn, new PublicKey(USDC_MINT)),
+ // USDC decimals resolve locally (no network read) and the blockhash goes
+ // through the shared cached guard, so a dead RPC only fails the bootstrap when
+ // no blockhash inside its validity window exists anywhere in the fleet. That
+ // failure is a typed 503 rpc_unavailable rather than a raw fetch error.
+ const [blockhash, decimals] = await Promise.all([
+ getRecentBlockhash(conn, SOLANA_RPC),
+ mintDecimals(conn, new PublicKey(USDC_MINT)),
]);
seedBlockhash(blockhash);
- return { buyer: payer, conn, blockhash, mintInfo };
+ return { buyer: payer, conn, blockhash, mintInfo: { decimals } };
}
// Settle a single x402 payment against `url`. Probes for the 402 challenge,
diff --git a/api/_mcp/tools/garments.js b/api/_mcp/tools/garments.js
index 067e82bb5..89274f5fb 100644
--- a/api/_mcp/tools/garments.js
+++ b/api/_mcp/tools/garments.js
@@ -11,6 +11,7 @@
import { limits } from '../../_lib/rate-limit.js';
+import { fetchUpstream } from '../../_lib/upstream-fetch.js';
const GARMENT_SLOTS = [
'top', 'bottom', 'footwear', 'outerwear', 'hair', 'headwear', 'glasses', 'accessory',
];
@@ -80,11 +81,11 @@ export const toolDefs = [
});
}
const { base, key } = workerConfig();
- const res = await fetch(`${base}/generate`, {
+ const res = await fetchUpstream(`${base}/generate`, {
method: 'POST',
headers: { authorization: `Bearer ${key}`, 'content-type': 'application/json' },
body: JSON.stringify({ prompt: args.prompt, slot: args.slot }),
- }).catch(() => null);
+ }, { name: 'garment-worker', timeoutMs: 30_000, attempts: 2, okWhen: () => true }).catch(() => null);
if (!res || !res.ok) {
throw new Error(`garment worker ${res ? `returned ${res.status}` : 'unreachable'}`);
}
@@ -132,9 +133,9 @@ export const toolDefs = [
}
if (!JOB_ID_RE.test(args.job_id)) throw new Error('malformed job id');
const { base, key } = workerConfig();
- const res = await fetch(`${base}/jobs/${args.job_id}`, {
+ const res = await fetchUpstream(`${base}/jobs/${args.job_id}`, {
headers: { authorization: `Bearer ${key}` },
- }).catch(() => null);
+ }, { name: 'garment-worker', timeoutMs: 15_000, attempts: 2, okWhen: () => true }).catch(() => null);
if (!res) throw new Error('garment worker unreachable');
if (res.status === 404) throw new Error('no such garment job');
if (!res.ok) throw new Error(`garment worker returned ${res.status}`);
@@ -201,7 +202,7 @@ export const toolDefs = [
retry_after: Math.ceil((rl.reset - Date.now()) / 1000),
});
}
- const res = await fetch(CATALOG_URL).catch(() => null);
+ const res = await fetchUpstream(CATALOG_URL, {}, { name: 'gcs-garments', timeoutMs: 15_000, attempts: 2, okWhen: () => true }).catch(() => null);
if (!res || !res.ok) throw new Error('garment catalog unreachable');
const raw = await res.json();
const garments = (Array.isArray(raw) ? raw : [])
diff --git a/api/_providers/gcp.js b/api/_providers/gcp.js
index 6aea61142..b6628acf8 100644
--- a/api/_providers/gcp.js
+++ b/api/_providers/gcp.js
@@ -40,6 +40,8 @@
// The extJobId is a JSON envelope base64url-encoded so it can carry both
// the service discriminator and the upstream task_id without another DB call.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
function readEnv(name) {
if (typeof process !== 'undefined' && process.env?.[name]) return process.env[name];
return null;
@@ -466,11 +468,11 @@ export function createRegenProvider({ reconstructUrl } = {}) {
const url = `${baseUrl.replace(/\/$/, '')}${workerReq.path}`;
let response;
try {
- response = await fetch(url, {
+ response = await fetchUpstream(url, {
method: 'POST',
headers: authHeaders,
body: JSON.stringify(workerReq.body),
- });
+ }, { name: 'gcp-forge-worker', timeoutMs: 60_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(
new Error(`gcp ${mode} service unreachable: ${err?.message}`),
@@ -534,10 +536,7 @@ export function createRegenProvider({ reconstructUrl } = {}) {
let response;
try {
- response = await fetch(
- `${baseUrl.replace(/\/$/, '')}${pollPath}`,
- { headers: authHeaders },
- );
+ response = await fetchUpstream(`${baseUrl.replace(/\/$/, '')}${pollPath}`, { headers: authHeaders }, { name: 'gcp-forge-worker', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `poll failed: ${err?.message}` };
}
diff --git a/api/_providers/huggingface.js b/api/_providers/huggingface.js
index 16ffd8684..45fa81be0 100644
--- a/api/_providers/huggingface.js
+++ b/api/_providers/huggingface.js
@@ -32,7 +32,17 @@
// is unset.
// HF_RECONSTRUCT_API_NAME — legacy single-API alias.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
const HF_INFERENCE_TIMEOUT_MS = 280_000; // leave headroom for response framing
+// Per-call deadlines. The SSE read is bounded by HF_INFERENCE_TIMEOUT_MS in
+// consumeSseUntilComplete; its fetch deadline sits above that as a backstop
+// (the signal covers the body read too, so it must not be shorter). The
+// enqueue and the GLB download are single attempts: the outer Space chain and
+// candidate loop already decide what to retry.
+const HF_ENQUEUE_TIMEOUT_MS = 60_000;
+const HF_SSE_TIMEOUT_MS = HF_INFERENCE_TIMEOUT_MS + 10_000;
+const HF_GLB_DOWNLOAD_TIMEOUT_MS = 120_000;
// A reconstructed GLB is served from the Space's ephemeral gradio /tmp path,
// which is purged minutes (sometimes seconds) after the run. Every consumer of
@@ -377,7 +387,7 @@ async function rehostGlbToR2(glbUrls, token) {
for (const glbUrl of candidates) {
for (let attempt = 1; attempt <= REHOST_MAX_ATTEMPTS; attempt++) {
try {
- const resp = await fetch(glbUrl, { headers: spaceAuthHeaders(token) });
+ const resp = await fetchUpstream(glbUrl, { headers: spaceAuthHeaders(token) }, { timeoutMs: HF_GLB_DOWNLOAD_TIMEOUT_MS, attempts: 1, okWhen: () => true });
if (!resp.ok) {
lastErr = Object.assign(new Error(`GLB fetch ${resp.status}`), { status: resp.status });
// 404/410 on this address = wrong route or the temp file is gone;
@@ -458,11 +468,11 @@ async function attemptOnSpace({ token, target, photos, params }) {
// Step 1 — POST /call/<api> to enqueue. Returns event_id.
let queueRes;
try {
- queueRes = await fetch(`${spaceUrl}/call/${api}`, {
+ queueRes = await fetchUpstream(`${spaceUrl}/call/${api}`, {
method: 'POST',
headers: { ...spaceAuthHeaders(token), 'content-type': 'application/json' },
body: JSON.stringify({ data: payload }),
- });
+ }, { timeoutMs: HF_ENQUEUE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
} catch (err) {
throw tag(Object.assign(new Error(`enqueue failed: ${err?.message}`), {
code: 'provider_unreachable',
@@ -485,9 +495,9 @@ async function attemptOnSpace({ token, target, photos, params }) {
// Step 2 — GET /call/<api>/<event_id> SSE; block until complete.
let streamRes;
try {
- streamRes = await fetch(`${spaceUrl}/call/${api}/${eventId}`, {
+ streamRes = await fetchUpstream(`${spaceUrl}/call/${api}/${eventId}`, {
headers: { ...spaceAuthHeaders(token), accept: 'text/event-stream' },
- });
+ }, { timeoutMs: HF_SSE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
} catch (err) {
throw tag(Object.assign(new Error(`SSE GET failed: ${err?.message}`), {
code: 'provider_unreachable',
diff --git a/api/_providers/meshy.js b/api/_providers/meshy.js
index 57eb20e5c..3c6fb862a 100644
--- a/api/_providers/meshy.js
+++ b/api/_providers/meshy.js
@@ -17,6 +17,8 @@
// BYOK only: the caller passes the user's Meshy API key (msy_…). No platform
// key — when absent, the forge endpoint never reaches this module.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
const MESHY_BASE = 'https://api.meshy.ai';
// Meshy accepts 100–300,000 for target_polycount. Clamp the tier budget into
@@ -63,11 +65,11 @@ export function createMeshyProvider(apiKey) {
async function postTask(path, body) {
let res;
try {
- res = await fetch(`${MESHY_BASE}${path}`, {
+ res = await fetchUpstream(`${MESHY_BASE}${path}`, {
method: 'POST',
headers,
body: JSON.stringify(body),
- });
+ }, { name: 'meshy', timeoutMs: 30_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`meshy unreachable: ${err?.message}`), {
code: 'provider_unreachable',
@@ -152,7 +154,7 @@ export function createMeshyProvider(apiKey) {
if (!base) return { status: 'failed', error: `unknown meshy task kind "${kind}"` };
let res;
try {
- res = await fetch(`${MESHY_BASE}${base}/${encodeURIComponent(taskId)}`, { headers });
+ res = await fetchUpstream(`${MESHY_BASE}${base}/${encodeURIComponent(taskId)}`, { headers }, { name: 'meshy', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `meshy poll failed: ${err?.message}` };
}
diff --git a/api/_providers/replicate.js b/api/_providers/replicate.js
index e6dc11a74..e7a99f3d2 100644
--- a/api/_providers/replicate.js
+++ b/api/_providers/replicate.js
@@ -33,6 +33,8 @@
// polls Replicate to translate the predictions API state into our 4-state
// machine.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
const REPLICATE_BASE = 'https://api.replicate.com/v1';
function readEnv(name) {
@@ -93,7 +95,7 @@ const _latestVersionCache = new Map();
async function resolveLatestVersion(owner, name, authHeaders) {
const slug = `${owner}/${name}`;
if (_latestVersionCache.has(slug)) return _latestVersionCache.get(slug);
- const res = await fetch(`${REPLICATE_BASE}/models/${owner}/${name}`, { headers: authHeaders });
+ const res = await fetchUpstream(`${REPLICATE_BASE}/models/${owner}/${name}`, { headers: authHeaders }, { name: 'replicate', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const data = await res.json().catch(() => ({}));
if (!res.ok || !data?.latest_version?.id) {
throw Object.assign(
@@ -299,7 +301,7 @@ async function createPrediction(endpoint, body, authHeaders) {
for (let attempt = 0; attempt < THROTTLE_MAX_ATTEMPTS; attempt++) {
let response;
try {
- response = await fetch(endpoint, { method: 'POST', headers: authHeaders, body });
+ response = await fetchUpstream(endpoint, { method: 'POST', headers: authHeaders, body }, { name: 'replicate', timeoutMs: 30_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`replicate submit failed: ${err?.message}`), {
code: 'provider_unreachable',
@@ -495,9 +497,9 @@ export function createRegenProvider({ apiToken } = {}) {
let response;
try {
- response = await fetch(`${REPLICATE_BASE}/predictions/${encodeURIComponent(extJobId)}`, {
+ response = await fetchUpstream(`${REPLICATE_BASE}/predictions/${encodeURIComponent(extJobId)}`, {
headers: authHeaders,
- });
+ }, { name: 'replicate', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `provider poll failed: ${err?.message}` };
}
diff --git a/api/_providers/rodin.js b/api/_providers/rodin.js
index 0125d991d..1eb8716c3 100644
--- a/api/_providers/rodin.js
+++ b/api/_providers/rodin.js
@@ -19,6 +19,8 @@
// BYOK only: the caller supplies their own Rodin key. No platform key — when
// absent, the forge endpoint never reaches this module.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
const RODIN_BASE = 'https://api.hyper3d.com/api/v2';
// Quad mode accepts 1,000–200,000 for the poly target; clamp the tier budget
@@ -97,7 +99,7 @@ export function createRodinProvider(apiKey) {
async function submit(form) {
let res;
try {
- res = await fetch(`${RODIN_BASE}/rodin`, { method: 'POST', headers: authHeader, body: form });
+ res = await fetchUpstream(`${RODIN_BASE}/rodin`, { method: 'POST', headers: authHeader, body: form }, { name: 'rodin', timeoutMs: 60_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`rodin unreachable: ${err?.message}`), {
code: 'provider_unreachable',
@@ -136,7 +138,7 @@ export function createRodinProvider(apiKey) {
async imageTo3d({ imageUrl, prompt, tier }) {
let imgRes;
try {
- imgRes = await fetch(imageUrl);
+ imgRes = await fetchUpstream(imageUrl, {}, { timeoutMs: 20_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`could not fetch reference image: ${err?.message}`), {
code: 'bad_image',
@@ -167,11 +169,11 @@ export function createRodinProvider(apiKey) {
async status({ kind, taskId }) {
let res;
try {
- res = await fetch(`${RODIN_BASE}/status`, {
+ res = await fetchUpstream(`${RODIN_BASE}/status`, {
method: 'POST',
headers: { ...authHeader, 'content-type': 'application/json' },
body: JSON.stringify({ subscription_key: kind }),
- });
+ }, { name: 'rodin', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `rodin poll failed: ${err?.message}` };
}
@@ -190,11 +192,11 @@ export function createRodinProvider(apiKey) {
// Done — fetch the downloadable file list and pull the GLB url.
let dl;
try {
- dl = await fetch(`${RODIN_BASE}/download`, {
+ dl = await fetchUpstream(`${RODIN_BASE}/download`, {
method: 'POST',
headers: { ...authHeader, 'content-type': 'application/json' },
body: JSON.stringify({ task_uuid: taskId }),
- });
+ }, { name: 'rodin', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `rodin download list failed: ${err?.message}` };
}
diff --git a/api/_providers/stability.js b/api/_providers/stability.js
index 964a7f626..dac870e32 100644
--- a/api/_providers/stability.js
+++ b/api/_providers/stability.js
@@ -12,6 +12,8 @@
// supplies their own Stability key (sk-…); when absent the forge endpoint never
// reaches this module.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
const STABILITY_BASE = 'https://api.stability.ai/v2beta/3d';
const ENDPOINT = 'stable-fast-3d';
@@ -31,7 +33,7 @@ export function createStabilityProvider(apiKey) {
async imageTo3d({ imageUrl, tier }) {
let imgRes;
try {
- imgRes = await fetch(imageUrl);
+ imgRes = await fetchUpstream(imageUrl, {}, { timeoutMs: 20_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`could not fetch reference image: ${err?.message}`), {
code: 'bad_image',
@@ -54,11 +56,11 @@ export function createStabilityProvider(apiKey) {
let res;
try {
- res = await fetch(`${STABILITY_BASE}/${ENDPOINT}`, {
+ res = await fetchUpstream(`${STABILITY_BASE}/${ENDPOINT}`, {
method: 'POST',
headers: { authorization: `Bearer ${apiKey}`, accept: 'model/gltf-binary' },
body: form,
- });
+ }, { name: 'stability', timeoutMs: 120_000, attempts: 1, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`stability unreachable: ${err?.message}`), {
code: 'provider_unreachable',
diff --git a/api/_providers/tripo.js b/api/_providers/tripo.js
index 5ff346c8c..b1c49557a 100644
--- a/api/_providers/tripo.js
+++ b/api/_providers/tripo.js
@@ -13,6 +13,8 @@
// BYOK only: the caller passes the user's Tripo key (tsk_…). The model version
// defaults to v3.1 and is overridable via TRIPO_MODEL_VERSION.
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
const TRIPO_BASE = 'https://api.tripo3d.ai/v2/openapi';
const DEFAULT_MODEL_VERSION = 'v3.1-20260211';
@@ -90,11 +92,11 @@ export function createTripoProvider(apiKey) {
async function createTask(body) {
let res;
try {
- res = await fetch(`${TRIPO_BASE}/task`, {
+ res = await fetchUpstream(`${TRIPO_BASE}/task`, {
method: 'POST',
headers,
body: JSON.stringify(body),
- });
+ }, { name: 'tripo', timeoutMs: 30_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`tripo unreachable: ${err?.message}`), {
code: 'provider_unreachable',
@@ -151,11 +153,11 @@ export function createTripoProvider(apiKey) {
try {
// Let fetch set the multipart content-type (with boundary) itself — only
// forward the bearer, never the JSON content-type from `headers`.
- res = await fetch(`${TRIPO_BASE}/upload`, {
+ res = await fetchUpstream(`${TRIPO_BASE}/upload`, {
method: 'POST',
headers: { authorization: headers.authorization },
body: form,
- });
+ }, { name: 'tripo', timeoutMs: 60_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(new Error(`tripo upload unreachable: ${err?.message}`), {
code: 'provider_unreachable',
@@ -235,7 +237,7 @@ export function createTripoProvider(apiKey) {
async status({ taskId }) {
let res;
try {
- res = await fetch(`${TRIPO_BASE}/task/${encodeURIComponent(taskId)}`, { headers });
+ res = await fetchUpstream(`${TRIPO_BASE}/task/${encodeURIComponent(taskId)}`, { headers }, { name: 'tripo', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `tripo poll failed: ${err?.message}` };
}
diff --git a/api/agent-wallet-bridge.js b/api/agent-wallet-bridge.js
index a7ef040e6..2d96b13e3 100644
--- a/api/agent-wallet-bridge.js
+++ b/api/agent-wallet-bridge.js
@@ -26,6 +26,7 @@ import { limits, clientIp } from './_lib/rate-limit.js';
import { createSolanaSigner } from './_lib/x402/a2a-client.js';
import { enforceCap, commit, rollbackReservation } from './_lib/x402-spending-cap.js';
import { env } from './_lib/env.js';
+import { fetchUpstream } from './_lib/upstream-fetch.js';
const SOLANA_RPC_URL = env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com';
const USDC_MINT_SOLANA = 'EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v';
@@ -115,11 +116,13 @@ async function walletStatus() {
}
async function fetch402(endpoint, method, body) {
- const res = await fetch(endpoint, {
+ // Any status is a legitimate answer here (we are probing for a 402), so the
+ // wrapper only adds the timeout and a retry on a dropped connection.
+ const res = await fetchUpstream(endpoint, {
method,
headers: { accept: 'application/json', ...(body ? { 'content-type': 'application/json' } : {}) },
body: body ? JSON.stringify(body) : undefined,
- });
+ }, { timeoutMs: 15_000, attempts: 2, okWhen: () => true });
if (res.status !== 402) {
const text = (await res.text()).slice(0, 300);
throw new Error(`expected a 402 challenge from ${endpoint}, got ${res.status}: ${text}`);
@@ -209,7 +212,9 @@ async function executePayment({ endpoint, method, body, payerAddress, emit }) {
if (echo) paymentPayload.extensions = { 'builder-code': echo };
emit({ stage: 'submitting' });
- const paidRes = await fetch(endpoint, {
+ // A signed payment must never be re-sent (a retry could double-spend), so
+ // this is a single attempt: timeout only.
+ const paidRes = await fetchUpstream(endpoint, {
method,
headers: {
accept: 'application/json',
@@ -217,7 +222,7 @@ async function executePayment({ endpoint, method, body, payerAddress, emit }) {
...(body ? { 'content-type': 'application/json' } : {}),
},
body: body ? JSON.stringify(body) : undefined,
- });
+ }, { timeoutMs: 30_000, attempts: 1, okWhen: () => true });
const resultText = await paidRes.text();
let result = null;
try { result = JSON.parse(resultText); } catch { result = { raw: resultText.slice(0, 1000) }; }
diff --git a/api/agents/[id]/memory-seed-x.js b/api/agents/[id]/memory-seed-x.js
index b4b01a3ca..c20a8fb39 100644
--- a/api/agents/[id]/memory-seed-x.js
+++ b/api/agents/[id]/memory-seed-x.js
@@ -37,6 +37,7 @@ import { revokeAgentSeedConsent } from '../../_lib/x-seed-consent.js';
import { X_SEED_REQUIRED_SCOPES, missingScopes } from '../../_lib/x-scopes.js';
import { decryptToken, encryptToken } from '../../auth/x/[action].js';
+import { fetchUpstream } from '../../_lib/upstream-fetch.js';
// ── Token refresh ─────────────────────────────────────────────────────────────
async function refreshXToken(conn) {
@@ -45,7 +46,7 @@ async function refreshXToken(conn) {
const creds = Buffer.from(`${env.X_OAUTH_CLIENT_ID}:${env.X_OAUTH_CLIENT_SECRET}`).toString(
'base64',
);
- const res = await fetch('https://api.twitter.com/2/oauth2/token', {
+ const res = await fetchUpstream('https://api.twitter.com/2/oauth2/token', {
method: 'POST',
headers: {
'content-type': 'application/x-www-form-urlencoded',
@@ -56,7 +57,7 @@ async function refreshXToken(conn) {
refresh_token: decRefresh,
client_id: env.X_OAUTH_CLIENT_ID,
}).toString(),
- });
+ }, { name: 'x-api', timeoutMs: 15_000, attempts: 1, okWhen: () => true });
if (!res.ok) {
const txt = await res.text();
throw Object.assign(new Error('token refresh failed: ' + txt), { status: 502 });
@@ -346,10 +347,7 @@ async function handlePost(req, res, agentId) {
);
}
- const profileRes = await fetch(
- 'https://api.twitter.com/2/users/me?user.fields=name,username,description,public_metrics',
- { headers: { authorization: `Bearer ${accessToken}` } },
- );
+ const profileRes = await fetchUpstream('https://api.twitter.com/2/users/me?user.fields=name,username,description,public_metrics', { headers: { authorization: `Bearer ${accessToken}` } }, { name: 'x-api', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
if (!profileRes.ok) {
const f = await describeXFailure(profileRes, 'profile');
return abandonSeed(res, agentId, f.status, f.code, f.description, f.extra);
@@ -371,11 +369,8 @@ async function handlePost(req, res, agentId) {
);
}
- const postsRes = await fetch(
- `https://api.twitter.com/2/users/${rawProfile.id}/tweets?max_results=${X_SEED_LIMITS.maxPosts}` +
- '&exclude=retweets,replies&tweet.fields=text,created_at',
- { headers: { authorization: `Bearer ${accessToken}` } },
- );
+ const postsRes = await fetchUpstream(`https://api.twitter.com/2/users/${rawProfile.id}/tweets?max_results=${X_SEED_LIMITS.maxPosts}` +
+ '&exclude=retweets,replies&tweet.fields=text,created_at', { headers: { authorization: `Bearer ${accessToken}` } }, { name: 'x-api', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
if (!postsRes.ok) {
const f = await describeXFailure(postsRes, 'posts');
return abandonSeed(res, agentId, f.status, f.code, f.description, f.extra);
diff --git a/api/agents/_id/memory/pin.js b/api/agents/_id/memory/pin.js
index 373d2adae..406db7a3d 100644
--- a/api/agents/_id/memory/pin.js
+++ b/api/agents/_id/memory/pin.js
@@ -5,6 +5,7 @@ import { cors, json, method, readJson, wrap, error, rateLimited } from '../../..
import { limits, clientIp } from '../../../_lib/rate-limit.js';
import { parse } from '../../../_lib/validate.js';
+import { pinToIPFS, ipfsPinningConfigured } from '../../../_lib/ipfs-pin.js';
const MAX_BYTES = 512 * 1024; // 512 KB per encrypted file
const bodySchema = z.object({
@@ -25,22 +26,6 @@ const bodySchema = z.object({
.refine((s) => s.replace(/[\r\n]/g, '').length % 4 === 0, 'data must be base64'),
});
-async function pinViaPinata(buf, filename) {
- const form = new FormData();
- form.append('file', new Blob([buf]), filename);
- const resp = await fetch('https://api.pinata.cloud/pinning/pinFileToIPFS', {
- method: 'POST',
- headers: { Authorization: `Bearer ${process.env.PINATA_JWT}` },
- body: form,
- });
- if (!resp.ok) {
- const detail = await resp.text().catch(() => '');
- throw Object.assign(new Error(`Pinata error ${resp.status}`), { status: 502, detail });
- }
- const data = await resp.json();
- return data.IpfsHash;
-}
-
// POST /api/agents/:id/memory/pin
// Body: { filename, data } — data is base64-encoded bytes
// Returns: { cid }
@@ -74,11 +59,14 @@ export default wrap(async (req, res) => {
return error(res, 413, 'payload_too_large', 'file exceeds 512 KB limit');
}
- if (!process.env.PINATA_JWT) {
+ if (!ipfsPinningConfigured()) {
return error(res, 503, 'pinning_unconfigured', 'no pinning provider configured');
}
- const cid = await pinViaPinata(buf, body.filename);
+ // Pinata first, web3.storage on failure: each rung carries its own deadline
+ // and breaker (api/_lib/ipfs-pin.js), so a sick provider fails over instead
+ // of holding this request open.
+ const { cid } = await pinToIPFS(buf, body.filename);
// Record the CID against this agent so the read proxy
// (GET /api/agents/:id/memory/:cid) can verify ownership of the CID itself,
diff --git a/api/agents/alpha.js b/api/agents/alpha.js
index ff63d67d2..12fca17f0 100644
--- a/api/agents/alpha.js
+++ b/api/agents/alpha.js
@@ -31,6 +31,8 @@ import { getBondingCurveState, getBuyQuote, getGraduationProgress } from '../_li
import { connectPumpFunFeed, recentBuffered, recentGraduations } from '../_lib/pumpfun-ws-feed.js';
import { llmComplete, LlmUnavailableError } from '../_lib/llm.js';
import { buildReadPrompt, parseReadJson, validateRead } from '../_lib/alpha-read.js';
+import { pumpFetchJson } from '../_lib/pump-feed-fetch.js';
+import { fetchUpstreamJson } from '../_lib/upstream-fetch.js';
const NETWORKS = new Set(['mainnet', 'devnet']);
const netOf = (v) => (NETWORKS.has(v) ? v : 'mainnet');
@@ -65,17 +67,23 @@ async function loadAgent(id, userId) {
// ── live launch candidates ──────────────────────────────────────────────────────
+// The read only consumes `created_timestamp` (launch age). pump.fun is the
+// source of truth; when it does not answer, DexScreener's pair listing carries
+// the pair creation time for the same mint, so the age still comes from a real
+// record rather than going missing on a pump.fun blip.
async function fetchPumpCoin(mint) {
- const ctrl = new AbortController();
- const tid = setTimeout(() => ctrl.abort(), 2500);
+ const { ok, body } = await pumpFetchJson(`${PUMPFUN_COIN_API}/${encodeURIComponent(mint)}`, { timeoutMs: 2500, retries: 1 });
+ if (ok && body && typeof body === 'object') return body;
try {
- const r = await fetch(`${PUMPFUN_COIN_API}/${encodeURIComponent(mint)}`, {
- signal: ctrl.signal,
- headers: { accept: 'application/json', 'user-agent': 'three.ws-alpha-copilot/1' },
+ const data = await fetchUpstreamJson(`https://api.dexscreener.com/latest/dex/tokens/${encodeURIComponent(mint)}`, {}, {
+ timeoutMs: 2500, attempts: 1, name: 'dexscreener',
});
- if (!r.ok) return null;
- return await r.json();
- } catch { return null; } finally { clearTimeout(tid); }
+ const pairs = (Array.isArray(data?.pairs) ? data.pairs : []).filter((p) => p?.chainId === 'solana');
+ if (!pairs.length) return null;
+ pairs.sort((a, b) => (a.pairCreatedAt ?? Infinity) - (b.pairCreatedAt ?? Infinity));
+ const createdMs = Number(pairs[0].pairCreatedAt);
+ return Number.isFinite(createdMs) ? { mint, created_timestamp: createdMs } : null;
+ } catch { return null; }
}
// Briefly tap the live PumpPortal mint feed to gather fresh launches. Bounded so
diff --git a/api/agents/eth-vanity.js b/api/agents/eth-vanity.js
index b1b978498..8efb8e8f1 100644
--- a/api/agents/eth-vanity.js
+++ b/api/agents/eth-vanity.js
@@ -34,29 +34,20 @@ import { sql } from '../_lib/db.js';
import { cors, json, method, error, readJson, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { keccak_256 } from '@noble/hashes/sha3';
+import { fetchAnyJson } from '../_lib/upstream-fetch.js';
+import { evmRpcEndpoints } from '../_lib/evm/rpc.js';
+import { CHAIN_BY_ID } from '../_lib/erc8004-chains.js';
const HEX_RE = /^0x[0-9a-f]+$/i;
const ADDR_RE = /^0x[0-9a-f]{40}$/i;
const HASH_RE = /^0x[0-9a-f]{64}$/i;
const TX_RE = /^0x[0-9a-f]{64}$/i;
-/**
- * Curated list of known-good public RPCs. We don't hold secrets here, and
- * anyone can spam these — but they're rate-limited per chain, and we only
- * call them on deploy confirmation (rare). If a chain isn't listed, the
- * deploy is recorded *unverified* with a warning flag.
- */
-const RPCS = {
- 1: 'https://eth.drpc.org', // cloudflare-eth.com is sunset (-32046)
- 8453: 'https://mainnet.base.org',
- 10: 'https://mainnet.optimism.io',
- 42161: 'https://arb1.arbitrum.io/rpc',
- 137: 'https://polygon-rpc.com',
- 56: 'https://bsc-dataseed.bnbchain.org',
- 43114: 'https://api.avax.network/ext/bc/C/rpc',
- 11155111: 'https://ethereum-sepolia-rpc.publicnode.com',
- 84532: 'https://sepolia.base.org',
-};
+// The chains a deployment can be verified on are exactly the chains the shared
+// EVM RPC layer knows (erc8004-chains.js): operator override, keyed Alchemy,
+// then the chain's curated public nodes, tried in that order. A chain outside
+// that table records the deploy *unverified* with a warning flag.
+const SUPPORTED_CHAIN_IDS = Object.keys(CHAIN_BY_ID).map(Number);
async function resolveAuth(req) {
const session = await getSessionUser(req);
@@ -90,24 +81,27 @@ function _verifyCreate2(deployer, salt, initCodeHash, predicted) {
/**
* Verify on-chain that bytecode exists at `address` on `chainId`. Uses
- * eth_getCode against a public RPC. Returns:
+ * eth_getCode across the chain's priority-ordered RPC endpoints, each on its
+ * own deadline. Returns:
* { ok: true, bytecodeHash } → contract is deployed
* { ok: false, reason } → not deployed / RPC unreachable / wrong chain
*
- * We don't hard-fail when the chain isn't in our RPC table — the caller
+ * We don't hard-fail when the chain isn't in the shared table; the caller
* can choose to record the deployment as unverified.
*/
async function _verifyDeployed(chainId, address) {
- const rpc = RPCS[Number(chainId)];
- if (!rpc) return { ok: false, reason: 'unsupported_chain', supported: Object.keys(RPCS).map(Number) };
+ const endpoints = evmRpcEndpoints(Number(chainId));
+ if (!CHAIN_BY_ID[Number(chainId)] || !endpoints.length) {
+ return { ok: false, reason: 'unsupported_chain', supported: SUPPORTED_CHAIN_IDS };
+ }
try {
- const r = await fetch(rpc, {
+ // Operator override, keyed Alchemy, then the chain's public nodes in turn,
+ // so one node rate-limiting us never records a real deployment as unverified.
+ const { value: data } = await fetchAnyJson(endpoints, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ jsonrpc: '2.0', id: 1, method: 'eth_getCode', params: [address, 'latest'] }),
- });
- if (!r.ok) return { ok: false, reason: `rpc_${r.status}` };
- const data = await r.json();
+ }, { name: `evm-getcode:${chainId}`, timeoutMs: 6_000, label: 'eth_getCode' });
if (data.error) return { ok: false, reason: data.error.message || 'rpc_error' };
const code = String(data.result || '0x');
if (code === '0x' || code === '0x0') return { ok: false, reason: 'no_code_at_address' };
diff --git a/api/agents/nfts.js b/api/agents/nfts.js
index c5a6c350f..cc2a1d5d8 100644
--- a/api/agents/nfts.js
+++ b/api/agents/nfts.js
@@ -20,6 +20,7 @@ import { cors, json, method, error, wrap, rateLimited, serverError } from '../_l
import { limits, clientIp } from '../_lib/rate-limit.js';
import { getSessionUser, authenticateBearer, extractBearer, hasScope } from '../_lib/auth.js';
import { dasRpcUrl } from '../_lib/nft-gate.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const BASE58_RE = /^[1-9A-HJ-NP-Za-km-z]{32,44}$/;
@@ -89,7 +90,7 @@ async function handlePortfolio(req, res) {
return error(res, e.status || 503, e.code || 'not_configured', e.message);
}
- const resp = await fetch(rpcUrl, {
+ const resp = await fetchUpstream(rpcUrl, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -109,7 +110,7 @@ async function handlePortfolio(req, res) {
},
},
}),
- });
+ }, { name: 'helius-das', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (!resp.ok) {
const txt = await resp.text().catch(() => resp.status.toString());
@@ -160,7 +161,7 @@ async function handleActivity(req, res) {
}
const url = baseUrl.replace('{address}', encodeURIComponent(wallet)) + `&limit=${limit}`;
- const resp = await fetch(url);
+ const resp = await fetchUpstream(url, {}, { name: 'helius-enhanced-tx', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (!resp.ok) {
const txt = await resp.text().catch(() => resp.status.toString());
diff --git a/api/agents/onchain/[action].js b/api/agents/onchain/[action].js
index 1e263c56f..a23c226b7 100644
--- a/api/agents/onchain/[action].js
+++ b/api/agents/onchain/[action].js
@@ -41,6 +41,7 @@ import {
collectionAuthoritySigner,
} from '../../_lib/solana-collection.js';
+import { pinToIPFS, ipfsPinningConfigured } from '../../_lib/ipfs-pin.js';
export default wrap(async (req, res) => {
const action = req.query?.action;
switch (action) {
@@ -94,7 +95,8 @@ const prepBodySchema = z.object({
.optional(),
});
-// Pin a manifest. Try web3.storage first for true IPFS pinning. If unavailable,
+// Pin a manifest. Try the IPFS provider chain first (Pinata, then web3.storage,
+// api/_lib/ipfs-pin.js) for true IPFS pinning. If unavailable,
// fall back to R2: compute the real CIDv1 (raw codec, sha2-256 multihash) from
// the bytes so the cid is verifiable IPFS content-addressing, and return the
// R2 HTTPS URL as the resolvable location. The on-chain registry accepts both
@@ -102,21 +104,12 @@ const prepBodySchema = z.object({
// is a first-class result — never a stub.
async function pinManifest(manifest) {
const bytes = Buffer.from(JSON.stringify(manifest), 'utf-8');
- const token = process.env.WEB3_STORAGE_TOKEN;
- if (token) {
+ if (ipfsPinningConfigured()) {
try {
- const r = await fetch('https://api.web3.storage/upload', {
- method: 'POST',
- headers: { Authorization: `Bearer ${token}` },
- body: bytes,
- });
- if (r.ok) {
- const data = await r.json();
- if (data.cid) return { cid: data.cid, uri: `ipfs://${data.cid}` };
- }
- console.warn('[onchain/prep] web3.storage returned', r.status);
+ const pinned = await pinToIPFS(bytes, 'agent-manifest.json');
+ if (pinned?.cid) return { cid: pinned.cid, uri: `ipfs://${pinned.cid}` };
} catch (e) {
- console.warn('[onchain/prep] web3.storage pin failed:', e.message);
+ console.warn('[onchain/prep] IPFS pin failed:', e.message);
}
}
// R2-backed fallback: compute the real raw-codec CIDv1 so the bytes are
diff --git a/api/agents/pumpfun/[action].js b/api/agents/pumpfun/[action].js
index 8a2c306b9..75b25d90e 100644
--- a/api/agents/pumpfun/[action].js
+++ b/api/agents/pumpfun/[action].js
@@ -30,6 +30,7 @@ import { loadAgentForSigning, solanaConnection } from '../../_lib/agent-pumpfun.
import { submitProtected } from '../../_lib/execution-engine.js';
import { reserveSpend, finalizeSpend, releaseSpend } from '../../_lib/agent-spend-policy.js';
import { grindMintKeypair } from '../../_lib/pump-vanity.js';
+import { isRpcOutageError } from '../../_lib/rpc-degrade.js';
import { sql } from '../../_lib/db.js';
import { Keypair, PublicKey } from '@solana/web3.js';
import { z } from 'zod';
@@ -1281,7 +1282,16 @@ async function handleSwap(req, res, id) {
// Detect graduation. If still on bonding curve, delegate to /buy or /sell.
const { PumpSdk, bondingCurvePda } = await import('@pump-fun/pump-sdk');
const pumpSdk = new PumpSdk();
- const bcInfo = await conn.getAccountInfo(bondingCurvePda(mint));
+ let bcInfo;
+ try {
+ bcInfo = await conn.getAccountInfo(bondingCurvePda(mint));
+ } catch (err) {
+ // Graduation cannot be decided without the curve account: answer with a
+ // typed retryable 503 rather than guessing a venue or 500ing.
+ if (!isRpcOutageError(err)) throw err;
+ res.setHeader('Retry-After', '15');
+ return error(res, 503, 'rpc_unavailable', 'Solana RPC is temporarily unavailable, retry shortly', { retry_after: 15 });
+ }
const bc = bcInfo ? pumpSdk.decodeBondingCurveNullable(bcInfo) : null;
const graduated = !bc || bc.complete;
diff --git a/api/agents/register/[action].js b/api/agents/register/[action].js
index c1a6119e1..ac3c865fb 100644
--- a/api/agents/register/[action].js
+++ b/api/agents/register/[action].js
@@ -17,6 +17,7 @@ import { PutObjectCommand } from '@aws-sdk/client-s3';
import { AbiCoder, getAddress, keccak256, toUtf8Bytes } from 'ethers';
import { z } from 'zod';
+import { fetchUpstream } from '../../_lib/upstream-fetch.js';
// ── prep ──────────────────────────────────────────────────────────────────────
const prepSchema = z.object({
@@ -51,11 +52,11 @@ async function pinRegistrationJson(jsonObj) {
const web3Token = process.env.WEB3_STORAGE_TOKEN;
if (web3Token) {
try {
- const res = await fetch('https://api.web3.storage/upload', {
+ const res = await fetchUpstream('https://api.web3.storage/upload', {
method: 'POST',
headers: { Authorization: `Bearer ${web3Token}` },
body: jsonBytes,
- });
+ }, { timeoutMs: 60_000, attempts: 2, okWhen: () => true });
if (res.ok) {
const r = await res.json();
if (r.cid) {
@@ -76,11 +77,11 @@ async function pinRegistrationJson(jsonObj) {
try {
const form = new FormData();
form.append('file', new Blob([jsonBytes], { type: 'application/json' }), 'agent-manifest.json');
- const res = await fetch('https://api.pinata.cloud/pinning/pinFileToIPFS', {
+ const res = await fetchUpstream('https://api.pinata.cloud/pinning/pinFileToIPFS', {
method: 'POST',
headers: { Authorization: `Bearer ${pinataJwt}` },
body: form,
- });
+ }, { timeoutMs: 60_000, attempts: 2, okWhen: () => true });
if (res.ok) {
const r = await res.json();
if (r.IpfsHash) {
diff --git a/api/agents/sns.js b/api/agents/sns.js
index 99a90859e..7e849d3e2 100644
--- a/api/agents/sns.js
+++ b/api/agents/sns.js
@@ -31,6 +31,7 @@ import { submitProtected } from '../_lib/execution-engine.js';
import { cors, json, method, error, readJson, rateLimited, serverError } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { env } from '../_lib/env.js';
+import { fetchUpstreamJson } from '../_lib/upstream-fetch.js';
import { loadAgentForSigning, solanaConnection } from '../_lib/agent-pumpfun.js';
const SNS_API = 'https://sns-api.bonfida.com';
@@ -50,10 +51,11 @@ function normalizeDomain(input) {
return trimmed;
}
+// Bonfida's API is the only SNS indexer; on-chain resolution below is the
+// fallback for a single domain, so the indexer call just needs to fail fast
+// (timeout + breaker) rather than hang a profile page on a slow response.
async function snsFetch(path) {
- const r = await fetch(`${SNS_API}${path}`, { headers: { accept: 'application/json' } });
- if (!r.ok) throw new Error(`sns api ${r.status}`);
- return r.json();
+ return fetchUpstreamJson(`${SNS_API}${path}`, {}, { name: 'bonfida-sns', timeoutMs: 6_000, attempts: 2, label: 'sns api' });
}
async function fetchOwnedDomains(address) {
diff --git a/api/agents/solana-trade.js b/api/agents/solana-trade.js
index e5e582f17..50ade9801 100644
--- a/api/agents/solana-trade.js
+++ b/api/agents/solana-trade.js
@@ -27,6 +27,7 @@ import { recoverSolanaAgentKeypair } from '../_lib/agent-wallet.js';
import { solanaConnection, solanaPublicConnection } from '../_lib/agent-pumpfun.js';
import { PublicKey } from '@solana/web3.js';
import { cacheSet } from '../_lib/cache.js';
+import { isRpcOutageError } from '../_lib/rpc-degrade.js';
import { logAudit } from '../_lib/audit.js';
import { explorerTxUrl } from '../_lib/avatar-wallet.js';
import {
@@ -620,7 +621,13 @@ export async function buildTradeInstructions({ side, conn, network, mintPk, owne
await import('@pump-fun/pump-sdk');
const online = new OnlinePumpSdk(conn);
const sdk = new PumpSdk();
- const mintInfo = await conn.getAccountInfo(mintPk);
+ let mintInfo;
+ try {
+ mintInfo = await conn.getAccountInfo(mintPk);
+ } catch (err) {
+ if (isRpcOutageError(err)) throw typed(503, 'rpc_unavailable', 'Solana RPC is temporarily unavailable, retry shortly');
+ throw err;
+ }
if (!mintInfo) throw typed(404, 'mint_not_found', 'mint not found on this network');
const tokenProgram = resolveTokenProgramForMintOwner(mintInfo.owner);
diff --git a/api/agents/solana-wallet.js b/api/agents/solana-wallet.js
index b09427568..fbea35911 100644
--- a/api/agents/solana-wallet.js
+++ b/api/agents/solana-wallet.js
@@ -24,6 +24,7 @@ import { grindMintKeypair, estimateAttempts, BASE58_ALPHABET, addressMatchesPatt
import { randomUUID } from 'node:crypto';
import { recordEvent } from '../_lib/usage.js';
import { cacheGet, cacheSet } from '../_lib/cache.js';
+import { withDeadline, staleEnvelope } from '../_lib/rpc-degrade.js';
import { logAudit } from '../_lib/audit.js';
import { explorerTxUrl } from '../_lib/avatar-wallet.js';
import {
@@ -934,19 +935,40 @@ async function handleHoldings(req, res, id) {
const conn = solanaConnection(network);
const owner = new PublicKey(address);
+ // Last-good tier: the most recent holdings a fully successful read produced.
+ // During an RPC outage the asset picker shows the real (older) list marked
+ // stale instead of a 502; only a wallet never read successfully gets the 503.
+ const lkgKey = `sol:holdings:lkg:${network}:${address}`;
+ const serveLastGood = async (balanceError) => {
+ const lkg = await cacheGet(lkgKey);
+ if (lkg && lkg.data && typeof lkg.at === 'number') {
+ res.setHeader('cache-control', 'no-store');
+ return json(res, 200, {
+ data: staleEnvelope({ ...lkg.data, is_owner: isOwner, balance_error: balanceError }, lkg.at),
+ });
+ }
+ res.setHeader('Retry-After', '15');
+ return error(res, 503, 'rpc_unavailable', 'could not read the wallet balance right now, retry shortly', { retry_after: 15 });
+ };
+
let sol = null;
- try {
- sol = (await conn.getBalance(owner, 'confirmed')) / 1e9;
- } catch {
- return error(res, 502, 'rpc_error', 'could not read the wallet balance — try again');
- }
+ const balResult = await _solRpcWithBackoffFallback(
+ conn,
+ solanaPublicConnection(network),
+ (c) => c.getBalance(owner, 'confirmed'),
+ 'holdings.getBalance',
+ );
+ if (!balResult.ok) return serveLastGood(classifyBalanceError(balResult.error));
+ sol = balResult.value / 1e9;
const tokens = [];
const usdcMint = USDC_MINT_BY_CLUSTER[network];
+ let programsRead = 0;
for (const programId of [TOKEN_PROGRAM_ID, TOKEN_2022_PROGRAM_ID]) {
let resp;
try {
resp = await conn.getParsedTokenAccountsByOwner(owner, { programId });
+ programsRead += 1;
} catch {
continue; // one token program failing shouldn't blank the whole list
}
@@ -966,7 +988,12 @@ async function handleHoldings(req, res, id) {
}
tokens.sort((a, b) => Number(b.ui_amount) - Number(a.ui_amount));
- return json(res, 200, { data: { address, network, sol, tokens, is_owner: isOwner } });
+ // Only a FULL read (balance + both token programs) is worth remembering: a
+ // half-read list would later be served as if it were the whole wallet.
+ if (programsRead === 2) {
+ cacheSet(lkgKey, { data: { address, network, sol, tokens }, at: Date.now() }, 24 * 3600).catch(() => {});
+ }
+ return json(res, 200, { data: { address, network, sol, tokens, is_owner: isOwner, ...(programsRead < 2 ? { partial: true } : {}) } });
}
// ── custody audit trail ───────────────────────────────────────────────────────
@@ -1323,7 +1350,16 @@ async function sweepWalletToAddress({ conn, fromKeypair, toPubkey, network }) {
// Build, sign, send and confirm a single v0 transaction. Throws on any failure or
// an unconfirmed/erroring result so callers treat the leg as not-yet-final.
async function sendV0({ conn, payer, ixs }) {
- const bh = await conn.getLatestBlockhash('confirmed');
+ let bh;
+ try {
+ bh = await conn.getLatestBlockhash('confirmed');
+ } catch (err) {
+ // Same typed 502 the unsigned-withdraw builder answers, so the caller sees
+ // "RPC blip, retry" rather than a sanitized 500 with no code to act on.
+ const e = new Error('could not fetch a recent blockhash, try again');
+ e.status = 502; e.code = 'rpc_error'; e.cause = err;
+ throw e;
+ }
const msg = new TransactionMessage({
payerKey: payer.publicKey,
recentBlockhash: bh.blockhash,
@@ -1705,6 +1741,8 @@ function warnNetworthThrottled(agentId, msg) {
// chain reads + real DB counts, plus the owner's reactivity preferences.
// PUT /api/agents/:id/solana/networth — owner-only: persist the reactivity prefs
// (CSRF-gated). Visitors render the agent exactly as the owner configured it.
+const NETWORTH_BALANCE_DEADLINE_MS = 6_000;
+
async function handleNetWorth(req, res, id) {
if (cors(req, res, { methods: 'GET,PUT,OPTIONS', credentials: true })) return;
if (!method(req, res, ['GET', 'PUT'])) return;
@@ -1776,7 +1814,10 @@ async function handleNetWorth(req, res, id) {
let balances = null;
try {
- balances = await getBalances({ chain: 'solana', address });
+ // Hard deadline: getBalances walks the RPC chain lane by lane, and a lane
+ // that stalls (instead of failing) would hold this request past the gateway
+ // timeout. Failing fast here lands in the same typed 503 path below.
+ balances = await withDeadline(getBalances({ chain: 'solana', address }), NETWORTH_BALANCE_DEADLINE_MS, 'networth balances');
} catch (e) {
// Hold-last-state contract: getBalances already serves a stale last-known-good
// snapshot when the live read fails, so reaching here means we have never read
diff --git a/api/agents/tokens/[action].js b/api/agents/tokens/[action].js
index f815f313a..7b00fa7ba 100644
--- a/api/agents/tokens/[action].js
+++ b/api/agents/tokens/[action].js
@@ -49,6 +49,7 @@ import {
PLAN_LIMITS,
} from '../../_lib/agent-token-plan.js';
+import { pinToIPFS, ipfsPinningConfigured } from '../../_lib/ipfs-pin.js';
// Heavy SDKs (Solana web3, Pump.fun, AWS S3 commands) are dynamic-imported
// inside the handlers that need them. Loading them at module top-level was
// pushing the function over Vercel's cold-start budget and producing
@@ -545,20 +546,14 @@ async function launchBodyFromPlan(raw, agentId) {
async function pinTokenMetadata(meta) {
const json = buildTokenMetadata(meta);
const bytes = Buffer.from(JSON.stringify(json), 'utf-8');
- const token = process.env.WEB3_STORAGE_TOKEN;
- if (token) {
+ // Pinata, then web3.storage, each with a deadline and a breaker
+ // (api/_lib/ipfs-pin.js); R2 below is the rung that never fails.
+ if (ipfsPinningConfigured()) {
try {
- const r = await fetch('https://api.web3.storage/upload', {
- method: 'POST',
- headers: { Authorization: `Bearer ${token}` },
- body: bytes,
- });
- if (r.ok) {
- const data = await r.json();
- if (data.cid) return { cid: data.cid, uri: `ipfs://${data.cid}` };
- }
+ const pinned = await pinToIPFS(bytes, 'token-metadata.json');
+ if (pinned?.cid) return { cid: pinned.cid, uri: `ipfs://${pinned.cid}` };
} catch (e) {
- console.warn('[token launch-prep] web3.storage pin failed:', e.message);
+ console.warn('[token launch-prep] IPFS pin failed:', e.message);
}
}
const hash = createHash('sha256').update(bytes).digest('hex');
diff --git a/api/asl-recognition.js b/api/asl-recognition.js
index 9201b3645..44cb69eef 100644
--- a/api/asl-recognition.js
+++ b/api/asl-recognition.js
@@ -20,6 +20,13 @@
import { cors, json, method, readJson, wrap, rateLimited } from './_lib/http.js';
import { limits, clientIp } from './_lib/rate-limit.js';
+import { fetchUpstream } from './_lib/upstream-fetch.js';
+
+// One breaker for the worker: the schema read is idempotent and retried, a
+// transcribe is a GPU inference and gets a single attempt with a long deadline.
+const WORKER_BREAKER = 'asl-recognition-worker';
+const SCHEMA_TIMEOUT_MS = 10_000;
+const TRANSCRIBE_TIMEOUT_MS = 60_000;
// 1500 frames × 390 floats serializes to ~10–20 MB; cap the body above that.
const MAX_BODY_BYTES = 24_000_000;
@@ -54,9 +61,9 @@ export default wrap(async (req, res) => {
if (_schemaCache && Date.now() - _schemaAt < SCHEMA_TTL_MS) {
return json(res, 200, _schemaCache);
}
- const r = await fetch(`${cfg.base}/schema`, {
+ const r = await fetchUpstream(`${cfg.base}/schema`, {
headers: { authorization: `Bearer ${cfg.key}` },
- });
+ }, { name: WORKER_BREAKER, timeoutMs: SCHEMA_TIMEOUT_MS, attempts: 3, okWhen: () => true });
if (!r.ok) return json(res, 502, { error: 'worker_error', status: r.status });
_schemaCache = await r.json();
_schemaAt = Date.now();
@@ -69,14 +76,14 @@ export default wrap(async (req, res) => {
if (!body || !Array.isArray(body.frames)) {
return json(res, 400, { error: 'bad_request', message: 'Body must be { frames: [[…]] }.' });
}
- const r = await fetch(`${cfg.base}/transcribe`, {
+ const r = await fetchUpstream(`${cfg.base}/transcribe`, {
method: 'POST',
headers: {
'content-type': 'application/json',
authorization: `Bearer ${cfg.key}`,
},
body: JSON.stringify({ frames: body.frames }),
- });
+ }, { name: WORKER_BREAKER, timeoutMs: TRANSCRIBE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
const out = await r.json().catch(() => ({}));
if (!r.ok) {
return json(res, r.status === 400 ? 400 : 502, {
diff --git a/api/auth/github/[action].js b/api/auth/github/[action].js
index 2ecc9aaf5..ff83324fe 100644
--- a/api/auth/github/[action].js
+++ b/api/auth/github/[action].js
@@ -29,6 +29,7 @@ import {
} from '../../_lib/github-token.js';
import { revokeGrant, verifyToken } from '../../_lib/github-api.js';
+import { fetchUpstream } from '../../_lib/upstream-fetch.js';
/** Where to send someone to mint a token with exactly the grants we accept. */
const TOKEN_CREATE_URL = `https://github.com/settings/tokens/new?description=three.ws%20agent%20memory&scopes=${RECOMMENDED_TOKEN_SCOPES.join(',')}`;
const TOKEN_MANAGE_URL = 'https://github.com/settings/tokens';
@@ -126,7 +127,7 @@ async function handleCallback(req, res) {
}
// Exchange code for access token
- const tokenRes = await fetch('https://github.com/login/oauth/access_token', {
+ const tokenRes = await fetchUpstream('https://github.com/login/oauth/access_token', {
method: 'POST',
headers: { accept: 'application/json', 'content-type': 'application/json' },
body: JSON.stringify({
@@ -135,7 +136,7 @@ async function handleCallback(req, res) {
code,
redirect_uri: `${env.APP_ORIGIN}/api/auth/github/callback`,
}),
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 1, okWhen: () => true });
if (!tokenRes.ok) {
return redirect(res, `${env.APP_ORIGIN}/settings?tab=connected-accounts&github=error`);
}
@@ -145,12 +146,12 @@ async function handleCallback(req, res) {
}
// Fetch GitHub user profile
- const profileRes = await fetch('https://api.github.com/user', {
+ const profileRes = await fetchUpstream('https://api.github.com/user', {
headers: {
authorization: `token ${tokenData.access_token}`,
'user-agent': 'three.ws/1.0',
},
- });
+ }, { name: 'github', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
if (!profileRes.ok) {
return redirect(res, `${env.APP_ORIGIN}/settings?tab=connected-accounts&github=error`);
}
diff --git a/api/auth/x/[action].js b/api/auth/x/[action].js
index ac0cad2e3..6ac19b494 100644
--- a/api/auth/x/[action].js
+++ b/api/auth/x/[action].js
@@ -27,6 +27,7 @@ import { env } from '../../_lib/env.js';
import { revokeAllSeedConsentsForUser } from '../../_lib/x-seed-consent.js';
import { resolveScopeSet } from '../../_lib/x-scopes.js';
+import { fetchUpstream } from '../../_lib/upstream-fetch.js';
// ── Signed-cookie PKCE state ─────────────────────────────────────────────────
const STATE_COOKIE = '__Host-xoa';
@@ -236,7 +237,7 @@ async function handleCallback(req, res) {
const creds = Buffer.from(`${env.X_OAUTH_CLIENT_ID}:${env.X_OAUTH_CLIENT_SECRET}`).toString(
'base64',
);
- const tokenRes = await fetch('https://api.twitter.com/2/oauth2/token', {
+ const tokenRes = await fetchUpstream('https://api.twitter.com/2/oauth2/token', {
method: 'POST',
headers: {
'content-type': 'application/x-www-form-urlencoded',
@@ -249,7 +250,7 @@ async function handleCallback(req, res) {
redirect_uri: `${env.APP_ORIGIN}/api/auth/x/callback`,
code_verifier: codeVerifier,
}).toString(),
- });
+ }, { name: 'x-api', timeoutMs: 15_000, attempts: 1, okWhen: () => true });
if (!tokenRes.ok) {
console.error('[x-oauth] token exchange failed', await tokenRes.text());
@@ -270,10 +271,7 @@ async function handleCallback(req, res) {
const expiresAt = new Date(Date.now() + (expires_in ?? 7200) * 1000).toISOString();
// Fetch X profile
- const profileRes = await fetch(
- 'https://api.twitter.com/2/users/me?user.fields=name,username,description,public_metrics',
- { headers: { authorization: `Bearer ${access_token}` } },
- );
+ const profileRes = await fetchUpstream('https://api.twitter.com/2/users/me?user.fields=name,username,description,public_metrics', { headers: { authorization: `Bearer ${access_token}` } }, { name: 'x-api', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
if (!profileRes.ok) {
console.error('[x-oauth] profile fetch failed', await profileRes.text());
return redirect(res, errorRedirect);
diff --git a/api/avatar/video-generate.js b/api/avatar/video-generate.js
index 07d6f5929..660f981f3 100644
--- a/api/avatar/video-generate.js
+++ b/api/avatar/video-generate.js
@@ -33,6 +33,12 @@ import { sql } from '../_lib/db.js';
import { publicUrl, thumbnailUrl } from '../_lib/r2.js';
import { getSessionUser } from '../_lib/auth.js';
import { env } from '../_lib/env.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
+// A submit starts a GPU job, so it is never retried; the deadline only bounds
+// the wait for the worker to accept it. Shared breaker with video-status.
+const WORKER_BREAKER = 'longcat-worker';
+const GENERATE_TIMEOUT_MS = 60_000;
import { resolveRenderParams, renderAvatarImage } from '../_lib/avatar-render.js';
// Rendering a fresh reference image runs headless chromium, so this handler can
@@ -279,7 +285,7 @@ export default wrap(async (req, res) => {
try {
let workerRes;
try {
- workerRes = await fetch(`${worker.url}/generate`, {
+ workerRes = await fetchUpstream(`${worker.url}/generate`, {
method: 'POST',
headers: {
'content-type': 'application/json',
@@ -290,7 +296,7 @@ export default wrap(async (req, res) => {
audio_url: audioUrl,
prompt: prompt || 'A person talking naturally.',
}),
- });
+ }, { name: WORKER_BREAKER, timeoutMs: GENERATE_TIMEOUT_MS, attempts: 1, okWhen: () => true });
} catch (err) {
await releaseReservation(reservationId);
return error(res, 502, 'worker_unreachable', err?.message || 'worker request failed');
diff --git a/api/avatar/video-status.js b/api/avatar/video-status.js
index 24379b958..5211335c0 100644
--- a/api/avatar/video-status.js
+++ b/api/avatar/video-status.js
@@ -32,6 +32,11 @@
import { cors, error, json, wrap } from '../_lib/http.js';
import { sql } from '../_lib/db.js';
import { getSessionUser } from '../_lib/auth.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
+// Job reads are idempotent: short deadline, retried, same breaker as the submit.
+const WORKER_BREAKER = 'longcat-worker';
+const STATUS_TIMEOUT_MS = 10_000;
// Mirror of workerConfig() in api/avatar/video-generate.js: resolve address and
// credential together and return null when either is unset. Throwing from inside
@@ -80,9 +85,9 @@ export default wrap(async (req, res) => {
let workerRes;
try {
- workerRes = await fetch(`${worker.url}/jobs/${encodeURIComponent(jobId)}`, {
+ workerRes = await fetchUpstream(`${worker.url}/jobs/${encodeURIComponent(jobId)}`, {
headers: { authorization: `Bearer ${worker.key}` },
- });
+ }, { name: WORKER_BREAKER, timeoutMs: STATUS_TIMEOUT_MS, attempts: 3, okWhen: () => true });
} catch (err) {
return error(res, 502, 'worker_unreachable', err?.message || 'worker request failed');
}
diff --git a/api/avatars/[id]/[action].js b/api/avatars/[id]/[action].js
index 019efab76..1daa974bf 100644
--- a/api/avatars/[id]/[action].js
+++ b/api/avatars/[id]/[action].js
@@ -17,7 +17,7 @@ import { getAvatar, resolveAvatarUrl } from '../../_lib/avatars.js';
import { r2, publicUrl, thumbnailUrl } from '../../_lib/r2.js';
import { env } from '../../_lib/env.js';
-const PINATA_ENDPOINT = 'https://api.pinata.cloud/pinning/pinFileToIPFS';
+import { pinToIPFS, ipfsPinningConfigured } from '../../_lib/ipfs-pin.js';
export default wrap(async (req, res) => {
let action = req.query?.action;
// Expose the GLB proxy at a `.glb`-terminating URL as well. Standard glTF
@@ -181,20 +181,18 @@ async function handlePinIpfs(req, res) {
const mode = await readStorageMode(id);
if (!mode) return error(res, 500, 'internal', 'storage_mode unavailable');
- const pinataJwt = env.PINATA_JWT;
let cid;
let isStub = false;
- if (pinataJwt && row.storage_key) {
+ if (ipfsPinningConfigured() && row.storage_key) {
try {
- cid = await pinToPinata({
- jwt: pinataJwt,
+ cid = await pinAvatarObject({
key: row.storage_key,
name: row.name || `avatar-${id}`,
contentType: row.content_type || 'model/gltf-binary',
});
} catch (err) {
- return error(res, 502, 'upstream_error', `Pinata upload failed: ${err.message}`);
+ return error(res, 502, 'upstream_error', `IPFS pin failed: ${err.message}`);
}
} else {
isStub = true;
@@ -210,27 +208,16 @@ async function handlePinIpfs(req, res) {
return json(res, 200, { storage_mode: next, stub: isStub });
}
-async function pinToPinata({ jwt, key, name, contentType }) {
+// Read the stored object out of R2 and pin it through the shared provider
+// chain (Pinata, then web3.storage; api/_lib/ipfs-pin.js), so one provider's
+// outage fails over instead of failing the pin.
+async function pinAvatarObject({ key, name, contentType }) {
const obj = await r2.send(new GetObjectCommand({ Bucket: env.S3_BUCKET, Key: key }));
const bytes = await streamToBuffer(obj.Body);
- const blob = new Blob([bytes], { type: contentType });
-
- const form = new FormData();
- form.append('file', blob, name);
- form.append('pinataMetadata', JSON.stringify({ name }));
-
- const res = await fetch(PINATA_ENDPOINT, {
- method: 'POST',
- headers: { Authorization: `Bearer ${jwt}` },
- body: form,
- });
- if (!res.ok) {
- const body = await res.text().catch(() => '');
- throw new Error(`HTTP ${res.status} ${body.slice(0, 200)}`);
- }
- const data = await res.json();
- if (!data?.IpfsHash) throw new Error('no IpfsHash in Pinata response');
- return data.IpfsHash;
+ const filename = /\.(glb|gltf|vrm)$/i.test(name) ? name : `${name}.${contentType === 'model/gltf+json' ? 'gltf' : 'glb'}`;
+ const pinned = await pinToIPFS(bytes, filename);
+ if (!pinned?.cid) throw new Error('no CID in pinning response');
+ return pinned.cid;
}
async function streamToBuffer(stream) {
diff --git a/api/avatars/_actions.js b/api/avatars/_actions.js
index b16047224..7c08238d3 100644
--- a/api/avatars/_actions.js
+++ b/api/avatars/_actions.js
@@ -74,6 +74,11 @@ function maskSubmitError(res, err) {
// threw `ReferenceError: sanitizeJobError is not defined` at runtime. Import the
// local binding and re-export it so both this module and its consumers resolve it.
import { sanitizeJobError } from '../_lib/provider-job-error.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+
+// Per-rung deadline for the vision classifiers so a hung provider moves the
+// chain to its next lane instead of holding the request.
+const VISION_TIMEOUT_MS = 30_000;
export { sanitizeJobError };
// ── presign ───────────────────────────────────────────────────────────────────
@@ -870,7 +875,7 @@ Respond with nothing else — no markdown, no explanation.`;
// vision provider is available so the caller can skip silently.
async function classifyAvatarImage({ thumbUrl, prompt, env }) {
const openaiVision = (name, key, url, model, extraHeaders = {}) => async () => {
- const r = await fetch(url, {
+ const r = await fetchUpstream(url, {
method: 'POST',
headers: { 'content-type': 'application/json', authorization: `Bearer ${key}`, ...extraHeaders },
body: JSON.stringify({
@@ -886,7 +891,7 @@ async function classifyAvatarImage({ thumbUrl, prompt, env }) {
},
],
}),
- });
+ }, { name: `${name}:vision`, timeoutMs: VISION_TIMEOUT_MS, attempts: 1, okWhen: () => true });
if (!r.ok) throw Object.assign(new Error(`${name} vision ${r.status}`), { code: 'vision_api_error' });
const d = await r.json();
return d.choices?.[0]?.message?.content || '';
@@ -912,7 +917,7 @@ async function classifyAvatarImage({ thumbUrl, prompt, env }) {
}
if (env.ANTHROPIC_API_KEY) {
attempts.push(async () => {
- const r = await fetch('https://api.anthropic.com/v1/messages', {
+ const r = await fetchUpstream('https://api.anthropic.com/v1/messages', {
method: 'POST',
headers: {
'content-type': 'application/json',
@@ -932,7 +937,7 @@ async function classifyAvatarImage({ thumbUrl, prompt, env }) {
},
],
}),
- });
+ }, { name: 'anthropic:vision', timeoutMs: VISION_TIMEOUT_MS, attempts: 1, okWhen: () => true });
if (!r.ok) throw Object.assign(new Error(`anthropic vision ${r.status}`), { code: 'vision_api_error' });
const d = await r.json();
// Pick the first TEXT block rather than content[0]: on a thinking-capable
diff --git a/api/aws-marketplace/subscription.js b/api/aws-marketplace/subscription.js
index ac42f36fd..2424c342a 100644
--- a/api/aws-marketplace/subscription.js
+++ b/api/aws-marketplace/subscription.js
@@ -39,6 +39,7 @@ import {
markCustomerStatus,
} from '../_lib/aws-marketplace-store.js';
import { env } from '../_lib/env.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
// Delegates to the shared readBody (api/_lib/http.js), which prefers the
// pre-parsed req.rawBody/req.body the Cloud Run server already captured —
@@ -182,7 +183,8 @@ async function handleSns(req, res, msg) {
return json(res, 400, { error: 'invalid_subscribe_url' });
}
try {
- await fetch(subscribeUrl);
+ // Confirming is idempotent (the token is single-purpose), so one retry is safe.
+ await fetchUpstream(subscribeUrl, {}, { timeoutMs: 8_000, attempts: 2 });
} catch (err) {
console.error('[aws-marketplace/subscription] failed to confirm SNS subscription', err?.message);
}
diff --git a/api/brain/chat.js b/api/brain/chat.js
index e8feb1add..304387aea 100644
--- a/api/brain/chat.js
+++ b/api/brain/chat.js
@@ -37,6 +37,8 @@ import { recordEvent } from '../_lib/usage.js';
import { costMicroUsd } from '../_lib/llm-pricing.js';
import { openrouterUsageFetch } from '../_lib/openrouter-usage.js';
+const WATSONX_HEADERS_TIMEOUT_MS = 45_000;
+
// Providers an anonymous (signed-out) caller may use: only the genuinely free
// tiers — the OpenRouter-routed open-weight default and the free NVIDIA NIM
// models. Every paid first-party model (Claude, GPT-5.x, o3, DashScope, DeepSeek)
@@ -693,11 +695,24 @@ async function streamWatsonx(res, { messages, system, maxTokens, t0 }) {
maxTokens,
});
- const upstream = await fetch(url, {
- method: 'POST',
- headers,
- body: JSON.stringify(body),
- });
+ // Bound the wait for response HEADERS only: a streamed completion may run
+ // longer than any sensible deadline, so the timer is cleared once watsonx
+ // answers, and only a hung connection is cut.
+ const headersCtrl = new AbortController();
+ const headersTimer = setTimeout(() => headersCtrl.abort(), WATSONX_HEADERS_TIMEOUT_MS);
+ let upstream;
+ try {
+ upstream = await fetch(url, {
+ method: 'POST',
+ headers,
+ body: JSON.stringify(body),
+ signal: headersCtrl.signal,
+ });
+ } catch (err) {
+ throw new Error(`watsonx unreachable: ${err?.name === 'AbortError' ? `no response headers within ${WATSONX_HEADERS_TIMEOUT_MS}ms` : err?.message || 'network error'}`);
+ } finally {
+ clearTimeout(headersTimer);
+ }
if (!upstream.ok || !upstream.body) {
const detail = await upstream.text().catch(() => '');
throw new Error(`watsonx ${upstream.status}: ${detail.slice(0, 200)}`);
diff --git a/api/chat/proxy.js b/api/chat/proxy.js
index 11a01d2d2..f6cfe902b 100644
--- a/api/chat/proxy.js
+++ b/api/chat/proxy.js
@@ -11,6 +11,8 @@ const UPGRADE_URL = `${env.APP_ORIGIN}/pricing`;
// afterwards; rolling over to a live model keeps that chat working instead of
// surfacing the upstream's raw "This model is unavailable for free" text.
const MAX_MODEL_ATTEMPTS = 3;
+// How long to wait for OpenRouter to start answering before rotating keys/models.
+const UPSTREAM_HEADERS_TIMEOUT_MS = 25_000;
/**
* Seconds to wait after an upstream 429, when OpenRouter sent no Retry-After.
@@ -107,16 +109,34 @@ export default wrap(async (req, res) => {
for (let attempt = 0; attempt < MAX_MODEL_ATTEMPTS; attempt += 1) {
tried.push(activeModel);
for (const [i, key] of keys.entries()) {
- upstream = await fetch('https://openrouter.ai/api/v1/chat/completions', {
- method: 'POST',
- headers: {
- Authorization: `Bearer ${key}`,
- 'Content-Type': 'application/json',
- 'HTTP-Referer': 'https://three.ws',
- 'X-Title': 'three.ws chat',
- },
- body: JSON.stringify({ ...body, model: activeModel }),
- });
+ // Bound the wait for response HEADERS only: the timer is cleared once
+ // OpenRouter answers, so a long streamed completion is never cut off,
+ // but a hung connection can no longer stall the chat UI indefinitely.
+ // A timeout is treated like a rate-limit: try the next key, then the
+ // next model, and finally report a clean 504.
+ const headersCtrl = new AbortController();
+ const headersTimer = setTimeout(() => headersCtrl.abort(), UPSTREAM_HEADERS_TIMEOUT_MS);
+ try {
+ upstream = await fetch('https://openrouter.ai/api/v1/chat/completions', {
+ method: 'POST',
+ headers: {
+ Authorization: `Bearer ${key}`,
+ 'Content-Type': 'application/json',
+ 'HTTP-Referer': 'https://three.ws',
+ 'X-Title': 'three.ws chat',
+ },
+ body: JSON.stringify({ ...body, model: activeModel }),
+ signal: headersCtrl.signal,
+ });
+ } catch (err) {
+ upstream = new Response(JSON.stringify({ error: { message: `openrouter unreachable: ${err?.name === 'AbortError' ? 'timed out waiting for headers' : err?.message || 'network error'}` } }), {
+ status: 504,
+ headers: { 'content-type': 'application/json' },
+ });
+ } finally {
+ clearTimeout(headersTimer);
+ }
+ if (upstream.status === 504 && i < keys.length - 1) continue;
if (![401, 402, 403, 429].includes(upstream.status) || i === keys.length - 1) break;
// Release the abandoned response so its connection returns to the pool.
await upstream.body?.cancel()?.catch?.(() => {});
diff --git a/api/coin/fear-greed.js b/api/coin/fear-greed.js
index d580b4bec..40b248c1a 100644
--- a/api/coin/fear-greed.js
+++ b/api/coin/fear-greed.js
@@ -9,7 +9,10 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
+import { fetchUpstreamJson, lastGood } from '../_lib/upstream-fetch.js';
const FNG_BASE = 'https://api.alternative.me/fng/';
+// The index updates once a day, so a day-old reading is still the current one.
+const STALE_MAX_AGE_MS = 24 * 60 * 60_000;
// One tiny per-instance cache keyed by limit shields the upstream from
// concurrent cold-instance misses; CDN absorbs the rest.
@@ -29,17 +32,60 @@ async function fetchFng(limit) {
const hit = _cache.get(limit);
if (hit && hit.expiresAt > now) return hit.value;
- const resp = await fetch(`${FNG_BASE}?limit=${limit}&format=json`, {
- headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(8000),
+ const { value } = await lastGood(`fng:${limit}`, () => fetchFngLive(limit), {
+ maxAgeMs: STALE_MAX_AGE_MS,
+ onFallback: (err, ageMs) => console.warn(`[fear-greed] every source failed (${err?.message}); serving ${Math.round(ageMs / 60_000)}m-old reading`),
});
- if (!resp.ok) {
- const err = new Error(`fng ${resp.status}`);
- err.status = resp.status;
- throw err;
+ return value;
+}
+
+// alternative.me is the canonical index. CoinMarketCap publishes its own
+// (same 0-100 scale, same classification bands) behind a free key, so it is
+// the second rung whenever COINMARKETCAP_API_KEY is set. Both normalise to the
+// same {value, timestamp, value_classification} row shape.
+async function fetchFngRows(limit) {
+ const sources = [
+ {
+ name: 'alternative.me',
+ load: async () => {
+ const raw = await fetchUpstreamJson(`${FNG_BASE}?limit=${limit}&format=json`, {
+ headers: { 'user-agent': 'three.ws/1.0' },
+ }, { name: 'alternative-me-fng', timeoutMs: 8_000, attempts: 2 });
+ return Array.isArray(raw?.data) ? raw.data : [];
+ },
+ },
+ process.env.COINMARKETCAP_API_KEY && {
+ name: 'coinmarketcap',
+ load: async () => {
+ const raw = await fetchUpstreamJson(`https://pro-api.coinmarketcap.com/v3/fear-and-greed/historical?limit=${limit}`, {
+ headers: { 'X-CMC_PRO_API_KEY': process.env.COINMARKETCAP_API_KEY },
+ }, { name: 'cmc-fng', timeoutMs: 8_000, attempts: 2 });
+ const rows = Array.isArray(raw?.data) ? raw.data : [];
+ return rows.map((d) => ({
+ value: d.value,
+ timestamp: Math.floor(Date.parse(d.timestamp) / 1000),
+ value_classification: d.value_classification,
+ }));
+ },
+ },
+ ].filter(Boolean);
+ let lastErr;
+ for (const src of sources) {
+ try {
+ const rows = await src.load();
+ if (rows.length) return rows;
+ lastErr = new Error(`${src.name}: empty payload`);
+ } catch (err) {
+ lastErr = err;
+ console.warn(`[fear-greed] ${src.name} failed: ${err?.message || err}`);
+ }
}
- const raw = await resp.json();
- const rows = Array.isArray(raw?.data) ? raw.data : [];
+ throw lastErr || new Error('no fear & greed source configured');
+}
+
+async function fetchFngLive(limit) {
+ const now = Date.now();
+ const rows = await fetchFngRows(limit);
// alternative.me returns newest-first; the chart wants oldest→newest.
const history = rows
.map((d) => {
diff --git a/api/coin/global.js b/api/coin/global.js
index dbb1ab3cf..9123b0acf 100644
--- a/api/coin/global.js
+++ b/api/coin/global.js
@@ -11,25 +11,30 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { fetchGlobalMarket } from '../_lib/market-fallbacks.js';
+import { fetchUpstreamJson, lastGoodValue } from '../_lib/upstream-fetch.js';
let _fng = null; // { value, expiresAt }
const FNG_TTL_MS = 300_000;
+// The index updates once a day, so a day-old reading is still the current one.
+const FNG_STALE_MAX_AGE_MS = 24 * 60 * 60_000;
// Exported for the paid Market Data API (api/_lib/market-data/) — the x402
// market-global endpoint sells the same fear & greed reading this page renders.
export async function fetchFearGreed() {
const now = Date.now();
if (_fng && _fng.expiresAt > now) return _fng.value;
- const resp = await fetch('https://api.alternative.me/fng/?limit=1', {
- headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(6000),
- });
- if (!resp.ok) throw new Error(`fng ${resp.status}`);
- const raw = await resp.json();
+ const value = await lastGoodValue('coin-global:fng', fetchFearGreedLive, { maxAgeMs: FNG_STALE_MAX_AGE_MS });
+ _fng = { value, expiresAt: now + FNG_TTL_MS };
+ return value;
+}
+
+async function fetchFearGreedLive() {
+ const raw = await fetchUpstreamJson('https://api.alternative.me/fng/?limit=1', {
+ headers: { 'user-agent': 'three.ws/1.0' },
+ }, { name: 'alternative-me-fng', timeoutMs: 6000, attempts: 2 });
const d = raw?.data?.[0];
const value = d ? { value: Number(d.value), label: d.value_classification || null } : null;
if (!value || !Number.isFinite(value.value)) throw new Error('fng payload');
- _fng = { value, expiresAt: now + FNG_TTL_MS };
return value;
}
diff --git a/api/community/world-gate.js b/api/community/world-gate.js
index 79d10f03a..253b19e4b 100644
--- a/api/community/world-gate.js
+++ b/api/community/world-gate.js
@@ -28,26 +28,41 @@ import {
UnconfiguredError,
} from '../_lib/coin-communities.js';
import { readWorldGate, writeWorldGate, normalizeMinTokens } from '../_lib/world-gate.js';
+import { pumpFetchJson, PUMP_FRONTEND_BASE } from '../_lib/pump-feed-fetch.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
-const PUMP_FRONTEND_BASE = process.env.PUMP_FRONTEND_BASE || 'https://frontend-api-v3.pump.fun';
+// A coin's creator never changes, so a resolved answer can be reused for a
+// while and, when pump.fun is down, served from the last good copy for an hour
+// rather than blocking every gate read and write behind the outage.
+export const CREATOR_CACHE_KEY = (mint) => `world-gate:creator:${mint}`;
+const CREATOR_TTL_S = 10 * 60;
+const CREATOR_LKG_TTL_S = 60 * 60;
// Resolve a coin's on-chain creator from pump.fun. Returns '' when unknown (a
// non-pump mint, or pump didn't answer) so the caller fails closed (not creator).
async function resolveCoinCreator(mint) {
try {
- const resp = await fetch(new URL(`/coins/${mint}`, PUMP_FRONTEND_BASE), {
- headers: { accept: 'application/json' },
- signal: AbortSignal.timeout(8000),
+ return await cacheWrapLastGood(CREATOR_CACHE_KEY(mint), CREATOR_TTL_S, () => fetchCoinCreator(mint), {
+ staleTtlSeconds: CREATOR_LKG_TTL_S,
});
- if (!resp.ok) return '';
- const body = await resp.json();
- const creator = typeof body?.creator === 'string' ? body.creator.trim() : '';
- return creator;
} catch {
return '';
}
}
+// Live pump.fun read with the feed's deadline + one bounded retry. A coin pump
+// does not know (404) is a definite answer and is thrown as such so the
+// last-good layer never papers over it with an older lookup.
+async function fetchCoinCreator(mint) {
+ const url = new URL(`/coins/${mint}`, PUMP_FRONTEND_BASE).toString();
+ const { ok, status, body } = await pumpFetchJson(url, { timeoutMs: 8000, retries: 1 });
+ if (!ok && status === 404) return '';
+ if (!ok) throw new Error(`pump.fun ${status || 'unreachable'}`);
+ const creator = typeof body?.creator === 'string' ? body.creator.trim() : '';
+ if (!creator) throw new Error('pump.fun answered without a creator');
+ return creator;
+}
+
// The signed-in user's linked Solana wallets, as { data: { wallets }, error }
// shaped like the SDK's so it composes directly with withAuthRefresh. Every
// linked wallet is returned: matching the creator is one in-memory comparison
diff --git a/api/cron/copy-fanout.js b/api/cron/copy-fanout.js
index b1792d01c..7ab0b7bf9 100644
--- a/api/cron/copy-fanout.js
+++ b/api/cron/copy-fanout.js
@@ -23,6 +23,8 @@ import { json, method, wrapCron } from '../_lib/http.js';
import { sql } from '../_lib/db.js';
import { planCopyOrder } from '../_lib/copy-engine.js';
import { requireCron } from '../_lib/cron-auth.js';
+import { pumpFetchJson, PUMP_FRONTEND_BASE } from '../_lib/pump-feed-fetch.js';
+import { fetchTokenMarketData } from '../_lib/market/token-market.js';
const NETWORKS = ['mainnet', 'devnet'];
const lamToSol = (l) => (l == null ? 0 : Number(BigInt(l)) / 1e9);
@@ -85,12 +87,15 @@ async function coinContext(mint, oracleScore) {
}
let ctx = null;
try {
- const r = await fetch(`https://frontend-api-v3.pump.fun/coins/${mint}`, {
- headers: { accept: 'application/json' }, signal: AbortSignal.timeout(5000),
- });
- if (r.ok) {
- const c = await r.json();
+ const { ok, body: c } = await pumpFetchJson(`${PUMP_FRONTEND_BASE}/coins/${mint}`, { timeoutMs: 5000, retries: 1 });
+ if (ok && c) {
ctx = { market_cap_usd: Number(c.usd_market_cap) || null, graduated: !!c.complete };
+ } else {
+ // pump.fun down: the shared multi-source reader still knows the market
+ // cap. Graduation is a pump.fun fact, so it stays unknown (null) rather
+ // than guessed; the engine treats a missing field as missing.
+ const md = await fetchTokenMarketData(mint);
+ if (md) ctx = { market_cap_usd: Number(md.market_cap) || null, graduated: null };
}
} catch { /* leave null — engine handles missing context */ }
_coinCache.set(mint, ctx);
diff --git a/api/cron/oracle-digest.js b/api/cron/oracle-digest.js
index a0d73302e..0218d4e76 100644
--- a/api/cron/oracle-digest.js
+++ b/api/cron/oracle-digest.js
@@ -20,6 +20,7 @@ import { sql } from '../_lib/db.js';
import { gmgnTokenUrl, referralUrl, TERMINAL_LABELS } from '../../src/shared/trading-terminals.js';
import { requireCron } from '../_lib/cron-auth.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const TIER_EMOJI = { prime: '🟣', strong: '🔵', lean: '🟡', watch: '⚪', avoid: '🔴' };
const ALERT_TIMEOUT_MS = 5000;
@@ -144,7 +145,7 @@ async function sendDigest(chatId, text) {
const ctrl = new AbortController();
const timer = setTimeout(() => ctrl.abort(), ALERT_TIMEOUT_MS);
try {
- const r = await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+ const r = await fetchUpstream(`https://api.telegram.org/bot${token}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -154,7 +155,7 @@ async function sendDigest(chatId, text) {
disable_web_page_preview: true,
}),
signal: ctrl.signal,
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const result = await r.json().catch(() => null);
return result?.ok === true;
} catch {
diff --git a/api/crypto/holders.js b/api/crypto/holders.js
index 542b16434..fc53a9a4d 100644
--- a/api/crypto/holders.js
+++ b/api/crypto/holders.js
@@ -25,6 +25,14 @@ import { cors, method, wrap, error, json, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { isValidSolanaAddress, isValidEvmAddress } from '../_lib/validate.js';
import { composeTokenHolders, DEFAULT_LIMIT, MAX_LIMIT } from '../_lib/crypto-token-holders.js';
+import { cacheGet, cacheSet } from '../_lib/cache.js';
+import { staleEnvelope } from '../_lib/rpc-degrade.js';
+
+// Last-good tier: the previous successful report for this mint+limit, served
+// marked stale during an RPC/indexer outage. Concentration moves slowly, so a
+// day-old distribution is still a real position-sizing answer; a 503 is not.
+const HOLDERS_LKG_TTL_S = 24 * 3600;
+const holdersLkgKey = (address, limit) => `crypto:holders:lkg:${address}:${limit}`;
const EXAMPLE = '/api/crypto/holders?address=FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump';
@@ -70,6 +78,13 @@ export default wrap(async (req, res) => {
});
}
if (result.status === 'upstream_down') {
+ const lkg = await cacheGet(holdersLkgKey(address, limit));
+ if (lkg && lkg.body && typeof lkg.at === 'number') {
+ return json(res, 200, staleEnvelope(lkg.body, lkg.at), {
+ 'cache-control': 'public, s-maxage=30, stale-while-revalidate=60',
+ 'x-holders-stale': '1',
+ });
+ }
return json(res, 503, {
error: 'upstream_unavailable',
error_description: 'holder data sources are temporarily unreachable, retry shortly',
@@ -90,6 +105,7 @@ export default wrap(async (req, res) => {
sources: result.sources,
};
if (result.note) body.note = result.note;
+ cacheSet(holdersLkgKey(address, limit), { body, at: Date.now() }, HOLDERS_LKG_TTL_S).catch(() => {});
// Distribution shifts trade by trade but a position-sizing read tolerates a
// minute of cache; this absorbs polling bursts without hammering the RPC.
diff --git a/api/defi/chain.js b/api/defi/chain.js
index f582461b9..5c62ac096 100644
--- a/api/defi/chain.js
+++ b/api/defi/chain.js
@@ -23,7 +23,30 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
-
+import { fetchUpstream, lastGood } from '../_lib/upstream-fetch.js';
+
+// DeFiLlama is this page's only upstream. A TTL cache alone still 502s the
+// moment it expires during an outage, so every loader also keeps its last good
+// result for hours and serves that (the handler's cache headers keep the
+// staleness honest to the CDN) rather than blanking the page.
+const STALE_MAX_AGE_MS = 6 * 60 * 60_000;
+// Keys currently served from a last-good copy (either tier), so the handler can
+// label the response `x-three-stale: 1` instead of passing it off as live. A key
+// leaves the set the moment its loader succeeds again.
+const _staleKeys = new Set();
+const cachedStale = (cache, key, load) => cached(cache, key, async () => {
+ const r = await lastGood(`defi-chain:${key}`, load, {
+ maxAgeMs: STALE_MAX_AGE_MS,
+ onFallback: (err, ageMs) => console.warn(`[upstream] defi-chain:${key}: serving last good value (${Math.round(ageMs / 60_000)}m old) after ${err?.message || err}`),
+ });
+ if (r.stale) _staleKeys.add(key);
+ else _staleKeys.delete(key);
+ return r.value;
+}, { onStale: (err) => {
+ _staleKeys.add(key);
+ console.warn(`[upstream] defi-chain:${key}: serving in-memory last good after ${err?.message || err}`);
+} });
+const staleHeaders = (...keys) => (keys.some((k) => _staleKeys.has(k)) ? { 'x-three-stale': '1' } : {});
const NAME_RE = /^[a-z0-9 ._-]{1,40}$/i;
const CHAINS_URL = 'https://api.llama.fi/v2/chains';
@@ -36,15 +59,9 @@ const finite = (n) => (Number.isFinite(n) ? n : null);
// ── Upstream helpers ─────────────────────────────────────────────────────────
async function fetchJson(url, { timeout = 10_000 } = {}) {
- const resp = await fetch(url, {
+ const resp = await fetchUpstream(url, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(timeout),
- });
- if (!resp.ok) {
- const err = new Error(`llama ${resp.status}`);
- err.status = resp.status;
- throw err;
- }
+ }, { timeoutMs: timeout, attempts: 2 });
return resp.json();
}
@@ -69,7 +86,7 @@ const _chainsCache = createCache({ max: 1, ttlMs: CHAINS_TTL_MS });
const _protocolsCache = createCache({ max: 1, ttlMs: CHAINS_TTL_MS });
async function loadChains() {
- return cached(_chainsCache, 'chains', async () => {
+ return cachedStale(_chainsCache, 'chains', async () => {
const raw = await fetchJson(CHAINS_URL);
if (!Array.isArray(raw)) throw new Error('unexpected upstream shape');
@@ -97,7 +114,7 @@ async function loadChains() {
// The full protocol feed, shared across every chain profile built in this TTL.
async function loadProtocols() {
- return cached(_protocolsCache, 'protocols', () => fetchJson(PROTOCOLS_URL));
+ return cachedStale(_protocolsCache, 'protocols', () => fetchJson(PROTOCOLS_URL));
}
// Resolve the request name to DeFiLlama's exact casing. Exact hit first, then a
@@ -200,7 +217,7 @@ function shapeFees(raw) {
const _chainCache = createCache({ max: 512, ttlMs: CHAIN_TTL_MS });
async function buildChain(canonical, chains) {
- return cached(_chainCache, canonical.name, () => fetchChain(canonical, chains));
+ return cachedStale(_chainCache, canonical.name, () => fetchChain(canonical, chains));
}
async function fetchChain(canonical, chains) {
@@ -289,6 +306,7 @@ export default wrap(async (req, res) => {
const payload = await buildChain(canonical, chains);
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders('chains', 'protocols', canonical.name),
});
} catch {
return error(
diff --git a/api/defi/chains.js b/api/defi/chains.js
index 0cd77ac7c..838346510 100644
--- a/api/defi/chains.js
+++ b/api/defi/chains.js
@@ -9,6 +9,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const UPSTREAM = 'https://api.llama.fi/v2/chains';
const TTL_MS = 300_000;
@@ -27,11 +28,9 @@ export async function buildChains() {
async function loadChains() {
const now = Date.now();
- const resp = await fetch(UPSTREAM, {
+ const resp = await fetchUpstream(UPSTREAM, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const raw = await resp.json();
if (!Array.isArray(raw)) throw new Error('unexpected upstream shape');
@@ -64,6 +63,14 @@ async function loadChains() {
};
}
+// `cached()` serves its last-good copy (without re-populating the fresh slot)
+// when the loader throws, so a fresh miss after a resolved value means the
+// payload is stale. Label it so the page can say so instead of passing it off
+// as live.
+function staleHeaders() {
+ return _cache.has(CACHE_KEY) ? {} : { 'x-three-stale': '1' };
+}
+
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
@@ -75,6 +82,7 @@ export default wrap(async (req, res) => {
const payload = await buildChains();
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders(),
});
} catch {
return error(
diff --git a/api/defi/dex-volumes.js b/api/defi/dex-volumes.js
index ceacac064..892cdd5f3 100644
--- a/api/defi/dex-volumes.js
+++ b/api/defi/dex-volumes.js
@@ -12,6 +12,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const UPSTREAM = 'https://api.llama.fi/overview/dexs?excludeTotalDataChartBreakdown=true';
const TTL_MS = 600_000;
@@ -47,11 +48,9 @@ export async function buildDexVolumes() {
async function loadDexVolumes() {
const now = Date.now();
- const resp = await fetch(UPSTREAM, {
+ const resp = await fetchUpstream(UPSTREAM, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const raw = await resp.json();
if (!raw || !Array.isArray(raw.protocols)) throw new Error('unexpected upstream shape');
@@ -94,6 +93,14 @@ async function loadDexVolumes() {
};
}
+// `cached()` serves its last-good copy (without re-populating the fresh slot)
+// when the loader throws, so a fresh miss after a resolved value means the
+// payload is stale. Label it so the page can say so instead of passing it off
+// as live.
+function staleHeaders() {
+ return _cache.has(CACHE_KEY) ? {} : { 'x-three-stale': '1' };
+}
+
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
@@ -105,6 +112,7 @@ export default wrap(async (req, res) => {
const payload = await buildDexVolumes();
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=600, stale-while-revalidate=600',
+ ...staleHeaders(),
});
} catch {
return error(
diff --git a/api/defi/fees.js b/api/defi/fees.js
index 6855a9024..514776136 100644
--- a/api/defi/fees.js
+++ b/api/defi/fees.js
@@ -13,6 +13,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const TTL_MS = 600_000;
const MAX_CHART_POINTS = 200;
@@ -55,11 +56,9 @@ export async function buildFees(type) {
async function loadFees(type) {
const now = Date.now();
- const resp = await fetch(upstreamFor(type), {
+ const resp = await fetchUpstream(upstreamFor(type), {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const raw = await resp.json();
if (!raw || !Array.isArray(raw.protocols)) throw new Error('unexpected upstream shape');
@@ -98,6 +97,14 @@ async function loadFees(type) {
};
}
+// `cached()` serves its last-good copy (without re-populating the fresh slot)
+// when the loader throws, so a fresh miss after a resolved value means the
+// payload is stale. Label it so the page can say so instead of passing it off
+// as live.
+function staleHeaders(type) {
+ return _cache.has(type) ? {} : { 'x-three-stale': '1' };
+}
+
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
@@ -112,6 +119,7 @@ export default wrap(async (req, res) => {
const payload = await buildFees(type);
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=600, stale-while-revalidate=600',
+ ...staleHeaders(type),
});
} catch {
return error(
diff --git a/api/defi/hacks.js b/api/defi/hacks.js
index a28c5d4ec..470ad79e7 100644
--- a/api/defi/hacks.js
+++ b/api/defi/hacks.js
@@ -16,7 +16,30 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
-
+import { fetchUpstream, lastGood } from '../_lib/upstream-fetch.js';
+
+// DeFiLlama is this page's only upstream. A TTL cache alone still 502s the
+// moment it expires during an outage, so every loader also keeps its last good
+// result for hours and serves that (the handler's cache headers keep the
+// staleness honest to the CDN) rather than blanking the page.
+const STALE_MAX_AGE_MS = 6 * 60 * 60_000;
+// Keys currently served from a last-good copy (either tier), so the handler can
+// label the response `x-three-stale: 1` instead of passing it off as live. A key
+// leaves the set the moment its loader succeeds again.
+const _staleKeys = new Set();
+const cachedStale = (cache, key, load) => cached(cache, key, async () => {
+ const r = await lastGood(`defi-hacks:${key}`, load, {
+ maxAgeMs: STALE_MAX_AGE_MS,
+ onFallback: (err, ageMs) => console.warn(`[upstream] defi-hacks:${key}: serving last good value (${Math.round(ageMs / 60_000)}m old) after ${err?.message || err}`),
+ });
+ if (r.stale) _staleKeys.add(key);
+ else _staleKeys.delete(key);
+ return r.value;
+}, { onStale: (err) => {
+ _staleKeys.add(key);
+ console.warn(`[upstream] defi-hacks:${key}: serving in-memory last good after ${err?.message || err}`);
+} });
+const staleHeaders = (...keys) => (keys.some((k) => _staleKeys.has(k)) ? { 'x-three-stale': '1' } : {});
const UPSTREAM = 'https://api.llama.fi/hacks';
const TTL_MS = 600_000;
const YEAR_MS = 365 * 24 * 60 * 60 * 1000;
@@ -40,17 +63,15 @@ function normalizeChains(chain) {
// sorted, normalized incident list plus the dataset-wide headline stats. The
// per-request search/pagination happens on top of this cached whole.
async function loadDataset() {
- return cached(_cache, CACHE_KEY, buildDataset);
+ return cachedStale(_cache, CACHE_KEY, buildDataset);
}
async function buildDataset() {
const now = Date.now();
- const resp = await fetch(UPSTREAM, {
+ const resp = await fetchUpstream(UPSTREAM, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const raw = await resp.json();
if (!Array.isArray(raw)) throw new Error('unexpected upstream shape');
@@ -165,6 +186,7 @@ export default wrap(async (req, res) => {
{
'cache-control':
'public, max-age=120, s-maxage=600, stale-while-revalidate=1200',
+ ...staleHeaders(CACHE_KEY),
},
);
} catch {
diff --git a/api/defi/protocol.js b/api/defi/protocol.js
index 9043b8988..dc72cc0db 100644
--- a/api/defi/protocol.js
+++ b/api/defi/protocol.js
@@ -18,7 +18,30 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream, lastGood } from '../_lib/upstream-fetch.js';
+// DeFiLlama is this page's only upstream. A TTL cache alone still 502s the
+// moment it expires during an outage, so every loader also keeps its last good
+// result for hours and serves that (the handler's cache headers keep the
+// staleness honest to the CDN) rather than blanking the page.
+const STALE_MAX_AGE_MS = 6 * 60 * 60_000;
+// Keys currently served from a last-good copy (either tier), so the handler can
+// label the response `x-three-stale: 1` instead of passing it off as live. A key
+// leaves the set the moment its loader succeeds again.
+const _staleKeys = new Set();
+const cachedStale = (cache, key, load) => cached(cache, key, async () => {
+ const r = await lastGood(`defi-protocol:${key}`, load, {
+ maxAgeMs: STALE_MAX_AGE_MS,
+ onFallback: (err, ageMs) => console.warn(`[upstream] defi-protocol:${key}: serving last good value (${Math.round(ageMs / 60_000)}m old) after ${err?.message || err}`),
+ });
+ if (r.stale) _staleKeys.add(key);
+ else _staleKeys.delete(key);
+ return r.value;
+}, { onStale: (err) => {
+ _staleKeys.add(key);
+ console.warn(`[upstream] defi-protocol:${key}: serving in-memory last good after ${err?.message || err}`);
+} });
+const staleHeaders = (...keys) => (keys.some((k) => _staleKeys.has(k)) ? { 'x-three-stale': '1' } : {});
// DeFiLlama slugs are not strictly [a-z0-9.-]: 14 of the ~6k protocols it tracks
// carry parentheses, a plus or a bang (`dinero-(pxeth)`, `synthetix-v1+v2`,
// `yay!`). Rejecting those characters turned every /yields row for such a
@@ -55,15 +78,9 @@ const SYNTH_SUFFIX = /-(borrowed|staking|pool2|vesting)$/i;
const AGGREGATE_KEYS = new Set(['borrowed', 'staking', 'pool2', 'vesting']);
async function fetchJson(url) {
- const resp = await fetch(url, {
+ const resp = await fetchUpstream(url, {
headers: { accept: 'application/json', 'user-agent': UA },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) {
- const err = new Error(`llama ${resp.status}`);
- err.status = resp.status;
- throw err;
- }
+ }, { timeoutMs: 10_000, attempts: 2 });
return resp.json();
}
@@ -233,7 +250,7 @@ function build(slug, proto, fees, revenue, dexs) {
}
async function load(slug) {
- return cached(_cache, slug, async () => {
+ return cachedStale(_cache, slug, async () => {
// Main call is required; the rest are best-effort enrichment fetched in
// parallel. The main fetch throwing a 400/404 propagates as not-found.
const [proto, fees, revenue, dexs] = await Promise.all([
@@ -264,6 +281,7 @@ export default wrap(async (req, res) => {
const payload = await load(slug);
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders(slug),
});
} catch (err) {
// DeFiLlama answers an unknown slug with 400 "Protocol not found" (and 404
diff --git a/api/defi/protocols.js b/api/defi/protocols.js
index a6575fb79..9be9ae7e8 100644
--- a/api/defi/protocols.js
+++ b/api/defi/protocols.js
@@ -9,6 +9,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const UPSTREAM = 'https://api.llama.fi/protocols';
const TTL_MS = 300_000;
@@ -31,11 +32,9 @@ export async function buildProtocols() {
async function loadProtocols() {
const now = Date.now();
- const resp = await fetch(UPSTREAM, {
+ const resp = await fetchUpstream(UPSTREAM, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const raw = await resp.json();
if (!Array.isArray(raw)) throw new Error('unexpected upstream shape');
@@ -84,6 +83,14 @@ async function loadProtocols() {
};
}
+// `cached()` serves its last-good copy (without re-populating the fresh slot)
+// when the loader throws, so a fresh miss after a resolved value means the
+// payload is stale. Label it so the page can say so instead of passing it off
+// as live.
+function staleHeaders() {
+ return _cache.has(CACHE_KEY) ? {} : { 'x-three-stale': '1' };
+}
+
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
@@ -95,6 +102,7 @@ export default wrap(async (req, res) => {
const payload = await buildProtocols();
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders(),
});
} catch {
return error(
diff --git a/api/defi/stablecoin.js b/api/defi/stablecoin.js
index af2e2914e..94d139095 100644
--- a/api/defi/stablecoin.js
+++ b/api/defi/stablecoin.js
@@ -15,6 +15,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const ID_RE = /^\d{1,6}$/;
const UPSTREAM = (id) => `https://stablecoins.llama.fi/stablecoin/${id}`;
@@ -176,12 +177,9 @@ function shape(raw, id) {
// the id is unknown upstream so the handler can map it to a clean 404.
async function build(id) {
return cached(_cache, id, async () => {
- const resp = await fetch(UPSTREAM(id), {
+ const resp = await fetchUpstream(UPSTREAM(id), {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (resp.status === 404) throw Object.assign(new Error('not found'), { status: 404 });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const body = await resp.json();
if (!body || typeof body !== 'object' || body.id == null) {
@@ -192,6 +190,14 @@ async function build(id) {
});
}
+// `cached()` serves its last-good copy (without re-populating the fresh slot)
+// when the loader throws, so a fresh miss after a resolved value means the
+// payload is stale. Label it so the page can say so instead of passing it off
+// as live.
+function staleHeaders(id) {
+ return _cache.has(id) ? {} : { 'x-three-stale': '1' };
+}
+
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
@@ -208,6 +214,7 @@ export default wrap(async (req, res) => {
const payload = await build(id);
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders(id),
});
} catch (err) {
if (err?.status === 404) {
diff --git a/api/defi/stablecoins.js b/api/defi/stablecoins.js
index b4d752bb0..14a9f4ed8 100644
--- a/api/defi/stablecoins.js
+++ b/api/defi/stablecoins.js
@@ -11,6 +11,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const UPSTREAM = 'https://stablecoins.llama.fi/stablecoins?includePrices=true';
const TTL_MS = 300_000; // 5 minutes, matches the CDN s-maxage below.
@@ -73,11 +74,9 @@ export async function buildStablecoins() {
}
async function loadStablecoins() {
- const resp = await fetch(UPSTREAM, {
+ const resp = await fetchUpstream(UPSTREAM, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!resp.ok) throw new Error(`llama ${resp.status}`);
+ }, { timeoutMs: 10_000, attempts: 2 });
const body = await resp.json();
const assets = Array.isArray(body?.peggedAssets) ? body.peggedAssets : null;
@@ -86,6 +85,14 @@ async function loadStablecoins() {
return shape(assets);
}
+// `cached()` serves its last-good copy (without re-populating the fresh slot)
+// when the loader throws, so a fresh miss after a resolved value means the
+// payload is stale. Label it so the page can say so instead of passing it off
+// as live.
+function staleHeaders() {
+ return _cache.has(CACHE_KEY) ? {} : { 'x-three-stale': '1' };
+}
+
export default wrap(async (req, res) => {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
@@ -97,6 +104,7 @@ export default wrap(async (req, res) => {
const payload = await buildStablecoins();
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders(),
});
} catch {
return error(
diff --git a/api/defi/yields.js b/api/defi/yields.js
index 1b8bd063e..247b60875 100644
--- a/api/defi/yields.js
+++ b/api/defi/yields.js
@@ -19,7 +19,30 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { createCache, cached } from '../_lib/mem-cache.js';
-
+import { fetchUpstream, lastGood } from '../_lib/upstream-fetch.js';
+
+// DeFiLlama is this page's only upstream. A TTL cache alone still 502s the
+// moment it expires during an outage, so every loader also keeps its last good
+// result for hours and serves that (the handler's cache headers keep the
+// staleness honest to the CDN) rather than blanking the page.
+const STALE_MAX_AGE_MS = 6 * 60 * 60_000;
+// Keys currently served from a last-good copy (either tier), so the handler can
+// label the response `x-three-stale: 1` instead of passing it off as live. A key
+// leaves the set the moment its loader succeeds again.
+const _staleKeys = new Set();
+const cachedStale = (cache, key, load) => cached(cache, key, async () => {
+ const r = await lastGood(`defi-yields:${key}`, load, {
+ maxAgeMs: STALE_MAX_AGE_MS,
+ onFallback: (err, ageMs) => console.warn(`[upstream] defi-yields:${key}: serving last good value (${Math.round(ageMs / 60_000)}m old) after ${err?.message || err}`),
+ });
+ if (r.stale) _staleKeys.add(key);
+ else _staleKeys.delete(key);
+ return r.value;
+}, { onStale: (err) => {
+ _staleKeys.add(key);
+ console.warn(`[upstream] defi-yields:${key}: serving in-memory last good after ${err?.message || err}`);
+} });
+const staleHeaders = (...keys) => (keys.some((k) => _staleKeys.has(k)) ? { 'x-three-stale': '1' } : {});
const POOLS_UPSTREAM = 'https://yields.llama.fi/pools';
const CHART_UPSTREAM = 'https://yields.llama.fi/chart/';
const POOLS_TTL_MS = 600_000;
@@ -102,12 +125,10 @@ export async function loadYieldPools() {
}
async function loadPools() {
- return cached(_pools, POOLS_KEY, async () => {
- const resp = await fetch(POOLS_UPSTREAM, {
+ return cachedStale(_pools, POOLS_KEY, async () => {
+ const resp = await fetchUpstream(POOLS_UPSTREAM, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(20_000),
- });
- if (!resp.ok) throw new Error(`llama yields ${resp.status}`);
+ }, { timeoutMs: 20_000, attempts: 2 });
const raw = await resp.json();
if (!Array.isArray(raw?.data)) throw new Error('unexpected upstream shape');
@@ -216,6 +237,7 @@ async function listMode(req, res, params) {
});
return json(res, 200, payload, {
'cache-control': 'public, max-age=120, s-maxage=300, stale-while-revalidate=600',
+ ...staleHeaders(POOLS_KEY),
});
}
@@ -238,17 +260,20 @@ function downsample(points, max) {
}
async function loadChart(pool) {
- return cached(_charts, pool, async () => {
+ return cachedStale(_charts, pool, async () => {
const now = Date.now();
- const resp = await fetch(`${CHART_UPSTREAM}${pool}`, {
- headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(15_000),
- });
// DeFiLlama answers an unknown pool id with a non-200 or an empty data set;
// both mean "no history for this pool", not an outage.
- if (resp.status === 404 || resp.status === 400) return null;
- if (!resp.ok) throw new Error(`llama chart ${resp.status}`);
+ let resp;
+ try {
+ resp = await fetchUpstream(`${CHART_UPSTREAM}${pool}`, {
+ headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
+ }, { timeoutMs: 15_000, attempts: 2 });
+ } catch (err) {
+ if (err?.status === 404 || err?.status === 400) return null;
+ throw err;
+ }
const raw = await resp.json();
if (!Array.isArray(raw?.data)) throw new Error('unexpected upstream shape');
@@ -293,6 +318,7 @@ async function chartMode(req, res, pool) {
}
return json(res, 200, value, {
'cache-control': 'public, max-age=300, s-maxage=600, stale-while-revalidate=1200',
+ ...staleHeaders(pool.toLowerCase()),
});
}
diff --git a/api/erc8004/[action].js b/api/erc8004/[action].js
index 5deee483b..ecd1c8ff3 100644
--- a/api/erc8004/[action].js
+++ b/api/erc8004/[action].js
@@ -29,6 +29,7 @@ import { attestValidation, AttestError } from '../_lib/validation-attest.js';
import { fetchSafePublicUrlPinned, SsrfBlockedError, MaxBytesExceededError } from '../_lib/ssrf-guard.js';
import { redactUrlSecrets } from '../_lib/scrub-secrets.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
export default wrap(async (req, res) => {
const action = req.query?.action;
@@ -616,13 +617,13 @@ async function uploadToWeb3Storage(token, ext, body, ct) {
const blob = new Blob([body], { type: ct });
formData.append('file', blob, filename);
- const response = await fetch('https://api.web3.storage/upload', {
+ const response = await fetchUpstream('https://api.web3.storage/upload', {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
},
body: formData,
- });
+ }, { timeoutMs: 60_000, attempts: 2, okWhen: () => true });
if (!response.ok) {
throw new Error(`web3.storage upload failed: ${response.status} ${response.statusText}`);
@@ -645,13 +646,13 @@ async function uploadToNftStorage(token, ext, body, ct) {
const blob = new Blob([body], { type: ct });
formData.append('file', blob, filename);
- const response = await fetch('https://api.nft.storage/upload', {
+ const response = await fetchUpstream('https://api.nft.storage/upload', {
method: 'POST',
headers: {
Authorization: `Bearer ${token}`,
},
body: formData,
- });
+ }, { timeoutMs: 60_000, attempts: 2, okWhen: () => true });
if (!response.ok) {
throw new Error(`nft.storage upload failed: ${response.status} ${response.statusText}`);
diff --git a/api/evm-rpc.js b/api/evm-rpc.js
new file mode 100644
index 000000000..79c29316f
--- /dev/null
+++ b/api/evm-rpc.js
@@ -0,0 +1,174 @@
+// Browser-safe EVM JSON-RPC proxy.
+//
+// Browser code used to talk to keyless public EVM hosts directly, led by
+// eth.llamarpc.com, a host the server-side failover (api/_lib/evm/rpc.js)
+// already demotes as a known-bad keyless endpoint. This proxy lets the browser
+// inherit that server chain instead: an explicit RPC_URL_<chainId> override,
+// then Alchemy where keyed, then the curated public tail, rotating on failure.
+//
+// Usage from browser:
+// POST /api/evm-rpc?chainId=8453 with a JSON-RPC body (single or batch)
+//
+// Hardening mirrors api/solana-rpc.js: per-IP and global rate limits (the chain
+// may front a keyed, paid upstream), a read-only method allowlist so an
+// anonymous caller can neither broadcast nor drain quota with scans, and a
+// batch cap so one POST cannot fan out into hundreds of upstream calls.
+
+import { cors, method, wrap, readJson, error, rateLimited } from './_lib/http.js';
+import { limits, clientIp } from './_lib/rate-limit.js';
+import { CHAIN_BY_ID } from './_lib/erc8004-chains.js';
+import { evmRpcEndpoints } from './_lib/evm/rpc.js';
+
+// Read-only methods a browser provider needs for contract reads, balances,
+// receipts and fee estimation. Nothing that broadcasts (eth_sendRawTransaction)
+// and nothing that filters or subscribes (eth_newFilter, eth_subscribe).
+export const ALLOWED_METHODS = new Set([
+ 'eth_call',
+ 'eth_getBalance',
+ 'eth_blockNumber',
+ 'eth_getLogs',
+ 'eth_getTransactionReceipt',
+ 'eth_getTransactionByHash',
+ 'eth_chainId',
+ 'eth_estimateGas',
+ 'eth_getCode',
+ 'eth_getStorageAt',
+ 'net_version',
+ 'eth_gasPrice',
+ 'eth_feeHistory',
+ 'eth_maxPriorityFeePerGas',
+ 'eth_getBlockByNumber',
+]);
+
+const MAX_BATCH = 10;
+const ENDPOINT_TIMEOUT_MS = 10_000;
+
+/** True when the chain id is one the server failover table knows. */
+export function isKnownChainId(chainId) {
+ return Number.isInteger(chainId) && chainId > 0 && Boolean(CHAIN_BY_ID[chainId]);
+}
+
+// Validate one or a batch of JSON-RPC payloads. Returns an error code string,
+// or null when every entry is a well-formed call to an allowlisted method.
+export function rejectReason(body) {
+ const entries = Array.isArray(body) ? body : [body];
+ if (entries.length === 0) return 'empty_request';
+ if (entries.length > MAX_BATCH) return 'batch_too_large';
+ for (const e of entries) {
+ if (!e || typeof e !== 'object') return 'malformed_request';
+ if (typeof e.method !== 'string') return 'malformed_request';
+ if (!ALLOWED_METHODS.has(e.method)) return 'method_not_allowed';
+ }
+ return null;
+}
+
+// A response body counts as usable only when it is a JSON-RPC envelope: an
+// object carrying `jsonrpc` (or an `id` plus `result`/`error`), or an array of
+// them. Keyless Ankr answers HTTP 200 with a bare {"error":"Unauthorized..."}
+// string, which must rotate rather than reach an ethers/viem parser.
+function isJsonRpcEnvelope(parsed, batch) {
+ const one = (v) =>
+ v && typeof v === 'object' && !Array.isArray(v) &&
+ (v.jsonrpc === '2.0' || ('id' in v && ('result' in v || (v.error && typeof v.error === 'object'))));
+ if (batch) return Array.isArray(parsed) && parsed.every(one);
+ return one(parsed);
+}
+
+/**
+ * POST the JSON-RPC body to each endpoint in order until one answers with a
+ * well-formed envelope. Network errors, timeouts, non-2xx statuses and
+ * non-envelope bodies rotate to the next endpoint. Resolves to the raw
+ * response text of the first good endpoint; throws when every endpoint failed.
+ *
+ * @param {string[]} urls
+ * @param {object|object[]} body
+ * @param {{ fetchImpl?: typeof fetch, timeoutMs?: number }} [opts]
+ * @returns {Promise<{ text: string, url: string }>}
+ */
+export async function forwardWithRotation(urls, body, { fetchImpl = fetch, timeoutMs = ENDPOINT_TIMEOUT_MS } = {}) {
+ const batch = Array.isArray(body);
+ const payload = JSON.stringify(body);
+ let lastErr = null;
+ for (const url of urls) {
+ const ctrl = new AbortController();
+ const timer = setTimeout(() => ctrl.abort(), timeoutMs);
+ try {
+ const res = await fetchImpl(url, {
+ method: 'POST',
+ headers: { 'content-type': 'application/json', accept: 'application/json' },
+ body: payload,
+ signal: ctrl.signal,
+ });
+ if (!res.ok) throw new Error(`http_${res.status}`);
+ const text = await res.text();
+ let parsed;
+ try {
+ parsed = JSON.parse(text);
+ } catch {
+ throw new Error('non_json_body');
+ }
+ if (!isJsonRpcEnvelope(parsed, batch)) throw new Error('not_jsonrpc_envelope');
+ return { text, url };
+ } catch (e) {
+ lastErr = e instanceof Error ? e : new Error(String(e));
+ } finally {
+ clearTimeout(timer);
+ }
+ }
+ throw new Error(`evm-rpc: all ${urls.length} endpoints failed (${lastErr?.message || 'no endpoints'})`);
+}
+
+export default wrap(async function handler(req, res) {
+ // Same posture as /api/solana-rpc: public, unauthenticated, method-allowlisted,
+ // open to any origin so statically hosted three.ws surfaces can use it. The
+ // per-IP and global rate limits below are the abuse control.
+ if (cors(req, res, { methods: 'POST,OPTIONS', origins: '*', credentials: false })) return;
+ if (!method(req, res, ['POST'])) return;
+
+ const ip = clientIp(req);
+ const [ipRl, globalRl] = await Promise.all([limits.evmRpcIp(ip), limits.evmRpcGlobal()]);
+ if (!ipRl.success || !globalRl.success) {
+ return rateLimited(res, ipRl, 'too many RPC requests');
+ }
+
+ const url = new URL(req.url, 'http://x');
+ const chainId = Number(url.searchParams.get('chainId'));
+ if (!isKnownChainId(chainId)) {
+ return error(res, 400, 'unknown_chain', 'chainId is missing or not a supported EVM chain');
+ }
+
+ let body;
+ try {
+ body = await readJson(req, 200_000);
+ } catch (e) {
+ return error(res, e.status || 400, 'bad_body', 'failed to read JSON body');
+ }
+
+ const reason = rejectReason(body);
+ if (reason) {
+ const msg =
+ reason === 'method_not_allowed'
+ ? 'this RPC method is not permitted through the proxy (read-only methods only)'
+ : reason === 'batch_too_large'
+ ? `batch exceeds ${MAX_BATCH} requests`
+ : 'malformed JSON-RPC request';
+ return error(res, reason === 'method_not_allowed' ? 403 : 400, reason, msg);
+ }
+
+ const endpoints = evmRpcEndpoints(chainId);
+ if (endpoints.length === 0) {
+ return error(res, 502, 'no_upstream', `no RPC endpoint configured for chain ${chainId}`);
+ }
+
+ let forwarded;
+ try {
+ forwarded = await forwardWithRotation(endpoints, body);
+ } catch {
+ return error(res, 502, 'upstream_error', 'every rpc upstream for this chain failed');
+ }
+
+ res.statusCode = 200;
+ res.setHeader('content-type', 'application/json; charset=utf-8');
+ res.setHeader('cache-control', 'no-store');
+ res.end(forwarded.text);
+});
diff --git a/api/garment-forge.js b/api/garment-forge.js
index b785babe8..ce1ac9980 100644
--- a/api/garment-forge.js
+++ b/api/garment-forge.js
@@ -24,6 +24,15 @@
import { cors, json, method, readJson, wrap, rateLimited } from './_lib/http.js';
import { limits, clientIp } from './_lib/rate-limit.js';
+import { fetchUpstream } from './_lib/upstream-fetch.js';
+
+// The worker is a Cloud Run service reached over public HTTPS: a submit is
+// not idempotent (it starts a GPU job), so it gets one attempt and a long
+// deadline; a job read is idempotent and retried. Both share one breaker so a
+// dead worker fails fast for every caller instead of each paying the deadline.
+const WORKER_BREAKER = 'garment-forge-worker';
+const GENERATE_TIMEOUT_MS = 60_000;
+const JOB_TIMEOUT_MS = 10_000;
const GARMENT_SLOTS = [
'top', 'bottom', 'footwear', 'outerwear', 'hair', 'headwear', 'glasses', 'accessory',
@@ -69,14 +78,14 @@ async function startJob(req, res, cfg) {
});
}
- const upstream = await fetch(`${cfg.base}/generate`, {
+ const upstream = await fetchUpstream(`${cfg.base}/generate`, {
method: 'POST',
headers: {
authorization: `Bearer ${cfg.key}`,
'content-type': 'application/json',
},
body: JSON.stringify({ prompt, slot }),
- }).catch(() => null);
+ }, { name: WORKER_BREAKER, timeoutMs: GENERATE_TIMEOUT_MS, attempts: 1, okWhen: () => true }).catch(() => null);
if (!upstream || !upstream.ok) {
return json(res, 502, {
error: 'garment_forge_unavailable',
@@ -98,9 +107,9 @@ async function pollJob(req, res, cfg, jobId) {
const rl = await limits.mcp3dStatus(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
- const upstream = await fetch(`${cfg.base}/jobs/${jobId}`, {
+ const upstream = await fetchUpstream(`${cfg.base}/jobs/${jobId}`, {
headers: { authorization: `Bearer ${cfg.key}` },
- }).catch(() => null);
+ }, { name: WORKER_BREAKER, timeoutMs: JOB_TIMEOUT_MS, attempts: 3, okWhen: () => true }).catch(() => null);
if (!upstream) {
return json(res, 502, { error: 'garment_forge_unavailable', message: 'Worker unreachable.' });
}
diff --git a/api/kol/[action].js b/api/kol/[action].js
index 67676ecd7..4c2ee09a8 100644
--- a/api/kol/[action].js
+++ b/api/kol/[action].js
@@ -9,6 +9,7 @@ import { requireAdmin } from '../_lib/admin.js';
import { createCache } from '../_lib/mem-cache.js';
import { loadWallets, saveImportedWallets } from '../../src/kol/wallet-store.js';
+import { getBalances, walletUsdTotal } from '../_lib/balances.js';
// ── wallets (Birdeye P&L proxy) ───────────────────────────────────────────────
const BIRDEYE_BASE = 'https://public-api.birdeye.so';
@@ -43,6 +44,32 @@ function _setCacheError(addr) {
_cache.set(addr, { error: true }, { ttl: NEG_CACHE_TTL_MS });
}
+// Holdings without Birdeye: the shared balances module reads the wallet over
+// the rotating Solana RPC chain and prices every position through Jupiter, so a
+// missing or rate-limited Birdeye key degrades to our own data, not to a 503.
+async function _fetchOwnHoldings(addr) {
+ const bal = await getBalances({ chain: 'solana', address: addr });
+ const items = [];
+ const nativeUsd = Number(bal?.native?.usd) || 0;
+ if (nativeUsd > 0) items.push({ symbol: 'SOL', valueUsd: nativeUsd });
+ for (const t of bal?.tokens || []) {
+ const valueUsd = Number(t?.usd) || 0;
+ if (valueUsd > 0) items.push({ symbol: t?.symbol || t?.mint?.slice(0, 4) || '?', valueUsd });
+ }
+ return { items, totalUsd: walletUsdTotal(bal), stale: !!bal?.stale };
+}
+
+async function _fetchPortfolio(addr, apiKey) {
+ if (apiKey) {
+ try {
+ return await _fetchBirdeye(addr, apiKey);
+ } catch (err) {
+ console.warn(`[kol] birdeye portfolio failed for ${addr.slice(0, 4)}…, falling back to RPC balances: ${err?.message || err}`);
+ }
+ }
+ return _fetchOwnHoldings(addr);
+}
+
async function _fetchBirdeye(addr, apiKey) {
const url = `${BIRDEYE_BASE}/v1/wallet/portfolio?wallet=${encodeURIComponent(addr)}&chain=solana`;
const ctrl = new AbortController();
@@ -126,8 +153,8 @@ async function handleWallets(req, res) {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
if (!method(req, res, ['GET'])) return;
- const apiKey = process.env.BIRDEYE_API_KEY;
- if (!apiKey) return error(res, 503, 'birdeye_not_configured', 'Birdeye API key not configured');
+ // Optional: without it every wallet is read through the RPC balances path.
+ const apiKey = process.env.BIRDEYE_API_KEY || null;
const rl = await limits.mcpIp(clientIp(req));
if (!rl.success) return rateLimited(res, rl);
@@ -148,10 +175,10 @@ async function handleWallets(req, res) {
await Promise.allSettled(
uncached.map(async (addr) => {
try {
- // _walletPnl never rejects, so only a Birdeye failure reaches the
- // catch: the holdings half is what this endpoint exists to proxy.
+ // _walletPnl never rejects, so only a holdings failure on BOTH the
+ // Birdeye and RPC paths reaches the catch.
const [portfolio, pnl] = await Promise.all([
- _fetchBirdeye(addr, apiKey),
+ _fetchPortfolio(addr, apiKey),
_walletPnl(addr),
]);
_setCache(addr, { ..._normalizeHoldings(addr, portfolio), ...pnl });
diff --git a/api/kol/trades.js b/api/kol/trades.js
index e2950d188..b2d3bb513 100644
--- a/api/kol/trades.js
+++ b/api/kol/trades.js
@@ -37,6 +37,8 @@ export default wrap(async (req, res) => {
try {
result = await fetchKolTrades({ mint, limit });
} catch (err) {
+ // No last-good copy either: honest retryable error with a back-off hint.
+ if (!err.status || err.status >= 500) res.setHeader('Retry-After', '15');
return error(
res,
err.status || 502,
@@ -46,5 +48,11 @@ export default wrap(async (req, res) => {
}
res.setHeader('x-kol-source', result.source || 'unconfigured');
- return json(res, 200, { mint, trades: result.trades, wallets: KOL_WALLETS?.length ?? 0 });
+ if (result.stale) res.setHeader('x-kol-stale', '1');
+ return json(res, 200, {
+ mint,
+ trades: result.trades,
+ wallets: KOL_WALLETS?.length ?? 0,
+ ...(result.stale ? { stale: true, as_of: result.as_of } : {}),
+ });
});
diff --git a/api/oracle-share.js b/api/oracle-share.js
index e865ba58f..90419f3ac 100644
--- a/api/oracle-share.js
+++ b/api/oracle-share.js
@@ -22,6 +22,8 @@ import { cors, method, wrap } from './_lib/http.js';
import { env } from './_lib/env.js';
import { terminalLinks } from '../src/shared/trading-terminals.js';
import { hitRateFor } from './_lib/oracle/conviction.js';
+import { pumpFetchJson } from './_lib/pump-feed-fetch.js';
+import { fetchUpstreamJson } from './_lib/upstream-fetch.js';
const MINT_RE = /^[1-9A-HJ-NP-Za-km-z]{32,44}$/;
const PUMP_V3 = 'https://frontend-api-v3.pump.fun';
@@ -115,17 +117,59 @@ export default wrap(async (req, res) => {
});
// Real pump.fun identity + bonding-curve snapshot for a mint. Keyless public
-// endpoint; mirrors the pump branch of api/_lib/oracle/market.js. Returns null on
-// any failure so the caller degrades gracefully.
+// endpoint; mirrors the pump branch of api/_lib/oracle/market.js. When pump.fun
+// does not answer, DexScreener's pair listing carries the identity fields the
+// share card renders (symbol, name, image, market cap, age, graduation), so the
+// card still names the coin instead of falling back to a bare mint. Returns null
+// only when both are down.
async function fetchPumpIdentity(mint) {
+ const pump = await fetchPumpIdentityLive(mint);
+ if (pump) return pump;
+ return fetchDexScreenerIdentity(mint);
+}
+
+async function fetchDexScreenerIdentity(mint) {
try {
- const r = await fetch(`${PUMP_V3}/coins-v2/${mint}`, {
- headers: { accept: 'application/json' },
- signal: AbortSignal.timeout(3500),
+ const data = await fetchUpstreamJson(`https://api.dexscreener.com/latest/dex/tokens/${mint}`, {}, {
+ timeoutMs: 3500, attempts: 2, name: 'dexscreener', label: 'dexscreener',
});
- if (!r.ok) return null;
- const d = await r.json();
- if (!d || !d.mint) return null;
+ const pairs = (Array.isArray(data?.pairs) ? data.pairs : []).filter((p) => p?.chainId === 'solana');
+ if (!pairs.length) return null;
+ pairs.sort((a, b) => (b.liquidity?.usd ?? 0) - (a.liquidity?.usd ?? 0));
+ const p = pairs[0];
+ const socials = Array.isArray(p.info?.socials) ? p.info.socials : [];
+ const social = (type) => socials.find((s) => s?.type === type)?.url || null;
+ const mcap = Number(p.marketCap ?? p.fdv);
+ return {
+ symbol: p.baseToken?.symbol || null,
+ name: p.baseToken?.name || null,
+ image: p.info?.imageUrl || null,
+ description: null,
+ creator: null,
+ created_at: Number.isFinite(Number(p.pairCreatedAt)) ? new Date(Number(p.pairCreatedAt)).toISOString() : null,
+ // A pump.fun coin still on its curve trades on the "pumpfun" dex; any
+ // other venue means it has graduated.
+ complete: p.dexId !== 'pumpfun',
+ bonding_curve_pct: p.dexId !== 'pumpfun' ? 100 : null,
+ real_sol_reserves: null,
+ reply_count: null,
+ is_live: false,
+ market_cap_usd: Number.isFinite(mcap) && mcap > 0 ? mcap : null,
+ links: {
+ website: Array.isArray(p.info?.websites) ? p.info.websites[0]?.url || null : null,
+ twitter: social('twitter'),
+ telegram: social('telegram'),
+ },
+ };
+ } catch {
+ return null;
+ }
+}
+
+async function fetchPumpIdentityLive(mint) {
+ try {
+ const { ok, body: d } = await pumpFetchJson(`${PUMP_V3}/coins-v2/${mint}`, { timeoutMs: 3500, retries: 1 });
+ if (!ok || !d || !d.mint) return null;
const complete = Boolean(d.complete);
const realTok = Number(d.real_token_reserves);
let bondingPct = complete ? 100 : null;
diff --git a/api/oracle/og.js b/api/oracle/og.js
index 609023c91..1c09b1049 100644
--- a/api/oracle/og.js
+++ b/api/oracle/og.js
@@ -20,6 +20,8 @@
import { cors, wrap } from '../_lib/http.js';
import { sql } from '../_lib/db.js';
import { isQuoteMint } from '../_lib/quote-mints.js';
+import { pumpFetchJson } from '../_lib/pump-feed-fetch.js';
+import { fetchTokenMarketData } from '../_lib/market/token-market.js';
const MINT_RE = /^[1-9A-HJ-NP-Za-km-z]{32,44}$/;
const PUMP_FRONTEND_V3 = 'https://frontend-api-v3.pump.fun';
@@ -155,18 +157,18 @@ async function buildCardData(mint) {
// Live USD market cap for a coin still on the curve. pump.fun emits
// `usd_market_cap` (with `market_cap_usd` as the v2 alias); this card used to ask
// for a `market_cap_in_usd` key that does not exist, so the number was fetched on
-// every render and always came back null. Any failure returns null and the card
-// falls back to the recorded outcome cap, or omits the block.
+// every render and always came back null. When pump.fun does not answer, the
+// shared multi-source market reader (Birdeye, DexScreener, GeckoTerminal, ...)
+// supplies the same figure. Any failure returns null and the card falls back to
+// the recorded outcome cap, or omits the block.
async function fetchLiveMarketCap(mint) {
try {
- const r = await fetch(`${PUMP_FRONTEND_V3}/coins-v2/${mint}`, {
- headers: { accept: 'application/json' },
- signal: AbortSignal.timeout(3000),
- });
- if (!r.ok) return null;
- const d = await r.json();
- const mcap = Number(d?.usd_market_cap ?? d?.market_cap_usd);
- return Number.isFinite(mcap) && mcap > 0 ? mcap : null;
+ const { ok, body: d } = await pumpFetchJson(`${PUMP_FRONTEND_V3}/coins-v2/${mint}`, { timeoutMs: 3000, retries: 1 });
+ const mcap = ok ? Number(d?.usd_market_cap ?? d?.market_cap_usd) : NaN;
+ if (Number.isFinite(mcap) && mcap > 0) return mcap;
+ const md = await fetchTokenMarketData(mint);
+ const fallback = Number(md?.market_cap);
+ return Number.isFinite(fallback) && fallback > 0 ? fallback : null;
} catch {
return null;
}
diff --git a/api/oracle/test-alert.js b/api/oracle/test-alert.js
index de53cc59d..574270732 100644
--- a/api/oracle/test-alert.js
+++ b/api/oracle/test-alert.js
@@ -18,6 +18,7 @@ import { limits, clientIp } from '../_lib/rate-limit.js';
import { sql } from '../_lib/db.js';
import { isUuid } from '../_lib/validate.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
const CHAT_ID_RE = /^-?\d{1,20}$|^@[A-Za-z0-9_]{3,32}$/;
async function resolveUserId(req) {
@@ -70,7 +71,7 @@ export default wrap(async (req, res) => {
let tgOk = false;
let tgErr = null;
try {
- const r = await fetch(`https://api.telegram.org/bot${token}/sendMessage`, {
+ const r = await fetchUpstream(`https://api.telegram.org/bot${token}/sendMessage`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
@@ -80,7 +81,7 @@ export default wrap(async (req, res) => {
disable_web_page_preview: true,
}),
signal: ctrl.signal,
- });
+ }, { name: 'telegram', timeoutMs: 10_000, attempts: 2, okWhen: () => true });
const result = await r.json().catch(() => null);
tgOk = result?.ok === true;
if (!tgOk) tgErr = result?.description || 'Telegram rejected the message';
diff --git a/api/pay/execute.js b/api/pay/execute.js
index fda59ab2b..7c21e5ed4 100644
--- a/api/pay/execute.js
+++ b/api/pay/execute.js
@@ -28,7 +28,7 @@ import {
getAssociatedTokenAddressSync,
createTransferCheckedInstruction,
createAssociatedTokenAccountIdempotentInstruction,
- TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID, getMint,
+ TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
} from '@solana/spl-token';
import { loadSeedKeypair } from '../_lib/x402/pay.js';
@@ -44,6 +44,7 @@ import {
USDC_SOLANA_MINT,
} from '../_lib/pay/probe.js';
import { solanaConnection } from '../_lib/solana/connection.js';
+import { mintDecimals, ataExists, getRecentBlockhash, isRpcUnavailable, RPC_RETRY_AFTER_SECONDS } from '../_lib/solana/read-guards.js';
import {
atomicsToUsd,
verifySessionToken,
@@ -198,24 +199,26 @@ async function buildSolanaPayload({ accept, amount, buyer, conn, resourceUrl })
const receiverAta = getAssociatedTokenAddressSync(
mint, payTo, false, TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
);
- const mintInfo = await getMint(conn, mint);
+ // Guarded reads (api/_lib/solana/read-guards.js): USDC decimals resolve
+ // locally, the ATA probe fails open to an idempotent create, and the blockhash
+ // is served from the shared cache inside its validity window during an outage.
+ const decimals = await mintDecimals(conn, mint);
const ixs = [
ComputeBudgetProgram.setComputeUnitLimit({ units: 60_000 }),
ComputeBudgetProgram.setComputeUnitPrice({ microLamports: 1 }),
];
- const receiverInfo = await conn.getAccountInfo(receiverAta);
- if (!receiverInfo) {
+ if (!(await ataExists(conn, receiverAta))) {
ixs.push(createAssociatedTokenAccountIdempotentInstruction(
feePayer, receiverAta, payTo, mint, TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
));
}
ixs.push(createTransferCheckedInstruction(
senderAta, mint, receiverAta, buyer.publicKey,
- amount, mintInfo.decimals, [], TOKEN_PROGRAM_ID,
+ amount, decimals, [], TOKEN_PROGRAM_ID,
));
- const { blockhash } = await conn.getLatestBlockhash('confirmed');
+ const blockhash = await getRecentBlockhash(conn, SOLANA_RPC);
const message = new TransactionMessage({
payerKey: feePayer,
recentBlockhash: blockhash,
@@ -361,6 +364,13 @@ export default wrap(async (req, res) => {
});
} catch (err) {
await rollbackReservation(sessionRecord.id, amountAtomics).catch(() => {});
+ if (isRpcUnavailable(err)) {
+ // No funds moved: the chain could not be read and nothing was cached.
+ // Typed + retryable so the agent backs off instead of treating it as a
+ // broken endpoint.
+ res.setHeader('retry-after', String(RPC_RETRY_AFTER_SECONDS));
+ return error(res, 503, 'rpc_unavailable', 'Solana RPC temporarily unavailable; payment not built, retry shortly', { retryable: true, retry_after: RPC_RETRY_AFTER_SECONDS });
+ }
return error(res, 502, 'build_failed', `Failed to build Solana payment: ${err?.message}`);
}
diff --git a/api/pump-fun-mcp.js b/api/pump-fun-mcp.js
index 47fa437c6..21d0e638d 100644
--- a/api/pump-fun-mcp.js
+++ b/api/pump-fun-mcp.js
@@ -49,6 +49,7 @@ import { scanFirstClaims } from './_lib/pump-claims.js';
import { eventField } from './_lib/pump-onchain-trades.js';
import { getSolPriceUsd } from '../src/shared/usd-price.js';
+import { pinToIPFS, ipfsPinningConfigured } from './_lib/ipfs-pin.js';
// ── On-chain handlers ──────────────────────────────────────────────────────
const TOTAL_PUMP_TOKEN_SUPPLY = 1_000_000_000; // 1B pump.fun standard
@@ -1059,41 +1060,26 @@ async function handleGetCoinIntel({ mint, network = 'mainnet' } = {}) {
// their coin. Requires bearer auth (API key) so Pinata credits aren't open.
//
// Two-step: (1) pin the image, (2) build the metadata JSON with the image CID
-// and pin that too. Both steps use the same Pinata/Web3.Storage credentials
-// already wired for the pinning API — no new env vars needed.
+// and pin that too. Both steps go through the shared provider chain in
+// api/_lib/ipfs-pin.js (Pinata, then web3.storage, each with a deadline and a
+// breaker) so one provider's outage never fails the launch prep.
-const PINATA_PIN_API = 'https://api.pinata.cloud/pinning/pinFileToIPFS';
-const W3S_PIN_API = 'https://api.web3.storage/upload';
const IMG_MAX_BYTES = 5 * 1024 * 1024; // 5 MB
const META_IMG_TIMEOUT_MS = 8_000;
async function _pinBuffer(buf, filename) {
- const pinataJwt = process.env.PINATA_JWT;
- if (pinataJwt) {
- const form = new FormData();
- form.append('file', new Blob([buf]), filename);
- const r = await fetch(PINATA_PIN_API, {
- method: 'POST',
- headers: { Authorization: `Bearer ${pinataJwt}` },
- body: form,
- });
- if (!r.ok) throw Object.assign(new Error(`Pinata ${r.status}`), { rpcCode: -32004 });
- return { cid: (await r.json()).IpfsHash, provider: 'pinata' };
+ if (!ipfsPinningConfigured()) {
+ throw rpcError(
+ -32004,
+ 'no IPFS pinning provider configured (PINATA_JWT or WEB3_STORAGE_TOKEN)',
+ );
}
- const w3sToken = process.env.WEB3_STORAGE_TOKEN;
- if (w3sToken) {
- const r = await fetch(W3S_PIN_API, {
- method: 'POST',
- headers: { Authorization: `Bearer ${w3sToken}`, 'X-NAME': filename },
- body: buf,
- });
- if (!r.ok) throw Object.assign(new Error(`Web3.Storage ${r.status}`), { rpcCode: -32004 });
- return { cid: (await r.json()).cid, provider: 'web3.storage' };
+ try {
+ const { cid, provider } = await pinToIPFS(buf, filename);
+ return { cid, provider };
+ } catch (err) {
+ throw Object.assign(new Error(`IPFS pin failed: ${err?.message || err}`), { rpcCode: -32004 });
}
- throw rpcError(
- -32004,
- 'no IPFS pinning provider configured (PINATA_JWT or WEB3_STORAGE_TOKEN)',
- );
}
async function _fetchImgBuf(imageUrl) {
diff --git a/api/pump/[action].js b/api/pump/[action].js
index 36f8b05c3..eb71510e5 100644
--- a/api/pump/[action].js
+++ b/api/pump/[action].js
@@ -74,6 +74,9 @@ import { pinToIPFS, ipfsPinningConfigured } from '../_lib/ipfs-pin.js';
import { THREE_WS_VANITY, hasThreeWsMark } from '../../src/solana/vanity/brand.js';
import { grindVanityNode, GrindExhaustedError } from '../../src/solana/vanity/grinder-node.js';
import { logger } from '../_lib/usage.js';
+import { cacheGet, cacheSet } from '../_lib/cache.js';
+import { pumpFetchJson } from '../_lib/pump-feed-fetch.js';
+import { staleEnvelope } from '../_lib/rpc-degrade.js';
const log = logger('pump.launch');
import {
@@ -2346,7 +2349,11 @@ async function handlePortfolio(req, res) {
const conn = solanaConnection(network);
const owner = new PublicKey(address);
- const [lamports, tokenResp, tradeRows, recentBuys] = await Promise.all([
+ // Each chain leg settles on its own: a throttled getBalance must not throw
+ // away a successful token-account read (or vice versa). A failed leg is
+ // named in `degraded` and the response carries `partial: true`; only when
+ // BOTH chain legs fail is there nothing honest to render.
+ const [lamportsRead, tokenRead, tradeRows, recentBuys] = await Promise.allSettled([
conn.getBalance(owner),
conn.getParsedTokenAccountsByOwner(owner, { programId: TOKEN_PROGRAM_ID }),
// Quote-aware cost basis from the trade ledger: net quote spent (buys −
@@ -2369,7 +2376,18 @@ async function handlePortfolio(req, res) {
ORDER BY created_at DESC
LIMIT 500
`.catch(() => []),
- ]);
+ ]).then((settled) => settled.map((r) => (r.status === 'fulfilled' ? r.value : r)));
+
+ const degraded = [];
+ if (lamportsRead?.status === 'rejected') degraded.push('balance');
+ if (tokenRead?.status === 'rejected') degraded.push('token_accounts');
+ if (degraded.length === 2) {
+ console.warn(`[pump/portfolio] both chain legs failed agentId=${agentId}: ${lamportsRead.reason?.message || lamportsRead.reason}`);
+ res.setHeader('Retry-After', '15');
+ return error(res, 503, 'rpc_unavailable', 'Solana RPC is temporarily unavailable, retry shortly', { retry_after: 15 });
+ }
+ const lamports = lamportsRead?.status === 'rejected' ? null : lamportsRead;
+ const tokenResp = tokenRead?.status === 'rejected' ? { value: [] } : tokenRead;
const holdings = tokenResp.value
.map((acc) => {
@@ -2489,9 +2507,10 @@ async function handlePortfolio(req, res) {
address,
network,
lamports,
- sol: lamports / LAMPORTS_PER_SOL,
+ sol: lamports == null ? null : lamports / LAMPORTS_PER_SOL,
holdings: priced,
subtotals,
+ ...(degraded.length ? { partial: true, degraded } : {}),
totalValueSol,
totalCostBasisSol,
unrealizedPnlSol,
@@ -3655,6 +3674,22 @@ async function handleTrending(req, res) {
const COIN_CACHE = new Map(); // mint → { at, body }
const COIN_TTL_MS = 30_000;
+// Shared last-good tier: a cold instance holds no COIN_CACHE, and pump.fun's
+// frontend API rate-limits in windows longer than a request budget. The last
+// body a healthy fetch produced is served marked stale until pump.fun answers.
+const COIN_LKG_TTL_S = 24 * 3600;
+const coinLkgKey = (mint) => `pump:coin:lkg:${mint}`;
+
+async function serveStaleCoin(res, mint, status, code, message) {
+ const hit = COIN_CACHE.get(mint) || (await cacheGet(coinLkgKey(mint)));
+ if (hit && hit.body && typeof hit.at === 'number') {
+ res.setHeader('cache-control', 'public, max-age=10');
+ res.setHeader('x-pump-stale', '1');
+ return json(res, 200, staleEnvelope(hit.body, hit.at));
+ }
+ res.setHeader('Retry-After', '15');
+ return error(res, status, code, message, { retry_after: 15 });
+}
async function handleCoin(req, res) {
if (cors(req, res, { methods: 'GET,OPTIONS', origins: '*' })) return;
@@ -3673,30 +3708,24 @@ async function handleCoin(req, res) {
return json(res, 200, hit.body);
}
- let resp;
- try {
- resp = await fetch(new URL(`/coins/${mint}`, PUMP_FRONTEND_BASE), {
- headers: { accept: 'application/json' },
- signal: AbortSignal.timeout(8000),
- });
- } catch {
- return error(res, 504, 'upstream_timeout', 'pump.fun did not respond');
- }
- if (!resp.ok) return error(res, 502, 'upstream_failed', `pump.fun returned ${resp.status}`);
- // pump.fun returns 200 with an empty body for an unknown/unmigrated mint;
- // resp.json() then throws "Unexpected end of JSON input" → an unhandled 500.
- // Treat an empty/invalid body as "coin not found" (404) instead.
- let parsed;
- try {
- parsed = await resp.json();
- } catch {
- return error(res, 404, 'coin_not_found', 'no pump.fun coin for that mint');
+ // Bounded retry on 429/5xx with Retry-After honoured; never throws.
+ const r = await pumpFetchJson(new URL(`/coins/${mint}`, PUMP_FRONTEND_BASE).toString(), { timeoutMs: 8000 });
+ if (!r.ok) {
+ // pump.fun returns 200 with an empty body for an unknown/unmigrated mint;
+ // pumpFetchJson reports that as ok:false with the 200 status. That is a
+ // real "coin not found", not an outage, so it never falls to last-good.
+ if (r.status >= 200 && r.status < 300) return error(res, 404, 'coin_not_found', 'no pump.fun coin for that mint');
+ if (r.status === 0) return serveStaleCoin(res, mint, 504, 'upstream_timeout', 'pump.fun did not respond');
+ if (r.status === 404) return error(res, 404, 'coin_not_found', 'no pump.fun coin for that mint');
+ return serveStaleCoin(res, mint, 502, 'upstream_failed', `pump.fun returned ${r.status}`);
}
+ const parsed = r.body;
if (!parsed || typeof parsed !== 'object') {
return error(res, 404, 'coin_not_found', 'no pump.fun coin for that mint');
}
const [body] = repairCoinImages([parsed]);
COIN_CACHE.set(mint, { at: now, body });
+ cacheSet(coinLkgKey(mint), { at: now, body }, COIN_LKG_TTL_S).catch(() => {});
res.setHeader('cache-control', 'public, max-age=15');
return json(res, 200, body);
}
@@ -3799,28 +3828,29 @@ async function handleSearch(req, res) {
upstream.searchParams.set('sort', 'market_cap');
upstream.searchParams.set('order', 'DESC');
upstream.searchParams.set('includeNsfw', 'false');
- let resp;
- try {
- resp = await fetch(upstream, {
- headers: { accept: 'application/json' },
- signal: AbortSignal.timeout(8000),
- });
- } catch {
- // Upstream timeout/network blip: an empty result set beats a 500 on search.
- return json(res, 200, []);
- }
- if (!resp.ok) return error(res, 502, 'upstream_failed', `pump.fun returned ${resp.status}`);
- // A 200 with an empty/truncated body makes resp.json() throw; degrade to "no
- // results" rather than an unhandled 500.
- let body;
- try {
- body = await resp.json();
- } catch {
- body = [];
+ const searchLkgKey = `pump:search:lkg:${q.toLowerCase()}`;
+ // Bounded retry on 429/5xx with Retry-After honoured; never throws. A 200
+ // with an empty/truncated body reports ok:false with a 2xx status, which is
+ // "no results", not an outage.
+ const r = await pumpFetchJson(upstream.toString(), { timeoutMs: 8000 });
+ if (!r.ok) {
+ if (r.status >= 200 && r.status < 300) return json(res, 200, []);
+ // Outage or rate-limit window: the last result set this query produced
+ // beats both a 502 and a misleading empty list.
+ const lkg = await cacheGet(searchLkgKey);
+ if (lkg && Array.isArray(lkg.rows) && typeof lkg.at === 'number') {
+ res.setHeader('x-pump-stale', '1');
+ return json(res, 200, lkg.rows, { 'x-pump-as-of': new Date(lkg.at).toISOString() });
+ }
+ if (r.status === 0) return json(res, 200, []);
+ res.setHeader('Retry-After', '15');
+ return error(res, 502, 'upstream_failed', `pump.fun returned ${r.status}`, { retry_after: 15 });
}
+ const body = r.body;
const arr = repairCoinImages(
Array.isArray(body) ? body : Array.isArray(body?.coins) ? body.coins : [],
);
+ if (arr.length) cacheSet(searchLkgKey, { rows: arr, at: Date.now() }, 3600).catch(() => {});
return json(res, 200, arr);
}
@@ -4974,6 +5004,21 @@ async function handleGithubResolve(req, res) {
return error(res, 400, 'validation_error', 'invalid github handle');
}
+ // A GitHub login maps to one numeric id for the account's whole life, so a
+ // resolved profile is safe to hold for days: a 60/hr unauthenticated rate
+ // limit window must not turn "rewards to @handle" into a 502.
+ const lkgKey = `github:user:lkg:${handle.toLowerCase()}`;
+ const GITHUB_LKG_TTL_S = 7 * 24 * 3600;
+ const serveLastGood = async (status, code, message) => {
+ const lkg = await cacheGet(lkgKey);
+ if (lkg && lkg.body && typeof lkg.at === 'number') {
+ res.setHeader('x-github-stale', '1');
+ return json(res, 200, staleEnvelope(lkg.body, lkg.at));
+ }
+ if (status === 429 || status === 502) res.setHeader('Retry-After', '15');
+ return error(res, status, code, message, ...(status === 429 || status === 502 ? [{ retry_after: 15 }] : []));
+ };
+
try {
const headers = { 'user-agent': 'three.ws', accept: 'application/vnd.github+json' };
if (process.env.GITHUB_TOKEN) headers.authorization = `Bearer ${process.env.GITHUB_TOKEN}`;
@@ -4983,12 +5028,12 @@ async function handleGithubResolve(req, res) {
});
if (gh.status === 404)
return error(res, 404, 'github_user_not_found', `@${handle} not found on GitHub`);
- if (gh.status === 403)
- return error(res, 429, 'github_rate_limited', 'GitHub rate limit — try again shortly');
- if (!gh.ok) return error(res, 502, 'github_error', `github responded ${gh.status}`);
+ if (gh.status === 403 || gh.status === 429)
+ return serveLastGood(429, 'github_rate_limited', 'GitHub rate limit, try again shortly');
+ if (!gh.ok) return serveLastGood(502, 'github_error', `github responded ${gh.status}`);
const j = await gh.json();
- if (!j?.id) return error(res, 502, 'github_error', 'github id missing');
- return json(res, 200, {
+ if (!j?.id) return serveLastGood(502, 'github_error', 'github id missing');
+ const body = {
ok: true,
login: j.login,
id: String(j.id), // social-fee user_id (≤20 chars, fits u64)
@@ -4996,9 +5041,17 @@ async function handleGithubResolve(req, res) {
avatar_url: j.avatar_url || null,
html_url: j.html_url || `https://github.com/${j.login}`,
type: j.type || 'User',
- });
+ };
+ cacheSet(lkgKey, { body, at: Date.now() }, GITHUB_LKG_TTL_S).catch(() => {});
+ return json(res, 200, body);
} catch (e) {
- return respondError(res, 502, 'github_error', e);
+ const lkg = await cacheGet(lkgKey);
+ if (lkg && lkg.body && typeof lkg.at === 'number') {
+ res.setHeader('x-github-stale', '1');
+ return json(res, 200, staleEnvelope(lkg.body, lkg.at));
+ }
+ res.setHeader('Retry-After', '15');
+ return respondError(res, 502, 'github_error', e, { retry_after: 15 });
}
}
diff --git a/api/rider/check.js b/api/rider/check.js
index 685f23e79..f16afaa24 100644
--- a/api/rider/check.js
+++ b/api/rider/check.js
@@ -14,6 +14,14 @@ import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { REQUIRED_AMOUNT } from '../_lib/rider.js';
import { TOKEN_MINT as THREE_MINT } from '../_lib/token/config.js';
+import { cacheGet, cacheSet } from '../_lib/cache.js';
+import { staleEnvelope, RPC_RETRY_AFTER_S } from '../_lib/rpc-degrade.js';
+
+// Last verdict per wallet. A pass check gates a ride, so the window is short:
+// long enough to ride out an RPC blip mid-session, short enough that a wallet
+// that just sold its $THREE is re-read within minutes.
+const VERDICT_LKG_TTL_S = 10 * 60;
+const verdictKey = (address) => `rider:check:lkg:${address}`;
async function heldBalance(owner) {
const connection = solanaConnection({ url: env.SOLANA_RPC_URL, commitment: 'confirmed' });
@@ -78,7 +86,17 @@ export default wrap(async (req, res) => {
// Never echo the rejection message: a web3.js network error embeds the
// keyed RPC URL, which would leak HELIUS_API_KEY to the caller.
console.error('[rider/check] $THREE balance read failed', balanceRead.reason);
- return error(res, 502, 'rpc_unavailable', 'could not read the $THREE balance for this wallet');
+ // The last verdict this wallet got, marked stale, beats refusing the
+ // gate outright; a wallet never checked before gets the typed 503.
+ const lkg = await cacheGet(verdictKey(address));
+ if (lkg && lkg.body && typeof lkg.at === 'number') {
+ res.setHeader('x-rider-stale', '1');
+ return json(res, 200, staleEnvelope(lkg.body, lkg.at));
+ }
+ res.setHeader('Retry-After', String(RPC_RETRY_AFTER_S));
+ return error(res, 503, 'rpc_unavailable', 'could not read the $THREE balance for this wallet, retry shortly', {
+ retry_after: RPC_RETRY_AFTER_S,
+ });
}
}
if (!paidPass && paymentRead.status === 'rejected') {
@@ -87,7 +105,7 @@ export default wrap(async (req, res) => {
if (!holderPass) throw paymentRead.reason;
}
- return json(res, 200, {
+ const body = {
has_pass: holderPass || paidPass,
holder_pass: holderPass,
paid_pass: paidPass,
@@ -96,5 +114,10 @@ export default wrap(async (req, res) => {
tx_signature: payment?.tx_signature ?? null,
required_amount: REQUIRED_AMOUNT,
mint: THREE_MINT,
- });
+ };
+ // Only a verdict both sources actually answered is worth remembering.
+ if (balanceRead.status === 'fulfilled' && paymentRead.status === 'fulfilled') {
+ cacheSet(verdictKey(address), { body, at: Date.now() }, VERDICT_LKG_TTL_S).catch(() => {});
+ }
+ return json(res, 200, body);
});
diff --git a/api/token/[action].js b/api/token/[action].js
index 227b6c7e6..e2ec8cb9c 100644
--- a/api/token/[action].js
+++ b/api/token/[action].js
@@ -17,6 +17,7 @@ import { isAdminUser } from '../_lib/admin.js';
import { cors, error, json, method, readJson, wrap, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { parse } from '../_lib/validate.js';
+import { isRpcOutageError } from '../_lib/rpc-degrade.js';
import { publicConfig, ATOMICS_PER_TOKEN } from '../_lib/token/config.js';
import { getTokenPriceUsd, quoteTokenForUsd } from '../_lib/token/price.js';
import {
@@ -56,6 +57,7 @@ const QUOTE_UNAVAILABLE_DETAIL = {
price_unavailable: 'Live $THREE price is briefly unavailable. Try again in a moment.',
treasury_unavailable: 'The $THREE payment rail is briefly unavailable. Try again shortly.',
rewards_unavailable: 'The $THREE payment rail is briefly unavailable. Try again shortly.',
+ rpc_unavailable: 'Solana RPC is briefly unavailable, so a live quote cannot be priced. Try again shortly.',
};
// ── GET /api/token/config ────────────────────────────────────────────────────
@@ -137,12 +139,16 @@ async function handleQuote(req, res) {
refId: body.ref_id ?? null,
});
} catch (err) {
- const detail = QUOTE_UNAVAILABLE_DETAIL[err?.code];
+ // A quote is never served stale: it prices a live payment. When the price
+ // or treasury read died because every RPC lane was exhausted, the honest
+ // answer is the same retryable 503 the named codes already get.
+ const code = QUOTE_UNAVAILABLE_DETAIL[err?.code] ? err.code : isRpcOutageError(err) ? 'rpc_unavailable' : null;
+ const detail = code ? QUOTE_UNAVAILABLE_DETAIL[code] : null;
if (detail) {
// Honest, safe-to-show 503 with the actionable code preserved + a back-off
// hint, so the client renders a clean "try again shortly" retry state.
res.setHeader('retry-after', '15');
- return error(res, 503, err.code, detail, { retry_after: 15 });
+ return error(res, 503, code, detail, { retry_after: 15 });
}
throw err; // genuine fault → wrap() logs, alerts, and sanitizes it.
}
diff --git a/api/tx/explain.js b/api/tx/explain.js
index 20ff93a69..bc46aee30 100644
--- a/api/tx/explain.js
+++ b/api/tx/explain.js
@@ -7,6 +7,7 @@ import { llmComplete, llmConfigured } from '../_lib/llm.js';
import { cacheGet, cacheSet } from '../_lib/cache.js';
import { solanaConnection } from '../_lib/solana/connection.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
// Reduce a raw parsed transaction to the same shape Helius's enhanced endpoint
// returns, so the fallback path is transparent to the LLM summary and the UI.
// Native transfers come from lamport balance deltas; token transfers from the
@@ -133,13 +134,14 @@ export default wrap(async (req, res) => {
// the transfer shape ourselves, so a Helius outage still explains the tx.
let enhanced = null;
try {
- const resp = await fetch(
+ const resp = await fetchUpstream(
`https://api.helius.xyz/v0/transactions/?api-key=${env.HELIUS_API_KEY}`,
{
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ transactions: [sig] }),
},
+ { name: 'helius-enhanced-tx', timeoutMs: 8_000, attempts: 2, okWhen: () => true },
);
if (resp.ok) {
const data = await resp.json();
@@ -191,12 +193,13 @@ export default wrap(async (req, res) => {
'https://ethereum-rpc.publicnode.com',
].filter(Boolean);
+ // Bounded per endpoint, or one hung node would starve the rest of the list.
const rpcAt = (url) => (id, methodName, params) =>
- fetch(url, {
+ fetchUpstream(url, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ jsonrpc: '2.0', id, method: methodName, params }),
- });
+ }, { timeoutMs: 6_000, attempts: 1, okWhen: () => true });
let txJson = null;
let receiptJson = null;
diff --git a/api/x402-pay.js b/api/x402-pay.js
index d05cac436..61e29105a 100644
--- a/api/x402-pay.js
+++ b/api/x402-pay.js
@@ -27,8 +27,9 @@ import {
import {
getAssociatedTokenAddressSync, createTransferCheckedInstruction,
createAssociatedTokenAccountIdempotentInstruction,
- TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID, getMint,
+ TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
} from '@solana/spl-token';
+import { mintDecimals, ataExists, getRecentBlockhash, isRpcUnavailable, RPC_RETRY_AFTER_SECONDS } from './_lib/solana/read-guards.js';
import { cors, json, readJson, wrap, rateLimited, setRateLimitHeaders, respondError } from './_lib/http.js';
import { limits, clientIp } from './_lib/rate-limit.js';
import { requireCsrf } from './_lib/csrf.js';
@@ -337,14 +338,17 @@ async function buildSolanaPaymentPayload({ accept, buyer, conn, resourceUrl }) {
const receiverAta = getAssociatedTokenAddressSync(
mint, payTo, false, TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
);
- const mintInfo = await getMint(conn, mint);
+ // Guarded reads (api/_lib/solana/read-guards.js): USDC decimals never touch the
+ // network, the ATA probe fails open to an idempotent create, and the blockhash
+ // is served from cache inside its validity window when every RPC lane is down.
+ // What still fails throws a typed 503 rpc_unavailable; no funds have moved yet.
+ const decimals = await mintDecimals(conn, mint);
const ixs = [
ComputeBudgetProgram.setComputeUnitLimit({ units: 60_000 }),
ComputeBudgetProgram.setComputeUnitPrice({ microLamports: 1 }),
];
- const receiverInfo = await conn.getAccountInfo(receiverAta);
- if (!receiverInfo) {
+ if (!(await ataExists(conn, receiverAta))) {
ixs.push(createAssociatedTokenAccountIdempotentInstruction(
feePayer, receiverAta, payTo, mint,
TOKEN_PROGRAM_ID, ASSOCIATED_TOKEN_PROGRAM_ID,
@@ -352,10 +356,10 @@ async function buildSolanaPaymentPayload({ accept, buyer, conn, resourceUrl }) {
}
ixs.push(createTransferCheckedInstruction(
senderAta, mint, receiverAta, buyer.publicKey,
- amount, mintInfo.decimals, [], TOKEN_PROGRAM_ID,
+ amount, decimals, [], TOKEN_PROGRAM_ID,
));
- const { blockhash } = await conn.getLatestBlockhash('confirmed');
+ const blockhash = await getRecentBlockhash(conn, SOLANA_RPC);
const message = new TransactionMessage({
payerKey: feePayer,
recentBlockhash: blockhash,
@@ -880,6 +884,8 @@ function payErrorEnvelope(err) {
error_description = 'No payment was attempted, so no funds were transferred from the agent wallet.';
} else if (code === 'invalid_payment' || code === 'payment_required' || code === 'builder_code_tampered') {
error_description = 'The payment was rejected before settlement; no funds were transferred.';
+ } else if (code === 'rpc_unavailable') {
+ error_description = 'The Solana RPC is temporarily unavailable, so the payment could not be built. No funds were transferred; retry shortly.';
}
const envelope = {
ok: false,
@@ -900,6 +906,12 @@ function payErrorStatus(err) {
return 500;
}
+// Response headers for a flow error: an RPC outage advertises Retry-After so
+// callers and CDNs back off instead of hammering a dead chain.
+function payErrorHeaders(err) {
+ return isRpcUnavailable(err) ? { 'retry-after': String(RPC_RETRY_AFTER_SECONDS) } : {};
+}
+
// Buy one of the in-house SHOWCASE_ENDPOINTS from the shared platform wallet and
// stream the payment lifecycle. This is what /agent-exchange runs: the two 3D
// avatars are a view over a real x402 purchase of /api/x402/crypto-intel, so the
@@ -964,7 +976,7 @@ async function handleShowcasePay(req, res, input, ip) {
try {
await runExternalFlow({ ...flowArgs, emit });
} catch (err) {
- return json(res, payErrorStatus(err), payErrorEnvelope(err));
+ return json(res, payErrorStatus(err), payErrorEnvelope(err), payErrorHeaders(err));
}
return json(res, 200, final);
}
@@ -1031,7 +1043,7 @@ async function handleExternalPay(req, res, input, ip) {
method,
});
} catch (err) {
- return json(res, payErrorStatus(err), payErrorEnvelope(err));
+ return json(res, payErrorStatus(err), payErrorEnvelope(err), payErrorHeaders(err));
}
}
@@ -1080,7 +1092,7 @@ async function handleExternalPay(req, res, input, ip) {
serviceLabel: typeof input.service_label === 'string' ? input.service_label : null,
});
} catch (err) {
- return json(res, payErrorStatus(err), payErrorEnvelope(err));
+ return json(res, payErrorStatus(err), payErrorEnvelope(err), payErrorHeaders(err));
}
return json(res, 200, final);
}
@@ -1226,7 +1238,7 @@ export default wrap(async (req, res) => {
try {
await runFlow({ tool, args, emit, buyer, resourceUrl, spendGuard });
} catch (err) {
- return json(res, payErrorStatus(err), payErrorEnvelope(err));
+ return json(res, payErrorStatus(err), payErrorEnvelope(err), payErrorHeaders(err));
}
return json(res, 200, final);
});
diff --git a/api/x402/defi-radar.js b/api/x402/defi-radar.js
index 6d7f143a7..9a1e53da6 100644
--- a/api/x402/defi-radar.js
+++ b/api/x402/defi-radar.js
@@ -16,6 +16,8 @@ import { installAccessControl } from '../_lib/x402/access-control.js';
import { withService } from '../_lib/x402/bazaar-helpers.js';
import { priceFor } from '../_lib/x402-prices.js';
import listing from '../_lib/service-catalog/services/defi-radar.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
const ROUTE = '/api/x402/defi-radar';
@@ -27,16 +29,19 @@ const DESCRIPTION = listing.description;
const TTL_MS = 300_000;
const MIN_MOVER_TVL_USD = 10_000_000; // a "top gainer" on $40k TVL is noise
+// A paid call refuses before settlement when its upstream is down, so the buyer
+// is never charged for nothing. That is right for a cold start and wrong for a
+// blip: the shared last-good copy below serves a recent board through a short
+// outage and the pre-settle 503 fires only when there has never been data.
+const STALE_TTL_S = 15 * 60;
let _cache = null; // { value, expiresAt }
const finite = (n) => (Number.isFinite(n) ? n : null);
async function fetchLlama(path) {
- const r = await fetch(`https://api.llama.fi${path}`, {
+ const r = await fetchUpstream(`https://api.llama.fi${path}`, {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!r.ok) throw new Error(`llama ${path} ${r.status}`);
+ }, { timeoutMs: 10_000, attempts: 2, label: `llama ${path}` });
return r.json();
}
@@ -45,7 +50,12 @@ async function fetchLlama(path) {
async function loadRadar() {
const now = Date.now();
if (_cache && _cache.expiresAt > now) return _cache.value;
+ const value = await cacheWrapLastGood('x402:defi-radar', TTL_MS / 1000, buildRadar, { staleTtlSeconds: STALE_TTL_S });
+ _cache = { value, expiresAt: now + TTL_MS };
+ return value;
+}
+async function buildRadar() {
const [protocols, fees, dexs] = await Promise.all([
fetchLlama('/protocols'),
fetchLlama('/overview/fees?excludeTotalDataChartBreakdown=true&dataType=dailyFees'),
@@ -98,7 +108,6 @@ async function loadRadar() {
fees: { total_24h_usd: finite(Number(fees.total24h)), top: feeRows },
dex: { total_24h_usd: finite(Number(dexs.total24h)), top: dexRows },
};
- _cache = { value, expiresAt: now + TTL_MS };
return value;
}
diff --git a/api/x402/hack-check.js b/api/x402/hack-check.js
index b2abb9874..3f3516293 100644
--- a/api/x402/hack-check.js
+++ b/api/x402/hack-check.js
@@ -17,6 +17,8 @@ import { installAccessControl } from '../_lib/x402/access-control.js';
import { withService } from '../_lib/x402/bazaar-helpers.js';
import { priceFor } from '../_lib/x402-prices.js';
import listing from '../_lib/service-catalog/services/hack-check.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
const ROUTE = '/api/x402/hack-check';
const DESCRIPTION = listing.description;
@@ -24,6 +26,11 @@ const DESCRIPTION = listing.description;
const TTL_MS = 600_000;
const YEAR_MS = 365 * 24 * 60 * 60 * 1000;
+// A paid call refuses before settlement when its upstream is down, so the buyer
+// is never charged for nothing. That is right for a cold start and wrong for a
+// blip: the shared last-good copy below serves a recent board through a short
+// outage and the pre-settle 503 fires only when there has never been data.
+const STALE_TTL_S = 15 * 60;
let _cache = null; // { hacks, stats, expiresAt }
function normalizeChains(chain) {
@@ -35,12 +42,18 @@ function normalizeChains(chain) {
async function loadDataset() {
const now = Date.now();
if (_cache && _cache.expiresAt > now) return _cache;
+ const { hacks, stats } = await cacheWrapLastGood(
+ 'x402:hack-check', TTL_MS / 1000, buildDataset, { staleTtlSeconds: STALE_TTL_S },
+ );
+ _cache = { hacks, stats, expiresAt: now + TTL_MS };
+ return _cache;
+}
- const r = await fetch('https://api.llama.fi/hacks', {
+async function buildDataset() {
+ const now = Date.now();
+ const r = await fetchUpstream('https://api.llama.fi/hacks', {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!r.ok) throw new Error(`llama hacks ${r.status}`);
+ }, { timeoutMs: 10_000, attempts: 2, label: 'llama hacks' });
const raw = await r.json();
if (!Array.isArray(raw)) throw new Error('unexpected upstream shape');
@@ -80,7 +93,7 @@ async function loadDataset() {
}
}
- _cache = {
+ return {
hacks,
stats: {
total_stolen_all_time: totalAllTime,
@@ -88,9 +101,7 @@ async function loadDataset() {
incidents_12mo: incidents12mo,
bridge_hack_share_pct: totalAllTime > 0 ? (bridgeAllTime / totalAllTime) * 100 : 0,
},
- expiresAt: now + TTL_MS,
};
- return _cache;
}
export const INPUT_SCHEMA = {
diff --git a/api/x402/market-mood.js b/api/x402/market-mood.js
index c03bbbde5..327a67448 100644
--- a/api/x402/market-mood.js
+++ b/api/x402/market-mood.js
@@ -25,6 +25,8 @@ import { withService } from '../_lib/x402/bazaar-helpers.js';
import { priceFor } from '../_lib/x402-prices.js';
import { getNews } from '../_lib/news.js';
import listing from '../_lib/service-catalog/services/market-mood.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
const ROUTE = '/api/x402/market-mood';
const DESCRIPTION = listing.description;
@@ -35,6 +37,13 @@ const FNG_WEIGHT = 0.6; // positioning
const NEWS_WEIGHT = 0.4; // narrative
const DRIVER_COUNT = 3;
+// A paid call refuses before settlement when a component is down, so the buyer
+// is never charged for a fabricated composite. That is right for a cold start
+// and wrong for a blip: the shared last-good copy below serves a recent reading
+// through a short outage and the pre-settle 503 fires only when there has never
+// been data.
+const STALE_TTL_S = 15 * 60;
+
let _cache = null; // { value, expiresAt }
function classify(v) {
@@ -46,11 +55,9 @@ function classify(v) {
}
async function fetchFearGreed() {
- const r = await fetch('https://api.alternative.me/fng/?limit=2', {
+ const r = await fetchUpstream('https://api.alternative.me/fng/?limit=2', {
headers: { accept: 'application/json' },
- signal: AbortSignal.timeout(8000),
- });
- if (!r.ok) throw new Error(`fng ${r.status}`);
+ }, { timeoutMs: 8000, attempts: 2, label: 'fng' });
const raw = await r.json();
const rows = Array.isArray(raw?.data) ? raw.data : [];
const today = Number(rows[0]?.value);
@@ -131,12 +138,12 @@ export function summarizeDvol(rows) {
async function fetchDvol(currency) {
const end = Date.now();
const start = end - 24 * 3_600_000;
- const r = await fetch(
+ const r = await fetchUpstream(
`https://www.deribit.com/api/v2/public/get_volatility_index_data?currency=${currency}` +
`&start_timestamp=${start}&end_timestamp=${end}&resolution=3600`,
- { headers: { accept: 'application/json' }, signal: AbortSignal.timeout(8000) },
+ { headers: { accept: 'application/json' } },
+ { timeoutMs: 8000, attempts: 2, label: 'dvol' },
);
- if (!r.ok) throw new Error(`dvol ${r.status}`);
return summarizeDvol((await r.json())?.result?.data);
}
@@ -153,7 +160,12 @@ async function fetchVolatility() {
async function loadMood() {
const now = Date.now();
if (_cache && _cache.expiresAt > now) return _cache.value;
+ const value = await cacheWrapLastGood('x402:market-mood', TTL_MS / 1000, buildMood, { staleTtlSeconds: STALE_TTL_S });
+ _cache = { value, expiresAt: now + TTL_MS };
+ return value;
+}
+async function buildMood() {
// Both components are required — Promise.all rejects if either is down and
// the caller refunds. A one-legged "composite" would be a fabricated signal.
// fetchVolatility never rejects (best-effort enrichment, null on outage).
@@ -194,7 +206,6 @@ async function loadMood() {
},
drivers: news.drivers,
};
- _cache = { value, expiresAt: now + TTL_MS };
return value;
}
diff --git a/api/x402/pump-agent-audit.js b/api/x402/pump-agent-audit.js
index aca8b2dd3..e20b7b3f5 100644
--- a/api/x402/pump-agent-audit.js
+++ b/api/x402/pump-agent-audit.js
@@ -26,6 +26,7 @@ import { sql } from '../_lib/db.js';
import { priceFor } from '../_lib/x402-prices.js';
import { recentPumpLaunches } from '../_lib/pump-launch-feed.js';
import { readBody } from '../_lib/http.js';
+import { pumpFetchJson } from '../_lib/pump-feed-fetch.js';
const ROUTE = '/api/x402/pump-agent-audit';
@@ -451,31 +452,19 @@ const WHALE_LIMIT_MAX = 25;
async function fetchTopCoins(limit) {
const url = `${PUMP_FRONTEND_BASE_AUDIT}/coins?offset=0&limit=${limit}&sort=market_cap&order=DESC&includeNsfw=false`;
- const r = await fetch(url, {
- headers: { accept: 'application/json', 'user-agent': 'three.ws-whale-oracle/1' },
- signal: AbortSignal.timeout(7000),
- });
- if (!r.ok) return [];
- const body = await r.json().catch(() => null);
+ const { ok, body } = await pumpFetchJson(url, { timeoutMs: 7000, retries: 1 });
+ if (!ok) return [];
const coins = Array.isArray(body) ? body : Array.isArray(body?.coins) ? body.coins : [];
return coins.filter((c) => c && typeof c.mint === 'string' && c.mint.length >= 32);
}
async function fetchCoinTrades(mint) {
- try {
- const r = await fetch(
- `${PUMP_SWAP_BASE_AUDIT}/v2/coins/${mint}/trades?limit=50`,
- {
- headers: { accept: 'application/json', 'user-agent': 'three.ws-whale-oracle/1' },
- signal: AbortSignal.timeout(6000),
- },
- );
- if (!r.ok) return [];
- const body = await r.json().catch(() => null);
- return Array.isArray(body) ? body : Array.isArray(body?.trades) ? body.trades : [];
- } catch {
- return [];
- }
+ const { ok, body } = await pumpFetchJson(
+ `${PUMP_SWAP_BASE_AUDIT}/v2/coins/${mint}/trades?limit=50`,
+ { timeoutMs: 6000, retries: 1 },
+ );
+ if (!ok) return [];
+ return Array.isArray(body) ? body : Array.isArray(body?.trades) ? body.trades : [];
}
function parseNum(v) {
diff --git a/api/x402/stablecoin-health.js b/api/x402/stablecoin-health.js
index f984d0629..325cb80c3 100644
--- a/api/x402/stablecoin-health.js
+++ b/api/x402/stablecoin-health.js
@@ -15,6 +15,8 @@ import { installAccessControl } from '../_lib/x402/access-control.js';
import { withService } from '../_lib/x402/bazaar-helpers.js';
import { priceFor } from '../_lib/x402-prices.js';
import listing from '../_lib/service-catalog/services/stablecoin-health.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
const ROUTE = '/api/x402/stablecoin-health';
const DESCRIPTION = listing.description;
@@ -23,6 +25,11 @@ const TTL_MS = 300_000;
const DRIFT_BPS = 25;
const DEPEG_BPS = 100;
+// A paid call refuses before settlement when its upstream is down, so the buyer
+// is never charged for nothing. That is right for a cold start and wrong for a
+// blip: the shared last-good copy below serves a recent board through a short
+// outage and the pre-settle 503 fires only when there has never been data.
+const STALE_TTL_S = 15 * 60;
let _cache = null; // { coins, totalCirculating, expiresAt }
const finite = (n) => (Number.isFinite(n) ? n : null);
@@ -74,12 +81,17 @@ export function toCoin(a) {
async function loadStablecoins() {
const now = Date.now();
if (_cache && _cache.expiresAt > now) return _cache;
+ const { coins, totalCirculating } = await cacheWrapLastGood(
+ 'x402:stablecoin-health', TTL_MS / 1000, buildStablecoins, { staleTtlSeconds: STALE_TTL_S },
+ );
+ _cache = { coins, totalCirculating, expiresAt: now + TTL_MS };
+ return _cache;
+}
- const r = await fetch('https://stablecoins.llama.fi/stablecoins?includePrices=true', {
+async function buildStablecoins() {
+ const r = await fetchUpstream('https://stablecoins.llama.fi/stablecoins?includePrices=true', {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(10_000),
- });
- if (!r.ok) throw new Error(`llama stablecoins ${r.status}`);
+ }, { timeoutMs: 10_000, attempts: 2, label: 'llama stablecoins' });
const raw = await r.json();
if (!Array.isArray(raw?.peggedAssets)) throw new Error('unexpected upstream shape');
@@ -92,9 +104,7 @@ async function loadStablecoins() {
coins.push(coin);
}
coins.sort((a, b) => b.circulating_usd - a.circulating_usd);
-
- _cache = { coins, totalCirculating, expiresAt: now + TTL_MS };
- return _cache;
+ return { coins, totalCirculating };
}
export const INPUT_SCHEMA = {
diff --git a/api/x402/three-intel.js b/api/x402/three-intel.js
index 47e0bbb92..da12bd9d7 100644
--- a/api/x402/three-intel.js
+++ b/api/x402/three-intel.js
@@ -23,6 +23,7 @@ import { installAccessControl } from '../_lib/x402/access-control.js';
import { withService } from '../_lib/x402/bazaar-helpers.js';
import { priceFor } from '../_lib/x402-prices.js';
import { env } from '../_lib/env.js';
+import { fetchTokenMarketData } from '../_lib/market/token-market.js';
import {
mppEnabled,
looksLikeMppPayment,
@@ -93,23 +94,21 @@ export const BAZAAR = {
}),
};
+// The shared market reader fails over across Birdeye, tokens.xyz, DexScreener,
+// GeckoTerminal, DefiLlama and Raydium and keeps a stale copy, so one indexer
+// having a bad minute no longer refuses every paid call. The 24h change is the
+// one field only some rungs carry; without it the signal cannot be computed
+// honestly, and computeThreeIntel refuses before settlement as before.
export async function fetchThreeMarket(mint) {
- const r = await fetch(`https://api.dexscreener.com/latest/dex/tokens/${mint}`, {
- headers: { Accept: 'application/json' },
- signal: AbortSignal.timeout(6000),
- });
- if (!r.ok) return null;
- const data = await r.json();
- const pairs = (data.pairs || []).filter((p) => p.chainId === 'solana');
- if (!pairs.length) return null;
- pairs.sort((a, b) => (b.liquidity?.usd ?? 0) - (a.liquidity?.usd ?? 0));
- const p = pairs[0];
+ const md = await fetchTokenMarketData(mint);
+ if (!md || !(md.price_usd > 0)) return null;
+ const num = (v) => (Number.isFinite(Number(v)) ? Number(v) : null);
return {
- price_usd: parseFloat(p.priceUsd) || null,
- change_24h: p.priceChange?.h24 ?? null,
- market_cap_usd: p.marketCap ?? p.fdv ?? null,
- liquidity_usd: p.liquidity?.usd ?? null,
- volume_24h_usd: p.volume?.h24 ?? null,
+ price_usd: md.price_usd,
+ change_24h: num(md.price_change_24h),
+ market_cap_usd: num(md.market_cap),
+ liquidity_usd: num(md.liquidity),
+ volume_24h_usd: num(md.volume_24h),
};
}
diff --git a/api/x402/yield-scan.js b/api/x402/yield-scan.js
index 59a5bc8de..3b72de7a0 100644
--- a/api/x402/yield-scan.js
+++ b/api/x402/yield-scan.js
@@ -16,6 +16,8 @@ import { installAccessControl } from '../_lib/x402/access-control.js';
import { withService } from '../_lib/x402/bazaar-helpers.js';
import { priceFor } from '../_lib/x402-prices.js';
import listing from '../_lib/service-catalog/services/yield-scan.js';
+import { fetchUpstream } from '../_lib/upstream-fetch.js';
+import { cacheWrapLastGood } from '../_lib/cache.js';
const ROUTE = '/api/x402/yield-scan';
const DESCRIPTION = listing.description;
@@ -24,6 +26,11 @@ const TTL_MS = 600_000;
const APY_SPIKE_RATIO = 3; // current APY > 3× its 30-day mean → likely transient
const HIGH_SIGMA = 1; // upstream sigma above this → volatile APY
+// A paid call refuses before settlement when its upstream is down, so the buyer
+// is never charged for nothing. That is right for a cold start and wrong for a
+// blip: the shared last-good copy below serves a recent board through a short
+// outage and the pre-settle 503 fires only when there has never been data.
+const STALE_TTL_S = 15 * 60;
let _cache = null; // { pools, expiresAt }
const finite = (n) => (Number.isFinite(n) ? n : null);
@@ -31,12 +38,15 @@ const finite = (n) => (Number.isFinite(n) ? n : null);
async function loadPools() {
const now = Date.now();
if (_cache && _cache.expiresAt > now) return _cache.pools;
+ const pools = await cacheWrapLastGood('x402:yield-scan', TTL_MS / 1000, buildPools, { staleTtlSeconds: STALE_TTL_S });
+ _cache = { pools, expiresAt: now + TTL_MS };
+ return pools;
+}
- const r = await fetch('https://yields.llama.fi/pools', {
+async function buildPools() {
+ const r = await fetchUpstream('https://yields.llama.fi/pools', {
headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(15_000),
- });
- if (!r.ok) throw new Error(`llama yields ${r.status}`);
+ }, { timeoutMs: 15_000, attempts: 2, label: 'llama yields' });
const raw = await r.json();
if (!Array.isArray(raw?.data)) throw new Error('unexpected upstream shape');
@@ -59,7 +69,6 @@ async function loadPools() {
prediction: typeof p.predictions?.predictedClass === 'string' ? p.predictions.predictedClass : null,
outlier: p.outlier === true,
}));
- _cache = { pools, expiresAt: now + TTL_MS };
return pools;
}
diff --git a/public/artifact.js b/public/artifact.js
index 4963343fb..05cd1ce7b 100644
--- a/public/artifact.js
+++ b/public/artifact.js
@@ -3,10 +3,59 @@
// Usage: <div data-agent-id="YOUR_ID"></div>
// <script type="module" src="https://three.ws/artifact.js"></script>
-import * as THREE from 'https://esm.sh/[email protected]';
-import { GLTFLoader } from 'https://esm.sh/[email protected]/examples/jsm/loaders/GLTFLoader.js';
-
const BASE = 'https://three.ws/';
+
+// three.js is fetched at boot from the first reachable CDN. This file stays
+// self-contained (Claude artifacts load it cross-origin), so the mirror loop
+// is inlined here rather than imported from /load-module.js.
+const THREE_VERSION = '0.160.0';
+const MODULE_TIMEOUT_MS = 15000;
+const THREE_MIRRORS = {
+ three: [
+ `https://esm.sh/three@${THREE_VERSION}`,
+ `https://cdn.jsdelivr.net/npm/three@${THREE_VERSION}/+esm`,
+ `https://unpkg.com/three@${THREE_VERSION}?module`,
+ ],
+ gltf: [
+ `https://esm.sh/three@${THREE_VERSION}/examples/jsm/loaders/GLTFLoader.js`,
+ `https://cdn.jsdelivr.net/npm/three@${THREE_VERSION}/examples/jsm/loaders/GLTFLoader.js/+esm`,
+ `https://unpkg.com/three@${THREE_VERSION}/examples/jsm/loaders/GLTFLoader.js?module`,
+ ],
+};
+let THREE = null;
+let GLTFLoader = null;
+
+function importWithDeadline(url) {
+ let timer;
+ const deadline = new Promise((_, reject) => {
+ timer = setTimeout(() => reject(new Error(`timed out loading ${url}`)), MODULE_TIMEOUT_MS);
+ });
+ return Promise.race([import(url), deadline]).finally(() => clearTimeout(timer));
+}
+
+async function importFirst(urls) {
+ let lastErr;
+ for (const url of urls) {
+ try {
+ return await importWithDeadline(url);
+ } catch (err) {
+ lastErr = err;
+ }
+ }
+ const hosts = urls.map((u) => new URL(u).host).join(', ');
+ throw Object.assign(new Error(`3D library unreachable (${hosts})`), { code: 'module_unavailable', cause: lastErr });
+}
+
+let _threeReady = null;
+function ensureThree() {
+ if (_threeReady) return _threeReady;
+ _threeReady = (async () => {
+ THREE = await importFirst(THREE_MIRRORS.three);
+ ({ GLTFLoader } = await importFirst(THREE_MIRRORS.gltf));
+ })();
+ _threeReady.catch(() => { _threeReady = null; });
+ return _threeReady;
+}
const REDUCED = window.matchMedia('(prefers-reduced-motion: reduce)').matches;
const CSS = `
@@ -164,6 +213,16 @@ async function mount(container) {
container.classList.add('a3d-wrap');
+ try {
+ await ensureThree();
+ } catch (e) {
+ showError(
+ container,
+ `The 3D viewer could not be loaded: ${e.message}. Check the network or content security policy, or open this agent on three.ws.`,
+ );
+ return;
+ }
+
let manifestUrl;
try {
manifestUrl = await fetchAgent(agentId);
diff --git a/public/changelog.json b/public/changelog.json
index 35fe6a40c..2ebb2dafb 100644
--- a/public/changelog.json
+++ b/public/changelog.json
@@ -1,5 +1,5 @@
{
- "generated_at": "2026-08-27T05:44:44.533Z",
+ "generated_at": "2026-08-27T05:50:04.377Z",
"generated_by": "scripts/build-page-index.mjs from data/pages.json + data/changelog.json",
"site": {
"name": "three.ws",
diff --git a/public/features.json b/public/features.json
index 97c806c12..18e06f652 100644
--- a/public/features.json
+++ b/public/features.json
@@ -1,5 +1,5 @@
{
- "generated_at": "2026-08-27T05:44:44.511Z",
+ "generated_at": "2026-08-27T05:50:04.361Z",
"generated_by": "scripts/build-page-index.mjs from data/pages.json",
"site": {
"name": "three.ws",
diff --git a/public/gallery/surprise.js b/public/gallery/surprise.js
index 6114ca671..f475695fd 100644
--- a/public/gallery/surprise.js
+++ b/public/gallery/surprise.js
@@ -15,6 +15,7 @@
import { stage as stageGuestAvatar } from '../../src/guest-avatar.js';
+import { ensureModelViewerOrFallback } from '../../src/shared/model-viewer-loader.js';
const ENDPOINT = '/api/avatars/surprise';
const REVIEW_URL = '/create-review';
@@ -33,11 +34,7 @@ function hueFromSeed(seed) {
}
function ensureModelViewer() {
- if (customElements.get('model-viewer')) return;
- const s = document.createElement('script');
- s.type = 'module';
- s.src = 'https://ajax.googleapis.com/ajax/libs/model-viewer/4.0.0/model-viewer.min.js';
- document.head.appendChild(s);
+ ensureModelViewerOrFallback(document);
}
function buildModal() {
diff --git a/public/load-module.js b/public/load-module.js
new file mode 100644
index 000000000..adbd15928
--- /dev/null
+++ b/public/load-module.js
@@ -0,0 +1,194 @@
+// Resilient loader for third-party browser modules and scripts.
+//
+// Every lazy CDN import on the platform used to be a single `import(url)` on
+// one host. When that host was slow, blocked by a corporate proxy, or having an
+// outage, the caller hung forever or died with the opaque "Failed to fetch
+// dynamically imported module" and the surface went blank. This module gives
+// every such site the same three things: an ordered mirror list (the same
+// package version on esm.sh, jsdelivr and unpkg), a deadline per attempt, and a
+// typed error that names every host it tried so the UI can say something true.
+//
+// Served raw at /load-module.js so plain public/ scripts (the launch studio,
+// the paywall core, the merchant console) can import it by URL; bundled code
+// reaches the same file through src/shared/load-module.js.
+
+const DEFAULT_TIMEOUT_MS = 15_000;
+const MODULE_UNAVAILABLE = 'module_unavailable';
+
+// Parses an esm.sh / jsdelivr / unpkg module URL into { pkg, version, subpath }.
+// Returns null for anything else (a self-hosted path, an unknown CDN), in
+// which case the URL is loaded as-is with no mirrors.
+export function parseCdnModuleUrl(url) {
+ let u;
+ try {
+ u = new URL(url);
+ } catch {
+ return null;
+ }
+ let path = u.pathname;
+ let host;
+ if (u.hostname === 'esm.sh') host = 'esm';
+ else if (u.hostname === 'cdn.jsdelivr.net' && path.startsWith('/npm/')) {
+ host = 'jsdelivr';
+ path = path.slice('/npm'.length);
+ } else if (u.hostname === 'unpkg.com') host = 'unpkg';
+ else return null;
+ if (host === 'jsdelivr') path = path.replace(/\/\+esm$/, '');
+ // /@scope/[email protected]/sub/path or /[email protected]
+ const m = path.match(/^\/((?:@[^/@]+\/)?[^/@]+)@([^/]+)(\/.*)?$/);
+ if (!m) return null;
+ return { host, pkg: m[1], version: m[2], subpath: m[3] || '', search: u.search };
+}
+
+// The same package version on each of the three CDNs, original host first.
+export function moduleMirrors(url) {
+ const parsed = parseCdnModuleUrl(url);
+ if (!parsed) return [url];
+ const { pkg, version, subpath, host, search } = parsed;
+ const spec = `${pkg}@${version}${subpath}`;
+ const byHost = {
+ esm: `https://esm.sh/${spec}${host === 'esm' ? search : ''}`,
+ jsdelivr: `https://cdn.jsdelivr.net/npm/${spec}/+esm`,
+ unpkg: `https://unpkg.com/${spec}?module`,
+ };
+ const order = [host, ...['esm', 'jsdelivr', 'unpkg'].filter((h) => h !== host)];
+ return order.map((h) => byHost[h]);
+}
+
+function hostOf(url) {
+ try {
+ return new URL(url).host;
+ } catch {
+ return String(url);
+ }
+}
+
+function withDeadline(promise, ms, label) {
+ if (!(ms > 0)) return promise;
+ let timer;
+ const deadline = new Promise((_, reject) => {
+ timer = setTimeout(() => reject(new Error(`timed out after ${ms}ms loading ${label}`)), ms);
+ });
+ return Promise.race([promise, deadline]).finally(() => clearTimeout(timer));
+}
+
+function unavailableError(hosts, lastErr, what) {
+ const err = new Error(
+ `${what} could not be loaded from ${hosts.join(', ')}. Check your connection, ad blocker or content security policy, then try again.`,
+ );
+ err.code = MODULE_UNAVAILABLE;
+ err.hosts = hosts;
+ err.cause = lastErr;
+ return err;
+}
+
+const _modules = new Map();
+
+/**
+ * Dynamically import the first reachable URL from an ordered list.
+ *
+ * @param {string|string[]} specifiers One CDN URL (expanded to its mirrors) or an explicit ordered list.
+ * @param {object} [opts]
+ * @param {number} [opts.timeoutMs] Per-attempt deadline (default 15s).
+ * @param {(url: string) => Promise<any>} [opts.importer] Injectable import() for tests.
+ * @returns {Promise<any>} The module namespace. Rejects with `{ code: 'module_unavailable', hosts }`.
+ */
+export function loadModule(specifiers, opts = {}) {
+ const urls = Array.isArray(specifiers) ? specifiers.slice() : moduleMirrors(specifiers);
+ if (!urls.length) return Promise.reject(new Error('loadModule: no URLs given'));
+ const key = urls.join('\n');
+ if (_modules.has(key)) return _modules.get(key);
+ const timeoutMs = opts.timeoutMs ?? DEFAULT_TIMEOUT_MS;
+ const importer = opts.importer || ((url) => import(/* @vite-ignore */ url));
+ const attempt = (async () => {
+ let lastErr;
+ for (const url of urls) {
+ try {
+ return await withDeadline(importer(url), timeoutMs, url);
+ } catch (err) {
+ lastErr = err;
+ }
+ }
+ throw unavailableError(urls.map(hostOf), lastErr, urls[0]);
+ })();
+ _modules.set(key, attempt);
+ // A failure is not memoised: the next call (a retry button) tries the chain again.
+ attempt.catch(() => _modules.delete(key));
+ return attempt;
+}
+
+const _scripts = new Map();
+
+function appendScript(url, { type, timeoutMs, globalName }) {
+ return new Promise((resolve, reject) => {
+ const existing = Array.from(document.scripts).some((el) => el.getAttribute('src') === url);
+ if (existing && (!globalName || globalThis[globalName] !== undefined)) {
+ resolve();
+ return;
+ }
+ const s = document.createElement('script');
+ if (type) s.type = type;
+ s.src = url;
+ s.crossOrigin = 'anonymous';
+ const timer = setTimeout(() => {
+ s.remove();
+ reject(new Error(`timed out after ${timeoutMs}ms loading ${url}`));
+ }, timeoutMs);
+ s.onload = () => {
+ clearTimeout(timer);
+ if (globalName && globalThis[globalName] === undefined) {
+ s.remove();
+ reject(new Error(`${url} loaded but did not define window.${globalName}`));
+ return;
+ }
+ resolve();
+ };
+ s.onerror = () => {
+ clearTimeout(timer);
+ s.remove();
+ reject(new Error(`failed to load ${url}`));
+ };
+ document.head.appendChild(s);
+ });
+}
+
+/**
+ * Append a classic or module <script> from the first reachable mirror.
+ *
+ * @param {string[]} urls Ordered mirrors of the same script.
+ * @param {object} [opts]
+ * @param {string} [opts.globalName] Resolve only once `window[globalName]` exists.
+ * @param {string} [opts.type] e.g. 'module'.
+ * @param {number} [opts.timeoutMs] Per-mirror deadline (default 15s).
+ * @returns {Promise<any>} `window[globalName]` when given, else undefined.
+ */
+export function loadScript(urls, opts = {}) {
+ const list = (Array.isArray(urls) ? urls : [urls]).filter(Boolean);
+ if (!list.length) return Promise.reject(new Error('loadScript: no URLs given'));
+ const { globalName, type, timeoutMs = DEFAULT_TIMEOUT_MS } = opts;
+ if (globalName && globalThis[globalName] !== undefined) return Promise.resolve(globalThis[globalName]);
+ const key = list.join('\n');
+ if (_scripts.has(key)) return _scripts.get(key);
+ const attempt = (async () => {
+ let lastErr;
+ for (const url of list) {
+ try {
+ await appendScript(url, { type, timeoutMs, globalName });
+ return globalName ? globalThis[globalName] : undefined;
+ } catch (err) {
+ lastErr = err;
+ }
+ }
+ throw unavailableError(list.map(hostOf), lastErr, list[0]);
+ })();
+ _scripts.set(key, attempt);
+ attempt.catch(() => _scripts.delete(key));
+ return attempt;
+}
+
+/** True when `err` came from a mirror chain that was exhausted. */
+export function isModuleUnavailable(err) {
+ return err?.code === MODULE_UNAVAILABLE;
+}
+
+export { MODULE_UNAVAILABLE };
diff --git a/public/studio/fees-panel.js b/public/studio/fees-panel.js
index 1860538cd..508765e74 100644
--- a/public/studio/fees-panel.js
+++ b/public/studio/fees-panel.js
@@ -101,9 +101,20 @@ function detectWallet() {
return window.phantom?.solana || window.solana || window.backpack || window.solflare || null;
}
+// web3.js comes from the first reachable CDN (esm.sh, jsdelivr, unpkg) under
+// a deadline; a total miss throws a message friendlyError() passes through.
let _web3;
async function loadWeb3() {
- if (!_web3) _web3 = await import('https://esm.sh/@solana/[email protected]');
+ if (_web3) return _web3;
+ const { loadModule } = await import('../load-module.js');
+ try {
+ _web3 = await loadModule('https://esm.sh/@solana/[email protected]');
+ } catch (err) {
+ if (err?.code === 'module_unavailable') {
+ throw new Error(`The Solana component could not be loaded (${err.hosts.join(', ')} unreachable or blocked). Check your connection or ad blocker and try again.`);
+ }
+ throw err;
+ }
return _web3;
}
diff --git a/public/studio/launch-panel.js b/public/studio/launch-panel.js
index bc9960aa4..a23124424 100644
--- a/public/studio/launch-panel.js
+++ b/public/studio/launch-panel.js
@@ -12,6 +12,31 @@
import { mountFeesPanel } from './fees-panel.js';
+// Third-party modules (web3.js, qrcode) load through /load-module.js: the
+// pinned version is tried on esm.sh, then jsdelivr and unpkg, each under a
+// deadline, and a total miss throws `{ code: 'module_unavailable', hosts }`.
+const WEB3_URL = 'https://esm.sh/@solana/[email protected]';
+const QRCODE_URL = 'https://esm.sh/[email protected]';
+let _loaderPromise = null;
+function loader() {
+ if (!_loaderPromise) _loaderPromise = import('../load-module.js');
+ return _loaderPromise;
+}
+async function loadWeb3() {
+ const { loadModule } = await loader();
+ return loadModule(WEB3_URL);
+}
+async function loadQrcode() {
+ const { loadModule } = await loader();
+ const mod = await loadModule(QRCODE_URL);
+ return mod.default ?? mod;
+}
+function moduleUnavailableText(err, what) {
+ return err?.code === 'module_unavailable'
+ ? `${what} could not be loaded (${err.hosts.join(', ')} unreachable or blocked). Check your connection or ad blocker and try again.`
+ : null;
+}
+
// ── Pure validation ─────────────────────────────────────────────────────────
export function validateLaunchForm({ name, symbol, description, initialBuy } = {}) {
@@ -728,7 +753,7 @@ export function mountLaunchPanel(container, { getAvatar, getUser, getPreviewView
async function fetchBalance(addr) {
try {
- const { Connection, PublicKey } = await import('https://esm.sh/@solana/[email protected]');
+ const { Connection, PublicKey } = await loadWeb3();
const conn = new Connection(RPC_URL, 'confirmed');
s.solBalance = (await conn.getBalance(new PublicKey(addr))) / 1e9;
} catch { s.solBalance = null; }
@@ -841,9 +866,8 @@ export function mountLaunchPanel(container, { getAvatar, getUser, getPreviewView
bd.querySelector('#lp-dep-close').focus();
try {
- const mod = await import('https://esm.sh/[email protected]');
+ const QRCode = await loadQrcode();
if (closed) return;
- const QRCode = mod.default ?? mod;
const canvas = document.createElement('canvas');
await QRCode.toCanvas(canvas, payUri, {
width: 220,
@@ -896,7 +920,7 @@ export function mountLaunchPanel(container, { getAvatar, getUser, getPreviewView
async function refreshAgentBalance() {
if (!s.agentWallet?.address) return;
try {
- const { Connection, PublicKey } = await import('https://esm.sh/@solana/[email protected]');
+ const { Connection, PublicKey } = await loadWeb3();
const conn = new Connection(RPC_URL, 'confirmed');
const lamports = await conn.getBalance(new PublicKey(s.agentWallet.address));
s.agentWallet = { ...s.agentWallet, lamports, sol: lamports / 1e9 };
@@ -998,9 +1022,8 @@ export function mountLaunchPanel(container, { getAvatar, getUser, getPreviewView
addrEl.addEventListener('keydown', (e) => { if (e.key === 'Enter' || e.key === ' ') { e.preventDefault(); copyAddr(); } });
try {
- const mod = await import('https://esm.sh/[email protected]');
+ const QRCode = await loadQrcode();
if (closed) return;
- const QRCode = mod.default ?? mod;
const canvas = document.createElement('canvas');
await QRCode.toCanvas(canvas, addr, { width: 220, margin: 1, color: { dark: '#0c0c0c', light: '#ffffff' } });
if (closed) return;
@@ -1325,7 +1348,7 @@ export function mountLaunchPanel(container, { getAvatar, getUser, getPreviewView
s.resolvedAgentId = prep.agent_id;
const { VersionedTransaction, Keypair, Connection } =
- await import('https://esm.sh/@solana/[email protected]');
+ await loadWeb3();
const tx = VersionedTransaction.deserialize(
Uint8Array.from(atob(prep.tx_base64), (c) => c.charCodeAt(0)),
);
@@ -1607,7 +1630,7 @@ export function mountLaunchPanel(container, { getAvatar, getUser, getPreviewView
try {
if (!s.pendingConfirm) return;
// One more on-chain check before finalizing
- const { Connection } = await import('https://esm.sh/@solana/[email protected]');
+ const { Connection } = await loadWeb3();
const conn = new Connection(RPC_URL, 'confirmed');
const { value } = await conn.getSignatureStatuses([s.pendingConfirm.sig], { searchTransactionHistory: true });
const st = value?.[0];
diff --git a/public/x402-pay-core.js b/public/x402-pay-core.js
index 1288af66a..ae15b6884 100644
--- a/public/x402-pay-core.js
+++ b/public/x402-pay-core.js
@@ -349,9 +349,21 @@ export function buildEvmPaymentPayload({ accept, signature, authorization, resou
let _solanaWeb3 = null;
async function loadSolanaWeb3() {
if (_solanaWeb3) return _solanaWeb3;
- // Dynamic import from esm.sh keeps the paywall tiny — web3.js is only fetched
- // when a Solana payment is actually attempted. Mirrors public/x402.js.
- _solanaWeb3 = await import('https://esm.sh/@solana/[email protected]?bundle');
+ // Dynamic import keeps the paywall tiny — web3.js is only fetched when a
+ // Solana payment is actually attempted. The shared loader tries esm.sh, then
+ // jsdelivr and unpkg for the same version, each under a deadline, and throws
+ // a typed error naming the hosts when all three are blocked.
+ const { loadModule } = await import('./load-module.js');
+ try {
+ _solanaWeb3 = await loadModule('https://esm.sh/@solana/[email protected]?bundle');
+ } catch (err) {
+ if (err?.code === 'module_unavailable') {
+ throw new Error(
+ 'The Solana wallet component could not be loaded (' + err.hosts.join(', ') + ' unreachable or blocked by this page\'s security policy). Pay with a Base wallet instead; that path needs no third-party code.',
+ );
+ }
+ throw err;
+ }
return _solanaWeb3;
}
diff --git a/public/x402/studio.js b/public/x402/studio.js
index 545a6cd2f..b1ca04e4a 100644
--- a/public/x402/studio.js
+++ b/public/x402/studio.js
@@ -6,6 +6,8 @@
// endpoints: /api/x402-merchant (settings), /api/x402-skus (products),
// /api/x402/pay-by-name (USDC sends), /api/sns (name resolution). No mocks.
+// Loaded on demand through /load-module.js, which tries esm.sh, jsdelivr and
+// unpkg for this exact version under a deadline each.
const SOLANA_WEB3 = 'https://esm.sh/@solana/[email protected]?bundle';
const USDC_DECIMALS = 6;
@@ -579,7 +581,17 @@ async function sendUsdc() {
status.textContent = 'Building transfer…';
const { data } = await api('/api/x402/pay-by-name', { method: 'POST', body: { name: to.replace(/^@/, ''), amount_usdc: Number(amt), mode: 'prep', payer_wallet: publicKey.toString() } });
status.innerHTML = `Sending <b>${data.amount_usdc} USDC</b> → ${shortAddr(data.recipient?.address)}. Approve in Phantom…`;
- const web3 = await import(/* @vite-ignore */ SOLANA_WEB3);
+ let web3;
+ try {
+ const { loadModule } = await import('../load-module.js');
+ web3 = await loadModule(SOLANA_WEB3);
+ } catch (err) {
+ status.textContent = err?.code === 'module_unavailable'
+ ? `The Solana signing component could not be loaded (${err.hosts.join(', ')} unreachable or blocked). Nothing was sent. Check your connection or ad blocker and try again.`
+ : `Failed: ${err?.message || err}`;
+ btn.disabled = false;
+ return;
+ }
const tx = web3.VersionedTransaction.deserialize(b64ToBytes(data.tx_base64));
const { signature } = await provider.signAndSendTransaction(tx);
status.innerHTML = `✓ Sent. <a href="https://solscan.io/tx/${signature}" target="_blank">View on Solscan ↗</a>`;
diff --git a/src/agent-picker.js b/src/agent-picker.js
index 35493be2f..35efd40c6 100644
--- a/src/agent-picker.js
+++ b/src/agent-picker.js
@@ -26,16 +26,13 @@ import './ui-juice.css';
import { enterStagger } from './ui-juice.js';
import { onchainBadgeHTML } from './shared/onchain-badge.js';
+import { ensureModelViewerOrFallback } from './shared/model-viewer-loader.js';
const PAGE_SIZE = 24;
-let _mvLoaded = false;
+// Shared loader: three CDNs, a per-source timeout and a visible fallback on
+// every <model-viewer> in this picker when none of them can be reached.
function ensureModelViewer() {
- if (_mvLoaded || customElements.get('model-viewer')) return;
- _mvLoaded = true;
- const s = document.createElement('script');
- s.type = 'module';
- s.src = 'https://ajax.googleapis.com/ajax/libs/model-viewer/3.5.0/model-viewer.min.js';
- document.head.appendChild(s);
+ ensureModelViewerOrFallback(document);
}
function esc(s) {
diff --git a/src/artifact/entry.js b/src/artifact/entry.js
index 508fe4277..3bb8ca8ce 100644
--- a/src/artifact/entry.js
+++ b/src/artifact/entry.js
@@ -5,6 +5,7 @@
// <div id="agent3d" data-agent-id="123"></div>
import { log } from '../shared/log.js';
+import { loadModule } from '../shared/load-module.js';
(async function initArtifact() {
const container = document.getElementById('agent3d');
if (!container) return;
@@ -46,14 +47,20 @@ import { log } from '../shared/log.js';
}
try {
- // Load three.js modules from CDN
- const THREE = await import('https://esm.sh/[email protected]');
- const { GLTFLoader } = await import(
- 'https://esm.sh/[email protected]/examples/jsm/loaders/GLTFLoader.js'
- );
- const { OrbitControls } = await import(
- 'https://esm.sh/[email protected]/examples/jsm/controls/OrbitControls.js'
- );
+ // Load three.js modules from CDN. Each import races a deadline across
+ // esm.sh, jsdelivr and unpkg; a total miss lands in the caption below.
+ let THREE;
+ let GLTFLoader;
+ let OrbitControls;
+ try {
+ THREE = await loadModule('https://esm.sh/[email protected]');
+ ({ GLTFLoader } = await loadModule('https://esm.sh/[email protected]/examples/jsm/loaders/GLTFLoader.js'));
+ ({ OrbitControls } = await loadModule('https://esm.sh/[email protected]/examples/jsm/controls/OrbitControls.js'));
+ } catch (err) {
+ log.warn('[artifact] three.js unavailable', err);
+ caption.textContent = 'The 3D viewer could not be loaded: its library is blocked or unreachable here. Open this agent on three.ws to see it in 3D.';
+ return;
+ }
// Fetch agent data
const agentRes = await fetch(`${origin}/api/agents/${agentId}`, {
diff --git a/src/assistant-frame.js b/src/assistant-frame.js
index 892b9ef9f..b801c54c1 100644
--- a/src/assistant-frame.js
+++ b/src/assistant-frame.js
@@ -596,12 +596,41 @@ async function chatErrorMessage(res) {
return description || `Chat failed (HTTP ${res.status}).`;
}
+// A chat turn (request plus the whole stream) that has not finished by now is
+// abandoned, so the thinking bubble can never sit forever on a wedged upstream.
+const CHAT_DEADLINE_MS = 60_000;
+
+/**
+ * Run one chat turn under the deadline: `fn` receives an AbortSignal to pass to
+ * fetch (aborting the fetch also ends its body reader), and an abort surfaces
+ * as a readable error on the existing bubble path.
+ */
+async function withChatDeadline(fn) {
+ const ctrl = new AbortController();
+ const timer = setTimeout(() => ctrl.abort(), CHAT_DEADLINE_MS);
+ try {
+ return await fn(ctrl.signal);
+ } catch (err) {
+ if (ctrl.signal.aborted || err?.name === 'AbortError') {
+ throw new Error('The reply took longer than 60 seconds. Try again, or switch lanes under Settings.');
+ }
+ throw err;
+ } finally {
+ clearTimeout(timer);
+ }
+}
+
/** Stream a reply from /api/chat (platform free chain). */
-async function chatFree(message, onChunk) {
+function chatFree(message, onChunk) {
+ return withChatDeadline((signal) => chatFreeStream(message, onChunk, signal));
+}
+
+async function chatFreeStream(message, onChunk, signal) {
const res = await fetch('/api/chat', {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ message, history: history.slice(0, -1), system_prompt: systemPrompt() }),
+ signal,
});
if (!res.ok || !res.body) throw new Error(await chatErrorMessage(res));
const reader = res.body.getReader();
@@ -641,7 +670,11 @@ async function chatFree(message, onChunk) {
}
/** Stream a reply straight from the visitor's own provider (BYOK). */
-async function chatByok(lane, message, onChunk) {
+function chatByok(lane, message, onChunk) {
+ return withChatDeadline((signal) => chatByokStream(lane, message, onChunk, signal));
+}
+
+async function chatByokStream(lane, message, onChunk, signal) {
const key = settings.keys[lane];
const model = settings.models[lane] || BYOK_DEFAULT_MODELS[lane];
const messages = [
@@ -658,6 +691,7 @@ async function chatByok(lane, message, onChunk) {
method: 'POST',
headers,
body: JSON.stringify({ model, messages, stream: true, max_tokens: 1024 }),
+ signal,
});
if (!res.ok || !res.body) {
if (res.status === 401 || res.status === 403) {
diff --git a/src/auth/walletconnect-bridge.js b/src/auth/walletconnect-bridge.js
index a47d308e3..6f1d1ee7c 100644
--- a/src/auth/walletconnect-bridge.js
+++ b/src/auth/walletconnect-bridge.js
@@ -1,13 +1,17 @@
// WalletConnect v2 → SIWE bridge.
// Lazy-loads @walletconnect/ethereum-provider from CDN so it never ships in
-// the main bundle unless the user explicitly triggers WC sign-in.
+// the main bundle unless the user explicitly triggers WC sign-in. The load
+// goes through the shared mirror chain (esm.sh, then jsdelivr and unpkg for
+// the same version) with a deadline, so one blocked CDN never strands sign-in.
+
+import { loadModule } from '../shared/load-module.js';
const WC_CDN = 'https://esm.sh/@walletconnect/[email protected]';
let _provider = null;
async function _loadEthereumProvider() {
- const { EthereumProvider } = await import(/* @vite-ignore */ WC_CDN);
+ const { EthereumProvider } = await loadModule(WC_CDN);
return EthereumProvider;
}
@@ -58,7 +62,12 @@ export async function signInWithWalletConnect({ projectId, chains } = {}) {
try {
EthereumProvider = await _loadEthereumProvider();
} catch (e) {
- throw _wcError('load_failed', `Failed to load WalletConnect provider: ${e.message}`);
+ throw _wcError(
+ 'load_failed',
+ e.code === 'module_unavailable'
+ ? `The WalletConnect component could not be loaded (${e.hosts.join(', ')} unreachable or blocked). Check your connection or ad blocker, or sign in with a browser wallet instead.`
+ : `Failed to load WalletConnect provider: ${e.message}`,
+ );
}
// 2. Init + open modal.
diff --git a/src/avatar-gallery-picker.js b/src/avatar-gallery-picker.js
index 70c5eb016..b56ae7459 100644
--- a/src/avatar-gallery-picker.js
+++ b/src/avatar-gallery-picker.js
@@ -22,16 +22,13 @@ import { enterStagger } from './ui-juice.js';
import { onchainBadgeHTML } from './shared/onchain-badge.js';
import { walletChipHTML, wireWalletChips } from './shared/agent-wallet-chip.js';
+import { ensureModelViewerOrFallback } from './shared/model-viewer-loader.js';
const PAGE_SIZE = 24;
-let _mvLoaded = false;
+// Shared loader: three CDNs, a per-source timeout and a visible fallback on
+// every <model-viewer> in this picker when none of them can be reached.
function ensureModelViewer() {
- if (_mvLoaded || customElements.get('model-viewer')) return;
- _mvLoaded = true;
- const s = document.createElement('script');
- s.type = 'module';
- s.src = 'https://ajax.googleapis.com/ajax/libs/model-viewer/3.5.0/model-viewer.min.js';
- document.head.appendChild(s);
+ ensureModelViewerOrFallback(document);
}
let _aaLoaded = false;
diff --git a/src/credits.js b/src/credits.js
index 1320200a1..12ee6c71e 100644
--- a/src/credits.js
+++ b/src/credits.js
@@ -10,6 +10,7 @@
import { getAdapter } from './onchain/adapters/index.js';
import { resolveTokenProgramId } from './shared/spl-token-program.js';
+import { solToUsd } from './shared/usd-price.js';
const SOL_MINT = 'So11111111111111111111111111111111111111112';
@@ -224,11 +225,18 @@ async function loadPrices() {
try {
const mint = state.asset === 'SOL' ? SOL_MINT : state.deposit?.three_mint;
if (!mint || state.prices[mint] != null) return updateEstimate();
- const r = await fetch(`https://lite-api.jup.ag/price/v3?ids=${mint}`);
- if (r.ok) {
- const d = await r.json();
- const p = Number(d?.[mint]?.usdPrice ?? d?.[mint]?.price);
+ if (state.asset === 'SOL') {
+ // Shared five-feed SOL/USD chain (src/shared/usd-price.js), null when
+ // every feed is down; never a single-source or hardcoded rate.
+ const p = await solToUsd(1);
if (p > 0) state.prices[mint] = p;
+ } else {
+ const r = await fetch(`https://lite-api.jup.ag/price/v3?ids=${mint}`);
+ if (r.ok) {
+ const d = await r.json();
+ const p = Number(d?.[mint]?.usdPrice ?? d?.[mint]?.price);
+ if (p > 0) state.prices[mint] = p;
+ }
}
} catch {
/* estimate is a nicety */
diff --git a/src/dashboard/dashboard.js b/src/dashboard/dashboard.js
index d6afe88fb..ddb30ac50 100644
--- a/src/dashboard/dashboard.js
+++ b/src/dashboard/dashboard.js
@@ -5129,7 +5129,7 @@ async function openQrModal(url, title) {
const host = overlay.querySelector('#qr-host');
try {
- const mod = await import('https://esm.sh/[email protected]');
+ const mod = await import('qrcode');
const QRCode = mod.default || mod;
const canvas = document.createElement('canvas');
await QRCode.toCanvas(canvas, url, {
@@ -5140,7 +5140,9 @@ async function openQrModal(url, title) {
host.innerHTML = '';
host.appendChild(canvas);
} catch (err) {
- host.innerHTML = `<div style="color:#a33;font-size:12px">QR generation failed: ${esc(err.message || 'error')}</div>`;
+ host.innerHTML =
+ `<div style="color:#a33;font-size:12px;line-height:1.5">The QR code could not be drawn (${esc(err.message || 'error')}). ` +
+ `Use <strong>Copy link</strong> above to share this address instead.</div>`;
}
}
diff --git a/src/erc8004/chain-edit.js b/src/erc8004/chain-edit.js
index d3817dff1..0ab8f2351 100644
--- a/src/erc8004/chain-edit.js
+++ b/src/erc8004/chain-edit.js
@@ -54,7 +54,6 @@ export async function mountChainEdit(rootEl, chainId, agentId) {
let currentGlbUrl = null;
try {
- const { JsonRpcProvider } = await import('ethers');
const provider = new JsonRpcProvider(chainMeta.rpcUrl, chainId);
const registry = new Contract(deployment.identityRegistry, IDENTITY_REGISTRY_ABI, provider);
currentTokenURI = await registry.tokenURI(agentId).catch(() => null);
diff --git a/src/erc8004/chains.js b/src/erc8004/chains.js
index af6384079..2f60fcddd 100644
--- a/src/erc8004/chains.js
+++ b/src/erc8004/chains.js
@@ -14,7 +14,11 @@ import { REGISTRY_DEPLOYMENTS } from './abi.js';
* @property {string} short Short label (e.g. "BSC", "Base").
* @property {string} symbol Native token symbol.
* @property {string} explorer Block explorer base URL (no trailing slash).
- * @property {string} rpc Public RPC URL (used for read-only calls).
+ * @property {string} rpc First public RPC URL (wallet_addEthereumChain and legacy single-URL callers).
+ * @property {string[]} rpcUrls Ordered keyless public RPC hosts; the same-origin
+ * /api/evm-rpc proxy is prepended at runtime by
+ * src/shared/evm-rpc-fallback.js. Never a host the
+ * server chain demotes (eth.llamarpc.com).
* @property {'mainnet'|'testnet'} kind
*/
@@ -27,7 +31,12 @@ export const CHAIN_INFO = {
short: 'ETH',
symbol: 'ETH',
explorer: 'https://etherscan.io',
- rpc: 'https://eth.llamarpc.com',
+ rpc: 'https://ethereum-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://ethereum-rpc.publicnode.com',
+ 'https://eth.drpc.org',
+ 'https://1rpc.io/eth',
+ ],
kind: 'mainnet',
},
10: {
@@ -36,7 +45,12 @@ export const CHAIN_INFO = {
short: 'OP',
symbol: 'ETH',
explorer: 'https://optimistic.etherscan.io',
- rpc: 'https://mainnet.optimism.io',
+ rpc: 'https://optimism-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://optimism-rpc.publicnode.com',
+ 'https://mainnet.optimism.io',
+ 'https://optimism.drpc.org',
+ ],
kind: 'mainnet',
},
56: {
@@ -45,7 +59,12 @@ export const CHAIN_INFO = {
short: 'BSC',
symbol: 'BNB',
explorer: 'https://bscscan.com',
- rpc: 'https://bsc-dataseed.bnbchain.org',
+ rpc: 'https://bsc-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://bsc-rpc.publicnode.com',
+ 'https://bsc-dataseed.bnbchain.org',
+ 'https://bsc.drpc.org',
+ ],
kind: 'mainnet',
},
100: {
@@ -54,7 +73,12 @@ export const CHAIN_INFO = {
short: 'GNO',
symbol: 'xDAI',
explorer: 'https://gnosisscan.io',
- rpc: 'https://rpc.gnosischain.com',
+ rpc: 'https://gnosis-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://gnosis-rpc.publicnode.com',
+ 'https://rpc.gnosischain.com',
+ 'https://gnosis.drpc.org',
+ ],
kind: 'mainnet',
},
137: {
@@ -63,7 +87,12 @@ export const CHAIN_INFO = {
short: 'MATIC',
symbol: 'POL',
explorer: 'https://polygonscan.com',
- rpc: 'https://polygon-rpc.com',
+ rpc: 'https://polygon-bor-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://polygon-bor-rpc.publicnode.com',
+ 'https://polygon-rpc.com',
+ 'https://polygon.drpc.org',
+ ],
kind: 'mainnet',
},
250: {
@@ -72,7 +101,12 @@ export const CHAIN_INFO = {
short: 'FTM',
symbol: 'FTM',
explorer: 'https://ftmscan.com',
- rpc: 'https://rpc.ftm.tools',
+ rpc: 'https://fantom-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://fantom-rpc.publicnode.com',
+ 'https://rpc.ftm.tools',
+ 'https://fantom.drpc.org',
+ ],
kind: 'mainnet',
},
324: {
@@ -82,6 +116,11 @@ export const CHAIN_INFO = {
symbol: 'ETH',
explorer: 'https://explorer.zksync.io',
rpc: 'https://mainnet.era.zksync.io',
+ rpcUrls: [
+ 'https://mainnet.era.zksync.io',
+ 'https://zksync.drpc.org',
+ 'https://1rpc.io/zksync2-era',
+ ],
kind: 'mainnet',
},
1284: {
@@ -91,6 +130,11 @@ export const CHAIN_INFO = {
symbol: 'GLMR',
explorer: 'https://moonscan.io',
rpc: 'https://rpc.api.moonbeam.network',
+ rpcUrls: [
+ 'https://rpc.api.moonbeam.network',
+ 'https://moonbeam-rpc.publicnode.com',
+ 'https://moonbeam.drpc.org',
+ ],
kind: 'mainnet',
},
5000: {
@@ -100,6 +144,11 @@ export const CHAIN_INFO = {
symbol: 'MNT',
explorer: 'https://explorer.mantle.xyz',
rpc: 'https://rpc.mantle.xyz',
+ rpcUrls: [
+ 'https://rpc.mantle.xyz',
+ 'https://mantle-rpc.publicnode.com',
+ 'https://mantle.drpc.org',
+ ],
kind: 'mainnet',
},
8453: {
@@ -108,7 +157,12 @@ export const CHAIN_INFO = {
short: 'BASE',
symbol: 'ETH',
explorer: 'https://basescan.org',
- rpc: 'https://mainnet.base.org',
+ rpc: 'https://base-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://base-rpc.publicnode.com',
+ 'https://mainnet.base.org',
+ 'https://base.drpc.org',
+ ],
kind: 'mainnet',
},
42161: {
@@ -117,7 +171,12 @@ export const CHAIN_INFO = {
short: 'ARB',
symbol: 'ETH',
explorer: 'https://arbiscan.io',
- rpc: 'https://arb1.arbitrum.io/rpc',
+ rpc: 'https://arbitrum-one-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://arbitrum-one-rpc.publicnode.com',
+ 'https://arb1.arbitrum.io/rpc',
+ 'https://arbitrum.drpc.org',
+ ],
kind: 'mainnet',
},
42220: {
@@ -127,6 +186,11 @@ export const CHAIN_INFO = {
symbol: 'CELO',
explorer: 'https://celoscan.io',
rpc: 'https://forno.celo.org',
+ rpcUrls: [
+ 'https://forno.celo.org',
+ 'https://celo.drpc.org',
+ 'https://1rpc.io/celo',
+ ],
kind: 'mainnet',
},
43114: {
@@ -135,7 +199,12 @@ export const CHAIN_INFO = {
short: 'AVAX',
symbol: 'AVAX',
explorer: 'https://snowtrace.io',
- rpc: 'https://api.avax.network/ext/bc/C/rpc',
+ rpc: 'https://avalanche-c-chain-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://avalanche-c-chain-rpc.publicnode.com',
+ 'https://api.avax.network/ext/bc/C/rpc',
+ 'https://avalanche.drpc.org',
+ ],
kind: 'mainnet',
},
59144: {
@@ -145,6 +214,11 @@ export const CHAIN_INFO = {
symbol: 'ETH',
explorer: 'https://lineascan.build',
rpc: 'https://rpc.linea.build',
+ rpcUrls: [
+ 'https://rpc.linea.build',
+ 'https://linea-rpc.publicnode.com',
+ 'https://linea.drpc.org',
+ ],
kind: 'mainnet',
},
534352: {
@@ -154,6 +228,11 @@ export const CHAIN_INFO = {
symbol: 'ETH',
explorer: 'https://scrollscan.com',
rpc: 'https://rpc.scroll.io',
+ rpcUrls: [
+ 'https://rpc.scroll.io',
+ 'https://scroll-rpc.publicnode.com',
+ 'https://scroll.drpc.org',
+ ],
kind: 'mainnet',
},
@@ -164,7 +243,12 @@ export const CHAIN_INFO = {
short: 'tBSC',
symbol: 'tBNB',
explorer: 'https://testnet.bscscan.com',
- rpc: 'https://data-seed-prebsc-1-s1.bnbchain.org:8545',
+ rpc: 'https://bsc-testnet-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://bsc-testnet-rpc.publicnode.com',
+ 'https://data-seed-prebsc-1-s1.bnbchain.org:8545',
+ 'https://bsc-testnet.drpc.org',
+ ],
kind: 'testnet',
},
11155111: {
@@ -173,7 +257,12 @@ export const CHAIN_INFO = {
short: 'SEP',
symbol: 'ETH',
explorer: 'https://sepolia.etherscan.io',
- rpc: 'https://rpc.sepolia.org',
+ rpc: 'https://ethereum-sepolia-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://ethereum-sepolia-rpc.publicnode.com',
+ 'https://rpc.sepolia.org',
+ 'https://sepolia.drpc.org',
+ ],
kind: 'testnet',
},
84532: {
@@ -182,7 +271,12 @@ export const CHAIN_INFO = {
short: 'baseSep',
symbol: 'ETH',
explorer: 'https://sepolia.basescan.org',
- rpc: 'https://sepolia.base.org',
+ rpc: 'https://base-sepolia-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://base-sepolia-rpc.publicnode.com',
+ 'https://sepolia.base.org',
+ 'https://base-sepolia.drpc.org',
+ ],
kind: 'testnet',
},
421614: {
@@ -191,7 +285,12 @@ export const CHAIN_INFO = {
short: 'arbSep',
symbol: 'ETH',
explorer: 'https://sepolia.arbiscan.io',
- rpc: 'https://sepolia-rollup.arbitrum.io/rpc',
+ rpc: 'https://arbitrum-sepolia-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://arbitrum-sepolia-rpc.publicnode.com',
+ 'https://sepolia-rollup.arbitrum.io/rpc',
+ 'https://arbitrum-sepolia.drpc.org',
+ ],
kind: 'testnet',
},
11155420: {
@@ -200,7 +299,12 @@ export const CHAIN_INFO = {
short: 'opSep',
symbol: 'ETH',
explorer: 'https://sepolia-optimism.etherscan.io',
- rpc: 'https://sepolia.optimism.io',
+ rpc: 'https://optimism-sepolia-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://optimism-sepolia-rpc.publicnode.com',
+ 'https://sepolia.optimism.io',
+ 'https://optimism-sepolia.drpc.org',
+ ],
kind: 'testnet',
},
80002: {
@@ -209,7 +313,12 @@ export const CHAIN_INFO = {
short: 'Amoy',
symbol: 'POL',
explorer: 'https://amoy.polygonscan.com',
- rpc: 'https://rpc-amoy.polygon.technology',
+ rpc: 'https://polygon-amoy-bor-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://polygon-amoy-bor-rpc.publicnode.com',
+ 'https://rpc-amoy.polygon.technology',
+ 'https://polygon-amoy.drpc.org',
+ ],
kind: 'testnet',
},
43113: {
@@ -218,7 +327,12 @@ export const CHAIN_INFO = {
short: 'Fuji',
symbol: 'AVAX',
explorer: 'https://testnet.snowtrace.io',
- rpc: 'https://api.avax-test.network/ext/bc/C/rpc',
+ rpc: 'https://avalanche-fuji-c-chain-rpc.publicnode.com',
+ rpcUrls: [
+ 'https://avalanche-fuji-c-chain-rpc.publicnode.com',
+ 'https://api.avax-test.network/ext/bc/C/rpc',
+ 'https://avalanche-fuji.drpc.org',
+ ],
kind: 'testnet',
},
};
diff --git a/src/kol/trades.js b/src/kol/trades.js
index cee340096..5dff61ed3 100644
--- a/src/kol/trades.js
+++ b/src/kol/trades.js
@@ -25,6 +25,9 @@ const HELIUS_TX_API = 'https://api.helius.xyz/v0/addresses';
const PER_WALLET_LIMIT = 100; // enhanced txns scanned per wallet
const FETCH_TIMEOUT_MS = 8_000;
const CACHE_TTL_S = 5; // trades move fast — a few seconds of memo is plenty
+// Last-good tier: the most recent feed a healthy fan-out produced, held long
+// enough to ride out a whole Helius outage. Served marked stale, never silently.
+const LKG_TTL_S = 24 * 3600;
function sourceFromTags(tags = []) {
if (tags.includes('whale')) return 'whale';
@@ -63,7 +66,9 @@ async function fetchWalletTxns(wallet, apiKey) {
* Fetch recent KOL-wallet trades for a mint.
*
* @param {{ mint: string, limit?: number }} opts
- * @returns {Promise<{ trades: Array<object>, source: 'helius'|null }>}
+ * @returns {Promise<{ trades: Array<object>, source: 'helius'|null, stale?: true, as_of?: string }>}
+ * `stale: true` + `as_of` (ISO) mark a last-good feed served because every
+ * provider lane failed just now; the rows are real, only older.
* `source: null` means the feed is unconfigured (no provider key) — an empty
* list there is "source off", distinct from a configured source that simply
* returned no matching trades. A configured source that *fails* throws (so the
@@ -88,6 +93,7 @@ export async function fetchKolTrades({ mint, limit = 20 } = {}) {
}
const cacheKey = `kol:trades:${mint}:${cap}`;
+ const lkgKey = `kol:trades:lkg:${mint}:${cap}`;
const cached = await cacheGet(cacheKey);
if (cached) return cached;
@@ -98,6 +104,11 @@ export async function fetchKolTrades({ mint, limit = 20 } = {}) {
);
if (results.length > 0 && results.every((r) => r.status === 'rejected')) {
const reason = results[0]?.reason;
+ const lkg = await cacheGet(lkgKey);
+ if (lkg && Array.isArray(lkg.trades) && typeof lkg.at === 'number') {
+ console.warn(`[kol/trades] every Helius lane failed (${reason?.message || 'unknown'}); serving last-good for ${mint}`);
+ return { trades: lkg.trades, source: 'helius', stale: true, as_of: new Date(lkg.at).toISOString() };
+ }
throw Object.assign(
new Error(`KOL trade provider unavailable: ${reason?.message || 'unknown'}`),
{ status: 502, code: 'provider_unavailable' },
@@ -145,5 +156,6 @@ export async function fetchKolTrades({ mint, limit = 20 } = {}) {
const payload = { trades, source: 'helius' };
await cacheSet(cacheKey, payload, CACHE_TTL_S);
+ cacheSet(lkgKey, { trades, at: Date.now() }, LKG_TTL_S).catch(() => {});
return payload;
}
diff --git a/src/manifest.js b/src/manifest.js
index 5ef567d4b..0bdcf5ff8 100644
--- a/src/manifest.js
+++ b/src/manifest.js
@@ -1,10 +1,11 @@
// Manifest loader — resolves agent://, ipfs://, https:// into a normalized manifest.
// See specs/AGENT_MANIFEST.md
-import { JsonRpcProvider, Contract } from 'ethers';
+import { Contract } from 'ethers';
import { IDENTITY_REGISTRY_ABI, REGISTRY_DEPLOYMENTS } from './erc8004/abi.js';
import { findAvatar3D } from './erc8004/queries.js';
import { resolveURI, fetchWithFallback } from './ipfs.js';
+import { getEvmProvider } from './shared/evm-rpc-fallback.js';
const CHAIN_ALIASES = {
base: 8453,
@@ -14,12 +15,6 @@ const CHAIN_ALIASES = {
mainnet: 1,
};
-const DEFAULT_RPCS = {
- 8453: 'https://mainnet.base.org',
- 84532: 'https://sepolia.base.org',
- 1: 'https://eth.llamarpc.com',
-};
-
export async function loadManifest(source, { rpcURL, registry: registryOverride } = {}) {
if (!source) throw new Error('loadManifest: source required');
@@ -52,7 +47,9 @@ async function loadFromAgentURI(uri, { rpcURL, registryOverride }) {
throw new Error(`No registry deployed on chain ${chainId}. Pass registry= override.`);
}
- const provider = new JsonRpcProvider(rpcURL || DEFAULT_RPCS[chainId]);
+ // Same-origin /api/evm-rpc proxy first (server failover), then the chain's
+ // public hosts; a caller-supplied rpcURL is pinned ahead of the public tail.
+ const provider = await getEvmProvider(chainId, { primaryUrl: rpcURL || null });
const registry = new Contract(registryAddr, IDENTITY_REGISTRY_ABI, provider);
const agentId = BigInt(agentIdStr);
diff --git a/src/permissions/toolkit.js b/src/permissions/toolkit.js
index 1bc346140..3cef52601 100644
--- a/src/permissions/toolkit.js
+++ b/src/permissions/toolkit.js
@@ -11,10 +11,10 @@ import {
AbiCoder,
Contract,
getAddress,
- JsonRpcProvider,
randomBytes,
TypedDataEncoder,
} from 'ethers';
+import { getEvmProvider, isRpcTransportError } from '../shared/evm-rpc-fallback.js';
import {
CAVEAT_ENFORCERS,
DELEGATION_MANAGER_ABI,
@@ -275,28 +275,42 @@ export async function redeemDelegation({ delegation, calls, signer, chainId }) {
* Read-only check that a delegation hash is valid on-chain.
* Checks: not disabled on-chain. Pass `delegation` for expiry + signature checks.
*
+ * Three outcomes, not two: `valid: true`, `valid: false` with a reason that
+ * names what is wrong with the delegation, or `valid: null` with
+ * `reason: 'rpc_unavailable'` when the chain could not be reached at all. An
+ * RPC outage is not evidence that a permission was revoked, and a caller that
+ * treats null as false is choosing fail-closed deliberately.
+ *
* @param {{
* hash: string,
* chainId: number,
* rpcUrl?: string,
* delegation?: object,
* }} opts
- * @returns {Promise<{ valid: boolean, reason?: string }>}
+ * @returns {Promise<{ valid: boolean|null, reason?: string }>}
* @throws {PermissionError} chain_not_supported (if chain unknown)
*/
export async function isDelegationValid({ hash, chainId, rpcUrl, delegation }) {
const addr = managerAddress(chainId); // throws if unsupported
- const url =
- rpcUrl ?? (typeof process !== 'undefined' ? process.env[`RPC_URL_${chainId}`] : undefined);
- if (!url) return { valid: false, reason: 'no rpcUrl provided for on-chain check' };
+ const pinned =
+ rpcUrl ?? (typeof process !== 'undefined' ? process.env?.[`RPC_URL_${chainId}`] : undefined);
+ let disabled;
try {
- const provider = new JsonRpcProvider(url);
+ // Pinned URL first, then the same-origin proxy and the public hosts.
+ const provider = await getEvmProvider(chainId, { primaryUrl: pinned || null });
const dm = new Contract(addr, DELEGATION_MANAGER_ABI, provider);
+ disabled = await dm.disabledDelegations(hash);
+ } catch (err) {
+ if (isRpcTransportError(err) || /no rpc configured/i.test(err?.message || '')) {
+ return { valid: null, reason: 'rpc_unavailable' };
+ }
+ return { valid: false, reason: err?.message ?? String(err) };
+ }
+ if (disabled) return { valid: false, reason: 'delegation_revoked' };
- const disabled = await dm.disabledDelegations(hash);
- if (disabled) return { valid: false, reason: 'delegation_revoked' };
+ try {
// Expiry check if full delegation object available.
if (delegation?.expiry) {
diff --git a/src/pump/launch-token-modal.js b/src/pump/launch-token-modal.js
index 9a33ccdd6..30ae42908 100644
--- a/src/pump/launch-token-modal.js
+++ b/src/pump/launch-token-modal.js
@@ -12,6 +12,7 @@ import { mountLaunchBondingCurve } from './bonding-curve-chart.js';
import { log } from '../shared/log.js';
import { ensureRiskAck } from '../shared/risk-ack.js';
import { THREE_WS_MARK } from '../solana/vanity/brand.js';
+import { solToUsd } from '../shared/usd-price.js';
const _isDev =
typeof location !== 'undefined' &&
@@ -134,20 +135,17 @@ export class LaunchTokenModal {
}
async _fetchSolPrice() {
- try {
- const r = await fetch(
- 'https://api.coingecko.com/api/v3/simple/price?ids=solana&vs_currencies=usd',
- );
- if (r.ok) {
- const data = await r.json();
- this._solPriceUsd = data?.solana?.usd || null;
- // If modal is open on step 1, re-render to show price hints
- if (this._overlay && this._step === 1) {
- this._renderStep1();
- }
- }
- } catch (e) {
- log.warn('Could not fetch SOL price', e);
+ // Shared five-feed SOL/USD chain; null when every feed is down, in which
+ // case the SOL amounts render without a USD hint rather than a stale one.
+ const price = await solToUsd(1);
+ if (price == null) {
+ log.warn('Could not fetch SOL price: every feed unavailable');
+ return;
+ }
+ this._solPriceUsd = price;
+ // If modal is open on step 1, re-render to show price hints
+ if (this._overlay && this._step === 1) {
+ this._renderStep1();
}
}
diff --git a/src/pump/pumpkit-whale.js b/src/pump/pumpkit-whale.js
index c2de9eb1e..eca3fb616 100644
--- a/src/pump/pumpkit-whale.js
+++ b/src/pump/pumpkit-whale.js
@@ -6,28 +6,24 @@
//
// All heavy imports are lazy so the module cold-starts cheaply.
+import { getSolPriceUsd } from '../shared/usd-price.js';
+
const RPC_MAINNET = 'https://api.mainnet-beta.solana.com';
const PUMP_PROGRAM = '6EF8rrecthR5Dkzon8Nwu78hRvfCKubJ14M5uBEwF6P';
const LAMPORTS_PER_SOL = 1_000_000_000;
const NATIVE_SOL = 'So11111111111111111111111111111111111111112';
-const PRICE_URL = `https://api.jup.ag/price/v2?ids=${NATIVE_SOL}`;
-const PRICE_TTL_MS = 60_000;
let _cachedSolPrice = 0;
-let _priceAt = 0;
+// SOL/USD through the shared four-provider chain (Jupiter, CoinGecko, Coinbase,
+// DefiLlama). The previous single call hit Jupiter's retired price/v2 host and
+// then silently used a hard-coded 150, which made every USD threshold wrong
+// rather than merely stale. Now the last real price is reused, and with no
+// price ever seen the caller gets 0 so a threshold is skipped, not faked.
async function fetchSolPrice() {
- if (Date.now() - _priceAt < PRICE_TTL_MS && _cachedSolPrice > 0) return _cachedSolPrice;
- try {
- const r = await fetch(PRICE_URL);
- const d = await r.json();
- const p = Number(d?.data?.[NATIVE_SOL]?.price ?? 0);
- if (p > 0) {
- _cachedSolPrice = p;
- _priceAt = Date.now();
- }
- } catch {}
- return _cachedSolPrice || 150; // rough fallback if price API is down
+ const p = await getSolPriceUsd().catch(() => 0);
+ if (p > 0) _cachedSolPrice = p;
+ return _cachedSolPrice;
}
/**
diff --git a/src/reputation-ui.js b/src/reputation-ui.js
index cfc6b37c4..1294e6b66 100644
--- a/src/reputation-ui.js
+++ b/src/reputation-ui.js
@@ -6,7 +6,7 @@
* review submissions with optimistic updates.
*/
-import { JsonRpcProvider } from 'ethers';
+import { getEvmProvider } from './shared/evm-rpc-fallback.js';
import { getReputation, getRecentReviews, submitReputation, getTotalStake } from './erc8004/reputation.js';
import { ensureWallet } from './erc8004/agent-registry.js';
import { REGISTRY_DEPLOYMENTS } from './erc8004/abi.js';
@@ -103,9 +103,8 @@ export class ReputationDashboard {
throw new Error(`Chain ${this.chainId} is not supported.`);
}
- // Pick an RPC URL per chain.
- const rpcUrl = this._getRpcUrl(this.chainId);
- const provider = new JsonRpcProvider(rpcUrl);
+ // Same-origin /api/evm-rpc proxy first, then the chain's public hosts.
+ const provider = await getEvmProvider(this.chainId);
const [reputation, reviews, totalStakeWei] = await Promise.all([
getReputation({ agentId: this.agentId, runner: provider, chainId: this.chainId }),
@@ -137,34 +136,6 @@ export class ReputationDashboard {
}
}
- _getRpcUrl(chainId) {
- const rpcMap = {
- 1: 'https://eth.llamarpc.com',
- 10: 'https://optimism.llamarpc.com',
- 56: 'https://bsc.llamarpc.com',
- 100: 'https://gnosischain.publicnode.com',
- 137: 'https://polygon.llamarpc.com',
- 250: 'https://fantom.publicnode.com',
- 324: 'https://zksync.publicnode.com',
- 1284: 'https://moonbeam.publicnode.com',
- 5000: 'https://mantle.publicnode.com',
- 8453: 'https://base.llamarpc.com',
- 42161: 'https://arbitrum.llamarpc.com',
- 42220: 'https://celo.publicnode.com',
- 43114: 'https://avalanche.publicnode.com',
- 59144: 'https://linea.publicnode.com',
- 534352: 'https://scroll.publicnode.com',
- 97: 'https://bsc-testnet.publicnode.com',
- 11155111: 'https://ethereum-sepolia.publicnode.com',
- 84532: 'https://base-sepolia.publicnode.com',
- 421614: 'https://arbitrum-sepolia.publicnode.com',
- 11155420: 'https://optimism-sepolia.publicnode.com',
- 80002: 'https://polygon-amoy.publicnode.com',
- 43113: 'https://avalanche-fuji.publicnode.com',
- };
- return rpcMap[chainId] || 'https://eth.llamarpc.com';
- }
-
/**
* Submit a reputation review.
*/
@@ -229,21 +200,40 @@ export class ReputationDashboard {
async _pollForConfirmation() {
try {
- const rpcUrl = this._getRpcUrl(this.chainId);
- const provider = new JsonRpcProvider(rpcUrl);
+ const provider = await getEvmProvider(this.chainId);
const receipt = await provider.getTransactionReceipt(this.lastTxHash);
if (receipt) {
// Transaction confirmed; reload reviews.
+ this._clearPollNote();
await this._fetchReputation();
this._render();
this._stopPolling();
}
} catch (err) {
log.warn('[ReputationDashboard] poll failed:', err.message);
+ this._showPollNote();
}
}
+ // A failed confirmation poll used to vanish into the console, leaving the
+ // optimistic "pending" review looking stuck forever. Say so on the page.
+ _showPollNote() {
+ if (!this.container) return;
+ let note = this.container.querySelector('.rep-poll-note');
+ if (!note) {
+ note = document.createElement('p');
+ note.className = 'rep-poll-note';
+ note.setAttribute('role', 'status');
+ this.container.prepend(note);
+ }
+ note.textContent = 'Could not confirm your review on-chain; refresh to check.';
+ }
+
+ _clearPollNote() {
+ this.container?.querySelector('.rep-poll-note')?.remove();
+ }
+
onReviewAdded(fn) {
this.reviewListeners.push(fn);
}
diff --git a/src/robinhood-purchase.js b/src/robinhood-purchase.js
index 414052c48..6cf29a4a4 100644
--- a/src/robinhood-purchase.js
+++ b/src/robinhood-purchase.js
@@ -87,7 +87,15 @@ const ERC20_ABI = [
let publicClient;
function getPublicClient() {
- if (!publicClient) publicClient = createPublicClient({ chain: HOOD_MAINNET, transport: http() });
+ // Robinhood Chain (4663) is not in the server failover table, so there is no
+ // same-origin proxy rung for it; the vendor RPC is the chain, with a bounded
+ // per-call timeout and two retries instead of viem's open-ended defaults.
+ if (!publicClient) {
+ publicClient = createPublicClient({
+ chain: HOOD_MAINNET,
+ transport: http(undefined, { timeout: 10_000, retryCount: 2 }),
+ });
+ }
return publicClient;
}
diff --git a/src/shared/api-json.js b/src/shared/api-json.js
new file mode 100644
index 000000000..b11cfee22
--- /dev/null
+++ b/src/shared/api-json.js
@@ -0,0 +1,92 @@
+// readJson(res): the one correct way to turn an /api response into data.
+//
+// Money-path callers used to do `fetch(...).then((r) => r.json())` and then
+// check `body.error`. Two failure shapes slipped through that: a gateway 502
+// whose body is HTML ("Unexpected token <" from JSON.parse, no status in the
+// message), and a non-2xx JSON body without an `error` key, which read as
+// success and let the flow continue past a rejected prep. This helper throws a
+// typed error on either, so every caller lands in its designed error state
+// with a message a person can act on.
+
+/**
+ * @typedef {Error & { status: number, code: string, data: any }} ApiJsonError
+ */
+
+function pickMessage(data, status) {
+ if (data && typeof data === 'object') {
+ const msg = data.error_description || data.message || data.detail;
+ if (typeof msg === 'string' && msg) return msg;
+ if (typeof data.error === 'string' && data.error) return data.error;
+ if (data.error && typeof data.error === 'object' && typeof data.error.message === 'string') return data.error.message;
+ }
+ if (status === 401) return 'Sign in to continue.';
+ if (status === 403) return 'You do not have permission to do that.';
+ if (status === 404) return 'That resource was not found.';
+ if (status === 429) return 'Too many requests. Wait a moment and try again.';
+ if (status >= 500) return 'The server had a problem handling that request. Try again in a moment.';
+ return `Request failed (HTTP ${status}).`;
+}
+
+function pickCode(data, status) {
+ if (data && typeof data === 'object') {
+ if (typeof data.error === 'string' && data.error) return data.error;
+ if (typeof data.code === 'string' && data.code) return data.code;
+ if (data.error && typeof data.error === 'object' && typeof data.error.code === 'string') return data.error.code;
+ }
+ return `http_${status}`;
+}
+
+/**
+ * Parse a fetch Response as JSON, throwing a typed error for any non-2xx status
+ * or unparseable body.
+ *
+ * @param {Response} res
+ * @returns {Promise<any>}
+ * @throws {ApiJsonError} `status` (HTTP status), `code` (server `error` key or `http_<status>`,
+ * `bad_json` for a 2xx that was not JSON), `data` (the parsed body when it was JSON).
+ */
+export async function readJson(res) {
+ const status = res.status;
+ let text = '';
+ try {
+ text = await res.text();
+ } catch (err) {
+ throw Object.assign(new Error('The response could not be read. Check your connection and try again.'), {
+ status,
+ code: `http_${status}`,
+ data: null,
+ cause: err,
+ });
+ }
+ let data;
+ let parsed = false;
+ if (text) {
+ try {
+ data = JSON.parse(text);
+ parsed = true;
+ } catch {
+ parsed = false;
+ }
+ }
+ if (!res.ok) {
+ throw Object.assign(new Error(pickMessage(parsed ? data : null, status)), {
+ status,
+ code: pickCode(parsed ? data : null, status),
+ data: parsed ? data : null,
+ });
+ }
+ if (!parsed) {
+ if (!text.trim()) return null;
+ throw Object.assign(new Error('The server returned an unreadable response. Try again in a moment.'), {
+ status,
+ code: 'bad_json',
+ data: null,
+ });
+ }
+ return data;
+}
+
+/** True when `err` was thrown by readJson (or shaped like it). */
+export function isApiJsonError(err) {
+ return !!err && typeof err.status === 'number' && typeof err.code === 'string';
+}
diff --git a/src/shared/evm-rpc-fallback.js b/src/shared/evm-rpc-fallback.js
new file mode 100644
index 000000000..128fb7a8c
--- /dev/null
+++ b/src/shared/evm-rpc-fallback.js
@@ -0,0 +1,99 @@
+// Browser-side EVM RPC failover, the client half of api/_lib/evm/rpc.js.
+//
+// Browser code used to pin one keyless public host per chain (often
+// eth.llamarpc.com, which the server chain demotes as a known-bad host), so a
+// single dead endpoint read as "chain unsupported". Every consumer now gets an
+// ordered list instead:
+// 1. the same-origin proxy `/api/evm-rpc?chainId=N`, which inherits the
+// server failover (RPC_URL override, Alchemy where keyed, curated public
+// tail). Only when the page is served from a three.ws origin (or a local
+// dev server): a widget embedded on a third-party site has no proxy.
+// 2. the chain's keyless public hosts from src/erc8004/chains.js.
+// The two helpers below turn that list into an ethers v6 FallbackProvider or a
+// viem fallback transport, so a consumer makes one call. Both libraries are
+// imported lazily so a consumer only pays for the one it already uses.
+
+import { CHAIN_INFO } from '../erc8004/chains.js';
+
+const PROXY_HOSTS = /(^|\.)three\.ws$|^localhost$|^127\.0\.0\.1$|^\[::1\]$/i;
+
+/** True when the page is served from an origin that hosts /api/evm-rpc. */
+export function hasEvmProxy() {
+ if (typeof window === 'undefined' || !window.location?.origin) return false;
+ return PROXY_HOSTS.test(window.location.hostname || '');
+}
+
+/** Same-origin proxy URL for a chain, or null off a three.ws origin. */
+export function evmProxyUrl(chainId) {
+ return hasEvmProxy() ? `${window.location.origin}/api/evm-rpc?chainId=${chainId}` : null;
+}
+
+/**
+ * Ordered RPC URL list for a chain: proxy first (when available), then a
+ * caller-pinned URL, then the public hosts. De-duplicated, empty when the chain
+ * is unknown and nothing was pinned.
+ *
+ * @param {number} chainId
+ * @param {{ primaryUrl?: string|null, proxy?: boolean }} [opts]
+ * @returns {string[]}
+ */
+export function evmRpcUrls(chainId, { primaryUrl = null, proxy = true } = {}) {
+ const info = CHAIN_INFO[chainId];
+ const urls = [
+ proxy ? evmProxyUrl(chainId) : null,
+ primaryUrl,
+ ...(info?.rpcUrls || (info?.rpc ? [info.rpc] : [])),
+ ];
+ return urls.filter((u, i, a) => u && a.indexOf(u) === i);
+}
+
+/**
+ * ethers v6 read provider with quorum-1 failover across evmRpcUrls(). The first
+ * endpoint to answer wins, tried in priority order; a single-URL chain gets a
+ * plain JsonRpcProvider. Throws when the chain has no endpoint at all.
+ *
+ * @param {number} chainId
+ * @param {{ primaryUrl?: string|null, stallTimeout?: number, proxy?: boolean }} [opts]
+ * @returns {Promise<import('ethers').AbstractProvider>}
+ */
+export async function getEvmProvider(chainId, { primaryUrl = null, stallTimeout = 2000, proxy = true } = {}) {
+ const { JsonRpcProvider, FallbackProvider, Network } = await import('ethers');
+ const urls = evmRpcUrls(chainId, { primaryUrl, proxy });
+ if (urls.length === 0) throw new Error(`No RPC configured for chainId ${chainId}`);
+ const network = Number.isInteger(chainId) && chainId > 0 ? Network.from(chainId) : undefined;
+ // staticNetwork skips the per-call eth_chainId round trip on every endpoint.
+ const mk = (u) => new JsonRpcProvider(u, network, network ? { staticNetwork: network } : undefined);
+ if (urls.length === 1) return mk(urls[0]);
+ const configs = urls.map((u, i) => ({ provider: mk(u), priority: i + 1, weight: 1, stallTimeout }));
+ return new FallbackProvider(configs, network, { quorum: 1 });
+}
+
+/**
+ * viem transport with sequential failover across evmRpcUrls(), strict priority
+ * order (`rank: false`). Drop-in for `http(url)` in createPublicClient.
+ *
+ * @param {number} chainId
+ * @param {{ primaryUrl?: string|null, timeout?: number, retryCount?: number, proxy?: boolean }} [opts]
+ */
+export async function getEvmTransport(chainId, { primaryUrl = null, timeout = 10_000, retryCount = 2, proxy = true } = {}) {
+ const { fallback, http } = await import('viem');
+ const httpOpts = { timeout, retryCount };
+ const urls = evmRpcUrls(chainId, { primaryUrl, proxy });
+ if (urls.length === 0) return http(undefined, httpOpts);
+ if (urls.length === 1) return http(urls[0], httpOpts);
+ return fallback(urls.map((u) => http(u, httpOpts)), { rank: false });
+}
+
+/**
+ * Classify a provider failure: true when the error is a transport problem
+ * (network, timeout, HTTP status, every endpoint down) rather than an on-chain
+ * answer such as a revert. Lets a caller report "rpc unavailable" instead of
+ * presenting an outage as a negative result.
+ */
+export function isRpcTransportError(err) {
+ const code = err?.code;
+ if (code === 'NETWORK_ERROR' || code === 'TIMEOUT' || code === 'SERVER_ERROR') return true;
+ if (err?.name === 'AbortError' || err?.name === 'TimeoutError') return true;
+ const msg = String(err?.message || err || '').toLowerCase();
+ return /network|timeout|timed out|failed to fetch|fetch failed|econnre|enotfound|http request failed|quorum|all .* endpoints? failed|rate limit|429|5\d\d/.test(msg);
+}
diff --git a/src/shared/failover-fetch.js b/src/shared/failover-fetch.js
index 5f74eeb85..335fcbc6e 100644
--- a/src/shared/failover-fetch.js
+++ b/src/shared/failover-fetch.js
@@ -104,3 +104,44 @@ export async function fetchFirstOrNull(providers, opts = {}) {
return fallback;
}
}
+
+const RETRYABLE_STATUSES = [408, 425, 429, 500, 502, 503, 504];
+
+/**
+ * fetch with a bounded retry for a single URL: network errors and retryable
+ * statuses (429, 5xx) are retried with exponential backoff; any other response
+ * is returned as-is on the first attempt. The last retryable response is
+ * returned (not thrown) once attempts are exhausted, so callers keep their
+ * normal `res.ok` handling. Use this for an idempotent GET against one host;
+ * a POST that must run exactly once (a swap build) stays single-shot.
+ *
+ * @param {string} url
+ * @param {RequestInit} [init]
+ * @param {object} [opts]
+ * @param {number} [opts.attempts=3] Total attempts, including the first.
+ * @param {number} [opts.backoffMs=400] Base delay; doubles per retry.
+ * @param {number} [opts.timeoutMs] Per-attempt deadline (composes with init.signal).
+ * @param {number[]} [opts.retryOn] Statuses that trigger a retry.
+ * @returns {Promise<Response>}
+ */
+export async function retryFetch(url, init = {}, { attempts = 3, backoffMs = 400, timeoutMs, retryOn = RETRYABLE_STATUSES } = {}) {
+ let lastErr;
+ for (let i = 0; i < attempts; i++) {
+ if (i > 0) await new Promise((r) => setTimeout(r, backoffMs * 2 ** (i - 1)));
+ const signals = [];
+ if (init.signal) signals.push(init.signal);
+ if (timeoutMs) signals.push(AbortSignal.timeout(timeoutMs));
+ const signal = signals.length > 1 ? AbortSignal.any(signals) : signals[0];
+ try {
+ const res = await fetch(url, signal ? { ...init, signal } : init);
+ if (!retryOn.includes(res.status) || i === attempts - 1) return res;
+ lastErr = new Error(`http_${res.status}`);
+ } catch (err) {
+ // The caller gave up (stale request): stop, and never mask that as a retry.
+ if (init.signal?.aborted) throw err;
+ lastErr = err;
+ if (i === attempts - 1) throw err;
+ }
+ }
+ throw lastErr;
+}
diff --git a/src/shared/leaflet-loader.js b/src/shared/leaflet-loader.js
index e485d0a70..18595accf 100644
--- a/src/shared/leaflet-loader.js
+++ b/src/shared/leaflet-loader.js
@@ -7,24 +7,52 @@
// app never pays for it.
import { fetchFirstOrNull } from './failover-fetch.js';
+import { loadModule } from './load-module.js';
+// One pinned version, three CDNs each for the JS and the CSS. loadModule
+// expands the esm.sh URL to its jsdelivr / unpkg mirrors; the stylesheet
+// carries its own explicit list because a <link> is not an import().
const LEAFLET_JS = 'https://esm.sh/[email protected]';
-const LEAFLET_CSS = 'https://unpkg.com/[email protected]/dist/leaflet.css';
+const LEAFLET_CSS = [
+ 'https://unpkg.com/[email protected]/dist/leaflet.css',
+ 'https://cdn.jsdelivr.net/npm/[email protected]/dist/leaflet.css',
+ 'https://esm.sh/[email protected]/dist/leaflet.css',
+];
+const CSS_TIMEOUT_MS = 10_000;
-let _cssPromise = null;
-export function ensureLeafletCss() {
- if (_cssPromise) return _cssPromise;
- _cssPromise = new Promise((resolve, reject) => {
- if (document.getElementById('leaflet-css')) { resolve(); return; }
+function loadStylesheet(href) {
+ return new Promise((resolve, reject) => {
const link = document.createElement('link');
- link.id = 'leaflet-css';
link.rel = 'stylesheet';
- link.href = LEAFLET_CSS;
+ link.href = href;
link.crossOrigin = '';
- link.onload = () => resolve();
- link.onerror = () => { link.remove(); reject(new Error('leaflet css failed to load')); };
+ const timer = setTimeout(() => { link.remove(); reject(new Error(`timed out loading ${href}`)); }, CSS_TIMEOUT_MS);
+ link.onload = () => { clearTimeout(timer); resolve(link); };
+ link.onerror = () => { clearTimeout(timer); link.remove(); reject(new Error(`failed to load ${href}`)); };
document.head.appendChild(link);
- }).catch((e) => { _cssPromise = null; throw e; });
+ });
+}
+
+let _cssPromise = null;
+export function ensureLeafletCss() {
+ if (_cssPromise) return _cssPromise;
+ _cssPromise = (async () => {
+ if (document.getElementById('leaflet-css')) return;
+ let lastErr;
+ for (const href of LEAFLET_CSS) {
+ try {
+ const link = await loadStylesheet(href);
+ link.id = 'leaflet-css';
+ return;
+ } catch (e) {
+ lastErr = e;
+ }
+ }
+ throw Object.assign(new Error('The map stylesheet could not be loaded from any mirror.'), {
+ code: 'module_unavailable',
+ cause: lastErr,
+ });
+ })().catch((e) => { _cssPromise = null; throw e; });
return _cssPromise;
}
@@ -32,7 +60,7 @@ let _jsPromise = null;
export function loadLeaflet() {
if (_jsPromise) return _jsPromise;
_jsPromise = (async () => {
- const [mod] = await Promise.all([import(/* @vite-ignore */ LEAFLET_JS), ensureLeafletCss()]);
+ const [mod] = await Promise.all([loadModule(LEAFLET_JS), ensureLeafletCss()]);
const L = mod?.default || mod;
if (!L || typeof L.map !== 'function') throw new Error('leaflet module missing');
return L;
diff --git a/src/shared/load-module.js b/src/shared/load-module.js
new file mode 100644
index 000000000..61b84b83c
--- /dev/null
+++ b/src/shared/load-module.js
@@ -0,0 +1,4 @@
+// Bundled entry for the resilient CDN loader. The implementation lives in
+// public/load-module.js so plain public/ scripts can import the same file by
+// URL (/load-module.js); this re-export keeps one copy of the mirror logic.
+export * from '../../public/load-module.js';
diff --git a/src/shared/model-viewer-loader.js b/src/shared/model-viewer-loader.js
new file mode 100644
index 000000000..6fe5e74f9
--- /dev/null
+++ b/src/shared/model-viewer-loader.js
@@ -0,0 +1,111 @@
+// One loader for the <model-viewer> custom element.
+//
+// Five pickers used to inject the Google-hosted script with no `onerror`, so
+// on a network that blocks ajax.googleapis.com (corporate proxies, some
+// regions, aggressive blockers) the element never upgraded and every
+// <model-viewer> sat as an inert empty box with no message. This module tries
+// each CDN in turn, resolves once the element is defined, and rejects with a
+// user-readable error when every host fails, so a caller can show a poster
+// image or a "3D preview unavailable" state instead of nothing.
+//
+// The same version is served from three independent CDNs; any one of them is
+// the whole library, so the first that loads wins and the rest are never hit.
+
+const VERSION = '4.0.0';
+const SOURCES = [
+ `https://ajax.googleapis.com/ajax/libs/model-viewer/${VERSION}/model-viewer.min.js`,
+ `https://cdn.jsdelivr.net/npm/@google/model-viewer@${VERSION}/dist/model-viewer.min.js`,
+ `https://unpkg.com/@google/model-viewer@${VERSION}/dist/model-viewer.min.js`,
+];
+// A script that neither loads nor errors (a black-holed CDN) is treated as
+// failed after this long, so the chain moves on instead of waiting forever.
+const PER_SOURCE_TIMEOUT_MS = 12_000;
+
+let _pending = null;
+
+function loadScript(src) {
+ return new Promise((resolve, reject) => {
+ const s = document.createElement('script');
+ s.type = 'module';
+ s.src = src;
+ s.crossOrigin = 'anonymous';
+ const timer = setTimeout(() => {
+ s.remove();
+ reject(new Error(`timed out loading ${src}`));
+ }, PER_SOURCE_TIMEOUT_MS);
+ s.onload = () => {
+ clearTimeout(timer);
+ resolve();
+ };
+ s.onerror = () => {
+ clearTimeout(timer);
+ s.remove();
+ reject(new Error(`failed to load ${src}`));
+ };
+ document.head.appendChild(s);
+ });
+}
+
+/**
+ * Resolve once `customElements.get('model-viewer')` is defined. Safe to call
+ * from many components at once: one load is shared. A failure is remembered
+ * only for the duration of the attempt, so a later call (a retry button) tries
+ * the chain again.
+ *
+ * @returns {Promise<void>}
+ */
+export function ensureModelViewer() {
+ if (typeof customElements !== 'undefined' && customElements.get('model-viewer')) return Promise.resolve();
+ if (_pending) return _pending;
+ _pending = (async () => {
+ let lastErr;
+ for (const src of SOURCES) {
+ try {
+ await loadScript(src);
+ if (customElements.get('model-viewer')) return;
+ await customElements.whenDefined('model-viewer');
+ return;
+ } catch (err) {
+ lastErr = err;
+ }
+ }
+ document.documentElement.dispatchEvent(new CustomEvent('model-viewer:unavailable', { bubbles: false }));
+ throw Object.assign(new Error('The 3D viewer could not be loaded. Check your connection or ad blocker and try again.'), {
+ code: 'model_viewer_unavailable',
+ cause: lastErr,
+ });
+ })().finally(() => {
+ _pending = null;
+ });
+ return _pending;
+}
+
+/**
+ * Fire-and-forget variant for pickers that render <model-viewer> tags eagerly:
+ * on failure every viewer in `root` gets a visible fallback (its `poster`
+ * image if set, else a short caption) rather than staying blank.
+ *
+ * @param {ParentNode} [root=document]
+ */
+export function ensureModelViewerOrFallback(root = document) {
+ return ensureModelViewer().catch((err) => {
+ for (const mv of root.querySelectorAll('model-viewer')) {
+ if (mv.dataset.mvFallback) continue;
+ mv.dataset.mvFallback = '1';
+ const poster = mv.getAttribute('poster');
+ const box = document.createElement('div');
+ box.className = 'mv-fallback';
+ box.setAttribute('role', 'img');
+ box.setAttribute('aria-label', mv.getAttribute('alt') || '3D preview unavailable');
+ box.style.cssText = 'display:flex;align-items:center;justify-content:center;width:100%;height:100%;min-height:120px;font:500 12px/1.4 system-ui,sans-serif;color:#9aa4b2;text-align:center;padding:8px;box-sizing:border-box;';
+ if (poster) {
+ box.style.background = `center / cover no-repeat url("${poster}")`;
+ box.textContent = '';
+ } else {
+ box.textContent = '3D preview unavailable';
+ }
+ mv.replaceWith(box);
+ }
+ console.warn('[model-viewer]', err?.message || err);
+ });
+}
diff --git a/src/shared/template-picker.js b/src/shared/template-picker.js
index 70eb67108..8c09af83d 100644
--- a/src/shared/template-picker.js
+++ b/src/shared/template-picker.js
@@ -27,16 +27,13 @@ import { stage as stageGuestAvatar } from '../guest-avatar.js';
import { openAvatarPicker } from '../avatar-gallery-picker.js';
import { log } from './log.js';
+import { ensureModelViewerOrFallback } from './model-viewer-loader.js';
const REVIEW_URL = '/create-review';
-let _mvLoaded = false;
+// Shared loader: three CDNs, a per-source timeout and a visible fallback on
+// every <model-viewer> in this picker when none of them can be reached.
function ensureModelViewer() {
- if (_mvLoaded || customElements.get('model-viewer')) return;
- _mvLoaded = true;
- const s = document.createElement('script');
- s.type = 'module';
- s.src = 'https://ajax.googleapis.com/ajax/libs/model-viewer/4.0.0/model-viewer.min.js';
- document.head.appendChild(s);
+ ensureModelViewerOrFallback(document);
}
function esc(s) {
diff --git a/src/shared/usd-price.js b/src/shared/usd-price.js
index 490629734..ff5beea7c 100644
--- a/src/shared/usd-price.js
+++ b/src/shared/usd-price.js
@@ -2,8 +2,8 @@
* Client-side SOL/USD price helper — single source of truth for USD equivalents.
*
* USDC: treated as exactly $1 (it's a dollar-pegged stablecoin).
- * SOL: four free, CORS-enabled feeds tried in order via the shared failover-
- * fetch (Jupiter → CoinGecko → Coinbase → DefiLlama), cached 60 s. No
+ * SOL: five free, CORS-enabled feeds tried in order via the shared failover-
+ * fetch (Jupiter → CoinGecko → Coinbase → DefiLlama → Kraken), cached 60 s. No
* single feed is a point of failure — mirrors the server-side
* api/_lib/sol-price.js chain, limited here to browser-CORS-safe hosts.
*
@@ -22,8 +22,8 @@ const asPrice = (v) => {
};
// Ordered, all keyless and CORS-enabled (Access-Control-Allow-Origin: *), so
-// they work from the browser. Kraken/Bitfinex are omitted here (no CORS header)
-// though the server chain uses them.
+// they work from the browser. Bitfinex is omitted here (no CORS header) though
+// the server chain uses it; Kraken's public Ticker does send the header.
const SOL_FEEDS = [
{
name: 'jupiter',
@@ -48,6 +48,12 @@ const SOL_FEEDS = [
url: `https://coins.llama.fi/prices/current/solana:${SOL_MINT}`,
parse: async (r) => asPrice((await r.json())?.coins?.[`solana:${SOL_MINT}`]?.price),
},
+ {
+ name: 'kraken',
+ url: 'https://api.kraken.com/0/public/Ticker?pair=SOLUSD',
+ // `c` is [last trade price, lot volume].
+ parse: async (r) => asPrice((await r.json())?.result?.SOLUSD?.c?.[0]),
+ },
];
let _solPrice = 0;
diff --git a/src/solana/jupiter-swap.js b/src/solana/jupiter-swap.js
index b0f9a4d48..b752887e5 100644
--- a/src/solana/jupiter-swap.js
+++ b/src/solana/jupiter-swap.js
@@ -11,7 +11,11 @@ import { detectSolanaWallet, SOLANA_RPC } from '../erc8004/solana-deploy.js';
export const WSOL_MINT = 'So11111111111111111111111111111111111111112';
export const USDC_MINT = 'EPjFWdd5AufqSSqeM2qN1xzybapC8G4wEGGkZwyTDt1v';
-const JUPITER_API_URL = 'https://quote-api.jup.ag/v6';
+// Jupiter consolidated quote-api.jup.ag/v6 and token.jup.ag onto lite-api.jup.ag;
+// the old hosts now fail every request.
+const JUPITER_API_URL = 'https://lite-api.jup.ag/swap/v1';
+const JUPITER_TOKENS_URL = 'https://lite-api.jup.ag/tokens/v2/tag?query=verified';
+const FETCH_TIMEOUT_MS = 10_000;
const QUOTE_TTL_MS = 15_000;
let _client = null;
@@ -127,7 +131,7 @@ const TOKEN_LIST_TTL = 5 * 60_000;
export async function getJupiterTopTokens() {
if (_tokenListCache && Date.now() - _tokenListTs < TOKEN_LIST_TTL) return _tokenListCache;
- const res = await fetch('https://token.jup.ag/strict');
+ const res = await fetch(JUPITER_TOKENS_URL, { signal: AbortSignal.timeout(FETCH_TIMEOUT_MS) });
if (!res.ok) throw new Error(`Jupiter token list fetch failed: ${res.status}`);
_tokenListCache = await res.json();
_tokenListTs = Date.now();
diff --git a/src/solana/sns-subdomain.js b/src/solana/sns-subdomain.js
index 15171489d..adbcf3dcf 100644
--- a/src/solana/sns-subdomain.js
+++ b/src/solana/sns-subdomain.js
@@ -23,7 +23,14 @@ import { Keypair, PublicKey, TransactionMessage, VersionedTransaction } from '@s
import bs58 from 'bs58';
import { solanaConnection } from '../../api/_lib/solana/connection.js';
-const DEFAULT_RPC_URL = process.env.SOLANA_RPC_URL || 'https://api.mainnet-beta.solana.com';
+// Server-side: SOLANA_RPC_URL (typically Helius), which solanaConnection() below
+// rotates across the failover lanes. Browser-side: the same-origin proxy, like
+// the sibling sns.js, because public mainnet-beta 403s most browser origins.
+const DEFAULT_RPC_URL =
+ (typeof process !== 'undefined' && process.env?.SOLANA_RPC_URL) ||
+ (typeof window !== 'undefined' && window.location?.origin
+ ? `${window.location.origin}/api/solana-rpc`
+ : 'https://three.ws/api/solana-rpc');
// SNS on-chain constants — do not change without verifying against bonfida's source.
const SNS_PROGRAM_ID = new PublicKey('namesLPneVptA9Z5rqUDD9tMTWEJwofgaYwp8cawRkX');
diff --git a/src/swap-jupiter.js b/src/swap-jupiter.js
index 712db956e..a188b48a5 100644
--- a/src/swap-jupiter.js
+++ b/src/swap-jupiter.js
@@ -50,6 +50,10 @@ const SOLANA_MINT_RE = /^[1-9A-HJ-NP-Za-km-z]{32,44}$/;
const JUP_QUOTE_URL = 'https://lite-api.jup.ag/swap/v1/quote';
const JUP_SWAP_URL = 'https://lite-api.jup.ag/swap/v1/swap';
const JUP_PRICE_URL = 'https://lite-api.jup.ag/price/v2';
+// A quote or swap build that has not answered by now is treated as failed, so
+// the modal's "Quoting…" / "Building swap transaction…" states can never stick.
+const QUOTE_TIMEOUT_MS = 12_000;
+const SWAP_BUILD_TIMEOUT_MS = 20_000;
// Mainnet RPC for swap submission. We route through the same-origin proxy so
// browser-side rate limits don't blow up the flow.
@@ -454,7 +458,7 @@ async function fetchQuote() {
$('sj-confirm').textContent = 'Quoting…';
try {
- const resp = await fetch(`${JUP_QUOTE_URL}?${params}`);
+ const resp = await fetch(`${JUP_QUOTE_URL}?${params}`, { signal: AbortSignal.timeout(QUOTE_TIMEOUT_MS) });
if (!resp.ok) {
const text = await resp.text();
throw new Error(`Jupiter quote ${resp.status}: ${text.slice(0, 200)}`);
@@ -823,6 +827,7 @@ async function executeSwap() {
dynamicComputeUnitLimit: true,
prioritizationFeeLamports: { priorityLevelWithMaxLamports: { maxLamports: 1_000_000, priorityLevel: 'medium' } },
}),
+ signal: AbortSignal.timeout(SWAP_BUILD_TIMEOUT_MS),
});
if (!swapResp.ok) {
const text = await swapResp.text();
diff --git a/src/vault.js b/src/vault.js
index 4cb2e716c..09d23c9e7 100644
--- a/src/vault.js
+++ b/src/vault.js
@@ -78,7 +78,10 @@ const state = {
message: null, // { tone, text }
};
-const publicClient = createPublicClient({ chain: BSC_TESTNET, transport: http() });
+const publicClient = createPublicClient({
+ chain: BSC_TESTNET,
+ transport: http(undefined, { timeout: 10_000, retryCount: 2 }),
+});
function apiUrl(path, extra = {}) {
const u = new URL(path, location.origin);
diff --git a/src/widget-types.js b/src/widget-types.js
index 84ef67bc0..95fca7f89 100644
--- a/src/widget-types.js
+++ b/src/widget-types.js
@@ -377,20 +377,30 @@ export const CHAIN_SLUGS = {
'polygon-amoy': 80002,
};
-export const PUBLIC_RPCS = {
- 1: 'https://eth.llamarpc.com',
- 10: 'https://mainnet.optimism.io',
- 56: 'https://bsc-dataseed.binance.org',
- 137: 'https://polygon-rpc.com',
- 8453: 'https://mainnet.base.org',
- 42161: 'https://arb1.arbitrum.io/rpc',
- 11155111: 'https://ethereum-sepolia-rpc.publicnode.com',
- 84532: 'https://sepolia.base.org',
- 11155420: 'https://sepolia.optimism.io',
- 421614: 'https://sepolia-rollup.arbitrum.io/rpc',
- 80002: 'https://rpc-amoy.polygon.technology',
+// Ordered keyless public RPC hosts per chain. Widgets run embedded on
+// third-party origins, so they cannot reach the same-origin /api/evm-rpc proxy
+// that three.ws pages use (src/shared/evm-rpc-fallback.js prepends it there);
+// this public tail is the whole chain for an embed. Never eth.llamarpc.com,
+// which bot-walls keyless POSTs.
+export const PUBLIC_RPC_URLS = {
+ 1: ['https://ethereum-rpc.publicnode.com', 'https://eth.drpc.org', 'https://1rpc.io/eth'],
+ 10: ['https://optimism-rpc.publicnode.com', 'https://mainnet.optimism.io', 'https://optimism.drpc.org'],
+ 56: ['https://bsc-rpc.publicnode.com', 'https://bsc-dataseed.bnbchain.org', 'https://bsc.drpc.org'],
+ 137: ['https://polygon-bor-rpc.publicnode.com', 'https://polygon-rpc.com', 'https://polygon.drpc.org'],
+ 8453: ['https://base-rpc.publicnode.com', 'https://mainnet.base.org', 'https://base.drpc.org'],
+ 42161: ['https://arbitrum-one-rpc.publicnode.com', 'https://arb1.arbitrum.io/rpc', 'https://arbitrum.drpc.org'],
+ 11155111: ['https://ethereum-sepolia-rpc.publicnode.com', 'https://rpc.sepolia.org', 'https://sepolia.drpc.org'],
+ 84532: ['https://base-sepolia-rpc.publicnode.com', 'https://sepolia.base.org', 'https://base-sepolia.drpc.org'],
+ 11155420: ['https://optimism-sepolia-rpc.publicnode.com', 'https://sepolia.optimism.io', 'https://optimism-sepolia.drpc.org'],
+ 421614: ['https://arbitrum-sepolia-rpc.publicnode.com', 'https://sepolia-rollup.arbitrum.io/rpc', 'https://arbitrum-sepolia.drpc.org'],
+ 80002: ['https://polygon-amoy-bor-rpc.publicnode.com', 'https://rpc-amoy.polygon.technology', 'https://polygon-amoy.drpc.org'],
};
+// First host of each list, for callers that still take a single URL.
+export const PUBLIC_RPCS = Object.fromEntries(
+ Object.entries(PUBLIC_RPC_URLS).map(([id, urls]) => [id, urls[0]]),
+);
+
export const EXPLORERS = {
1: 'https://etherscan.io',
10: 'https://optimistic.etherscan.io',
diff --git a/src/widgets/bonding-curve.js b/src/widgets/bonding-curve.js
index 6bf628b3b..a2b13d470 100644
--- a/src/widgets/bonding-curve.js
+++ b/src/widgets/bonding-curve.js
@@ -24,6 +24,7 @@
*/
import { hasThreeWsMark } from '../solana/vanity/brand.js';
+import { solToUsd } from '../shared/usd-price.js';
const LAMPORTS_PER_SOL = 1_000_000_000;
const SOL_MINT = 'So11111111111111111111111111111111111111112';
@@ -474,30 +475,11 @@ function injectStyles(doc) {
// SOL/USD price — fetched once, refreshed lazily, shared across mounts.
// ---------------------------------------------------------------------------
-let _solUsd = { value: null, at: 0, inflight: null };
-const SOL_USD_TTL = 60_000;
-
+// Shared, cached SOL/USD read through the five-feed chain in
+// src/shared/usd-price.js (Jupiter, CoinGecko, Coinbase, DefiLlama, Kraken).
+// Resolves null when every feed is down; the caller falls back to SOL-only.
export async function getSolUsd() {
- const now = Date.now();
- if (_solUsd.value != null && now - _solUsd.at < SOL_USD_TTL) return _solUsd.value;
- if (_solUsd.inflight) return _solUsd.inflight;
- _solUsd.inflight = (async () => {
- try {
- const r = await fetch(`https://lite-api.jup.ag/price/v3?ids=${SOL_MINT}`);
- if (!r.ok) throw new Error(`HTTP ${r.status}`);
- const d = await r.json();
- const usd = Number(d?.[SOL_MINT]?.usdPrice ?? d?.[SOL_MINT]?.price);
- if (Number.isFinite(usd) && usd > 0) {
- _solUsd = { value: usd, at: Date.now(), inflight: null };
- return usd;
- }
- } catch {
- /* graceful — caller falls back to SOL-only display */
- }
- _solUsd.inflight = null;
- return _solUsd.value;
- })();
- return _solUsd.inflight;
+ return solToUsd(1);
}
const _easeOutCubic = (t) => 1 - Math.pow(1 - t, 3);
diff --git a/tests/api-json.test.js b/tests/api-json.test.js
new file mode 100644
index 000000000..f6e035f12
--- /dev/null
+++ b/tests/api-json.test.js
@@ -0,0 +1,42 @@
+import { describe, it, expect } from 'vitest';
+import { readJson, isApiJsonError } from '../src/shared/api-json.js';
+
+function res(status, body, ok = status >= 200 && status < 300) {
+ return new Response(body, { status, statusText: ok ? 'OK' : 'ERR' });
+}
+
+describe('readJson', () => {
+ it('returns parsed JSON on 2xx', async () => {
+ await expect(readJson(res(200, JSON.stringify({ data: { x: 1 } })))).resolves.toEqual({ data: { x: 1 } });
+ });
+ it('returns null on an empty 2xx body', async () => {
+ await expect(readJson(res(204, null))).resolves.toBeNull();
+ });
+ it('throws http_<status> with a readable message on a gateway HTML 502', async () => {
+ const err = await readJson(res(502, '<html><body>Bad Gateway</body></html>')).catch((e) => e);
+ expect(err.status).toBe(502);
+ expect(err.code).toBe('http_502');
+ expect(err.message).not.toMatch(/Unexpected token/);
+ expect(err.message).toMatch(/server had a problem/i);
+ expect(isApiJsonError(err)).toBe(true);
+ });
+ it('surfaces the server error key and description on a JSON 4xx', async () => {
+ const err = await readJson(
+ res(400, JSON.stringify({ error: 'insufficient_funds', error_description: 'Top up 0.02 SOL first.' })),
+ ).catch((e) => e);
+ expect(err.status).toBe(400);
+ expect(err.code).toBe('insufficient_funds');
+ expect(err.message).toBe('Top up 0.02 SOL first.');
+ expect(err.data.error).toBe('insufficient_funds');
+ });
+ it('does not treat a non-2xx JSON body without an error key as success', async () => {
+ const err = await readJson(res(500, JSON.stringify({ ok: false }))).catch((e) => e);
+ expect(err.status).toBe(500);
+ expect(err.code).toBe('http_500');
+ });
+ it('throws bad_json on a 2xx body that is not JSON', async () => {
+ const err = await readJson(res(200, 'not json')).catch((e) => e);
+ expect(err.code).toBe('bad_json');
+ expect(err.status).toBe(200);
+ });
+});
diff --git a/tests/evm-rpc-proxy.test.js b/tests/evm-rpc-proxy.test.js
new file mode 100644
index 000000000..7baa1542d
--- /dev/null
+++ b/tests/evm-rpc-proxy.test.js
@@ -0,0 +1,147 @@
+// /api/evm-rpc: the browser's same-origin EVM JSON-RPC proxy. Locks down the
+// read-only method allowlist and the endpoint rotation, so a keyless host that
+// answers 403 (or a 200 carrying a non-JSON-RPC body) never reaches the browser.
+
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+vi.mock('../api/_lib/http.js', () => ({
+ wrap: (fn) => fn,
+ cors: () => false,
+ method: () => true,
+ readJson: async (req) => req.body,
+ rateLimited: (res) => {
+ res._json = { status: 429, body: { error: 'rate_limited' } };
+ return res;
+ },
+ error: (res, status, code, message) => {
+ res._json = { status, body: { error: code, error_description: message } };
+ return res;
+ },
+}));
+vi.mock('../api/_lib/rate-limit.js', () => ({
+ limits: {
+ evmRpcIp: vi.fn(async () => ({ success: true })),
+ evmRpcGlobal: vi.fn(async () => ({ success: true })),
+ },
+ clientIp: () => '1.2.3.4',
+}));
+const endpoints = ['https://a.test/', 'https://b.test/', 'https://c.test/'];
+vi.mock('../api/_lib/evm/rpc.js', () => ({
+ evmRpcEndpoints: vi.fn(() => endpoints),
+}));
+
+import handler, { rejectReason, forwardWithRotation, ALLOWED_METHODS } from '../api/evm-rpc.js';
+
+const OK = { jsonrpc: '2.0', id: 1, result: '0x1' };
+const jsonRes = (body, status = 200) =>
+ new Response(typeof body === 'string' ? body : JSON.stringify(body), { status });
+
+function makeRes() {
+ const headers = {};
+ return {
+ statusCode: 0,
+ setHeader: (k, v) => { headers[k] = v; },
+ end(text) { this.text = text; },
+ headers,
+ };
+}
+
+describe('/api/evm-rpc method allowlist', () => {
+ it('accepts every read-only method and refuses everything else', () => {
+ for (const m of ALLOWED_METHODS) {
+ expect(rejectReason({ jsonrpc: '2.0', id: 1, method: m, params: [] })).toBeNull();
+ }
+ expect(rejectReason({ method: 'eth_sendRawTransaction', params: ['0x'] })).toBe('method_not_allowed');
+ expect(rejectReason({ method: 'eth_subscribe' })).toBe('method_not_allowed');
+ expect(rejectReason({ method: 'eth_newFilter' })).toBe('method_not_allowed');
+ });
+
+ it('refuses malformed, empty and oversized batches', () => {
+ expect(rejectReason([])).toBe('empty_request');
+ expect(rejectReason('nope')).toBe('malformed_request');
+ expect(rejectReason({ id: 1 })).toBe('malformed_request');
+ expect(rejectReason(Array.from({ length: 11 }, () => ({ method: 'eth_chainId' })))).toBe('batch_too_large');
+ expect(rejectReason([{ method: 'eth_chainId' }, { method: 'eth_sendRawTransaction' }])).toBe('method_not_allowed');
+ });
+
+ it('answers 403 for a write method and 400 for an unknown chain through the handler', async () => {
+ let res = makeRes();
+ await handler({ url: '/api/evm-rpc?chainId=1', body: { method: 'eth_sendRawTransaction' } }, res);
+ expect(res._json.status).toBe(403);
+ expect(res._json.body.error).toBe('method_not_allowed');
+
+ res = makeRes();
+ await handler({ url: '/api/evm-rpc?chainId=999999', body: { method: 'eth_chainId' } }, res);
+ expect(res._json.status).toBe(400);
+ expect(res._json.body.error).toBe('unknown_chain');
+ });
+});
+
+describe('/api/evm-rpc endpoint rotation', () => {
+ let calls;
+ beforeEach(() => { calls = []; });
+
+ it('rotates past a 403, a non-JSON-RPC 200 body and a network error to the first good host', async () => {
+ const fetchImpl = vi.fn(async (url) => {
+ calls.push(url);
+ if (url === 'https://a.test/') return jsonRes('forbidden', 403);
+ if (url === 'https://b.test/') return jsonRes({ error: 'Unauthorized: You must authenticate your request with an API key' });
+ return jsonRes(OK);
+ });
+ const out = await forwardWithRotation(endpoints, { jsonrpc: '2.0', id: 1, method: 'eth_chainId' }, { fetchImpl });
+ expect(out.url).toBe('https://c.test/');
+ expect(JSON.parse(out.text)).toEqual(OK);
+ expect(calls).toEqual(endpoints);
+ });
+
+ it('rotates past a timeout', async () => {
+ const fetchImpl = vi.fn((url, init) => {
+ calls.push(url);
+ if (url === 'https://a.test/') {
+ return new Promise((_, reject) => {
+ init.signal.addEventListener('abort', () => reject(new Error('aborted')));
+ });
+ }
+ return Promise.resolve(jsonRes(OK));
+ });
+ const out = await forwardWithRotation(endpoints, { method: 'eth_blockNumber' }, { fetchImpl, timeoutMs: 20 });
+ expect(out.url).toBe('https://b.test/');
+ expect(calls).toEqual(['https://a.test/', 'https://b.test/']);
+ });
+
+ it('returns a legitimate JSON-RPC error envelope without rotating (a revert is an answer, not an outage)', async () => {
+ const revert = { jsonrpc: '2.0', id: 1, error: { code: 3, message: 'execution reverted' } };
+ const fetchImpl = vi.fn(async () => jsonRes(revert));
+ const out = await forwardWithRotation(endpoints, { method: 'eth_call' }, { fetchImpl });
+ expect(out.url).toBe('https://a.test/');
+ expect(fetchImpl).toHaveBeenCalledTimes(1);
+ });
+
+ it('answers 502 upstream_error once every endpoint has failed', async () => {
+ const orig = global.fetch;
+ global.fetch = vi.fn(async () => jsonRes('nope', 500));
+ try {
+ const res = makeRes();
+ await handler({ url: '/api/evm-rpc?chainId=8453', body: { jsonrpc: '2.0', id: 1, method: 'eth_chainId' } }, res);
+ expect(res._json.status).toBe(502);
+ expect(res._json.body.error).toBe('upstream_error');
+ expect(global.fetch).toHaveBeenCalledTimes(endpoints.length);
+ } finally {
+ global.fetch = orig;
+ }
+ });
+
+ it('streams the first good upstream body through with no-store caching', async () => {
+ const orig = global.fetch;
+ global.fetch = vi.fn(async () => jsonRes(OK));
+ try {
+ const res = makeRes();
+ await handler({ url: '/api/evm-rpc?chainId=1', body: { jsonrpc: '2.0', id: 1, method: 'eth_chainId' } }, res);
+ expect(res.statusCode).toBe(200);
+ expect(res.headers['cache-control']).toBe('no-store');
+ expect(JSON.parse(res.text)).toEqual(OK);
+ } finally {
+ global.fetch = orig;
+ }
+ });
+});
diff --git a/tests/load-module.test.js b/tests/load-module.test.js
new file mode 100644
index 000000000..4b8feedfe
--- /dev/null
+++ b/tests/load-module.test.js
@@ -0,0 +1,97 @@
+// @vitest-environment jsdom
+import { describe, it, expect, vi } from 'vitest';
+import { loadModule, loadScript, moduleMirrors, parseCdnModuleUrl } from '../src/shared/load-module.js';
+
+describe('moduleMirrors', () => {
+ it('expands an esm.sh URL to jsdelivr and unpkg equivalents, original first', () => {
+ expect(moduleMirrors('https://esm.sh/[email protected]')).toEqual([
+ 'https://esm.sh/[email protected]',
+ 'https://cdn.jsdelivr.net/npm/[email protected]/+esm',
+ 'https://unpkg.com/[email protected]?module',
+ ]);
+ });
+ it('keeps scoped packages, subpaths and the esm.sh query', () => {
+ expect(moduleMirrors('https://esm.sh/@solana/[email protected]?bundle')).toEqual([
+ 'https://esm.sh/@solana/[email protected]?bundle',
+ 'https://cdn.jsdelivr.net/npm/@solana/[email protected]/+esm',
+ 'https://unpkg.com/@solana/[email protected]?module',
+ ]);
+ expect(moduleMirrors('https://esm.sh/[email protected]/examples/jsm/loaders/GLTFLoader.js')[1]).toBe(
+ 'https://cdn.jsdelivr.net/npm/[email protected]/examples/jsm/loaders/GLTFLoader.js/+esm',
+ );
+ });
+ it('starts from jsdelivr when the original is jsdelivr', () => {
+ const list = moduleMirrors('https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm');
+ expect(list[0]).toBe('https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/+esm');
+ expect(list).toContain('https://esm.sh/@mediapipe/[email protected]');
+ expect(list).toContain('https://unpkg.com/@mediapipe/[email protected]?module');
+ });
+ it('leaves non-CDN URLs alone', () => {
+ expect(moduleMirrors('/three/draco/gltf/decoder.js')).toEqual(['/three/draco/gltf/decoder.js']);
+ expect(parseCdnModuleUrl('https://example.com/x.js')).toBeNull();
+ });
+});
+
+describe('loadModule', () => {
+ it('returns the first importer that succeeds and skips failed mirrors', async () => {
+ const importer = vi.fn(async (url) => {
+ if (url.includes('esm.sh')) throw new TypeError('Failed to fetch dynamically imported module');
+ return { default: `ok:${url}` };
+ });
+ const mod = await loadModule('https://esm.sh/[email protected]', { importer });
+ expect(mod.default).toBe('ok:https://cdn.jsdelivr.net/npm/[email protected]/+esm');
+ expect(importer).toHaveBeenCalledTimes(2);
+ });
+ it('memoises per list so a second call does not re-import', async () => {
+ const importer = vi.fn(async () => ({ v: 1 }));
+ const a = loadModule(['https://esm.sh/[email protected]'], { importer });
+ const b = loadModule(['https://esm.sh/[email protected]'], { importer });
+ expect(a).toBe(b);
+ await a;
+ expect(importer).toHaveBeenCalledTimes(1);
+ });
+ it('rejects with a typed error naming every host when all mirrors fail', async () => {
+ const importer = vi.fn(async () => { throw new Error('nope'); });
+ const err = await loadModule('https://esm.sh/[email protected]', { importer }).catch((e) => e);
+ expect(err.code).toBe('module_unavailable');
+ expect(err.hosts).toEqual(['esm.sh', 'cdn.jsdelivr.net', 'unpkg.com']);
+ expect(err.message).toMatch(/esm\.sh, cdn\.jsdelivr\.net, unpkg\.com/);
+ // A failed chain is not memoised: a retry tries again.
+ const importer2 = vi.fn(async () => ({ ok: true }));
+ await expect(loadModule('https://esm.sh/[email protected]', { importer: importer2 })).resolves.toEqual({ ok: true });
+ });
+ it('treats a hung import as failed after the deadline and moves on', async () => {
+ vi.useFakeTimers();
+ const importer = vi.fn((url) => (url.includes('esm.sh') ? new Promise(() => {}) : Promise.resolve({ fast: url })));
+ const p = loadModule('https://esm.sh/[email protected]', { importer, timeoutMs: 50 });
+ await vi.advanceTimersByTimeAsync(60);
+ await expect(p).resolves.toEqual({ fast: 'https://cdn.jsdelivr.net/npm/[email protected]/+esm' });
+ vi.useRealTimers();
+ });
+});
+
+describe('loadScript', () => {
+ it('falls through to the next mirror on error and resolves the global', async () => {
+ const p = loadScript(['https://a.example/x.js', 'https://b.example/x.js'], { globalName: 'TestGlobalX' });
+ await Promise.resolve();
+ const first = document.querySelector('script[src="https://a.example/x.js"]');
+ expect(first).not.toBeNull();
+ first.onerror();
+ await Promise.resolve();
+ await Promise.resolve();
+ const second = document.querySelector('script[src="https://b.example/x.js"]');
+ expect(second).not.toBeNull();
+ globalThis.TestGlobalX = { ready: true };
+ second.onload();
+ await expect(p).resolves.toEqual({ ready: true });
+ delete globalThis.TestGlobalX;
+ });
+ it('rejects with the hosts tried when every mirror errors', async () => {
+ const p = loadScript(['https://c.example/y.js']);
+ await Promise.resolve();
+ document.querySelector('script[src="https://c.example/y.js"]').onerror();
+ const err = await p.catch((e) => e);
+ expect(err.code).toBe('module_unavailable');
+ expect(err.hosts).toEqual(['c.example']);
+ });
+});
diff --git a/tests/money-rpc-resilience.test.js b/tests/money-rpc-resilience.test.js
new file mode 100644
index 000000000..525ded0c9
--- /dev/null
+++ b/tests/money-rpc-resilience.test.js
@@ -0,0 +1,238 @@
+// Single-point-of-failure removal on the money and RPC paths (2026-08-27 audit).
+//
+// Pins, per module, the one behaviour that used to be missing:
+// - token/jupiter.js: a quote retries through a 429/5xx; a swap build retries
+// ONCE on a 5xx/network failure and never on a 4xx; deadlines exist.
+// - token/price.js: DefiLlama answers when Jupiter and the market chain miss.
+// - x402-spending-price.js: the spend-cap oracle walks the shared price chain
+// and serves last-known-good on a total outage; only a never-priced symbol
+// throws.
+// - coin/randomness.js: a dead drand relay rolls to the next one; a 404 is a
+// real answer and stops the chain.
+// - gcp-auth.js: a failed refresh keeps serving a token that is still valid.
+//
+// No live network: global.fetch is stubbed per case.
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { sha256 } from '@noble/hashes/sha256';
+
+vi.mock('../api/_lib/cache.js', async (importOriginal) => {
+ const real = await importOriginal();
+ return real;
+});
+vi.mock('../api/_lib/db.js', () => ({ sql: vi.fn(async () => []) }));
+
+import { jupiterQuote, jupiterSwapTx } from '../api/_lib/token/jupiter.js';
+import { getTokenPriceUsd } from '../api/_lib/token/price.js';
+import { TOKEN_MINT } from '../api/_lib/token/config.js';
+import { toMicroUsd } from '../api/_lib/x402-spending-price.js';
+import { fetchDrandRound, DRAND_RELAYS, QUICKNET_CHAIN_HASH } from '../api/_lib/coin/randomness.js';
+import { getGcpAccessToken, _resetGcpTokenCache } from '../api/_lib/gcp-auth.js';
+import { _resetBreakers } from '../api/_lib/resilience.js';
+import { __resetMarketCache } from '../api/_lib/market/token-market.js';
+import { cacheDel } from '../api/_lib/cache.js';
+
+const json = (body, status = 200) =>
+ new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
+
+const realFetch = global.fetch;
+beforeEach(() => {
+ _resetBreakers();
+ __resetMarketCache();
+ vi.spyOn(console, 'warn').mockImplementation(() => {});
+});
+afterEach(() => {
+ global.fetch = realFetch;
+ vi.restoreAllMocks();
+});
+
+describe('jupiter quote + swap build', () => {
+ const QUOTE = { inputMint: 'a', outputMint: 'b', amount: 1000, slippageBps: 50 };
+
+ it('retries a quote through a transient 503 and returns the route', async () => {
+ let n = 0;
+ global.fetch = vi.fn(async () => (++n === 1 ? json({ error: 'down' }, 503) : json({ outAmount: '42' })));
+ const q = await jupiterQuote(QUOTE);
+ expect(q.outAmount).toBe('42');
+ expect(global.fetch).toHaveBeenCalledTimes(2);
+ });
+
+ it('gives every attempt a deadline signal', async () => {
+ global.fetch = vi.fn(async (url, init) => {
+ expect(init.signal).toBeInstanceOf(AbortSignal);
+ return json({ outAmount: '1' });
+ });
+ await jupiterQuote(QUOTE);
+ });
+
+ it('surfaces a 400 on the quote at once with the jupiter_error shape', async () => {
+ global.fetch = vi.fn(async () => json({ error: 'no route' }, 400));
+ await expect(jupiterQuote(QUOTE)).rejects.toMatchObject({ code: 'jupiter_error', status: 400 });
+ expect(global.fetch).toHaveBeenCalledTimes(1);
+ });
+
+ it('retries the swap build exactly once on a network failure', async () => {
+ let n = 0;
+ global.fetch = vi.fn(async () => {
+ if (++n === 1) throw Object.assign(new TypeError('fetch failed'), { code: 'ECONNRESET' });
+ return json({ swapTransaction: 'AQID' });
+ });
+ const tx = await jupiterSwapTx({ quote: { outAmount: '1' }, userPublicKey: 'pk' });
+ expect(tx).toBe('AQID');
+ expect(global.fetch).toHaveBeenCalledTimes(2);
+ });
+
+ it('retries the swap build once on a 5xx, then gives up with the jupiter_error shape', async () => {
+ global.fetch = vi.fn(async () => json({ error: 'boom' }, 502));
+ await expect(jupiterSwapTx({ quote: { outAmount: '1' }, userPublicKey: 'pk' }))
+ .rejects.toMatchObject({ code: 'jupiter_error', status: 502 });
+ expect(global.fetch).toHaveBeenCalledTimes(2);
+ });
+
+ it('never retries a swap build that was answered with a 4xx or a 429', async () => {
+ global.fetch = vi.fn(async () => json({ error: 'slippage' }, 422));
+ await expect(jupiterSwapTx({ quote: { outAmount: '1' }, userPublicKey: 'pk' }))
+ .rejects.toMatchObject({ code: 'jupiter_error', status: 422 });
+ expect(global.fetch).toHaveBeenCalledTimes(1);
+
+ global.fetch = vi.fn(async () => json({ error: 'slow down' }, 429));
+ await expect(jupiterSwapTx({ quote: { outAmount: '1' }, userPublicKey: 'pk' }))
+ .rejects.toMatchObject({ code: 'jupiter_error', status: 429 });
+ expect(global.fetch).toHaveBeenCalledTimes(1);
+ });
+});
+
+describe('token price: DefiLlama rung', () => {
+ it('prices from DefiLlama when Jupiter and the rest of the market chain miss', async () => {
+ const key = `solana:${TOKEN_MINT}`;
+ global.fetch = vi.fn(async (url) => {
+ const u = String(url);
+ if (u.includes('coins.llama.fi')) return json({ coins: { [key]: { price: 0.00031 } } });
+ return json({ error: 'down' }, 503);
+ });
+ const p = await getTokenPriceUsd({ fresh: true });
+ expect(p.priceUsd).toBe(0.00031);
+ expect(p.source).toBe('llama');
+ const llamaCalls = global.fetch.mock.calls.filter(([u]) => String(u).includes('coins.llama.fi'));
+ expect(llamaCalls[0][0]).toBe(`https://coins.llama.fi/prices/current/${key}`);
+ });
+});
+
+describe('x402 spending price oracle', () => {
+ const solRequirement = { asset: 'So11111111111111111111111111111111111111112', extra: { name: 'SOL', decimals: 9 } };
+
+ beforeEach(async () => {
+ const { __resetLocalCache } = await import('../api/_lib/x402-spending-price.js');
+ __resetLocalCache();
+ await cacheDel('spot-usd:SOL');
+ await cacheDel('spot-usd:SOL:lkg');
+ await cacheDel('spot-usd:BNB');
+ await cacheDel('spot-usd:BNB:lkg');
+ });
+
+ it('falls through the shared chain when CoinGecko is down and prices the spend', async () => {
+ global.fetch = vi.fn(async (url) => {
+ const u = String(url);
+ if (u.includes('coins.llama.fi')) return json({ coins: { 'coingecko:solana': { price: 150 } } });
+ return json({}, 503);
+ });
+ // 1 SOL at $150 = 150_000_000 micro-USD.
+ const micro = await toMicroUsd('1000000000', solRequirement);
+ expect(micro).toBe(150_000_000n);
+ expect(global.fetch.mock.calls.some(([u]) => String(u).includes('coins.llama.fi'))).toBe(true);
+ });
+
+ it('serves the last-known-good price when every source is down', async () => {
+ // Warm the last-good tier through a real load first (local micro-cache
+ // cold), then fail everything and read again.
+ global.fetch = vi.fn(async (url) => (String(url).includes('coins.llama.fi')
+ ? json({ coins: { 'coingecko:solana': { price: 150 } } })
+ : json({}, 500)));
+ const first = await toMicroUsd('1000000000', solRequirement);
+ expect(first).toBe(150_000_000n);
+ // Evict the live keys so the next read must reload, then take every source down.
+ await cacheDel('spot-usd:SOL');
+ const { __resetLocalCache } = await import('../api/_lib/x402-spending-price.js');
+ __resetLocalCache();
+ global.fetch = vi.fn(async () => { throw new TypeError('network down'); });
+ const again = await toMicroUsd('1000000000', solRequirement);
+ expect(again).toBe(150_000_000n);
+ });
+
+ it('throws only when a symbol has never been priced', async () => {
+ global.fetch = vi.fn(async () => { throw new TypeError('network down'); });
+ await expect(toMicroUsd('1000', { asset: '0xdeadbeef', extra: { name: 'BNB', decimals: 18 } }))
+ .rejects.toThrow(/spending-price: every price source for BNB-USD failed/);
+ });
+});
+
+describe('drand relay chain', () => {
+ it('rolls past a dead relay to the next one', async () => {
+ const path = `/${QUICKNET_CHAIN_HASH}/public/7`;
+ const signature = 'abcdef';
+ const randomness = Buffer.from(sha256(Buffer.from(signature, 'hex'))).toString('hex');
+ global.fetch = vi.fn(async (url) => {
+ const u = String(url);
+ expect(u.endsWith(path)).toBe(true);
+ if (u.startsWith(DRAND_RELAYS[0])) throw Object.assign(new TypeError('fetch failed'), { code: 'ECONNREFUSED' });
+ return json({ round: 7, randomness, signature });
+ });
+ const out = await fetchDrandRound(7);
+ expect(out.round).toBe(7);
+ const hosts = global.fetch.mock.calls.map(([u]) => new URL(String(u)).origin);
+ expect(hosts).toContain(new URL(DRAND_RELAYS[0]).origin);
+ expect(hosts).toContain(new URL(DRAND_RELAYS[1]).origin);
+ });
+
+ it('treats a 404 as "round not published yet" and does not ask another relay', async () => {
+ global.fetch = vi.fn(async () => json({}, 404));
+ await expect(fetchDrandRound(999_999_999)).rejects.toThrow(/drand_round_unavailable/);
+ const hosts = new Set(global.fetch.mock.calls.map(([u]) => new URL(String(u)).origin));
+ expect(hosts.size).toBe(1);
+ });
+});
+
+describe('gcp-auth token cache', () => {
+ const priorSa = process.env.GCP_SERVICE_ACCOUNT_JSON;
+ beforeEach(() => {
+ _resetGcpTokenCache();
+ delete process.env.GCP_SERVICE_ACCOUNT_JSON;
+ });
+ afterEach(() => {
+ if (priorSa !== undefined) process.env.GCP_SERVICE_ACCOUNT_JSON = priorSa;
+ else delete process.env.GCP_SERVICE_ACCOUNT_JSON;
+ vi.useRealTimers();
+ });
+
+ it('retries the metadata server through a transient failure', async () => {
+ let n = 0;
+ global.fetch = vi.fn(async () => (++n === 1 ? json({}, 503) : json({ access_token: 't1', expires_in: 3600 })));
+ expect(await getGcpAccessToken()).toBe('t1');
+ expect(global.fetch).toHaveBeenCalledTimes(2);
+ });
+
+ it('keeps serving a still-valid token when the refresh fails', async () => {
+ vi.useFakeTimers();
+ vi.setSystemTime(new Date('2026-08-27T00:00:00Z'));
+ global.fetch = vi.fn(async () => json({ access_token: 't1', expires_in: 3600 }));
+ expect(await getGcpAccessToken()).toBe('t1');
+ // Inside the refresh-ahead window (>5 min before expiry): no fetch at all.
+ global.fetch = vi.fn(async () => { throw new TypeError('metadata down'); });
+ vi.setSystemTime(new Date('2026-08-27T00:50:00Z'));
+ expect(await getGcpAccessToken()).toBe('t1');
+ expect(global.fetch).not.toHaveBeenCalled();
+ // Past the refresh point but with >60s of validity left: refresh fails, the
+ // cached token is still served.
+ vi.setSystemTime(new Date('2026-08-27T00:57:00Z'));
+ const p = getGcpAccessToken();
+ await vi.runAllTimersAsync();
+ expect(await p).toBe('t1');
+ expect(global.fetch).toHaveBeenCalled();
+ // Inside the last 60s: the stale token is no longer safe to hand out.
+ vi.setSystemTime(new Date('2026-08-27T00:59:30Z'));
+ const q = getGcpAccessToken();
+ const settled = q.then(() => 'ok', (e) => e.code);
+ await vi.runAllTimersAsync();
+ expect(await settled).toBe('metadata_unavailable');
+ });
+});