Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(market): keep the symbol checker, portfolio changes, heatmap and paid datapoints alive through an upstream throttle

nirholasc31a006ba4f0
diff --git a/api/_lib/market-data/datapoints.js b/api/_lib/market-data/datapoints.js
index a2c2fe529..5f2589404 100644
--- a/api/_lib/market-data/datapoints.js
+++ b/api/_lib/market-data/datapoints.js
@@ -30,6 +30,7 @@
// outage → 503.
import { fetchGlobalMarket } from '../market-fallbacks.js';
+import { fetchUpstreamJson, lastGood } from '../upstream-fetch.js';
import { isPlausibleCoinId } from '../coingecko.js';
import { buildCoinDetail, MINT_RE } from '../../coin/detail.js';
import { buildExchanges } from '../../coin/exchanges.js';
@@ -56,17 +57,29 @@ const finite = (n) => (Number.isFinite(n) ? n : null);
const TTL_MS = 600_000;
const _full = new Map(); // key → { value, expiresAt }
+// The positive-only TTL map above has no answer for a failed load, so a
+// DeFiLlama blip threw straight through every paid Market Data datapoint and the
+// free /api/x402/d enumerator with it. The retry absorbs the transient and the
+// last-known-good tier absorbs the rest: these feeds move slowly enough that an
+// hour-old protocol list is a good answer, and infinitely better than a 503 on
+// an endpoint a buyer has paid to call.
async function fullSet(key, url, slim) {
const now = Date.now();
const hit = _full.get(key);
if (hit && hit.expiresAt > now) return hit.value;
- const resp = await fetch(url, {
- headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },
- signal: AbortSignal.timeout(15_000),
- });
- if (!resp.ok) throw new Error(`${key} upstream ${resp.status}`);
- const value = slim(await resp.json());
- _full.set(key, { value, expiresAt: now + TTL_MS });
+ const { value } = await lastGood(
+ `market-datapoints:${key}`,
+ async () => {
+ const raw = await fetchUpstreamJson(
+ url,
+ { headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' } },
+ { name: `market-datapoints:${key}`, timeoutMs: 15_000, attempts: 2 },
+ );
+ return slim(raw);
+ },
+ { maxAgeMs: 60 * 60_000 },
+ );
+ _full.set(key, { value, expiresAt: Date.now() + TTL_MS });
return value;
}
diff --git a/api/crypto/portfolio.js b/api/crypto/portfolio.js
index 0a99982d0..ce1df0551 100644
--- a/api/crypto/portfolio.js
+++ b/api/crypto/portfolio.js
@@ -23,6 +23,7 @@ import { getBalances, heliusHealth } from '../_lib/balances.js';
import { isValidAddressForChain } from '../_lib/splits.js';
import { solChange24hPct } from '../_lib/sol-price.js';
import { cacheWrap } from '../_lib/cache.js';
+import { fetchUpstreamJson, lastGood } from '../_lib/upstream-fetch.js';
import { buildOverview } from '../_lib/portfolio-overview.js';
const CHAIN_ALIASES = {
@@ -76,13 +77,23 @@ async function fetchDexScreenerChanges(tokens) {
for (let i = 0; i < mints.length; i += CHANGE_BATCH) {
batches.push(mints.slice(i, i + CHANGE_BATCH));
}
- const settled = await Promise.allSettled(batches.map(async (batch) => {
- const r = await fetch(`${DEXSCREENER_TOKENS}${batch.map(encodeURIComponent).join(',')}`, {
- headers: { Accept: 'application/json' },
- signal: AbortSignal.timeout(6000),
- });
- if (!r.ok) return [];
- const data = await r.json().catch(() => null);
+ // Each batch retries and keeps its own last-known-good pair list. Before
+ // this, allSettled swallowed a failure into an empty array and every holding
+ // in that batch silently rendered a blank 24h change, indistinguishable from
+ // a token that genuinely has none. A short-lived remembered answer keeps the
+ // column populated through a throttle, and `staleBatches` records when it did.
+ let staleBatches = 0;
+ const settled = await Promise.allSettled(batches.map(async (batch, i) => {
+ const { value: data, stale } = await lastGood(
+ `portfolio-change:${batch.join(',')}`,
+ () => fetchUpstreamJson(
+ `${DEXSCREENER_TOKENS}${batch.map(encodeURIComponent).join(',')}`,
+ { headers: { Accept: 'application/json' } },
+ { name: `dexscreener:portfolio:${i}`, timeoutMs: 6_000, attempts: 2 },
+ ),
+ { maxAgeMs: 10 * 60_000 },
+ );
+ if (stale) staleBatches++;
return Array.isArray(data?.pairs) ? data.pairs : [];
}));
diff --git a/api/crypto/symbol.js b/api/crypto/symbol.js
index f1541ebe2..dd2acb0e4 100644
--- a/api/crypto/symbol.js
+++ b/api/crypto/symbol.js
@@ -19,6 +19,8 @@
import { wrap, cors, method, json, error, readJson, rateLimited, setRateLimitHeaders } from '../_lib/http.js';
import { clientIp, limits } from '../_lib/rate-limit.js';
+import { lastGood } from '../_lib/upstream-fetch.js';
+import { withRetry } from '../_lib/resilience.js';
const DEXSCREENER_SEARCH = 'https://api.dexscreener.com/latest/dex/search';
@@ -70,12 +72,26 @@ export function symbolSimilarity(a, b) {
// a non-OK upstream so the caller can degrade that symbol gracefully.
async function searchTokens(symbol, chain, fetchImpl) {
const url = `${DEXSCREENER_SEARCH}?q=${encodeURIComponent(symbol)}`;
- const r = await fetchImpl(url, {
- headers: { Accept: 'application/json' },
- signal: AbortSignal.timeout(6000),
- });
- if (!r.ok) throw new Error(`dexscreener search ${r.status}`);
- const data = await r.json();
+ // One collision source, and a bare 429 from it turned every answer on the
+ // page into "could not verify". A retry absorbs the common transient, and a
+ // last-known-good result for this exact symbol absorbs the rest: a
+ // ten-minute-old collision list is still an honest answer about whether a
+ // ticker is taken, and far more useful than a row of question marks.
+ const { value: data, stale } = await lastGood(
+ `symbol-search:${chain || 'any'}:${symbol.toLowerCase()}`,
+ () => withRetry(
+ async () => {
+ const r = await fetchImpl(url, {
+ headers: { Accept: 'application/json' },
+ signal: AbortSignal.timeout(6000),
+ });
+ if (!r.ok) throw Object.assign(new Error(`dexscreener search ${r.status}`), { status: r.status });
+ return r.json();
+ },
+ { attempts: 2, label: 'dexscreener:search' },
+ ),
+ { maxAgeMs: 10 * 60_000 },
+ );
const pairs = Array.isArray(data?.pairs) ? data.pairs : [];
const byMint = new Map();
for (const p of pairs) {
@@ -94,14 +110,15 @@ async function searchTokens(symbol, chain, fetchImpl) {
});
}
}
- return [...byMint.values()];
+ return { tokens: [...byMint.values()], stale };
}
// Classify one candidate symbol against the live registry.
async function checkOne(symbol, chain, fetchImpl) {
let candidates;
+ let stale = false;
try {
- candidates = await searchTokens(symbol, chain, fetchImpl);
+ ({ tokens: candidates, stale } = await searchTokens(symbol, chain, fetchImpl));
} catch {
// Registry source down for this symbol: degrade, never fail the batch.
// `available: null` signals "could not verify", distinct from a hard
@@ -134,9 +151,30 @@ async function checkOne(symbol, chain, fetchImpl) {
}
fuzzy.sort((a, b) => b.similarity - a.similarity);
+ // A stale answer is NOT a verdict here, and this is the one surface where
+ // that distinction has teeth: someone reads `available: true` and launches a
+ // ticker on it. A collision registered in the last few minutes is exactly
+ // what a cached answer cannot see, so availability stays null when the live
+ // source could not be reached. The remembered collisions still ship, as
+ // context rather than as a claim: "last time we could check, these existed"
+ // is worth showing, and it is a strictly better degraded state than the
+ // empty one this used to fall back to.
+ if (stale) {
+ return {
+ symbol,
+ available: null,
+ exactCollisions: exact.slice(0, MAX_COLLISIONS),
+ fuzzyCollisions: fuzzy.slice(0, MAX_COLLISIONS),
+ degraded: true,
+ stale: true,
+ note: 'collision source unavailable: showing the last known collisions, availability could not be verified live',
+ };
+ }
+
return {
symbol,
available: exact.length === 0,
+ stale: false,
exactCollisions: exact.slice(0, MAX_COLLISIONS),
fuzzyCollisions: fuzzy.slice(0, MAX_COLLISIONS),
};
diff --git a/api/intel/heatmap.js b/api/intel/heatmap.js
index a9a38c9af..b60cd57ca 100644
--- a/api/intel/heatmap.js
+++ b/api/intel/heatmap.js
@@ -34,6 +34,7 @@
import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';
import { limits, clientIp } from '../_lib/rate-limit.js';
import { normalizeGatewayURL } from '../../src/ipfs.js';
+import { fetchUpstreamJson, lastGood } from '../_lib/upstream-fetch.js';
// The one and only coin three.ws promotes. Always present, always featured.
const THREE_MINT = 'FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump';
@@ -54,15 +55,26 @@ export function _resetHeatmapCache() {
_cache = { value: null, storedAt: 0, expiresAt: 0, limit: 0 };
}
-async function fetchJson(url, init) {
- let res;
+// Every upstream read on this page goes through here, so this is where the
+// page's resilience lives. Two things it did not have: a retry (a single 429
+// from pump.fun or DexScreener blanked the field) and a memory (the in-process
+// stale tier at STALE_MAX_MS is per-instance, so a cold instance during an
+// outage had nothing at all and 502'd a public page). Both are added here
+// rather than at each call site, so the trending feed and the per-batch market
+// reads inherit them together.
+async function fetchJson(url, init, lastGoodKey = null) {
+ const load = () => fetchUpstreamJson(
+ String(url),
+ { ...init },
+ { name: 'intel-heatmap', timeoutMs: UPSTREAM_TIMEOUT_MS, attempts: 2 },
+ );
try {
- res = await fetch(url, { ...init, signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) });
+ if (!lastGoodKey) return await load();
+ const { value } = await lastGood(lastGoodKey, load, { maxAgeMs: 15 * 60_000 });
+ return value;
} catch {
return null;
}
- if (!res.ok) return null;
- return res.json().catch(() => null);
}
// Trending mints from the public pump.fun feed (no key). Returns mint pubkeys
@@ -74,7 +86,7 @@ async function fetchTrendingMints(limit) {
url.searchParams.set('sort', 'market_cap');
url.searchParams.set('order', 'DESC');
url.searchParams.set('includeNsfw', 'false');
- const body = await fetchJson(url, { headers: { accept: 'application/json' } });
+ const body = await fetchJson(url, { headers: { accept: 'application/json' } }, `heatmap:trending:${limit}`);
const coins = Array.isArray(body) ? body : Array.isArray(body?.coins) ? body.coins : null;
if (!Array.isArray(coins)) return [];
return coins
@@ -89,7 +101,7 @@ async function fetchDexBatch(mints) {
for (let i = 0; i < mints.length; i += DS_BATCH) {
const chunk = mints.slice(i, i + DS_BATCH);
const url = `${DEXSCREENER_BASE}/latest/dex/tokens/${chunk.join(',')}`;
- const data = await fetchJson(url, { headers: { accept: 'application/json' } });
+ const data = await fetchJson(url, { headers: { accept: 'application/json' } }, `heatmap:dex:${chunk.join(',')}`);
const pairs = Array.isArray(data?.pairs) ? data.pairs : [];
for (const p of pairs) {
const mint = p?.baseToken?.address;
@@ -118,9 +130,13 @@ async function fetchDexBatch(mints) {
// pump.fun metadata for a single mint (name/symbol/image): fills gaps when a
// token has no Dexscreener pair yet, and supplies $THREE's canonical metadata.
async function fetchPumpMeta(mint) {
- const data = await fetchJson(`${PUMP_FRONTEND_BASE}/coins/${mint}`, {
- headers: { accept: 'application/json' },
- });
+ // Token metadata barely changes, so a remembered copy is as good as a live
+ // one and keeps names and images on the tiles through a pump.fun throttle.
+ const data = await fetchJson(
+ `${PUMP_FRONTEND_BASE}/coins/${mint}`,
+ { headers: { accept: 'application/json' } },
+ `heatmap:meta:${mint}`,
+ );
if (!data || data.error) return null;
return {
symbol: data.symbol || null,