Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(market): keep the symbol checker, portfolio changes, heatmap and paid datapoints alive through an upstream throttle
nirholasc31a006ba4f0
diff --git a/api/_lib/market-data/datapoints.js b/api/_lib/market-data/datapoints.jsindex a2c2fe529..5f2589404 100644--- a/api/_lib/market-data/datapoints.js+++ b/api/_lib/market-data/datapoints.js@@ -30,6 +30,7 @@// outage → 503.import { fetchGlobalMarket } from '../market-fallbacks.js';+import { fetchUpstreamJson, lastGood } from '../upstream-fetch.js';import { isPlausibleCoinId } from '../coingecko.js';import { buildCoinDetail, MINT_RE } from '../../coin/detail.js';import { buildExchanges } from '../../coin/exchanges.js';@@ -56,17 +57,29 @@ const finite = (n) => (Number.isFinite(n) ? n : null);const TTL_MS = 600_000;const _full = new Map(); // key → { value, expiresAt }+// The positive-only TTL map above has no answer for a failed load, so a+// DeFiLlama blip threw straight through every paid Market Data datapoint and the+// free /api/x402/d enumerator with it. The retry absorbs the transient and the+// last-known-good tier absorbs the rest: these feeds move slowly enough that an+// hour-old protocol list is a good answer, and infinitely better than a 503 on+// an endpoint a buyer has paid to call.async function fullSet(key, url, slim) {const now = Date.now();const hit = _full.get(key);if (hit && hit.expiresAt > now) return hit.value;- const resp = await fetch(url, {- headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' },- signal: AbortSignal.timeout(15_000),- });- if (!resp.ok) throw new Error(`${key} upstream ${resp.status}`);- const value = slim(await resp.json());- _full.set(key, { value, expiresAt: now + TTL_MS });+ const { value } = await lastGood(+ `market-datapoints:${key}`,+ async () => {+ const raw = await fetchUpstreamJson(+ url,+ { headers: { accept: 'application/json', 'user-agent': 'three.ws/1.0' } },+ { name: `market-datapoints:${key}`, timeoutMs: 15_000, attempts: 2 },+ );+ return slim(raw);+ },+ { maxAgeMs: 60 * 60_000 },+ );+ _full.set(key, { value, expiresAt: Date.now() + TTL_MS });return value;}diff --git a/api/crypto/portfolio.js b/api/crypto/portfolio.jsindex 0a99982d0..ce1df0551 100644--- a/api/crypto/portfolio.js+++ b/api/crypto/portfolio.js@@ -23,6 +23,7 @@ import { getBalances, heliusHealth } from '../_lib/balances.js';import { isValidAddressForChain } from '../_lib/splits.js';import { solChange24hPct } from '../_lib/sol-price.js';import { cacheWrap } from '../_lib/cache.js';+import { fetchUpstreamJson, lastGood } from '../_lib/upstream-fetch.js';import { buildOverview } from '../_lib/portfolio-overview.js';const CHAIN_ALIASES = {@@ -76,13 +77,23 @@ async function fetchDexScreenerChanges(tokens) {for (let i = 0; i < mints.length; i += CHANGE_BATCH) {batches.push(mints.slice(i, i + CHANGE_BATCH));}- const settled = await Promise.allSettled(batches.map(async (batch) => {- const r = await fetch(`${DEXSCREENER_TOKENS}${batch.map(encodeURIComponent).join(',')}`, {- headers: { Accept: 'application/json' },- signal: AbortSignal.timeout(6000),- });- if (!r.ok) return [];- const data = await r.json().catch(() => null);+ // Each batch retries and keeps its own last-known-good pair list. Before+ // this, allSettled swallowed a failure into an empty array and every holding+ // in that batch silently rendered a blank 24h change, indistinguishable from+ // a token that genuinely has none. A short-lived remembered answer keeps the+ // column populated through a throttle, and `staleBatches` records when it did.+ let staleBatches = 0;+ const settled = await Promise.allSettled(batches.map(async (batch, i) => {+ const { value: data, stale } = await lastGood(+ `portfolio-change:${batch.join(',')}`,+ () => fetchUpstreamJson(+ `${DEXSCREENER_TOKENS}${batch.map(encodeURIComponent).join(',')}`,+ { headers: { Accept: 'application/json' } },+ { name: `dexscreener:portfolio:${i}`, timeoutMs: 6_000, attempts: 2 },+ ),+ { maxAgeMs: 10 * 60_000 },+ );+ if (stale) staleBatches++;return Array.isArray(data?.pairs) ? data.pairs : [];}));diff --git a/api/crypto/symbol.js b/api/crypto/symbol.jsindex f1541ebe2..dd2acb0e4 100644--- a/api/crypto/symbol.js+++ b/api/crypto/symbol.js@@ -19,6 +19,8 @@import { wrap, cors, method, json, error, readJson, rateLimited, setRateLimitHeaders } from '../_lib/http.js';import { clientIp, limits } from '../_lib/rate-limit.js';+import { lastGood } from '../_lib/upstream-fetch.js';+import { withRetry } from '../_lib/resilience.js';const DEXSCREENER_SEARCH = 'https://api.dexscreener.com/latest/dex/search';@@ -70,12 +72,26 @@ export function symbolSimilarity(a, b) {// a non-OK upstream so the caller can degrade that symbol gracefully.async function searchTokens(symbol, chain, fetchImpl) {const url = `${DEXSCREENER_SEARCH}?q=${encodeURIComponent(symbol)}`;- const r = await fetchImpl(url, {- headers: { Accept: 'application/json' },- signal: AbortSignal.timeout(6000),- });- if (!r.ok) throw new Error(`dexscreener search ${r.status}`);- const data = await r.json();+ // One collision source, and a bare 429 from it turned every answer on the+ // page into "could not verify". A retry absorbs the common transient, and a+ // last-known-good result for this exact symbol absorbs the rest: a+ // ten-minute-old collision list is still an honest answer about whether a+ // ticker is taken, and far more useful than a row of question marks.+ const { value: data, stale } = await lastGood(+ `symbol-search:${chain || 'any'}:${symbol.toLowerCase()}`,+ () => withRetry(+ async () => {+ const r = await fetchImpl(url, {+ headers: { Accept: 'application/json' },+ signal: AbortSignal.timeout(6000),+ });+ if (!r.ok) throw Object.assign(new Error(`dexscreener search ${r.status}`), { status: r.status });+ return r.json();+ },+ { attempts: 2, label: 'dexscreener:search' },+ ),+ { maxAgeMs: 10 * 60_000 },+ );const pairs = Array.isArray(data?.pairs) ? data.pairs : [];const byMint = new Map();for (const p of pairs) {@@ -94,14 +110,15 @@ async function searchTokens(symbol, chain, fetchImpl) {});}}- return [...byMint.values()];+ return { tokens: [...byMint.values()], stale };}// Classify one candidate symbol against the live registry.async function checkOne(symbol, chain, fetchImpl) {let candidates;+ let stale = false;try {- candidates = await searchTokens(symbol, chain, fetchImpl);+ ({ tokens: candidates, stale } = await searchTokens(symbol, chain, fetchImpl));} catch {// Registry source down for this symbol: degrade, never fail the batch.// `available: null` signals "could not verify", distinct from a hard@@ -134,9 +151,30 @@ async function checkOne(symbol, chain, fetchImpl) {}fuzzy.sort((a, b) => b.similarity - a.similarity);+ // A stale answer is NOT a verdict here, and this is the one surface where+ // that distinction has teeth: someone reads `available: true` and launches a+ // ticker on it. A collision registered in the last few minutes is exactly+ // what a cached answer cannot see, so availability stays null when the live+ // source could not be reached. The remembered collisions still ship, as+ // context rather than as a claim: "last time we could check, these existed"+ // is worth showing, and it is a strictly better degraded state than the+ // empty one this used to fall back to.+ if (stale) {+ return {+ symbol,+ available: null,+ exactCollisions: exact.slice(0, MAX_COLLISIONS),+ fuzzyCollisions: fuzzy.slice(0, MAX_COLLISIONS),+ degraded: true,+ stale: true,+ note: 'collision source unavailable: showing the last known collisions, availability could not be verified live',+ };+ }+return {symbol,available: exact.length === 0,+ stale: false,exactCollisions: exact.slice(0, MAX_COLLISIONS),fuzzyCollisions: fuzzy.slice(0, MAX_COLLISIONS),};diff --git a/api/intel/heatmap.js b/api/intel/heatmap.jsindex a9a38c9af..b60cd57ca 100644--- a/api/intel/heatmap.js+++ b/api/intel/heatmap.js@@ -34,6 +34,7 @@import { cors, json, method, wrap, error, rateLimited } from '../_lib/http.js';import { limits, clientIp } from '../_lib/rate-limit.js';import { normalizeGatewayURL } from '../../src/ipfs.js';+import { fetchUpstreamJson, lastGood } from '../_lib/upstream-fetch.js';// The one and only coin three.ws promotes. Always present, always featured.const THREE_MINT = 'FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump';@@ -54,15 +55,26 @@ export function _resetHeatmapCache() {_cache = { value: null, storedAt: 0, expiresAt: 0, limit: 0 };}-async function fetchJson(url, init) {- let res;+// Every upstream read on this page goes through here, so this is where the+// page's resilience lives. Two things it did not have: a retry (a single 429+// from pump.fun or DexScreener blanked the field) and a memory (the in-process+// stale tier at STALE_MAX_MS is per-instance, so a cold instance during an+// outage had nothing at all and 502'd a public page). Both are added here+// rather than at each call site, so the trending feed and the per-batch market+// reads inherit them together.+async function fetchJson(url, init, lastGoodKey = null) {+ const load = () => fetchUpstreamJson(+ String(url),+ { ...init },+ { name: 'intel-heatmap', timeoutMs: UPSTREAM_TIMEOUT_MS, attempts: 2 },+ );try {- res = await fetch(url, { ...init, signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS) });+ if (!lastGoodKey) return await load();+ const { value } = await lastGood(lastGoodKey, load, { maxAgeMs: 15 * 60_000 });+ return value;} catch {return null;}- if (!res.ok) return null;- return res.json().catch(() => null);}// Trending mints from the public pump.fun feed (no key). Returns mint pubkeys@@ -74,7 +86,7 @@ async function fetchTrendingMints(limit) {url.searchParams.set('sort', 'market_cap');url.searchParams.set('order', 'DESC');url.searchParams.set('includeNsfw', 'false');- const body = await fetchJson(url, { headers: { accept: 'application/json' } });+ const body = await fetchJson(url, { headers: { accept: 'application/json' } }, `heatmap:trending:${limit}`);const coins = Array.isArray(body) ? body : Array.isArray(body?.coins) ? body.coins : null;if (!Array.isArray(coins)) return [];return coins@@ -89,7 +101,7 @@ async function fetchDexBatch(mints) {for (let i = 0; i < mints.length; i += DS_BATCH) {const chunk = mints.slice(i, i + DS_BATCH);const url = `${DEXSCREENER_BASE}/latest/dex/tokens/${chunk.join(',')}`;- const data = await fetchJson(url, { headers: { accept: 'application/json' } });+ const data = await fetchJson(url, { headers: { accept: 'application/json' } }, `heatmap:dex:${chunk.join(',')}`);const pairs = Array.isArray(data?.pairs) ? data.pairs : [];for (const p of pairs) {const mint = p?.baseToken?.address;@@ -118,9 +130,13 @@ async function fetchDexBatch(mints) {// pump.fun metadata for a single mint (name/symbol/image): fills gaps when a// token has no Dexscreener pair yet, and supplies $THREE's canonical metadata.async function fetchPumpMeta(mint) {- const data = await fetchJson(`${PUMP_FRONTEND_BASE}/coins/${mint}`, {- headers: { accept: 'application/json' },- });+ // Token metadata barely changes, so a remembered copy is as good as a live+ // one and keeps names and images on the tiles through a pump.fun throttle.+ const data = await fetchJson(+ `${PUMP_FRONTEND_BASE}/coins/${mint}`,+ { headers: { accept: 'application/json' } },+ `heatmap:meta:${mint}`,+ );if (!data || data.error) return null;return {symbol: data.symbol || null,