Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(crypto-holders): stop one free scraper from draining the Helius plan behind /api/crypto/holders
nirholas51bddac92432
diff --git a/api/_lib/crypto-token-holders.js b/api/_lib/crypto-token-holders.jsindex 591dbf4ee..4d6a3b8b9 100644--- a/api/_lib/crypto-token-holders.js+++ b/api/_lib/crypto-token-holders.js@@ -19,7 +19,12 @@// deriveConcentration; docs/crypto-api.md mirrors the exact numbers.import { parseMintAccount } from '../v1/token/security.js';-import { solanaRpcEndpoints, makeRotatingFetch } from './solana/connection.js';+import {+ solanaRpcEndpoints,+ makeRotatingFetch,+ isEndpointCooling,+ markEndpointCooldown,+} from './solana/connection.js';import { withDeadline } from './rpc-degrade.js';// Budget for ONE JSON-RPC read across the WHOLE failover chain, not per@@ -135,10 +140,18 @@ async function rpcCall(rpcFetch, method, params) {* `complete` is true iff every account was enumerated within the page cap —* only then is a holder count honest.*/-export async function heliusHolderWalk(mint, { fetchImpl = fetch } = {}) {+export async function heliusHolderWalk(+ mint,+ { fetchImpl = fetch, isCooling = isEndpointCooling, markCooldown = markEndpointCooldown } = {},+) {const key = process.env.HELIUS_API_KEY;if (!key) return null;+ // Same URL the canonical RPC chain parks, so the fleet-wide quota verdict+ // applies here too. Walking into a plan that is already out spent a doomed+ // round trip per read and kept the cap pinned (2,000+ "max usage reached"+ // replies in 6h on 2026-10-10).const url = `https://mainnet.helius-rpc.com/?api-key=${key}`;+ if (isCooling(url)) return null;const accounts = [];let complete = false;for (let page = 1; page <= HELIUS_MAX_PAGES; page++) {@@ -146,6 +159,7 @@ export async function heliusHolderWalk(mint, { fetchImpl = fetch } = {}) {try {batch = await heliusPage(url, mint, page, fetchImpl);} catch (err) {+ if (err.quota) markCooldown(url, 429, err.message);// A page that fails mid-walk does not invalidate the pages already in// hand: 1000 accounts is far more than any top-N needs, and discarding// them dropped the read onto the keyless lane, whose@@ -164,6 +178,10 @@ export async function heliusHolderWalk(mint, { fetchImpl = fetch } = {}) {return { accounts, complete };}+// Helius words an exhausted plan as "max usage reached" (HTTP 429, or 200 with a+// plain-text body) or JSON-RPC -32429.+const HELIUS_QUOTA_RE = /max usage reached|-32429/i;+// One page of the walk. Helius answers an exhausted plan with HTTP 200 and the// plain-text body "max usage reached", so parsing the body as JSON unconditionally// surfaced a bare SyntaxError with no hint that the key was the problem. Read the@@ -181,21 +199,29 @@ async function heliusPage(url, mint, page, fetchImpl) {signal: AbortSignal.timeout(RPC_TIMEOUT_MS),});const text = await r.text();- if (!r.ok) throw new Error(`helius getTokenAccounts ${r.status}: ${text.slice(0, 80)}`);+ const quota = r.status === 429 || HELIUS_QUOTA_RE.test(text);+ if (!r.ok) throw Object.assign(new Error(`helius getTokenAccounts ${r.status}: ${text.slice(0, 80)}`), { quota });let json;try {json = JSON.parse(text);} catch {- throw new Error(`helius getTokenAccounts: non-JSON response "${text.slice(0, 60)}"`);+ throw Object.assign(new Error(`helius getTokenAccounts: non-JSON response "${text.slice(0, 60)}"`), { quota });+ }+ if (json?.error) {+ const detail = JSON.stringify(json.error);+ throw Object.assign(new Error(`helius getTokenAccounts: ${detail.slice(0, 120)}`), { quota: HELIUS_QUOTA_RE.test(detail) });}- if (json?.error) throw new Error(`helius getTokenAccounts: ${JSON.stringify(json.error).slice(0, 120)}`);const batch = json?.result?.token_accounts;if (!Array.isArray(batch)) throw new Error('helius getTokenAccounts: malformed response');return batch;}-// Default network dependency bundle — injectable for tests.-export function realHolderDeps() {+// Default network dependency bundle, injectable for tests. `heliusAllowed` is+// the caller's budget gate for the keyed walk: each read costs up to+// HELIUS_MAX_PAGES DAS calls against a paid plan shared with balances, the+// sniper and the indexers, so a free keyless endpoint must not let one client+// spend it. A denied budget is not an error, the read takes the keyless lane.+export function realHolderDeps({ heliusAllowed = async () => true } = {}) {const rpcFetch = makeRotatingFetch(solanaRpcEndpoints('mainnet'));return {fetchMintAccount: (address) =>@@ -213,7 +239,7 @@ export function realHolderDeps() {});return owners;},- fetchHelius: (address) => heliusHolderWalk(address),+ fetchHelius: async (address) => ((await heliusAllowed()) ? heliusHolderWalk(address) : null),};}diff --git a/api/_lib/rate-limit.js b/api/_lib/rate-limit.jsindex ad9d4167f..804945e70 100644--- a/api/_lib/rate-limit.js+++ b/api/_lib/rate-limit.js@@ -1520,6 +1520,12 @@ export const limits = {// (which caching can't stop). One bucket across all DAS endpoints.heliusDasGlobal: () =>getLimiter('helius-das:global', { limit: 3000, window: '1 h' }).limit('global'),+ // Per-IP share of that ceiling for /api/crypto/holders, whose keyed walk is+ // up to five DAS pages. Without it one scraper (3,500 reads/h from a single+ // IP, 2026-10-10) spent the whole Helius plan; past it the caller still gets+ // a real answer from the keyless lane.+ cryptoHoldersHeliusIp: (ip) =>+ getLimiter('crypto-holders:helius:ip', { limit: 30, window: '1 h' }).limit(ip),// Free Crypto Data API family (api/crypto/*). Keyless, no-account reads an agent// makes mid-task (wallet portfolio, token snapshots). Some paths fan out to the// keyed Helius/public-RPC upstreams, so a generous-but-bounded per-IP burst keepsdiff --git a/api/crypto/holders.js b/api/crypto/holders.jsindex fc53a9a4d..ff21d17ff 100644--- a/api/crypto/holders.js+++ b/api/crypto/holders.js@@ -24,7 +24,7 @@import { cors, method, wrap, error, json, rateLimited } from '../_lib/http.js';import { limits, clientIp } from '../_lib/rate-limit.js';import { isValidSolanaAddress, isValidEvmAddress } from '../_lib/validate.js';-import { composeTokenHolders, DEFAULT_LIMIT, MAX_LIMIT } from '../_lib/crypto-token-holders.js';+import { composeTokenHolders, realHolderDeps, DEFAULT_LIMIT, MAX_LIMIT } from '../_lib/crypto-token-holders.js';import { cacheGet, cacheSet } from '../_lib/cache.js';import { staleEnvelope } from '../_lib/rpc-degrade.js';@@ -34,6 +34,11 @@ import { staleEnvelope } from '../_lib/rpc-degrade.js';const HOLDERS_LKG_TTL_S = 24 * 3600;const holdersLkgKey = (address, limit) => `crypto:holders:lkg:${address}:${limit}`;+// Fresh tier on the same record: a repeat read inside the CDN's own max-age is+// answered from it, so a poller that misses the edge cache never re-walks the+// chain for a distribution it was handed seconds ago.+const HOLDERS_FRESH_MS = 60_000;+const EXAMPLE = '/api/crypto/holders?address=FeMbDoX7R1Psc4GEcvJdsbNbZA3bfztcyDCatJVJpump';export default wrap(async (req, res) => {@@ -69,7 +74,22 @@ export default wrap(async (req, res) => {});}- const result = await composeTokenHolders({ address, limit });+ const lkgKey = holdersLkgKey(address, limit);+ const cached = await cacheGet(lkgKey);+ if (cached && cached.body && typeof cached.at === 'number' && Date.now() - cached.at < HOLDERS_FRESH_MS) {+ return json(res, 200, cached.body, {+ 'cache-control': 'public, s-maxage=60, stale-while-revalidate=60',+ 'x-holders-cache': 'hit',+ });+ }++ // The keyed walk spends the shared Helius DAS ceiling, so it runs only while+ // this caller's hourly share and the global cap both have room.+ const heliusAllowed = async () => {+ const [ipDas, globalDas] = await Promise.all([limits.cryptoHoldersHeliusIp(ip), limits.heliusDasGlobal()]);+ return ipDas.success && globalDas.success;+ };+ const result = await composeTokenHolders({ address, limit }, realHolderDeps({ heliusAllowed }));const ts = new Date().toISOString();if (result.status === 'not_found') {@@ -78,9 +98,8 @@ export default wrap(async (req, res) => {});}if (result.status === 'upstream_down') {- const lkg = await cacheGet(holdersLkgKey(address, limit));- if (lkg && lkg.body && typeof lkg.at === 'number') {- return json(res, 200, staleEnvelope(lkg.body, lkg.at), {+ if (cached && cached.body && typeof cached.at === 'number') {+ return json(res, 200, staleEnvelope(cached.body, cached.at), {'cache-control': 'public, s-maxage=30, stale-while-revalidate=60','x-holders-stale': '1',});@@ -105,7 +124,7 @@ export default wrap(async (req, res) => {sources: result.sources,};if (result.note) body.note = result.note;- cacheSet(holdersLkgKey(address, limit), { body, at: Date.now() }, HOLDERS_LKG_TTL_S).catch(() => {});+ cacheSet(lkgKey, { body, at: Date.now() }, HOLDERS_LKG_TTL_S).catch(() => {});// Distribution shifts trade by trade but a position-sizing read tolerates a// minute of cache; this absorbs polling bursts without hammering the RPC.diff --git a/docs/crypto-api.md b/docs/crypto-api.mdindex 35496f501..bf6d6a851 100644--- a/docs/crypto-api.md+++ b/docs/crypto-api.md@@ -819,6 +819,13 @@ surfaces can never contradict each other.`getTokenAccounts` — pages through the token's accounts and aggregates byowner wallet, so a whale split across accounts reads as one holder. Walkcapped at 5,000 accounts; beyond it `holderCount` is `null` with a `note`.+ The walk is metered: each caller gets 30 keyed reads per hour within the+ platform's shared indexer ceiling, and it is skipped while the indexer plan is+ in a quota cooldown. Past either limit the read is answered from the keyless+ path below (`holderCount: null`, marked in `note`), never refused.+- **Repeat reads:** the same `address` + `limit` within 60 seconds is answered+ from the previous result (`x-holders-cache: hit`), matching the response's own+ `s-maxage=60`.- **Keyless fallback (always available):** Solana RPC `getTokenLargestAccounts`(the chain's top 20 token accounts) + `getMultipleAccounts` to resolve owners+ the mint's supply for percentages. Coarse but real, and marked in `note`.diff --git a/tests/crypto-token-holders.test.js b/tests/crypto-token-holders.test.jsindex ec436d8aa..5e6a4e1e2 100644--- a/tests/crypto-token-holders.test.js+++ b/tests/crypto-token-holders.test.js@@ -7,6 +7,7 @@ import {deriveConcentration,composeTokenHolders,heliusHolderWalk,+ realHolderDeps,CONCENTRATION_HIGH_PCT,CONCENTRATION_MEDIUM_PCT,} from '../api/_lib/crypto-token-holders.js';@@ -220,6 +221,8 @@ describe('composeTokenHolders — states', () => {// refuse. These pin the failure shapes that took it down without a trace.describe('heliusHolderWalk failure shapes', () => {const MINT = THREE;+ // The fleet-wide lane cooldown is module state; isolate each case from it.+ const noBreaker = { isCooling: () => false, markCooldown: () => {} };const page = (n) => ({ok: true,status: 200,@@ -243,7 +246,7 @@ describe('heliusHolderWalk failure shapes', () => {const had = process.env.HELIUS_API_KEY;delete process.env.HELIUS_API_KEY;try {- expect(await heliusHolderWalk(MINT, { fetchImpl: async () => page(0) })).toBeNull();+ expect(await heliusHolderWalk(MINT, { fetchImpl: async () => page(0), ...noBreaker })).toBeNull();} finally {if (had !== undefined) process.env.HELIUS_API_KEY = had;}@@ -252,14 +255,14 @@ describe('heliusHolderWalk failure shapes', () => {it('names an exhausted plan instead of throwing a bare JSON parse error', async () => {await withKey(async () => {const fetchImpl = async () => ({ ok: true, status: 200, text: async () => 'max usage reached' });- await expect(heliusHolderWalk(MINT, { fetchImpl })).rejects.toThrow(/non-JSON response .*max usage reached/);+ await expect(heliusHolderWalk(MINT, { fetchImpl, ...noBreaker })).rejects.toThrow(/non-JSON response .*max usage reached/);});});it('names a JSON-RPC error envelope', async () => {await withKey(async () => {const fetchImpl = async () => ({ ok: true, status: 200, text: async () => JSON.stringify({ error: { code: -32603, message: 'nope' } }) });- await expect(heliusHolderWalk(MINT, { fetchImpl })).rejects.toThrow(/nope/);+ await expect(heliusHolderWalk(MINT, { fetchImpl, ...noBreaker })).rejects.toThrow(/nope/);});});@@ -271,7 +274,7 @@ describe('heliusHolderWalk failure shapes', () => {if (call === 1) return page(1000);throw new Error('gateway timeout');};- const out = await heliusHolderWalk(MINT, { fetchImpl });+ const out = await heliusHolderWalk(MINT, { fetchImpl, ...noBreaker });expect(out.accounts).toHaveLength(1000);// Incomplete, so the holder COUNT is still withheld rather than guessed.expect(out.complete).toBe(false);@@ -281,15 +284,58 @@ describe('heliusHolderWalk failure shapes', () => {it('still throws when the very first page fails and there is nothing to keep', async () => {await withKey(async () => {const fetchImpl = async () => { throw new Error('gateway timeout'); };- await expect(heliusHolderWalk(MINT, { fetchImpl })).rejects.toThrow(/gateway timeout/);+ await expect(heliusHolderWalk(MINT, { fetchImpl, ...noBreaker })).rejects.toThrow(/gateway timeout/);});});it('marks a short page as a complete walk', async () => {await withKey(async () => {- const out = await heliusHolderWalk(MINT, { fetchImpl: async () => page(3) });+ const out = await heliusHolderWalk(MINT, { fetchImpl: async () => page(3), ...noBreaker });expect(out.complete).toBe(true);expect(out.accounts).toHaveLength(3);});});++ it('skips the walk while the Helius lane is parked in the quota cooldown', async () => {+ await withKey(async () => {+ let called = false;+ const fetchImpl = async () => { called = true; return page(3); };+ const out = await heliusHolderWalk(MINT, { fetchImpl, isCooling: () => true, markCooldown: () => {} });+ expect(out).toBeNull();+ expect(called).toBe(false);+ });+ });++ it('parks the Helius lane fleet-wide when the plan reports max usage', async () => {+ await withKey(async () => {+ const marks = [];+ const fetchImpl = async () => ({ ok: false, status: 429, text: async () => 'max usage reached' });+ await expect(+ heliusHolderWalk(MINT, { fetchImpl, isCooling: () => false, markCooldown: (...a) => marks.push(a) }),+ ).rejects.toThrow(/429: max usage reached/);+ expect(marks).toHaveLength(1);+ expect(marks[0][0]).toMatch(/mainnet\.helius-rpc\.com/);+ expect(marks[0][1]).toBe(429);+ });+ });++ it('does not park the lane for a transient failure', async () => {+ await withKey(async () => {+ const marks = [];+ const fetchImpl = async () => { throw new Error('gateway timeout'); };+ await expect(+ heliusHolderWalk(MINT, { fetchImpl, isCooling: () => false, markCooldown: (...a) => marks.push(a) }),+ ).rejects.toThrow(/gateway timeout/);+ expect(marks).toHaveLength(0);+ });+ });+});++describe('realHolderDeps Helius budget gate', () => {+ it('takes the keyless lane without touching Helius when the budget says no', async () => {+ let asked = 0;+ const d = realHolderDeps({ heliusAllowed: async () => { asked++; return false; } });+ expect(await d.fetchHelius(THREE)).toBeNull();+ expect(asked).toBe(1);+ });});