Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(aggregator): fail over to the next Solana RPC host when a pooled host answers 401 or 403

nirholas61c2fde580ee
diff --git a/api/_lib/aggregator.js b/api/_lib/aggregator.js
index 895780fb5..1c79f3a56 100644
--- a/api/_lib/aggregator.js
+++ b/api/_lib/aggregator.js
@@ -44,6 +44,13 @@ const MAX_UPSTREAM_ATTEMPTS = 6;
// everywhere, so it is never retried.
const RETRYABLE_CODES = new Set(['upstream_rate_limited', 'upstream_unreachable']);
const isRetryable = (err) => RETRYABLE_CODES.has(err?.code) || err?.status === 502;
+// In a pool of interchangeable hosts, a 401 or 403 is about that host (an
+// expired or quota-dead key, a node blocking our egress IP; the RPC registry
+// already cools such a host for hours), not about the request, so the next
+// host answers it. Observed 2026-10-09: the keyed Solana primary answered 403
+// and largest-holders failed outright while the public hosts below it were
+// healthy. A single-host provider still surfaces the 4xx unchanged.
+const isRetryableInPool = (err) => isRetryable(err) || err?.status === 401 || err?.status === 403;
// Short per-host memory of retryable failures. Without it, every request
// re-discovers a dead primary the hard way (worst case the full 20s timeout)
@@ -244,7 +251,7 @@ export async function executeUpstream({ provider, endpoint, query = {}, body, ap
markGood(provider.base);
return out;
} catch (err) {
- if (!isRetryable(err)) throw err;
+ if (!(hasPool ? isRetryableInPool(err) : isRetryable(err))) throw err;
markBad(provider.base);
reportFailure(provider.base, err);
logHostFailure(provider, provider.base, err);
@@ -281,7 +288,7 @@ export async function executeUpstream({ provider, endpoint, query = {}, body, ap
markGood(base);
return out;
} catch (next) {
- if (!isRetryable(next)) throw next;
+ if (!isRetryableInPool(next)) throw next;
markBad(base);
reportFailure(base, next);
logHostFailure(provider, base, next);
diff --git a/tests/aggregator-upstream-failover.test.js b/tests/aggregator-upstream-failover.test.js
index 766a2e4dc..ba8a82c23 100644
--- a/tests/aggregator-upstream-failover.test.js
+++ b/tests/aggregator-upstream-failover.test.js
@@ -7,8 +7,9 @@
// even though the platform keeps a priority-ordered pool of six more endpoints
// for exactly this case (api/_lib/solana/connection.js).
//
-// The rule under test: a retryable failure (429, unreachable, 5xx) walks the
-// provider's alternate hosts; a caller-fault failure (4xx) never does; and a
+// The rule under test: a retryable failure (429, unreachable, 5xx, and a
+// pooled host's own 401/403) walks the provider's alternate hosts; a
+// caller-fault failure (any other 4xx) never does; and a
// provider that declares no alternates behaves exactly as before, with one
// attempt and no extra cost.
//
@@ -111,6 +112,48 @@ describe('aggregator upstream failover', () => {
expect(seen).toEqual(['primary.invalid']);
});
+ it('falls over when a pooled host refuses with 401 or 403', async () => {
+ for (const status of [401, 403]) {
+ resetUpstreamHealth();
+ const seen = [];
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (input) => {
+ seen.push(hostOf(input));
+ return seen.length === 1 ? reply(status, { error: 'key rejected' }) : reply(200, { ok: true });
+ }),
+ );
+
+ const out = await executeUpstream({
+ provider: providerWithPool(['https://backup-a.invalid']),
+ endpoint,
+ query: {},
+ apiKey: null,
+ });
+
+ expect(out).toEqual({ ok: true });
+ expect(seen).toEqual(['primary.invalid', 'backup-a.invalid']);
+ }
+ });
+
+ it('surfaces a 403 unchanged for a provider with no alternates', async () => {
+ const seen = [];
+ vi.stubGlobal(
+ 'fetch',
+ vi.fn(async (input) => {
+ seen.push(hostOf(input));
+ return reply(403, { error: 'forbidden' });
+ }),
+ );
+
+ const provider = { id: 'single', name: 'Single host', base: 'https://primary.invalid', requiresKey: false, applyKey: () => {} };
+ await expect(executeUpstream({ provider, endpoint, query: {}, apiKey: null })).rejects.toMatchObject({
+ status: 403,
+ code: 'upstream_error',
+ });
+ expect(seen).toEqual(['primary.invalid']);
+ });
+
it('surfaces the failure once the whole pool is exhausted', async () => {
vi.stubGlobal('fetch', vi.fn(async () => reply(429, { error: 'slow down' })));