Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(vision): skip a throttled lane instead of re-picking it, and give every rung a real slice of the deadline

nirholas63b3aa0e8668
diff --git a/api/_lib/vision.js b/api/_lib/vision.js
index 8a66d62d8..2b4c1b659 100644
--- a/api/_lib/vision.js
+++ b/api/_lib/vision.js
@@ -27,6 +27,12 @@
import { isIP } from 'node:net';
import { env } from './env.js';
+import {
+ AUTH_COOLDOWN_SECONDS,
+ clearProviderCooldown,
+ markProviderCooldown,
+ providersInCooldown,
+} from './provider-health.js';
import { recordEvent } from './usage.js';
import { costMicroUsd } from './llm-pricing.js';
import { validatePublicUrl, isPrivateAddress, SsrfError } from './ssrf.js';
@@ -50,6 +56,96 @@ const NVIDIA_VISION_MODELS = [
// llm-pricing.js, keeping the backstop cheap and the spend ledger truthful.
const OPENAI_VISION_MODEL = 'gpt-5.4-nano';
+// ── Lane health and budget policy ───────────────────────────────────────────
+//
+// Two failures were costing /api/vision its 502s and 504s in production (79
+// and 35 respectively in the week to 2026-08-27), and neither was a provider
+// being genuinely unable to answer:
+//
+// 1. NO COOLDOWN. Unlike the chat chain, vision never recorded a throttling
+// lane, so every incoming request re-picked the same rate-limited NIM as
+// attempt zero, waited out its timeout, and only then failed over. Under
+// any sustained throttle that is the whole latency budget spent on a lane
+// already known to be refusing. The chat chain's remedy applies verbatim:
+// skip a cooling lane, and clear the cooldown the moment one answers.
+//
+// 2. THE FIRST LANE COULD EAT THE WHOLE DEADLINE. The per-attempt timeout was
+// min(timeoutMs, remaining), so a single hung lane consumed the entire
+// 24s budget and the request 504'd having tried exactly one provider, with
+// a healthy Vertex anchor sitting untried behind it. Splitting the
+// remaining budget across the lanes that are still to come guarantees
+// every rung a real attempt, which is the only reason a chain exists.
+//
+// The two free NIM rungs share one host, so a transport failure or a 429 there
+// is a statement about the HOST, not the model: cooling only the model that
+// happened to be asked would send the very next request to its twin on the same
+// sick host. Model-specific rejections (a 404 for a retired model id, a 400)
+// cool just that lane.
+const VISION_LANE_COOLDOWN_SECONDS = 45;
+// Below this a lane cannot complete a VLM call, so handing it a smaller slice
+// only burns budget the next rung could have used.
+const MIN_LANE_ATTEMPT_MS = 3_500;
+// Share of the remaining deadline the image inline fetch may take. It runs
+// BEFORE any lane, so an uncapped one starves the whole chain: at the measured
+// 20s timeout against a 24s deadline it left 4s for every provider combined.
+const INLINE_BUDGET_SHARE = 0.25;
+const INLINE_MAX_MS = 8_000;
+
+/** Cooldown key for one lane. Model-scoped: two lanes on one host are distinct rungs. */
+function laneKey(p) {
+ return `vision:${p.name}:${p.model}`;
+}
+
+/** The host a lane talks to, used to cool every sibling rung when the host itself is sick. */
+function laneHost(p) {
+ try {
+ return new URL(p.url).host;
+ } catch {
+ return p.url;
+ }
+}
+
+/**
+ * Cool `lane` for `seconds`, and every sibling lane sharing its host when the
+ * failure was about the host (transport error, rate limit, auth) rather than
+ * about the model. Fire-and-forget: provider-health never throws.
+ */
+function coolLane(chain, lane, { seconds, reason, hostWide }) {
+ const targets = hostWide
+ ? chain.filter((p) => laneHost(p) === laneHost(lane))
+ : [lane];
+ for (const t of targets) void markProviderCooldown(laneKey(t), seconds, reason);
+}
+
+/**
+ * Per-attempt timeout for the lane about to be tried. Splits what is left of the
+ * deadline evenly across the lanes still to come so the first rung can never
+ * consume the budget of the rest, and never exceeds the caller's own timeoutMs.
+ * With no deadline the caller's timeout stands unchanged.
+ *
+ * @param {number} remainingMs ms left on the overall deadline (Infinity when none)
+ * @param {number} lanesLeft lanes still to try, including this one
+ * @param {number} timeoutMs caller's per-attempt ceiling
+ */
+export function laneAttemptTimeout(remainingMs, lanesLeft, timeoutMs) {
+ if (!Number.isFinite(remainingMs)) return timeoutMs;
+ const share = remainingMs / Math.max(1, lanesLeft);
+ // The floor may exceed the share when the budget is nearly spent; capping it
+ // by what is actually left keeps the attempt inside the deadline either way.
+ return Math.max(1, Math.min(timeoutMs, remainingMs, Math.max(MIN_LANE_ATTEMPT_MS, share)));
+}
+
+/**
+ * Budget for the pre-chain image inline fetch: a slice of the remaining
+ * deadline, never more than INLINE_MAX_MS and never more than the caller's
+ * timeout. Exported for the budget regression test.
+ */
+export function inlineImageBudget(remainingMs, timeoutMs) {
+ const cap = Math.min(timeoutMs, INLINE_MAX_MS);
+ if (!Number.isFinite(remainingMs)) return cap;
+ return Math.max(1_000, Math.min(cap, Math.floor(remainingMs * INLINE_BUDGET_SHARE)));
+}
+
// Thrown when no vision provider is available at all. Carries an HTTP status so
// a handler that *chose* to surface it can return 503 — but consumers should
// generally catch it and degrade silently instead.
@@ -281,7 +377,7 @@ export async function describeImage({
// deadline so a slow image host can't blow the whole budget before a lane runs.
if (imageUrl && !imageBase64) {
try {
- const budget = Math.max(1_000, Math.min(timeoutMs, deadlineAt - Date.now()));
+ const budget = inlineImageBudget(deadlineAt - Date.now(), timeoutMs);
const inlined = await inlineImageFromUrl(imageUrl, { timeoutMs: budget });
imageBase64 = inlined.imageBase64;
mimeType = inlined.mimeType;
@@ -296,8 +392,30 @@ export async function describeImage({
imagePart({ imageUrl, imageBase64, mimeType }),
];
+ // Put the lanes a recent request found throttled or key-dead at the BACK
+ // rather than dropping them: a chain that skips every cooling lane and finds
+ // nothing left must still answer, so the cooled ones remain as a last resort.
+ // One cache round-trip, and it is skipped entirely for a single-lane chain.
+ let order = chain;
+ if (chain.length > 1) {
+ const cooling = await providersInCooldown(chain.map(laneKey));
+ if (cooling.size) {
+ const hot = chain.filter((p) => !cooling.has(laneKey(p)));
+ const cold = chain.filter((p) => cooling.has(laneKey(p)));
+ if (hot.length) order = [...hot, ...cold];
+ }
+ }
+
let lastErr;
- for (const p of chain) {
+ // Hosts this request has already proven sick (a throttle, a dead key, an
+ // unreachable socket). Skipping their remaining rungs inside THIS request is
+ // the difference between paying one doomed attempt and paying one per model
+ // the host happens to serve, and the deadline it saves is what lets the next
+ // healthy lane answer at all.
+ const sickHosts = new Set();
+ for (let i = 0; i < order.length; i++) {
+ const p = order[i];
+ if (sickHosts.has(laneHost(p))) continue;
// Stop walking the chain once the overall budget is spent — returning a clean
// 504 here beats letting the platform hard-kill the function mid-request.
const remaining = deadlineAt - Date.now();
@@ -308,7 +426,7 @@ export async function describeImage({
});
break;
}
- const attemptTimeout = Math.min(timeoutMs, remaining);
+ const attemptTimeout = laneAttemptTimeout(remaining, order.length - i, timeoutMs);
const startedAt = Date.now();
let upstream;
try {
@@ -322,11 +440,36 @@ export async function describeImage({
signal: AbortSignal.timeout(attemptTimeout),
});
} catch (e) {
+ // Unreachable or timed out. Cool this lane only: a hang is as often one
+ // heavy model refusing to answer inside its slice as it is a dead host,
+ // and benching the sibling rung on that guess would discard the very
+ // redundancy the second rung exists to provide.
+ coolLane(order, p, { seconds: VISION_LANE_COOLDOWN_SECONDS, reason: 'health', hostWide: false });
lastErr = Object.assign(new Error(`${p.name} vision unreachable: ${e.message}`), { status: 502, code: 'provider_unreachable' });
continue;
}
if (!upstream.ok) {
const body = await upstream.text().catch(() => '');
+ // 401/403/402 is a key or billing fault that will not clear on its own, so
+ // it parks the lane for the long window instead of being re-probed every
+ // request. 429 and 5xx are the host throttling or failing, which cools
+ // every rung sharing it. Anything else (a 404 for a retired model id, a
+ // 400) is specific to this model and cools this lane alone.
+ const st = upstream.status;
+ const authFault = st === 401 || st === 403 || st === 402;
+ // A 429 is the one verdict that is unambiguously about the HOST and the
+ // account behind it: every model served there is throttled by the same
+ // quota, so both the bench and the in-request skip cover every sibling
+ // rung. Everything else stays lane-scoped, because NIM answers a
+ // per-model fault with a 403 (model not enabled for this account) or a
+ // 500 just as readily as a host-level one, and benching a working twin
+ // on that guess would cost the chain its redundancy.
+ coolLane(order, p, {
+ seconds: authFault ? AUTH_COOLDOWN_SECONDS : VISION_LANE_COOLDOWN_SECONDS,
+ reason: authFault ? 'auth' : 'health',
+ hostWide: st === 429,
+ });
+ if (st === 429) sickHosts.add(laneHost(p));
lastErr = Object.assign(
new Error(`${p.name} vision ${upstream.status}: ${body.slice(0, 200)}`),
{ status: 502, code: normalizeStatus(upstream.status) },
@@ -336,6 +479,10 @@ export async function describeImage({
const data = await upstream.json();
const usage = p.extractUsage(data);
recordVisionSpend(p, usage, Date.now() - startedAt, track);
+ // A lane that just served a real request is healthy whatever an earlier
+ // window recorded; waiting out the rest of a disproved cooldown only keeps
+ // a recovered lane off the menu.
+ void clearProviderCooldown(laneKey(p));
return {
text: (p.extractText(data) || '').trim(),
provider: p.name,
diff --git a/tests/api/vision-lane-health.test.js b/tests/api/vision-lane-health.test.js
new file mode 100644
index 000000000..ef93f3e5a
--- /dev/null
+++ b/tests/api/vision-lane-health.test.js
@@ -0,0 +1,176 @@
+// Lane health and budget policy for the vision chain: a throttled host is
+// skipped by the next request instead of being re-picked as attempt zero, and
+// no single lane can spend the whole deadline that the lanes behind it need.
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { clearProviderCooldown } from '../../api/_lib/provider-health.js';
+
+const usageState = { events: [] };
+vi.mock('../../api/_lib/usage.js', () => ({
+ recordEvent: (evt) => usageState.events.push(evt),
+}));
+
+import { describeImage, laneAttemptTimeout, inlineImageBudget } from '../../api/_lib/vision.js';
+
+const ORIGINAL_FETCH = globalThis.fetch;
+const ENV_KEYS = ['NVIDIA_API_KEY', 'OPENAI_API_KEY', 'GOOGLE_CLOUD_PROJECT'];
+const ORIGINAL_ENV = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));
+
+const LANE_KEYS = [
+ 'vision:nvidia:nvidia/nemotron-nano-12b-v2-vl',
+ 'vision:nvidia:meta/llama-3.2-11b-vision-instruct',
+ 'vision:openai:gpt-5.4-nano',
+];
+
+function stubFetch(routes) {
+ const calls = [];
+ globalThis.fetch = vi.fn(async (url, opts = {}) => {
+ const u = String(url);
+ calls.push({ url: u, body: opts.body ? JSON.parse(opts.body) : null, signal: opts.signal });
+ for (const [match, responder] of routes) {
+ if (u.includes(match)) return responder(calls[calls.length - 1]);
+ }
+ throw new Error(`unexpected fetch in test: ${u}`);
+ });
+ return calls;
+}
+const chatOk = (content) =>
+ new Response(JSON.stringify({ model: 'm', choices: [{ message: { content } }], usage: {} }), {
+ status: 200,
+ headers: { 'content-type': 'application/json' },
+ });
+const httpErr = (status, body = 'err') => new Response(body, { status });
+
+beforeEach(async () => {
+ usageState.events = [];
+ await Promise.all(LANE_KEYS.map((k) => clearProviderCooldown(k)));
+ process.env.NVIDIA_API_KEY = 'nvapi-test';
+ process.env.OPENAI_API_KEY = 'sk-x';
+ delete process.env.GOOGLE_CLOUD_PROJECT;
+});
+afterEach(async () => {
+ globalThis.fetch = ORIGINAL_FETCH;
+ await Promise.all(LANE_KEYS.map((k) => clearProviderCooldown(k)));
+ for (const k of ENV_KEYS) {
+ if (ORIGINAL_ENV[k] === undefined) delete process.env[k];
+ else process.env[k] = ORIGINAL_ENV[k];
+ }
+ vi.restoreAllMocks();
+});
+
+describe('laneAttemptTimeout', () => {
+ it('splits the remaining deadline across the lanes still to come', () => {
+ // 18s left, three lanes to go: 6s each, so the last rung still gets a turn.
+ expect(laneAttemptTimeout(18_000, 3, 20_000)).toBe(6_000);
+ expect(laneAttemptTimeout(12_000, 2, 20_000)).toBe(6_000);
+ });
+ it('never exceeds the caller timeout and never outlives the deadline', () => {
+ expect(laneAttemptTimeout(60_000, 1, 20_000)).toBe(20_000);
+ expect(laneAttemptTimeout(2_000, 3, 20_000)).toBe(2_000);
+ });
+ it('keeps a workable floor rather than handing a lane an unusable slice', () => {
+ // 10s left across 8 lanes is 1.25s each, too short for any VLM call, so the
+ // floor applies and the chain simply gets through fewer lanes honestly.
+ expect(laneAttemptTimeout(10_000, 8, 20_000)).toBe(3_500);
+ });
+ it('leaves the caller timeout alone when there is no deadline', () => {
+ expect(laneAttemptTimeout(Infinity, 3, 20_000)).toBe(20_000);
+ });
+});
+
+describe('inlineImageBudget', () => {
+ it('takes only a slice of the deadline so the chain is not starved', () => {
+ // The bug this replaces: min(20s timeout, 24s remaining) = 20s for the image
+ // fetch alone, leaving 4s for every provider combined.
+ expect(inlineImageBudget(24_000, 20_000)).toBe(6_000);
+ });
+ it('is capped for a generous deadline and floored for a nearly spent one', () => {
+ expect(inlineImageBudget(120_000, 20_000)).toBe(8_000);
+ expect(inlineImageBudget(1_000, 20_000)).toBe(1_000);
+ });
+});
+
+describe('vision lane cooldowns', () => {
+ it('sends the next request past a throttled NIM host instead of re-picking it', async () => {
+ const first = stubFetch([
+ ['integrate.api.nvidia.com', () => httpErr(429, 'rate limit')],
+ ['api.openai.com', () => chatOk('backstop')],
+ ]);
+ const a = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ expect(a.provider).toBe('openai');
+ // 429 is the one verdict that is unambiguously about the host and its
+ // quota, so the sibling rung is skipped inside this very request: one
+ // throttled attempt is paid, not one per model the host serves.
+ expect(first.filter((c) => c.url.includes('nvidia')).length).toBe(1);
+
+ const second = stubFetch([
+ ['integrate.api.nvidia.com', () => chatOk('nim back')],
+ ['api.openai.com', () => chatOk('backstop')],
+ ]);
+ const b = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ // The cooled lanes are re-ordered behind the healthy one, never dropped.
+ expect(b.provider).toBe('openai');
+ expect(second[0].url).toContain('openai.com');
+ });
+
+ it('keeps trying the twin rung on a transport failure, which may be one slow model', async () => {
+ let nimCalls = 0;
+ const calls = stubFetch([
+ ['integrate.api.nvidia.com', () => {
+ if (++nimCalls === 1) throw Object.assign(new Error('aborted'), { name: 'TimeoutError' });
+ return chatOk('twin answered');
+ }],
+ ]);
+ const r = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ expect(r.text).toBe('twin answered');
+ expect(calls.filter((c) => c.url.includes('nvidia')).length).toBe(2);
+ });
+
+ it('benches only the failing model on a 5xx, keeping its twin on the same host', async () => {
+ let nimCalls = 0;
+ stubFetch([
+ ['integrate.api.nvidia.com', () => (++nimCalls === 1 ? httpErr(500, 'boom') : chatOk('twin'))],
+ ]);
+ const a = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ expect(a.provider).toBe('nvidia');
+
+ // Next request still leads with a NIM lane, because a model-level 500 is no
+ // evidence against the host or against the sibling model.
+ const next = stubFetch([['integrate.api.nvidia.com', () => chatOk('ok')]]);
+ await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ expect(next[0].url).toContain('integrate.api.nvidia.com');
+ });
+
+ it('clears a lane cooldown as soon as that lane serves a real request', async () => {
+ stubFetch([
+ ['integrate.api.nvidia.com', () => httpErr(429)],
+ ['api.openai.com', () => chatOk('backstop')],
+ ]);
+ await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+
+ // Force the cooled NIM lanes to answer (every lane cooling means the whole
+ // chain is tried anyway) and confirm the bench is lifted afterwards.
+ await Promise.all(LANE_KEYS.map((k) => clearProviderCooldown(k)));
+ const back = stubFetch([['integrate.api.nvidia.com', () => chatOk('recovered')]]);
+ const r = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ expect(r.text).toBe('recovered');
+ expect(back[0].url).toContain('integrate.api.nvidia.com');
+ });
+
+ it('still answers when every lane is cooling, rather than failing cold', async () => {
+ stubFetch([
+ ['integrate.api.nvidia.com', () => httpErr(429)],
+ ['api.openai.com', () => httpErr(429)],
+ ]);
+ await expect(
+ describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' }),
+ ).rejects.toThrow();
+
+ const retry = stubFetch([
+ ['integrate.api.nvidia.com', () => chatOk('recovered')],
+ ['api.openai.com', () => chatOk('backstop')],
+ ]);
+ const r = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });
+ expect(r.text).toBeTruthy();
+ expect(retry.length).toBeGreaterThan(0);
+ });
+});
diff --git a/tests/api/vision.test.js b/tests/api/vision.test.js
index 1a41794ec..d626c2433 100644
--- a/tests/api/vision.test.js
+++ b/tests/api/vision.test.js
@@ -5,6 +5,7 @@
// consumer's DEGRADED path (vision unavailable / error must never hard-fail).
import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+import { clearProviderCooldown } from '../../api/_lib/provider-health.js';
// Capture recordEvent so we can assert spend tracking without a DB.
const usageState = { events: [] };
@@ -63,8 +64,24 @@ function httpErr(status, body = 'err') {
return new Response(body, { status });
}
-beforeEach(() => {
+// Every lane key describeImage can cool, so a test never inherits another's bench.
+const VISION_LANE_KEYS = [
+ 'vision:nvidia:nvidia/nemotron-nano-12b-v2-vl',
+ 'vision:nvidia:meta/llama-3.2-11b-vision-instruct',
+ 'vision:vertex-gemini:gemini-2.5-flash',
+ 'vision:openai:gpt-5.4-nano',
+];
+async function clearVisionCooldowns() {
+ await Promise.all(VISION_LANE_KEYS.map((k) => clearProviderCooldown(k)));
+}
+
+beforeEach(async () => {
usageState.events = [];
+ // Lane cooldowns are process-wide and outlive a single test, exactly as they
+ // do across requests in production. Clear them so each case asserts the
+ // chain's ORDER from a clean slate rather than inheriting the bench a
+ // previous case's failing lane earned.
+ await clearVisionCooldowns();
process.env.NVIDIA_API_KEY = 'nvapi-test';
delete process.env.OPENAI_API_KEY;
delete process.env.GOOGLE_CLOUD_PROJECT;