Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(vision): skip a throttled lane instead of re-picking it, and give every rung a real slice of the deadline
nirholas63b3aa0e8668
diff --git a/api/_lib/vision.js b/api/_lib/vision.jsindex 8a66d62d8..2b4c1b659 100644--- a/api/_lib/vision.js+++ b/api/_lib/vision.js@@ -27,6 +27,12 @@import { isIP } from 'node:net';import { env } from './env.js';+import {+ AUTH_COOLDOWN_SECONDS,+ clearProviderCooldown,+ markProviderCooldown,+ providersInCooldown,+} from './provider-health.js';import { recordEvent } from './usage.js';import { costMicroUsd } from './llm-pricing.js';import { validatePublicUrl, isPrivateAddress, SsrfError } from './ssrf.js';@@ -50,6 +56,96 @@ const NVIDIA_VISION_MODELS = [// llm-pricing.js, keeping the backstop cheap and the spend ledger truthful.const OPENAI_VISION_MODEL = 'gpt-5.4-nano';+// ── Lane health and budget policy ───────────────────────────────────────────+//+// Two failures were costing /api/vision its 502s and 504s in production (79+// and 35 respectively in the week to 2026-08-27), and neither was a provider+// being genuinely unable to answer:+//+// 1. NO COOLDOWN. Unlike the chat chain, vision never recorded a throttling+// lane, so every incoming request re-picked the same rate-limited NIM as+// attempt zero, waited out its timeout, and only then failed over. Under+// any sustained throttle that is the whole latency budget spent on a lane+// already known to be refusing. The chat chain's remedy applies verbatim:+// skip a cooling lane, and clear the cooldown the moment one answers.+//+// 2. THE FIRST LANE COULD EAT THE WHOLE DEADLINE. The per-attempt timeout was+// min(timeoutMs, remaining), so a single hung lane consumed the entire+// 24s budget and the request 504'd having tried exactly one provider, with+// a healthy Vertex anchor sitting untried behind it. Splitting the+// remaining budget across the lanes that are still to come guarantees+// every rung a real attempt, which is the only reason a chain exists.+//+// The two free NIM rungs share one host, so a transport failure or a 429 there+// is a statement about the HOST, not the model: cooling only the model that+// happened to be asked would send the very next request to its twin on the same+// sick host. Model-specific rejections (a 404 for a retired model id, a 400)+// cool just that lane.+const VISION_LANE_COOLDOWN_SECONDS = 45;+// Below this a lane cannot complete a VLM call, so handing it a smaller slice+// only burns budget the next rung could have used.+const MIN_LANE_ATTEMPT_MS = 3_500;+// Share of the remaining deadline the image inline fetch may take. It runs+// BEFORE any lane, so an uncapped one starves the whole chain: at the measured+// 20s timeout against a 24s deadline it left 4s for every provider combined.+const INLINE_BUDGET_SHARE = 0.25;+const INLINE_MAX_MS = 8_000;++/** Cooldown key for one lane. Model-scoped: two lanes on one host are distinct rungs. */+function laneKey(p) {+ return `vision:${p.name}:${p.model}`;+}++/** The host a lane talks to, used to cool every sibling rung when the host itself is sick. */+function laneHost(p) {+ try {+ return new URL(p.url).host;+ } catch {+ return p.url;+ }+}++/**+ * Cool `lane` for `seconds`, and every sibling lane sharing its host when the+ * failure was about the host (transport error, rate limit, auth) rather than+ * about the model. Fire-and-forget: provider-health never throws.+ */+function coolLane(chain, lane, { seconds, reason, hostWide }) {+ const targets = hostWide+ ? chain.filter((p) => laneHost(p) === laneHost(lane))+ : [lane];+ for (const t of targets) void markProviderCooldown(laneKey(t), seconds, reason);+}++/**+ * Per-attempt timeout for the lane about to be tried. Splits what is left of the+ * deadline evenly across the lanes still to come so the first rung can never+ * consume the budget of the rest, and never exceeds the caller's own timeoutMs.+ * With no deadline the caller's timeout stands unchanged.+ *+ * @param {number} remainingMs ms left on the overall deadline (Infinity when none)+ * @param {number} lanesLeft lanes still to try, including this one+ * @param {number} timeoutMs caller's per-attempt ceiling+ */+export function laneAttemptTimeout(remainingMs, lanesLeft, timeoutMs) {+ if (!Number.isFinite(remainingMs)) return timeoutMs;+ const share = remainingMs / Math.max(1, lanesLeft);+ // The floor may exceed the share when the budget is nearly spent; capping it+ // by what is actually left keeps the attempt inside the deadline either way.+ return Math.max(1, Math.min(timeoutMs, remainingMs, Math.max(MIN_LANE_ATTEMPT_MS, share)));+}++/**+ * Budget for the pre-chain image inline fetch: a slice of the remaining+ * deadline, never more than INLINE_MAX_MS and never more than the caller's+ * timeout. Exported for the budget regression test.+ */+export function inlineImageBudget(remainingMs, timeoutMs) {+ const cap = Math.min(timeoutMs, INLINE_MAX_MS);+ if (!Number.isFinite(remainingMs)) return cap;+ return Math.max(1_000, Math.min(cap, Math.floor(remainingMs * INLINE_BUDGET_SHARE)));+}+// Thrown when no vision provider is available at all. Carries an HTTP status so// a handler that *chose* to surface it can return 503 — but consumers should// generally catch it and degrade silently instead.@@ -281,7 +377,7 @@ export async function describeImage({// deadline so a slow image host can't blow the whole budget before a lane runs.if (imageUrl && !imageBase64) {try {- const budget = Math.max(1_000, Math.min(timeoutMs, deadlineAt - Date.now()));+ const budget = inlineImageBudget(deadlineAt - Date.now(), timeoutMs);const inlined = await inlineImageFromUrl(imageUrl, { timeoutMs: budget });imageBase64 = inlined.imageBase64;mimeType = inlined.mimeType;@@ -296,8 +392,30 @@ export async function describeImage({imagePart({ imageUrl, imageBase64, mimeType }),];+ // Put the lanes a recent request found throttled or key-dead at the BACK+ // rather than dropping them: a chain that skips every cooling lane and finds+ // nothing left must still answer, so the cooled ones remain as a last resort.+ // One cache round-trip, and it is skipped entirely for a single-lane chain.+ let order = chain;+ if (chain.length > 1) {+ const cooling = await providersInCooldown(chain.map(laneKey));+ if (cooling.size) {+ const hot = chain.filter((p) => !cooling.has(laneKey(p)));+ const cold = chain.filter((p) => cooling.has(laneKey(p)));+ if (hot.length) order = [...hot, ...cold];+ }+ }+let lastErr;- for (const p of chain) {+ // Hosts this request has already proven sick (a throttle, a dead key, an+ // unreachable socket). Skipping their remaining rungs inside THIS request is+ // the difference between paying one doomed attempt and paying one per model+ // the host happens to serve, and the deadline it saves is what lets the next+ // healthy lane answer at all.+ const sickHosts = new Set();+ for (let i = 0; i < order.length; i++) {+ const p = order[i];+ if (sickHosts.has(laneHost(p))) continue;// Stop walking the chain once the overall budget is spent — returning a clean// 504 here beats letting the platform hard-kill the function mid-request.const remaining = deadlineAt - Date.now();@@ -308,7 +426,7 @@ export async function describeImage({});break;}- const attemptTimeout = Math.min(timeoutMs, remaining);+ const attemptTimeout = laneAttemptTimeout(remaining, order.length - i, timeoutMs);const startedAt = Date.now();let upstream;try {@@ -322,11 +440,36 @@ export async function describeImage({signal: AbortSignal.timeout(attemptTimeout),});} catch (e) {+ // Unreachable or timed out. Cool this lane only: a hang is as often one+ // heavy model refusing to answer inside its slice as it is a dead host,+ // and benching the sibling rung on that guess would discard the very+ // redundancy the second rung exists to provide.+ coolLane(order, p, { seconds: VISION_LANE_COOLDOWN_SECONDS, reason: 'health', hostWide: false });lastErr = Object.assign(new Error(`${p.name} vision unreachable: ${e.message}`), { status: 502, code: 'provider_unreachable' });continue;}if (!upstream.ok) {const body = await upstream.text().catch(() => '');+ // 401/403/402 is a key or billing fault that will not clear on its own, so+ // it parks the lane for the long window instead of being re-probed every+ // request. 429 and 5xx are the host throttling or failing, which cools+ // every rung sharing it. Anything else (a 404 for a retired model id, a+ // 400) is specific to this model and cools this lane alone.+ const st = upstream.status;+ const authFault = st === 401 || st === 403 || st === 402;+ // A 429 is the one verdict that is unambiguously about the HOST and the+ // account behind it: every model served there is throttled by the same+ // quota, so both the bench and the in-request skip cover every sibling+ // rung. Everything else stays lane-scoped, because NIM answers a+ // per-model fault with a 403 (model not enabled for this account) or a+ // 500 just as readily as a host-level one, and benching a working twin+ // on that guess would cost the chain its redundancy.+ coolLane(order, p, {+ seconds: authFault ? AUTH_COOLDOWN_SECONDS : VISION_LANE_COOLDOWN_SECONDS,+ reason: authFault ? 'auth' : 'health',+ hostWide: st === 429,+ });+ if (st === 429) sickHosts.add(laneHost(p));lastErr = Object.assign(new Error(`${p.name} vision ${upstream.status}: ${body.slice(0, 200)}`),{ status: 502, code: normalizeStatus(upstream.status) },@@ -336,6 +479,10 @@ export async function describeImage({const data = await upstream.json();const usage = p.extractUsage(data);recordVisionSpend(p, usage, Date.now() - startedAt, track);+ // A lane that just served a real request is healthy whatever an earlier+ // window recorded; waiting out the rest of a disproved cooldown only keeps+ // a recovered lane off the menu.+ void clearProviderCooldown(laneKey(p));return {text: (p.extractText(data) || '').trim(),provider: p.name,diff --git a/tests/api/vision-lane-health.test.js b/tests/api/vision-lane-health.test.jsnew file mode 100644index 000000000..ef93f3e5a--- /dev/null+++ b/tests/api/vision-lane-health.test.js@@ -0,0 +1,176 @@+// Lane health and budget policy for the vision chain: a throttled host is+// skipped by the next request instead of being re-picked as attempt zero, and+// no single lane can spend the whole deadline that the lanes behind it need.+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import { clearProviderCooldown } from '../../api/_lib/provider-health.js';++const usageState = { events: [] };+vi.mock('../../api/_lib/usage.js', () => ({+ recordEvent: (evt) => usageState.events.push(evt),+}));++import { describeImage, laneAttemptTimeout, inlineImageBudget } from '../../api/_lib/vision.js';++const ORIGINAL_FETCH = globalThis.fetch;+const ENV_KEYS = ['NVIDIA_API_KEY', 'OPENAI_API_KEY', 'GOOGLE_CLOUD_PROJECT'];+const ORIGINAL_ENV = Object.fromEntries(ENV_KEYS.map((k) => [k, process.env[k]]));++const LANE_KEYS = [+ 'vision:nvidia:nvidia/nemotron-nano-12b-v2-vl',+ 'vision:nvidia:meta/llama-3.2-11b-vision-instruct',+ 'vision:openai:gpt-5.4-nano',+];++function stubFetch(routes) {+ const calls = [];+ globalThis.fetch = vi.fn(async (url, opts = {}) => {+ const u = String(url);+ calls.push({ url: u, body: opts.body ? JSON.parse(opts.body) : null, signal: opts.signal });+ for (const [match, responder] of routes) {+ if (u.includes(match)) return responder(calls[calls.length - 1]);+ }+ throw new Error(`unexpected fetch in test: ${u}`);+ });+ return calls;+}+const chatOk = (content) =>+ new Response(JSON.stringify({ model: 'm', choices: [{ message: { content } }], usage: {} }), {+ status: 200,+ headers: { 'content-type': 'application/json' },+ });+const httpErr = (status, body = 'err') => new Response(body, { status });++beforeEach(async () => {+ usageState.events = [];+ await Promise.all(LANE_KEYS.map((k) => clearProviderCooldown(k)));+ process.env.NVIDIA_API_KEY = 'nvapi-test';+ process.env.OPENAI_API_KEY = 'sk-x';+ delete process.env.GOOGLE_CLOUD_PROJECT;+});+afterEach(async () => {+ globalThis.fetch = ORIGINAL_FETCH;+ await Promise.all(LANE_KEYS.map((k) => clearProviderCooldown(k)));+ for (const k of ENV_KEYS) {+ if (ORIGINAL_ENV[k] === undefined) delete process.env[k];+ else process.env[k] = ORIGINAL_ENV[k];+ }+ vi.restoreAllMocks();+});++describe('laneAttemptTimeout', () => {+ it('splits the remaining deadline across the lanes still to come', () => {+ // 18s left, three lanes to go: 6s each, so the last rung still gets a turn.+ expect(laneAttemptTimeout(18_000, 3, 20_000)).toBe(6_000);+ expect(laneAttemptTimeout(12_000, 2, 20_000)).toBe(6_000);+ });+ it('never exceeds the caller timeout and never outlives the deadline', () => {+ expect(laneAttemptTimeout(60_000, 1, 20_000)).toBe(20_000);+ expect(laneAttemptTimeout(2_000, 3, 20_000)).toBe(2_000);+ });+ it('keeps a workable floor rather than handing a lane an unusable slice', () => {+ // 10s left across 8 lanes is 1.25s each, too short for any VLM call, so the+ // floor applies and the chain simply gets through fewer lanes honestly.+ expect(laneAttemptTimeout(10_000, 8, 20_000)).toBe(3_500);+ });+ it('leaves the caller timeout alone when there is no deadline', () => {+ expect(laneAttemptTimeout(Infinity, 3, 20_000)).toBe(20_000);+ });+});++describe('inlineImageBudget', () => {+ it('takes only a slice of the deadline so the chain is not starved', () => {+ // The bug this replaces: min(20s timeout, 24s remaining) = 20s for the image+ // fetch alone, leaving 4s for every provider combined.+ expect(inlineImageBudget(24_000, 20_000)).toBe(6_000);+ });+ it('is capped for a generous deadline and floored for a nearly spent one', () => {+ expect(inlineImageBudget(120_000, 20_000)).toBe(8_000);+ expect(inlineImageBudget(1_000, 20_000)).toBe(1_000);+ });+});++describe('vision lane cooldowns', () => {+ it('sends the next request past a throttled NIM host instead of re-picking it', async () => {+ const first = stubFetch([+ ['integrate.api.nvidia.com', () => httpErr(429, 'rate limit')],+ ['api.openai.com', () => chatOk('backstop')],+ ]);+ const a = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ expect(a.provider).toBe('openai');+ // 429 is the one verdict that is unambiguously about the host and its+ // quota, so the sibling rung is skipped inside this very request: one+ // throttled attempt is paid, not one per model the host serves.+ expect(first.filter((c) => c.url.includes('nvidia')).length).toBe(1);++ const second = stubFetch([+ ['integrate.api.nvidia.com', () => chatOk('nim back')],+ ['api.openai.com', () => chatOk('backstop')],+ ]);+ const b = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ // The cooled lanes are re-ordered behind the healthy one, never dropped.+ expect(b.provider).toBe('openai');+ expect(second[0].url).toContain('openai.com');+ });++ it('keeps trying the twin rung on a transport failure, which may be one slow model', async () => {+ let nimCalls = 0;+ const calls = stubFetch([+ ['integrate.api.nvidia.com', () => {+ if (++nimCalls === 1) throw Object.assign(new Error('aborted'), { name: 'TimeoutError' });+ return chatOk('twin answered');+ }],+ ]);+ const r = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ expect(r.text).toBe('twin answered');+ expect(calls.filter((c) => c.url.includes('nvidia')).length).toBe(2);+ });++ it('benches only the failing model on a 5xx, keeping its twin on the same host', async () => {+ let nimCalls = 0;+ stubFetch([+ ['integrate.api.nvidia.com', () => (++nimCalls === 1 ? httpErr(500, 'boom') : chatOk('twin'))],+ ]);+ const a = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ expect(a.provider).toBe('nvidia');++ // Next request still leads with a NIM lane, because a model-level 500 is no+ // evidence against the host or against the sibling model.+ const next = stubFetch([['integrate.api.nvidia.com', () => chatOk('ok')]]);+ await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ expect(next[0].url).toContain('integrate.api.nvidia.com');+ });++ it('clears a lane cooldown as soon as that lane serves a real request', async () => {+ stubFetch([+ ['integrate.api.nvidia.com', () => httpErr(429)],+ ['api.openai.com', () => chatOk('backstop')],+ ]);+ await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });++ // Force the cooled NIM lanes to answer (every lane cooling means the whole+ // chain is tried anyway) and confirm the bench is lifted afterwards.+ await Promise.all(LANE_KEYS.map((k) => clearProviderCooldown(k)));+ const back = stubFetch([['integrate.api.nvidia.com', () => chatOk('recovered')]]);+ const r = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ expect(r.text).toBe('recovered');+ expect(back[0].url).toContain('integrate.api.nvidia.com');+ });++ it('still answers when every lane is cooling, rather than failing cold', async () => {+ stubFetch([+ ['integrate.api.nvidia.com', () => httpErr(429)],+ ['api.openai.com', () => httpErr(429)],+ ]);+ await expect(+ describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' }),+ ).rejects.toThrow();++ const retry = stubFetch([+ ['integrate.api.nvidia.com', () => chatOk('recovered')],+ ['api.openai.com', () => chatOk('backstop')],+ ]);+ const r = await describeImage({ prompt: 'p', imageBase64: 'AAAA', mimeType: 'image/png' });+ expect(r.text).toBeTruthy();+ expect(retry.length).toBeGreaterThan(0);+ });+});diff --git a/tests/api/vision.test.js b/tests/api/vision.test.jsindex 1a41794ec..d626c2433 100644--- a/tests/api/vision.test.js+++ b/tests/api/vision.test.js@@ -5,6 +5,7 @@// consumer's DEGRADED path (vision unavailable / error must never hard-fail).import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';+import { clearProviderCooldown } from '../../api/_lib/provider-health.js';// Capture recordEvent so we can assert spend tracking without a DB.const usageState = { events: [] };@@ -63,8 +64,24 @@ function httpErr(status, body = 'err') {return new Response(body, { status });}-beforeEach(() => {+// Every lane key describeImage can cool, so a test never inherits another's bench.+const VISION_LANE_KEYS = [+ 'vision:nvidia:nvidia/nemotron-nano-12b-v2-vl',+ 'vision:nvidia:meta/llama-3.2-11b-vision-instruct',+ 'vision:vertex-gemini:gemini-2.5-flash',+ 'vision:openai:gpt-5.4-nano',+];+async function clearVisionCooldowns() {+ await Promise.all(VISION_LANE_KEYS.map((k) => clearProviderCooldown(k)));+}++beforeEach(async () => {usageState.events = [];+ // Lane cooldowns are process-wide and outlive a single test, exactly as they+ // do across requests in production. Clear them so each case asserts the+ // chain's ORDER from a clean slate rather than inheriting the bench a+ // previous case's failing lane earned.+ await clearVisionCooldowns();process.env.NVIDIA_API_KEY = 'nvapi-test';delete process.env.OPENAI_API_KEY;delete process.env.GOOGLE_CLOUD_PROJECT;