Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): stop the Vertex billing hold and HF credit exhaustion from re-paying their own timeout on every image request
nirholas9d2cd957d37b
diff --git a/api/_lib/forge-reference-image.js b/api/_lib/forge-reference-image.jsindex 6fbb1e1b8..169dfb5da 100644--- a/api/_lib/forge-reference-image.js+++ b/api/_lib/forge-reference-image.js@@ -59,9 +59,10 @@ import { getGcpAccessToken } from './gcp-auth.js';import { persistImageBase64 } from './image-persist.js';import { subjectNegativePrompt } from '../forge-enhance.js';import { isProviderRefusal } from './ai-image-lanes.js';-import { textToImage, ladderBudgetMs, laneTimeoutMs } from '../_mcp3d/text-to-image.js';+import { textToImage, ladderBudgetMs, laneTimeoutMs, VERTEX_IMAGE_COOLDOWN_KEY } from '../_mcp3d/text-to-image.js';import { parseJsonLoose } from './vision.js';import { getRedis } from './redis.js';+import { markProviderCooldown, providersInCooldown, AUTH_COOLDOWN_SECONDS } from './provider-health.js';// Reference-set cache: the SAME enhanced prompt (already director-rewritten,// so two users typing differently-worded ideas that converge on one directed@@ -376,7 +377,15 @@ export async function generateReferenceImage(return { ...cached, lane: 'vertex-reference-cached' };}- if (vertexImageEnabled()) {+ // A project-wide billing hold or safety block fails this lane identically on+ // every call until ops fixes it. Share the breaker with text-to-image.js's+ // own Vertex rung (VERTEX_IMAGE_COOLDOWN_KEY): either call site marking it+ // silences both for AUTH_COOLDOWN_SECONDS, instead of this step paying the+ // full Vertex timeout only to fall through to a textToImage() that pays it+ // again for the exact same failure.+ const vertexCooling =+ vertexImageEnabled() && (await providersInCooldown([VERTEX_IMAGE_COOLDOWN_KEY])).has(VERTEX_IMAGE_COOLDOWN_KEY);+ if (vertexImageEnabled() && !vertexCooling) {try {const instruction = buildReferenceInstruction(prompt, negatives);const leaderMs = laneTimeoutMs(VERTEX_REQUEST_CEILING_MS, { budgetMs: budget, remainingMs: remaining(), laneRemains: true });@@ -426,6 +435,10 @@ export async function generateReferenceImage(// Any Vertex failure (unconfigured token, safety block, throttle, tier// change) degrades to the existing chain - the reference image still gets// made, just on the prior path. Warn, don't error.+ const status = err?.providerStatus;+ if (status === 401 || status === 402 || status === 403) {+ markProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});+ }console.warn(`[forge-reference-image] vertex lane failed, falling through: ${err?.message}`);}}diff --git a/api/_mcp3d/text-to-image.js b/api/_mcp3d/text-to-image.jsindex 033f0d30c..b1e26e374 100644--- a/api/_mcp3d/text-to-image.js+++ b/api/_mcp3d/text-to-image.js@@ -21,7 +21,7 @@// free on the NVIDIA NIM catalog as base64 JPEG (no poll; returns inline).// Replicate backstop: black-forest-labs/flux-schnell — same family, $0.003/run.-import { markProviderCooldown, providersInCooldown } from '../_lib/provider-health.js';+import { markProviderCooldown, providersInCooldown, AUTH_COOLDOWN_SECONDS } from '../_lib/provider-health.js';import { fetchUpstream } from '../_lib/upstream-fetch.js';import { reserveProviderRateSlot, SCALE_LIMITS } from '../_lib/forge-scale.js';import { persistImageBase64, persistImageBytes, looksLikeImageBytes } from '../_lib/image-persist.js';@@ -81,6 +81,23 @@ async function pollReplicatePrediction(getUrl, token, { timeoutMs = REPLICATE_POconst NIM_FLUX_COOLDOWN_KEY = 'forge-nim-flux';const NIM_FLUX_COOLDOWN_SECONDS = 60;+// Same breaker, but for the Vertex image lane and the two HF-routed providers,+// keyed on an auth/billing fault (401/402/403) rather than a transient one: a+// dunning hold or an exhausted credit balance will not clear itself in 60s, so+// marking it 'health'-length just burns the full request timeout again on the+// very next call. AUTH_COOLDOWN_SECONDS (5 min) matches the same breaker+// already used for Vertex in forge-quality-gate.js and for every LLM lane in+// chat.js/concierge.js. VERTEX_IMAGE_COOLDOWN_KEY is exported so+// forge-reference-image.js's own, separate Vertex-reference attempt shares the+// same breaker: one 403 should silence Vertex for both call sites, not just+// this one.+export const VERTEX_IMAGE_COOLDOWN_KEY = 'forge-vertex-image';+const HF_IMAGE_COOLDOWN_PREFIX = 'forge-hf-image-';+function isAuthFault(err) {+ const status = err?.providerStatus;+ return status === 401 || status === 402 || status === 403;+}+// Whether a thrown nimFluxImage error means the lane itself is degraded (timeout,// unreachable, throttle, or 5xx) — worth a cooldown — as opposed to a 4xx client// fault (bad input / key), which a cooldown would wrongly punish a healthy lane for.@@ -595,6 +612,12 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = falsetry {const { generateImage, isConfigured } = await import('./vertex-imagen.js');if (!isConfigured()) return null;+ // A lane parked on a billing/auth fault beats nothing only when it is the+ // only lane left; with a fallback behind it, skip the known-dead call+ // instead of re-paying its full timeout on every request.+ if (laneRemains && (await providersInCooldown([VERTEX_IMAGE_COOLDOWN_KEY])).has(VERTEX_IMAGE_COOLDOWN_KEY)) {+ return null;+ }if (!canTry()) {if (!laneRemains) throw budgetExhausted(budget);return null;@@ -605,6 +628,9 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false),);} catch (err) {+ if (isAuthFault(err)) {+ markProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});+ }if (!laneRemains) throw err;console.warn(`vertex imagen failed, falling back: ${err?.message}`);return null;@@ -657,8 +683,15 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false// ── Hugging Face routed providers (fal-ai, then nscale) ──────────────────if (hasHf) {+ const hfCooling = await providersInCooldown(+ HF_IMAGE_PROVIDERS.map((provider) => `${HF_IMAGE_COOLDOWN_PREFIX}${provider.id}`),+ );for (const provider of HF_IMAGE_PROVIDERS) {if (!canTry()) break;+ // An out-of-credit or revoked-key provider parked in cooldown: the other+ // HF rung (and everything downstream) is independent, so skip only this+ // one rung instead of re-paying its timeout every request.+ if (hfCooling.has(`${HF_IMAGE_COOLDOWN_PREFIX}${provider.id}`)) continue;try {return logImageProvider(await hfRoutedImage(provider, prompt, aspectRatio, hasSeed ? seed : undefined, {@@ -666,6 +699,9 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false}),);} catch (err) {+ if (isAuthFault(err)) {+ markProviderCooldown(`${HF_IMAGE_COOLDOWN_PREFIX}${provider.id}`, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});+ }console.warn(`hf ${provider.id} failed, falling back: ${err?.message}`);}}diff --git a/tests/text-to-image-provider-cooldown.test.js b/tests/text-to-image-provider-cooldown.test.jsnew file mode 100644index 000000000..8d0bda5b9--- /dev/null+++ b/tests/text-to-image-provider-cooldown.test.js@@ -0,0 +1,118 @@+/**+ * Root cause of the 2026-10-12 /api/3d/generate 502 storm: the Vertex image+ * lane (billing dunning hold, 403 on every call) and the two HF-routed+ * providers (fal-ai/nscale, 402 "no remaining credits") had no circuit+ * breaker, unlike NIM FLUX. Every single request re-paid the full timeout of+ * each known-dead lane before ever reaching keyless Pollinations, which often+ * blew the shared ladder budget and surfaced as a 502 instead of a served+ * image. This pins that an auth/billing fault (401/402/403) parks the lane+ * for AUTH_COOLDOWN_SECONDS so the next request skips straight past it.+ */+import { describe, it, expect, vi, beforeEach } from 'vitest';++process.env.NVIDIA_API_KEY = 'nim-test-key';+process.env.HF_TOKEN = 'hf-token';+process.env.GOOGLE_CLOUD_PROJECT = 'test-project';+delete process.env.REPLICATE_API_TOKEN;+delete process.env.LIVEPEER_FEDERATION_ENABLED;+delete process.env.TEXT_TO_IMAGE_BUDGET_MS;+delete process.env.VERTEX_IMAGEN_FIRST;++vi.mock('../api/_lib/redis.js', () => ({ getRedis: () => null }));+vi.mock('../api/_lib/forge-scale.js', () => ({ reserveProviderRateSlot: async () => ({ ok: true, waitMs: 0 }), SCALE_LIMITS: {} }));+vi.mock('../api/_providers/livepeer.js', () => ({ livepeerFederationEnabled: () => false, livepeerTextToImage: async () => { throw new Error('unused'); } }));++const vertex = vi.hoisted(() => ({ calls: 0 }));+vi.mock('../api/_mcp3d/vertex-imagen.js', () => ({+ isConfigured: () => true,+ generateImage: async () => {+ vertex.calls++;+ throw Object.assign(new Error('Vertex AI image generation returned 403: Lightning dunning decision is deny'), {+ providerStatus: 403,+ });+ },+}));++const persisted = vi.hoisted(() => ({ calls: 0 }));+vi.mock('../api/_lib/image-persist.js', async (importOriginal) => {+ const real = await importOriginal();+ return {+ ...real,+ persistImageBytes: async () => { persisted.calls++; return `https://cdn.test/img-${persisted.calls}.png`; },+ persistImageBase64: async () => { persisted.calls++; return `https://cdn.test/img-${persisted.calls}.png`; },+ };+});++// Not mocking ../api/_lib/provider-health.js: the whole point of this test is+// to exercise the REAL cooldown read/write against the in-memory cache+// fallback (getRedis -> null), the same way tests/forge-reference-image-budget+// .test.js already does for the Vertex-budget regression.++const { textToImage, VERTEX_IMAGE_COOLDOWN_KEY } = await import('../api/_mcp3d/text-to-image.js');+const { clearProviderCooldown } = await import('../api/_lib/provider-health.js');++const JPG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]);+const json = (status, body) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });+const bytes = (b, type) => new Response(b, { status: 200, headers: { 'content-type': type } });++const lane = {};+const hits = [];+globalThis.fetch = vi.fn(async (url) => {+ const u = String(url);+ hits.push(u);+ if (u.includes('ai.api.nvidia.com')) return lane.nim();+ if (u.includes('router.huggingface.co/fal-ai')) return lane.fal();+ if (u.includes('router.huggingface.co/nscale')) return lane.nscale();+ if (u.includes('image.pollinations.ai')) return lane.pollinations();+ throw new Error(`unexpected fetch ${u}`);+});++beforeEach(async () => {+ hits.length = 0;+ vertex.calls = 0;+ persisted.calls = 0;+ lane.nim = () => json(502, { error: 'gateway' });+ lane.fal = () => json(402, { error: 'You have no remaining credits.' });+ lane.nscale = () => json(402, { error: 'You have no remaining credits.' });+ lane.pollinations = () => bytes(JPG, 'image/jpeg');+ // Each test starts with every breaker clear: these keys live in the real+ // (in-memory, since getRedis -> null) shared cache, so a cooldown one test+ // sets would otherwise leak into the next.+ await Promise.all([+ clearProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY),+ clearProviderCooldown('forge-hf-image-fal-ai'),+ clearProviderCooldown('forge-hf-image-nscale'),+ ]);+});++describe('a billing-hold Vertex lane cools down after one auth fault', () => {+ it('tries Vertex once, then skips it on the next request while the cooldown holds', async () => {+ const first = await textToImage('a fox', { budgetMs: 20_000 });+ expect(first.model).toBe('pollinations/flux');+ expect(vertex.calls).toBe(1);++ const second = await textToImage('a fox', { budgetMs: 20_000 });+ expect(second.model).toBe('pollinations/flux');+ // Without the breaker this would be 2: the known-dead Vertex lane would be+ // re-dialed (and re-timed-out) on every single request.+ expect(vertex.calls).toBe(1);+ });+});++describe('an out-of-credit HF provider cools down after one 402, independently of its sibling', () => {+ it('tries fal-ai once, then skips it (not nscale) on the next request', async () => {+ lane.nscale = () => json(200, { data: [{ b64_json: Buffer.from(JPG).toString('base64') }] });++ const first = await textToImage('a fox', { budgetMs: 20_000 });+ expect(first.model).toBe('hf/nscale/flux-schnell');+ expect(hits.filter((u) => u.includes('fal-ai'))).toHaveLength(1);+ expect(hits.filter((u) => u.includes('nscale'))).toHaveLength(1);++ hits.length = 0;+ const second = await textToImage('a fox', { budgetMs: 20_000 });+ expect(second.model).toBe('hf/nscale/flux-schnell');+ // Without the breaker fal-ai would be re-dialed (and re-fail) every request.+ expect(hits.filter((u) => u.includes('fal-ai'))).toHaveLength(0);+ expect(hits.filter((u) => u.includes('nscale'))).toHaveLength(1);+ });+});