Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): stop the Vertex billing hold and HF credit exhaustion from re-paying their own timeout on every image request

nirholas9d2cd957d37b
diff --git a/api/_lib/forge-reference-image.js b/api/_lib/forge-reference-image.js
index 6fbb1e1b8..169dfb5da 100644
--- a/api/_lib/forge-reference-image.js
+++ b/api/_lib/forge-reference-image.js
@@ -59,9 +59,10 @@ import { getGcpAccessToken } from './gcp-auth.js';
import { persistImageBase64 } from './image-persist.js';
import { subjectNegativePrompt } from '../forge-enhance.js';
import { isProviderRefusal } from './ai-image-lanes.js';
-import { textToImage, ladderBudgetMs, laneTimeoutMs } from '../_mcp3d/text-to-image.js';
+import { textToImage, ladderBudgetMs, laneTimeoutMs, VERTEX_IMAGE_COOLDOWN_KEY } from '../_mcp3d/text-to-image.js';
import { parseJsonLoose } from './vision.js';
import { getRedis } from './redis.js';
+import { markProviderCooldown, providersInCooldown, AUTH_COOLDOWN_SECONDS } from './provider-health.js';
// Reference-set cache: the SAME enhanced prompt (already director-rewritten,
// so two users typing differently-worded ideas that converge on one directed
@@ -376,7 +377,15 @@ export async function generateReferenceImage(
return { ...cached, lane: 'vertex-reference-cached' };
}
- if (vertexImageEnabled()) {
+ // A project-wide billing hold or safety block fails this lane identically on
+ // every call until ops fixes it. Share the breaker with text-to-image.js's
+ // own Vertex rung (VERTEX_IMAGE_COOLDOWN_KEY): either call site marking it
+ // silences both for AUTH_COOLDOWN_SECONDS, instead of this step paying the
+ // full Vertex timeout only to fall through to a textToImage() that pays it
+ // again for the exact same failure.
+ const vertexCooling =
+ vertexImageEnabled() && (await providersInCooldown([VERTEX_IMAGE_COOLDOWN_KEY])).has(VERTEX_IMAGE_COOLDOWN_KEY);
+ if (vertexImageEnabled() && !vertexCooling) {
try {
const instruction = buildReferenceInstruction(prompt, negatives);
const leaderMs = laneTimeoutMs(VERTEX_REQUEST_CEILING_MS, { budgetMs: budget, remainingMs: remaining(), laneRemains: true });
@@ -426,6 +435,10 @@ export async function generateReferenceImage(
// Any Vertex failure (unconfigured token, safety block, throttle, tier
// change) degrades to the existing chain - the reference image still gets
// made, just on the prior path. Warn, don't error.
+ const status = err?.providerStatus;
+ if (status === 401 || status === 402 || status === 403) {
+ markProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});
+ }
console.warn(`[forge-reference-image] vertex lane failed, falling through: ${err?.message}`);
}
}
diff --git a/api/_mcp3d/text-to-image.js b/api/_mcp3d/text-to-image.js
index 033f0d30c..b1e26e374 100644
--- a/api/_mcp3d/text-to-image.js
+++ b/api/_mcp3d/text-to-image.js
@@ -21,7 +21,7 @@
// free on the NVIDIA NIM catalog as base64 JPEG (no poll; returns inline).
// Replicate backstop: black-forest-labs/flux-schnell — same family, $0.003/run.
-import { markProviderCooldown, providersInCooldown } from '../_lib/provider-health.js';
+import { markProviderCooldown, providersInCooldown, AUTH_COOLDOWN_SECONDS } from '../_lib/provider-health.js';
import { fetchUpstream } from '../_lib/upstream-fetch.js';
import { reserveProviderRateSlot, SCALE_LIMITS } from '../_lib/forge-scale.js';
import { persistImageBase64, persistImageBytes, looksLikeImageBytes } from '../_lib/image-persist.js';
@@ -81,6 +81,23 @@ async function pollReplicatePrediction(getUrl, token, { timeoutMs = REPLICATE_PO
const NIM_FLUX_COOLDOWN_KEY = 'forge-nim-flux';
const NIM_FLUX_COOLDOWN_SECONDS = 60;
+// Same breaker, but for the Vertex image lane and the two HF-routed providers,
+// keyed on an auth/billing fault (401/402/403) rather than a transient one: a
+// dunning hold or an exhausted credit balance will not clear itself in 60s, so
+// marking it 'health'-length just burns the full request timeout again on the
+// very next call. AUTH_COOLDOWN_SECONDS (5 min) matches the same breaker
+// already used for Vertex in forge-quality-gate.js and for every LLM lane in
+// chat.js/concierge.js. VERTEX_IMAGE_COOLDOWN_KEY is exported so
+// forge-reference-image.js's own, separate Vertex-reference attempt shares the
+// same breaker: one 403 should silence Vertex for both call sites, not just
+// this one.
+export const VERTEX_IMAGE_COOLDOWN_KEY = 'forge-vertex-image';
+const HF_IMAGE_COOLDOWN_PREFIX = 'forge-hf-image-';
+function isAuthFault(err) {
+ const status = err?.providerStatus;
+ return status === 401 || status === 402 || status === 403;
+}
+
// Whether a thrown nimFluxImage error means the lane itself is degraded (timeout,
// unreachable, throttle, or 5xx) — worth a cooldown — as opposed to a 4xx client
// fault (bad input / key), which a cooldown would wrongly punish a healthy lane for.
@@ -595,6 +612,12 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false
try {
const { generateImage, isConfigured } = await import('./vertex-imagen.js');
if (!isConfigured()) return null;
+ // A lane parked on a billing/auth fault beats nothing only when it is the
+ // only lane left; with a fallback behind it, skip the known-dead call
+ // instead of re-paying its full timeout on every request.
+ if (laneRemains && (await providersInCooldown([VERTEX_IMAGE_COOLDOWN_KEY])).has(VERTEX_IMAGE_COOLDOWN_KEY)) {
+ return null;
+ }
if (!canTry()) {
if (!laneRemains) throw budgetExhausted(budget);
return null;
@@ -605,6 +628,9 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false
),
);
} catch (err) {
+ if (isAuthFault(err)) {
+ markProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});
+ }
if (!laneRemains) throw err;
console.warn(`vertex imagen failed, falling back: ${err?.message}`);
return null;
@@ -657,8 +683,15 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false
// ── Hugging Face routed providers (fal-ai, then nscale) ──────────────────
if (hasHf) {
+ const hfCooling = await providersInCooldown(
+ HF_IMAGE_PROVIDERS.map((provider) => `${HF_IMAGE_COOLDOWN_PREFIX}${provider.id}`),
+ );
for (const provider of HF_IMAGE_PROVIDERS) {
if (!canTry()) break;
+ // An out-of-credit or revoked-key provider parked in cooldown: the other
+ // HF rung (and everything downstream) is independent, so skip only this
+ // one rung instead of re-paying its timeout every request.
+ if (hfCooling.has(`${HF_IMAGE_COOLDOWN_PREFIX}${provider.id}`)) continue;
try {
return logImageProvider(
await hfRoutedImage(provider, prompt, aspectRatio, hasSeed ? seed : undefined, {
@@ -666,6 +699,9 @@ export async function textToImage(prompt, { aspectRatio = '1:1', skipNim = false
}),
);
} catch (err) {
+ if (isAuthFault(err)) {
+ markProviderCooldown(`${HF_IMAGE_COOLDOWN_PREFIX}${provider.id}`, AUTH_COOLDOWN_SECONDS, 'auth').catch(() => {});
+ }
console.warn(`hf ${provider.id} failed, falling back: ${err?.message}`);
}
}
diff --git a/tests/text-to-image-provider-cooldown.test.js b/tests/text-to-image-provider-cooldown.test.js
new file mode 100644
index 000000000..8d0bda5b9
--- /dev/null
+++ b/tests/text-to-image-provider-cooldown.test.js
@@ -0,0 +1,118 @@
+/**
+ * Root cause of the 2026-10-12 /api/3d/generate 502 storm: the Vertex image
+ * lane (billing dunning hold, 403 on every call) and the two HF-routed
+ * providers (fal-ai/nscale, 402 "no remaining credits") had no circuit
+ * breaker, unlike NIM FLUX. Every single request re-paid the full timeout of
+ * each known-dead lane before ever reaching keyless Pollinations, which often
+ * blew the shared ladder budget and surfaced as a 502 instead of a served
+ * image. This pins that an auth/billing fault (401/402/403) parks the lane
+ * for AUTH_COOLDOWN_SECONDS so the next request skips straight past it.
+ */
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+
+process.env.NVIDIA_API_KEY = 'nim-test-key';
+process.env.HF_TOKEN = 'hf-token';
+process.env.GOOGLE_CLOUD_PROJECT = 'test-project';
+delete process.env.REPLICATE_API_TOKEN;
+delete process.env.LIVEPEER_FEDERATION_ENABLED;
+delete process.env.TEXT_TO_IMAGE_BUDGET_MS;
+delete process.env.VERTEX_IMAGEN_FIRST;
+
+vi.mock('../api/_lib/redis.js', () => ({ getRedis: () => null }));
+vi.mock('../api/_lib/forge-scale.js', () => ({ reserveProviderRateSlot: async () => ({ ok: true, waitMs: 0 }), SCALE_LIMITS: {} }));
+vi.mock('../api/_providers/livepeer.js', () => ({ livepeerFederationEnabled: () => false, livepeerTextToImage: async () => { throw new Error('unused'); } }));
+
+const vertex = vi.hoisted(() => ({ calls: 0 }));
+vi.mock('../api/_mcp3d/vertex-imagen.js', () => ({
+ isConfigured: () => true,
+ generateImage: async () => {
+ vertex.calls++;
+ throw Object.assign(new Error('Vertex AI image generation returned 403: Lightning dunning decision is deny'), {
+ providerStatus: 403,
+ });
+ },
+}));
+
+const persisted = vi.hoisted(() => ({ calls: 0 }));
+vi.mock('../api/_lib/image-persist.js', async (importOriginal) => {
+ const real = await importOriginal();
+ return {
+ ...real,
+ persistImageBytes: async () => { persisted.calls++; return `https://cdn.test/img-${persisted.calls}.png`; },
+ persistImageBase64: async () => { persisted.calls++; return `https://cdn.test/img-${persisted.calls}.png`; },
+ };
+});
+
+// Not mocking ../api/_lib/provider-health.js: the whole point of this test is
+// to exercise the REAL cooldown read/write against the in-memory cache
+// fallback (getRedis -> null), the same way tests/forge-reference-image-budget
+// .test.js already does for the Vertex-budget regression.
+
+const { textToImage, VERTEX_IMAGE_COOLDOWN_KEY } = await import('../api/_mcp3d/text-to-image.js');
+const { clearProviderCooldown } = await import('../api/_lib/provider-health.js');
+
+const JPG = new Uint8Array([0xff, 0xd8, 0xff, 0xe0, 1, 2, 3]);
+const json = (status, body) => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } });
+const bytes = (b, type) => new Response(b, { status: 200, headers: { 'content-type': type } });
+
+const lane = {};
+const hits = [];
+globalThis.fetch = vi.fn(async (url) => {
+ const u = String(url);
+ hits.push(u);
+ if (u.includes('ai.api.nvidia.com')) return lane.nim();
+ if (u.includes('router.huggingface.co/fal-ai')) return lane.fal();
+ if (u.includes('router.huggingface.co/nscale')) return lane.nscale();
+ if (u.includes('image.pollinations.ai')) return lane.pollinations();
+ throw new Error(`unexpected fetch ${u}`);
+});
+
+beforeEach(async () => {
+ hits.length = 0;
+ vertex.calls = 0;
+ persisted.calls = 0;
+ lane.nim = () => json(502, { error: 'gateway' });
+ lane.fal = () => json(402, { error: 'You have no remaining credits.' });
+ lane.nscale = () => json(402, { error: 'You have no remaining credits.' });
+ lane.pollinations = () => bytes(JPG, 'image/jpeg');
+ // Each test starts with every breaker clear: these keys live in the real
+ // (in-memory, since getRedis -> null) shared cache, so a cooldown one test
+ // sets would otherwise leak into the next.
+ await Promise.all([
+ clearProviderCooldown(VERTEX_IMAGE_COOLDOWN_KEY),
+ clearProviderCooldown('forge-hf-image-fal-ai'),
+ clearProviderCooldown('forge-hf-image-nscale'),
+ ]);
+});
+
+describe('a billing-hold Vertex lane cools down after one auth fault', () => {
+ it('tries Vertex once, then skips it on the next request while the cooldown holds', async () => {
+ const first = await textToImage('a fox', { budgetMs: 20_000 });
+ expect(first.model).toBe('pollinations/flux');
+ expect(vertex.calls).toBe(1);
+
+ const second = await textToImage('a fox', { budgetMs: 20_000 });
+ expect(second.model).toBe('pollinations/flux');
+ // Without the breaker this would be 2: the known-dead Vertex lane would be
+ // re-dialed (and re-timed-out) on every single request.
+ expect(vertex.calls).toBe(1);
+ });
+});
+
+describe('an out-of-credit HF provider cools down after one 402, independently of its sibling', () => {
+ it('tries fal-ai once, then skips it (not nscale) on the next request', async () => {
+ lane.nscale = () => json(200, { data: [{ b64_json: Buffer.from(JPG).toString('base64') }] });
+
+ const first = await textToImage('a fox', { budgetMs: 20_000 });
+ expect(first.model).toBe('hf/nscale/flux-schnell');
+ expect(hits.filter((u) => u.includes('fal-ai'))).toHaveLength(1);
+ expect(hits.filter((u) => u.includes('nscale'))).toHaveLength(1);
+
+ hits.length = 0;
+ const second = await textToImage('a fox', { budgetMs: 20_000 });
+ expect(second.model).toBe('hf/nscale/flux-schnell');
+ // Without the breaker fal-ai would be re-dialed (and re-fail) every request.
+ expect(hits.filter((u) => u.includes('fal-ai'))).toHaveLength(0);
+ expect(hits.filter((u) => u.includes('nscale'))).toHaveLength(1);
+ });
+});