Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): fail over to self-hosted Hunyuan3D or NVIDIA NIM when the free HuggingFace lane cannot serve an auto-routed request, instead of answering 502

nirholas31b416e89ea0
diff --git a/api/_lib/forge-tiers.js b/api/_lib/forge-tiers.js
index deec667ff..32919c2a6 100644
--- a/api/_lib/forge-tiers.js
+++ b/api/_lib/forge-tiers.js
@@ -602,6 +602,30 @@ export function resolveBackendIdWithHealth({ path, tier, backend, userImages = f
return defaultBackendForHealthAware(p, tierId, userImages, health, subjectClass);
}
+// Where a request goes after the free HuggingFace Spaces lane failed to serve it.
+//
+// The health-aware router picks HuggingFace whenever every self-host lane reads
+// 'down' at that instant. That happens far more often than a real outage: one
+// failed job (a bad reference image is enough) cools the Hunyuan3D lane for 90
+// seconds fleet-wide, and with self-host TRELLIS also out, HuggingFace (reported
+// 'unknown', never probed) became the pick. Its Spaces sit GPU-quota-dead for
+// most of the day, so the request answered 502 while our own Hunyuan3D worker
+// sat healthy and idle (forge_avatar, 2026-09-18). A health snapshot is a
+// routing hint; the submit is the authority. So an auto-routed request that HF
+// could not serve now tries our self-host Hunyuan3D worker, and a text prompt
+// with no self-host lane still gets the free NVIDIA NIM text lane.
+//
+// An explicitly chosen HuggingFace engine keeps its designed busy state: the
+// caller named that engine, so it is never silently swapped.
+//
+// Returns 'hunyuan3d' | 'nvidia' | null (null = surface the busy state).
+export function laneAfterHfFailure({ explicit = false, userImages = false } = {}) {
+ if (explicit) return null;
+ if (backendIsConfigured('hunyuan3d')) return 'hunyuan3d';
+ if (!userImages && backendIsConfigured('nvidia')) return 'nvidia';
+ return null;
+}
+
function readEnv(name) {
if (typeof process !== 'undefined' && process.env && process.env[name]) return process.env[name];
return null;
diff --git a/api/forge.js b/api/forge.js
index 15be852c1..ba5621e60 100644
--- a/api/forge.js
+++ b/api/forge.js
@@ -77,6 +77,7 @@ import {
estimateCredits,
preferFreeReconstruct,
backendIsConfigured,
+ laneAfterHfFailure,
backendAcceptsInlineViews,
buildCatalog,
selfhostQualityForTier,
@@ -2199,12 +2200,26 @@ async function startJob(req, res) {
})
)
return;
- return json(res, 502, {
- error: 'provider_busy',
- backend: 'huggingface',
- message:
- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',
- });
+ // Failover instead of a dead end: see laneAfterHfFailure for why an
+ // auto-routed request lands here while our own GPU worker is healthy.
+ const nextLane = laneAfterHfFailure({ explicit: backendExplicit, userImages: isImageMode });
+ if (nextLane === 'hunyuan3d') {
+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to self-hosted Hunyuan3D');
+ backendId = 'hunyuan3d';
+ provider = createGcpProvider();
+ } else if (nextLane === 'nvidia' && !nvidiaTried) {
+ nvidiaTried = true;
+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to free NVIDIA NIM');
+ if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;
+ }
+ if (backendId === 'huggingface') {
+ return json(res, 502, {
+ error: 'provider_busy',
+ backend: 'huggingface',
+ message:
+ 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',
+ });
+ }
}
// Free-first: exhaust the free reconstruct lane (HuggingFace Spaces) BEFORE
diff --git a/api/gpt-forge.js b/api/gpt-forge.js
index 1fb0e7b7d..74b0ff90a 100644
--- a/api/gpt-forge.js
+++ b/api/gpt-forge.js
@@ -83,6 +83,7 @@ import {
estimateCredits,
preferFreeReconstruct,
backendIsConfigured,
+ laneAfterHfFailure,
buildCatalog,
selfhostQualityForTier,
} from './_lib/forge-tiers.js';
@@ -2129,12 +2130,26 @@ async function startJob(req, res) {
})
)
return;
- return json(res, 502, {
- error: 'provider_busy',
- backend: 'huggingface',
- message:
- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',
- });
+ // Failover instead of a dead end: see laneAfterHfFailure for why an
+ // auto-routed request lands here while our own GPU worker is healthy.
+ const nextLane = laneAfterHfFailure({ explicit: backendExplicit, userImages: isImageMode });
+ if (nextLane === 'hunyuan3d') {
+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to self-hosted Hunyuan3D');
+ backendId = 'hunyuan3d';
+ provider = createGcpProvider();
+ } else if (nextLane === 'nvidia' && !nvidiaTried) {
+ nvidiaTried = true;
+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to free NVIDIA NIM');
+ if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;
+ }
+ if (backendId === 'huggingface') {
+ return json(res, 502, {
+ error: 'provider_busy',
+ backend: 'huggingface',
+ message:
+ 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',
+ });
+ }
}
// Free-first: exhaust the free reconstruct lane (HuggingFace Spaces) BEFORE
diff --git a/tests/api/forge-hf-failover.test.js b/tests/api/forge-hf-failover.test.js
new file mode 100644
index 000000000..dab575cb2
--- /dev/null
+++ b/tests/api/forge-hf-failover.test.js
@@ -0,0 +1,55 @@
+// laneAfterHfFailure: where an auto-routed forge request goes after the free
+// HuggingFace Spaces lane could not serve it.
+//
+// Production 2026-09-18: forge_avatar answered "Could not generate the model"
+// because the health-aware router picked HuggingFace (every self-host lane read
+// 'down': Hunyuan3D in a 90 s cooldown after one failed job, self-host TRELLIS
+// with a failed model load), the Spaces were GPU-quota-dead, and the HF branch
+// returned a 502 with no failover while our own Hunyuan3D worker was healthy.
+
+import { describe, it, expect, beforeEach, afterEach } from 'vitest';
+import { laneAfterHfFailure } from '../../api/_lib/forge-tiers.js';
+
+const ENV = ['GCP_HUNYUAN3D_URL', 'GCP_RECONSTRUCTION_KEY', 'NVIDIA_API_KEY'];
+
+describe('laneAfterHfFailure', () => {
+ let saved;
+ beforeEach(() => {
+ saved = Object.fromEntries(ENV.map((k) => [k, process.env[k]]));
+ for (const k of ENV) delete process.env[k];
+ });
+ afterEach(() => {
+ for (const k of ENV) {
+ if (saved[k] === undefined) delete process.env[k];
+ else process.env[k] = saved[k];
+ }
+ });
+
+ it('fails over to the self-hosted Hunyuan3D worker when it is configured', () => {
+ process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example.run.app';
+ process.env.GCP_RECONSTRUCTION_KEY = 'k';
+ process.env.NVIDIA_API_KEY = 'nv';
+ expect(laneAfterHfFailure({})).toBe('hunyuan3d');
+ expect(laneAfterHfFailure({ userImages: true })).toBe('hunyuan3d');
+ });
+
+ it('sends a text prompt to the free NVIDIA lane when no self-host worker is wired', () => {
+ process.env.NVIDIA_API_KEY = 'nv';
+ expect(laneAfterHfFailure({ userImages: false })).toBe('nvidia');
+ });
+
+ it('never sends a photo to the text-only NVIDIA lane', () => {
+ process.env.NVIDIA_API_KEY = 'nv';
+ expect(laneAfterHfFailure({ userImages: true })).toBeNull();
+ });
+
+ it('keeps an explicitly chosen HuggingFace engine on its designed busy state', () => {
+ process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example.run.app';
+ process.env.GCP_RECONSTRUCTION_KEY = 'k';
+ expect(laneAfterHfFailure({ explicit: true })).toBeNull();
+ });
+
+ it('has nowhere to go when nothing else is configured', () => {
+ expect(laneAfterHfFailure({})).toBeNull();
+ });
+});