Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): fail over to self-hosted Hunyuan3D or NVIDIA NIM when the free HuggingFace lane cannot serve an auto-routed request, instead of answering 502
nirholas31b416e89ea0
diff --git a/api/_lib/forge-tiers.js b/api/_lib/forge-tiers.jsindex deec667ff..32919c2a6 100644--- a/api/_lib/forge-tiers.js+++ b/api/_lib/forge-tiers.js@@ -602,6 +602,30 @@ export function resolveBackendIdWithHealth({ path, tier, backend, userImages = freturn defaultBackendForHealthAware(p, tierId, userImages, health, subjectClass);}+// Where a request goes after the free HuggingFace Spaces lane failed to serve it.+//+// The health-aware router picks HuggingFace whenever every self-host lane reads+// 'down' at that instant. That happens far more often than a real outage: one+// failed job (a bad reference image is enough) cools the Hunyuan3D lane for 90+// seconds fleet-wide, and with self-host TRELLIS also out, HuggingFace (reported+// 'unknown', never probed) became the pick. Its Spaces sit GPU-quota-dead for+// most of the day, so the request answered 502 while our own Hunyuan3D worker+// sat healthy and idle (forge_avatar, 2026-09-18). A health snapshot is a+// routing hint; the submit is the authority. So an auto-routed request that HF+// could not serve now tries our self-host Hunyuan3D worker, and a text prompt+// with no self-host lane still gets the free NVIDIA NIM text lane.+//+// An explicitly chosen HuggingFace engine keeps its designed busy state: the+// caller named that engine, so it is never silently swapped.+//+// Returns 'hunyuan3d' | 'nvidia' | null (null = surface the busy state).+export function laneAfterHfFailure({ explicit = false, userImages = false } = {}) {+ if (explicit) return null;+ if (backendIsConfigured('hunyuan3d')) return 'hunyuan3d';+ if (!userImages && backendIsConfigured('nvidia')) return 'nvidia';+ return null;+}+function readEnv(name) {if (typeof process !== 'undefined' && process.env && process.env[name]) return process.env[name];return null;diff --git a/api/forge.js b/api/forge.jsindex 15be852c1..ba5621e60 100644--- a/api/forge.js+++ b/api/forge.js@@ -77,6 +77,7 @@ import {estimateCredits,preferFreeReconstruct,backendIsConfigured,+ laneAfterHfFailure,backendAcceptsInlineViews,buildCatalog,selfhostQualityForTier,@@ -2199,12 +2200,26 @@ async function startJob(req, res) {}))return;- return json(res, 502, {- error: 'provider_busy',- backend: 'huggingface',- message:- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',- });+ // Failover instead of a dead end: see laneAfterHfFailure for why an+ // auto-routed request lands here while our own GPU worker is healthy.+ const nextLane = laneAfterHfFailure({ explicit: backendExplicit, userImages: isImageMode });+ if (nextLane === 'hunyuan3d') {+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to self-hosted Hunyuan3D');+ backendId = 'hunyuan3d';+ provider = createGcpProvider();+ } else if (nextLane === 'nvidia' && !nvidiaTried) {+ nvidiaTried = true;+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to free NVIDIA NIM');+ if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;+ }+ if (backendId === 'huggingface') {+ return json(res, 502, {+ error: 'provider_busy',+ backend: 'huggingface',+ message:+ 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',+ });+ }}// Free-first: exhaust the free reconstruct lane (HuggingFace Spaces) BEFOREdiff --git a/api/gpt-forge.js b/api/gpt-forge.jsindex 1fb0e7b7d..74b0ff90a 100644--- a/api/gpt-forge.js+++ b/api/gpt-forge.js@@ -83,6 +83,7 @@ import {estimateCredits,preferFreeReconstruct,backendIsConfigured,+ laneAfterHfFailure,buildCatalog,selfhostQualityForTier,} from './_lib/forge-tiers.js';@@ -2129,12 +2130,26 @@ async function startJob(req, res) {}))return;- return json(res, 502, {- error: 'provider_busy',- backend: 'huggingface',- message:- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',- });+ // Failover instead of a dead end: see laneAfterHfFailure for why an+ // auto-routed request lands here while our own GPU worker is healthy.+ const nextLane = laneAfterHfFailure({ explicit: backendExplicit, userImages: isImageMode });+ if (nextLane === 'hunyuan3d') {+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to self-hosted Hunyuan3D');+ backendId = 'hunyuan3d';+ provider = createGcpProvider();+ } else if (nextLane === 'nvidia' && !nvidiaTried) {+ nvidiaTried = true;+ console.warn('[forge] free HuggingFace lane could not serve this request; failing over to free NVIDIA NIM');+ if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;+ }+ if (backendId === 'huggingface') {+ return json(res, 502, {+ error: 'provider_busy',+ backend: 'huggingface',+ message:+ 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',+ });+ }}// Free-first: exhaust the free reconstruct lane (HuggingFace Spaces) BEFOREdiff --git a/tests/api/forge-hf-failover.test.js b/tests/api/forge-hf-failover.test.jsnew file mode 100644index 000000000..dab575cb2--- /dev/null+++ b/tests/api/forge-hf-failover.test.js@@ -0,0 +1,55 @@+// laneAfterHfFailure: where an auto-routed forge request goes after the free+// HuggingFace Spaces lane could not serve it.+//+// Production 2026-09-18: forge_avatar answered "Could not generate the model"+// because the health-aware router picked HuggingFace (every self-host lane read+// 'down': Hunyuan3D in a 90 s cooldown after one failed job, self-host TRELLIS+// with a failed model load), the Spaces were GPU-quota-dead, and the HF branch+// returned a 502 with no failover while our own Hunyuan3D worker was healthy.++import { describe, it, expect, beforeEach, afterEach } from 'vitest';+import { laneAfterHfFailure } from '../../api/_lib/forge-tiers.js';++const ENV = ['GCP_HUNYUAN3D_URL', 'GCP_RECONSTRUCTION_KEY', 'NVIDIA_API_KEY'];++describe('laneAfterHfFailure', () => {+ let saved;+ beforeEach(() => {+ saved = Object.fromEntries(ENV.map((k) => [k, process.env[k]]));+ for (const k of ENV) delete process.env[k];+ });+ afterEach(() => {+ for (const k of ENV) {+ if (saved[k] === undefined) delete process.env[k];+ else process.env[k] = saved[k];+ }+ });++ it('fails over to the self-hosted Hunyuan3D worker when it is configured', () => {+ process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example.run.app';+ process.env.GCP_RECONSTRUCTION_KEY = 'k';+ process.env.NVIDIA_API_KEY = 'nv';+ expect(laneAfterHfFailure({})).toBe('hunyuan3d');+ expect(laneAfterHfFailure({ userImages: true })).toBe('hunyuan3d');+ });++ it('sends a text prompt to the free NVIDIA lane when no self-host worker is wired', () => {+ process.env.NVIDIA_API_KEY = 'nv';+ expect(laneAfterHfFailure({ userImages: false })).toBe('nvidia');+ });++ it('never sends a photo to the text-only NVIDIA lane', () => {+ process.env.NVIDIA_API_KEY = 'nv';+ expect(laneAfterHfFailure({ userImages: true })).toBeNull();+ });++ it('keeps an explicitly chosen HuggingFace engine on its designed busy state', () => {+ process.env.GCP_HUNYUAN3D_URL = 'https://hunyuan.example.run.app';+ process.env.GCP_RECONSTRUCTION_KEY = 'k';+ expect(laneAfterHfFailure({ explicit: true })).toBeNull();+ });++ it('has nowhere to go when nothing else is configured', () => {+ expect(laneAfterHfFailure({})).toBeNull();+ });+});