Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): pause the HuggingFace lane after a chain-wide Space failure and hop explicit picks to a working engine
nirholas6adae161edc9
diff --git a/api/_lib/forge-health.js b/api/_lib/forge-health.jsindex 23a1131f1..efc75fc4d 100644--- a/api/_lib/forge-health.js+++ b/api/_lib/forge-health.js@@ -25,6 +25,7 @@ import { getRedisBurn } from './redis-usage.js';import { probeLlmHealth } from './llm-health.js';import { readGenerationMetrics } from './forge-events.js';import { providersInCooldown } from './provider-health.js';+import { hfLaneCooling } from './forge-lane-health.js';const PROBE_TIMEOUT_MS = 4_000;const CACHE_TTL_MS = 60_000;@@ -179,6 +180,19 @@ async function probeHuggingFace() {return result(id, 'degraded', 'Hugging Face is rate-limiting — the free Spaces may queue.', { http_status: res.status, latency_ms: latency });}if (res.ok) {+ // whoami only proves the token. The Spaces behind the lane fail on their+ // own (GPU quota exhausted, Space errored) while the token stays valid, so+ // cross-check the pause the real generation path sets after a chain-wide+ // failure. Reported down, not degraded: the engine picker disables a down+ // lane and moves the user to a working one, which is the point.+ if (await hfLaneCooling()) {+ return result(+ id,+ 'down',+ 'Hugging Face accepted the token, but every free Space failed a real generation in the last few minutes; the lane is paused while they recover.',+ { latency_ms: latency },+ );+ }return result(id, 'ok', 'Hugging Face accepted the token; the free Spaces lane is reachable (queue waits vary).', { latency_ms: latency });}return result(id, 'down', `Hugging Face returned an unexpected HTTP ${res.status}.`, { http_status: res.status, latency_ms: latency });diff --git a/api/_lib/forge-lane-health.js b/api/_lib/forge-lane-health.jsindex ea4fa53ed..7af3a57fa 100644--- a/api/_lib/forge-lane-health.js+++ b/api/_lib/forge-lane-health.js@@ -51,6 +51,24 @@ export async function markLaneUnhealthy(backendId, seconds = LANE_COOLDOWN_SECONawait markProviderCooldown(laneCooldownKey(backendId), seconds).catch(() => {});}+// The free HuggingFace Spaces lane fails as a whole: every Space in its chain+// sits GPU-quota-exhausted or errored for long stretches of the day, and each+// attempt still costs the caller 3-15s of Space round trips before it gives up.+// On 2026-10-08 a user who had picked that engine retried it four times in six+// minutes, each one a 502, while our own TRELLIS worker finished jobs alongside.+// One chain-wide failure therefore pauses the lane fleet-wide for this long:+// the free-first path skips straight to the next lane, the health report reads+// the lane as down so the engine picker stops offering it, and the first+// request after the window re-tests the Spaces for real.+export const HF_LANE_COOLDOWN_SECONDS = 300;++/** True while a recent chain-wide HuggingFace failure has the lane paused. */+export async function hfLaneCooling() {+ const key = laneCooldownKey('huggingface');+ const cooling = await providersInCooldown([key]).catch(() => new Map());+ return cooling.has(key);+}+const PROBE_TIMEOUT_MS = 2_500;// A reachable worker that answers within this is warm; slower means it is most// likely a scale-to-zero container spinning up, so the lane is reported cold anddiff --git a/api/forge.js b/api/forge.jsindex ee9a6f9ff..5e9942ab6 100644--- a/api/forge.js+++ b/api/forge.js@@ -82,7 +82,12 @@ import {buildCatalog,selfhostQualityForTier,} from './_lib/forge-tiers.js';-import { laneHealthSnapshot, markLaneUnhealthy } from './_lib/forge-lane-health.js';+import {+ laneHealthSnapshot,+ markLaneUnhealthy,+ hfLaneCooling,+ HF_LANE_COOLDOWN_SECONDS,+} from './_lib/forge-lane-health.js';import { isInlineImageRef } from './_lib/image-persist.js';import { resolveProviderKey } from './_lib/forge-provider-key.js';import { validateForgeImage } from './_lib/forge-image-validate.js';@@ -794,6 +799,26 @@ async function runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, optreturn true;}+// The caller explicitly picked the free HuggingFace engine and it could not+// serve the request. The pick is honoured (never silently swapped), but this is+// a temporary capacity state, not a gateway fault: answer 503 with Retry-After,+// and name the configured engines that can take a fresh retry so the client can+// hop to one visibly instead of looping its countdown on the same dead Spaces.+// The lane always holds reference views by now (uploaded, or synthesized from+// the prompt), so the suggestions are scoped to image-capable lanes.+function hfLaneBusy(res) {+ const retryBackends = retryBackendSuggestions({ attempted: ['huggingface'], hasImage: true });+ res.setHeader('retry-after', '30');+ return json(res, 503, {+ error: 'provider_busy',+ backend: 'huggingface',+ message:+ 'The free Hugging Face Spaces are out of GPU quota or warming up right now. Switch engines, or try again in a moment.',+ retry_after: 30,+ ...(retryBackends.length ? { retryable: true, retry_backends: retryBackends } : {}),+ });+}+// Free Hugging Face Spaces image→3D lane (Hunyuan3D / TRELLIS / TripoSR on free// GPU Spaces: the same provider the avatar reconstruction pipeline runs). The// platform photo→3D default is the Replicate TRELLIS lane; when that account is@@ -860,6 +885,11 @@ async function runHfImageLane({// lane reports "not served" (false) and the caller degrades, to the paid// reconstruct fallback on the free-first path, or a designed "free lane busy"// error on an explicit free pick: instead of piling onto an exhausted pool.+ // A chain-wide failure in the last few minutes paused the lane (see+ // HF_LANE_COOLDOWN_SECONDS): report "not served" at once instead of making+ // this caller wait out the same dead Spaces again.+ if (await hfLaneCooling()) return false;+const slot = await acquireBlockingSlot('hf', {max: SCALE_LIMITS.hfConcurrent,ttlMs: SCALE_LIMITS.hfSlotTtlMs,@@ -883,6 +913,7 @@ async function runHfImageLane({if (!resultGlbUrl) throw new Error('HuggingFace returned no GLB');} catch (err) {console.warn(`[forge] free HuggingFace image lane failed: ${err?.message || err}`);+ await markLaneUnhealthy('huggingface', HF_LANE_COOLDOWN_SECONDS);return false;} finally {await slot.release();@@ -2225,12 +2256,7 @@ async function startJobWithBody(req, res, body) {if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;}if (backendId === 'huggingface') {- return json(res, 502, {- error: 'provider_busy',- backend: 'huggingface',- message:- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',- });+ return hfLaneBusy(res);}}diff --git a/api/gpt-forge.js b/api/gpt-forge.jsindex 854fb35d1..912a58e1f 100644--- a/api/gpt-forge.js+++ b/api/gpt-forge.js@@ -87,7 +87,12 @@ import {buildCatalog,selfhostQualityForTier,} from './_lib/forge-tiers.js';-import { laneHealthSnapshot, markLaneUnhealthy } from './_lib/forge-lane-health.js';+import {+ laneHealthSnapshot,+ markLaneUnhealthy,+ hfLaneCooling,+ HF_LANE_COOLDOWN_SECONDS,+} from './_lib/forge-lane-health.js';import { resolveProviderKey } from './_lib/forge-provider-key.js';import { validateForgeImage } from './_lib/forge-image-validate.js';import { encodeJobToken, decodeJobToken } from './_lib/forge-job-token.js';@@ -827,6 +832,26 @@ async function runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, optreturn true;}+// The caller explicitly picked the free HuggingFace engine and it could not+// serve the request. The pick is honoured (never silently swapped), but this is+// a temporary capacity state, not a gateway fault: answer 503 with Retry-After,+// and name the configured engines that can take a fresh retry so the client can+// hop to one visibly instead of looping its countdown on the same dead Spaces.+// The lane always holds reference views by now (uploaded, or synthesized from+// the prompt), so the suggestions are scoped to image-capable lanes.+function hfLaneBusy(res) {+ const retryBackends = retryBackendSuggestions({ attempted: ['huggingface'], hasImage: true });+ res.setHeader('retry-after', '30');+ return json(res, 503, {+ error: 'provider_busy',+ backend: 'huggingface',+ message:+ 'The free Hugging Face Spaces are out of GPU quota or warming up right now. Switch engines, or try again in a moment.',+ retry_after: 30,+ ...(retryBackends.length ? { retryable: true, retry_backends: retryBackends } : {}),+ });+}+// Free Hugging Face Spaces image→3D lane (Hunyuan3D / TRELLIS / TripoSR on free// GPU Spaces: the same provider the avatar reconstruction pipeline runs). The// platform photo→3D default is the Replicate TRELLIS lane; when that account is@@ -889,6 +914,11 @@ async function runHfImageLane({// lane reports "not served" (false) and the caller degrades, to the paid// reconstruct fallback on the free-first path, or a designed "free lane busy"// error on an explicit free pick: instead of piling onto an exhausted pool.+ // A chain-wide failure in the last few minutes paused the lane (see+ // HF_LANE_COOLDOWN_SECONDS): report "not served" at once instead of making+ // this caller wait out the same dead Spaces again.+ if (await hfLaneCooling()) return false;+const slot = await acquireBlockingSlot('hf', {max: SCALE_LIMITS.hfConcurrent,ttlMs: SCALE_LIMITS.hfSlotTtlMs,@@ -912,6 +942,7 @@ async function runHfImageLane({if (!resultGlbUrl) throw new Error('HuggingFace returned no GLB');} catch (err) {console.warn(`[forge] free HuggingFace image lane failed: ${err?.message || err}`);+ await markLaneUnhealthy('huggingface', HF_LANE_COOLDOWN_SECONDS);return false;} finally {await slot.release();@@ -2152,12 +2183,7 @@ async function startJob(req, res) {if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;}if (backendId === 'huggingface') {- return json(res, 502, {- error: 'provider_busy',- backend: 'huggingface',- message:- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',- });+ return hfLaneBusy(res);}}diff --git a/data/changelog.json b/data/changelog.jsonindex ac6aba803..7f7592a7c 100644--- a/data/changelog.json+++ b/data/changelog.json@@ -1,6 +1,15 @@{"$comment": "Curated changelog entries for three.ws, the editorial layer of the public changelog. New PAGE launches are tracked automatically via the `added` field in data/pages.json; this file is for everything else holders care about: improvements, fixes, SDK releases, trading/payments capabilities, security work. scripts/build-page-index.mjs merges both sources into CHANGELOG.md, public/changelog.json, and public/changelog.xml. Newest first. Allowed tags: feature, improvement, fix, sdk, infra, docs, security, desktop (desktop entries become the desktop app's release notes). `link` is optional and must be a live three.ws path.","entries": [+ {+ "date": "2026-10-08",+ "title": "Picking the free Hugging Face engine no longer leaves you stuck when its Spaces are down",+ "summary": "When every free Hugging Face Space is out of GPU quota, the Forge now pauses that engine for a few minutes, greys it out in the engine picker with the reason, and moves a generation you started on it to a working engine, showing you the switch, instead of retrying the same dead Spaces with a countdown.",+ "tags": [+ "fix"+ ],+ "link": "/forge"+ },{"date": "2026-10-08","title": "Chart Companion: a 3D agent that watches the chart with you",diff --git a/docs/forge-pipeline.md b/docs/forge-pipeline.mdindex 956c2725b..98a984871 100644--- a/docs/forge-pipeline.md+++ b/docs/forge-pipeline.md@@ -125,6 +125,8 @@ Routing answers one question: given `(path, tier, userImages, prompt)`, which co**Circuit breakers.** A failing lane is sidelined, not retried in a loop: per-lane cooldowns (`forge-lane:<id>`, 90s) plus dedicated NIM cooldowns (120s model, 30s gateway) short-circuit routing away from a degraded upstream. Cooldowns are shared cross-instance via `api/_lib/provider-health.js` and expire on their own, so a recovered lane is retried promptly. Every entry is env-gated, so partial deployments degrade to a shorter list instead of erroring.+**The HuggingFace Spaces pause.** The free HF lane fails as a whole (every Space in its chain GPU-quota-exhausted or errored), and each attempt still costs the caller 3-15 s of Space round trips. So one chain-wide failure records `forge-lane:huggingface` for `HF_LANE_COOLDOWN_SECONDS` (300 s, `api/_lib/forge-lane-health.js`). While it holds, `runHfImageLane` reports "not served" at once, the health-aware router reads the lane as down, and `GET /api/forge?health=1` reports `huggingface` as `down` (whoami alone only proves the token), so the engine picker disables it. A caller who explicitly picked the HF engine gets `503 provider_busy` with `Retry-After: 30` and `retry_backends` naming the configured image-capable engines; the pick is never swapped silently, and `/forge` hops to the first suggestion visibly through the same lane-failed path a mid-job failure uses. The first request after the window re-tests the Spaces for real.+## 5. Job lifecycle### Submitdiff --git a/src/forge-studio/forge.js b/src/forge-studio/forge.jsindex ec4b78841..a38f29a2c 100644--- a/src/forge-studio/forge.js+++ b/src/forge-studio/forge.js@@ -1379,6 +1379,23 @@ async function startJob({ prompt, imageUrls, skipValidation, payment }) {body: JSON.stringify(body),});const data = await res.json().catch(() => ({}));+ // An explicitly picked free engine that could not serve the request (503+ // provider_busy). Checked before the generic 503 handling below, which would+ // misfile it. When the server names configured engines that can take the+ // request, hop to one visibly (the lane_failed path flips the picker and says+ // so) rather than count down into the same dead lane; otherwise it is the+ // recoverable high-demand countdown.+ if (data.error === 'provider_busy') {+ const e = new Error(data.message || 'The free 3D engines are at capacity right now.');+ if (Array.isArray(data.retry_backends) && data.retry_backends.length) {+ e.kind = 'lane_failed';+ e.retryBackends = data.retry_backends;+ } else {+ e.kind = 'busy';+ e.retryAfter = Number(data.retry_after) > 0 ? Math.ceil(Number(data.retry_after)) : 15;+ }+ throw e;+ }if (res.status === 503 || data.error === 'unconfigured') {const e = new Error(data.message || 'unconfigured');e.kind = 'unconfigured';diff --git a/src/forge.js b/src/forge.jsindex 9f3345276..a0b30e572 100644--- a/src/forge.js+++ b/src/forge.js@@ -1684,6 +1684,23 @@ async function startJob({ prompt, imageUrls, skipValidation, payment, seed, progbody: JSON.stringify(body),});const data = await res.json().catch(() => ({}));+ // An explicitly picked free engine that could not serve the request (503+ // provider_busy). Checked before the generic 503 handling below, which would+ // misfile it. When the server names configured engines that can take the+ // request, hop to one visibly (the lane_failed path flips the picker and says+ // so) rather than count down into the same dead lane; otherwise it is the+ // recoverable high-demand countdown.+ if (data.error === 'provider_busy') {+ const e = new Error(data.message || 'The free 3D engines are at capacity right now.');+ if (Array.isArray(data.retry_backends) && data.retry_backends.length) {+ e.kind = 'lane_failed';+ e.retryBackends = data.retry_backends;+ } else {+ e.kind = 'busy';+ e.retryAfter = Number(data.retry_after) > 0 ? Math.ceil(Number(data.retry_after)) : 15;+ }+ throw e;+ }// Only a genuine deployment gap earns the unconfigured panel: it names env// vars and offers no retry, which is the right answer for an operator and the// wrong one for a visitor. The other 503s (`generation_unavailable`) arediff --git a/tests/api/forge-free-first.test.js b/tests/api/forge-free-first.test.jsindex cf934c47c..a38d0d1bf 100644--- a/tests/api/forge-free-first.test.js+++ b/tests/api/forge-free-first.test.js@@ -255,9 +255,16 @@ describe('forge free-first reconstruct ordering', () => {const res = makeRes();await handler(req, res);- expect(res.statusCode).toBe(502);+ // A temporary capacity state, not a gateway fault: 503 + Retry-After.+ expect(res.statusCode).toBe(503);+ expect(res.headers['retry-after']).toBe('30');expect(res.body.error).toBe('provider_busy');expect(res.body.backend).toBe('huggingface');+ expect(res.body.retry_after).toBe(30);+ // Names the configured engines a fresh retry could use, never HF itself.+ expect(res.body.retryable).toBe(true);+ expect(res.body.retry_backends).toContain('trellis');+ expect(res.body.retry_backends).not.toContain('huggingface');expect(replicateSubmit).not.toHaveBeenCalled();});@@ -289,3 +296,48 @@ describe('forge free-first reconstruct ordering', () => {}});});++// Production 2026-10-08: every HuggingFace Space sat GPU-quota-dead while a user+// who had picked that engine retried it four times in six minutes, each one a+// 3-15s wait ending in a 502. One chain-wide failure now pauses the lane+// fleet-wide, so the next request answers at once (the health report side is+// pinned in tests/api/forge-health.test.js).+describe('forge HuggingFace lane pause after a chain-wide failure', () => {+ const photoPick = () =>+ makeReq({+ image_urls: ['https://cdn.example/photo.png'],+ backend: 'huggingface',+ tier: 'standard',+ path: 'image',+ skip_validation: true,+ });++ it('pauses the lane so the next explicit pick skips the dead Spaces', async () => {+ const { clearProviderCooldown } = await import('../../api/_lib/provider-health.js');+ const { laneCooldownKey, hfLaneCooling } = await import('../../api/_lib/forge-lane-health.js');+ process.env.HF_TOKEN = 'test-hf-token';+ hfSubmit.mockClear();+ hfSubmit.mockRejectedValueOnce(new Error('all 3 huggingface Space(s) failed: GPU quota exhausted'));+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});+ try {+ const first = makeRes();+ await handler(photoPick(), first);+ expect(first.statusCode).toBe(503);+ expect(first.body.error).toBe('provider_busy');+ expect(hfSubmit).toHaveBeenCalledTimes(1);+ expect(await hfLaneCooling()).toBe(true);++ const second = makeRes();+ await handler(photoPick(), second);+ expect(second.statusCode).toBe(503);+ expect(second.body.error).toBe('provider_busy');+ // Paused: the Spaces were not called again.+ expect(hfSubmit).toHaveBeenCalledTimes(1);+ } finally {+ warn.mockRestore();+ await clearProviderCooldown(laneCooldownKey('huggingface'));+ delete process.env.HF_TOKEN;+ }+ });++});diff --git a/tests/api/forge-health.test.js b/tests/api/forge-health.test.jsindex 79328c44b..12317cbdc 100644--- a/tests/api/forge-health.test.js+++ b/tests/api/forge-health.test.js@@ -164,6 +164,26 @@ describe('forge-health — per-backend verdicts', () => {expect((await probeForgeHealth()).backends.huggingface.status).toBe('degraded');});+ it('reports HuggingFace down while a chain-wide Space failure has the lane paused', async () => {+ // whoami only proves the token; the Spaces fail on their own. The real+ // generation path pauses the lane after every Space failed, and the report+ // must say so, or the engine picker keeps offering a dead lane.+ const { markLaneUnhealthy, laneCooldownKey } = await import('../../api/_lib/forge-lane-health.js');+ const { clearProviderCooldown } = await import('../../api/_lib/provider-health.js');+ process.env.HF_TOKEN = 'hf_test';+ mockUpstreams([['huggingface.co', 200]]);+ try {+ await markLaneUnhealthy('huggingface', 300);+ const health = await probeForgeHealth({ force: true });+ expect(health.backends.huggingface.status).toBe('down');+ expect(health.backends.huggingface.message).toMatch(/paused/);+ } finally {+ await clearProviderCooldown(laneCooldownKey('huggingface'));+ }+ const healed = await probeForgeHealth({ force: true });+ expect(healed.backends.huggingface.status).toBe('ok');+ });+it('never probes HuggingFace when HF_TOKEN is unset', async () => {mockUpstreams([]);const health = await probeForgeHealth();