Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): pause the HuggingFace lane after a chain-wide Space failure and hop explicit picks to a working engine

nirholas6adae161edc9
diff --git a/api/_lib/forge-health.js b/api/_lib/forge-health.js
index 23a1131f1..efc75fc4d 100644
--- a/api/_lib/forge-health.js
+++ b/api/_lib/forge-health.js
@@ -25,6 +25,7 @@ import { getRedisBurn } from './redis-usage.js';
import { probeLlmHealth } from './llm-health.js';
import { readGenerationMetrics } from './forge-events.js';
import { providersInCooldown } from './provider-health.js';
+import { hfLaneCooling } from './forge-lane-health.js';
const PROBE_TIMEOUT_MS = 4_000;
const CACHE_TTL_MS = 60_000;
@@ -179,6 +180,19 @@ async function probeHuggingFace() {
return result(id, 'degraded', 'Hugging Face is rate-limiting — the free Spaces may queue.', { http_status: res.status, latency_ms: latency });
}
if (res.ok) {
+ // whoami only proves the token. The Spaces behind the lane fail on their
+ // own (GPU quota exhausted, Space errored) while the token stays valid, so
+ // cross-check the pause the real generation path sets after a chain-wide
+ // failure. Reported down, not degraded: the engine picker disables a down
+ // lane and moves the user to a working one, which is the point.
+ if (await hfLaneCooling()) {
+ return result(
+ id,
+ 'down',
+ 'Hugging Face accepted the token, but every free Space failed a real generation in the last few minutes; the lane is paused while they recover.',
+ { latency_ms: latency },
+ );
+ }
return result(id, 'ok', 'Hugging Face accepted the token; the free Spaces lane is reachable (queue waits vary).', { latency_ms: latency });
}
return result(id, 'down', `Hugging Face returned an unexpected HTTP ${res.status}.`, { http_status: res.status, latency_ms: latency });
diff --git a/api/_lib/forge-lane-health.js b/api/_lib/forge-lane-health.js
index ea4fa53ed..7af3a57fa 100644
--- a/api/_lib/forge-lane-health.js
+++ b/api/_lib/forge-lane-health.js
@@ -51,6 +51,24 @@ export async function markLaneUnhealthy(backendId, seconds = LANE_COOLDOWN_SECON
await markProviderCooldown(laneCooldownKey(backendId), seconds).catch(() => {});
}
+// The free HuggingFace Spaces lane fails as a whole: every Space in its chain
+// sits GPU-quota-exhausted or errored for long stretches of the day, and each
+// attempt still costs the caller 3-15s of Space round trips before it gives up.
+// On 2026-10-08 a user who had picked that engine retried it four times in six
+// minutes, each one a 502, while our own TRELLIS worker finished jobs alongside.
+// One chain-wide failure therefore pauses the lane fleet-wide for this long:
+// the free-first path skips straight to the next lane, the health report reads
+// the lane as down so the engine picker stops offering it, and the first
+// request after the window re-tests the Spaces for real.
+export const HF_LANE_COOLDOWN_SECONDS = 300;
+
+/** True while a recent chain-wide HuggingFace failure has the lane paused. */
+export async function hfLaneCooling() {
+ const key = laneCooldownKey('huggingface');
+ const cooling = await providersInCooldown([key]).catch(() => new Map());
+ return cooling.has(key);
+}
+
const PROBE_TIMEOUT_MS = 2_500;
// A reachable worker that answers within this is warm; slower means it is most
// likely a scale-to-zero container spinning up, so the lane is reported cold and
diff --git a/api/forge.js b/api/forge.js
index ee9a6f9ff..5e9942ab6 100644
--- a/api/forge.js
+++ b/api/forge.js
@@ -82,7 +82,12 @@ import {
buildCatalog,
selfhostQualityForTier,
} from './_lib/forge-tiers.js';
-import { laneHealthSnapshot, markLaneUnhealthy } from './_lib/forge-lane-health.js';
+import {
+ laneHealthSnapshot,
+ markLaneUnhealthy,
+ hfLaneCooling,
+ HF_LANE_COOLDOWN_SECONDS,
+} from './_lib/forge-lane-health.js';
import { isInlineImageRef } from './_lib/image-persist.js';
import { resolveProviderKey } from './_lib/forge-provider-key.js';
import { validateForgeImage } from './_lib/forge-image-validate.js';
@@ -794,6 +799,26 @@ async function runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opt
return true;
}
+// The caller explicitly picked the free HuggingFace engine and it could not
+// serve the request. The pick is honoured (never silently swapped), but this is
+// a temporary capacity state, not a gateway fault: answer 503 with Retry-After,
+// and name the configured engines that can take a fresh retry so the client can
+// hop to one visibly instead of looping its countdown on the same dead Spaces.
+// The lane always holds reference views by now (uploaded, or synthesized from
+// the prompt), so the suggestions are scoped to image-capable lanes.
+function hfLaneBusy(res) {
+ const retryBackends = retryBackendSuggestions({ attempted: ['huggingface'], hasImage: true });
+ res.setHeader('retry-after', '30');
+ return json(res, 503, {
+ error: 'provider_busy',
+ backend: 'huggingface',
+ message:
+ 'The free Hugging Face Spaces are out of GPU quota or warming up right now. Switch engines, or try again in a moment.',
+ retry_after: 30,
+ ...(retryBackends.length ? { retryable: true, retry_backends: retryBackends } : {}),
+ });
+}
+
// Free Hugging Face Spaces image→3D lane (Hunyuan3D / TRELLIS / TripoSR on free
// GPU Spaces: the same provider the avatar reconstruction pipeline runs). The
// platform photo→3D default is the Replicate TRELLIS lane; when that account is
@@ -860,6 +885,11 @@ async function runHfImageLane({
// lane reports "not served" (false) and the caller degrades, to the paid
// reconstruct fallback on the free-first path, or a designed "free lane busy"
// error on an explicit free pick: instead of piling onto an exhausted pool.
+ // A chain-wide failure in the last few minutes paused the lane (see
+ // HF_LANE_COOLDOWN_SECONDS): report "not served" at once instead of making
+ // this caller wait out the same dead Spaces again.
+ if (await hfLaneCooling()) return false;
+
const slot = await acquireBlockingSlot('hf', {
max: SCALE_LIMITS.hfConcurrent,
ttlMs: SCALE_LIMITS.hfSlotTtlMs,
@@ -883,6 +913,7 @@ async function runHfImageLane({
if (!resultGlbUrl) throw new Error('HuggingFace returned no GLB');
} catch (err) {
console.warn(`[forge] free HuggingFace image lane failed: ${err?.message || err}`);
+ await markLaneUnhealthy('huggingface', HF_LANE_COOLDOWN_SECONDS);
return false;
} finally {
await slot.release();
@@ -2225,12 +2256,7 @@ async function startJobWithBody(req, res, body) {
if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;
}
if (backendId === 'huggingface') {
- return json(res, 502, {
- error: 'provider_busy',
- backend: 'huggingface',
- message:
- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',
- });
+ return hfLaneBusy(res);
}
}
diff --git a/api/gpt-forge.js b/api/gpt-forge.js
index 854fb35d1..912a58e1f 100644
--- a/api/gpt-forge.js
+++ b/api/gpt-forge.js
@@ -87,7 +87,12 @@ import {
buildCatalog,
selfhostQualityForTier,
} from './_lib/forge-tiers.js';
-import { laneHealthSnapshot, markLaneUnhealthy } from './_lib/forge-lane-health.js';
+import {
+ laneHealthSnapshot,
+ markLaneUnhealthy,
+ hfLaneCooling,
+ HF_LANE_COOLDOWN_SECONDS,
+} from './_lib/forge-lane-health.js';
import { resolveProviderKey } from './_lib/forge-provider-key.js';
import { validateForgeImage } from './_lib/forge-image-validate.js';
import { encodeJobToken, decodeJobToken } from './_lib/forge-job-token.js';
@@ -827,6 +832,26 @@ async function runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opt
return true;
}
+// The caller explicitly picked the free HuggingFace engine and it could not
+// serve the request. The pick is honoured (never silently swapped), but this is
+// a temporary capacity state, not a gateway fault: answer 503 with Retry-After,
+// and name the configured engines that can take a fresh retry so the client can
+// hop to one visibly instead of looping its countdown on the same dead Spaces.
+// The lane always holds reference views by now (uploaded, or synthesized from
+// the prompt), so the suggestions are scoped to image-capable lanes.
+function hfLaneBusy(res) {
+ const retryBackends = retryBackendSuggestions({ attempted: ['huggingface'], hasImage: true });
+ res.setHeader('retry-after', '30');
+ return json(res, 503, {
+ error: 'provider_busy',
+ backend: 'huggingface',
+ message:
+ 'The free Hugging Face Spaces are out of GPU quota or warming up right now. Switch engines, or try again in a moment.',
+ retry_after: 30,
+ ...(retryBackends.length ? { retryable: true, retry_backends: retryBackends } : {}),
+ });
+}
+
// Free Hugging Face Spaces image→3D lane (Hunyuan3D / TRELLIS / TripoSR on free
// GPU Spaces: the same provider the avatar reconstruction pipeline runs). The
// platform photo→3D default is the Replicate TRELLIS lane; when that account is
@@ -889,6 +914,11 @@ async function runHfImageLane({
// lane reports "not served" (false) and the caller degrades, to the paid
// reconstruct fallback on the free-first path, or a designed "free lane busy"
// error on an explicit free pick: instead of piling onto an exhausted pool.
+ // A chain-wide failure in the last few minutes paused the lane (see
+ // HF_LANE_COOLDOWN_SECONDS): report "not served" at once instead of making
+ // this caller wait out the same dead Spaces again.
+ if (await hfLaneCooling()) return false;
+
const slot = await acquireBlockingSlot('hf', {
max: SCALE_LIMITS.hfConcurrent,
ttlMs: SCALE_LIMITS.hfSlotTtlMs,
@@ -912,6 +942,7 @@ async function runHfImageLane({
if (!resultGlbUrl) throw new Error('HuggingFace returned no GLB');
} catch (err) {
console.warn(`[forge] free HuggingFace image lane failed: ${err?.message || err}`);
+ await markLaneUnhealthy('huggingface', HF_LANE_COOLDOWN_SECONDS);
return false;
} finally {
await slot.release();
@@ -2152,12 +2183,7 @@ async function startJob(req, res) {
if (await runNvidiaTextLane({ req, res, ip, prompt, aspect, tier, path, opts, cacheKey })) return;
}
if (backendId === 'huggingface') {
- return json(res, 502, {
- error: 'provider_busy',
- backend: 'huggingface',
- message:
- 'The free 3D Spaces are all busy or warming up right now. Try again in a moment, or pick another engine.',
- });
+ return hfLaneBusy(res);
}
}
diff --git a/data/changelog.json b/data/changelog.json
index ac6aba803..7f7592a7c 100644
--- a/data/changelog.json
+++ b/data/changelog.json
@@ -1,6 +1,15 @@
{
"$comment": "Curated changelog entries for three.ws, the editorial layer of the public changelog. New PAGE launches are tracked automatically via the `added` field in data/pages.json; this file is for everything else holders care about: improvements, fixes, SDK releases, trading/payments capabilities, security work. scripts/build-page-index.mjs merges both sources into CHANGELOG.md, public/changelog.json, and public/changelog.xml. Newest first. Allowed tags: feature, improvement, fix, sdk, infra, docs, security, desktop (desktop entries become the desktop app's release notes). `link` is optional and must be a live three.ws path.",
"entries": [
+ {
+ "date": "2026-10-08",
+ "title": "Picking the free Hugging Face engine no longer leaves you stuck when its Spaces are down",
+ "summary": "When every free Hugging Face Space is out of GPU quota, the Forge now pauses that engine for a few minutes, greys it out in the engine picker with the reason, and moves a generation you started on it to a working engine, showing you the switch, instead of retrying the same dead Spaces with a countdown.",
+ "tags": [
+ "fix"
+ ],
+ "link": "/forge"
+ },
{
"date": "2026-10-08",
"title": "Chart Companion: a 3D agent that watches the chart with you",
diff --git a/docs/forge-pipeline.md b/docs/forge-pipeline.md
index 956c2725b..98a984871 100644
--- a/docs/forge-pipeline.md
+++ b/docs/forge-pipeline.md
@@ -125,6 +125,8 @@ Routing answers one question: given `(path, tier, userImages, prompt)`, which co
**Circuit breakers.** A failing lane is sidelined, not retried in a loop: per-lane cooldowns (`forge-lane:<id>`, 90s) plus dedicated NIM cooldowns (120s model, 30s gateway) short-circuit routing away from a degraded upstream. Cooldowns are shared cross-instance via `api/_lib/provider-health.js` and expire on their own, so a recovered lane is retried promptly. Every entry is env-gated, so partial deployments degrade to a shorter list instead of erroring.
+**The HuggingFace Spaces pause.** The free HF lane fails as a whole (every Space in its chain GPU-quota-exhausted or errored), and each attempt still costs the caller 3-15 s of Space round trips. So one chain-wide failure records `forge-lane:huggingface` for `HF_LANE_COOLDOWN_SECONDS` (300 s, `api/_lib/forge-lane-health.js`). While it holds, `runHfImageLane` reports "not served" at once, the health-aware router reads the lane as down, and `GET /api/forge?health=1` reports `huggingface` as `down` (whoami alone only proves the token), so the engine picker disables it. A caller who explicitly picked the HF engine gets `503 provider_busy` with `Retry-After: 30` and `retry_backends` naming the configured image-capable engines; the pick is never swapped silently, and `/forge` hops to the first suggestion visibly through the same lane-failed path a mid-job failure uses. The first request after the window re-tests the Spaces for real.
+
## 5. Job lifecycle
### Submit
diff --git a/src/forge-studio/forge.js b/src/forge-studio/forge.js
index ec4b78841..a38f29a2c 100644
--- a/src/forge-studio/forge.js
+++ b/src/forge-studio/forge.js
@@ -1379,6 +1379,23 @@ async function startJob({ prompt, imageUrls, skipValidation, payment }) {
body: JSON.stringify(body),
});
const data = await res.json().catch(() => ({}));
+ // An explicitly picked free engine that could not serve the request (503
+ // provider_busy). Checked before the generic 503 handling below, which would
+ // misfile it. When the server names configured engines that can take the
+ // request, hop to one visibly (the lane_failed path flips the picker and says
+ // so) rather than count down into the same dead lane; otherwise it is the
+ // recoverable high-demand countdown.
+ if (data.error === 'provider_busy') {
+ const e = new Error(data.message || 'The free 3D engines are at capacity right now.');
+ if (Array.isArray(data.retry_backends) && data.retry_backends.length) {
+ e.kind = 'lane_failed';
+ e.retryBackends = data.retry_backends;
+ } else {
+ e.kind = 'busy';
+ e.retryAfter = Number(data.retry_after) > 0 ? Math.ceil(Number(data.retry_after)) : 15;
+ }
+ throw e;
+ }
if (res.status === 503 || data.error === 'unconfigured') {
const e = new Error(data.message || 'unconfigured');
e.kind = 'unconfigured';
diff --git a/src/forge.js b/src/forge.js
index 9f3345276..a0b30e572 100644
--- a/src/forge.js
+++ b/src/forge.js
@@ -1684,6 +1684,23 @@ async function startJob({ prompt, imageUrls, skipValidation, payment, seed, prog
body: JSON.stringify(body),
});
const data = await res.json().catch(() => ({}));
+ // An explicitly picked free engine that could not serve the request (503
+ // provider_busy). Checked before the generic 503 handling below, which would
+ // misfile it. When the server names configured engines that can take the
+ // request, hop to one visibly (the lane_failed path flips the picker and says
+ // so) rather than count down into the same dead lane; otherwise it is the
+ // recoverable high-demand countdown.
+ if (data.error === 'provider_busy') {
+ const e = new Error(data.message || 'The free 3D engines are at capacity right now.');
+ if (Array.isArray(data.retry_backends) && data.retry_backends.length) {
+ e.kind = 'lane_failed';
+ e.retryBackends = data.retry_backends;
+ } else {
+ e.kind = 'busy';
+ e.retryAfter = Number(data.retry_after) > 0 ? Math.ceil(Number(data.retry_after)) : 15;
+ }
+ throw e;
+ }
// Only a genuine deployment gap earns the unconfigured panel: it names env
// vars and offers no retry, which is the right answer for an operator and the
// wrong one for a visitor. The other 503s (`generation_unavailable`) are
diff --git a/tests/api/forge-free-first.test.js b/tests/api/forge-free-first.test.js
index cf934c47c..a38d0d1bf 100644
--- a/tests/api/forge-free-first.test.js
+++ b/tests/api/forge-free-first.test.js
@@ -255,9 +255,16 @@ describe('forge free-first reconstruct ordering', () => {
const res = makeRes();
await handler(req, res);
- expect(res.statusCode).toBe(502);
+ // A temporary capacity state, not a gateway fault: 503 + Retry-After.
+ expect(res.statusCode).toBe(503);
+ expect(res.headers['retry-after']).toBe('30');
expect(res.body.error).toBe('provider_busy');
expect(res.body.backend).toBe('huggingface');
+ expect(res.body.retry_after).toBe(30);
+ // Names the configured engines a fresh retry could use, never HF itself.
+ expect(res.body.retryable).toBe(true);
+ expect(res.body.retry_backends).toContain('trellis');
+ expect(res.body.retry_backends).not.toContain('huggingface');
expect(replicateSubmit).not.toHaveBeenCalled();
});
@@ -289,3 +296,48 @@ describe('forge free-first reconstruct ordering', () => {
}
});
});
+
+// Production 2026-10-08: every HuggingFace Space sat GPU-quota-dead while a user
+// who had picked that engine retried it four times in six minutes, each one a
+// 3-15s wait ending in a 502. One chain-wide failure now pauses the lane
+// fleet-wide, so the next request answers at once (the health report side is
+// pinned in tests/api/forge-health.test.js).
+describe('forge HuggingFace lane pause after a chain-wide failure', () => {
+ const photoPick = () =>
+ makeReq({
+ image_urls: ['https://cdn.example/photo.png'],
+ backend: 'huggingface',
+ tier: 'standard',
+ path: 'image',
+ skip_validation: true,
+ });
+
+ it('pauses the lane so the next explicit pick skips the dead Spaces', async () => {
+ const { clearProviderCooldown } = await import('../../api/_lib/provider-health.js');
+ const { laneCooldownKey, hfLaneCooling } = await import('../../api/_lib/forge-lane-health.js');
+ process.env.HF_TOKEN = 'test-hf-token';
+ hfSubmit.mockClear();
+ hfSubmit.mockRejectedValueOnce(new Error('all 3 huggingface Space(s) failed: GPU quota exhausted'));
+ const warn = vi.spyOn(console, 'warn').mockImplementation(() => {});
+ try {
+ const first = makeRes();
+ await handler(photoPick(), first);
+ expect(first.statusCode).toBe(503);
+ expect(first.body.error).toBe('provider_busy');
+ expect(hfSubmit).toHaveBeenCalledTimes(1);
+ expect(await hfLaneCooling()).toBe(true);
+
+ const second = makeRes();
+ await handler(photoPick(), second);
+ expect(second.statusCode).toBe(503);
+ expect(second.body.error).toBe('provider_busy');
+ // Paused: the Spaces were not called again.
+ expect(hfSubmit).toHaveBeenCalledTimes(1);
+ } finally {
+ warn.mockRestore();
+ await clearProviderCooldown(laneCooldownKey('huggingface'));
+ delete process.env.HF_TOKEN;
+ }
+ });
+
+});
diff --git a/tests/api/forge-health.test.js b/tests/api/forge-health.test.js
index 79328c44b..12317cbdc 100644
--- a/tests/api/forge-health.test.js
+++ b/tests/api/forge-health.test.js
@@ -164,6 +164,26 @@ describe('forge-health — per-backend verdicts', () => {
expect((await probeForgeHealth()).backends.huggingface.status).toBe('degraded');
});
+ it('reports HuggingFace down while a chain-wide Space failure has the lane paused', async () => {
+ // whoami only proves the token; the Spaces fail on their own. The real
+ // generation path pauses the lane after every Space failed, and the report
+ // must say so, or the engine picker keeps offering a dead lane.
+ const { markLaneUnhealthy, laneCooldownKey } = await import('../../api/_lib/forge-lane-health.js');
+ const { clearProviderCooldown } = await import('../../api/_lib/provider-health.js');
+ process.env.HF_TOKEN = 'hf_test';
+ mockUpstreams([['huggingface.co', 200]]);
+ try {
+ await markLaneUnhealthy('huggingface', 300);
+ const health = await probeForgeHealth({ force: true });
+ expect(health.backends.huggingface.status).toBe('down');
+ expect(health.backends.huggingface.message).toMatch(/paused/);
+ } finally {
+ await clearProviderCooldown(laneCooldownKey('huggingface'));
+ }
+ const healed = await probeForgeHealth({ force: true });
+ expect(healed.backends.huggingface.status).toBe('ok');
+ });
+
it('never probes HuggingFace when HF_TOKEN is unset', async () => {
mockUpstreams([]);
const health = await probeForgeHealth();