Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): cool the nvidia breaker from cron-observed stalls, give stuck text-to-3D jobs one resubmit

nirholas7fa4b06534d9
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.js
index 9e353b9cc..81f72ebea 100644
--- a/api/cron/forge-finalize.js
+++ b/api/cron/forge-finalize.js
@@ -52,9 +52,11 @@ import {
MAX_FAILOVER_HOPS,
} from '../_lib/forge-failover.js';
import { markLaneUnhealthy } from '../_lib/forge-lane-health.js';
+import { markProviderCooldown } from '../_lib/provider-health.js';
import { resolveTier } from '../_lib/forge-tiers.js';
import { acquireLock, cacheGet, cacheSet } from '../_lib/cache.js';
import { getRedis } from '../_lib/redis.js';
+import { NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds } from '../forge.js';
export const maxDuration = 60;
@@ -146,6 +148,32 @@ export function decideFailedSweep({
return 'redispatch';
}
+/**
+ * NVIDIA NIM is the only text-to-3D lane (forge-failover.js excludes it from
+ * redispatch: text-only, and a poll-time redispatch always reconstructs from a
+ * stored reference image nvidia rows never have), so a stalled poll has no
+ * alternate lane to fail over to. The only two defenses are: defer to the
+ * attended browser poll while one may still be running, then give the job one
+ * real resubmit before the hard TTL reaps it as dead.
+ *
+ * @returns {'defer'|'resubmit'|'terminal'}
+ * - defer: an attended client may still be polling, or this job already got
+ * its one resubmit; leave the row alone.
+ * - resubmit: past the attended window, before the hard TTL, not yet retried.
+ * - terminal: past the hard TTL; nothing left to try.
+ */
+export function decideNvidiaStall({
+ ageMinutes,
+ alreadyResubmitted = false,
+ attendedBudgetMinutes = ATTENDED_POLL_BUDGET_MINUTES,
+ hardTtlMinutes = HARD_TTL_MINUTES,
+} = {}) {
+ if (!Number.isFinite(ageMinutes) || ageMinutes < attendedBudgetMinutes) return 'defer';
+ if (ageMinutes >= hardTtlMinutes) return 'terminal';
+ if (alreadyResubmitted) return 'defer';
+ return 'resubmit';
+}
+
// Unattended twin of the poll handler's automatic lane failover (api/forge.js
// pollJob). The 30-minute worker orphan class lands here by construction: the
// worker only declares an orphan long after every browser has stopped polling,
@@ -405,6 +433,46 @@ export default wrapCron(async (req, res) => {
out.still_running++;
continue;
}
+ if ((!status || status.status === 'running' || status.status === 'queued') && row.prompt) {
+ // Still running/queued, or the poll itself errored (status null): no
+ // alternate lane exists for text-only nvidia, so give the job one real
+ // resubmit before the hard TTL reaps it as dead. A distinct lock from
+ // nvcf:resub: below means this can never double-fire with the
+ // nvcf_expired recovery.
+ const stallDecision = decideNvidiaStall({
+ ageMinutes,
+ alreadyResubmitted: Boolean(
+ await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),
+ ),
+ });
+ if (stallDecision === 'resubmit' && (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))) {
+ try {
+ const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
+ await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);
+ if (resub.taskId) {
+ await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
+ console.warn(
+ `[forge-finalize] NVCF request stalled; resubmitted unattended job (creation ${row.id})`,
+ );
+ out.resubmitted++;
+ continue;
+ }
+ if (resub.resultGlbUrl) {
+ const durable = await materializeCreation({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ glbUrl: resub.resultGlbUrl,
+ });
+ if (durable) {
+ out.done++;
+ continue;
+ }
+ }
+ } catch (err) {
+ console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);
+ }
+ }
+ }
if (status?.code === 'nvcf_expired' && row.prompt) {
// The request id is dead but the inputs live on the row — the same
// never-dead-end recovery the attended poll runs, sharing its
@@ -437,8 +505,12 @@ export default wrapCron(async (req, res) => {
console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);
}
}
- // Lock held by a prior recovery or the resubmit failed: fall through
- // to the TTL check so a second expiry fails honestly, not instantly.
+ // Lock held by a prior recovery or the resubmit failed: the lane is
+ // cron-observed broken with no alternate text-to-3D lane to send new
+ // submissions to, so cool the same breaker forge.js's submit path
+ // reads before every new free-text request. Fall through to the TTL
+ // check so a second expiry fails honestly, not instantly.
+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
} else if (status?.status === 'failed') {
await markFailed({
replicateJobId: row.replicate_job_id,
@@ -458,6 +530,13 @@ export default wrapCron(async (req, res) => {
}
if (ageMinutes > HARD_TTL_MINUTES) {
+ if (row.backend === 'nvidia') {
+ // Reaping the bare TTL on nvidia is itself a cron-observed failure of
+ // the lane: cool the submit-time breaker so new free-text requests
+ // skip straight to the paid/image-intermediate path instead of landing
+ // on the same dead end for another HARD_TTL_MINUTES.
+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
+ }
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
diff --git a/api/forge.js b/api/forge.js
index 5a3498de8..acfa9c5e5 100644
--- a/api/forge.js
+++ b/api/forge.js
@@ -175,7 +175,7 @@ import { meshDirectorFor, meshSubjectClass, resolveLogoPrompt } from './_lib/for
// runs) lets subsequent requests skip a degraded lane and go straight to a
// working one; it expires on its own so a recovered lane is retried promptly.
// Best-effort via the shared cache: a miss just means "not cooling".
-const NIM_TRELLIS_COOLDOWN_KEY = 'forge-nim-trellis';
+export const NIM_TRELLIS_COOLDOWN_KEY = 'forge-nim-trellis';
// Sideline the free NIM lane after a failure. The cooldown exists to avoid
// re-paying the expensive submit-timeout HANG, so a socket timeout / unreachable
// host (no HTTP status came back) earns the full window. A fast gateway 5xx (a
diff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.js
index ae6266790..aa2c66327 100644
--- a/tests/forge-finalize.test.js
+++ b/tests/forge-finalize.test.js
@@ -4,7 +4,7 @@
// Importing the cron module is safe: db/neon clients are created lazily.
import { describe, it, expect } from 'vitest';
-import { decodeWorkerEnvelope, decideFailedSweep } from '../api/cron/forge-finalize.js';
+import { decodeWorkerEnvelope, decideFailedSweep, decideNvidiaStall } from '../api/cron/forge-finalize.js';
function pack(obj) {
return Buffer.from(JSON.stringify(obj), 'utf8').toString('base64url');
@@ -81,3 +81,34 @@ describe('decideFailedSweep', () => {
expect(decideFailedSweep({ ageMinutes: undefined, hasReferenceImage: true })).toBe('redispatch');
});
});
+
+// The free NVIDIA NIM text lane has no alternate lane to fail a stuck prompt
+// over to (forge-failover.js excludes it: text-only, no stored reference image
+// to reconstruct from), so a stalled poll gets exactly one resubmit attempt
+// before the hard TTL reaps it, instead of silently burning the full 45 minutes
+// on a request that will never resolve on its own.
+describe('decideNvidiaStall', () => {
+ it('defers while an attended client may still be polling', () => {
+ expect(decideNvidiaStall({ ageMinutes: 3 })).toBe('defer');
+ expect(decideNvidiaStall({ ageMinutes: 12.9 })).toBe('defer');
+ });
+
+ it('resubmits once the attended window has passed and the hard TTL has not hit', () => {
+ expect(decideNvidiaStall({ ageMinutes: 13 })).toBe('resubmit');
+ expect(decideNvidiaStall({ ageMinutes: 30 })).toBe('resubmit');
+ });
+
+ it('never resubmits a job twice', () => {
+ expect(decideNvidiaStall({ ageMinutes: 30, alreadyResubmitted: true })).toBe('defer');
+ });
+
+ it('goes terminal at or past the hard TTL regardless of the resubmit flag', () => {
+ expect(decideNvidiaStall({ ageMinutes: 45 })).toBe('terminal');
+ expect(decideNvidiaStall({ ageMinutes: 50, alreadyResubmitted: true })).toBe('terminal');
+ expect(decideNvidiaStall({ ageMinutes: 45, alreadyResubmitted: false })).toBe('terminal');
+ });
+
+ it('treats an unknowable age as inside the attended window rather than resubmitting blind', () => {
+ expect(decideNvidiaStall({ ageMinutes: undefined })).toBe('defer');
+ });
+});