Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): cool the nvidia breaker from cron-observed stalls, give stuck text-to-3D jobs one resubmit
nirholas7fa4b06534d9
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.jsindex 9e353b9cc..81f72ebea 100644--- a/api/cron/forge-finalize.js+++ b/api/cron/forge-finalize.js@@ -52,9 +52,11 @@ import {MAX_FAILOVER_HOPS,} from '../_lib/forge-failover.js';import { markLaneUnhealthy } from '../_lib/forge-lane-health.js';+import { markProviderCooldown } from '../_lib/provider-health.js';import { resolveTier } from '../_lib/forge-tiers.js';import { acquireLock, cacheGet, cacheSet } from '../_lib/cache.js';import { getRedis } from '../_lib/redis.js';+import { NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds } from '../forge.js';export const maxDuration = 60;@@ -146,6 +148,32 @@ export function decideFailedSweep({return 'redispatch';}+/**+ * NVIDIA NIM is the only text-to-3D lane (forge-failover.js excludes it from+ * redispatch: text-only, and a poll-time redispatch always reconstructs from a+ * stored reference image nvidia rows never have), so a stalled poll has no+ * alternate lane to fail over to. The only two defenses are: defer to the+ * attended browser poll while one may still be running, then give the job one+ * real resubmit before the hard TTL reaps it as dead.+ *+ * @returns {'defer'|'resubmit'|'terminal'}+ * - defer: an attended client may still be polling, or this job already got+ * its one resubmit; leave the row alone.+ * - resubmit: past the attended window, before the hard TTL, not yet retried.+ * - terminal: past the hard TTL; nothing left to try.+ */+export function decideNvidiaStall({+ ageMinutes,+ alreadyResubmitted = false,+ attendedBudgetMinutes = ATTENDED_POLL_BUDGET_MINUTES,+ hardTtlMinutes = HARD_TTL_MINUTES,+} = {}) {+ if (!Number.isFinite(ageMinutes) || ageMinutes < attendedBudgetMinutes) return 'defer';+ if (ageMinutes >= hardTtlMinutes) return 'terminal';+ if (alreadyResubmitted) return 'defer';+ return 'resubmit';+}+// Unattended twin of the poll handler's automatic lane failover (api/forge.js// pollJob). The 30-minute worker orphan class lands here by construction: the// worker only declares an orphan long after every browser has stopped polling,@@ -405,6 +433,46 @@ export default wrapCron(async (req, res) => {out.still_running++;continue;}+ if ((!status || status.status === 'running' || status.status === 'queued') && row.prompt) {+ // Still running/queued, or the poll itself errored (status null): no+ // alternate lane exists for text-only nvidia, so give the job one real+ // resubmit before the hard TTL reaps it as dead. A distinct lock from+ // nvcf:resub: below means this can never double-fire with the+ // nvcf_expired recovery.+ const stallDecision = decideNvidiaStall({+ ageMinutes,+ alreadyResubmitted: Boolean(+ await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),+ ),+ });+ if (stallDecision === 'resubmit' && (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))) {+ try {+ const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });+ await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);+ if (resub.taskId) {+ await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);+ console.warn(+ `[forge-finalize] NVCF request stalled; resubmitted unattended job (creation ${row.id})`,+ );+ out.resubmitted++;+ continue;+ }+ if (resub.resultGlbUrl) {+ const durable = await materializeCreation({+ replicateJobId: row.replicate_job_id,+ clientKey: row.client_key,+ glbUrl: resub.resultGlbUrl,+ });+ if (durable) {+ out.done++;+ continue;+ }+ }+ } catch (err) {+ console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);+ }+ }+ }if (status?.code === 'nvcf_expired' && row.prompt) {// The request id is dead but the inputs live on the row — the same// never-dead-end recovery the attended poll runs, sharing its@@ -437,8 +505,12 @@ export default wrapCron(async (req, res) => {console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);}}- // Lock held by a prior recovery or the resubmit failed: fall through- // to the TTL check so a second expiry fails honestly, not instantly.+ // Lock held by a prior recovery or the resubmit failed: the lane is+ // cron-observed broken with no alternate text-to-3D lane to send new+ // submissions to, so cool the same breaker forge.js's submit path+ // reads before every new free-text request. Fall through to the TTL+ // check so a second expiry fails honestly, not instantly.+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});} else if (status?.status === 'failed') {await markFailed({replicateJobId: row.replicate_job_id,@@ -458,6 +530,13 @@ export default wrapCron(async (req, res) => {}if (ageMinutes > HARD_TTL_MINUTES) {+ if (row.backend === 'nvidia') {+ // Reaping the bare TTL on nvidia is itself a cron-observed failure of+ // the lane: cool the submit-time breaker so new free-text requests+ // skip straight to the paid/image-intermediate path instead of landing+ // on the same dead end for another HARD_TTL_MINUTES.+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});+ }await markFailed({replicateJobId: row.replicate_job_id,clientKey: row.client_key,diff --git a/api/forge.js b/api/forge.jsindex 5a3498de8..acfa9c5e5 100644--- a/api/forge.js+++ b/api/forge.js@@ -175,7 +175,7 @@ import { meshDirectorFor, meshSubjectClass, resolveLogoPrompt } from './_lib/for// runs) lets subsequent requests skip a degraded lane and go straight to a// working one; it expires on its own so a recovered lane is retried promptly.// Best-effort via the shared cache: a miss just means "not cooling".-const NIM_TRELLIS_COOLDOWN_KEY = 'forge-nim-trellis';+export const NIM_TRELLIS_COOLDOWN_KEY = 'forge-nim-trellis';// Sideline the free NIM lane after a failure. The cooldown exists to avoid// re-paying the expensive submit-timeout HANG, so a socket timeout / unreachable// host (no HTTP status came back) earns the full window. A fast gateway 5xx (adiff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.jsindex ae6266790..aa2c66327 100644--- a/tests/forge-finalize.test.js+++ b/tests/forge-finalize.test.js@@ -4,7 +4,7 @@// Importing the cron module is safe: db/neon clients are created lazily.import { describe, it, expect } from 'vitest';-import { decodeWorkerEnvelope, decideFailedSweep } from '../api/cron/forge-finalize.js';+import { decodeWorkerEnvelope, decideFailedSweep, decideNvidiaStall } from '../api/cron/forge-finalize.js';function pack(obj) {return Buffer.from(JSON.stringify(obj), 'utf8').toString('base64url');@@ -81,3 +81,34 @@ describe('decideFailedSweep', () => {expect(decideFailedSweep({ ageMinutes: undefined, hasReferenceImage: true })).toBe('redispatch');});});++// The free NVIDIA NIM text lane has no alternate lane to fail a stuck prompt+// over to (forge-failover.js excludes it: text-only, no stored reference image+// to reconstruct from), so a stalled poll gets exactly one resubmit attempt+// before the hard TTL reaps it, instead of silently burning the full 45 minutes+// on a request that will never resolve on its own.+describe('decideNvidiaStall', () => {+ it('defers while an attended client may still be polling', () => {+ expect(decideNvidiaStall({ ageMinutes: 3 })).toBe('defer');+ expect(decideNvidiaStall({ ageMinutes: 12.9 })).toBe('defer');+ });++ it('resubmits once the attended window has passed and the hard TTL has not hit', () => {+ expect(decideNvidiaStall({ ageMinutes: 13 })).toBe('resubmit');+ expect(decideNvidiaStall({ ageMinutes: 30 })).toBe('resubmit');+ });++ it('never resubmits a job twice', () => {+ expect(decideNvidiaStall({ ageMinutes: 30, alreadyResubmitted: true })).toBe('defer');+ });++ it('goes terminal at or past the hard TTL regardless of the resubmit flag', () => {+ expect(decideNvidiaStall({ ageMinutes: 45 })).toBe('terminal');+ expect(decideNvidiaStall({ ageMinutes: 50, alreadyResubmitted: true })).toBe('terminal');+ expect(decideNvidiaStall({ ageMinutes: 45, alreadyResubmitted: false })).toBe('terminal');+ });++ it('treats an unknowable age as inside the attended window rather than resubmitting blind', () => {+ expect(decideNvidiaStall({ ageMinutes: undefined })).toBe('defer');+ });+});