Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): record an in-flight marker before the nvidia expiry resubmit and alert on its failure
nirholasd269cc4ef156
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.jsindex 4f538c6e7..f0b3f8065 100644--- a/api/cron/forge-finalize.js+++ b/api/cron/forge-finalize.js@@ -55,6 +55,7 @@ import { markLaneUnhealthy } from '../_lib/forge-lane-health.js';import { markProviderCooldown } from '../_lib/provider-health.js';import { resolveTier } from '../_lib/forge-tiers.js';import { acquireLock, cacheGet, cacheSet } from '../_lib/cache.js';+import { sendOpsAlert } from '../_lib/alerts.js';import { getRedis } from '../_lib/redis.js';import { NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds } from '../forge.js';import { mapPoolSettled } from '../_lib/pool.js';@@ -159,6 +160,25 @@ export function decideFailedSweep({return 'redispatch';}+const NVCF_RESUB_INFLIGHT_TTL_S = 5 * 60;++// Written BEFORE the paid NVCF submission. A tick cut off mid-flight (Cloud Run+// withdraws CPU once wrapCron answers) leaves this marker as proof an attempt+// began, and a slow-but-alive attempt is never re-fired by a later tick.+async function markResubmitInflight(jobId) {+ await cacheSet(`nvcf:resub-inflight:${jobId}`, String(Date.now()), NVCF_RESUB_INFLIGHT_TTL_S);+}++// console.warn from this handler has never reached Cloud Logging, so a failed+// resubmit goes to the durable ops_alerts sink as well.+async function alertResubmitFailed(row, err) {+ await sendOpsAlert(+ 'forge-finalize: NVCF resubmit failed',+ `creation ${row.id}: ${err?.message || err}`,+ { signature: 'forge-finalize:nvcf-resubmit-failed', severity: 'warn' },+ ).catch(() => {});+}+/*** NVIDIA NIM is the only text-to-3D lane (forge-failover.js excludes it from* redispatch: text-only, and a poll-time redispatch always reconstructs from a@@ -529,8 +549,13 @@ export async function sweepRow(row, { gcp, nvidia }) {await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),),});- if (stallDecision === 'resubmit' && (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))) {+ if (+ stallDecision === 'resubmit' &&+ !(await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`).catch(() => null)) &&+ (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))+ ) {try {+ await markResubmitInflight(row.replicate_job_id);const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);if (resub.taskId) {@@ -554,6 +579,7 @@ export async function sweepRow(row, { gcp, nvidia }) {}} catch (err) {console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);+ await alertResubmitFailed(row, err);}}}@@ -563,8 +589,12 @@ export async function sweepRow(row, { gcp, nvidia }) {// once-per-job resub lock so the two paths can't double-submit.// The alias keeps the row's handle stable; the next tick polls the// resubmitted task and materializes onto this same row.- if (await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S)) {+ if (+ !(await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`).catch(() => null)) &&+ (await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))+ ) {try {+ await markResubmitInflight(row.replicate_job_id);const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });if (resub.taskId) {await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);@@ -587,6 +617,7 @@ export async function sweepRow(row, { gcp, nvidia }) {}} catch (err) {console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);+ await alertResubmitFailed(row, err);}}// Lock held by a prior recovery or the resubmit failed: the lane isdiff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.jsindex 79eab1ac0..34a4c30b8 100644--- a/tests/forge-finalize.test.js+++ b/tests/forge-finalize.test.js@@ -15,7 +15,8 @@ import { mapPoolSettled } from '../api/_lib/pool.js';// hoisted above every import (including transitive ones like// api/_lib/rate-limit.js's own `sql` import), so the mocked values must come// from vi.hoisted rather than a plain top-level const.-const { sql, gcpStatus, nvidiaStatus, notifyForgeFailed, markProviderCooldown } = vi.hoisted(() => ({+const { sql, gcpStatus, nvidiaStatus, notifyForgeFailed, markProviderCooldown, sendOpsAlert } = vi.hoisted(() => ({+ sendOpsAlert: vi.fn(async () => {}),sql: vi.fn(async () => []),gcpStatus: vi.fn(async () => {throw new Error('reap pass must not poll gcp');@@ -49,6 +50,8 @@ vi.mock('../api/_lib/forge-notify.js', () => ({vi.mock('../api/_lib/provider-health.js', () => ({ markProviderCooldown }));+vi.mock('../api/_lib/alerts.js', () => ({ sendOpsAlert }));+const {decodeWorkerEnvelope,decideFailedSweep,@@ -304,3 +307,64 @@ describe('TTL reap runs ahead of the live poll', () => {expect(notifyForgeFailed).not.toHaveBeenCalled();});});++// Measured 2026-10-25: 15 text-prompt nvidia rows failed "NVCF request not+// found or expired" in 2 days and not one `[forge-finalize]` log line reached+// Cloud Logging in 7 days, so nobody could tell whether the same-lane resubmit+// ever ran. The resubmit must leave a durable marker BEFORE the paid NVCF+// submission, so a tick cut off mid-flight is distinguishable from "never tried".+describe('nvidia resubmit on expiry', () => {+ function nvidiaRow(n) {+ return {+ id: `nv-row-${n}`,+ replicate_job_id: `3f2b9c1e-0000-4000-8000-00000000${String(n).padStart(4, '0')}`,+ client_key: `client-${n}`,+ user_id: null,+ prompt: 'a test prompt',+ preview_image_url: null,+ backend: 'nvidia',+ tier: 'draft',+ path: 'image',+ created_at: new Date(Date.now() - 5 * 60_000).toISOString(),+ destination: null,+ internal: false,+ };+ }+ const expiredStatus = async () => ({+ status: 'failed',+ code: 'nvcf_expired',+ error: 'NVCF request not found or expired',+ });++ it('resubmits a text-prompt nvidia row whose request id expired', async () => {+ const textTo3d = vi.fn(async () => ({ taskId: 'resubmit-task-id' }));+ const out = await sweepRow(nvidiaRow(1), { gcp: null, nvidia: { status: expiredStatus, textTo3d } });+ expect(out.resubmitted).toBe(1);+ expect(out.failed).toBe(0);+ expect(out.done).toBe(0);+ expect(textTo3d).toHaveBeenCalledTimes(1);+ });++ it('records an in-flight marker before the NVCF submission is made', async () => {+ const { cacheGet } = await import('../api/_lib/cache.js');+ const row = nvidiaRow(2);+ let markerSeenDuringSubmit = null;+ const textTo3d = vi.fn(async () => {+ markerSeenDuringSubmit = await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`);+ return { taskId: 'resubmit-task-id-2' };+ });+ await sweepRow(row, { gcp: null, nvidia: { status: expiredStatus, textTo3d } });+ expect(markerSeenDuringSubmit).toBeTruthy();+ });++ it('routes a failed resubmit through the durable ops alert, not only console.warn', async () => {+ sendOpsAlert.mockClear();+ const textTo3d = vi.fn(async () => {+ throw new Error('NVCF down');+ });+ const out = await sweepRow(nvidiaRow(3), { gcp: null, nvidia: { status: expiredStatus, textTo3d } });+ expect(out.resubmitted).toBe(0);+ expect(sendOpsAlert).toHaveBeenCalledTimes(1);+ expect(sendOpsAlert.mock.calls[0][1]).toContain('NVCF down');+ });+});