Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): record an in-flight marker before the nvidia expiry resubmit and alert on its failure

nirholasd269cc4ef156
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.js
index 4f538c6e7..f0b3f8065 100644
--- a/api/cron/forge-finalize.js
+++ b/api/cron/forge-finalize.js
@@ -55,6 +55,7 @@ import { markLaneUnhealthy } from '../_lib/forge-lane-health.js';
import { markProviderCooldown } from '../_lib/provider-health.js';
import { resolveTier } from '../_lib/forge-tiers.js';
import { acquireLock, cacheGet, cacheSet } from '../_lib/cache.js';
+import { sendOpsAlert } from '../_lib/alerts.js';
import { getRedis } from '../_lib/redis.js';
import { NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds } from '../forge.js';
import { mapPoolSettled } from '../_lib/pool.js';
@@ -159,6 +160,25 @@ export function decideFailedSweep({
return 'redispatch';
}
+const NVCF_RESUB_INFLIGHT_TTL_S = 5 * 60;
+
+// Written BEFORE the paid NVCF submission. A tick cut off mid-flight (Cloud Run
+// withdraws CPU once wrapCron answers) leaves this marker as proof an attempt
+// began, and a slow-but-alive attempt is never re-fired by a later tick.
+async function markResubmitInflight(jobId) {
+ await cacheSet(`nvcf:resub-inflight:${jobId}`, String(Date.now()), NVCF_RESUB_INFLIGHT_TTL_S);
+}
+
+// console.warn from this handler has never reached Cloud Logging, so a failed
+// resubmit goes to the durable ops_alerts sink as well.
+async function alertResubmitFailed(row, err) {
+ await sendOpsAlert(
+ 'forge-finalize: NVCF resubmit failed',
+ `creation ${row.id}: ${err?.message || err}`,
+ { signature: 'forge-finalize:nvcf-resubmit-failed', severity: 'warn' },
+ ).catch(() => {});
+}
+
/**
* NVIDIA NIM is the only text-to-3D lane (forge-failover.js excludes it from
* redispatch: text-only, and a poll-time redispatch always reconstructs from a
@@ -529,8 +549,13 @@ export async function sweepRow(row, { gcp, nvidia }) {
await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),
),
});
- if (stallDecision === 'resubmit' && (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))) {
+ if (
+ stallDecision === 'resubmit' &&
+ !(await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`).catch(() => null)) &&
+ (await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))
+ ) {
try {
+ await markResubmitInflight(row.replicate_job_id);
const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);
if (resub.taskId) {
@@ -554,6 +579,7 @@ export async function sweepRow(row, { gcp, nvidia }) {
}
} catch (err) {
console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);
+ await alertResubmitFailed(row, err);
}
}
}
@@ -563,8 +589,12 @@ export async function sweepRow(row, { gcp, nvidia }) {
// once-per-job resub lock so the two paths can't double-submit.
// The alias keeps the row's handle stable; the next tick polls the
// resubmitted task and materializes onto this same row.
- if (await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S)) {
+ if (
+ !(await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`).catch(() => null)) &&
+ (await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))
+ ) {
try {
+ await markResubmitInflight(row.replicate_job_id);
const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
if (resub.taskId) {
await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
@@ -587,6 +617,7 @@ export async function sweepRow(row, { gcp, nvidia }) {
}
} catch (err) {
console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);
+ await alertResubmitFailed(row, err);
}
}
// Lock held by a prior recovery or the resubmit failed: the lane is
diff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.js
index 79eab1ac0..34a4c30b8 100644
--- a/tests/forge-finalize.test.js
+++ b/tests/forge-finalize.test.js
@@ -15,7 +15,8 @@ import { mapPoolSettled } from '../api/_lib/pool.js';
// hoisted above every import (including transitive ones like
// api/_lib/rate-limit.js's own `sql` import), so the mocked values must come
// from vi.hoisted rather than a plain top-level const.
-const { sql, gcpStatus, nvidiaStatus, notifyForgeFailed, markProviderCooldown } = vi.hoisted(() => ({
+const { sql, gcpStatus, nvidiaStatus, notifyForgeFailed, markProviderCooldown, sendOpsAlert } = vi.hoisted(() => ({
+ sendOpsAlert: vi.fn(async () => {}),
sql: vi.fn(async () => []),
gcpStatus: vi.fn(async () => {
throw new Error('reap pass must not poll gcp');
@@ -49,6 +50,8 @@ vi.mock('../api/_lib/forge-notify.js', () => ({
vi.mock('../api/_lib/provider-health.js', () => ({ markProviderCooldown }));
+vi.mock('../api/_lib/alerts.js', () => ({ sendOpsAlert }));
+
const {
decodeWorkerEnvelope,
decideFailedSweep,
@@ -304,3 +307,64 @@ describe('TTL reap runs ahead of the live poll', () => {
expect(notifyForgeFailed).not.toHaveBeenCalled();
});
});
+
+// Measured 2026-10-25: 15 text-prompt nvidia rows failed "NVCF request not
+// found or expired" in 2 days and not one `[forge-finalize]` log line reached
+// Cloud Logging in 7 days, so nobody could tell whether the same-lane resubmit
+// ever ran. The resubmit must leave a durable marker BEFORE the paid NVCF
+// submission, so a tick cut off mid-flight is distinguishable from "never tried".
+describe('nvidia resubmit on expiry', () => {
+ function nvidiaRow(n) {
+ return {
+ id: `nv-row-${n}`,
+ replicate_job_id: `3f2b9c1e-0000-4000-8000-00000000${String(n).padStart(4, '0')}`,
+ client_key: `client-${n}`,
+ user_id: null,
+ prompt: 'a test prompt',
+ preview_image_url: null,
+ backend: 'nvidia',
+ tier: 'draft',
+ path: 'image',
+ created_at: new Date(Date.now() - 5 * 60_000).toISOString(),
+ destination: null,
+ internal: false,
+ };
+ }
+ const expiredStatus = async () => ({
+ status: 'failed',
+ code: 'nvcf_expired',
+ error: 'NVCF request not found or expired',
+ });
+
+ it('resubmits a text-prompt nvidia row whose request id expired', async () => {
+ const textTo3d = vi.fn(async () => ({ taskId: 'resubmit-task-id' }));
+ const out = await sweepRow(nvidiaRow(1), { gcp: null, nvidia: { status: expiredStatus, textTo3d } });
+ expect(out.resubmitted).toBe(1);
+ expect(out.failed).toBe(0);
+ expect(out.done).toBe(0);
+ expect(textTo3d).toHaveBeenCalledTimes(1);
+ });
+
+ it('records an in-flight marker before the NVCF submission is made', async () => {
+ const { cacheGet } = await import('../api/_lib/cache.js');
+ const row = nvidiaRow(2);
+ let markerSeenDuringSubmit = null;
+ const textTo3d = vi.fn(async () => {
+ markerSeenDuringSubmit = await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`);
+ return { taskId: 'resubmit-task-id-2' };
+ });
+ await sweepRow(row, { gcp: null, nvidia: { status: expiredStatus, textTo3d } });
+ expect(markerSeenDuringSubmit).toBeTruthy();
+ });
+
+ it('routes a failed resubmit through the durable ops alert, not only console.warn', async () => {
+ sendOpsAlert.mockClear();
+ const textTo3d = vi.fn(async () => {
+ throw new Error('NVCF down');
+ });
+ const out = await sweepRow(nvidiaRow(3), { gcp: null, nvidia: { status: expiredStatus, textTo3d } });
+ expect(out.resubmitted).toBe(0);
+ expect(sendOpsAlert).toHaveBeenCalledTimes(1);
+ expect(sendOpsAlert.mock.calls[0][1]).toContain('NVCF down');
+ });
+});