Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// GET /api/cron/forge-finalize: server-side completion for forge generations.
//
// Historically a forge_creations row only flipped from 'generating' to
// 'done'/'failed' when the creator's browser polled GET /api/forge?job=… to
// completion (materializeCreation runs inside that poll). Close the tab and the
// generation is orphaned at 'generating' forever, even though the GPU worker
// finished and the GLB is sitting in GCS. This cron makes completion
// server-authoritative so users can leave the page mid-generation:
//
// 1. Sweep 'generating' rows older than GRACE (the browser poll path gets
// first shot: most generations still finish attended).
// 2. Rows whose replicate_job_id is a GCP worker envelope (the async
// self-host lanes: trellis_selfhost, hunyuan3d, sketch/triposg: exactly
// the minutes-long lanes that get orphaned) are polled directly on the
// worker. done → materializeCreation (same durable-copy writer the
// browser poll uses, so gallery/showcase/streaks all light up
// identically). failed → markFailed.
// 3. Anything still not terminal after HARD_TTL is marked failed as timed
// out, so no row is ever stuck at 'generating' forever. Envelope rows are
// still polled once first: recovering a finished model always beats
// declaring it dead.
// 4. A creation finalized here completed UNATTENDED: the user wasn't
// watching: so this is the one place completion notifications fire
// (bell + push + email via api/_lib/forge-notify). The attended path
// never notifies: the result is already on screen.
//
// Idempotent + swarm-safe: materializeCreation short-circuits on already-done
// rows, markFailed is guarded by status != 'done', and the batch is bounded so
// overlapping ticks just split the work.
//
// Auth: the shared, fail-closed cron gate (api/_lib/cron-auth.js), same as every
// other cron in /api/cron/.
import { sql } from '../_lib/db.js';
import { cors, json, method, wrapCron } from '../_lib/http.js';
import { requireCron } from '../_lib/cron-auth.js';
import {
materializeCreation,
markFailed,
markSupersededBy,
forgeStoreEnabled,
createCreation,
} from '../_lib/forge-store.js';
import { notifyForgeComplete, notifyForgeFailed } from '../_lib/forge-notify.js';
import { createRegenProvider as createGcpProvider } from '../_providers/gcp.js';
import { createNvidiaProvider } from '../_providers/nvidia.js';
import { decodeJobToken, encodeJobToken } from '../_lib/forge-job-token.js';
import {
pickRedispatchLane,
submitFailoverJob,
bindJobSuccessor,
MAX_FAILOVER_HOPS,
} from '../_lib/forge-failover.js';
import { markLaneUnhealthy } from '../_lib/forge-lane-health.js';
import { markProviderCooldown } from '../_lib/provider-health.js';
import { resolveTier } from '../_lib/forge-tiers.js';
import { acquireLock, cacheGet, cacheSet } from '../_lib/cache.js';
import { sendOpsAlert } from '../_lib/alerts.js';
import { getRedis } from '../_lib/redis.js';
import { NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds } from '../forge.js';
import { mapPoolSettled } from '../_lib/pool.js';
export const maxDuration = 60;
const BATCH_LIMIT = Number(process.env.FORGE_FINALIZE_BATCH) || 25;
// Each row's status poll is an independent network round trip
// (api/_providers/gcp.js status() allows up to 15s x 2 attempts on its own), so
// sweeping BATCH_LIMIT rows one at a time can take minutes once a lane like
// trellis_selfhost is unhealthy and every poll on it runs its full timeout.
// That is exactly what blew the 60s cron budget on 2026-10-24 (see
// docs/ops/production-log-triage.md "HTTP timeout with no cron context"): rows
// are independent (keyed by their own replicate_job_id, no shared mutable
// state besides the stateless provider clients), so sweeping them concurrently
// bounds the tick's wall-clock time to the slowest row instead of the sum.
export const ROW_CONCURRENCY = Number(process.env.FORGE_FINALIZE_CONCURRENCY) || 8;
// Leave fresh rows to the attended browser poll path.
const GRACE_MINUTES = 2;
// Past this age a non-terminal generation is declared dead. The slowest real
// lane (self-host TRELLIS high tier behind a full queue) finishes well inside
// 30 minutes; beyond it the worker task has been evicted or lost.
const HARD_TTL_MINUTES = 45;
// Browser clients poll attended for up to 12 minutes (src/forge.js MAX_POLL_MS)
// and run their own poll-time lane failover. The cron defers acting on a failed
// self-host poll until that window has passed, so it can never race a live
// browser into a duplicate redispatch of the same generation.
const ATTENDED_POLL_BUDGET_MINUTES = 13;
// Hop/attempted memory for cron-built failover chains, keyed by the successor's
// worker envelope. Same lifetime as the poll path's successor bindings.
const CRON_HOP_PREFIX = 'fr:cron-hop:';
const CRON_HOP_TTL_S = 2 * 3600;
// Mirrors the poll path's NVCF alias/resub keys (api/forge.js pollNvidiaStatus)
// so attended and unattended recovery share one once-per-job budget.
const NVCF_ALIAS_TTL_S = 3600;
// The async GCP worker lanes wrap their upstream task in a base64url JSON
// envelope ({ mode, taskId, baseUrl, resultKey }): see api/_providers/gcp.js.
// That shape is the reliable discriminator between "a worker we can poll" and
// an opaque third-party id (Replicate prediction, HF job) we cannot.
export function decodeWorkerEnvelope(replicateJobId) {
try {
const obj = JSON.parse(Buffer.from(String(replicateJobId), 'base64url').toString('utf8'));
return obj?.taskId && obj?.baseUrl ? obj : null;
} catch {
return null;
}
}
async function readCronHop(replicateJobId) {
const r = getRedis();
if (!r) return null;
try {
const raw = await r.get(`${CRON_HOP_PREFIX}${replicateJobId}`);
const value = typeof raw === 'string' ? JSON.parse(raw) : raw;
return value && Number.isFinite(value.hop) ? value : null;
} catch {
return null;
}
}
async function writeCronHop(replicateJobId, value) {
const r = getRedis();
if (!r) return;
try {
await r.set(`${CRON_HOP_PREFIX}${replicateJobId}`, JSON.stringify(value), { ex: CRON_HOP_TTL_S });
} catch {
// Best-effort: a lost hop record loosens the chain cap by one, nothing more.
}
}
/**
* Decide what this sweep may do with a self-host poll that came back FAILED.
* Pure: no clock, no I/O, no Redis - the caller supplies the resolved facts.
* Factored out (like decideSelfhostMissing in _lib/forge-selfhost-recovery.js)
* because the swarm-safety rule here is subtle enough to deserve its own tests:
* acting too early races a live browser into duplicate GPU work, and acting
* never is what made the orphan class permanent.
*
* @returns {'defer'|'redispatch'|'terminal'}
* - defer: an attended client may still handle it; leave the row alone.
* - redispatch: resubmit to the next healthy lane.
* - terminal: nothing left to try; mark it failed and notify.
*/
export function decideFailedSweep({
ageMinutes,
hop = 0,
hasReferenceImage = false,
path = null,
attendedBudgetMinutes = ATTENDED_POLL_BUDGET_MINUTES,
maxHops = MAX_FAILOVER_HOPS,
} = {}) {
// Inside the attended window a live browser runs its own poll-time failover.
// Acting here would duplicate that GPU work on the same generation.
if (Number.isFinite(ageMinutes) && ageMinutes < attendedBudgetMinutes) return 'defer';
// A redispatch reconstructs from the stored reference view, so a job without
// one has nothing to resubmit. Sketch keeps its single purpose-built lane
// (reconstruct models do badly on drawings).
if (!hasReferenceImage || path === 'sketch') return 'terminal';
// Bounded chain, same cap as the attended path.
if (hop >= maxHops) return 'terminal';
return 'redispatch';
}
const NVCF_RESUB_INFLIGHT_TTL_S = 5 * 60;
// Written BEFORE the paid NVCF submission. A tick cut off mid-flight (Cloud Run
// withdraws CPU once wrapCron answers) leaves this marker as proof an attempt
// began, and a slow-but-alive attempt is never re-fired by a later tick.
async function markResubmitInflight(jobId) {
await cacheSet(`nvcf:resub-inflight:${jobId}`, String(Date.now()), NVCF_RESUB_INFLIGHT_TTL_S);
}
// console.warn from this handler has never reached Cloud Logging, so a failed
// resubmit goes to the durable ops_alerts sink as well.
async function alertResubmitFailed(row, err) {
await sendOpsAlert(
'forge-finalize: NVCF resubmit failed',
`creation ${row.id}: ${err?.message || err}`,
{ signature: 'forge-finalize:nvcf-resubmit-failed', severity: 'warn' },
).catch(() => {});
}
/**
* NVIDIA NIM is the only text-to-3D lane (forge-failover.js excludes it from
* redispatch: text-only, and a poll-time redispatch always reconstructs from a
* stored reference image nvidia rows never have), so a stalled poll has no
* alternate lane to fail over to. The only two defenses are: defer to the
* attended browser poll while one may still be running, then give the job one
* real resubmit before the hard TTL reaps it as dead.
*
* @returns {'defer'|'resubmit'|'terminal'}
* - defer: an attended client may still be polling, or this job already got
* its one resubmit; leave the row alone.
* - resubmit: past the attended window, before the hard TTL, not yet retried.
* - terminal: past the hard TTL; nothing left to try.
*/
export function decideNvidiaStall({
ageMinutes,
alreadyResubmitted = false,
attendedBudgetMinutes = ATTENDED_POLL_BUDGET_MINUTES,
hardTtlMinutes = HARD_TTL_MINUTES,
} = {}) {
if (!Number.isFinite(ageMinutes) || ageMinutes < attendedBudgetMinutes) return 'defer';
if (ageMinutes >= hardTtlMinutes) return 'terminal';
if (alreadyResubmitted) return 'defer';
return 'resubmit';
}
// Unattended twin of the poll handler's automatic lane failover (api/forge.js
// pollJob). The 30-minute worker orphan class lands here by construction: the
// worker only declares an orphan long after every browser has stopped polling,
// so the cron is the ONLY consumer of that failure, and before this it
// dead-ended the job permanently. The original inputs are still on the row;
// resubmit to the next healthy lane instead. The successor gets its own
// creation row (same client/user), so this same cron sweeps it to completion
// and the completion notification still fires for a model, not a failure.
//
// Returns `{ successorId }` when the request now continues on another lane (the
// caller links the two rows so the ledger shows a recovery, not a loss), or
// null when nothing was redispatched.
async function tryCronRedispatch(row, ageMinutes) {
const prior = await readCronHop(row.replicate_job_id);
const hop = prior ? prior.hop : 0;
const decision = decideFailedSweep({
ageMinutes,
hop,
hasReferenceImage: Boolean(row.preview_image_url),
path: row.path,
});
if (decision !== 'redispatch') return null;
const attempted = [...new Set([...(prior?.attempted || []), row.backend].filter(Boolean))];
const nextLane = await pickRedispatchLane({ attempted });
if (!nextLane) {
// The one recoverable failure that stays lost, and it used to leave no
// trace: every other outcome here logs, so a silent return read as "we
// never tried". Measured on 2026-08-14: of the 14 self-host orphans in the
// prior week, 13 failed over and the single loss came through this branch,
// with nothing in the logs to say whether a lane was picked at all. A
// declined failover is a capacity signal (every alternative lane already
// attempted or cooled by markLaneUnhealthy), so say so.
console.warn(
`[forge-finalize] job failed on ${row.backend}; no redispatch lane available (attempted: ${attempted.join(', ') || 'none'}), leaving it terminal`,
);
return null;
}
try {
const submitted = await submitFailoverJob({
backend: nextLane,
imageUrl: row.preview_image_url,
prompt: row.prompt,
tierId: row.tier,
path: row.path,
});
// The redispatch reconstructs from the primary stored view, so a
// multi-view original degrades visibly (views_used: 1), never silently:
// the same provenance contract as the attended failover.
const successorId = await createCreation({
clientKey: row.client_key,
userId: row.user_id ?? null,
prompt: row.prompt,
previewImageUrl: row.preview_image_url,
replicateJobId: submitted.extJobId,
viewsRequested: 1,
viewsUsed: 1,
multiview: false,
backend: nextLane,
tier: row.tier,
path: row.path,
destination: row.destination,
internal: row.internal === true,
});
await writeCronHop(submitted.extJobId, { hop: hop + 1, attempted });
// A late attended poller still holds the original f1 handle. Job tokens
// are deterministic (HMAC over the same payload), so re-encoding the
// envelope reconstructs the exact string that client polls, so bind the
// successor chain on it so resolveLiveJob chases to the live job.
await bindJobSuccessor(
encodeJobToken({ provider: 'gcp', kind: null, taskId: row.replicate_job_id }),
{ handle: submitted.handle, backend: nextLane, hop: hop + 1, attempted },
);
console.warn(
`[forge-finalize] job failed on ${row.backend}; unattended auto-failover #${hop + 1} → ${nextLane}`,
);
return { successorId: successorId ?? null };
} catch (err) {
console.warn(`[forge-finalize] redispatch failed: ${err?.message || err}`);
return null;
}
}
// Poll-free, BATCH_LIMIT-free hard-TTL reap: a single bulk UPDATE that flips
// every row already past HARD_TTL_MINUTES to 'failed', no matter how old or
// which backend/path it belongs to. This runs ahead of (and independently of)
// the oldest-created_at-first, BATCH_LIMIT-bounded live-poll query below, so a
// batch monopolized by one unhealthy backend (trellis_selfhost owning every
// slot because its rows are the oldest) can never block a different, never-
// selected lane's rows from ever being reaped: exactly the bug measured
// 2026-10-25 in prompts/finish/436: 192 `rerig` rows sat 'generating' forever
// because they never won a batch slot against older trellis_selfhost rows.
// Message/column shape matches forge-store.js's markFailed exactly (error,
// completed_at, updated_at) so no downstream matcher (forge-error-report.mjs's
// `error ilike '%timed out%'`) sees a different error string.
export async function reapExpiredRows() {
const rows = await sql`
update forge_creations
set status = 'failed',
error = 'generation timed out after ' || round(extract(epoch from (now() - created_at)) / 60) || ' minutes',
completed_at = coalesce(completed_at, now()),
updated_at = now()
where status = 'generating'
and replicate_job_id is not null
and created_at < now() - (${HARD_TTL_MINUTES} * interval '1 minute')
returning id, user_id, prompt, backend, tier, path
`;
for (const row of rows) {
if (row.backend === 'nvidia') {
// Reaping the bare TTL on nvidia is itself a cron-observed failure of
// the lane: cool the submit-time breaker so new free-text requests
// skip straight to the paid/image-intermediate path instead of landing
// on the same dead end for another HARD_TTL_MINUTES.
await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
}
if (row.user_id) {
notifyForgeFailed({
userId: row.user_id,
creationId: row.id,
prompt: row.prompt,
error: 'generation timed out',
backend: row.backend,
tier: row.tier,
});
}
}
return rows.length;
}
export default wrapCron(async (req, res) => {
if (cors(req, res, { methods: 'GET,POST,OPTIONS' })) return;
if (!method(req, res, ['GET', 'POST'])) return;
if (!requireCron(req, res)) return;
if (!forgeStoreEnabled()) {
return json(res, 200, { enabled: false, swept: 0 });
}
const t0 = Date.now();
const reaped = await reapExpiredRows();
const rows = await sql`
select id, replicate_job_id, client_key, user_id, prompt, preview_image_url,
backend, tier, path, created_at, destination, internal
from forge_creations
where status = 'generating'
and replicate_job_id is not null
and created_at < now() - (${GRACE_MINUTES} * interval '1 minute')
order by created_at asc
limit ${BATCH_LIMIT}
`;
let gcp = null;
try {
gcp = createGcpProvider();
} catch {
// No GCP_RECONSTRUCTION_KEY on this deployment: envelope rows can't be
// polled here; the hard TTL below still reaps them.
}
let nvidia = null;
try {
nvidia = createNvidiaProvider();
} catch {
// NIM unconfigured: forge-token rows can't be polled here; hard TTL reaps.
}
const results = await mapPoolSettled(rows, ROW_CONCURRENCY, (row) => sweepRow(row, { gcp, nvidia }));
const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: reaped, still_running: 0, unpollable: 0 };
for (const result of results) {
if (result.ok) {
for (const key of Object.keys(out)) out[key] += result.value[key] || 0;
} else {
console.warn(`[forge-finalize] row sweep threw: ${result.error?.message || result.error}`);
out.unpollable++;
}
}
return json(res, 200, {
enabled: true,
reaped,
swept: rows.length,
...out,
elapsed_ms: Date.now() - t0,
});
}, { maxDurationMs: maxDuration * 1000 });
// One row's worth of the sweep above, factored out so it can run concurrently
// across the batch (mapPoolSettled in the caller) instead of blocking the next
// row on this row's network round trips. Returns the per-row delta to merge
// into the tick's aggregate counters; never throws (mapPoolSettled would catch
// it anyway, but a thrown row loses its counters, so every awaited call here
// already carries its own `.catch`).
export async function sweepRow(row, { gcp, nvidia }) {
const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: 0, still_running: 0, unpollable: 0 };
const ageMinutes = (Date.now() - Date.parse(row.created_at)) / 60_000;
const envelope = decodeWorkerEnvelope(row.replicate_job_id);
if (envelope && gcp) {
const status = await gcp.status(row.replicate_job_id).catch(() => null);
if (status?.status === 'done' && status.resultGlbUrl) {
const durable = await materializeCreation({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
glbUrl: status.resultGlbUrl,
});
if (durable) {
out.done++;
if (row.user_id) {
notifyForgeComplete({
userId: row.user_id,
creationId: durable.id,
prompt: row.prompt,
previewImageUrl: durable.previewImageUrl ?? row.preview_image_url ?? null,
});
}
return out;
}
// Copy-to-durable failed (transient storage/network): leave the row
// for the next tick rather than failing a finished model.
out.still_running++;
return out;
}
if (status?.status === 'failed') {
// Inside the attended window a live browser may poll this failure
// seconds from now and run its own failover; acting here would race
// it into duplicate GPU work. Leave the row: an attended client
// resolves it, and an abandoned one ages into the branch below.
if (decideFailedSweep({ ageMinutes, hasReferenceImage: true }) === 'defer') {
out.still_running++;
return out;
}
if (row.backend) await markLaneUnhealthy(row.backend).catch(() => {});
const redispatched = await tryCronRedispatch(row, ageMinutes);
if (redispatched) {
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
error: status.error || 'generation failed',
});
// Link the attempt to its successor so the ledger reads as a
// recovery: without it the health sensor and the error report both
// count this row as a user-visible loss, when the request is in
// fact still running on another lane.
if (redispatched.successorId) {
await markSupersededBy({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
successorId: redispatched.successorId,
});
}
out.failed_over++;
// No failure notification: the successor row carries the job on,
// and its completion (or terminal failure) notifies instead.
return out;
}
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
error: status.error || 'generation failed',
});
out.failed++;
if (row.user_id) {
notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: status.error, backend: row.backend, tier: row.tier });
}
return out;
}
// queued / running / poll blip: fall through to the TTL check.
} else if (!envelope) {
// The paid x402 lane stores the FULL signed forge token (provider +
// upstream task id) so unattended agent purchases can be completed
// server-side here, so an agent that got its poll_url answer and moved on
// still ends with a finished gallery row, not a TTL orphan.
const forgeToken = decodeJobToken(row.replicate_job_id);
if (forgeToken?.provider === 'nvidia' && forgeToken.taskId && nvidia) {
const status = await nvidia
.status({ kind: forgeToken.kind, taskId: forgeToken.taskId })
.catch(() => null);
if (status?.status === 'done' && status.resultGlbUrl) {
const durable = await materializeCreation({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
glbUrl: status.resultGlbUrl,
});
if (durable) {
out.done++;
return out;
}
out.still_running++;
return out;
}
if (status?.status === 'failed') {
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
error: status.error || 'generation failed',
});
out.failed++;
return out;
}
// queued / running / poll blip: fall through to the TTL check.
} else if (row.backend === 'nvidia' && nvidia) {
// Browser/free-lane NIM rows store the BARE NVCF request id (only the
// x402 lane stores the signed token), so they used to be unpollable
// here and died at the hard TTL as "generation timed out" even when
// the GPU had finished. Poll the bare id directly, chasing the alias
// an attended expiry recovery may have written first.
const alias = await cacheGet(`nvcf:alias:${row.replicate_job_id}`).catch(() => null);
const taskId = typeof alias === 'string' && alias ? alias : row.replicate_job_id;
const status = await nvidia.status({ taskId }).catch(() => null);
if (status?.status === 'done' && status.resultGlbUrl) {
const durable = await materializeCreation({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
glbUrl: status.resultGlbUrl,
});
if (durable) {
out.done++;
if (row.user_id) {
notifyForgeComplete({
userId: row.user_id,
creationId: durable.id,
prompt: row.prompt,
previewImageUrl: durable.previewImageUrl ?? row.preview_image_url ?? null,
});
}
return out;
}
out.still_running++;
return out;
}
if ((!status || status.status === 'running' || status.status === 'queued') && row.prompt) {
// Still running/queued, or the poll itself errored (status null): no
// alternate lane exists for text-only nvidia, so give the job one real
// resubmit before the hard TTL reaps it as dead. A distinct lock from
// nvcf:resub: below means this can never double-fire with the
// nvcf_expired recovery.
const stallDecision = decideNvidiaStall({
ageMinutes,
alreadyResubmitted: Boolean(
await cacheGet(`nvcf:stall-resub:${row.replicate_job_id}`).catch(() => null),
),
});
if (
stallDecision === 'resubmit' &&
!(await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`).catch(() => null)) &&
(await acquireLock(`nvcf:stall-lock:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))
) {
try {
await markResubmitInflight(row.replicate_job_id);
const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
await cacheSet(`nvcf:stall-resub:${row.replicate_job_id}`, '1', NVCF_ALIAS_TTL_S);
if (resub.taskId) {
await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
console.warn(
`[forge-finalize] NVCF request stalled; resubmitted unattended job (creation ${row.id})`,
);
out.resubmitted++;
return out;
}
if (resub.resultGlbUrl) {
const durable = await materializeCreation({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
glbUrl: resub.resultGlbUrl,
});
if (durable) {
out.done++;
return out;
}
}
} catch (err) {
console.warn(`[forge-finalize] NVCF stall resubmit failed: ${err?.message || err}`);
await alertResubmitFailed(row, err);
}
}
}
if (status?.code === 'nvcf_expired' && row.prompt) {
// The request id is dead but the inputs live on the row: the same
// never-dead-end recovery the attended poll runs, sharing its
// once-per-job resub lock so the two paths can't double-submit.
// The alias keeps the row's handle stable; the next tick polls the
// resubmitted task and materializes onto this same row.
if (
!(await cacheGet(`nvcf:resub-inflight:${row.replicate_job_id}`).catch(() => null)) &&
(await acquireLock(`nvcf:resub:${row.replicate_job_id}`, NVCF_ALIAS_TTL_S))
) {
try {
await markResubmitInflight(row.replicate_job_id);
const resub = await nvidia.textTo3d({ prompt: row.prompt, tier: resolveTier(row.tier) });
if (resub.taskId) {
await cacheSet(`nvcf:alias:${row.replicate_job_id}`, resub.taskId, NVCF_ALIAS_TTL_S);
console.warn(
`[forge-finalize] NVCF request expired; resubmitted unattended job (creation ${row.id})`,
);
out.resubmitted++;
return out;
}
if (resub.resultGlbUrl) {
const durable = await materializeCreation({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
glbUrl: resub.resultGlbUrl,
});
if (durable) {
out.done++;
return out;
}
}
} catch (err) {
console.warn(`[forge-finalize] NVCF resubmit failed: ${err?.message || err}`);
await alertResubmitFailed(row, err);
}
}
// Lock held by a prior recovery or the resubmit failed: the lane is
// cron-observed broken with no alternate text-to-3D lane to send new
// submissions to, so cool the same breaker forge.js's submit path
// reads before every new free-text request. Fall through to the TTL
// check so a second expiry fails honestly, not instantly.
await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
} else if (status?.status === 'failed') {
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
error: status.error || 'generation failed',
});
out.failed++;
if (row.user_id) {
notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: status.error, backend: row.backend, tier: row.tier });
}
return out;
}
// queued / running / poll blip: fall through to the TTL check.
} else {
out.unpollable++;
}
}
// reapExpiredRows runs a poll-free, BATCH_LIMIT-free hard-TTL sweep ahead of
// the live-poll query that selects `rows` for this batch (see the handler
// above), so any row that reaches this point is, by construction, already
// inside HARD_TTL_MINUTES: there is no second TTL branch to duplicate here.
if (envelope) {
out.still_running++;
}
return out;
}