Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): decouple the hard-TTL reap from the live-poll batch so a slow backend stops burying other lanes

nirholas76b65c15577b
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.js
index 928605598..4f538c6e7 100644
--- a/api/cron/forge-finalize.js
+++ b/api/cron/forge-finalize.js
@@ -267,6 +267,52 @@ async function tryCronRedispatch(row, ageMinutes) {
}
}
+// Poll-free, BATCH_LIMIT-free hard-TTL reap: a single bulk UPDATE that flips
+// every row already past HARD_TTL_MINUTES to 'failed', no matter how old or
+// which backend/path it belongs to. This runs ahead of (and independently of)
+// the oldest-created_at-first, BATCH_LIMIT-bounded live-poll query below, so a
+// batch monopolized by one unhealthy backend (trellis_selfhost owning every
+// slot because its rows are the oldest) can never block a different, never-
+// selected lane's rows from ever being reaped: exactly the bug measured
+// 2026-10-25 in prompts/finish/436: 192 `rerig` rows sat 'generating' forever
+// because they never won a batch slot against older trellis_selfhost rows.
+// Message/column shape matches forge-store.js's markFailed exactly (error,
+// completed_at, updated_at) so no downstream matcher (forge-error-report.mjs's
+// `error ilike '%timed out%'`) sees a different error string.
+export async function reapExpiredRows() {
+ const rows = await sql`
+ update forge_creations
+ set status = 'failed',
+ error = 'generation timed out after ' || round(extract(epoch from (now() - created_at)) / 60) || ' minutes',
+ completed_at = coalesce(completed_at, now()),
+ updated_at = now()
+ where status = 'generating'
+ and replicate_job_id is not null
+ and created_at < now() - (${HARD_TTL_MINUTES} * interval '1 minute')
+ returning id, user_id, prompt, backend, tier, path
+ `;
+ for (const row of rows) {
+ if (row.backend === 'nvidia') {
+ // Reaping the bare TTL on nvidia is itself a cron-observed failure of
+ // the lane: cool the submit-time breaker so new free-text requests
+ // skip straight to the paid/image-intermediate path instead of landing
+ // on the same dead end for another HARD_TTL_MINUTES.
+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
+ }
+ if (row.user_id) {
+ notifyForgeFailed({
+ userId: row.user_id,
+ creationId: row.id,
+ prompt: row.prompt,
+ error: 'generation timed out',
+ backend: row.backend,
+ tier: row.tier,
+ });
+ }
+ }
+ return rows.length;
+}
+
export default wrapCron(async (req, res) => {
if (cors(req, res, { methods: 'GET,POST,OPTIONS' })) return;
if (!method(req, res, ['GET', 'POST'])) return;
@@ -277,6 +323,8 @@ export default wrapCron(async (req, res) => {
const t0 = Date.now();
+ const reaped = await reapExpiredRows();
+
const rows = await sql`
select id, replicate_job_id, client_key, user_id, prompt, preview_image_url,
backend, tier, path, created_at, destination, internal
@@ -303,7 +351,7 @@ export default wrapCron(async (req, res) => {
}
const results = await mapPoolSettled(rows, ROW_CONCURRENCY, (row) => sweepRow(row, { gcp, nvidia }));
- const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: 0, still_running: 0, unpollable: 0 };
+ const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: reaped, still_running: 0, unpollable: 0 };
for (const result of results) {
if (result.ok) {
for (const key of Object.keys(out)) out[key] += result.value[key] || 0;
@@ -315,6 +363,7 @@ export default wrapCron(async (req, res) => {
return json(res, 200, {
enabled: true,
+ reaped,
swept: rows.length,
...out,
elapsed_ms: Date.now() - t0,
@@ -564,24 +613,11 @@ export async function sweepRow(row, { gcp, nvidia }) {
}
}
- if (ageMinutes > HARD_TTL_MINUTES) {
- if (row.backend === 'nvidia') {
- // Reaping the bare TTL on nvidia is itself a cron-observed failure of
- // the lane: cool the submit-time breaker so new free-text requests
- // skip straight to the paid/image-intermediate path instead of landing
- // on the same dead end for another HARD_TTL_MINUTES.
- await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});
- }
- await markFailed({
- replicateJobId: row.replicate_job_id,
- clientKey: row.client_key,
- error: `generation timed out after ${Math.round(ageMinutes)} minutes`,
- });
- out.timed_out++;
- if (row.user_id) {
- notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: 'generation timed out', backend: row.backend, tier: row.tier });
- }
- } else if (envelope) {
+ // reapExpiredRows runs a poll-free, BATCH_LIMIT-free hard-TTL sweep ahead of
+ // the live-poll query that selects `rows` for this batch (see the handler
+ // above), so any row that reaches this point is, by construction, already
+ // inside HARD_TTL_MINUTES: there is no second TTL branch to duplicate here.
+ if (envelope) {
out.still_running++;
}
diff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.js
index 1adcd8521..79eab1ac0 100644
--- a/tests/forge-finalize.test.js
+++ b/tests/forge-finalize.test.js
@@ -3,10 +3,61 @@
// (see api/_providers/gcp.js) vs. opaque third-party ids the cron cannot poll.
// Importing the cron module is safe: db/neon clients are created lazily.
-import { describe, it, expect } from 'vitest';
-import { decodeWorkerEnvelope, decideFailedSweep, decideNvidiaStall, sweepRow, ROW_CONCURRENCY } from '../api/cron/forge-finalize.js';
+import { describe, it, expect, vi, beforeEach } from 'vitest';
+import { NIM_TRELLIS_COOLDOWN_KEY } from '../api/forge.js';
import { mapPoolSettled } from '../api/_lib/pool.js';
+// TTL-reap tests below need to prove zero live-poll calls happen, so the
+// provider factories and the notification/cooldown side effects are mocked
+// here (module-wide for this file). The pre-existing describe blocks never
+// trigger these paths (their rows stay 'running', never 'failed' or past
+// HARD_TTL_MINUTES), so mocking them is a no-op for those tests. vi.mock is
+// hoisted above every import (including transitive ones like
+// api/_lib/rate-limit.js's own `sql` import), so the mocked values must come
+// from vi.hoisted rather than a plain top-level const.
+const { sql, gcpStatus, nvidiaStatus, notifyForgeFailed, markProviderCooldown } = vi.hoisted(() => ({
+ sql: vi.fn(async () => []),
+ gcpStatus: vi.fn(async () => {
+ throw new Error('reap pass must not poll gcp');
+ }),
+ nvidiaStatus: vi.fn(async () => {
+ throw new Error('reap pass must not poll nvidia');
+ }),
+ notifyForgeFailed: vi.fn(),
+ markProviderCooldown: vi.fn(async () => {}),
+}));
+
+vi.mock('../api/_lib/db.js', () => ({
+ sql,
+ isDbUnavailableError: () => false,
+ isDbCapacityError: () => false,
+ isStoragePressured: () => false,
+}));
+
+vi.mock('../api/_providers/gcp.js', () => ({
+ createRegenProvider: () => ({ status: gcpStatus }),
+}));
+
+vi.mock('../api/_providers/nvidia.js', () => ({
+ createNvidiaProvider: () => ({ status: nvidiaStatus, textTo3d: vi.fn() }),
+}));
+
+vi.mock('../api/_lib/forge-notify.js', () => ({
+ notifyForgeComplete: vi.fn(),
+ notifyForgeFailed,
+}));
+
+vi.mock('../api/_lib/provider-health.js', () => ({ markProviderCooldown }));
+
+const {
+ decodeWorkerEnvelope,
+ decideFailedSweep,
+ decideNvidiaStall,
+ sweepRow,
+ ROW_CONCURRENCY,
+ reapExpiredRows,
+} = await import('../api/cron/forge-finalize.js');
+
function pack(obj) {
return Buffer.from(JSON.stringify(obj), 'utf8').toString('base64url');
}
@@ -185,3 +236,71 @@ describe('sweepRow concurrency', () => {
expect(totalStillRunning).toBe(4);
});
});
+
+// Measured 2026-10-25: 780+ `forge_creations` rows sat 'generating' for
+// ~24h against a documented 45-minute HARD_TTL_MINUTES. Root cause: the old
+// TTL reap only ran inside sweepRow, AFTER a row won a slot in the
+// oldest-created_at-first, BATCH_LIMIT-bounded live-poll batch. Once one
+// backend (trellis_selfhost) owned every slot because its rows were the
+// oldest, a different, never-selected lane's rows (the `rerig` cluster,
+// backend/path both null) never reached any reap logic at all and never
+// resolved. reapExpiredRows is a single poll-free, BATCH_LIMIT-free UPDATE
+// that runs ahead of that batch, so no backend can monopolize the reap.
+describe('TTL reap runs ahead of the live poll', () => {
+ beforeEach(() => {
+ sql.mockReset();
+ gcpStatus.mockClear();
+ nvidiaStatus.mockClear();
+ notifyForgeFailed.mockClear();
+ markProviderCooldown.mockClear();
+ });
+
+ function expiredRow(id, { backend = 'trellis_selfhost', path = null, userId = null } = {}) {
+ return { id, user_id: userId, prompt: `prompt-${id}`, backend, tier: 'draft', path };
+ }
+
+ it('reaps every row past HARD_TTL_MINUTES in one poll-free query, regardless of batch position', async () => {
+ // Simulates the production monopoly: 25 trellis_selfhost rows would
+ // fill an oldest-first batch on their own, and a `rerig` row (backend
+ // and path both null, like the real stalled cluster) would never win
+ // a slot against them. All are already past the hard TTL.
+ const monopolizingRows = Array.from({ length: 25 }, (_, i) => expiredRow(`trellis-${i}`));
+ const neverSelectedRow = expiredRow('rerig-1', { backend: null, path: null, userId: 'user-1' });
+ const fixtureRows = [...monopolizingRows, neverSelectedRow];
+
+ sql.mockImplementation(async (strings) => {
+ const text = strings.join(' ');
+ expect(text).toContain('update forge_creations');
+ // The whole point of this pass: no BATCH_LIMIT gates it.
+ expect(text).not.toContain('limit');
+ return fixtureRows;
+ });
+
+ const reapedCount = await reapExpiredRows();
+
+ expect(reapedCount).toBe(fixtureRows.length);
+ // The never-selected lane's row is reaped by this same pass: it never
+ // had to win a batch slot against trellis_selfhost's older rows.
+ expect(notifyForgeFailed).toHaveBeenCalledWith(
+ expect.objectContaining({ userId: 'user-1', creationId: 'rerig-1' }),
+ );
+ // No live poll of any kind: the reap pass never touches a provider.
+ expect(gcpStatus).not.toHaveBeenCalled();
+ expect(nvidiaStatus).not.toHaveBeenCalled();
+ });
+
+ it('cools the nvidia breaker for reaped nvidia rows, same as the old TTL branch', async () => {
+ const row = expiredRow('nvidia-1', { backend: 'nvidia', userId: 'user-2' });
+ sql.mockImplementation(async () => [row]);
+
+ await reapExpiredRows();
+
+ expect(markProviderCooldown).toHaveBeenCalledWith(NIM_TRELLIS_COOLDOWN_KEY, expect.any(Number));
+ });
+
+ it('does nothing when nothing is past the hard TTL', async () => {
+ sql.mockImplementation(async () => []);
+ expect(await reapExpiredRows()).toBe(0);
+ expect(notifyForgeFailed).not.toHaveBeenCalled();
+ });
+});