Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): decouple the hard-TTL reap from the live-poll batch so a slow backend stops burying other lanes
nirholas76b65c15577b
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.jsindex 928605598..4f538c6e7 100644--- a/api/cron/forge-finalize.js+++ b/api/cron/forge-finalize.js@@ -267,6 +267,52 @@ async function tryCronRedispatch(row, ageMinutes) {}}+// Poll-free, BATCH_LIMIT-free hard-TTL reap: a single bulk UPDATE that flips+// every row already past HARD_TTL_MINUTES to 'failed', no matter how old or+// which backend/path it belongs to. This runs ahead of (and independently of)+// the oldest-created_at-first, BATCH_LIMIT-bounded live-poll query below, so a+// batch monopolized by one unhealthy backend (trellis_selfhost owning every+// slot because its rows are the oldest) can never block a different, never-+// selected lane's rows from ever being reaped: exactly the bug measured+// 2026-10-25 in prompts/finish/436: 192 `rerig` rows sat 'generating' forever+// because they never won a batch slot against older trellis_selfhost rows.+// Message/column shape matches forge-store.js's markFailed exactly (error,+// completed_at, updated_at) so no downstream matcher (forge-error-report.mjs's+// `error ilike '%timed out%'`) sees a different error string.+export async function reapExpiredRows() {+ const rows = await sql`+ update forge_creations+ set status = 'failed',+ error = 'generation timed out after ' || round(extract(epoch from (now() - created_at)) / 60) || ' minutes',+ completed_at = coalesce(completed_at, now()),+ updated_at = now()+ where status = 'generating'+ and replicate_job_id is not null+ and created_at < now() - (${HARD_TTL_MINUTES} * interval '1 minute')+ returning id, user_id, prompt, backend, tier, path+ `;+ for (const row of rows) {+ if (row.backend === 'nvidia') {+ // Reaping the bare TTL on nvidia is itself a cron-observed failure of+ // the lane: cool the submit-time breaker so new free-text requests+ // skip straight to the paid/image-intermediate path instead of landing+ // on the same dead end for another HARD_TTL_MINUTES.+ await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});+ }+ if (row.user_id) {+ notifyForgeFailed({+ userId: row.user_id,+ creationId: row.id,+ prompt: row.prompt,+ error: 'generation timed out',+ backend: row.backend,+ tier: row.tier,+ });+ }+ }+ return rows.length;+}+export default wrapCron(async (req, res) => {if (cors(req, res, { methods: 'GET,POST,OPTIONS' })) return;if (!method(req, res, ['GET', 'POST'])) return;@@ -277,6 +323,8 @@ export default wrapCron(async (req, res) => {const t0 = Date.now();+ const reaped = await reapExpiredRows();+const rows = await sql`select id, replicate_job_id, client_key, user_id, prompt, preview_image_url,backend, tier, path, created_at, destination, internal@@ -303,7 +351,7 @@ export default wrapCron(async (req, res) => {}const results = await mapPoolSettled(rows, ROW_CONCURRENCY, (row) => sweepRow(row, { gcp, nvidia }));- const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: 0, still_running: 0, unpollable: 0 };+ const out = { done: 0, failed: 0, failed_over: 0, resubmitted: 0, timed_out: reaped, still_running: 0, unpollable: 0 };for (const result of results) {if (result.ok) {for (const key of Object.keys(out)) out[key] += result.value[key] || 0;@@ -315,6 +363,7 @@ export default wrapCron(async (req, res) => {return json(res, 200, {enabled: true,+ reaped,swept: rows.length,...out,elapsed_ms: Date.now() - t0,@@ -564,24 +613,11 @@ export async function sweepRow(row, { gcp, nvidia }) {}}- if (ageMinutes > HARD_TTL_MINUTES) {- if (row.backend === 'nvidia') {- // Reaping the bare TTL on nvidia is itself a cron-observed failure of- // the lane: cool the submit-time breaker so new free-text requests- // skip straight to the paid/image-intermediate path instead of landing- // on the same dead end for another HARD_TTL_MINUTES.- await markProviderCooldown(NIM_TRELLIS_COOLDOWN_KEY, nimCooldownSeconds()).catch(() => {});- }- await markFailed({- replicateJobId: row.replicate_job_id,- clientKey: row.client_key,- error: `generation timed out after ${Math.round(ageMinutes)} minutes`,- });- out.timed_out++;- if (row.user_id) {- notifyForgeFailed({ userId: row.user_id, creationId: row.id, prompt: row.prompt, error: 'generation timed out', backend: row.backend, tier: row.tier });- }- } else if (envelope) {+ // reapExpiredRows runs a poll-free, BATCH_LIMIT-free hard-TTL sweep ahead of+ // the live-poll query that selects `rows` for this batch (see the handler+ // above), so any row that reaches this point is, by construction, already+ // inside HARD_TTL_MINUTES: there is no second TTL branch to duplicate here.+ if (envelope) {out.still_running++;}diff --git a/tests/forge-finalize.test.js b/tests/forge-finalize.test.jsindex 1adcd8521..79eab1ac0 100644--- a/tests/forge-finalize.test.js+++ b/tests/forge-finalize.test.js@@ -3,10 +3,61 @@// (see api/_providers/gcp.js) vs. opaque third-party ids the cron cannot poll.// Importing the cron module is safe: db/neon clients are created lazily.-import { describe, it, expect } from 'vitest';-import { decodeWorkerEnvelope, decideFailedSweep, decideNvidiaStall, sweepRow, ROW_CONCURRENCY } from '../api/cron/forge-finalize.js';+import { describe, it, expect, vi, beforeEach } from 'vitest';+import { NIM_TRELLIS_COOLDOWN_KEY } from '../api/forge.js';import { mapPoolSettled } from '../api/_lib/pool.js';+// TTL-reap tests below need to prove zero live-poll calls happen, so the+// provider factories and the notification/cooldown side effects are mocked+// here (module-wide for this file). The pre-existing describe blocks never+// trigger these paths (their rows stay 'running', never 'failed' or past+// HARD_TTL_MINUTES), so mocking them is a no-op for those tests. vi.mock is+// hoisted above every import (including transitive ones like+// api/_lib/rate-limit.js's own `sql` import), so the mocked values must come+// from vi.hoisted rather than a plain top-level const.+const { sql, gcpStatus, nvidiaStatus, notifyForgeFailed, markProviderCooldown } = vi.hoisted(() => ({+ sql: vi.fn(async () => []),+ gcpStatus: vi.fn(async () => {+ throw new Error('reap pass must not poll gcp');+ }),+ nvidiaStatus: vi.fn(async () => {+ throw new Error('reap pass must not poll nvidia');+ }),+ notifyForgeFailed: vi.fn(),+ markProviderCooldown: vi.fn(async () => {}),+}));++vi.mock('../api/_lib/db.js', () => ({+ sql,+ isDbUnavailableError: () => false,+ isDbCapacityError: () => false,+ isStoragePressured: () => false,+}));++vi.mock('../api/_providers/gcp.js', () => ({+ createRegenProvider: () => ({ status: gcpStatus }),+}));++vi.mock('../api/_providers/nvidia.js', () => ({+ createNvidiaProvider: () => ({ status: nvidiaStatus, textTo3d: vi.fn() }),+}));++vi.mock('../api/_lib/forge-notify.js', () => ({+ notifyForgeComplete: vi.fn(),+ notifyForgeFailed,+}));++vi.mock('../api/_lib/provider-health.js', () => ({ markProviderCooldown }));++const {+ decodeWorkerEnvelope,+ decideFailedSweep,+ decideNvidiaStall,+ sweepRow,+ ROW_CONCURRENCY,+ reapExpiredRows,+} = await import('../api/cron/forge-finalize.js');+function pack(obj) {return Buffer.from(JSON.stringify(obj), 'utf8').toString('base64url');}@@ -185,3 +236,71 @@ describe('sweepRow concurrency', () => {expect(totalStillRunning).toBe(4);});});++// Measured 2026-10-25: 780+ `forge_creations` rows sat 'generating' for+// ~24h against a documented 45-minute HARD_TTL_MINUTES. Root cause: the old+// TTL reap only ran inside sweepRow, AFTER a row won a slot in the+// oldest-created_at-first, BATCH_LIMIT-bounded live-poll batch. Once one+// backend (trellis_selfhost) owned every slot because its rows were the+// oldest, a different, never-selected lane's rows (the `rerig` cluster,+// backend/path both null) never reached any reap logic at all and never+// resolved. reapExpiredRows is a single poll-free, BATCH_LIMIT-free UPDATE+// that runs ahead of that batch, so no backend can monopolize the reap.+describe('TTL reap runs ahead of the live poll', () => {+ beforeEach(() => {+ sql.mockReset();+ gcpStatus.mockClear();+ nvidiaStatus.mockClear();+ notifyForgeFailed.mockClear();+ markProviderCooldown.mockClear();+ });++ function expiredRow(id, { backend = 'trellis_selfhost', path = null, userId = null } = {}) {+ return { id, user_id: userId, prompt: `prompt-${id}`, backend, tier: 'draft', path };+ }++ it('reaps every row past HARD_TTL_MINUTES in one poll-free query, regardless of batch position', async () => {+ // Simulates the production monopoly: 25 trellis_selfhost rows would+ // fill an oldest-first batch on their own, and a `rerig` row (backend+ // and path both null, like the real stalled cluster) would never win+ // a slot against them. All are already past the hard TTL.+ const monopolizingRows = Array.from({ length: 25 }, (_, i) => expiredRow(`trellis-${i}`));+ const neverSelectedRow = expiredRow('rerig-1', { backend: null, path: null, userId: 'user-1' });+ const fixtureRows = [...monopolizingRows, neverSelectedRow];++ sql.mockImplementation(async (strings) => {+ const text = strings.join(' ');+ expect(text).toContain('update forge_creations');+ // The whole point of this pass: no BATCH_LIMIT gates it.+ expect(text).not.toContain('limit');+ return fixtureRows;+ });++ const reapedCount = await reapExpiredRows();++ expect(reapedCount).toBe(fixtureRows.length);+ // The never-selected lane's row is reaped by this same pass: it never+ // had to win a batch slot against trellis_selfhost's older rows.+ expect(notifyForgeFailed).toHaveBeenCalledWith(+ expect.objectContaining({ userId: 'user-1', creationId: 'rerig-1' }),+ );+ // No live poll of any kind: the reap pass never touches a provider.+ expect(gcpStatus).not.toHaveBeenCalled();+ expect(nvidiaStatus).not.toHaveBeenCalled();+ });++ it('cools the nvidia breaker for reaped nvidia rows, same as the old TTL branch', async () => {+ const row = expiredRow('nvidia-1', { backend: 'nvidia', userId: 'user-2' });+ sql.mockImplementation(async () => [row]);++ await reapExpiredRows();++ expect(markProviderCooldown).toHaveBeenCalledWith(NIM_TRELLIS_COOLDOWN_KEY, expect.any(Number));+ });++ it('does nothing when nothing is past the hard TTL', async () => {+ sql.mockImplementation(async () => []);+ expect(await reapExpiredRows()).toBe(0);+ expect(notifyForgeFailed).not.toHaveBeenCalled();+ });+});