Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge-seed): fail a seed generation the lane never resolves so it stops holding a pending slot forever

nirholas8117e6720972
diff --git a/api/cron/forge-seed-cron.js b/api/cron/forge-seed-cron.js
index 000ac384c..807afe34a 100644
--- a/api/cron/forge-seed-cron.js
+++ b/api/cron/forge-seed-cron.js
@@ -163,6 +163,13 @@ export function riggableShape(metrics) {
// does. Measured against started_at (the whole job, generation included)
// because the rig submit has no timestamp column of its own.
const RIG_STALL_MS = 45 * 60 * 1000;
+// The same guard for the generation itself. A job the lane lost keeps answering
+// queued or running (or a poll keeps failing on transport), and every pending
+// row counts against maxPending(), so one such row shrinks seeding capacity for
+// good and, at SEED_CRON_MAX_PENDING=1, stops it outright: a hunyuan3d job sat
+// 'pending' from 2026-09-17 to 2026-10-09 that way. The forge client gives a
+// running job 12 minutes and does not count queue time, so an hour is generous.
+const GENERATION_STALL_MS = 60 * 60 * 1000;
// Gates per tick. Each one fetches the finished GLB and parses its glTF chunk;
// two keeps the phase in the low seconds even when a burst lands together.
const gateBatchSize = () => intEnv('SEED_CRON_GATE', 2, { min: 1, max: 10 });
@@ -344,9 +351,9 @@ function assertFreeBackend(backend) {
//
// A finished generation becomes 'generated', not 'done': the mesh exists but has
// not faced the catalog gate yet. Publishing happens only after advanceGates().
-async function pollPending(origin) {
+export async function pollPending(origin) {
const rows = await sql`
- select id, user_id, raw_client_id, job_id, prompt, model_category
+ select id, user_id, raw_client_id, job_id, prompt, model_category, started_at
from forge_seed_jobs
where status = 'pending'
and started_at < now() - (${MIN_JOB_AGE_SECONDS} || ' seconds')::interval
@@ -356,6 +363,16 @@ async function pollPending(origin) {
if (!rows.length) return [];
const results = [];
+ const stalled = (job) => Date.now() - new Date(job.started_at).getTime() > GENERATION_STALL_MS;
+ const expire = async (job, lastStatus) => {
+ const error = `generation stalled past ${Math.round(GENERATION_STALL_MS / 60000)}m (last poll: ${lastStatus})`;
+ await sql`
+ update forge_seed_jobs
+ set status = 'failed', error = ${error}, finished_at = now()
+ where id = ${job.id} and status = 'pending'
+ `;
+ results.push({ job_id: job.job_id, status: 'failed', error, prompt: job.prompt });
+ };
await Promise.all(rows.map(async (job) => {
try {
const cronSecret = process.env.CRON_SECRET || env.CRON_SECRET || '';
@@ -392,10 +409,18 @@ async function pollPending(origin) {
where id = ${job.id}
`;
results.push({ job_id: job.job_id, status: 'failed', prompt: job.prompt });
+ } else if (stalled(job)) {
+ await expire(job, poll.body?.status || `HTTP ${poll.status ?? 'unknown'}`);
} else {
results.push({ job_id: job.job_id, status: poll.body?.status || 'running' });
}
} catch (err) {
+ if (stalled(job)) {
+ await expire(job, `poll error: ${err?.message || 'unknown'}`).catch((e) =>
+ results.push({ job_id: job.job_id, status: 'poll_error', error: e?.message }),
+ );
+ return;
+ }
results.push({ job_id: job.job_id, status: 'poll_error', error: err?.message });
}
}));
diff --git a/tests/cron-seed-pending-stall.test.js b/tests/cron-seed-pending-stall.test.js
new file mode 100644
index 000000000..5db2b8eb7
--- /dev/null
+++ b/tests/cron-seed-pending-stall.test.js
@@ -0,0 +1,83 @@
+// The forge seed cron gives up on a generation the lane never resolves.
+//
+// Every 'pending' row counts against maxPending(), and pollPending only ever
+// moved a row on when its poll said done or failed. A job the lane lost kept
+// answering queued (or the poll kept erroring), so the row held its slot for
+// good: one hunyuan3d job sat pending from 2026-09-17 to 2026-10-09, and once
+// SEED_CRON_MAX_PENDING was set to 1 it stopped seeding entirely.
+
+import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';
+
+const queries = [];
+let pendingRows = [];
+vi.mock('../api/_lib/db.js', () => ({
+ sql: (strings, ...values) => {
+ const text = strings.join('?');
+ queries.push({ text, values });
+ if (/select[\s\S]*from forge_seed_jobs[\s\S]*status = 'pending'/.test(text)) return Promise.resolve(pendingRows);
+ return Promise.resolve([]);
+ },
+}));
+
+const { pollPending } = await import('../api/cron/forge-seed-cron.js');
+
+const row = (id, ageMinutes) => ({
+ id,
+ user_id: 'user-1',
+ raw_client_id: 'client-1',
+ job_id: `forge-${id}`,
+ prompt: 'a lighthouse keeper in an oilskin coat',
+ model_category: 'avatar',
+ started_at: new Date(Date.now() - ageMinutes * 60_000).toISOString(),
+});
+
+const failUpdates = () => queries.filter((q) => /update forge_seed_jobs[\s\S]*status = 'failed'/.test(q.text));
+
+function answer(body, status = 200) {
+ vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } })));
+}
+
+describe('pollPending stall expiry', () => {
+ beforeEach(() => {
+ queries.length = 0;
+ pendingRows = [];
+ });
+ afterEach(() => vi.unstubAllGlobals());
+
+ it('leaves a young job that is still queued alone', async () => {
+ pendingRows = [row('young', 5)];
+ answer({ status: 'queued' });
+ const results = await pollPending('https://three.ws');
+ expect(results).toEqual([{ job_id: 'forge-young', status: 'queued' }]);
+ expect(failUpdates()).toHaveLength(0);
+ });
+
+ it('fails a job still queued past the stall window and frees its slot', async () => {
+ pendingRows = [row('lost', 3 * 24 * 60)];
+ answer({ status: 'queued' });
+ const [result] = await pollPending('https://three.ws');
+ expect(result.status).toBe('failed');
+ expect(result.error).toMatch(/stalled past 60m \(last poll: queued\)/);
+ const [update] = failUpdates();
+ expect(update.values).toContain('lost');
+ // Guarded so a row another tick already closed out is never reopened.
+ expect(update.text).toMatch(/and status = 'pending'/);
+ });
+
+ it('fails a stalled job whose poll errors on transport', async () => {
+ pendingRows = [row('unreachable', 90)];
+ vi.stubGlobal('fetch', vi.fn(async () => { throw new TypeError('fetch failed'); }));
+ const [result] = await pollPending('https://three.ws');
+ expect(result.status).toBe('failed');
+ expect(result.error).toMatch(/poll error/);
+ expect(failUpdates()).toHaveLength(1);
+ });
+
+ it('still publishes a stalled job that turns out to be done', async () => {
+ pendingRows = [row('late', 90)];
+ answer({ status: 'done', glb_url: 'https://three.ws/m.glb', creation_id: 'c-1', backend: 'trellis_selfhost' });
+ const [result] = await pollPending('https://three.ws');
+ expect(result.status).toBe('generated');
+ expect(failUpdates()).toHaveLength(0);
+ });
+});