Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge-seed): fail a seed generation the lane never resolves so it stops holding a pending slot forever
nirholas8117e6720972
diff --git a/api/cron/forge-seed-cron.js b/api/cron/forge-seed-cron.jsindex 000ac384c..807afe34a 100644--- a/api/cron/forge-seed-cron.js+++ b/api/cron/forge-seed-cron.js@@ -163,6 +163,13 @@ export function riggableShape(metrics) {// does. Measured against started_at (the whole job, generation included)// because the rig submit has no timestamp column of its own.const RIG_STALL_MS = 45 * 60 * 1000;+// The same guard for the generation itself. A job the lane lost keeps answering+// queued or running (or a poll keeps failing on transport), and every pending+// row counts against maxPending(), so one such row shrinks seeding capacity for+// good and, at SEED_CRON_MAX_PENDING=1, stops it outright: a hunyuan3d job sat+// 'pending' from 2026-09-17 to 2026-10-09 that way. The forge client gives a+// running job 12 minutes and does not count queue time, so an hour is generous.+const GENERATION_STALL_MS = 60 * 60 * 1000;// Gates per tick. Each one fetches the finished GLB and parses its glTF chunk;// two keeps the phase in the low seconds even when a burst lands together.const gateBatchSize = () => intEnv('SEED_CRON_GATE', 2, { min: 1, max: 10 });@@ -344,9 +351,9 @@ function assertFreeBackend(backend) {//// A finished generation becomes 'generated', not 'done': the mesh exists but has// not faced the catalog gate yet. Publishing happens only after advanceGates().-async function pollPending(origin) {+export async function pollPending(origin) {const rows = await sql`- select id, user_id, raw_client_id, job_id, prompt, model_category+ select id, user_id, raw_client_id, job_id, prompt, model_category, started_atfrom forge_seed_jobswhere status = 'pending'and started_at < now() - (${MIN_JOB_AGE_SECONDS} || ' seconds')::interval@@ -356,6 +363,16 @@ async function pollPending(origin) {if (!rows.length) return [];const results = [];+ const stalled = (job) => Date.now() - new Date(job.started_at).getTime() > GENERATION_STALL_MS;+ const expire = async (job, lastStatus) => {+ const error = `generation stalled past ${Math.round(GENERATION_STALL_MS / 60000)}m (last poll: ${lastStatus})`;+ await sql`+ update forge_seed_jobs+ set status = 'failed', error = ${error}, finished_at = now()+ where id = ${job.id} and status = 'pending'+ `;+ results.push({ job_id: job.job_id, status: 'failed', error, prompt: job.prompt });+ };await Promise.all(rows.map(async (job) => {try {const cronSecret = process.env.CRON_SECRET || env.CRON_SECRET || '';@@ -392,10 +409,18 @@ async function pollPending(origin) {where id = ${job.id}`;results.push({ job_id: job.job_id, status: 'failed', prompt: job.prompt });+ } else if (stalled(job)) {+ await expire(job, poll.body?.status || `HTTP ${poll.status ?? 'unknown'}`);} else {results.push({ job_id: job.job_id, status: poll.body?.status || 'running' });}} catch (err) {+ if (stalled(job)) {+ await expire(job, `poll error: ${err?.message || 'unknown'}`).catch((e) =>+ results.push({ job_id: job.job_id, status: 'poll_error', error: e?.message }),+ );+ return;+ }results.push({ job_id: job.job_id, status: 'poll_error', error: err?.message });}}));diff --git a/tests/cron-seed-pending-stall.test.js b/tests/cron-seed-pending-stall.test.jsnew file mode 100644index 000000000..5db2b8eb7--- /dev/null+++ b/tests/cron-seed-pending-stall.test.js@@ -0,0 +1,83 @@+// The forge seed cron gives up on a generation the lane never resolves.+//+// Every 'pending' row counts against maxPending(), and pollPending only ever+// moved a row on when its poll said done or failed. A job the lane lost kept+// answering queued (or the poll kept erroring), so the row held its slot for+// good: one hunyuan3d job sat pending from 2026-09-17 to 2026-10-09, and once+// SEED_CRON_MAX_PENDING was set to 1 it stopped seeding entirely.++import { describe, it, expect, vi, beforeEach, afterEach } from 'vitest';++const queries = [];+let pendingRows = [];+vi.mock('../api/_lib/db.js', () => ({+ sql: (strings, ...values) => {+ const text = strings.join('?');+ queries.push({ text, values });+ if (/select[\s\S]*from forge_seed_jobs[\s\S]*status = 'pending'/.test(text)) return Promise.resolve(pendingRows);+ return Promise.resolve([]);+ },+}));++const { pollPending } = await import('../api/cron/forge-seed-cron.js');++const row = (id, ageMinutes) => ({+ id,+ user_id: 'user-1',+ raw_client_id: 'client-1',+ job_id: `forge-${id}`,+ prompt: 'a lighthouse keeper in an oilskin coat',+ model_category: 'avatar',+ started_at: new Date(Date.now() - ageMinutes * 60_000).toISOString(),+});++const failUpdates = () => queries.filter((q) => /update forge_seed_jobs[\s\S]*status = 'failed'/.test(q.text));++function answer(body, status = 200) {+ vi.stubGlobal('fetch', vi.fn(async () => new Response(JSON.stringify(body), { status, headers: { 'content-type': 'application/json' } })));+}++describe('pollPending stall expiry', () => {+ beforeEach(() => {+ queries.length = 0;+ pendingRows = [];+ });+ afterEach(() => vi.unstubAllGlobals());++ it('leaves a young job that is still queued alone', async () => {+ pendingRows = [row('young', 5)];+ answer({ status: 'queued' });+ const results = await pollPending('https://three.ws');+ expect(results).toEqual([{ job_id: 'forge-young', status: 'queued' }]);+ expect(failUpdates()).toHaveLength(0);+ });++ it('fails a job still queued past the stall window and frees its slot', async () => {+ pendingRows = [row('lost', 3 * 24 * 60)];+ answer({ status: 'queued' });+ const [result] = await pollPending('https://three.ws');+ expect(result.status).toBe('failed');+ expect(result.error).toMatch(/stalled past 60m \(last poll: queued\)/);+ const [update] = failUpdates();+ expect(update.values).toContain('lost');+ // Guarded so a row another tick already closed out is never reopened.+ expect(update.text).toMatch(/and status = 'pending'/);+ });++ it('fails a stalled job whose poll errors on transport', async () => {+ pendingRows = [row('unreachable', 90)];+ vi.stubGlobal('fetch', vi.fn(async () => { throw new TypeError('fetch failed'); }));+ const [result] = await pollPending('https://three.ws');+ expect(result.status).toBe('failed');+ expect(result.error).toMatch(/poll error/);+ expect(failUpdates()).toHaveLength(1);+ });++ it('still publishes a stalled job that turns out to be done', async () => {+ pendingRows = [row('late', 90)];+ answer({ status: 'done', glb_url: 'https://three.ws/m.glb', creation_id: 'c-1', backend: 'trellis_selfhost' });+ const [result] = await pollPending('https://three.ws');+ expect(result.status).toBe('generated');+ expect(failUpdates()).toHaveLength(0);+ });+});