Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
fix(forge): classify the two failure shapes that were hiding the real lost-task rate
nirholase0c1572fbb66
diff --git a/api/_lib/ops/forge-error-class.js b/api/_lib/ops/forge-error-class.jsindex 9f8a61330..2ab5b61c7 100644--- a/api/_lib/ops/forge-error-class.js+++ b/api/_lib/ops/forge-error-class.js@@ -65,7 +65,12 @@ export function normalizeForgeError(raw) {/** @type {Array<{ id: string, label: string, test: RegExp }>} */const CLASSES = [{ id: 'timeout', label: 'timed out before the lane returned', test: /\btimed out\b|\btimeout\b|deadline exceeded|etimedout/ },- { id: 'lost_task', label: 'lane lost the task (poll found nothing)', test: /task not found|prediction not found|job not found|unknown task|no such task|missing task/ },+ // The two dominant real shapes both mean "the lane lost the job", and both+ // used to land elsewhere: the self-host workers' own orphan reaper wording+ // (workers/model-*/main.py) fell through to `other`, and NVIDIA's+ // "NVCF request not found or expired" read as a generic 404. Measured on+ // 2026-08-14 they were 19 of the prior week's 23 failures.+ { id: 'lost_task', label: 'lane lost the task (poll found nothing)', test: /task not found|prediction not found|job not found|request not found|unknown task|no such task|missing task|orphaned/ },{ id: 'aborted', label: 'request aborted mid-flight', test: /\baborted\b|abortsignal|operation was canceled|canceled by (the )?client/ },{ id: 'out_of_memory', label: 'worker ran out of memory', test: /out of memory|\boom\b|cuda out of memory|killed \(signal 9\)|exit 144/ },{ id: 'rate_limited', label: 'lane rate limited or over quota', test: /rate limit|too many requests|\b429\b|quota exceeded|over quota/ },diff --git a/api/_lib/ops/forge-health-sensor.js b/api/_lib/ops/forge-health-sensor.jsindex d145b0b8b..caebb9464 100644--- a/api/_lib/ops/forge-health-sensor.js+++ b/api/_lib/ops/forge-health-sensor.js@@ -12,7 +12,12 @@// pages instead of hiding.//// Metric: done / (done + failed) over a rolling window. Still-running and queued-// rows are excluded (not yet an outcome). The window is 6h because forge volume+// rows are excluded (not yet an outcome), and so are attempts that were+// automatically re-dispatched to another lane (`superseded_by IS NOT NULL`):+// the successor row carries that same request to its real outcome, so counting+// the attempt too would page on the failover machinery working. Measured on+// 2026-08-14, 13 of the 14 trellis_selfhost orphan failures in the prior week+// were recovered that way and every one of them counted against the lane here. The window is 6h because forge volume// is modest (~7 image jobs/hour) — long enough to carry a judge-able sample and// smooth the bursty failure pattern, short enough to surface a sustained// regression. Below MIN_ATTEMPTS the verdict is `unknown` (neutral, never pages).@@ -81,7 +86,7 @@ export function classifyForgeBuckets(buckets, { minAttempts = MIN_ATTEMPTS } = {return {status: 'unknown',done, failed, attempts, rate: null, worstBackend, worstPath, topReason,- detail: `only ${attempts} finished generations in ${WINDOW_INTERVAL} — too few to judge`,+ detail: `only ${attempts} finished generations in ${WINDOW_INTERVAL}, too few to judge`,};}@@ -130,6 +135,7 @@ export async function gatherForgeHealth() {count(*)::int AS nFROM forge_creationsWHERE created_at >= now() - ${WINDOW_INTERVAL}::interval+ AND superseded_by IS NULLGROUP BY status, backend, path, reason`);diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.jsindex 7f68352fa..c43664acc 100644--- a/api/cron/forge-finalize.js+++ b/api/cron/forge-finalize.js@@ -34,7 +34,13 @@import { sql } from '../_lib/db.js';import { cors, json, method, wrapCron } from '../_lib/http.js';import { requireCron } from '../_lib/cron-auth.js';-import { materializeCreation, markFailed, forgeStoreEnabled, createCreation } from '../_lib/forge-store.js';+import {+ materializeCreation,+ markFailed,+ markSupersededBy,+ forgeStoreEnabled,+ createCreation,+} from '../_lib/forge-store.js';import { notifyForgeComplete, notifyForgeFailed } from '../_lib/forge-notify.js';import { createRegenProvider as createGcpProvider } from '../_providers/gcp.js';import { createNvidiaProvider } from '../_providers/nvidia.js';@@ -148,6 +154,10 @@ export function decideFailedSweep({// resubmit to the next healthy lane instead. The successor gets its own// creation row (same client/user), so this same cron sweeps it to completion// and the completion notification still fires for a model, not a failure.+//+// Returns `{ successorId }` when the request now continues on another lane (the+// caller links the two rows so the ledger shows a recovery, not a loss), or+// null when nothing was redispatched.async function tryCronRedispatch(row, ageMinutes) {const prior = await readCronHop(row.replicate_job_id);const hop = prior ? prior.hop : 0;@@ -157,10 +167,10 @@ async function tryCronRedispatch(row, ageMinutes) {hasReferenceImage: Boolean(row.preview_image_url),path: row.path,});- if (decision !== 'redispatch') return false;+ if (decision !== 'redispatch') return null;const attempted = [...new Set([...(prior?.attempted || []), row.backend].filter(Boolean))];const nextLane = await pickRedispatchLane({ attempted });- if (!nextLane) return false;+ if (!nextLane) return null;try {const submitted = await submitFailoverJob({backend: nextLane,@@ -172,7 +182,7 @@ async function tryCronRedispatch(row, ageMinutes) {// The redispatch reconstructs from the primary stored view, so a// multi-view original degrades visibly (views_used: 1), never silently:// the same provenance contract as the attended failover.- await createCreation({+ const successorId = await createCreation({clientKey: row.client_key,userId: row.user_id ?? null,prompt: row.prompt,@@ -197,10 +207,10 @@ async function tryCronRedispatch(row, ageMinutes) {console.warn(`[forge-finalize] job failed on ${row.backend}; unattended auto-failover #${hop + 1} → ${nextLane}`,);- return true;+ return { successorId: successorId ?? null };} catch (err) {console.warn(`[forge-finalize] redispatch failed: ${err?.message || err}`);- return false;+ return null;}}@@ -282,12 +292,24 @@ export default wrapCron(async (req, res) => {continue;}if (row.backend) await markLaneUnhealthy(row.backend).catch(() => {});- if (await tryCronRedispatch(row, ageMinutes)) {+ const redispatched = await tryCronRedispatch(row, ageMinutes);+ if (redispatched) {await markFailed({replicateJobId: row.replicate_job_id,clientKey: row.client_key,error: status.error || 'generation failed',});+ // Link the attempt to its successor so the ledger reads as a+ // recovery: without it the health sensor and the error report both+ // count this row as a user-visible loss, when the request is in+ // fact still running on another lane.+ if (redispatched.successorId) {+ await markSupersededBy({+ replicateJobId: row.replicate_job_id,+ clientKey: row.client_key,+ successorId: redispatched.successorId,+ });+ }out.failed_over++;// No failure notification: the successor row carries the job on,// and its completion (or terminal failure) notifies instead.diff --git a/api/forge.js b/api/forge.jsindex abb77ebf1..1703d12a5 100644--- a/api/forge.js+++ b/api/forge.js@@ -88,6 +88,7 @@ import {createCreation,materializeCreation,markFailed,+ markSupersededBy,findByJob,} from './_lib/forge-store.js';import { getSessionUser } from './_lib/auth.js';@@ -2948,7 +2949,7 @@ async function pollJob(req, res, jobId) {// and the success path materializes into the store as usual. The// redispatch reconstructs from the primary stored view, so a// multi-view original degrades visibly (views_used: 1), never silently.- await createCreation({+ const successorId = await createCreation({clientKey,userId: await sessionUserIdFromReq(req),ipHash: hashIp(clientIp(req)),@@ -2971,6 +2972,12 @@ async function pollJob(req, res, jobId) {attempted,});if (bound) {+ // Link the failed attempt to the successor now that the chain is+ // durable, so the outcome ledger records a recovery rather than a+ // loss (the health sensor and npm run forge:errors both read it).+ if (successorId) {+ await markSupersededBy({ replicateJobId: upstreamId, clientKey, successorId });+ }console.warn(`[forge] job failed on ${failedBackend}; auto-failover #${hop + 1} → ${nextLane}`,);diff --git a/scripts/forge-error-report.mjs b/scripts/forge-error-report.mjsindex aef82c78d..5e2737215 100644--- a/scripts/forge-error-report.mjs+++ b/scripts/forge-error-report.mjs@@ -16,6 +16,7 @@* node scripts/forge-error-report.mjs --days 30* node scripts/forge-error-report.mjs --days 7 --json* node scripts/forge-error-report.mjs --class lost_task # every message in one class+ * node scripts/forge-error-report.mjs --include-recovered # count failed-over attempts too** Reads DATABASE_URL from .env.local, then .env, then the shell (same order as* scripts/apply-migrations.mjs). Read-only: it runs SELECTs and nothing else.@@ -53,6 +54,11 @@ const flag = (name, fallback) => {const DAYS = Math.max(1, Math.min(365, Number(flag('days', '7')) || 7));const JSON_OUT = args.includes('--json');const ONLY_CLASS = flag('class', null);+// A failed attempt that was automatically re-dispatched to another lane is not+// a user-visible failure: the successor row carries the same request to its real+// outcome. Ranking those by default puts a class the platform already recovers+// from at rank 1, which is exactly how this report would waste a triager's day.+const INCLUDE_RECOVERED = args.includes('--include-recovered');if (!process.env.DATABASE_URL) {console.error('DATABASE_URL is not set. Add it to .env.local or export it in your shell.');@@ -83,6 +89,7 @@ async function main() {SELECT count(*) FILTER (WHERE status = 'done')::int AS done,count(*) FILTER (WHERE status = 'failed')::int AS failed,count(*) FILTER (WHERE status NOT IN ('done', 'failed'))::int AS open,+ count(*) FILTER (WHERE superseded_by IS NOT NULL)::int AS recovered,count(*)::int AS rowsFROM forge_creationsWHERE created_at >= now() - ${since}::interval@@ -99,6 +106,7 @@ async function main() {FROM forge_creationsWHERE status = 'failed'AND created_at >= now() - ${since}::interval+ AND (${INCLUDE_RECOVERED} OR superseded_by IS NULL)ORDER BY created_at DESC`;@@ -135,16 +143,24 @@ async function main() {})).sort((a, b) => b.count - a.count);+ const done = totals?.done ?? 0;+ const failedRows = totals?.failed ?? 0;+ const recovered = totals?.recovered ?? 0;+ // The attempts nobody recovered: one row per request the user actually lost.+ const lost = Math.max(0, failedRows - recovered);+ const judged = done + lost;+const report = {windowDays: DAYS,database: maskUrl(process.env.DATABASE_URL),generations: totals?.rows ?? 0,- done: totals?.done ?? 0,- failed: totals?.failed ?? 0,+ done,+ failed: failedRows,+ recovered,+ lost,stillOpen: totals?.open ?? 0,- successRate: (totals?.done ?? 0) + (totals?.failed ?? 0) > 0- ? (totals.done) / (totals.done + totals.failed)- : null,+ includesRecovered: INCLUDE_RECOVERED,+ successRate: judged > 0 ? done / judged : null,classes: ranked,};@@ -154,13 +170,21 @@ async function main() {}console.log(`Forge generation errors, last ${DAYS} day(s), on ${report.database}\n`);- console.log(` generations ${report.generations} done ${report.done} failed ${report.failed} still running ${report.stillOpen}`);- console.log(` success rate ${report.successRate == null ? 'n/a (no finished generations)' : pct(report.done, report.done + report.failed)}\n`);+ console.log(` generations ${report.generations} done ${done} failed ${failedRows} still running ${report.stillOpen}`);+ console.log(` of those failures, ${recovered} were re-dispatched to another lane and finished there; ${lost} were lost`);+ console.log(` success rate ${report.successRate == null ? 'n/a (no finished generations)' : pct(done, judged)} (recovered attempts are not counted against it)\n`);if (!ranked.length) {- console.log(' No failed generations in this window. Nothing to rank.');+ console.log(+ recovered && !INCLUDE_RECOVERED+ ? ` No unrecovered failures in this window. Re-run with --include-recovered to rank the ${recovered} that failed over.`+ : ' No failed generations in this window. Nothing to rank.',+ );return;}+ if (recovered && !INCLUDE_RECOVERED) {+ console.log(` (${recovered} recovered attempt(s) excluded; --include-recovered to rank them too)\n`);+ }if (ONLY_CLASS) {const picked = ranked.filter((c) => c.id === ONLY_CLASS);diff --git a/tests/forge-error-class.test.js b/tests/forge-error-class.test.jsindex 10abbab47..22440ccf7 100644--- a/tests/forge-error-class.test.js+++ b/tests/forge-error-class.test.js@@ -40,6 +40,16 @@ describe('classifyForgeError', () => {expect(idOf('upstream returned 404 for the asset')).toBe('not_found_4xx');});+ // The two messages that actually dominate the ledger, verbatim from+ // production on 2026-08-14 (19 of that week's 23 failures). Both mean the+ // lane lost the job; neither matched before they were measured.+ it('reads the real production lost-task wordings as lost_task', () => {+ expect(idOf(+ 'task orphaned: no progress within 30 minutes (runner instance likely restarted mid-job); retry the request',+ )).toBe('lost_task');+ expect(idOf('NVCF request not found or expired')).toBe('lost_task');+ });+it('classifies the transport and vendor failures forge actually records', () => {expect(idOf('upstream returned 502 Bad Gateway')).toBe('upstream_5xx');expect(idOf('service unavailable')).toBe('upstream_5xx');