Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(forge): classify the two failure shapes that were hiding the real lost-task rate

nirholase0c1572fbb66
diff --git a/api/_lib/ops/forge-error-class.js b/api/_lib/ops/forge-error-class.js
index 9f8a61330..2ab5b61c7 100644
--- a/api/_lib/ops/forge-error-class.js
+++ b/api/_lib/ops/forge-error-class.js
@@ -65,7 +65,12 @@ export function normalizeForgeError(raw) {
/** @type {Array<{ id: string, label: string, test: RegExp }>} */
const CLASSES = [
{ id: 'timeout', label: 'timed out before the lane returned', test: /\btimed out\b|\btimeout\b|deadline exceeded|etimedout/ },
- { id: 'lost_task', label: 'lane lost the task (poll found nothing)', test: /task not found|prediction not found|job not found|unknown task|no such task|missing task/ },
+ // The two dominant real shapes both mean "the lane lost the job", and both
+ // used to land elsewhere: the self-host workers' own orphan reaper wording
+ // (workers/model-*/main.py) fell through to `other`, and NVIDIA's
+ // "NVCF request not found or expired" read as a generic 404. Measured on
+ // 2026-08-14 they were 19 of the prior week's 23 failures.
+ { id: 'lost_task', label: 'lane lost the task (poll found nothing)', test: /task not found|prediction not found|job not found|request not found|unknown task|no such task|missing task|orphaned/ },
{ id: 'aborted', label: 'request aborted mid-flight', test: /\baborted\b|abortsignal|operation was canceled|canceled by (the )?client/ },
{ id: 'out_of_memory', label: 'worker ran out of memory', test: /out of memory|\boom\b|cuda out of memory|killed \(signal 9\)|exit 144/ },
{ id: 'rate_limited', label: 'lane rate limited or over quota', test: /rate limit|too many requests|\b429\b|quota exceeded|over quota/ },
diff --git a/api/_lib/ops/forge-health-sensor.js b/api/_lib/ops/forge-health-sensor.js
index d145b0b8b..caebb9464 100644
--- a/api/_lib/ops/forge-health-sensor.js
+++ b/api/_lib/ops/forge-health-sensor.js
@@ -12,7 +12,12 @@
// pages instead of hiding.
//
// Metric: done / (done + failed) over a rolling window. Still-running and queued
-// rows are excluded (not yet an outcome). The window is 6h because forge volume
+// rows are excluded (not yet an outcome), and so are attempts that were
+// automatically re-dispatched to another lane (`superseded_by IS NOT NULL`):
+// the successor row carries that same request to its real outcome, so counting
+// the attempt too would page on the failover machinery working. Measured on
+// 2026-08-14, 13 of the 14 trellis_selfhost orphan failures in the prior week
+// were recovered that way and every one of them counted against the lane here. The window is 6h because forge volume
// is modest (~7 image jobs/hour) — long enough to carry a judge-able sample and
// smooth the bursty failure pattern, short enough to surface a sustained
// regression. Below MIN_ATTEMPTS the verdict is `unknown` (neutral, never pages).
@@ -81,7 +86,7 @@ export function classifyForgeBuckets(buckets, { minAttempts = MIN_ATTEMPTS } = {
return {
status: 'unknown',
done, failed, attempts, rate: null, worstBackend, worstPath, topReason,
- detail: `only ${attempts} finished generations in ${WINDOW_INTERVAL} — too few to judge`,
+ detail: `only ${attempts} finished generations in ${WINDOW_INTERVAL}, too few to judge`,
};
}
@@ -130,6 +135,7 @@ export async function gatherForgeHealth() {
count(*)::int AS n
FROM forge_creations
WHERE created_at >= now() - ${WINDOW_INTERVAL}::interval
+ AND superseded_by IS NULL
GROUP BY status, backend, path, reason
`
);
diff --git a/api/cron/forge-finalize.js b/api/cron/forge-finalize.js
index 7f68352fa..c43664acc 100644
--- a/api/cron/forge-finalize.js
+++ b/api/cron/forge-finalize.js
@@ -34,7 +34,13 @@
import { sql } from '../_lib/db.js';
import { cors, json, method, wrapCron } from '../_lib/http.js';
import { requireCron } from '../_lib/cron-auth.js';
-import { materializeCreation, markFailed, forgeStoreEnabled, createCreation } from '../_lib/forge-store.js';
+import {
+ materializeCreation,
+ markFailed,
+ markSupersededBy,
+ forgeStoreEnabled,
+ createCreation,
+} from '../_lib/forge-store.js';
import { notifyForgeComplete, notifyForgeFailed } from '../_lib/forge-notify.js';
import { createRegenProvider as createGcpProvider } from '../_providers/gcp.js';
import { createNvidiaProvider } from '../_providers/nvidia.js';
@@ -148,6 +154,10 @@ export function decideFailedSweep({
// resubmit to the next healthy lane instead. The successor gets its own
// creation row (same client/user), so this same cron sweeps it to completion
// and the completion notification still fires for a model, not a failure.
+//
+// Returns `{ successorId }` when the request now continues on another lane (the
+// caller links the two rows so the ledger shows a recovery, not a loss), or
+// null when nothing was redispatched.
async function tryCronRedispatch(row, ageMinutes) {
const prior = await readCronHop(row.replicate_job_id);
const hop = prior ? prior.hop : 0;
@@ -157,10 +167,10 @@ async function tryCronRedispatch(row, ageMinutes) {
hasReferenceImage: Boolean(row.preview_image_url),
path: row.path,
});
- if (decision !== 'redispatch') return false;
+ if (decision !== 'redispatch') return null;
const attempted = [...new Set([...(prior?.attempted || []), row.backend].filter(Boolean))];
const nextLane = await pickRedispatchLane({ attempted });
- if (!nextLane) return false;
+ if (!nextLane) return null;
try {
const submitted = await submitFailoverJob({
backend: nextLane,
@@ -172,7 +182,7 @@ async function tryCronRedispatch(row, ageMinutes) {
// The redispatch reconstructs from the primary stored view, so a
// multi-view original degrades visibly (views_used: 1), never silently:
// the same provenance contract as the attended failover.
- await createCreation({
+ const successorId = await createCreation({
clientKey: row.client_key,
userId: row.user_id ?? null,
prompt: row.prompt,
@@ -197,10 +207,10 @@ async function tryCronRedispatch(row, ageMinutes) {
console.warn(
`[forge-finalize] job failed on ${row.backend}; unattended auto-failover #${hop + 1} → ${nextLane}`,
);
- return true;
+ return { successorId: successorId ?? null };
} catch (err) {
console.warn(`[forge-finalize] redispatch failed: ${err?.message || err}`);
- return false;
+ return null;
}
}
@@ -282,12 +292,24 @@ export default wrapCron(async (req, res) => {
continue;
}
if (row.backend) await markLaneUnhealthy(row.backend).catch(() => {});
- if (await tryCronRedispatch(row, ageMinutes)) {
+ const redispatched = await tryCronRedispatch(row, ageMinutes);
+ if (redispatched) {
await markFailed({
replicateJobId: row.replicate_job_id,
clientKey: row.client_key,
error: status.error || 'generation failed',
});
+ // Link the attempt to its successor so the ledger reads as a
+ // recovery: without it the health sensor and the error report both
+ // count this row as a user-visible loss, when the request is in
+ // fact still running on another lane.
+ if (redispatched.successorId) {
+ await markSupersededBy({
+ replicateJobId: row.replicate_job_id,
+ clientKey: row.client_key,
+ successorId: redispatched.successorId,
+ });
+ }
out.failed_over++;
// No failure notification: the successor row carries the job on,
// and its completion (or terminal failure) notifies instead.
diff --git a/api/forge.js b/api/forge.js
index abb77ebf1..1703d12a5 100644
--- a/api/forge.js
+++ b/api/forge.js
@@ -88,6 +88,7 @@ import {
createCreation,
materializeCreation,
markFailed,
+ markSupersededBy,
findByJob,
} from './_lib/forge-store.js';
import { getSessionUser } from './_lib/auth.js';
@@ -2948,7 +2949,7 @@ async function pollJob(req, res, jobId) {
// and the success path materializes into the store as usual. The
// redispatch reconstructs from the primary stored view, so a
// multi-view original degrades visibly (views_used: 1), never silently.
- await createCreation({
+ const successorId = await createCreation({
clientKey,
userId: await sessionUserIdFromReq(req),
ipHash: hashIp(clientIp(req)),
@@ -2971,6 +2972,12 @@ async function pollJob(req, res, jobId) {
attempted,
});
if (bound) {
+ // Link the failed attempt to the successor now that the chain is
+ // durable, so the outcome ledger records a recovery rather than a
+ // loss (the health sensor and npm run forge:errors both read it).
+ if (successorId) {
+ await markSupersededBy({ replicateJobId: upstreamId, clientKey, successorId });
+ }
console.warn(
`[forge] job failed on ${failedBackend}; auto-failover #${hop + 1} → ${nextLane}`,
);
diff --git a/scripts/forge-error-report.mjs b/scripts/forge-error-report.mjs
index aef82c78d..5e2737215 100644
--- a/scripts/forge-error-report.mjs
+++ b/scripts/forge-error-report.mjs
@@ -16,6 +16,7 @@
* node scripts/forge-error-report.mjs --days 30
* node scripts/forge-error-report.mjs --days 7 --json
* node scripts/forge-error-report.mjs --class lost_task # every message in one class
+ * node scripts/forge-error-report.mjs --include-recovered # count failed-over attempts too
*
* Reads DATABASE_URL from .env.local, then .env, then the shell (same order as
* scripts/apply-migrations.mjs). Read-only: it runs SELECTs and nothing else.
@@ -53,6 +54,11 @@ const flag = (name, fallback) => {
const DAYS = Math.max(1, Math.min(365, Number(flag('days', '7')) || 7));
const JSON_OUT = args.includes('--json');
const ONLY_CLASS = flag('class', null);
+// A failed attempt that was automatically re-dispatched to another lane is not
+// a user-visible failure: the successor row carries the same request to its real
+// outcome. Ranking those by default puts a class the platform already recovers
+// from at rank 1, which is exactly how this report would waste a triager's day.
+const INCLUDE_RECOVERED = args.includes('--include-recovered');
if (!process.env.DATABASE_URL) {
console.error('DATABASE_URL is not set. Add it to .env.local or export it in your shell.');
@@ -83,6 +89,7 @@ async function main() {
SELECT count(*) FILTER (WHERE status = 'done')::int AS done,
count(*) FILTER (WHERE status = 'failed')::int AS failed,
count(*) FILTER (WHERE status NOT IN ('done', 'failed'))::int AS open,
+ count(*) FILTER (WHERE superseded_by IS NOT NULL)::int AS recovered,
count(*)::int AS rows
FROM forge_creations
WHERE created_at >= now() - ${since}::interval
@@ -99,6 +106,7 @@ async function main() {
FROM forge_creations
WHERE status = 'failed'
AND created_at >= now() - ${since}::interval
+ AND (${INCLUDE_RECOVERED} OR superseded_by IS NULL)
ORDER BY created_at DESC
`;
@@ -135,16 +143,24 @@ async function main() {
}))
.sort((a, b) => b.count - a.count);
+ const done = totals?.done ?? 0;
+ const failedRows = totals?.failed ?? 0;
+ const recovered = totals?.recovered ?? 0;
+ // The attempts nobody recovered: one row per request the user actually lost.
+ const lost = Math.max(0, failedRows - recovered);
+ const judged = done + lost;
+
const report = {
windowDays: DAYS,
database: maskUrl(process.env.DATABASE_URL),
generations: totals?.rows ?? 0,
- done: totals?.done ?? 0,
- failed: totals?.failed ?? 0,
+ done,
+ failed: failedRows,
+ recovered,
+ lost,
stillOpen: totals?.open ?? 0,
- successRate: (totals?.done ?? 0) + (totals?.failed ?? 0) > 0
- ? (totals.done) / (totals.done + totals.failed)
- : null,
+ includesRecovered: INCLUDE_RECOVERED,
+ successRate: judged > 0 ? done / judged : null,
classes: ranked,
};
@@ -154,13 +170,21 @@ async function main() {
}
console.log(`Forge generation errors, last ${DAYS} day(s), on ${report.database}\n`);
- console.log(` generations ${report.generations} done ${report.done} failed ${report.failed} still running ${report.stillOpen}`);
- console.log(` success rate ${report.successRate == null ? 'n/a (no finished generations)' : pct(report.done, report.done + report.failed)}\n`);
+ console.log(` generations ${report.generations} done ${done} failed ${failedRows} still running ${report.stillOpen}`);
+ console.log(` of those failures, ${recovered} were re-dispatched to another lane and finished there; ${lost} were lost`);
+ console.log(` success rate ${report.successRate == null ? 'n/a (no finished generations)' : pct(done, judged)} (recovered attempts are not counted against it)\n`);
if (!ranked.length) {
- console.log(' No failed generations in this window. Nothing to rank.');
+ console.log(
+ recovered && !INCLUDE_RECOVERED
+ ? ` No unrecovered failures in this window. Re-run with --include-recovered to rank the ${recovered} that failed over.`
+ : ' No failed generations in this window. Nothing to rank.',
+ );
return;
}
+ if (recovered && !INCLUDE_RECOVERED) {
+ console.log(` (${recovered} recovered attempt(s) excluded; --include-recovered to rank them too)\n`);
+ }
if (ONLY_CLASS) {
const picked = ranked.filter((c) => c.id === ONLY_CLASS);
diff --git a/tests/forge-error-class.test.js b/tests/forge-error-class.test.js
index 10abbab47..22440ccf7 100644
--- a/tests/forge-error-class.test.js
+++ b/tests/forge-error-class.test.js
@@ -40,6 +40,16 @@ describe('classifyForgeError', () => {
expect(idOf('upstream returned 404 for the asset')).toBe('not_found_4xx');
});
+ // The two messages that actually dominate the ledger, verbatim from
+ // production on 2026-08-14 (19 of that week's 23 failures). Both mean the
+ // lane lost the job; neither matched before they were measured.
+ it('reads the real production lost-task wordings as lost_task', () => {
+ expect(idOf(
+ 'task orphaned: no progress within 30 minutes (runner instance likely restarted mid-job); retry the request',
+ )).toBe('lost_task');
+ expect(idOf('NVCF request not found or expired')).toBe('lost_task');
+ });
+
it('classifies the transport and vendor failures forge actually records', () => {
expect(idOf('upstream returned 502 Bad Gateway')).toBe('upstream_5xx');
expect(idOf('service unavailable')).toBe('upstream_5xx');