Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

feat(ops): rank real forge failure classes from the outcome ledger instead of raw error text

nirholasa0b691198862
diff --git a/api/_lib/ops/forge-error-class.js b/api/_lib/ops/forge-error-class.js
new file mode 100644
index 000000000..9f8a61330
--- /dev/null
+++ b/api/_lib/ops/forge-error-class.js
@@ -0,0 +1,105 @@
+// @ts-check
+// Forge failure classifier. Turns the free-text `forge_creations.error` column
+// into a stable class you can count.
+//
+// The outcome ledger stores whatever the lane said, verbatim (markFailed clamps
+// it to 500 chars and nothing else). That is the right thing to store: the raw
+// message is the only forensic record of what a vendor or worker actually
+// returned. It is the wrong thing to GROUP BY. Every message carries the
+// specifics of its own failure (a prediction id, a task uuid, a signed URL, a
+// byte count, a minute count), so 40 instances of one recurring failure count as
+// 40 distinct "classes" and the ranking that should say "the self-host lane is
+// losing its tasks" says nothing at all.
+//
+// The pre-aggregation the health sensor does in SQL (`split_part(error, ':', 1)`)
+// is the cheap version of this and holds only while messages happen to lead with
+// a stable prefix: `generation timed out after 41 minutes` and
+// `generation timed out after 63 minutes` are already two reasons under it, and
+// a bare vendor sentence with no colon becomes its own class every time.
+//
+// So: normalize first (strip the varying parts), then match a known class, and
+// fall back to the normalized text rather than force-fitting. Same principle as
+// forge-classify.js: high precision, honest 'other', no invented buckets.
+
+/**
+ * Lowercase and strip the unbounded identifiers (URLs, uuids, long hex/base58
+ * ids) while KEEPING numbers. This is what the class patterns match against,
+ * because the numbers are half the signal: `502`, `429`, `404` are the class.
+ * @param {unknown} raw
+ * @returns {string}
+ */
+function cleanForgeError(raw) {
+ const text = String(raw ?? '').trim();
+ if (!text) return '';
+ return text
+ .toLowerCase()
+ .replace(/https?:\/\/\S+/g, ' <url> ')
+ .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, '<id>')
+ .replace(/\b[0-9a-f]{16,}\b/g, '<id>')
+ .replace(/\b[a-z0-9]{20,}\b/g, '<id>')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+/**
+ * Strip the parts of a failure message that vary per occurrence, numbers
+ * included, so two instances of the same failure normalize to one string. This
+ * is the GROUPING key for messages no class recognizes: without it,
+ * `generation timed out after 41 minutes` and the same failure 63 minutes later
+ * are two rows in a frequency ranking that exists to collapse them.
+ * @param {unknown} raw
+ * @returns {string}
+ */
+export function normalizeForgeError(raw) {
+ const cleaned = cleanForgeError(raw);
+ if (!cleaned) return '';
+ return cleaned
+ .replace(/\b\d+(\.\d+)?\s*(ms|s|sec|secs|seconds|m|min|mins|minutes|h|hours|mb|gb|kb|bytes)\b/g, '<n> $2')
+ .replace(/\b\d+\b/g, '<n>')
+ .replace(/\s+/g, ' ')
+ .trim();
+}
+
+// Ordered most-specific first: the first pattern that matches wins, so
+// "task not found" reads as a lost self-host task rather than a generic 404.
+/** @type {Array<{ id: string, label: string, test: RegExp }>} */
+const CLASSES = [
+ { id: 'timeout', label: 'timed out before the lane returned', test: /\btimed out\b|\btimeout\b|deadline exceeded|etimedout/ },
+ { id: 'lost_task', label: 'lane lost the task (poll found nothing)', test: /task not found|prediction not found|job not found|unknown task|no such task|missing task/ },
+ { id: 'aborted', label: 'request aborted mid-flight', test: /\baborted\b|abortsignal|operation was canceled|canceled by (the )?client/ },
+ { id: 'out_of_memory', label: 'worker ran out of memory', test: /out of memory|\boom\b|cuda out of memory|killed \(signal 9\)|exit 144/ },
+ { id: 'rate_limited', label: 'lane rate limited or over quota', test: /rate limit|too many requests|\b429\b|quota exceeded|over quota/ },
+ { id: 'payment_required', label: 'lane refused on billing or credits', test: /\b402\b|payment required|insufficient (credit|funds)|billing/ },
+ { id: 'unauthorized', label: 'lane rejected our credentials', test: /\b401\b|\b403\b|unauthorized|forbidden|invalid api key|authentication failed/ },
+ { id: 'upstream_5xx', label: 'lane returned a server error', test: /\b5\d\d\b|internal server error|bad gateway|service unavailable|gateway timeout/ },
+ { id: 'not_found_4xx', label: 'lane returned not found', test: /\b404\b|\b410\b|not found|gone/ },
+ { id: 'network', label: 'network failure reaching the lane', test: /econnreset|econnrefused|enotfound|socket hang up|network error|fetch failed|dns/ },
+ { id: 'bad_input_image', label: 'reference image rejected', test: /image (is )?(too large|invalid|unusable|not usable)|invalid image|unsupported image|no face|cannot decode image/ },
+ { id: 'bad_output_mesh', label: 'produced mesh was unusable', test: /invalid glb|empty mesh|no mesh|mesh (is )?(invalid|empty)|glb (is )?(invalid|empty|too large)|zero bytes/ },
+ { id: 'content_filtered', label: 'prompt or image blocked by a content filter', test: /content (policy|filter)|nsfw|safety (system|filter)|flagged/ },
+ { id: 'storage', label: 'could not persist the artifact', test: /\bs3\b|object storage|upload failed|bucket|access ?denied .*key/ },
+ { id: 'generic_failure', label: 'lane failed without saying why', test: /^generation failed$|^failed$|^error$|^unknown error$/ },
+];
+
+/**
+ * Classify one `forge_creations.error` value into a stable class.
+ *
+ * Returns `{ id: 'other', label: <normalized message> }` when nothing matches,
+ * so an unrecognized failure still groups with its own kind (identical messages
+ * share a normalized form) instead of being force-fit into a named class.
+ * An empty/absent message classifies as `none`.
+ * @param {unknown} raw
+ * @returns {{ id: string, label: string, normalized: string }}
+ */
+export function classifyForgeError(raw) {
+ const cleaned = cleanForgeError(raw);
+ if (!cleaned) return { id: 'none', label: 'no error recorded', normalized: '' };
+ const normalized = normalizeForgeError(raw);
+ for (const c of CLASSES) {
+ if (c.test.test(cleaned)) return { id: c.id, label: c.label, normalized };
+ }
+ return { id: 'other', label: normalized.slice(0, 80), normalized };
+}
+
+/** Every named class id, in match order. Consumers use it to render a full table. */
+export const FORGE_ERROR_CLASS_IDS = CLASSES.map((c) => c.id);
diff --git a/scripts/forge-error-report.mjs b/scripts/forge-error-report.mjs
new file mode 100644
index 000000000..aef82c78d
--- /dev/null
+++ b/scripts/forge-error-report.mjs
@@ -0,0 +1,202 @@
+#!/usr/bin/env node
+/**
+ * Forge generation error report. Ranks what is actually failing, from the
+ * outcome ledger (`forge_creations`), over a real window.
+ *
+ * /api/healthz already answers "is generation healthy right now?" over a fixed
+ * 6-hour window (api/_lib/ops/forge-health-sensor.js). That window is tuned to
+ * page on a burst, and at ~7 image jobs an hour it is regularly too small to
+ * judge at all (a quiet afternoon reads `unknown`). Triage needs the other
+ * question: over the last week, which failure class recurs most, on which lane,
+ * and what does it actually say? This is that query, as a command, so nobody
+ * hand-writes it against production again.
+ *
+ * Usage:
+ * node scripts/forge-error-report.mjs # last 7 days
+ * node scripts/forge-error-report.mjs --days 30
+ * node scripts/forge-error-report.mjs --days 7 --json
+ * node scripts/forge-error-report.mjs --class lost_task # every message in one class
+ *
+ * Reads DATABASE_URL from .env.local, then .env, then the shell (same order as
+ * scripts/apply-migrations.mjs). Read-only: it runs SELECTs and nothing else.
+ */
+
+import { readFileSync } from 'node:fs';
+import { fileURLToPath } from 'node:url';
+import path from 'node:path';
+import { neon } from '@neondatabase/serverless';
+import { classifyForgeError } from '../api/_lib/ops/forge-error-class.js';
+
+const HERE = path.dirname(fileURLToPath(import.meta.url));
+const REPO_ROOT = path.resolve(HERE, '..');
+
+for (const envFile of ['.env.local', '.env']) {
+ try {
+ const raw = readFileSync(path.resolve(REPO_ROOT, envFile), 'utf8');
+ for (const line of raw.split('\n')) {
+ const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);
+ if (!m || process.env[m[1]]) continue;
+ let val = m[2].trim();
+ if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {
+ val = val.slice(1, -1);
+ }
+ process.env[m[1]] = val;
+ }
+ } catch { /* file not present */ }
+}
+
+const args = process.argv.slice(2);
+const flag = (name, fallback) => {
+ const i = args.indexOf(`--${name}`);
+ return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : fallback;
+};
+const DAYS = Math.max(1, Math.min(365, Number(flag('days', '7')) || 7));
+const JSON_OUT = args.includes('--json');
+const ONLY_CLASS = flag('class', null);
+
+if (!process.env.DATABASE_URL) {
+ console.error('DATABASE_URL is not set. Add it to .env.local or export it in your shell.');
+ console.error('Production\'s value lives on the Cloud Run service:');
+ console.error(' gcloud run services describe three-ws-api --region us-central1 \\');
+ console.error(' --project aerial-vehicle-466722-p5 --format=yaml | grep -A1 DATABASE_URL');
+ process.exit(2);
+}
+
+const sql = neon(process.env.DATABASE_URL);
+
+function maskUrl(url) {
+ try {
+ const u = new URL(url);
+ if (u.password) u.password = '***';
+ return u.toString();
+ } catch {
+ return '<DATABASE_URL>';
+ }
+}
+
+const pct = (n, d) => (d > 0 ? `${((n / d) * 100).toFixed(1)}%` : 'n/a');
+
+async function main() {
+ const since = `${DAYS} days`;
+
+ const [totals] = await sql`
+ SELECT count(*) FILTER (WHERE status = 'done')::int AS done,
+ count(*) FILTER (WHERE status = 'failed')::int AS failed,
+ count(*) FILTER (WHERE status NOT IN ('done', 'failed'))::int AS open,
+ count(*)::int AS rows
+ FROM forge_creations
+ WHERE created_at >= now() - ${since}::interval
+ `;
+
+ // Raw failures, newest first. The classifier runs here rather than in SQL:
+ // grouping on the stored text would count each occurrence's own ids and
+ // durations as a separate class, which is the whole reason this report needs
+ // to exist. Bounded by the window and by forge's real volume.
+ const failures = await sql`
+ SELECT id, error, created_at,
+ COALESCE(backend, '(none)') AS backend,
+ COALESCE(path, '(none)') AS path
+ FROM forge_creations
+ WHERE status = 'failed'
+ AND created_at >= now() - ${since}::interval
+ ORDER BY created_at DESC
+ `;
+
+ /** @type {Map<string, { id: string, label: string, n: number, lanes: Map<string, number>, paths: Map<string, number>, first: string, last: string, samples: string[] }>} */
+ const classes = new Map();
+ for (const row of failures) {
+ const c = classifyForgeError(row.error);
+ const key = c.id === 'other' ? `other:${c.normalized}` : c.id;
+ let entry = classes.get(key);
+ if (!entry) {
+ entry = { id: c.id, label: c.label, n: 0, lanes: new Map(), paths: new Map(), first: row.created_at, last: row.created_at, samples: [] };
+ classes.set(key, entry);
+ }
+ entry.n += 1;
+ entry.lanes.set(row.backend, (entry.lanes.get(row.backend) || 0) + 1);
+ entry.paths.set(row.path, (entry.paths.get(row.path) || 0) + 1);
+ if (row.created_at < entry.first) entry.first = row.created_at;
+ if (row.created_at > entry.last) entry.last = row.created_at;
+ if (entry.samples.length < 3 && row.error) entry.samples.push(String(row.error).slice(0, 200));
+ }
+
+ const ranked = [...classes.entries()]
+ .map(([key, e]) => ({
+ key,
+ id: e.id,
+ label: e.label,
+ count: e.n,
+ share: failures.length ? e.n / failures.length : 0,
+ lanes: [...e.lanes.entries()].sort((a, b) => b[1] - a[1]).map(([lane, n]) => ({ lane, n })),
+ paths: [...e.paths.entries()].sort((a, b) => b[1] - a[1]).map(([p, n]) => ({ path: p, n })),
+ firstSeen: e.first,
+ lastSeen: e.last,
+ samples: e.samples,
+ }))
+ .sort((a, b) => b.count - a.count);
+
+ const report = {
+ windowDays: DAYS,
+ database: maskUrl(process.env.DATABASE_URL),
+ generations: totals?.rows ?? 0,
+ done: totals?.done ?? 0,
+ failed: totals?.failed ?? 0,
+ stillOpen: totals?.open ?? 0,
+ successRate: (totals?.done ?? 0) + (totals?.failed ?? 0) > 0
+ ? (totals.done) / (totals.done + totals.failed)
+ : null,
+ classes: ranked,
+ };
+
+ if (JSON_OUT) {
+ console.log(JSON.stringify(report, null, 2));
+ return;
+ }
+
+ console.log(`Forge generation errors, last ${DAYS} day(s), on ${report.database}\n`);
+ console.log(` generations ${report.generations} done ${report.done} failed ${report.failed} still running ${report.stillOpen}`);
+ console.log(` success rate ${report.successRate == null ? 'n/a (no finished generations)' : pct(report.done, report.done + report.failed)}\n`);
+
+ if (!ranked.length) {
+ console.log(' No failed generations in this window. Nothing to rank.');
+ return;
+ }
+
+ if (ONLY_CLASS) {
+ const picked = ranked.filter((c) => c.id === ONLY_CLASS);
+ if (!picked.length) {
+ console.error(` No class "${ONLY_CLASS}" in this window. Classes present: ${[...new Set(ranked.map((c) => c.id))].join(', ')}`);
+ process.exitCode = 1;
+ return;
+ }
+ for (const c of picked) {
+ console.log(` ${c.id}: ${c.count} failure(s), ${pct(c.count, failures.length)} of all failures`);
+ console.log(` lanes: ${c.lanes.map((l) => `${l.lane}×${l.n}`).join(', ')}`);
+ console.log(` paths: ${c.paths.map((p) => `${p.path}×${p.n}`).join(', ')}`);
+ console.log(` first ${new Date(c.firstSeen).toISOString()} last ${new Date(c.lastSeen).toISOString()}`);
+ for (const s of c.samples) console.log(` · ${s}`);
+ console.log('');
+ }
+ return;
+ }
+
+ console.log(' rank n share class worst lane message');
+ ranked.forEach((c, i) => {
+ const lane = c.lanes[0] ? `${c.lanes[0].lane}×${c.lanes[0].n}` : '(none)';
+ console.log(
+ ` ${String(i + 1).padEnd(5)} ${String(c.count).padEnd(5)} ${pct(c.count, failures.length).padEnd(7)} ` +
+ `${c.id.padEnd(22)} ${lane.padEnd(21)} ${(c.samples[0] || c.label).slice(0, 70)}`,
+ );
+ });
+
+ const top = ranked[0];
+ console.log(`\n Top class: ${top.id} (${top.label})`);
+ console.log(` ${top.count} of ${failures.length} failures (${pct(top.count, failures.length)}), lanes ${top.lanes.map((l) => `${l.lane}×${l.n}`).join(', ')}`);
+ console.log(` last seen ${new Date(top.lastSeen).toISOString()}`);
+ console.log(` drill in: node scripts/forge-error-report.mjs --days ${DAYS} --class ${top.id}`);
+}
+
+main().catch((e) => {
+ console.error('FAILED:', e?.message || e);
+ process.exitCode = 1;
+});
diff --git a/tests/forge-error-class.test.js b/tests/forge-error-class.test.js
new file mode 100644
index 000000000..10abbab47
--- /dev/null
+++ b/tests/forge-error-class.test.js
@@ -0,0 +1,99 @@
+// api/_lib/ops/forge-error-class.js: free-text forge failure -> countable class.
+//
+// The point of the classifier is that a frequency ranking over
+// `forge_creations.error` is meaningless without it, because every message
+// carries its own ids and durations. So the tests that matter are the collapsing
+// ones: two instances of the same failure, worded differently by the lane, must
+// land on one class and one grouping key. The rest pins match ordering (a lost
+// task must not read as a generic 404) and the honest 'other' fallback.
+
+import { describe, it, expect } from 'vitest';
+import {
+ classifyForgeError,
+ normalizeForgeError,
+ FORGE_ERROR_CLASS_IDS,
+} from '../api/_lib/ops/forge-error-class.js';
+
+const idOf = (s) => classifyForgeError(s).id;
+
+describe('classifyForgeError', () => {
+ it('collapses the same failure worded with different specifics', () => {
+ const a = classifyForgeError('generation timed out after 41 minutes');
+ const b = classifyForgeError('generation timed out after 63 minutes');
+ expect(a.id).toBe('timeout');
+ expect(b.id).toBe(a.id);
+ expect(b.normalized).toBe(a.normalized);
+ });
+
+ it('collapses messages that differ only by an embedded id or url', () => {
+ const a = classifyForgeError('task not found: 7f3a2b19c4d5e6f708192a3b4c5d6e7f');
+ const b = classifyForgeError('task not found: 0011223344556677889900aabbccddee');
+ expect(a.id).toBe('lost_task');
+ expect(a.normalized).toBe(b.normalized);
+ expect(normalizeForgeError('fetch failed for https://api.example.com/v1/x'))
+ .toBe(normalizeForgeError('fetch failed for https://api.example.com/v1/y'));
+ });
+
+ it('reads a lost task as lost_task, not as a generic not-found', () => {
+ expect(idOf('task not found')).toBe('lost_task');
+ expect(idOf('prediction not found for id 8ac1f0d2b3e4')).toBe('lost_task');
+ expect(idOf('upstream returned 404 for the asset')).toBe('not_found_4xx');
+ });
+
+ it('classifies the transport and vendor failures forge actually records', () => {
+ expect(idOf('upstream returned 502 Bad Gateway')).toBe('upstream_5xx');
+ expect(idOf('service unavailable')).toBe('upstream_5xx');
+ expect(idOf('429 Too Many Requests')).toBe('rate_limited');
+ expect(idOf('quota exceeded for this model')).toBe('rate_limited');
+ expect(idOf('401 unauthorized: invalid api key')).toBe('unauthorized');
+ expect(idOf('payment required: insufficient credit')).toBe('payment_required');
+ expect(idOf('socket hang up')).toBe('network');
+ expect(idOf('ECONNRESET while streaming the result')).toBe('network');
+ expect(idOf('CUDA out of memory')).toBe('out_of_memory');
+ expect(idOf('request aborted')).toBe('aborted');
+ expect(idOf('deadline exceeded')).toBe('timeout');
+ });
+
+ it('classifies the pipeline-side failures', () => {
+ expect(idOf('invalid image: cannot decode image')).toBe('bad_input_image');
+ expect(idOf('invalid glb: empty mesh')).toBe('bad_output_mesh');
+ expect(idOf('blocked by the safety filter')).toBe('content_filtered');
+ expect(idOf('upload failed to bucket three-ws-assets')).toBe('storage');
+ });
+
+ it('keeps the say-nothing messages as their own class', () => {
+ expect(idOf('generation failed')).toBe('generic_failure');
+ expect(idOf('unknown error')).toBe('generic_failure');
+ });
+
+ it('falls back to other rather than force-fitting an unknown message', () => {
+ const c = classifyForgeError('the renderer produced a scene with no camera');
+ expect(c.id).toBe('other');
+ expect(c.label).toContain('renderer');
+ expect(c.normalized).toBe('the renderer produced a scene with no camera');
+ });
+
+ it('reports an absent error as none', () => {
+ for (const empty of [null, undefined, '', ' ']) {
+ expect(classifyForgeError(empty).id).toBe('none');
+ }
+ });
+
+ it('exposes every class id in match order, with no duplicates', () => {
+ expect(FORGE_ERROR_CLASS_IDS[0]).toBe('timeout');
+ expect(FORGE_ERROR_CLASS_IDS).toContain('lost_task');
+ expect(new Set(FORGE_ERROR_CLASS_IDS).size).toBe(FORGE_ERROR_CLASS_IDS.length);
+ });
+});
+
+describe('normalizeForgeError', () => {
+ it('strips counts and durations so occurrences share one grouping key', () => {
+ expect(normalizeForgeError('mesh is 1234 bytes')).toBe(normalizeForgeError('mesh is 91 bytes'));
+ expect(normalizeForgeError('gave up after 3 retries')).toBe(normalizeForgeError('gave up after 9 retries'));
+ });
+
+ it('is empty for an absent message', () => {
+ expect(normalizeForgeError(null)).toBe('');
+ expect(normalizeForgeError(' ')).toBe('');
+ });
+});