Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
feat(ops): rank real forge failure classes from the outcome ledger instead of raw error text
nirholasa0b691198862
diff --git a/api/_lib/ops/forge-error-class.js b/api/_lib/ops/forge-error-class.jsnew file mode 100644index 000000000..9f8a61330--- /dev/null+++ b/api/_lib/ops/forge-error-class.js@@ -0,0 +1,105 @@+// @ts-check+// Forge failure classifier. Turns the free-text `forge_creations.error` column+// into a stable class you can count.+//+// The outcome ledger stores whatever the lane said, verbatim (markFailed clamps+// it to 500 chars and nothing else). That is the right thing to store: the raw+// message is the only forensic record of what a vendor or worker actually+// returned. It is the wrong thing to GROUP BY. Every message carries the+// specifics of its own failure (a prediction id, a task uuid, a signed URL, a+// byte count, a minute count), so 40 instances of one recurring failure count as+// 40 distinct "classes" and the ranking that should say "the self-host lane is+// losing its tasks" says nothing at all.+//+// The pre-aggregation the health sensor does in SQL (`split_part(error, ':', 1)`)+// is the cheap version of this and holds only while messages happen to lead with+// a stable prefix: `generation timed out after 41 minutes` and+// `generation timed out after 63 minutes` are already two reasons under it, and+// a bare vendor sentence with no colon becomes its own class every time.+//+// So: normalize first (strip the varying parts), then match a known class, and+// fall back to the normalized text rather than force-fitting. Same principle as+// forge-classify.js: high precision, honest 'other', no invented buckets.++/**+ * Lowercase and strip the unbounded identifiers (URLs, uuids, long hex/base58+ * ids) while KEEPING numbers. This is what the class patterns match against,+ * because the numbers are half the signal: `502`, `429`, `404` are the class.+ * @param {unknown} raw+ * @returns {string}+ */+function cleanForgeError(raw) {+ const text = String(raw ?? '').trim();+ if (!text) return '';+ return text+ .toLowerCase()+ .replace(/https?:\/\/\S+/g, ' <url> ')+ .replace(/[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/g, '<id>')+ .replace(/\b[0-9a-f]{16,}\b/g, '<id>')+ .replace(/\b[a-z0-9]{20,}\b/g, '<id>')+ .replace(/\s+/g, ' ')+ .trim();+}++/**+ * Strip the parts of a failure message that vary per occurrence, numbers+ * included, so two instances of the same failure normalize to one string. This+ * is the GROUPING key for messages no class recognizes: without it,+ * `generation timed out after 41 minutes` and the same failure 63 minutes later+ * are two rows in a frequency ranking that exists to collapse them.+ * @param {unknown} raw+ * @returns {string}+ */+export function normalizeForgeError(raw) {+ const cleaned = cleanForgeError(raw);+ if (!cleaned) return '';+ return cleaned+ .replace(/\b\d+(\.\d+)?\s*(ms|s|sec|secs|seconds|m|min|mins|minutes|h|hours|mb|gb|kb|bytes)\b/g, '<n> $2')+ .replace(/\b\d+\b/g, '<n>')+ .replace(/\s+/g, ' ')+ .trim();+}++// Ordered most-specific first: the first pattern that matches wins, so+// "task not found" reads as a lost self-host task rather than a generic 404.+/** @type {Array<{ id: string, label: string, test: RegExp }>} */+const CLASSES = [+ { id: 'timeout', label: 'timed out before the lane returned', test: /\btimed out\b|\btimeout\b|deadline exceeded|etimedout/ },+ { id: 'lost_task', label: 'lane lost the task (poll found nothing)', test: /task not found|prediction not found|job not found|unknown task|no such task|missing task/ },+ { id: 'aborted', label: 'request aborted mid-flight', test: /\baborted\b|abortsignal|operation was canceled|canceled by (the )?client/ },+ { id: 'out_of_memory', label: 'worker ran out of memory', test: /out of memory|\boom\b|cuda out of memory|killed \(signal 9\)|exit 144/ },+ { id: 'rate_limited', label: 'lane rate limited or over quota', test: /rate limit|too many requests|\b429\b|quota exceeded|over quota/ },+ { id: 'payment_required', label: 'lane refused on billing or credits', test: /\b402\b|payment required|insufficient (credit|funds)|billing/ },+ { id: 'unauthorized', label: 'lane rejected our credentials', test: /\b401\b|\b403\b|unauthorized|forbidden|invalid api key|authentication failed/ },+ { id: 'upstream_5xx', label: 'lane returned a server error', test: /\b5\d\d\b|internal server error|bad gateway|service unavailable|gateway timeout/ },+ { id: 'not_found_4xx', label: 'lane returned not found', test: /\b404\b|\b410\b|not found|gone/ },+ { id: 'network', label: 'network failure reaching the lane', test: /econnreset|econnrefused|enotfound|socket hang up|network error|fetch failed|dns/ },+ { id: 'bad_input_image', label: 'reference image rejected', test: /image (is )?(too large|invalid|unusable|not usable)|invalid image|unsupported image|no face|cannot decode image/ },+ { id: 'bad_output_mesh', label: 'produced mesh was unusable', test: /invalid glb|empty mesh|no mesh|mesh (is )?(invalid|empty)|glb (is )?(invalid|empty|too large)|zero bytes/ },+ { id: 'content_filtered', label: 'prompt or image blocked by a content filter', test: /content (policy|filter)|nsfw|safety (system|filter)|flagged/ },+ { id: 'storage', label: 'could not persist the artifact', test: /\bs3\b|object storage|upload failed|bucket|access ?denied .*key/ },+ { id: 'generic_failure', label: 'lane failed without saying why', test: /^generation failed$|^failed$|^error$|^unknown error$/ },+];++/**+ * Classify one `forge_creations.error` value into a stable class.+ *+ * Returns `{ id: 'other', label: <normalized message> }` when nothing matches,+ * so an unrecognized failure still groups with its own kind (identical messages+ * share a normalized form) instead of being force-fit into a named class.+ * An empty/absent message classifies as `none`.+ * @param {unknown} raw+ * @returns {{ id: string, label: string, normalized: string }}+ */+export function classifyForgeError(raw) {+ const cleaned = cleanForgeError(raw);+ if (!cleaned) return { id: 'none', label: 'no error recorded', normalized: '' };+ const normalized = normalizeForgeError(raw);+ for (const c of CLASSES) {+ if (c.test.test(cleaned)) return { id: c.id, label: c.label, normalized };+ }+ return { id: 'other', label: normalized.slice(0, 80), normalized };+}++/** Every named class id, in match order. Consumers use it to render a full table. */+export const FORGE_ERROR_CLASS_IDS = CLASSES.map((c) => c.id);diff --git a/scripts/forge-error-report.mjs b/scripts/forge-error-report.mjsnew file mode 100644index 000000000..aef82c78d--- /dev/null+++ b/scripts/forge-error-report.mjs@@ -0,0 +1,202 @@+#!/usr/bin/env node+/**+ * Forge generation error report. Ranks what is actually failing, from the+ * outcome ledger (`forge_creations`), over a real window.+ *+ * /api/healthz already answers "is generation healthy right now?" over a fixed+ * 6-hour window (api/_lib/ops/forge-health-sensor.js). That window is tuned to+ * page on a burst, and at ~7 image jobs an hour it is regularly too small to+ * judge at all (a quiet afternoon reads `unknown`). Triage needs the other+ * question: over the last week, which failure class recurs most, on which lane,+ * and what does it actually say? This is that query, as a command, so nobody+ * hand-writes it against production again.+ *+ * Usage:+ * node scripts/forge-error-report.mjs # last 7 days+ * node scripts/forge-error-report.mjs --days 30+ * node scripts/forge-error-report.mjs --days 7 --json+ * node scripts/forge-error-report.mjs --class lost_task # every message in one class+ *+ * Reads DATABASE_URL from .env.local, then .env, then the shell (same order as+ * scripts/apply-migrations.mjs). Read-only: it runs SELECTs and nothing else.+ */++import { readFileSync } from 'node:fs';+import { fileURLToPath } from 'node:url';+import path from 'node:path';+import { neon } from '@neondatabase/serverless';+import { classifyForgeError } from '../api/_lib/ops/forge-error-class.js';++const HERE = path.dirname(fileURLToPath(import.meta.url));+const REPO_ROOT = path.resolve(HERE, '..');++for (const envFile of ['.env.local', '.env']) {+ try {+ const raw = readFileSync(path.resolve(REPO_ROOT, envFile), 'utf8');+ for (const line of raw.split('\n')) {+ const m = line.match(/^([A-Z_][A-Z0-9_]*)=(.*)$/);+ if (!m || process.env[m[1]]) continue;+ let val = m[2].trim();+ if ((val.startsWith('"') && val.endsWith('"')) || (val.startsWith("'") && val.endsWith("'"))) {+ val = val.slice(1, -1);+ }+ process.env[m[1]] = val;+ }+ } catch { /* file not present */ }+}++const args = process.argv.slice(2);+const flag = (name, fallback) => {+ const i = args.indexOf(`--${name}`);+ return i >= 0 && args[i + 1] && !args[i + 1].startsWith('--') ? args[i + 1] : fallback;+};+const DAYS = Math.max(1, Math.min(365, Number(flag('days', '7')) || 7));+const JSON_OUT = args.includes('--json');+const ONLY_CLASS = flag('class', null);++if (!process.env.DATABASE_URL) {+ console.error('DATABASE_URL is not set. Add it to .env.local or export it in your shell.');+ console.error('Production\'s value lives on the Cloud Run service:');+ console.error(' gcloud run services describe three-ws-api --region us-central1 \\');+ console.error(' --project aerial-vehicle-466722-p5 --format=yaml | grep -A1 DATABASE_URL');+ process.exit(2);+}++const sql = neon(process.env.DATABASE_URL);++function maskUrl(url) {+ try {+ const u = new URL(url);+ if (u.password) u.password = '***';+ return u.toString();+ } catch {+ return '<DATABASE_URL>';+ }+}++const pct = (n, d) => (d > 0 ? `${((n / d) * 100).toFixed(1)}%` : 'n/a');++async function main() {+ const since = `${DAYS} days`;++ const [totals] = await sql`+ SELECT count(*) FILTER (WHERE status = 'done')::int AS done,+ count(*) FILTER (WHERE status = 'failed')::int AS failed,+ count(*) FILTER (WHERE status NOT IN ('done', 'failed'))::int AS open,+ count(*)::int AS rows+ FROM forge_creations+ WHERE created_at >= now() - ${since}::interval+ `;++ // Raw failures, newest first. The classifier runs here rather than in SQL:+ // grouping on the stored text would count each occurrence's own ids and+ // durations as a separate class, which is the whole reason this report needs+ // to exist. Bounded by the window and by forge's real volume.+ const failures = await sql`+ SELECT id, error, created_at,+ COALESCE(backend, '(none)') AS backend,+ COALESCE(path, '(none)') AS path+ FROM forge_creations+ WHERE status = 'failed'+ AND created_at >= now() - ${since}::interval+ ORDER BY created_at DESC+ `;++ /** @type {Map<string, { id: string, label: string, n: number, lanes: Map<string, number>, paths: Map<string, number>, first: string, last: string, samples: string[] }>} */+ const classes = new Map();+ for (const row of failures) {+ const c = classifyForgeError(row.error);+ const key = c.id === 'other' ? `other:${c.normalized}` : c.id;+ let entry = classes.get(key);+ if (!entry) {+ entry = { id: c.id, label: c.label, n: 0, lanes: new Map(), paths: new Map(), first: row.created_at, last: row.created_at, samples: [] };+ classes.set(key, entry);+ }+ entry.n += 1;+ entry.lanes.set(row.backend, (entry.lanes.get(row.backend) || 0) + 1);+ entry.paths.set(row.path, (entry.paths.get(row.path) || 0) + 1);+ if (row.created_at < entry.first) entry.first = row.created_at;+ if (row.created_at > entry.last) entry.last = row.created_at;+ if (entry.samples.length < 3 && row.error) entry.samples.push(String(row.error).slice(0, 200));+ }++ const ranked = [...classes.entries()]+ .map(([key, e]) => ({+ key,+ id: e.id,+ label: e.label,+ count: e.n,+ share: failures.length ? e.n / failures.length : 0,+ lanes: [...e.lanes.entries()].sort((a, b) => b[1] - a[1]).map(([lane, n]) => ({ lane, n })),+ paths: [...e.paths.entries()].sort((a, b) => b[1] - a[1]).map(([p, n]) => ({ path: p, n })),+ firstSeen: e.first,+ lastSeen: e.last,+ samples: e.samples,+ }))+ .sort((a, b) => b.count - a.count);++ const report = {+ windowDays: DAYS,+ database: maskUrl(process.env.DATABASE_URL),+ generations: totals?.rows ?? 0,+ done: totals?.done ?? 0,+ failed: totals?.failed ?? 0,+ stillOpen: totals?.open ?? 0,+ successRate: (totals?.done ?? 0) + (totals?.failed ?? 0) > 0+ ? (totals.done) / (totals.done + totals.failed)+ : null,+ classes: ranked,+ };++ if (JSON_OUT) {+ console.log(JSON.stringify(report, null, 2));+ return;+ }++ console.log(`Forge generation errors, last ${DAYS} day(s), on ${report.database}\n`);+ console.log(` generations ${report.generations} done ${report.done} failed ${report.failed} still running ${report.stillOpen}`);+ console.log(` success rate ${report.successRate == null ? 'n/a (no finished generations)' : pct(report.done, report.done + report.failed)}\n`);++ if (!ranked.length) {+ console.log(' No failed generations in this window. Nothing to rank.');+ return;+ }++ if (ONLY_CLASS) {+ const picked = ranked.filter((c) => c.id === ONLY_CLASS);+ if (!picked.length) {+ console.error(` No class "${ONLY_CLASS}" in this window. Classes present: ${[...new Set(ranked.map((c) => c.id))].join(', ')}`);+ process.exitCode = 1;+ return;+ }+ for (const c of picked) {+ console.log(` ${c.id}: ${c.count} failure(s), ${pct(c.count, failures.length)} of all failures`);+ console.log(` lanes: ${c.lanes.map((l) => `${l.lane}×${l.n}`).join(', ')}`);+ console.log(` paths: ${c.paths.map((p) => `${p.path}×${p.n}`).join(', ')}`);+ console.log(` first ${new Date(c.firstSeen).toISOString()} last ${new Date(c.lastSeen).toISOString()}`);+ for (const s of c.samples) console.log(` · ${s}`);+ console.log('');+ }+ return;+ }++ console.log(' rank n share class worst lane message');+ ranked.forEach((c, i) => {+ const lane = c.lanes[0] ? `${c.lanes[0].lane}×${c.lanes[0].n}` : '(none)';+ console.log(+ ` ${String(i + 1).padEnd(5)} ${String(c.count).padEnd(5)} ${pct(c.count, failures.length).padEnd(7)} ` ++ `${c.id.padEnd(22)} ${lane.padEnd(21)} ${(c.samples[0] || c.label).slice(0, 70)}`,+ );+ });++ const top = ranked[0];+ console.log(`\n Top class: ${top.id} (${top.label})`);+ console.log(` ${top.count} of ${failures.length} failures (${pct(top.count, failures.length)}), lanes ${top.lanes.map((l) => `${l.lane}×${l.n}`).join(', ')}`);+ console.log(` last seen ${new Date(top.lastSeen).toISOString()}`);+ console.log(` drill in: node scripts/forge-error-report.mjs --days ${DAYS} --class ${top.id}`);+}++main().catch((e) => {+ console.error('FAILED:', e?.message || e);+ process.exitCode = 1;+});diff --git a/tests/forge-error-class.test.js b/tests/forge-error-class.test.jsnew file mode 100644index 000000000..10abbab47--- /dev/null+++ b/tests/forge-error-class.test.js@@ -0,0 +1,99 @@+// api/_lib/ops/forge-error-class.js: free-text forge failure -> countable class.+//+// The point of the classifier is that a frequency ranking over+// `forge_creations.error` is meaningless without it, because every message+// carries its own ids and durations. So the tests that matter are the collapsing+// ones: two instances of the same failure, worded differently by the lane, must+// land on one class and one grouping key. The rest pins match ordering (a lost+// task must not read as a generic 404) and the honest 'other' fallback.++import { describe, it, expect } from 'vitest';+import {+ classifyForgeError,+ normalizeForgeError,+ FORGE_ERROR_CLASS_IDS,+} from '../api/_lib/ops/forge-error-class.js';++const idOf = (s) => classifyForgeError(s).id;++describe('classifyForgeError', () => {+ it('collapses the same failure worded with different specifics', () => {+ const a = classifyForgeError('generation timed out after 41 minutes');+ const b = classifyForgeError('generation timed out after 63 minutes');+ expect(a.id).toBe('timeout');+ expect(b.id).toBe(a.id);+ expect(b.normalized).toBe(a.normalized);+ });++ it('collapses messages that differ only by an embedded id or url', () => {+ const a = classifyForgeError('task not found: 7f3a2b19c4d5e6f708192a3b4c5d6e7f');+ const b = classifyForgeError('task not found: 0011223344556677889900aabbccddee');+ expect(a.id).toBe('lost_task');+ expect(a.normalized).toBe(b.normalized);+ expect(normalizeForgeError('fetch failed for https://api.example.com/v1/x'))+ .toBe(normalizeForgeError('fetch failed for https://api.example.com/v1/y'));+ });++ it('reads a lost task as lost_task, not as a generic not-found', () => {+ expect(idOf('task not found')).toBe('lost_task');+ expect(idOf('prediction not found for id 8ac1f0d2b3e4')).toBe('lost_task');+ expect(idOf('upstream returned 404 for the asset')).toBe('not_found_4xx');+ });++ it('classifies the transport and vendor failures forge actually records', () => {+ expect(idOf('upstream returned 502 Bad Gateway')).toBe('upstream_5xx');+ expect(idOf('service unavailable')).toBe('upstream_5xx');+ expect(idOf('429 Too Many Requests')).toBe('rate_limited');+ expect(idOf('quota exceeded for this model')).toBe('rate_limited');+ expect(idOf('401 unauthorized: invalid api key')).toBe('unauthorized');+ expect(idOf('payment required: insufficient credit')).toBe('payment_required');+ expect(idOf('socket hang up')).toBe('network');+ expect(idOf('ECONNRESET while streaming the result')).toBe('network');+ expect(idOf('CUDA out of memory')).toBe('out_of_memory');+ expect(idOf('request aborted')).toBe('aborted');+ expect(idOf('deadline exceeded')).toBe('timeout');+ });++ it('classifies the pipeline-side failures', () => {+ expect(idOf('invalid image: cannot decode image')).toBe('bad_input_image');+ expect(idOf('invalid glb: empty mesh')).toBe('bad_output_mesh');+ expect(idOf('blocked by the safety filter')).toBe('content_filtered');+ expect(idOf('upload failed to bucket three-ws-assets')).toBe('storage');+ });++ it('keeps the say-nothing messages as their own class', () => {+ expect(idOf('generation failed')).toBe('generic_failure');+ expect(idOf('unknown error')).toBe('generic_failure');+ });++ it('falls back to other rather than force-fitting an unknown message', () => {+ const c = classifyForgeError('the renderer produced a scene with no camera');+ expect(c.id).toBe('other');+ expect(c.label).toContain('renderer');+ expect(c.normalized).toBe('the renderer produced a scene with no camera');+ });++ it('reports an absent error as none', () => {+ for (const empty of [null, undefined, '', ' ']) {+ expect(classifyForgeError(empty).id).toBe('none');+ }+ });++ it('exposes every class id in match order, with no duplicates', () => {+ expect(FORGE_ERROR_CLASS_IDS[0]).toBe('timeout');+ expect(FORGE_ERROR_CLASS_IDS).toContain('lost_task');+ expect(new Set(FORGE_ERROR_CLASS_IDS).size).toBe(FORGE_ERROR_CLASS_IDS.length);+ });+});++describe('normalizeForgeError', () => {+ it('strips counts and durations so occurrences share one grouping key', () => {+ expect(normalizeForgeError('mesh is 1234 bytes')).toBe(normalizeForgeError('mesh is 91 bytes'));+ expect(normalizeForgeError('gave up after 3 retries')).toBe(normalizeForgeError('gave up after 9 retries'));+ });++ it('is empty for an absent message', () => {+ expect(normalizeForgeError(null)).toBe('');+ expect(normalizeForgeError(' ')).toBe('');+ });+});