Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

fix(vision): restore the image judge with a free OpenRouter rung after all three lanes died

nirholasabf076b087ef
diff --git a/api/_lib/seed-quality.js b/api/_lib/seed-quality.js
index 06f2e18cf..1a907febd 100644
--- a/api/_lib/seed-quality.js
+++ b/api/_lib/seed-quality.js
@@ -5,10 +5,10 @@
// asset enters the public catalog:
// • api/cron/forge-seed-cron.js — the per-minute trickle, in-process on
// Cloud Run (Vertex reachable via the attached service account).
-// • the bulk batch runner (gcp-credits campaign, work order 05 task 5, still
-// open: not in the tree yet) - driven from a workstation with no GCP
-// credentials, so it routes render + judge through the production HTTP
-// surfaces instead.
+// • scripts/gcp/seed-avatars.mjs - the bulk batch runner, driven from a
+// workstation with no GCP credentials, so it routes render (and by default
+// the judge too) through the production HTTP surfaces instead. It has no
+// 70 s function wall, so unlike the cron it runs BOTH stages below.
//
// Two stages, cheapest first:
//
@@ -192,9 +192,8 @@ export const SEED_WATCHLIST = Object.freeze([
* @param {{ origin: string, fetchImpl?: typeof fetch, timeoutMs?: number }} opts
*/
export function remoteTransport({ origin, fetchImpl = fetch, timeoutMs = 120_000 }) {
- const base = String(origin || '').replace(/\/+$/, '');
-
async function visionAsk({ png, prompt }) {
+ const base = String(origin || '').replace(/\/+$/, '');
const res = await fetchImpl(`${base}/api/vision`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
@@ -209,9 +208,81 @@ export function remoteTransport({ origin, fetchImpl = fetch, timeoutMs = 120_000
if (!res.ok) throw new Error(`vision ${res.status}: ${JSON.stringify(data).slice(0, 300)}`);
return String(data?.text || '');
}
+ return buildTransport({ name: 'remote', origin, fetchImpl, timeoutMs, visionAsk });
+}
+
+/**
+ * Same gate as remoteTransport, with the judge running IN THIS PROCESS against
+ * api/_lib/vision.js's provider chain instead of the deployed POST /api/vision.
+ * Rendering still goes to `origin`, because the renderer is a GPU service with
+ * no in-process equivalent; only the judge moves.
+ *
+ * This is how a vision-chain change is measured before it ships. A fix to the
+ * chain cannot be exercised through the deployed endpoint until the deploy
+ * lands, so a bulk run that needs the fixed chain points its judge here and
+ * keeps its accept rate honest in the meantime. Same prompts, same parser, same
+ * verdict maths as remoteTransport; the only difference is which process calls
+ * the model, so the numbers are comparable across the two.
+ *
+ * Needs whatever credentials that chain reads (OPENROUTER_API_KEY, NVIDIA_API_KEY,
+ * or GCP application-default credentials for the Vertex anchor) in this process.
+ *
+ * @param {{ origin: string, fetchImpl?: typeof fetch, timeoutMs?: number }} opts
+ */
+export function localJudgeTransport({ origin, fetchImpl = fetch, timeoutMs = 120_000 }) {
+ async function visionAsk({ png, prompt }) {
+ const { describeImage } = await import('./vision.js');
+ const out = await describeImage({
+ imageBase64: png.toString('base64'),
+ prompt,
+ maxTokens: 700,
+ timeoutMs,
+ });
+ return String(out?.text || '');
+ }
+ return buildTransport({ name: 'local-judge', origin, fetchImpl, timeoutMs, visionAsk });
+}
+
+/**
+ * Shared transport body: the renderer call and the two judge calls, over an
+ * injected `visionAsk`. Both transports differ only in that function, and the
+ * prompts and parsing must stay identical between them or their accept rates
+ * stop being comparable, which is why they are written once here.
+ */
+function buildTransport({ name, origin, fetchImpl, timeoutMs, visionAsk }) {
+ const base = String(origin || '').replace(/\/+$/, '');
+
+ /**
+ * Ask for a JSON verdict, and give a model that answered in prose exactly one
+ * more chance with the instruction restated as the last thing it reads.
+ *
+ * The free vision lanes obey "reply with only JSON" most of the time and
+ * occasionally open with "The image shows..." instead. Treating that as an
+ * infrastructure failure loses the whole asset: it is recorded as ungated,
+ * so the run learns nothing about that prompt and re-spends GPU time on it
+ * at the next resume. One retry is cheap next to a regeneration, and it is
+ * NOT a quality shortcut: the retry only re-asks for the format, never
+ * relaxes the verdict, and a second prose reply still fails the gate.
+ */
+ async function askJson({ png, prompt }) {
+ const first = await visionAsk({ png, prompt });
+ try {
+ return parseJsonReply(first);
+ } catch (err) {
+ const retry = await visionAsk({
+ png,
+ prompt: `${prompt}\n\nYour previous reply was prose and could not be parsed. Reply with ONLY the JSON object, starting with { and ending with }. No prose, no markdown fence.`,
+ });
+ try {
+ return parseJsonReply(retry);
+ } catch {
+ throw err;
+ }
+ }
+ }
return {
- name: 'remote',
+ name,
async render({ glbUrl, width = 768, height = 768, theta = 0, phi = 78 }) {
const res = await fetchImpl(`${base}/api/render/avatar-clip`, {
method: 'POST',
@@ -233,7 +304,7 @@ export function remoteTransport({ origin, fetchImpl = fetch, timeoutMs = 120_000
},
async judgeRealism({ png, prompt, viewLabel }) {
const { buildJudgePrompt } = await import('./quality-bench.js');
- const text = await visionAsk({
+ const parsed = await askJson({
png,
prompt: buildJudgePrompt({
prompt,
@@ -242,7 +313,6 @@ export function remoteTransport({ origin, fetchImpl = fetch, timeoutMs = 120_000
viewLabel,
}),
});
- const parsed = parseJsonReply(text);
const out = {};
for (const k of ['photorealism', 'geometryIntegrity', 'textureFidelity', 'promptAdherence']) {
const n = Number(parsed[k]);
@@ -253,8 +323,7 @@ export function remoteTransport({ origin, fetchImpl = fetch, timeoutMs = 120_000
return out;
},
async judgeRigReadiness({ png, prompt, category }) {
- const text = await visionAsk({ png, prompt: buildRigReadinessPrompt({ prompt, category }) });
- return normalizeRigReadiness(parseJsonReply(text));
+ return normalizeRigReadiness(await askJson({ png, prompt: buildRigReadinessPrompt({ prompt, category }) }));
},
};
}
diff --git a/api/_lib/vision.js b/api/_lib/vision.js
index 6b9bf8b73..ac31723af 100644
--- a/api/_lib/vision.js
+++ b/api/_lib/vision.js
@@ -57,6 +57,34 @@ import {
const NVIDIA_VISION_MODELS = [
'meta/llama-3.2-11b-vision-instruct',
];
+// Free OpenRouter vision lanes, in order.
+//
+// The text chain (llm.js providerChain) has carried an OpenRouter rung for a
+// long time; vision never did, and that gap took the whole endpoint down on
+// 2026-09-09: NVIDIA answered 500, the paid OpenAI backstop answered 429
+// `billing_not_active`, and Vertex answered 403 `Lightning dunning decision is
+// deny for project` (a project-level billing hold that denies Vertex AND the
+// AI-Studio Gemini endpoint on the same key, so both Google rungs die together).
+// Three rungs, one dead chain, and every consumer that judges an image with it
+// (catalog seeding's quality gate, forge validation, alt text) went dark.
+//
+// Only the `:free` suffix is used here. llm-pricing.js prices OpenRouter by
+// exactly that suffix (isOpenRouterFreeModel), so a `:free` route is metered at
+// zero and the spend ledger stays truthful; OpenRouter's `openrouter/free`
+// router meta-model answers fine but would be metered as paid, so it is
+// deliberately not in this list.
+//
+// Verified against the live platform key on 2026-09-09: nemotron-omni answered
+// 3/3 in a median 1.5 s with clean parseable JSON, which is what the seed judge
+// needs (api/_lib/seed-quality.js parses every reply as JSON). The two Gemma
+// routes were saturated (429) in the same probe and are kept behind it as spare
+// capacity: a 429 falls through in ~150 ms and, unlike the 410 that retired the
+// NIM model above, it recovers when the shared free pool drains.
+const OPENROUTER_VISION_MODELS = [
+ 'nvidia/nemotron-3-nano-omni-30b-a3b-reasoning:free',
+ 'google/gemma-4-31b-it:free',
+ 'google/gemma-4-26b-a4b-it:free',
+];
// Paid last-resort tail. gpt-5.4-nano is vision-capable and already priced in
// llm-pricing.js, keeping the backstop cheap and the spend ledger truthful.
const OPENAI_VISION_MODEL = 'gpt-5.4-nano';
@@ -164,7 +192,7 @@ export function inlineImageBudget(remainingMs, timeoutMs) {
// a handler that *chose* to surface it can return 503 — but consumers should
// generally catch it and degrade silently instead.
export class VisionUnavailableError extends Error {
- constructor(message = 'No vision provider available. Configure NVIDIA_API_KEY (free), GOOGLE_CLOUD_PROJECT (Vertex Gemini credits anchor), or OPENAI_API_KEY (paid backstop).') {
+ constructor(message = 'No vision provider available. Configure NVIDIA_API_KEY or OPENROUTER_API_KEY (free), GOOGLE_CLOUD_PROJECT (Vertex Gemini credits anchor), or OPENAI_API_KEY (paid backstop).') {
super(message);
this.name = 'VisionUnavailableError';
this.code = 'vision_unavailable';
@@ -211,6 +239,21 @@ export function visionChain() {
}));
}
}
+ // Free OpenRouter routes, alongside the NIM lanes and ahead of the credits
+ // anchor: same free tier, same zero meter, and they are the rungs that keep
+ // vision answering when both NVIDIA and Google are down at once. Ordering
+ // them here leaves the Vertex anchor exactly where it was in the chain
+ // relative to the paid tail; nothing is evicted.
+ if (env.OPENROUTER_API_KEY) {
+ for (const model of OPENROUTER_VISION_MODELS) {
+ chain.push(openaiCompatVisionProvider({
+ name: 'openrouter',
+ key: env.OPENROUTER_API_KEY,
+ url: 'https://openrouter.ai/api/v1/chat/completions',
+ model,
+ }));
+ }
+ }
// Vertex Gemini credits anchor: multimodal (Gemini Flash reads image_url
// parts, data URIs included, through the same OpenAI-compatible endpoint),
// keyless (OAuth token minted per request), billed to GCP credits. Sits
diff --git a/scripts/gcp/seed-avatars.mjs b/scripts/gcp/seed-avatars.mjs
index f17632820..deb4968a6 100644
--- a/scripts/gcp/seed-avatars.mjs
+++ b/scripts/gcp/seed-avatars.mjs
@@ -20,6 +20,7 @@
* node scripts/gcp/seed-avatars.mjs --limit=500 --concurrency=4
* node scripts/gcp/seed-avatars.mjs --categories=accessory
* node scripts/gcp/seed-avatars.mjs --no-vision # mesh stage only
+ * node scripts/gcp/seed-avatars.mjs --judge=local # judge in-process
* node scripts/gcp/seed-avatars.mjs --report # checkpoint stats
*
* RESUMABLE. Every decision is written to a checkpoint keyed by the bare prompt
@@ -44,7 +45,7 @@ import { dirname, join, resolve } from 'node:path';
import { fileURLToPath } from 'node:url';
import { SEED_PROMPTS, composeSeedPrompt } from '../../api/_lib/seed-prompts.js';
-import { evaluateSeedAsset, remoteTransport } from '../../api/_lib/seed-quality.js';
+import { evaluateSeedAsset, remoteTransport, localJudgeTransport } from '../../api/_lib/seed-quality.js';
import { isFreeBackend } from '../../api/_lib/forge-tiers.js';
const __dirname = dirname(fileURLToPath(import.meta.url));
@@ -66,6 +67,11 @@ const LIMIT = args.limit ? Number(args.limit) : 25;
const CONCURRENCY = Math.max(1, Math.min(Number(args.concurrency) || 3, 8));
const CATEGORIES = typeof args.categories === 'string' ? args.categories.split(',').map((s) => s.trim()) : null;
const VISION = args['no-vision'] !== true;
+// Where the vision judge runs. 'remote' (default) calls the deployed
+// POST /api/vision; 'local' runs the same prompts against this process's own
+// api/_lib/vision.js chain, which is the only way to measure a chain fix before
+// its deploy lands. Rendering is remote either way (see seed-quality.js).
+const JUDGE = args.judge === 'local' ? 'local' : 'remote';
const REPORT_ONLY = !!args.report;
const RETRY_REJECTS = !!args['retry-rejects'];
@@ -289,12 +295,13 @@ async function main() {
// Render + judge run through the live platform's own HTTP surfaces, so the
// batch enforces the identical gate from a machine with no GCP credentials.
- const transport = VISION ? remoteTransport({ origin: ORIGIN, timeoutMs: GATE_TIMEOUT_MS }) : null;
+ const makeTransport = JUDGE === 'local' ? localJudgeTransport : remoteTransport;
+ const transport = VISION ? makeTransport({ origin: ORIGIN, timeoutMs: GATE_TIMEOUT_MS }) : null;
log('Seeding the avatar catalog from api/_lib/seed-prompts.js');
log(` origin ${ORIGIN}`);
log(` queue ${queue.length} prompt(s) of ${pool.length} (${Object.keys(state.prompts).length} already decided)`);
- log(` gate mesh${VISION ? ' + vision judge (render + Vertex via /api/vision)' : ' only (--no-vision)'}`);
+ log(` gate mesh${VISION ? ` + vision judge (remote render, ${JUDGE} judge)` : ' only (--no-vision)'}`);
log(` concurrency ${CONCURRENCY}`);
log('');