Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// GET /api/cron/quality-bench — weekly bounded realism-regression smoke test.
//
// This is NOT the full 23-prompt x all-lanes x all-tiers baseline (that one is
// run by hand/CI via `node scripts/quality-bench.mjs` and its result committed
// to data/quality-bench/runs/ — Cloud Run's filesystem is ephemeral and this
// handler can't commit to git). Instead it runs a small fixed subset — enough
// prompts to span a few subject classes, on the platform's default standard-tier
// image lane — end to end for real (real forge generation, real render, real
// Vertex Gemini judge), and diffs the result against the most recent COMMITTED
// baseline run for the same lane/tier. It logs a structured summary line via
// console.log/console.error either way — Cloud Logging IS "the run log" here —
// so `gcloud logging read` surfaces a regression without anyone needing to run
// the full bench first. For a real before/after gate on a specific change, run
// `node scripts/quality-bench.mjs --compare=latest,previous` by hand — see
// data/quality-bench/README.md.
import { readFile, readdir } from 'node:fs/promises';
import path from 'node:path';
import { error, json, method, wrapCron } from '../_lib/http.js';
import { loadCatalog, runOne } from '../_lib/quality-bench.js';
import { requireCron } from '../_lib/cron-auth.js';
export const maxDuration = 300;
// Nothing enforces `maxDuration` here. It is a Vercel declaration, and this runs
// on Cloud Run, where the only ceilings are the Cloud Scheduler job's 320s
// attempt deadline and Cloud Run's own 900s request timeout. So the sweep has to
// bound itself. Until it did, "kept deliberately short" was an intention rather
// than a limit: the 2026-08-10 run spent 900s, collected a 504 from Cloud Run,
// left Scheduler recording the job as DEADLINE_EXCEEDED, and then logged its
// summary 21 minutes later into a request nobody was reading.
const RESPONSE_RESERVE_MS = 20_000;
const BUDGET_MS = maxDuration * 1000 - RESPONSE_RESERVE_MS;
// Starting another prompt with less runway than this just burns forge capacity
// on a result that cannot land, so the sweep skips it and says so instead.
const MIN_PROMPT_MS = 45_000;
// A small, fixed, cheap-to-run cross-section of the full bench: one
// people/organic subject, one hard-surface subject, one architecture subject.
const SMOKE_PROMPT_IDS = ['qb01', 'qb09', 'qb12'];
const SMOKE_TIER = 'standard';
const REGRESSION_THRESHOLD = 1.0;
const BENCH_DIR = path.join(process.cwd(), 'data', 'quality-bench');
const RUNS_DIR = path.join(BENCH_DIR, 'runs');
async function loadPrompts() {
const raw = JSON.parse(await readFile(path.join(BENCH_DIR, 'prompts.json'), 'utf8'));
return raw.prompts;
}
// Mean score for (lane, tier) across every committed run's results — the
// standing baseline this sweep compares a fresh smoke run against.
async function baselineMean(lane, tier) {
let files = [];
try {
files = (await readdir(RUNS_DIR)).filter((f) => f.endsWith('.json'));
} catch {
return null;
}
const scores = [];
for (const file of files) {
try {
const run = JSON.parse(await readFile(path.join(RUNS_DIR, file), 'utf8'));
for (const r of run.results || []) {
if (r.lane === lane && r.tier === tier && typeof r.meanScore === 'number') scores.push(r.meanScore);
}
} catch {
// skip a corrupt run file
}
}
if (!scores.length) return null;
return scores.reduce((a, b) => a + b, 0) / scores.length;
}
export default wrapCron(async (req, res) => {
if (!method(req, res, ['GET'])) return;
if (!requireCron(req, res)) return;
// Clock starts before the catalog fetch: loading the catalog is part of the
// budget, not free time in front of it.
const startedAt = Date.now();
const deadlineAt = startedAt + BUDGET_MS;
// The sweep drives our own deployment over HTTP, so it needs this service's
// own origin. Cloud Run always sets x-forwarded-proto, which is what makes
// production https; hardcoding the scheme instead made the bench unrunnable
// against a plain local server, so the one cron in this batch that could not
// be proven outside production was the one whose job is proving quality.
// Same derivation api/ar.js and api/irl/share.js already use.
const host = req.headers.host || 'three.ws';
const proto = req.headers['x-forwarded-proto'] || (/^localhost|127\.0\.0\.1/.test(host) ? 'http' : 'https');
const baseUrl = `${proto}://${host}`;
let catalog;
try {
catalog = await loadCatalog(baseUrl);
} catch (err) {
error(res, 502, 'catalog_unavailable', String(err?.message || err));
return;
}
const lane = catalog.default_backend_for_tier?.[SMOKE_TIER]?.image;
if (!lane) {
error(res, 503, 'no_lane', `no default image lane for tier "${SMOKE_TIER}"`);
return;
}
const allPrompts = await loadPrompts();
const prompts = SMOKE_PROMPT_IDS.map((id) => allPrompts.find((p) => p.id === id)).filter(Boolean);
const results = [];
const skippedForBudget = [];
for (const p of prompts) {
if (Date.now() + MIN_PROMPT_MS > deadlineAt) {
skippedForBudget.push(p.id);
continue;
}
results.push(await runOne(baseUrl, p, lane, SMOKE_TIER, { deadlineAt }));
}
const scored = results.filter((r) => typeof r.meanScore === 'number');
const smokeMean = scored.length ? scored.reduce((s, r) => s + r.meanScore, 0) / scored.length : null;
const baseline = await baselineMean(lane, SMOKE_TIER);
// A sweep that ran out of clock covers fewer prompts than the baseline it
// would be compared against, so its delta measures the budget, not the
// models. Report it loudly and do not let it trip the regression alarm.
const complete = !skippedForBudget.length && !results.some((r) => r.status === 'budget_exhausted');
const summary = {
kind: 'quality-bench-smoke',
lane,
tier: SMOKE_TIER,
promptIds: SMOKE_PROMPT_IDS,
smokeMean,
baselineMean: baseline,
delta: smokeMean != null && baseline != null ? smokeMean - baseline : null,
complete,
skippedForBudget,
budgetMs: BUDGET_MS,
elapsedMs: Date.now() - startedAt,
perPrompt: results.map((r) => ({ promptId: r.promptId, status: r.status, meanScore: r.meanScore, error: r.error || null })),
};
const regressed = complete && summary.delta != null && summary.delta < -REGRESSION_THRESHOLD;
if (regressed) {
console.error('[cron] quality-bench REGRESSION', summary);
} else if (!complete) {
console.warn('[cron] quality-bench INCOMPLETE (ran out of budget)', summary);
} else {
console.log('[cron] quality-bench smoke run', summary);
}
// Always 200: a regression is a reported finding, not a failed invocation.
// A non-2xx here would make Cloud Scheduler retry the (expensive) bench run.
json(res, 200, { ok: true, regressed, ...summary });
}, { maxDurationMs: maxDuration * 1000 });