Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
#!/usr/bin/env node
/**
* reconstruct-load-test — does the selfie→avatar lane actually hold the launch
* target of 10,000 avatars/day?
*
* The reconstruction worker (workers/avatar-reconstruction) answers POST
* /reconstruct in tens of milliseconds and then does the real work in a FastAPI
* BackgroundTask behind an in-process semaphore (MAX_CONCURRENT_JOBS). That
* shape makes throughput invisible to every cheap probe: the submit latency is
* flat no matter how deep the backlog is, and Cloud Run's request-concurrency
* signal sees an idle instance while eight vCPUs grind. The only way to know
* what the lane holds is to submit real photos, poll every job to a terminal
* state, and measure completion rate against wall clock. That is this script.
*
* Two arrival models, because they answer different questions:
*
* --n 5 BURST. Fire N jobs at once and watch the
* completion times cluster. Waves of C mean one
* instance is serving everything with
* MAX_CONCURRENT_JOBS=C; that is the fastest way
* to see the real per-instance concurrency.
* --rate 10000 --duration 600 SUSTAINED. Open-loop arrivals at a target
* expressed in jobs/day (10000/day = one job
* every 8.64 s), held for --duration seconds.
* Open loop matters: a closed loop that waits for
* each job before sending the next can never
* build a backlog, so it always "passes".
*
* Targets:
*
* --target worker POST /reconstruct + GET /jobs/:id straight at the Cloud
* Run service. Needs GCP_RECONSTRUCTION_URL and
* GCP_RECONSTRUCTION_KEY. This is the mode for sustained
* runs: no per-user rate limit, no platform bookkeeping,
* and it isolates the worker from everything around it.
* --target platform POST /api/avatars/reconstruct + GET
* /api/avatars/regenerate-status through three.ws with a
* real signed-in account (RECON_LOAD_EMAIL /
* RECON_LOAD_PASSWORD, falling back to AUDIT_EMAIL /
* AUDIT_PASSWORD). Measures what a user actually waits
* for, including provider selection and the avatar
* materialization that follows a finished mesh. Capped by
* the platform's 60 uploads/hour/user limit, so a long
* sustained run needs --target worker.
*
* Photos are the 41 published reference faces the fidelity suite already uses
* (gs://three-ws-avatar-reconstructions/eval-refs, publicly readable, Vertex
* generated — no real person's photo is submitted). They are listed live rather
* than bundled, and each job gets a different face so nothing is served from a
* cache and every job exercises face detection for real.
*
* Usage:
* node scripts/reconstruct-load-test.mjs --target platform --n 5
* node scripts/reconstruct-load-test.mjs --rate 10000 --duration 600
* node scripts/reconstruct-load-test.mjs --rate 30000 --duration 900 --json
*
* Exit codes: 0 target met · 1 target missed or jobs failed · 2 bad usage ·
* 3 credentials missing.
*/
import { inspectGlb, isValidGlbHeader } from '../api/_lib/glb-inspect.js';
import { fileURLToPath } from 'node:url';
import { resolve } from 'node:path';
const REF_BUCKET = 'three-ws-avatar-reconstructions';
const REF_PREFIX = 'eval-refs/';
const DEFAULT_ORIGIN = 'https://three.ws';
// A job that has not reached a terminal state by here is counted as stuck, not
// waited on forever. The worker's own Cloud Run request timeout is 120 s and a
// healthy job finishes in single-digit seconds, so ten minutes only trips when
// the lane is genuinely backed up.
const JOB_DEADLINE_MS = 10 * 60 * 1000;
const POLL_MS = 2000;
const sleep = (ms) => new Promise((r) => setTimeout(r, ms));
export function parseArgs(argv) {
const opts = {
target: 'auto',
origin: DEFAULT_ORIGIN,
n: null,
ratePerDay: null,
durationSec: 600,
// Fraction of the arrival rate that must actually complete for the run to
// pass. Below 1.0 on purpose: an open-loop run that lands 96% of a target
// it held for ten minutes has proved the capacity, and demanding 100%
// makes a single slow photo fetch fail an otherwise healthy lane.
minCompletionRatio: 0.95,
minSuccessRatio: 0.9,
photos: null,
json: false,
help: false,
};
for (let i = 0; i < argv.length; i++) {
const next = () => argv[++i];
switch (argv[i]) {
case '--target': opts.target = next(); break;
case '--origin': opts.origin = String(next()).replace(/\/+$/, ''); break;
case '--n': opts.n = Number(next()); break;
case '--rate': opts.ratePerDay = Number(next()); break;
case '--duration': opts.durationSec = Number(next()); break;
case '--min-completion': opts.minCompletionRatio = Number(next()); break;
case '--min-success': opts.minSuccessRatio = Number(next()); break;
case '--photos': opts.photos = String(next()).split(',').map((s) => s.trim()).filter(Boolean); break;
case '--json': opts.json = true; break;
case '-h': case '--help': opts.help = true; break;
default:
return { ...opts, error: `unknown flag: ${argv[i]}` };
}
}
if (opts.n == null && opts.ratePerDay == null) opts.n = 5;
if (opts.n != null && !(opts.n > 0)) return { ...opts, error: '--n must be a positive number' };
if (opts.ratePerDay != null && !(opts.ratePerDay > 0)) return { ...opts, error: '--rate must be a positive number of jobs/day' };
if (opts.ratePerDay != null && !(opts.durationSec > 0)) return { ...opts, error: '--duration must be a positive number of seconds' };
if (!['auto', 'worker', 'platform'].includes(opts.target)) return { ...opts, error: `--target must be worker or platform` };
return opts;
}
/**
* Submission offsets in ms from t0.
*
* Burst mode returns all zeros; sustained mode spaces arrivals evenly at the
* target rate. Even spacing rather than Poisson jitter is deliberate — this
* measures whether the lane sustains a rate, and Poisson bunching would make
* consecutive runs of the same configuration disagree with each other.
*/
export function arrivalOffsets({ n, ratePerDay, durationSec }) {
if (ratePerDay) {
const intervalMs = 86_400_000 / ratePerDay;
const count = Math.max(1, Math.floor((durationSec * 1000) / intervalMs));
return Array.from({ length: count }, (_, i) => Math.round(i * intervalMs));
}
return Array.from({ length: n }, () => 0);
}
export function percentile(values, p) {
if (!values.length) return null;
const sorted = [...values].sort((a, b) => a - b);
const i = Math.min(sorted.length - 1, Math.max(0, Math.ceil((p / 100) * sorted.length) - 1));
return sorted[i];
}
/** Object keys out of a GCS XML bucket listing. */
export function photoKeysFromListing(xml) {
return [...String(xml).matchAll(/<Key>([^<]+)<\/Key>/g)]
.map((m) => m[1])
.filter((k) => /\.(png|jpe?g|webp)$/i.test(k));
}
/**
* Verdict for a finished run.
*
* `achievedPerDay` extrapolates from jobs that actually reached a good GLB over
* the wall clock of the run, which is the only number that means anything for
* capacity planning: submits that 202'd and then stranded are not throughput.
*/
export function summarize(records, { targetPerDay = null, wallMs, minCompletionRatio, minSuccessRatio }) {
const byOutcome = {};
for (const r of records) byOutcome[r.outcome] = (byOutcome[r.outcome] || 0) + 1;
const ok = records.filter((r) => r.outcome === 'ok');
const latencies = ok.map((r) => r.elapsedMs);
const terminal = records.filter((r) => r.outcome !== 'stuck' && r.outcome !== 'pending');
const achievedPerDay = wallMs > 0 ? (ok.length / (wallMs / 86_400_000)) : 0;
const successRatio = records.length ? ok.length / records.length : 0;
const completionRatio = targetPerDay ? achievedPerDay / targetPerDay : null;
const failures = [
...new Set(records.filter((r) => r.outcome !== 'ok').map((r) => `${r.outcome}${r.note ? `: ${r.note}` : ''}`)),
].slice(0, 8);
return {
submitted: records.length,
completed: ok.length,
terminal: terminal.length,
byOutcome,
p50Ms: percentile(latencies, 50),
p95Ms: percentile(latencies, 95),
maxMs: latencies.length ? Math.max(...latencies) : null,
queueWaitP95Ms: percentile(ok.map((r) => r.queuedMs).filter((v) => v != null), 95),
achievedPerDay,
targetPerDay,
completionRatio,
successRatio,
failures,
pass:
successRatio >= minSuccessRatio &&
(completionRatio == null || completionRatio >= minCompletionRatio),
};
}
/**
* Per-instance job concurrency implied by a burst run.
*
* With C jobs allowed in flight per instance, a burst of N against a single
* instance completes in ceil(N/C) waves of roughly equal latency. Counting how
* many jobs finished inside the first job's latency window recovers C without
* any access to the service's logs, which is what makes this usable from a
* machine that only has the public endpoint.
*/
export function impliedConcurrency(records) {
const ok = records.filter((r) => r.outcome === 'ok').sort((a, b) => a.finishedAt - b.finishedAt);
if (ok.length < 2) return null;
const first = ok[0];
const window = first.elapsedMs * 1.25;
return ok.filter((r) => r.finishedAt - first.startedAt <= window).length;
}
// ── photo corpus ─────────────────────────────────────────────────────────────
async function listReferencePhotos() {
const res = await fetch(
`https://storage.googleapis.com/${REF_BUCKET}?list-type=2&prefix=${encodeURIComponent(REF_PREFIX)}&max-keys=100`,
);
if (!res.ok) throw new Error(`could not list reference photos (HTTP ${res.status})`);
const keys = photoKeysFromListing(await res.text());
if (!keys.length) throw new Error('reference photo bucket returned no images');
return keys.map((k) => `https://storage.googleapis.com/${REF_BUCKET}/${k}`);
}
// ── drivers ──────────────────────────────────────────────────────────────────
/** Straight at the Cloud Run worker: the contract in workers/avatar-reconstruction. */
function workerDriver({ url, key }) {
const auth = { authorization: `Bearer ${key}`, 'content-type': 'application/json' };
return {
name: `worker ${url}`,
async submit({ photo, jobId }) {
const res = await fetch(`${url}/reconstruct`, {
method: 'POST',
headers: auth,
body: JSON.stringify({ images: [photo], job_id: jobId, body_type: 'neutral' }),
});
const body = await res.json().catch(() => ({}));
if (!res.ok) {
const err = new Error(String(body.detail || body.error || `HTTP ${res.status}`));
err.status = res.status;
throw err;
}
return { ref: body.job_id || jobId };
},
async poll(ref) {
const res = await fetch(`${url}/jobs/${encodeURIComponent(ref)}`, { headers: auth });
if (res.status === 404) return { status: 'queued' };
if (!res.ok) return { status: 'unknown' };
const body = await res.json().catch(() => ({}));
return { status: body.status, glbUrl: body.glb_url || null, error: body.error || null };
},
};
}
/** Through the product: sign in, submit, poll the same endpoints the site calls. */
async function platformDriver({ origin, email, password }) {
const res = await fetch(`${origin}/api/auth/login`, {
method: 'POST',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({ email, password }),
});
if (!res.ok) {
const body = await res.text().catch(() => '');
throw new Error(`login failed (HTTP ${res.status}) ${body.slice(0, 160)}`);
}
const cookie = (res.headers.getSetCookie?.() || [])
.map((c) => c.split(';')[0])
.join('; ');
if (!cookie) throw new Error('login returned no session cookie');
const headers = { cookie, 'content-type': 'application/json' };
return {
name: `platform ${origin}`,
async submit({ photo, jobId }) {
const r = await fetch(`${origin}/api/avatars/reconstruct`, {
method: 'POST',
headers,
body: JSON.stringify({ name: `capacity ${jobId}`, photos: [photo], visibility: 'private' }),
});
const body = await r.json().catch(() => ({}));
if (!r.ok) {
const err = new Error(String(body.message || body.error || `HTTP ${r.status}`));
err.status = r.status;
throw err;
}
return { ref: body.jobId, provider: body.provider || null };
},
async poll(ref) {
const r = await fetch(`${origin}/api/avatars/regenerate-status?jobId=${encodeURIComponent(ref)}`, { headers });
if (!r.ok) return { status: 'unknown' };
const body = await r.json().catch(() => ({}));
// The platform status payload is camelCase (resultGlbUrl), the worker's
// is snake_case (glb_url). Reading only one of them makes every job
// look stuck forever while it is quietly finishing.
return {
status: body.status,
glbUrl: body.resultGlbUrl || body.result_glb_url || null,
error: body.error || body.error_description || null,
};
},
};
}
async function resolveDriver(opts) {
const workerUrl = process.env.GCP_RECONSTRUCTION_URL;
const workerKey = process.env.GCP_RECONSTRUCTION_KEY;
const email = process.env.RECON_LOAD_EMAIL || process.env.AUDIT_EMAIL;
const password = process.env.RECON_LOAD_PASSWORD || process.env.AUDIT_PASSWORD;
const wantWorker = opts.target === 'worker' || (opts.target === 'auto' && workerUrl && workerKey);
if (wantWorker) {
if (!workerUrl || !workerKey) {
throw Object.assign(
new Error(
'worker mode needs GCP_RECONSTRUCTION_URL and GCP_RECONSTRUCTION_KEY (both live on the three-ws-api Cloud Run service and in .env)',
),
{ exitCode: 3 },
);
}
return workerDriver({ url: workerUrl.replace(/\/+$/, ''), key: workerKey });
}
if (!email || !password) {
throw Object.assign(
new Error(
'platform mode needs RECON_LOAD_EMAIL / RECON_LOAD_PASSWORD (or AUDIT_EMAIL / AUDIT_PASSWORD) for a real account on the target origin',
),
{ exitCode: 3 },
);
}
return platformDriver({ origin: opts.origin, email, password });
}
// ── run ──────────────────────────────────────────────────────────────────────
async function runJob(driver, { photo, index, stamp }) {
const jobId = `loadtest-${stamp}-${index}`;
const rec = {
index,
photo: photo.split('/').pop(),
provider: null,
outcome: 'pending',
startedAt: Date.now(),
finishedAt: null,
elapsedMs: null,
queuedMs: null,
meshes: null,
note: '',
};
let ref;
try {
const submitted = await driver.submit({ photo, jobId });
ref = submitted.ref;
rec.provider = submitted.provider ?? null;
if (!ref) {
rec.outcome = 'no_job_id';
rec.finishedAt = Date.now();
rec.elapsedMs = rec.finishedAt - rec.startedAt;
return rec;
}
} catch (err) {
rec.outcome = err?.status === 429 ? 'rate_limited' : 'submit_failed';
rec.note = String(err?.message || err).slice(0, 140);
rec.finishedAt = Date.now();
rec.elapsedMs = rec.finishedAt - rec.startedAt;
return rec;
}
const deadline = rec.startedAt + JOB_DEADLINE_MS;
let lastStatus = 'queued';
while (Date.now() < deadline) {
await sleep(POLL_MS);
let update;
try {
update = await driver.poll(ref);
} catch {
continue;
}
if (update.status && update.status !== 'unknown') {
// The first observation of a non-queued state is the closest thing to a
// queue-wait measurement available from outside: it is when the worker
// took the job off the semaphore.
if (rec.queuedMs == null && update.status !== 'queued') rec.queuedMs = Date.now() - rec.startedAt;
lastStatus = update.status;
}
if (update.status === 'done' && update.glbUrl) {
rec.finishedAt = Date.now();
rec.elapsedMs = rec.finishedAt - rec.startedAt;
await verifyGlb(rec, update.glbUrl);
return rec;
}
if (update.status === 'failed' || update.status === 'error') {
rec.outcome = 'failed';
rec.note = String(update.error || 'reconstruction failed').slice(0, 140);
rec.finishedAt = Date.now();
rec.elapsedMs = rec.finishedAt - rec.startedAt;
return rec;
}
}
rec.outcome = 'stuck';
rec.note = `never left "${lastStatus}" inside ${Math.round(JOB_DEADLINE_MS / 1000)}s`;
rec.finishedAt = Date.now();
rec.elapsedMs = rec.finishedAt - rec.startedAt;
return rec;
}
/**
* A done job only counts once its GLB parses with geometry. A lane that returns
* 200 and an empty file is a failure that a status-only test reports as success.
*/
async function verifyGlb(rec, glbUrl) {
try {
const res = await fetch(glbUrl);
if (!res.ok) {
rec.outcome = 'glb_unreachable';
rec.note = `GLB fetch HTTP ${res.status}`;
return;
}
const buf = Buffer.from(await res.arrayBuffer());
if (!isValidGlbHeader(buf)) {
rec.outcome = 'bad_glb';
rec.note = `not a GLB (${buf.length} bytes)`;
return;
}
const info = inspectGlb(buf);
rec.meshes = info?.meshCount ?? null;
rec.outcome = info && info.meshCount > 0 ? 'ok' : 'glb_no_geometry';
} catch (err) {
rec.outcome = 'glb_unreachable';
rec.note = String(err?.message || err).slice(0, 140);
}
}
const secs = (ms) => (ms == null ? 'n/a' : `${(ms / 1000).toFixed(1)}s`);
function printHelp() {
console.log(`reconstruct-load-test — throughput proof for the selfie→avatar lane
--target worker|platform where to submit (default: worker when
GCP_RECONSTRUCTION_URL + GCP_RECONSTRUCTION_KEY are
set, else platform)
--origin <url> platform origin (default ${DEFAULT_ORIGIN})
--n <count> burst mode: submit count jobs at once (default 5)
--rate <jobs/day> sustained mode: hold this arrival rate
--duration <seconds> sustained mode run length (default 600)
--min-completion <ratio> pass threshold vs --rate (default 0.95)
--min-success <ratio> pass threshold on job success (default 0.9)
--photos <urls> comma-separated photo URLs (default: the published
eval-refs reference faces)
--json machine-readable summary
`);
}
async function main() {
const opts = parseArgs(process.argv.slice(2));
if (opts.help) return printHelp(), 0;
if (opts.error) {
console.error(opts.error);
return 2;
}
let driver;
try {
driver = await resolveDriver(opts);
} catch (err) {
console.error(String(err?.message || err));
return err?.exitCode || 3;
}
const photos = opts.photos?.length ? opts.photos : await listReferencePhotos();
const offsets = arrivalOffsets(opts);
const stamp = String(process.hrtime.bigint()).slice(-8);
const mode = opts.ratePerDay
? `sustained ${opts.ratePerDay}/day for ${opts.durationSec}s (${offsets.length} jobs, one every ${(86_400 / opts.ratePerDay).toFixed(1)}s)`
: `burst of ${offsets.length}`;
if (!opts.json) {
console.log(`[reconstruct-load] ${mode} against ${driver.name}`);
console.log(`[reconstruct-load] ${photos.length} reference faces, one per job (round robin)`);
}
const t0 = Date.now();
const records = await Promise.all(
offsets.map(async (offset, index) => {
if (offset > 0) await sleep(offset);
return runJob(driver, { photo: photos[index % photos.length], index, stamp });
}),
);
const wallMs = Date.now() - t0;
const summary = summarize(records, {
targetPerDay: opts.ratePerDay,
wallMs,
minCompletionRatio: opts.minCompletionRatio,
minSuccessRatio: opts.minSuccessRatio,
});
const concurrency = opts.ratePerDay ? null : impliedConcurrency(records);
if (opts.json) {
console.log(JSON.stringify({ mode, driver: driver.name, wallMs, summary, concurrency, records }, null, 2));
return summary.pass ? 0 : 1;
}
console.log('\nidx photo outcome elapsed queued meshes note');
for (const r of records.sort((a, b) => a.index - b.index)) {
console.log(
[
String(r.index).padEnd(3),
String(r.photo).slice(0, 38).padEnd(38),
String(r.outcome).padEnd(14),
secs(r.elapsedMs).padStart(7),
secs(r.queuedMs).padStart(7),
String(r.meshes ?? '-').padStart(6),
r.note ? ` ${r.note}` : '',
].join(' '),
);
}
console.log(`\noutcomes: ${JSON.stringify(summary.byOutcome)}`);
console.log(`latency: p50=${secs(summary.p50Ms)} p95=${secs(summary.p95Ms)} max=${secs(summary.maxMs)}`);
console.log(`queue wait p95: ${secs(summary.queueWaitP95Ms)}`);
if (concurrency) {
console.log(`implied per-instance job concurrency: ${concurrency} (jobs finished inside the first job's latency)`);
}
console.log(
`throughput: ${summary.completed} verified GLBs in ${secs(wallMs)} = ${Math.round(summary.achievedPerDay).toLocaleString('en-US')}/day` +
(summary.targetPerDay ? ` against a ${summary.targetPerDay.toLocaleString('en-US')}/day target (${(summary.completionRatio * 100).toFixed(0)}%)` : ''),
);
if (summary.failures.length) console.log(`failures: ${summary.failures.join(' | ')}`);
console.log(summary.pass ? 'PASS' : 'FAIL');
return summary.pass ? 0 : 1;
}
const invokedDirectly = process.argv[1] && fileURLToPath(import.meta.url) === resolve(process.argv[1]);
if (invokedDirectly) {
process.exit(await main());
}