Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* Selfie pipeline -- bridges the capture UI on /create/selfie to the native
* avatar reconstruction backend.
*
* Flow:
* [selfie:submit] -> downscale photos (frontal required, sides optional)
* -> dispatches selfie:preview { dataUrl } so the build view can show the
* user's own photo on a placeholder body while the real GLB renders
* -> POST /api/avatars/reconstruct
* -> dispatches selfie:building { jobId }
* -> poll /api/avatars/regenerate-status?jobId=...
* -> dispatches selfie:progress { label } on each poll tick
* -> on { status: 'done', resultAvatarId } -> dispatches selfie:done { avatarId }
* -> on { status: 'failed' } or timeout -> dispatches selfie:build-error
* { message, slot? } where slot is the photo that likely caused the failure
*/
import { assessPhoto, refineSelfie, warmRefiner } from './selfie-refine.js';
import { log } from './shared/log.js';
// Prefetch the detection + segmentation models so the refine step is instant by
// the time the user has framed and submitted their photo. Fire-and-forget.
warmRefiner();
const SUBMIT_ENDPOINT = '/api/avatars/reconstruct';
const STATUS_ENDPOINT = '/api/avatars/regenerate-status';
const MAX_DIM = 1024;
const JPEG_QUALITY = 0.88;
const POLL_INTERVAL_MS = 3000;
const POLL_TIMEOUT_MS = 8 * 60 * 1000;
const RATE_LIMIT_COOLDOWN_MS = 60_000;
/** Extra polls to allow a 'done' job whose avatar has not materialized yet. */
const MATERIALIZE_RETRY_POLLS = 3;
// BYOK key session storage keys — readable by the page to pre-populate the
// key entry form and by the pipeline to attach the key to submit requests.
const BYOK_KEY_STORAGE = 'selfie:byok:key';
const BYOK_PROVIDER_STORAGE = 'selfie:byok:provider';
/** Read the BYOK credentials stored for this session, if any. */
export function getStoredByokKey() {
try {
const key = sessionStorage.getItem(BYOK_KEY_STORAGE);
const provider = sessionStorage.getItem(BYOK_PROVIDER_STORAGE);
if (key && provider) return { key, provider };
} catch (_) {}
return null;
}
/** Persist BYOK credentials to the session. */
export function storeByokKey(provider, key) {
try {
sessionStorage.setItem(BYOK_PROVIDER_STORAGE, provider);
sessionStorage.setItem(BYOK_KEY_STORAGE, key);
} catch (_) {}
}
/** Clear any stored BYOK credentials (e.g. after an invalid-key error). */
export function clearStoredByokKey() {
try {
sessionStorage.removeItem(BYOK_KEY_STORAGE);
sessionStorage.removeItem(BYOK_PROVIDER_STORAGE);
} catch (_) {}
}
let _building = false;
let _lastSlotsSent = /** @type {string[]} */ ([]);
let _jobStartTime = 0;
let _rateLimitedUntil = 0;
document.addEventListener('selfie:submit', (event) => {
const ev = /** @type {CustomEvent} */ (event);
_building = false;
run(ev.detail).catch((err) => {
log.error('[selfie-pipeline]', err);
if (err?.redirect) {
window.location.assign(err.redirect);
return;
}
// No platform backend + no BYOK key: surface the key-entry form.
if (err?.needsByok) {
document.dispatchEvent(new CustomEvent('selfie:needs-byok', {
detail: { providers: err.providers || ['meshy', 'tripo'] },
}));
return;
}
if (_building) {
document.dispatchEvent(new CustomEvent('selfie:build-error', {
detail: {
message: err.userMessage || 'Something went wrong. Try again.',
slot: err.slot || null,
},
}));
} else {
setStatus(err.userMessage || 'Something went wrong. Try again.', { error: true });
if (err.slot) highlightSlotError(err.slot);
if (err.rateLimited) {
startRateLimitCooldown();
} else {
resetSubmit();
}
}
});
});
// Resume polling if we have a pending job from a previous page load.
(function resumePendingJob() {
try {
const pending = sessionStorage.getItem('selfie:pendingJobId');
if (!pending) return;
sessionStorage.removeItem('selfie:pendingJobId');
_building = true;
_jobStartTime = Date.now();
document.dispatchEvent(new CustomEvent('selfie:building', { detail: { jobId: pending } }));
pollUntilDone(pending).then((finalJob) => {
if (finalJob.resultAvatarId) {
document.dispatchEvent(new CustomEvent('selfie:done', {
detail: { avatarId: finalJob.resultAvatarId },
}));
}
}).catch((err) => {
log.error('[selfie-pipeline] resume failed:', err);
if (err?.redirect) {
window.location.assign(err.redirect);
return;
}
document.dispatchEvent(new CustomEvent('selfie:build-error', {
detail: { message: err.userMessage || 'Could not resume. Try again.', slot: null },
}));
});
} catch (_) {}
})();
/**
* @param {{
* files: Record<'frontal'|'left'|'right', File | null>,
* avatarType: 'v1' | 'v2',
* method: 'camera' | 'upload' | null,
* }} detail
*/
async function run(detail) {
_building = false;
if (!detail?.files?.frontal) {
throw withMessage(new Error('missing frontal'), 'Add a front-facing photo to start.', 'frontal');
}
if (Date.now() < _rateLimitedUntil) {
const secsLeft = Math.ceil((_rateLimitedUntil - Date.now()) / 1000);
const e = withMessage(
new Error('rate_limited'),
`Too many attempts -- wait ${secsLeft}s and try again.`,
);
/** @type {any} */ (e).rateLimited = true;
throw e;
}
setStatus('Preparing photos...');
const slots = /** @type {const} */ (['frontal', 'left', 'right']);
_lastSlotsSent = [];
const photos = [];
const warnings = [];
for (const slot of slots) {
const file = detail.files[slot];
if (!file) continue;
const bitmap = await loadBitmap(file);
try {
// Gate the input. The frontal must contain a face — that's the only hard
// stop. Everything else (soft/blurry/illustration/far) is a non-fatal
// warning the user sees but can proceed past.
setStatus(slot === 'frontal' ? 'Checking your photo...' : `Checking ${slot} angle...`);
const assessment = await assessPhoto(bitmap);
if (slot === 'frontal' && assessment.verdict === 'block') {
throw withMessage(
new Error(`quality block: ${assessment.primary}`),
assessment.message,
'frontal',
);
}
if (assessment.primary && assessment.primary !== 'multiple-faces') {
warnings.push({ slot, primary: assessment.primary, message: assessment.message });
}
// Isolate the subject from its background and reframe to a clean
// head-and-shoulders square — the composition the reconstruction lane
// handles best. This is the fix for the "whole photo on a card" failure.
setStatus('Isolating subject...');
let dataUrl;
try {
const refined = await refineSelfie(bitmap, {
faceBox: assessment.faceBox,
onStep: (label) => setStatus(label),
});
dataUrl = refined.dataUrl;
} catch (refineErr) {
// Refinement is an enhancement, never a gate: fall back to a plain
// downscale so a model/canvas hiccup can't block avatar creation.
log.warn('[selfie-pipeline] refine failed, using raw photo:', refineErr);
dataUrl = await fileToDataUrl(file);
}
if (!dataUrl || dataUrl.length < 500) {
throw withMessage(new Error('invalid image data'), 'One of your photos could not be processed. Try a different file.', slot);
}
photos.push(dataUrl);
_lastSlotsSent.push(slot);
} finally {
try { bitmap.close?.(); } catch (_) {}
}
}
if (warnings.length) {
document.dispatchEvent(new CustomEvent('selfie:quality', { detail: { warnings } }));
}
// Preview the refined frontal — the user sees exactly what the engine sees.
document.dispatchEvent(new CustomEvent('selfie:preview', {
detail: { dataUrl: photos[0] },
}));
setStatus('Submitting to avatar engine...');
const byok = getStoredByokKey();
const submitBody = {
name: defaultAvatarName(),
photos,
visibility: 'private',
params: { style: detail.avatarType },
...(byok ? { provider_key: byok.key, provider_name: byok.provider } : {}),
};
const submitRes = await fetch(SUBMIT_ENDPOINT, {
method: 'POST',
credentials: 'include',
headers: { 'content-type': 'application/json' },
body: JSON.stringify(submitBody),
});
if (!submitRes.ok) {
const payload = await submitRes.json().catch(() => ({}));
// The platform has no backend and the user has no stored BYOK key.
// Dispatch a dedicated event so the page can show the key entry form.
if (payload.code === 'regen_needs_byok') {
const e = new Error('regen_needs_byok');
/** @type {any} */ (e).needsByok = true;
/** @type {any} */ (e).providers = payload.providers || ['meshy', 'tripo'];
throw e;
}
throw mapApiError(submitRes.status, payload);
}
const submitData = await submitRes.json();
if (!submitData.jobId) {
throw withMessage(new Error('no jobId'), 'The avatar engine did not return a job.');
}
_building = true;
_jobStartTime = Date.now();
try { sessionStorage.setItem('selfie:pendingJobId', submitData.jobId); } catch (_) {}
document.dispatchEvent(new CustomEvent('selfie:building', { detail: { jobId: submitData.jobId } }));
const finalJob = await pollUntilDone(submitData.jobId);
try { sessionStorage.removeItem('selfie:pendingJobId'); } catch (_) {}
if (!finalJob.resultAvatarId) {
throw withMessage(
new Error('done without resultAvatarId'),
'Avatar finished but could not be saved. Try again.',
);
}
document.dispatchEvent(new CustomEvent('selfie:done', {
detail: { avatarId: finalJob.resultAvatarId },
}));
}
/**
* Poll the status endpoint until the job reaches a terminal state, throwing
* with a user-facing message on failure or timeout.
* @param {string} jobId
*/
async function pollUntilDone(jobId) {
const deadline = Date.now() + POLL_TIMEOUT_MS;
let attempt = 0;
let consecutiveErrors = 0;
// A 'done' job with no avatar id means the server's materialize stage threw
// after the mesh was already generated. That stage re-runs on every poll, so
// a transient (a storage blip, a lock) clears itself if we look again instead
// of failing the user out of a reconstruction they already waited through.
// Bounded: a deterministic failure must not spin until the poll deadline.
let unmaterializedPolls = 0;
const url = `${STATUS_ENDPOINT}?jobId=${encodeURIComponent(jobId)}`;
while (Date.now() < deadline) {
const interval = attempt === 0 ? 1500 : backoffInterval(consecutiveErrors);
await sleep(interval);
attempt += 1;
let res;
try {
res = await fetch(url, { credentials: 'include' });
consecutiveErrors = 0;
} catch (err) {
consecutiveErrors += 1;
log.warn('[selfie-pipeline] poll fetch failed (%d):', consecutiveErrors, err);
if (consecutiveErrors >= 5) {
throw withMessage(
new Error('poll network failure'),
'Lost connection to the avatar engine. Check your internet and try again.',
);
}
continue;
}
if (!res.ok) {
if (res.status === 404) {
throw withMessage(new Error('job vanished'), 'Job disappeared. Try again.');
}
if (res.status === 401 || res.status === 403) {
// Session expired (or never authenticated) mid-build. Bounce to
// login and resume this job on return — don't spin for 8 minutes
// on a status that retrying can never recover from.
try { sessionStorage.setItem('selfie:pendingJobId', jobId); } catch (_) {}
throw mapApiError(res.status, await res.json().catch(() => ({})));
}
consecutiveErrors += 1;
continue;
}
const job = await res.json().catch(() => ({}));
const elapsed = Date.now() - _jobStartTime;
document.dispatchEvent(new CustomEvent('selfie:progress', {
detail: { label: statusLabel(job.status, attempt, elapsed), attempt },
}));
if (job.status === 'done') {
if (job.resultAvatarId) return job;
unmaterializedPolls += 1;
if (unmaterializedPolls <= MATERIALIZE_RETRY_POLLS) continue;
return job;
}
if (job.status === 'failed') {
throw withMessage(
new Error(job.error || 'reconstruct failed'),
friendlyJobError(job.error, job.errorKind),
inferFailingSlot(job.error),
);
}
if (elapsed > 10 * 60 * 1000 && job.status === 'running') {
throw withMessage(
new Error('stalled job'),
'This is taking unusually long. Your avatar may still be processing -- check your dashboard in a few minutes.',
);
}
}
throw withMessage(
new Error('poll timeout'),
'Avatar is taking longer than expected. Check your dashboard in a few minutes.',
);
}
/** @param {number} errors */
function backoffInterval(errors) {
if (errors <= 0) return POLL_INTERVAL_MS;
return Math.min(POLL_INTERVAL_MS * Math.pow(1.5, errors), 30_000);
}
/**
* @param {'queued'|'running'|'rigging'|'done'|'failed'|undefined} status
* @param {number} attempt
* @param {number} elapsedMs
*/
function statusLabel(status, attempt, elapsedMs) {
if (status === 'queued') return 'Queued -- waiting for a reconstruction slot...';
if (status === 'rigging') return 'Auto-rigging skeleton and skinning...';
if (status === 'running') {
if (attempt <= 4) return 'Generating 3D mesh from your photo...';
if (attempt <= 12) return 'Building geometry and textures...';
if (attempt <= 25) return 'Auto-rigging skeleton and skinning...';
if (attempt <= 35) return 'Finishing avatar...';
if (elapsedMs > 5 * 60 * 1000) return 'Still working -- this one is taking a bit longer...';
return 'Almost there -- finalizing...';
}
if (status === 'done') return 'Done!';
return 'Processing...';
}
/**
* Recognise a job error and phrase it as an instruction the user can act on.
* Returns null when nothing matched, so callers can decide what an unrecognised
* error deserves rather than being handed a guess dressed up as an answer.
* @param {string | null | undefined} raw
* @returns {string | null}
*/
function recogniseJobError(raw) {
if (!raw) return null;
const lower = raw.toLowerCase();
if (lower.includes('face') && lower.includes('detect'))
return 'We could not find a face in your front photo. Try a clearer, well-lit shot.';
if (lower.includes('nsfw'))
return 'Your photo was flagged by content safety. Try a different shot.';
if (lower.includes('blur'))
return 'Your photo looks blurry. Try again with a sharper shot in better light.';
if (lower.includes('unreachable') || lower.includes('service') || lower.includes('502') || lower.includes('503'))
return 'The avatar engine is temporarily unavailable. Please try again in a few minutes.';
if (lower.includes('timeout') || lower.includes('timed out'))
return 'The engine took too long. Try again in a moment.';
if (lower.includes('oom') || lower.includes('memory'))
return 'The engine ran out of resources. Try again with a simpler photo.';
return null;
}
/**
* Turn a failed job into the sentence the build screen shows.
*
* errorKind 'input' means the API vetted this string as caller-facing copy: a
* worker's "no face detected in any of the provided photos", or a plan refusal.
* Our own phrasing still wins wherever we recognise the fault, because it says
* what to do next. What changes is the unrecognised case: a caller-facing
* message is relayed rather than overwritten by the catch-all, which used to
* turn "your library is full" into "try again with a clearer photo" and send
* the user off to retake a photo that was never the problem.
*
* @param {string | null | undefined} raw
* @param {string | null | undefined} [errorKind]
*/
function friendlyJobError(raw, errorKind) {
const known = recogniseJobError(raw);
if (known) return known;
if (errorKind === 'input' && raw) return String(raw);
if (!raw) return 'Avatar reconstruction failed. Try clearer photos in better light.';
return 'Avatar reconstruction failed. Try again with a clearer photo.';
}
/**
* @param {string | null | undefined} raw
*/
function inferFailingSlot(raw) {
if (!_lastSlotsSent.length) return null;
if (!raw) return _lastSlotsSent[0];
const lower = raw.toLowerCase();
if (lower.includes('left')) return _lastSlotsSent.includes('left') ? 'left' : null;
if (lower.includes('right')) return _lastSlotsSent.includes('right') ? 'right' : null;
if (lower.includes('front') || lower.includes('face')) return _lastSlotsSent[0];
return _lastSlotsSent[0];
}
function defaultAvatarName() {
const d = new Date();
const stamp = `${d.getFullYear()}-${String(d.getMonth() + 1).padStart(2, '0')}-${String(d.getDate()).padStart(2, '0')}`;
return `Selfie avatar · ${stamp}`;
}
/**
* @param {File} file
*/
async function fileToDataUrl(file) {
const bitmap = await loadBitmap(file);
const { width, height } = fit(bitmap.width, bitmap.height, MAX_DIM);
const canvas = document.createElement('canvas');
canvas.width = width;
canvas.height = height;
const ctx = canvas.getContext('2d');
if (!ctx) throw new Error('2d canvas unsupported');
ctx.drawImage(bitmap, 0, 0, width, height);
try { bitmap.close?.(); } catch (_) {}
return canvas.toDataURL('image/jpeg', JPEG_QUALITY);
}
/** @param {File} file @returns {Promise<ImageBitmap | HTMLImageElement>} */
async function loadBitmap(file) {
if (typeof createImageBitmap === 'function') {
try {
return await createImageBitmap(file);
} catch (_) {}
}
return new Promise((resolve, reject) => {
const img = new Image();
img.onload = () => resolve(img);
img.onerror = () => reject(new Error('could not decode image'));
img.src = URL.createObjectURL(file);
});
}
/**
* @param {number} w
* @param {number} h
* @param {number} max
*/
function fit(w, h, max) {
if (w <= max && h <= max) return { width: w, height: h };
const s = Math.min(max / w, max / h);
return { width: Math.round(w * s), height: Math.round(h * s) };
}
/**
* @param {number} status
* @param {{ error?: string, error_description?: string }} payload
*/
function mapApiError(status, payload) {
const code = payload.error;
// BYOK-specific errors must be checked before the generic 401/402 handlers.
if (status === 401 && (code === 'invalid_key' || code === 'missing_key')) {
clearStoredByokKey();
return withMessage(
new Error(code),
'Your API key was rejected. Check the key and try again.',
);
}
if (status === 402 && code === 'insufficient_credits')
return withMessage(
new Error(code),
'Your API key is out of credits. Top up your account and try again.',
);
if (status === 401) {
// Return to /create/selfie so the post-login round-trip resumes on the right page.
const here = typeof location !== 'undefined' ? location.pathname : '/create/selfie';
const next = encodeURIComponent(here);
const e = withMessage(
new Error(code || 'unauthorized'),
'Please sign in to create an avatar.',
);
/** @type {any} */ (e).redirect = `/login?next=${next}`;
return e;
}
if (status === 402)
return withMessage(
new Error(code || 'plan_limit'),
payload.error_description || 'Avatar quota reached. Upgrade your plan to create more.',
);
if (status === 413)
return withMessage(new Error(code || 'too_large'), 'Photos are too large. Try again.');
if (status === 429) {
const e = withMessage(
new Error(code || 'rate_limited'),
'Too many attempts -- wait a minute and try again.',
);
/** @type {any} */ (e).rateLimited = true;
return e;
}
if (status === 501)
return withMessage(
new Error(code || 'not_configured'),
'Avatar engine is not available right now. Please try again later.',
);
if (status === 502)
return withMessage(
new Error(code || 'upstream_error'),
'Avatar engine is having trouble. Try again shortly.',
);
return withMessage(
new Error(code || `http_${status}`),
payload.error_description || 'Could not create avatar.',
);
}
/**
* @param {Error} err
* @param {string} userMessage
* @param {string | null} [slot]
*/
function withMessage(err, userMessage, slot) {
/** @type {any} */ (err).userMessage = userMessage;
if (slot) /** @type {any} */ (err).slot = slot;
return err;
}
/** @param {number} ms */
function sleep(ms) {
return new Promise((resolve) => setTimeout(resolve, ms));
}
// ── UI helpers ────────────────────────────────────────────────────────────
const submitBtn = /** @type {HTMLButtonElement | null} */ (document.getElementById('submit-btn'));
let errorBanner = /** @type {HTMLElement | null} */ (null);
let _cooldownTimer = 0;
/** @param {string} text @param {{ error?: boolean }} [opts] */
function setStatus(text, opts = {}) {
if (submitBtn) {
const labelEl = submitBtn.querySelector('.label');
const metaEl = submitBtn.querySelector('.meta');
if (labelEl) labelEl.textContent = text;
else submitBtn.textContent = text;
if (metaEl) metaEl.remove();
submitBtn.disabled = true;
submitBtn.classList.remove('ready');
}
if (opts.error) {
const isNew = !errorBanner;
if (!errorBanner) {
errorBanner = document.createElement('p');
errorBanner.className = 'unsupported show';
errorBanner.setAttribute('role', 'alert');
errorBanner.style.maxWidth = '720px';
errorBanner.style.margin = '0 auto 16px';
const bar = document.getElementById('submit-bar');
bar?.parentNode?.insertBefore(errorBanner, bar);
}
errorBanner.textContent = text;
// The submit bar is sticky, so it is reachable from anywhere on the page,
// but the banner lands at the bar's NATURAL position, near the bottom of a
// phone-height page. Tapping the button from higher up therefore put the
// only explanation off-screen while resetSubmit restored the button to its
// idle label, which reads as "nothing happened". Bring the banner to the
// user. Only on first appearance: the rate-limit cooldown rewrites this
// text every second and must not fight their scrolling.
if (isNew) {
errorBanner.scrollIntoView({
block: 'center',
behavior: window.matchMedia?.('(prefers-reduced-motion: reduce)').matches ? 'auto' : 'smooth',
});
}
} else if (errorBanner) {
errorBanner.remove();
errorBanner = null;
}
}
function resetSubmit() {
if (!submitBtn) return;
submitBtn.disabled = false;
submitBtn.classList.add('ready');
const labelEl = submitBtn.querySelector('.label');
if (labelEl) labelEl.textContent = 'Build my avatar';
else submitBtn.textContent = 'Build my avatar';
}
/** @param {string} slot */
function highlightSlotError(slot) {
const frame = document.querySelector(
slot === 'frontal'
? '.hero-slot[data-slot="frontal"]'
: `.slot-frame[data-slot="${slot}"]`,
);
frame?.classList.add('error');
if (slot !== 'frontal') {
const fid = /** @type {HTMLDetailsElement|null} */ (document.getElementById('fidelity-boost'));
if (fid) fid.open = true;
}
}
function startRateLimitCooldown() {
_rateLimitedUntil = Date.now() + RATE_LIMIT_COOLDOWN_MS;
if (_cooldownTimer) clearInterval(_cooldownTimer);
_cooldownTimer = window.setInterval(() => {
const remaining = Math.ceil((_rateLimitedUntil - Date.now()) / 1000);
if (remaining <= 0) {
clearInterval(_cooldownTimer);
_cooldownTimer = 0;
_rateLimitedUntil = 0;
resetSubmit();
return;
}
setStatus(`Rate limited -- try again in ${remaining}s`, { error: true });
}, 1000);
}