Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// Avatar regeneration provider loader — shared by the regenerate / reconstruct
// endpoints, the reconstruct-finalize stage, and the Replicate webhook.
//
// Dynamically imports a provider module by name so we don't pay the cost of
// loading e.g. the Replicate SDK on every request when regeneration is unused.
// Cached per-process after first load.
//
// Provider name precedence (platform-level, env-configured):
// 1. Explicit env: AVATAR_REGEN_PROVIDER=replicate|huggingface|gcp
// 2. Inferred from credentials: REPLICATE_API_TOKEN → replicate,
// GCP_RECONSTRUCTION_URL → gcp, HF_TOKEN → huggingface.
//
// BYOK providers (Meshy, Tripo) are NOT in this precedence chain — they are
// request-scoped (the caller supplies the key) and are loaded via
// getRegenProviderByName(). The reconstruct endpoint checks platform first and
// falls back to a user's stored BYOK key when the platform is unconfigured.
// Tier used when adapting BYOK geometry providers for the regen interface.
// Mirrors the forge "standard" tier: solid polycount, PBR textures, no HD
// surcharge so the selfie pipeline isn't pay-gated without warning.
const REGEN_TIER = Object.freeze({ polycount: 30_000, pbr: true, hd: false });
// BYOK provider names that support the reconstruct (image → 3D) mode.
export const BYOK_REGEN_PROVIDERS = Object.freeze(['meshy', 'tripo']);
// Wraps createMeshyProvider to speak the regen submit/status interface.
// ext_job_id is the raw Meshy task id — job.provider === 'meshy' tells the
// status handler which adapter to use, so no kind prefix is needed.
function createMeshyRegenAdapter(key) {
let _meshy;
async function meshy() {
if (!_meshy) {
const mod = await import('../_providers/meshy.js');
_meshy = mod.createMeshyProvider(key);
}
return _meshy;
}
return {
supportsMode(mode) {
return mode === 'reconstruct';
},
supportsMultiview() {
return false;
},
async submit({ mode, params, sourceUrl }) {
if (mode !== 'reconstruct') {
throw Object.assign(
new Error(`meshy BYOK regen does not support mode "${mode}"`),
{ code: 'mode_unconfigured', status: 501 },
);
}
const images = Array.isArray(params?.images) ? params.images : [];
const imageUrl = images[0] || sourceUrl;
if (!imageUrl) {
throw Object.assign(new Error('meshy regen: no image URL provided'), {
code: 'invalid_request',
status: 400,
});
}
const provider = await meshy();
const result = await provider.imageTo3d({
imageUrl,
prompt: params?.name || '',
tier: REGEN_TIER,
});
// result.taskId is the Meshy task id; stored directly as ext_job_id.
return {
extJobId: result.taskId,
eta: 90,
backend: 'meshy',
};
},
async status(extJobId) {
if (!extJobId) return { status: 'failed', error: 'missing ext_job_id' };
const provider = await meshy();
// Meshy status always uses kind 'image-to-3d' for selfie submissions.
return provider.status({ kind: 'image-to-3d', taskId: extJobId });
},
};
}
// Wraps createTripoProvider to speak the regen submit/status interface. Used
// both as a BYOK provider (user's own key, mode 'reconstruct') and, when
// TRIPO_API_KEY is set, as a platform provider for 'rerig' (see
// PLATFORM_PROVIDER_ORDER / loadPlatformProvider below): the house-key auto-
// rig lane that drives maybeAutoRigAvatar().
function createTripoRegenAdapter(key) {
let _tripo;
async function tripo() {
if (!_tripo) {
const mod = await import('../_providers/tripo.js');
_tripo = mod.createTripoProvider(key);
}
return _tripo;
}
return {
supportsMode(mode) {
return mode === 'reconstruct' || mode === 'rerig';
},
supportsMultiview() {
return false;
},
async submit({ mode, params, sourceUrl }) {
if (mode === 'rerig') {
if (!sourceUrl) {
throw Object.assign(new Error('tripo rerig: no source model URL provided'), {
code: 'invalid_request',
status: 400,
});
}
const provider = await tripo();
// spec:'mixamo' asks Tripo for Mixamo-named bones, which this repo's
// glb-canonicalize.js already knows how to map onto the canonical
// skeleton, so no new bone-naming code is needed to play idle/walk on it.
const result = await provider.rig({ input: sourceUrl, spec: 'mixamo' });
return {
extJobId: result.taskId,
eta: 90,
backend: 'tripo',
};
}
if (mode !== 'reconstruct') {
throw Object.assign(
new Error(`tripo BYOK regen does not support mode "${mode}"`),
{ code: 'mode_unconfigured', status: 501 },
);
}
const images = Array.isArray(params?.images) ? params.images : [];
const imageUrl = images[0] || sourceUrl;
if (!imageUrl) {
throw Object.assign(new Error('tripo regen: no image URL provided'), {
code: 'invalid_request',
status: 400,
});
}
const provider = await tripo();
const result = await provider.imageTo3d({ imageUrl, tier: REGEN_TIER });
return {
extJobId: result.taskId,
eta: 60,
backend: 'tripo',
};
},
async status(extJobId) {
if (!extJobId) return { status: 'failed', error: 'missing ext_job_id' };
const provider = await tripo();
return provider.status({ taskId: extJobId });
},
};
}
// Instantiate a named BYOK regen adapter with the given API key. Only the
// providers listed in BYOK_REGEN_PROVIDERS are supported here.
export function getRegenProviderByName(name, key) {
if (!key) {
throw Object.assign(new Error(`${name} regen: API key is required`), {
code: 'missing_key',
status: 400,
});
}
switch (name) {
case 'meshy':
return { name, instance: createMeshyRegenAdapter(key) };
case 'tripo':
return { name, instance: createTripoRegenAdapter(key) };
default:
throw Object.assign(new Error(`unknown BYOK regen provider: ${name}`), {
code: 'regen_provider_unknown',
status: 501,
});
}
}
// Per-process provider instances, keyed by canonical name. A Map (not a single
// slot) so the failover path can hold several platform providers live at once
// without thrashing the cache between them.
const _platformProviderCache = new Map();
// Canonical platform provider names in paid → free precedence. Used both to
// resolve the single primary and to enumerate failover candidates. 'tripo'
// sits last: it is an additional house-key lane (billed through our own
// credits, see forge-tiers.js), not a replacement for the self-hosted/free
// options above it, and it only ever contributes 'rerig' candidates.
const PLATFORM_PROVIDER_ORDER = Object.freeze(['replicate', 'gcp', 'huggingface', 'tripo']);
function canonicalProviderName(name) {
const n = (name || '').trim().toLowerCase();
return n === 'hf' ? 'huggingface' : n;
}
// True when the named platform provider has its required credentials present.
function platformProviderConfigured(name) {
switch (canonicalProviderName(name)) {
case 'replicate':
return Boolean(process.env.REPLICATE_API_TOKEN);
case 'gcp':
// The gcp adapter needs both the service URL (endpoint) and the auth
// key — requiring both here keeps a URL-only deploy out of the failover
// rotation instead of enumerating a provider whose constructor throws.
return Boolean(process.env.GCP_RECONSTRUCTION_URL && process.env.GCP_RECONSTRUCTION_KEY);
case 'huggingface':
return Boolean(process.env.HF_TOKEN);
case 'tripo':
return Boolean(process.env.TRIPO_API_KEY);
default:
return false;
}
}
export function resolveProviderName() {
const explicit = canonicalProviderName(process.env.AVATAR_REGEN_PROVIDER);
if (explicit) return explicit;
// Auto-detect order is paid → free: prefer Replicate's pinned-version
// reliability, then our own GCP Cloud Run service, then the HF Spaces queue
// (free GPU but variable wait + cold starts).
if (process.env.REPLICATE_API_TOKEN) return 'replicate';
if (process.env.GCP_RECONSTRUCTION_URL) return 'gcp';
if (process.env.HF_TOKEN) return 'huggingface';
return 'none';
}
// Load (and cache) a single named platform provider instance. Dynamic-imports
// the adapter so an unused provider's SDK never loads. Throws with a classified
// code for an unknown name; provider constructors themselves throw when their
// credentials are missing.
async function loadPlatformProvider(rawName) {
const name = canonicalProviderName(rawName);
if (name === 'none' || !name) return { name: 'none', instance: null };
if (_platformProviderCache.has(name)) {
return { name, instance: _platformProviderCache.get(name) };
}
let instance;
if (name === 'replicate') {
instance = (await import('../_providers/replicate.js')).createRegenProvider();
} else if (name === 'huggingface') {
instance = (await import('../_providers/huggingface.js')).createRegenProvider();
} else if (name === 'gcp') {
instance = (await import('../_providers/gcp.js')).createRegenProvider();
} else if (name === 'tripo') {
instance = createTripoRegenAdapter(process.env.TRIPO_API_KEY);
} else {
throw Object.assign(new Error(`unknown AVATAR_REGEN_PROVIDER: ${name}`), {
code: 'regen_provider_unknown',
status: 501,
});
}
_platformProviderCache.set(name, instance);
return { name, instance };
}
export async function getRegenProvider() {
const name = resolveProviderName();
if (name === 'none' || !name) return { name, instance: null };
return loadPlatformProvider(name);
}
// Resolve the first configured provider (in paid → free precedence) that
// actually supports `mode`. The plain getRegenProvider() returns only the single
// highest-precedence provider, which breaks rigging on the common production
// setup where Replicate is primary for `reconstruct` (its built-in TRELLIS
// default) but has no rerig model — meanwhile the self-hosted GCP UniRig pipeline
// CAN rig. Without this failover, every forged avatar comes out un-rigged because
// the rig gate asks Replicate (which can't) and gives up. Mirrors the MCP studio
// path's regenProvider(mode). Returns { name, instance } or { name:'none' }.
export async function getRegenProviderForMode(mode) {
const candidates = await getRegenProviderCandidates();
for (const candidate of candidates) {
try {
if (candidate.instance && candidate.instance.supportsMode(mode)) return candidate;
} catch {
// A provider whose supportsMode throws is treated as not-capable.
}
}
return { name: 'none', instance: null };
}
// Enumerate every configured platform provider in priority order so the submit
// path can fail over from one to the next. The explicit AVATAR_REGEN_PROVIDER
// (if set and credentialed) leads; the credential-inferred order fills in the
// rest, de-duplicated. A provider whose constructor throws (bad config) is
// logged and skipped rather than blocking the others. Returns [] when nothing
// is configured.
export async function getRegenProviderCandidates() {
const order = [];
const seen = new Set();
const push = (rawName) => {
const name = canonicalProviderName(rawName);
if (name && name !== 'none' && !seen.has(name) && platformProviderConfigured(name)) {
seen.add(name);
order.push(name);
}
};
push(process.env.AVATAR_REGEN_PROVIDER);
for (const name of PLATFORM_PROVIDER_ORDER) push(name);
const out = [];
for (const name of order) {
try {
const resolved = await loadPlatformProvider(name);
if (resolved.instance) out.push(resolved);
} catch (err) {
console.warn('[regen] platform provider load failed:', name, '—', err?.message);
}
}
return out;
}
// Resolve the provider for a job that's already in flight, routing by the
// provider name stored on the job row. BYOK providers (meshy, tripo) need the
// user's stored key to poll — fetched from the DB via the request session.
// Platform providers (replicate, gcp, huggingface) use env credentials as usual.
export async function getRegenProviderForJob(jobProvider, req) {
if (!jobProvider) return { name: 'none', instance: null };
// Platform providers: load the adapter for the provider THIS job was
// submitted to, not whichever one currently leads the precedence order.
// getRegenProvider() returns the primary, so the moment a second platform
// credential is added (REPLICATE_API_TOKEN sits above gcp in
// PLATFORM_PROVIDER_ORDER) every in-flight gcp job would be polled with the
// Replicate adapter, handed a base64 gcp envelope as a prediction id. The
// poll then reports a failure that has nothing to do with the real job, and
// the capture the user is waiting on dies on a provider that never ran it.
// The reconstruct/auto-rig sweeps already refuse to poll across a provider
// mismatch; this makes the browser poll agree with them.
if (['replicate', 'gcp', 'huggingface', 'hf'].includes(jobProvider)) {
const name = canonicalProviderName(jobProvider);
// Credentials rotated away since submission: report the job's own
// provider with no instance so the caller leaves the row untouched and
// the sweep ages it out with an honest reason, rather than polling a
// different backend for an id it has never seen.
if (!platformProviderConfigured(name)) return { name, instance: null };
try {
return await loadPlatformProvider(name);
} catch (err) {
console.warn('[regen] job provider load failed:', name, '-', err?.message);
return { name, instance: null };
}
}
// BYOK providers: re-resolve the user's stored key at poll time. The key
// is never stored on the job row itself (it's a secret); the user's session
// is the authority. Tripo doubles as a platform-level 'rerig' lane (house
// key, see PLATFORM_PROVIDER_ORDER above): jobs submitted that way have no
// corresponding user session key, so fall back to TRIPO_API_KEY rather than
// reporting a false "no instance" for an in-flight house-key job.
if (BYOK_REGEN_PROVIDERS.includes(jobProvider)) {
try {
const { resolveProviderKey } = await import('./forge-provider-key.js');
const key = await resolveProviderKey(req, null, jobProvider);
if (key) return getRegenProviderByName(jobProvider, key);
if (jobProvider === 'tripo' && platformProviderConfigured('tripo')) {
return loadPlatformProvider('tripo');
}
return { name: jobProvider, instance: null };
} catch {
return { name: jobProvider, instance: null };
}
}
return { name: 'none', instance: null };
}