Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// GCP Cloud Run provider — implements the same contract as the Replicate provider.
//
// submit(request) → { extJobId, eta }
// status(extJobId) → { status, resultGlbUrl?, resultImageUrl?, error? }
//
// Supported modes and the Cloud Run service they route to:
//
// reconstruct → GCP_RECONSTRUCTION_URL (avatar-pipeline-controller)
// POST /reconstruct → { job_id } GET /jobs/:id → { status, glb_url }
//
// trellis → MODEL_TRELLIS_URL (workers/model-trellis)
// Native single-image→3D via Microsoft TRELLIS. Standard task
// shape: POST /infer → { task_id } GET /tasks/:id → result_gcs_url.
//
// trellis2 → MODEL_TRELLIS2_URL (workers/model-trellis2)
// pixal3d → MODEL_PIXAL3D_URL (workers/model-pixal3d)
// "Match my photo": 1-4 reference views fused pixel-aligned
// (unlike every other image lane above, it reads every view in
// the array, not just the first). `view_labels` tags each view
// front/back/left/right; `fill_other_sides` synthesizes the
// other three sides from one photo via MV-Adapter.
// modly → MODEL_MODLY_URL (workers/modly, TripoSG)
// modly_hunyuan → MODEL_MODLY_URL (workers/modly, Hunyuan3D 2 Mini Turbo)
// One Modly worker serves both; the request's `model` field
// picks the extension. Untextured geometry, repaired and
// decimated to the poly budget by Modly's mesh optimizer.
// hunyuan → GCP_HUNYUAN3D_URL (workers/model-hunyuan3d)
// Image→3D via Tencent Hunyuan3D-2 (shape DiT + multiview
// paint). Same standard task shape as the TRELLIS worker:
// POST /infer → { task_id } GET /tasks/:id → result_gcs_url.
// NOT the avatar controller's /reconstruct + /jobs/:id shape.
//
// triposplat → MODEL_TRIPOSPLAT_URL (workers/model-triposplat, MIT)
// One image → 3D Gaussian splat (.splat + .ply). Standard
// /infer + /tasks/:id shape; the result is a splat, not a GLB.
// sam3d → MODEL_SAM3D_URL (workers/model-sam3d-objects, SAM License)
// One photo → one GLB with a named node per object, laid out
// like the photo. Needs a 32 GB+ GPU. Standard /infer +
// /tasks/:id shape.
// remesh → GCP_REMESH_URL (workers/remesh)
// stylize → GCP_STYLIZE_URL (workers/stylize)
// retex → GCP_TEXTURE_URL (workers/texture)
// rembg → GCP_REMBG_URL (workers/rembg)
// segment → GCP_SEGMENT_URL (workers/segment)
// sketch → GCP_TRIPOSG_URL (workers/model-triposg, scribble mode)
// rerig → GCP_UNIRIG_URL (workers/rig, /rig + /tasks/:id)
// video2scene → GCP_VIDEO2SCENE_URL (workers/model-video2scene, LingBot-Map)
// Streaming video → 3D point cloud. Standard /infer + /tasks/:id
// shape; the result is a .ply point cloud, not a GLB mesh.
// video2motion → GCP_VIDEO2MOTION_URL (workers/model-video2motion, MediaPipe)
// Video of a person → AnimationClip JSON + person-mask video +
// normalized video + per-frame screen anchors (body swap).
//
// All workers share the same bearer secret (GCP_RECONSTRUCTION_KEY).
// Each worker exposes the same task shape:
// POST /… → { task_id, status: "queued" }
// GET /tasks/:id → { task_id, status, result_url|result_gcs_url?, error? }
//
// The extJobId is a JSON envelope base64url-encoded so it can carry both
// the service discriminator and the upstream task_id without another DB call.
import { fetchUpstream } from '../_lib/upstream-fetch.js';
function readEnv(name) {
if (typeof process !== 'undefined' && process.env?.[name]) return process.env[name];
return null;
}
function packJobId(payload) {
return Buffer.from(JSON.stringify(payload), 'utf8').toString('base64url');
}
function unpackJobId(extJobId) {
try {
return JSON.parse(Buffer.from(extJobId, 'base64url').toString('utf8'));
} catch {
return null;
}
}
function translateStatus(s) {
switch (s) {
case 'queued': return 'queued';
case 'running': return 'running';
case 'done': return 'done';
case 'failed': return 'failed';
default: return 'queued';
}
}
// Resolve which Cloud Run base URL handles a given mode.
function serviceUrlForMode(mode) {
switch (mode) {
case 'reconstruct':
return readEnv('GCP_RECONSTRUCTION_URL');
case 'remesh':
return readEnv('GCP_REMESH_URL');
case 'stylize':
return readEnv('GCP_STYLIZE_URL');
case 'retex':
case 'retex_region':
// Region (magic-brush) edits run on the same texture worker.
return readEnv('GCP_TEXTURE_URL');
case 'rembg':
return readEnv('GCP_REMBG_URL');
case 'segment':
return readEnv('GCP_SEGMENT_URL');
case 'sketch':
case 'triposg':
// One TripoSG worker (workers/model-triposg) serves both: `sketch` is its
// scribble pipeline, `triposg` its photo pipeline.
return readEnv('GCP_TRIPOSG_URL');
case 'trellis':
// Self-hosted Microsoft TRELLIS image→3D worker (workers/model-trellis):
// native single-image reconstruction, no FLUX intermediate. Distinct from
// the avatar pipeline's `reconstruct` (face-only, /reconstruct + /jobs/:id);
// this worker speaks the standard /infer + /tasks/:id task shape.
return readEnv('MODEL_TRELLIS_URL');
case 'trellis2':
// Self-hosted Microsoft TRELLIS.2 image→3D worker (workers/model-trellis2),
// the default image lane. Same /infer + /tasks/:id task shape.
return readEnv('MODEL_TRELLIS2_URL');
case 'pixal3d':
// Self-hosted TencentARC Pixal3D worker (workers/model-pixal3d): the
// "Match my photo" lane. Unlike every other image lane above, it fuses
// every one of its 1-4 reference views instead of reading the first.
// Same /infer + /tasks/:id task shape.
return readEnv('MODEL_PIXAL3D_URL');
case 'anigen':
// Self-hosted VAST-AI-Research AniGen worker (workers/model-anigen):
// the only image lane that hands back an already-rigged GLB (mesh +
// skeleton + skin weights in one pass), so a caller can skip the
// separate rerig step. Same /infer + /tasks/:id task shape; conditions
// on the first reference image only.
return readEnv('MODEL_ANIGEN_URL');
case 'partcrafter':
// Self-hosted PartCrafter worker (workers/model-partcrafter): the
// editable-parts image lane. One reference view decoded into N named
// part nodes in a single pass. Same /infer + /tasks/:id task shape.
return readEnv('MODEL_PARTCRAFTER_URL');
case 'holopart':
// Self-hosted HoloPart worker (workers/model-holopart): completes the
// occluded geometry of an already part-segmented mesh (e.g. the
// output of `segment`), preserving part node names. Same
// /infer + /tasks/:id task shape.
return readEnv('MODEL_HOLOPART_URL');
case 'sam3d':
// Self-hosted SAM 3D Objects worker (workers/model-sam3d-objects).
return readEnv('MODEL_SAM3D_URL');
case 'triposplat':
// Self-hosted TripoSplat worker (workers/model-triposplat): image to
// Gaussian splat. Same /infer + /tasks/:id task shape.
return readEnv('MODEL_TRIPOSPLAT_URL');
case 'modly':
case 'modly_hunyuan':
// Self-hosted Modly worker (workers/modly): one service, two models.
// Same /infer + /tasks/:id task shape as the TRELLIS workers.
return readEnv('MODEL_MODLY_URL');
case 'hunyuan':
// Self-hosted Hunyuan3D-2 image→3D worker (workers/model-hunyuan3d).
// Same standard /infer + /tasks/:id task shape as the TRELLIS worker.
return readEnv('GCP_HUNYUAN3D_URL');
case 'text2motion':
return readEnv('GCP_TEXT2MOTION_URL');
case 'video2scene':
// Streaming video → 3D point-cloud worker (workers/model-video2scene,
// LingBot-Map). Standard /infer + /tasks/:id task shape; result is a PLY.
return readEnv('GCP_VIDEO2SCENE_URL');
case 'video2motion':
// Video of a person → animation clip + body-swap plates
// (workers/model-video2motion, MediaPipe pose + segmentation). Standard
// /infer + /tasks/:id task shape; result is an AnimationClip JSON plus
// mask/video/meta sidecar URLs.
return readEnv('GCP_VIDEO2MOTION_URL');
case 'rerig':
// Rigging: the standalone rig worker only (workers/rig, Cloud Run
// `model-rig`: /rig + /tasks/:id, `mesh_gcs_url` request schema).
// There is deliberately NO fallback to GCP_RECONSTRUCTION_URL: neither
// deployed service behind it (avatar-reconstruction or the pipeline
// controller) exposes a /rig endpoint, so the old fallback made
// supportsMode('rerig') report true and then 404 on every submit,
// which silently downgraded reconstructions to unrigged meshes.
// With no GCP_UNIRIG_URL the mode is honestly unsupported and the
// finalize stage skips the rig chain cleanly.
return readEnv('GCP_UNIRIG_URL');
default:
return null;
}
}
// Build the request body and path suffix for each worker's API shape.
function buildWorkerRequest(request) {
const { mode, sourceUrl, params } = request;
if (mode === 'reconstruct') {
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos, body_type: params?.bodyType || 'neutral' };
// Quality-tier provenance + poly budget for the self-host high-detail path.
// Only forwarded when set, so the default request shape is unchanged.
if (Number.isFinite(Number(params?.target_polycount))) {
body.target_polycount = Math.round(Number(params.target_polycount));
}
if (params?.tier) body.tier = params.tier;
if (params?.path) body.path = params.path;
return {
path: '/reconstruct',
resultKey: 'glb_url',
body,
};
}
if (mode === 'trellis') {
// Self-hosted TRELLIS image→3D (workers/model-trellis). Reconstructs a
// textured mesh from the reference views — user photos, or the synthesized
// view for a text prompt. `body_type` rides along for parity with the
// shared worker request shape.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos, body_type: params?.bodyType || 'neutral' };
// Per-tier sampler/export budgets (see SELFHOST_TRELLIS_QUALITY in
// forge-tiers.js). The worker clamps every knob server-side and treats an
// absent field as its own defaults, so this is pure opt-in quality.
if (params?.quality && typeof params.quality === 'object') {
body.quality = params.quality;
}
// Opt-in background pre-matting: the worker routes each view through its
// sibling rembg service before reconstruction (workers/model-trellis
// main.py `do_matte`). Only forwarded when the caller asks — the worker
// otherwise defaults matte off for every tier below `max`, so the free/
// default lane is unchanged unless we explicitly request it here.
if (params?.matte === true) body.matte = true;
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'trellis2') {
// Self-hosted TRELLIS.2 image→3D (workers/model-trellis2). One reference
// view in, a PBR GLB with alpha out. `resolution` (512, 1024 or 1536) is
// the voxel resolution; absent, the worker maps `tier` to one.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos, body_type: params?.bodyType || 'neutral' };
if (Number.isFinite(Number(params?.resolution))) body.resolution = Math.round(Number(params.resolution));
if (params?.tier) body.tier = params.tier;
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
// The worker clamps both (512 to 4096 px, 50k to 2M faces).
if (Number.isFinite(Number(params?.texture_size))) body.texture_size = Math.round(Number(params.texture_size));
if (Number.isFinite(Number(params?.decimation_target))) body.decimation_target = Math.round(Number(params.decimation_target));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'pixal3d') {
// Self-hosted Pixal3D (workers/model-pixal3d): 1-4 reference views in, a
// pixel-aligned PBR GLB out. `view_labels` tags each view front/back/left/
// right (omitted entries, or the whole field omitted, let the worker infer
// the front view from its camera estimator); `fill_other_sides` asks the
// worker to synthesize the other three sides from a single supplied photo.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos, body_type: params?.bodyType || 'neutral' };
if (Array.isArray(params?.view_labels)) body.view_labels = params.view_labels;
if (params?.fill_other_sides === true) body.fill_other_sides = true;
if (Number.isFinite(Number(params?.resolution))) body.resolution = Math.round(Number(params.resolution));
if (params?.tier) body.tier = params.tier;
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
if (Number.isFinite(Number(params?.texture_size))) body.texture_size = Math.round(Number(params.texture_size));
if (Number.isFinite(Number(params?.decimation_target))) body.decimation_target = Math.round(Number(params.decimation_target));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'anigen') {
// Self-hosted AniGen (workers/model-anigen): one reference view in, a
// rigged GLB (mesh + skeleton + skin weights) out. The worker reads only
// the first image; extra views are accepted but ignored.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos };
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'partcrafter') {
// Self-hosted PartCrafter (workers/model-partcrafter): one reference
// view in, a part-separated GLB (N named part nodes) out. The worker
// reads only the first image; extra views are accepted but ignored.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos, num_parts: Math.round(Number(params?.num_parts) || 4) };
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'sam3d') {
// Self-hosted SAM 3D Objects: one photo plus either per-object masks or
// boxes (never both). The worker rebuilds each object and lays them out
// like the photo in a single scene GLB.
const body = { image_url: sourceUrl };
if (Array.isArray(params?.maskUrls) && params.maskUrls.length) body.mask_urls = params.maskUrls;
else if (Array.isArray(params?.boxes) && params.boxes.length) body.boxes = params.boxes;
if (Number.isFinite(Number(params?.maxObjects))) body.max_objects = Math.round(Number(params.maxObjects));
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'triposplat') {
// Self-hosted TripoSplat (workers/model-triposplat): the worker
// conditions on the first reference image only.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos };
if (Number.isFinite(Number(params?.num_gaussians))) body.num_gaussians = Math.round(Number(params.num_gaussians));
return {
path: '/infer',
resultKey: 'splat_url',
body,
};
}
if (mode === 'holopart') {
// Self-hosted HoloPart (workers/model-holopart): an already part-
// segmented mesh in (one named geometry per part, e.g. workers/segment's
// own output), a mesh with every part's occluded geometry completed out.
// Part node names are preserved verbatim by the worker.
const body = { mesh_url: sourceUrl };
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'modly' || mode === 'modly_hunyuan') {
return buildModlyRequest(mode, sourceUrl, params);
}
if (mode === 'hunyuan') {
// Self-hosted Hunyuan3D-2 image→3D (workers/model-hunyuan3d). Shape DiT
// conditions on the primary reference view; the worker isolates the
// subject (rembg), cleans the mesh, decimates to the poly budget, and
// paints it with the multiview texture pipeline.
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = { images: photos, body_type: params?.bodyType || 'neutral' };
if (Number.isFinite(Number(params?.target_polycount))) {
body.target_polycount = Math.round(Number(params.target_polycount));
}
// Tier + path select the worker's per-tier generation budget and record
// provenance; absent fields mean its high-quality defaults.
if (params?.tier) body.tier = params.tier;
if (params?.path) body.path = params.path;
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'remesh') {
return {
path: '/process',
resultKey: 'result_url',
body: {
mesh: sourceUrl,
remesh_mode: params?.remesh_mode || 'triangle',
operation: params?.operation || 'full',
target_faces: params?.target_faces || 50_000,
texture_size: params?.texture_size || 1024,
output_format: params?.output_format || 'glb',
},
};
}
if (mode === 'stylize') {
return {
path: '/process',
resultKey: 'result_url',
body: {
mesh: sourceUrl,
style: params?.style || 'voxel',
resolution: params?.resolution ?? null,
output_format: params?.output_format || 'glb',
},
};
}
if (mode === 'retex') {
return {
path: '/texture',
resultKey: 'result_url',
body: {
mesh: sourceUrl,
prompt: params?.prompt || '',
negative_prompt: params?.negative_prompt || 'blurry, low quality, distorted',
num_views: params?.num_views || 8,
texture_size: params?.texture_size || 1024,
// person|metal|wood|fabric|plastic|glass — bakes measured-value
// roughness/metallic factors instead of the flat default and
// nudges the SDXL prompt with material-appropriate descriptors
// (workers/texture/main.py MATERIAL_CLASS_PBR). Optional: omit to
// keep the old flat-guess behavior.
material_class: params?.material_class || null,
},
};
}
if (mode === 'retex_region') {
// Magic brush: repaint only the masked UV region of the existing texture.
// The mask arrives as inline base64 (browser canvas) or a public URL.
return {
path: '/retexture_region',
resultKey: 'result_url',
body: {
mesh: sourceUrl,
prompt: params?.prompt || '',
negative_prompt:
params?.negative_prompt || 'blurry, low quality, distorted, watermark, seam',
mask_b64: params?.mask_b64 || null,
mask: params?.mask || null,
color: params?.color || null,
texture_size: params?.texture_size || 1024,
strength: params?.strength ?? 0.85,
feather: params?.feather ?? 24,
seed: params?.seed ?? 0,
},
};
}
if (mode === 'rembg') {
return {
path: '/remove',
resultKey: 'result_url',
body: {
image: sourceUrl,
model: params?.model || 'rmbg2',
},
};
}
if (mode === 'segment') {
return {
path: '/segment',
resultKey: 'result_url',
body: {
mesh: sourceUrl,
method: params?.method || 'auto',
max_parts: params?.max_parts || 24,
min_part_faces: params?.min_part_faces || 64,
crease_angle: params?.crease_angle ?? 40,
...(params?.only_part ? { only_part: params.only_part } : {}),
},
};
}
if (mode === 'sketch') {
// Sketch→3D on the TripoSG worker's scribble pipeline: the source is the
// drawing, the prompt names what it depicts (the model is prompt-
// conditioned). Geometry only — no textures.
const body = {
images: [sourceUrl],
prompt: params?.prompt || '',
mode: 'scribble',
};
if (Number.isFinite(Number(params?.target_polycount))) {
body.target_polycount = Math.round(Number(params.target_polycount));
}
if (Number.isFinite(Number(params?.scribble_confidence))) {
body.scribble_confidence = Number(params.scribble_confidence);
}
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'triposg') {
// Photo→3D on the TripoSG worker's image pipeline (in-process background
// removal, 50-step sampler). Geometry only, no textures, so forge uses it
// as the last image lane, behind every lane that returns a textured GLB.
const body = { images: [sourceUrl], mode: 'image' };
if (Number.isFinite(Number(params?.target_polycount))) {
body.target_polycount = Math.round(Number(params.target_polycount));
}
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'video2scene') {
// Streaming video → 3D point cloud (workers/model-video2scene, LingBot-Map).
// The source is a public video URL; an `images` array of frames is the
// alternate input. Reconstruction params (sampling/quality) ride in `params`
// and are forwarded only when set so the worker's defaults stand otherwise.
const body = {};
if (Array.isArray(params?.images) && params.images.length) {
body.images = params.images;
} else {
body.video_url = sourceUrl;
}
for (const key of [
'mode', 'fps', 'keyframe_interval', 'num_scale_frames',
'window_size', 'overlap_size', 'conf_percentile', 'max_points', 'voxel_size',
]) {
if (params?.[key] !== undefined && params[key] !== null) body[key] = params[key];
}
if (params?.mask_sky !== undefined) body.mask_sky = Boolean(params.mask_sky);
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
if (mode === 'rerig') {
// The standalone rig worker behind GCP_UNIRIG_URL (workers/rig, the
// Make-It-Animatable service; the env name predates it) takes
// `mesh_gcs_url` (any https URL, the name is historical) and reports
// `rigged_gcs_url`. It is the only rerig backend: serviceUrlForMode
// resolves rerig from GCP_UNIRIG_URL alone.
return {
path: '/rig',
resultKey: 'rigged_gcs_url',
body: {
mesh_gcs_url: sourceUrl,
template: params?.template || 'wolf3d_neutral',
blendshapes: params?.blendshapes !== false,
},
};
}
if (mode === 'video2motion') {
// Video of a person → motion clip + compositing plates. The source is a
// public video URL; capture params ride in `params`.
const body = { video_url: sourceUrl };
if (params?.fps !== undefined && params.fps !== null) body.fps = params.fps;
if (params?.max_seconds !== undefined && params.max_seconds !== null) {
body.max_seconds = params.max_seconds;
}
return {
path: '/infer',
resultKey: 'result_url',
body,
};
}
if (mode === 'text2motion') {
// Text→motion has no source mesh: the prompt rides in params. The worker
// returns a three.js AnimationClip JSON URL (result_url), not a GLB.
return {
path: '/infer',
resultKey: 'result_url',
body: {
prompt: params?.prompt || '',
duration_seconds: params?.duration_seconds ?? 4,
fps: params?.fps ?? 30,
},
};
}
return null;
}
// The Modly worker's model id per forge mode, and the face budget its
// decimate step accepts (workers/modly request_policy.py DECIMATE_MIN/MAX).
export const MODLY_MODEL_FOR_MODE = Object.freeze({
modly: 'triposg',
modly_hunyuan: 'hunyuan3d-mini-turbo',
});
const MODLY_DECIMATE_MIN = 100;
const MODLY_DECIMATE_MAX = 1_000_000;
// Self-hosted Modly image to 3D (workers/modly). One reference view in, an
// untextured GLB out. The worker maps `tier` to its sampler budget; the
// post-process plan repairs the raw marching-cubes mesh and then decimates it to
// the caller's poly budget, so the tier's polycount is honoured. A budget outside
// the worker's bounds is clamped here rather than 422ing the whole generation.
function buildModlyRequest(mode, sourceUrl, params) {
const photos = Array.isArray(params?.images) && params.images.length
? params.images
: [sourceUrl].filter(Boolean);
const body = {
images: photos,
body_type: params?.bodyType || 'neutral',
model: MODLY_MODEL_FOR_MODE[mode],
};
if (params?.tier) body.tier = params.tier;
if (Number.isFinite(Number(params?.seed))) body.seed = Math.floor(Number(params.seed));
const postprocess = { repair: true };
const target = Number(params?.target_polycount);
if (Number.isFinite(target) && target > 0) {
postprocess.decimate = {
target_faces: Math.min(MODLY_DECIMATE_MAX, Math.max(MODLY_DECIMATE_MIN, Math.round(target))),
};
}
body.postprocess = postprocess;
return {
path: '/infer',
resultKey: 'result_gcs_url',
body,
};
}
// Expected wall-clock ETAs (seconds) per mode — used to populate the
// progress indicator on the client side.
const MODE_ETA = {
reconstruct: 120,
trellis: 60,
trellis2: 75,
// Pixel-aligned single-view path is close to TRELLIS.2's own cost; the
// multi-view fusion path (2-4 images) and the optional MV-Adapter
// "fill the other sides" rung both run meaningfully heavier.
pixal3d: 100,
// Mesh + skeleton prediction + skin-weight transfer in one pass.
anigen: 90,
// N parts decoded in one diffusion-transformer pass; scales mildly with
// the requested part count, which estimateEtaSeconds does not model here.
partcrafter: 110,
// Per-part point-cloud conditioning + diffusion decode for every part in
// an already-segmented mesh.
holopart: 100,
// Single-pass flow-matching denoise plus a decoder replay on the L4.
triposplat: 45,
// Two diffusion stages per object, several objects, on a 96 GB-class GPU.
sam3d: 150,
// TripoSG's 20 to 50 flow steps plus repair and decimate on the L4.
modly: 60,
// Hunyuan3D 2 Mini Turbo is a few-step distilled sampler.
modly_hunyuan: 45,
// 50-step shape diffusion + multiview paint on the L4: much heavier than
// TRELLIS's single pass.
hunyuan: 300,
remesh: 30,
stylize: 25,
retex: 180,
retex_region: 60,
rembg: 5,
segment: 45,
sketch: 45,
// The photo pipeline runs 50 sampler steps against scribble's 16.
triposg: 60,
text2motion: 30,
rerig: 45,
video2scene: 240,
// CPU pose+segmentation over every frame: scales with clip length; a 30 s
// clip lands around this.
video2motion: 120,
};
// `reconstructUrl` overrides where `reconstruct` jobs go. The forge Hunyuan3D
// lane no longer needs it (it has its own `hunyuan` mode above); this remains
// as an escape hatch for pointing `reconstruct` at an alternate controller.
// Polling needs no override: the job envelope packs the base URL it was
// submitted to.
export function createRegenProvider({ reconstructUrl } = {}) {
const apiKey = readEnv('GCP_RECONSTRUCTION_KEY');
if (!apiKey) {
throw new Error('GCP_RECONSTRUCTION_KEY env var is required for the gcp provider');
}
const authHeaders = {
authorization: `Bearer ${apiKey}`,
'content-type': 'application/json',
};
const urlForMode = (mode) =>
mode === 'reconstruct' && reconstructUrl ? reconstructUrl : serviceUrlForMode(mode);
return {
supportsMode(mode) {
return Boolean(urlForMode(mode));
},
// The reconstruction worker accepts an `images` array and fuses every
// supplied view, so multi-view conditioning is available whenever the
// reconstruct service is configured.
supportsMultiview() {
return Boolean(urlForMode('reconstruct'));
},
async submit(request) {
const { mode } = request;
const baseUrl = urlForMode(mode);
if (!baseUrl) {
throw Object.assign(
new Error(`gcp provider: no service URL configured for mode "${mode}"`),
{ code: 'mode_unconfigured', status: 501 },
);
}
const workerReq = buildWorkerRequest(request);
if (!workerReq) {
throw Object.assign(
new Error(`gcp provider: unsupported mode "${mode}"`),
{ code: 'mode_unconfigured', status: 501 },
);
}
const url = `${baseUrl.replace(/\/$/, '')}${workerReq.path}`;
let response;
try {
response = await fetchUpstream(url, {
method: 'POST',
headers: authHeaders,
body: JSON.stringify(workerReq.body),
}, { name: 'gcp-forge-worker', timeoutMs: 60_000, attempts: 2, okWhen: () => true });
} catch (err) {
throw Object.assign(
new Error(`gcp ${mode} service unreachable: ${err?.message}`),
{ code: 'provider_unreachable', status: 502 },
);
}
const data = await response.json().catch(() => ({}));
if (!response.ok) {
throw Object.assign(
new Error(data?.detail || data?.error || `gcp service returned ${response.status}`),
{ code: 'provider_error', status: 502, providerStatus: response.status },
);
}
// Workers return either task_id (new) or job_id (legacy reconstruct).
const taskId = data.task_id || data.job_id;
if (!taskId) {
throw Object.assign(
new Error('gcp service returned no task_id'),
{ code: 'provider_error', status: 502 },
);
}
// Report how the job was conditioned. For reconstruct and Pixal3D the
// worker fuses every image in the body; the Hunyuan3D worker conditions
// on the primary view only (report 1 so a multi-view request is never
// silently claimed as fused); other modes are single-source.
const viewsUsed =
mode === 'hunyuan' || mode === 'triposplat' || mode === 'trellis2' || mode === 'modly' || mode === 'modly_hunyuan' || mode === 'anigen' || mode === 'partcrafter' || mode === 'sam3d'
? 1
: (mode === 'reconstruct' || mode === 'trellis' || mode === 'pixal3d') && Array.isArray(workerReq.body.images)
? workerReq.body.images.length
: 0;
return {
extJobId: packJobId({ mode, taskId, baseUrl, resultKey: workerReq.resultKey }),
eta: MODE_ETA[mode] ?? 60,
backend: 'gcp',
multiview: viewsUsed > 1,
viewsUsed,
};
},
async status(extJobId) {
if (!extJobId) {
return { status: 'failed', error: 'missing ext_job_id' };
}
const job = unpackJobId(extJobId);
if (!job?.taskId || !job?.baseUrl) {
return { status: 'failed', error: 'malformed ext_job_id' };
}
const { mode, taskId, baseUrl, resultKey } = job;
// The job id is unsigned and client-supplied, and the poll carries the
// worker bearer key, so only a configured worker URL may be polled.
const trimUrl = (u) => (typeof u === 'string' ? u.replace(/\/+$/, '') : '');
const allowedBase = new Set(
[urlForMode(mode), serviceUrlForMode(mode), reconstructUrl].filter(Boolean).map(trimUrl),
);
if (typeof baseUrl !== 'string' || !allowedBase.has(trimUrl(baseUrl))) {
return { status: 'failed', error: 'malformed ext_job_id' };
}
// Legacy reconstruct path uses /jobs/:id; new workers use /tasks/:id.
const pollPath = mode === 'reconstruct'
? `/jobs/${encodeURIComponent(taskId)}`
: `/tasks/${encodeURIComponent(taskId)}`;
let response;
try {
response = await fetchUpstream(`${baseUrl.replace(/\/$/, '')}${pollPath}`, { headers: authHeaders }, { name: 'gcp-forge-worker', timeoutMs: 15_000, attempts: 2, okWhen: () => true });
} catch (err) {
return { status: 'running', error: `poll failed: ${err?.message}` };
}
if (response.status === 404) {
// A 404 means the durable task record isn't visible to the instance
// this poll landed on — usually the post-submit cross-instance window
// (worker at high concurrency, no session affinity), not a real loss.
// Tag it recoverable so the router can grace/recover instead of failing
// on the first miss (see _lib/forge-selfhost-recovery.js).
return { status: 'failed', error: 'task not found on gcp service', code: 'gcp_task_missing' };
}
const data = await response.json().catch(() => ({}));
if (!response.ok) {
return { status: 'running', error: `gcp returned ${response.status}` };
}
const status = translateStatus(data.status);
const result = { status };
if (status === 'done') {
const url = data[resultKey] || data.result_url || data.glb_url || data.result_gcs_url;
if (!url) {
result.status = 'failed';
result.error = `${mode} finished but no result URL in response`;
} else if (mode === 'rembg') {
result.resultImageUrl = url;
} else if (mode === 'text2motion') {
// The result is a three.js AnimationClip JSON, not a mesh.
result.resultClipUrl = url;
if (typeof data.frames === 'number') result.frames = data.frames;
if (typeof data.fps === 'number') result.fps = data.fps;
} else if (mode === 'video2motion') {
// Clip JSON plus the body-swap compositing plates.
result.resultClipUrl = url;
if (data.meta_url) result.metaUrl = data.meta_url;
if (data.video_url) result.videoUrl = data.video_url;
if (data.mask_url) result.maskUrl = data.mask_url;
if (typeof data.frames === 'number') result.frames = data.frames;
if (typeof data.fps === 'number') result.fps = data.fps;
} else if (mode === 'triposplat') {
// A Gaussian splat, not a mesh: the compact .splat is the
// primary result and the .ply rides along for download.
result.resultSplatUrl = url;
if (data.ply_url) result.plyUrl = data.ply_url;
if (typeof data.num_gaussians === 'number') result.numGaussians = data.num_gaussians;
} else if (mode === 'sam3d') {
// A laid-out scene GLB plus the per-object manifest.
result.resultGlbUrl = url;
if (Array.isArray(data.objects)) result.objects = data.objects;
if (typeof data.object_count === 'number') result.objectCount = data.object_count;
if (Array.isArray(data.skipped) && data.skipped.length) result.skipped = data.skipped;
} else if (mode === 'video2scene') {
// The result is a .ply point cloud, not a GLB mesh. Surface the
// reconstruction telemetry so the client can show point/frame counts.
result.resultPointCloudUrl = url;
if (typeof data.num_points === 'number') result.numPoints = data.num_points;
if (typeof data.frames === 'number') result.frames = data.frames;
if (typeof data.bytes === 'number') result.bytes = data.bytes;
// The worker caps a job at its frame budget; say so rather than let a
// partial reconstruction look like the whole clip.
if (typeof data.frames_truncated === 'boolean') result.framesTruncated = data.frames_truncated;
if (typeof data.sky_points_removed === 'number') result.skyPointsRemoved = data.sky_points_removed;
} else {
result.resultGlbUrl = url;
}
// AniGen's GLB already carries a skeleton and skin weights. Tell the
// caller so the UI can skip offering a separate rerig step.
if (mode === 'anigen') {
result.rigged = true;
if (typeof data.joint_count === 'number') result.jointCount = data.joint_count;
}
// Surface remesh telemetry so callers can show topology stats.
if (mode === 'remesh') {
if (typeof data.face_count === 'number') result.faceCount = data.face_count;
if (typeof data.quad_ratio === 'number') result.quadRatio = data.quad_ratio;
if (typeof data.textured === 'boolean') result.textured = data.textured;
if (data.texture_url) result.textureUrl = data.texture_url;
if (data.mode) result.mode = data.mode;
}
// The stylize worker reports the rebuilt mesh's face count and the
// forge's stylize panel renders it beside the result, so it has to
// survive the hop: without this it arrived as null every time.
if (mode === 'stylize' && typeof data.face_count === 'number') {
result.faceCount = data.face_count;
}
// Surface the parts manifest so callers can render a selectable
// part list without a second fetch. PartCrafter and HoloPart share
// the same `parts`/`part_count` shape as segment (id, name, region,
// face_count, vertex_count, bbox, centroid, volume, color), so
// Restyle Studio's part picker reads all three sources identically.
if (mode === 'segment' || mode === 'partcrafter' || mode === 'holopart') {
if (data.manifest_url) result.manifestUrl = data.manifest_url;
if (Array.isArray(data.parts)) result.parts = data.parts;
if (typeof data.part_count === 'number') result.partCount = data.part_count;
if (typeof data.source_faces === 'number') result.sourceFaces = data.source_faces;
if (data.method) result.segmentMethod = data.method;
if (Array.isArray(data.warnings) && data.warnings.length) result.warnings = data.warnings;
}
}
if (status === 'failed') {
result.error = data.error || `${mode} failed`;
// The avatar workers classify failures ('input' = the caller's
// photos were unusable and `error` is caller-facing copy;
// 'internal' = service fault, `error` is an opaque correlation
// id). Surface the taxonomy so the status endpoint can relay
// actionable capture errors instead of masking every failure
// into the generic retry message.
if (data.error_kind === 'input' || data.error_kind === 'internal') {
result.errorKind = data.error_kind;
}
}
return result;
},
};
}