Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
/**
* OSS Avatar pipeline — Combined demo.
*
* Loads a body GLB + a frontal selfie, generates the MediaPipe face mesh, and
* attaches it to the body. If the body has a head bone, the face mesh is
* parented to it (so it animates with the body). Otherwise it sits at the top
* of the body's bounding box. The whole scene exports as one GLB.
*/
import {
Box3,
Vector3,
BufferGeometry,
BufferAttribute,
CanvasTexture,
SRGBColorSpace,
MeshStandardMaterial,
DoubleSide,
Mesh,
Scene,
Color,
PerspectiveCamera,
WebGLRenderer,
HemisphereLight,
DirectionalLight,
GridHelper,
} from 'three';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { getMeshoptDecoder } from '../viewer/internal.js';
import { GLTFExporter } from 'three/addons/exporters/GLTFExporter.js';
import { FilesetResolver, FaceLandmarker } from '@mediapipe/tasks-vision';
import { TRIANGULATION } from './triangulation.js';
import { log } from '../shared/log.js';
import { applyCinematicDefaults, detectQualityTier, loadEnvironment, updateGroundContactShadow } from '../shared/cinematic-render.js';
import { modelUrl, visionWasmBase } from '../shared/mediapipe-assets.js';
const MODEL_URL =
modelUrl('face_landmarker/face_landmarker/float16/1/face_landmarker.task');
// Runtime location comes from the shared resolver (see face-mocap.js).
const DEFAULT_BODY = '/avatars/default.glb';
const HEAD_BONE_PATTERNS = [
/^head$/i,
/head_/i,
/_head/i,
/^J_Bip_C_Head$/, // VRM
/mixamorig:?Head/i,
];
const viewerHost = document.getElementById('viewer-host');
const viewerBadge = document.getElementById('viewer-badge');
const viewerOverlay = document.getElementById('viewer-overlay');
const bodyStatus = document.getElementById('body-status');
const captureStatus = document.getElementById('capture-status');
const buildStatus = document.getElementById('build-status');
const exportStatus = document.getElementById('export-status');
const btnBodyDefault = document.getElementById('btn-body-default');
const btnBodyUpload = document.getElementById('btn-body-upload');
const bodyFileInput = document.getElementById('body-file-input');
const btnBuild = document.getElementById('btn-build');
const btnDownload = document.getElementById('btn-download');
const btnSave = document.getElementById('btn-save');
let landmarker = null;
let landmarkerLoading = null;
let viewer = null;
let bodyRoot = null;
let groundShadow = null;
let faceMesh = null;
let selfieImg = null;
let lastGlbBlob = null;
function setStatus(el, msg, kind) {
el.textContent = msg;
el.classList.remove('ok', 'err');
if (kind === 'ok') el.classList.add('ok');
if (kind === 'err') el.classList.add('err');
}
function refreshBuildEnabled() {
btnBuild.disabled = !(bodyRoot && selfieImg);
}
// ── Body loading ───────────────────────────────────────────────────────────
btnBodyDefault.addEventListener('click', () => loadBody(DEFAULT_BODY));
btnBodyUpload.addEventListener('click', () => bodyFileInput.click());
bodyFileInput.addEventListener('change', () => {
const file = bodyFileInput.files?.[0];
if (!file) return;
const url = URL.createObjectURL(file);
loadBody(url, file.name);
});
async function loadBody(url, label) {
ensureViewer();
setStatus(bodyStatus, `loading ${label || url}…`);
try {
const loader = new GLTFLoader();
loader.setMeshoptDecoder(await getMeshoptDecoder());
const gltf = await loader.loadAsync(url);
if (bodyRoot) {
viewer.scene.remove(bodyRoot);
bodyRoot = null;
}
bodyRoot = gltf.scene;
bodyRoot.name = 'body-root';
bodyRoot.traverse((n) => { if (n.isMesh) n.castShadow = true; });
viewer.scene.add(bodyRoot);
fitToObject(viewer, bodyRoot);
groundShadow = updateGroundContactShadow(viewer.scene, bodyRoot, groundShadow);
viewerOverlay.style.display = 'none';
const headBone = findHeadBone(bodyRoot);
const bones = countBones(bodyRoot);
setStatus(
bodyStatus,
`loaded · ${bones} bones · head bone: ${headBone ? headBone.name : 'none (will use bbox)'}`,
'ok',
);
viewerBadge.textContent = `body · ${bones} bones`;
refreshBuildEnabled();
} catch (err) {
setStatus(bodyStatus, `load failed: ${err.message}`, 'err');
}
}
function findHeadBone(root) {
let found = null;
root.traverse((node) => {
if (found || !node.isBone) return;
for (const re of HEAD_BONE_PATTERNS) {
if (re.test(node.name)) {
found = node;
return;
}
}
});
return found;
}
function countBones(root) {
let n = 0;
root.traverse((node) => {
if (node.isBone) n++;
});
return n;
}
// ── Selfie capture ─────────────────────────────────────────────────────────
document.querySelectorAll('.photo-slot').forEach((slot) => {
const input = slot.querySelector('input[type=file]');
input.addEventListener('change', async () => {
const file = input.files?.[0];
if (!file) return;
const url = URL.createObjectURL(file);
const img = await loadImage(url);
selfieImg = img;
let prev = slot.querySelector('img');
if (!prev) {
prev = document.createElement('img');
slot.insertBefore(prev, slot.firstChild);
}
prev.src = url;
setStatus(captureStatus, `loaded · ${img.naturalWidth}×${img.naturalHeight}`, 'ok');
refreshBuildEnabled();
});
});
function loadImage(src) {
return new Promise((resolve, reject) => {
const img = new Image();
img.crossOrigin = 'anonymous';
img.onload = () => resolve(img);
img.onerror = reject;
img.src = src;
});
}
// ── MediaPipe ──────────────────────────────────────────────────────────────
async function getLandmarker() {
if (landmarker) return landmarker;
if (landmarkerLoading) return landmarkerLoading;
landmarkerLoading = (async () => {
const fileset = await FilesetResolver.forVisionTasks(await visionWasmBase());
landmarker = await FaceLandmarker.createFromOptions(fileset, {
baseOptions: { modelAssetPath: MODEL_URL, delegate: 'GPU' },
runningMode: 'IMAGE',
numFaces: 1,
outputFacialTransformationMatrixes: true,
});
return landmarker;
})();
return landmarkerLoading;
}
// ── Combine ────────────────────────────────────────────────────────────────
btnBuild.addEventListener('click', async () => {
if (!bodyRoot || !selfieImg) return;
btnBuild.disabled = true;
try {
setStatus(buildStatus, 'loading MediaPipe…');
const lm = await getLandmarker();
setStatus(buildStatus, 'detecting landmarks…');
const result = lm.detect(selfieImg);
if (!result.faceLandmarks || !result.faceLandmarks.length) {
throw new Error('no face detected in photo');
}
const landmarks = result.faceLandmarks[0];
if (faceMesh) {
faceMesh.removeFromParent();
faceMesh.geometry.dispose();
if (faceMesh.material.map) faceMesh.material.map.dispose();
faceMesh.material.dispose();
faceMesh = null;
}
const mesh = buildFaceMesh(landmarks, selfieImg);
const headBone = findHeadBone(bodyRoot);
if (headBone) {
// Parent to head bone, position at bone origin.
// Bone local-space puts face roughly forward.
headBone.add(mesh);
mesh.position.set(0, 0.1, 0.1);
} else {
// No head bone: position at top of body bbox.
const box = new Box3().setFromObject(bodyRoot);
const center = new Vector3();
box.getCenter(center);
mesh.position.set(center.x, box.max.y - 0.12, center.z + 0.08);
bodyRoot.add(mesh);
}
// Scale face to roughly match a human head (~0.15m tall)
mesh.scale.setScalar(0.7);
faceMesh = mesh;
viewerBadge.textContent = `combined · ${countBones(bodyRoot)} bones · ${landmarks.length} face verts`;
setStatus(
buildStatus,
`attached to ${headBone ? `bone "${headBone.name}"` : 'body bbox'} · ${landmarks.length} face verts`,
'ok',
);
btnDownload.disabled = false;
btnSave.disabled = false;
setStatus(exportStatus, 'ready to export');
} catch (err) {
setStatus(buildStatus, `build failed: ${err.message}`, 'err');
log.error('[os-combined] build failed', err);
} finally {
btnBuild.disabled = false;
}
});
function buildFaceMesh(landmarks, photoImg) {
const geom = new BufferGeometry();
const aspect = photoImg.naturalWidth / photoImg.naturalHeight;
const scale = 0.2;
const positions = new Float32Array(landmarks.length * 3);
const uvs = new Float32Array(landmarks.length * 2);
for (let i = 0; i < landmarks.length; i++) {
const p = landmarks[i];
positions[i * 3 + 0] = (p.x - 0.5) * aspect * scale;
positions[i * 3 + 1] = -(p.y - 0.5) * scale;
positions[i * 3 + 2] = -p.z * scale;
uvs[i * 2 + 0] = p.x;
uvs[i * 2 + 1] = 1 - p.y;
}
geom.setAttribute('position', new BufferAttribute(positions, 3));
geom.setAttribute('uv', new BufferAttribute(uvs, 2));
geom.setIndex(TRIANGULATION);
geom.computeVertexNormals();
const canvas = document.createElement('canvas');
canvas.width = photoImg.naturalWidth;
canvas.height = photoImg.naturalHeight;
canvas.getContext('2d').drawImage(photoImg, 0, 0);
const texture = new CanvasTexture(canvas);
texture.colorSpace = SRGBColorSpace;
texture.flipY = false;
const material = new MeshStandardMaterial({
map: texture,
side: DoubleSide,
roughness: 0.85,
metalness: 0,
});
const mesh = new Mesh(geom, material);
mesh.name = 'os-face-mesh';
return mesh;
}
// ── Viewer ─────────────────────────────────────────────────────────────────
function ensureViewer() {
if (viewer) return viewer;
const width = viewerHost.clientWidth;
const height = viewerHost.clientHeight || 600;
const scene = new Scene();
scene.background = new Color(0x050505);
const camera = new PerspectiveCamera(35, width / height, 0.01, 100);
camera.position.set(0, 1.4, 2.4);
const renderer = new WebGLRenderer({ antialias: true });
renderer.setPixelRatio(Math.min(window.devicePixelRatio, 2));
renderer.setSize(width, height);
const tier = detectQualityTier();
applyCinematicDefaults(renderer, { tier });
viewerHost.appendChild(renderer.domElement);
loadEnvironment(renderer, scene, tier === 'mobile' ? null : 'studio');
scene.add(new HemisphereLight(0xffffff, 0x333333, 1.2));
const dir = new DirectionalLight(0xffffff, 1.5);
dir.position.set(2, 4, 3);
dir.castShadow = true;
dir.shadow.mapSize.set(1024, 1024);
scene.add(dir);
const grid = new GridHelper(4, 8, 0x222222, 0x111111);
scene.add(grid);
const controls = new OrbitControls(camera, renderer.domElement);
controls.target.set(0, 1, 0);
controls.enableDamping = true;
controls.update();
const onResize = () => {
const w = viewerHost.clientWidth;
const h = viewerHost.clientHeight || 600;
camera.aspect = w / h;
camera.updateProjectionMatrix();
renderer.setSize(w, h);
};
window.addEventListener('resize', onResize);
(function tick() {
controls.update();
renderer.render(scene, camera);
requestAnimationFrame(tick);
})();
viewer = { scene, camera, renderer, controls };
return viewer;
}
function fitToObject({ camera, controls }, obj) {
const box = new Box3().setFromObject(obj);
const size = new Vector3();
const center = new Vector3();
box.getSize(size);
box.getCenter(center);
const maxDim = Math.max(size.x, size.y, size.z);
const dist = (maxDim / Math.tan((Math.PI / 180) * camera.fov / 2)) * 1.4;
camera.position.set(center.x, center.y + size.y * 0.2, center.z + dist);
controls.target.copy(center);
controls.update();
}
// ── Export ─────────────────────────────────────────────────────────────────
async function exportGlb() {
if (!bodyRoot) return null;
if (lastGlbBlob) return lastGlbBlob;
const exporter = new GLTFExporter();
const arrayBuffer = await new Promise((resolve, reject) => {
exporter.parse(
bodyRoot,
(result) => resolve(result),
(err) => reject(err),
{ binary: true, embedImages: true },
);
});
const blob = new Blob([arrayBuffer], { type: 'model/gltf-binary' });
lastGlbBlob = blob;
return blob;
}
btnDownload.addEventListener('click', async () => {
try {
setStatus(exportStatus, 'exporting merged GLB…');
const blob = await exportGlb();
if (!blob) return;
const url = URL.createObjectURL(blob);
const a = document.createElement('a');
a.href = url;
a.download = `os-avatar-${Date.now()}.glb`;
a.click();
setTimeout(() => URL.revokeObjectURL(url), 1000);
setStatus(exportStatus, `downloaded · ${(blob.size / 1024).toFixed(1)} KB`, 'ok');
} catch (err) {
setStatus(exportStatus, `export failed: ${err.message}`, 'err');
}
});
btnSave.addEventListener('click', async () => {
try {
setStatus(exportStatus, 'exporting GLB…');
const blob = await exportGlb();
if (!blob) return;
setStatus(exportStatus, 'requesting presigned URL…');
const initRes = await fetch('/api/avatars/presign', {
method: 'POST',
credentials: 'include',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
size_bytes: blob.size,
content_type: 'model/gltf-binary',
}),
});
if (!initRes.ok) {
const txt = await initRes.text();
throw new Error(`presign ${initRes.status}: ${txt.slice(0, 120)}`);
}
const init = await initRes.json();
setStatus(exportStatus, 'uploading to R2…');
const putRes = await fetch(init.upload_url, {
method: 'PUT',
headers: { 'content-type': 'model/gltf-binary' },
body: blob,
});
if (!putRes.ok) throw new Error(`R2 PUT ${putRes.status}`);
setStatus(exportStatus, 'creating avatar record…');
const createRes = await fetch('/api/avatars', {
method: 'POST',
credentials: 'include',
headers: { 'content-type': 'application/json' },
body: JSON.stringify({
name: 'OSS combined avatar',
storage_key: init.storage_key,
size_bytes: blob.size,
visibility: 'private',
}),
});
if (!createRes.ok) {
const txt = await createRes.text();
throw new Error(`/api/avatars ${createRes.status}: ${txt.slice(0, 120)}`);
}
const created = await createRes.json();
setStatus(
exportStatus,
`saved · avatar id ${created.id || created.avatar?.id || 'ok'}`,
'ok',
);
} catch (err) {
setStatus(exportStatus, `save failed: ${err.message}`, 'err');
}
});