Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.
JavaScript 3d-generationai-agentsanimationavatardigital-humangenerative-aiglbgltfimage-to-3dllmmcpmcp-servermodel-context-protocolsolanatext-to-3dthreejsweb-componentswebglwebxrx402
// /xr — immersive AR experience with full/half-body avatar.
//
// Provides a standalone Three.js scene wired to WebXRSession for real
// immersive-ar. Falls back to camera-passthrough on non-WebXR devices.
// The viewer shim object matches the interface expected by WebXRSession.
import {
AmbientLight,
CircleGeometry,
DirectionalLight,
HemisphereLight,
Mesh,
MeshStandardMaterial,
PerspectiveCamera,
Scene,
ShadowMaterial,
WebGLRenderer,
} from 'three';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { getMeshoptDecoder } from './viewer/internal.js';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { AnimationManager } from './animation-manager.js';
import { WebXRSession } from './ar/webxr.js';
import { wireShareButton } from './irl/share-frame.js';
import { canUseQuickLook } from './ar/quick-look.js';
import { canUseSceneViewer, openSceneViewer } from './ar/scene-viewer.js';
import { log } from './shared/log.js';
import { createAvatarPicker } from './avatar-picker.js';
import { applyCinematicDefaults, detectQualityTier, loadEnvironment } from './shared/cinematic-render.js';
// ── DOM ──────────────────────────────────────────────────────────────────────
const canvas = document.getElementById('xr-canvas');
const videoEl = document.getElementById('xr-camera-feed');
const enterArBtn = document.getElementById('xr-enter-ar');
const bodyToggle = document.getElementById('xr-body-toggle');
const fullBtn = document.getElementById('xr-body-full');
const halfBtn = document.getElementById('xr-body-half');
const statusEl = document.getElementById('xr-status');
const statusDot = document.getElementById('xr-status-dot');
const deviceChip = document.getElementById('xr-device-chip');
const avatarLabel = document.getElementById('xr-avatar-label');
const changeAvatarBtn = document.getElementById('xr-change-avatar');
const loadingEl = document.getElementById('xr-loading');
const lockBtn = document.getElementById('xr-lock-btn');
function setStatus(msg, type = 'idle') {
if (!statusEl) return;
statusEl.textContent = msg;
if (statusDot) {
statusDot.className = 'xr-dot xr-dot--' + type;
}
}
// ── Three.js setup ───────────────────────────────────────────────────────────
const renderer = new WebGLRenderer({ canvas, antialias: true, alpha: true, preserveDrawingBuffer: true });
renderer.setSize(window.innerWidth, window.innerHeight, false);
// Filmic tone mapping + correct sRGB output + soft VSM shadows: the shared bar
// every viewer on the platform now matches (src/shared/cinematic-render.js).
const qualityTier = detectQualityTier();
applyCinematicDefaults(renderer, { tier: qualityTier });
const scene = new Scene();
loadEnvironment(renderer, scene, qualityTier === 'mobile' ? null : 'studio');
const camera = new PerspectiveCamera(35, window.innerWidth / window.innerHeight, 0.01, 100);
camera.position.set(0, 1.4, 2.6);
const controls = new OrbitControls(camera, renderer.domElement);
controls.target.set(0, 1, 0);
controls.enableDamping = true;
controls.minDistance = 0.8;
controls.maxDistance = 6;
controls.minPolarAngle = 0.2;
controls.maxPolarAngle = Math.PI * 0.65;
controls.update();
let orbitLocked = false;
function setOrbitLocked(next) {
orbitLocked = next;
controls.enabled = !next;
if (lockBtn) {
lockBtn.setAttribute('aria-pressed', String(next));
lockBtn.classList.toggle('xr-lock--active', next);
lockBtn.querySelector('.xr-lock-label').textContent = next ? 'Locked' : 'Lock';
}
setStatus(next ? 'Avatar pinned — orbit disabled' : 'Orbit unlocked', 'idle');
}
if (lockBtn) lockBtn.addEventListener('click', () => setOrbitLocked(!orbitLocked));
// Lights
const hemi = new HemisphereLight(0xbcd6ff, 0x202830, 0.7);
hemi.position.set(0, 5, 0);
scene.add(hemi);
const ambient = new AmbientLight(0xffffff, 0.4);
scene.add(ambient);
const sun = new DirectionalLight(0xffffff, 1.5);
sun.position.set(4, 8, 6);
sun.castShadow = true;
sun.shadow.mapSize.set(1024, 1024);
sun.shadow.camera.near = 0.5;
sun.shadow.camera.far = 30;
sun.shadow.camera.left = -5;
sun.shadow.camera.right = 5;
sun.shadow.camera.top = 5;
sun.shadow.camera.bottom = -5;
sun.shadow.bias = -0.0005;
scene.add(sun);
scene.add(sun.target);
// Ground — opaque ring in preview, shadow-only in AR
const groundOpaque = new Mesh(
new CircleGeometry(3, 64),
new MeshStandardMaterial({ color: 0x151821, roughness: 0.95, metalness: 0 }),
);
groundOpaque.rotation.x = -Math.PI / 2;
groundOpaque.receiveShadow = true;
scene.add(groundOpaque);
const groundShadow = new Mesh(
new CircleGeometry(3, 64),
new ShadowMaterial({ opacity: 0.4 }),
);
groundShadow.rotation.x = -Math.PI / 2;
groundShadow.receiveShadow = true;
groundShadow.visible = false;
scene.add(groundShadow);
// ── Animation + viewer shim ───────────────────────────────────────────────────
const animationManager = new AnimationManager();
/** Viewer shim — matches the interface WebXRSession expects. */
const xrViewer = {
renderer,
scene,
content: null,
controls,
mixer: null,
animationManager,
_afterAnimateHooks: [],
_rafId: null,
prevTime: null,
activeCamera: camera,
_needsRender: true,
_updateRenderLoop() { _startRAF(); },
};
function _rafTick(time) {
xrViewer._rafId = requestAnimationFrame(_rafTick);
const dt = xrViewer.prevTime ? (time - xrViewer.prevTime) / 1000 : 0.016;
xrViewer.prevTime = time;
if (xrViewer.mixer) xrViewer.mixer.update(dt);
animationManager.update(dt);
for (const hook of xrViewer._afterAnimateHooks) hook(dt);
controls.update();
renderer.render(scene, camera);
}
function _startRAF() {
if (xrViewer._rafId !== null) return;
xrViewer._rafId = requestAnimationFrame(_rafTick);
}
// ── State ──────────────────────────────────────────────────────────────────────
let halfBodyActive = false;
let halfBodyBones = []; // { bone, scale } pairs captured on enter
let xrSession = null;
let arCameraActive = false;
let mediaStream = null;
// ── Half-body logic ───────────────────────────────────────────────────────────
const LOWER_BODY_FRAGMENTS = ['upleg','leg','thigh','knee','shin','calf','foot','toe','ankle'];
function _normalizeBone(name) {
return String(name || '').toLowerCase()
.replace(/^mixamorig:?_?/, '').replace(/^cc_base_/, '')
.replace(/^armature[:_|]/, '').replace(/^rig[:_]/, '');
}
function _isLowerBody(name) {
const n = _normalizeBone(name);
return LOWER_BODY_FRAGMENTS.some((f) => n.includes(f));
}
function applyHalfBody() {
const content = xrViewer.content;
if (!content) return;
halfBodyBones = [];
content.traverse((obj) => {
if (!obj.isSkinnedMesh || !obj.skeleton) return;
for (const bone of obj.skeleton.bones || []) {
if (!_isLowerBody(bone.name)) continue;
if (halfBodyBones.some((b) => b.bone === bone)) continue;
halfBodyBones.push({ bone, scale: bone.scale.clone() });
bone.scale.set(0.0001, 0.0001, 0.0001);
}
});
content.traverse((obj) => {
if (obj.isSkinnedMesh && obj.skeleton) obj.skeleton.update();
});
}
function removeHalfBody() {
for (const { bone, scale } of halfBodyBones) bone.scale.copy(scale);
halfBodyBones = [];
const content = xrViewer.content;
if (content) {
content.traverse((obj) => {
if (obj.isSkinnedMesh && obj.skeleton) obj.skeleton.update();
});
}
}
function setHalfBody(active) {
halfBodyActive = active;
if (active) {
applyHalfBody();
fullBtn.classList.remove('active');
halfBtn.classList.add('active');
} else {
removeHalfBody();
fullBtn.classList.add('active');
halfBtn.classList.remove('active');
}
}
// ── AR: camera passthrough (non-WebXR fallback) ───────────────────────────────
async function enableCameraAR() {
try {
mediaStream = await navigator.mediaDevices.getUserMedia({
video: { facingMode: { ideal: 'environment' } },
audio: false,
});
} catch (err) {
const msg = err?.name === 'NotAllowedError' ? 'camera permission denied' : `camera unavailable: ${err?.message}`;
setStatus(msg, 'error');
return false;
}
videoEl.srcObject = mediaStream;
try { await videoEl.play(); } catch {}
arCameraActive = true;
document.getElementById('xr-stage').classList.add('is-ar');
groundOpaque.visible = false;
groundShadow.visible = true;
renderer.setClearColor(0x000000, 0);
scene.background = null;
if (halfBodyActive) applyHalfBody();
setStatus('Camera AR active — your avatar in your space', 'ar');
enterArBtn.textContent = 'Exit AR';
enterArBtn.classList.add('active');
return true;
}
function disableCameraAR() {
if (mediaStream) {
for (const track of mediaStream.getTracks()) { try { track.stop(); } catch {} }
mediaStream = null;
}
videoEl.srcObject = null;
arCameraActive = false;
document.getElementById('xr-stage').classList.remove('is-ar');
groundOpaque.visible = true;
groundShadow.visible = false;
renderer.setClearColor(0x000000, 1);
setStatus('Preview', 'idle');
enterArBtn.textContent = _arBtnLabel();
enterArBtn.classList.remove('active');
}
// ── AR: WebXR immersive-ar ────────────────────────────────────────────────────
async function enterWebXR() {
if (xrSession) {
await xrSession.end();
return;
}
enterArBtn.disabled = true;
enterArBtn.textContent = 'Starting XR…';
setStatus('Requesting XR session…', 'loading');
try {
xrSession = new WebXRSession(xrViewer, {
halfBody: halfBodyActive,
onEnd: () => {
xrSession = null;
groundOpaque.visible = true;
groundShadow.visible = false;
enterArBtn.disabled = false;
enterArBtn.textContent = _arBtnLabel();
enterArBtn.classList.remove('active');
setStatus('Preview', 'idle');
},
});
await xrSession.start();
groundOpaque.visible = false;
groundShadow.visible = true;
setStatus('WebXR active — tap real surface to place avatar', 'ar');
enterArBtn.disabled = false;
enterArBtn.textContent = 'Exit XR';
enterArBtn.classList.add('active');
} catch (err) {
xrSession = null;
setStatus('XR failed: ' + (err?.message || err), 'error');
enterArBtn.disabled = false;
enterArBtn.textContent = _arBtnLabel();
}
}
// ── AR button logic ───────────────────────────────────────────────────────────
let _arMode = 'none'; // 'none' | 'webxr' | 'camera' | 'sceneviewer' | 'quicklook'
let _avatarGlbUrl = '/avatars/default.glb';
function _arBtnLabel() {
switch (_arMode) {
case 'webxr': return 'Enter AR';
case 'camera': return 'Enter AR';
case 'sceneviewer': return 'View in AR';
case 'quicklook': return 'View in AR';
default: return 'AR not available';
}
}
async function _detectARMode() {
if (navigator.xr && await navigator.xr.isSessionSupported('immersive-ar').catch(() => false)) {
_arMode = 'webxr';
if (deviceChip) {
deviceChip.textContent = 'WebXR';
deviceChip.className = 'xr-chip xr-chip--webxr';
}
setStatus('WebXR supported — tap Enter AR to place your avatar', 'ready');
} else if (canUseSceneViewer()) {
_arMode = 'sceneviewer';
if (deviceChip) {
deviceChip.textContent = 'ARCore';
deviceChip.className = 'xr-chip xr-chip--arcore';
}
setStatus('ARCore available', 'ready');
} else if (canUseQuickLook()) {
_arMode = 'quicklook';
if (deviceChip) {
deviceChip.textContent = 'Quick Look';
deviceChip.className = 'xr-chip xr-chip--ios';
}
setStatus('Quick Look available', 'ready');
} else if (navigator.mediaDevices?.getUserMedia) {
_arMode = 'camera';
if (deviceChip) {
deviceChip.textContent = 'Camera AR';
deviceChip.className = 'xr-chip xr-chip--camera';
}
setStatus('Camera passthrough available', 'ready');
} else {
_arMode = 'none';
if (deviceChip) {
deviceChip.textContent = 'No AR';
deviceChip.className = 'xr-chip xr-chip--none';
}
enterArBtn.disabled = true;
setStatus('AR not available on this device', 'idle');
}
if (enterArBtn && _arMode !== 'none') {
enterArBtn.textContent = _arBtnLabel();
enterArBtn.disabled = false;
}
}
enterArBtn.addEventListener('click', async () => {
if (xrSession) { await xrSession.end(); return; }
if (arCameraActive) { disableCameraAR(); return; }
if (_arMode === 'webxr') {
await enterWebXR();
} else if (_arMode === 'sceneviewer') {
openSceneViewer(_avatarGlbUrl, { title: 'three.ws avatar', link: location.href });
} else if (_arMode === 'camera') {
await enableCameraAR();
}
});
// ── Body toggle ───────────────────────────────────────────────────────────────
fullBtn.addEventListener('click', () => {
if (!halfBodyActive) return;
setHalfBody(false);
// If already in XR, restart with updated mode
if (xrSession) {
xrSession.end().then(() => enterWebXR());
}
});
halfBtn.addEventListener('click', () => {
if (halfBodyActive) return;
setHalfBody(true);
if (xrSession) {
xrSession.end().then(() => enterWebXR());
}
});
// ── Avatar loading ────────────────────────────────────────────────────────────
let _currentAvatarId = null;
function _clearAvatar() {
if (xrViewer.content) {
scene.remove(xrViewer.content);
xrViewer.content = null;
}
animationManager.detach();
xrViewer.mixer = null;
}
async function _resolveAvatarUrl(id) {
if (!id) return '/avatars/default.glb';
const res = await fetch(`/api/avatars/${encodeURIComponent(id)}`);
if (!res.ok) return '/avatars/default.glb';
const { avatar } = await res.json();
return avatar?.url || '/avatars/default.glb';
}
async function loadAvatar(idOrUrl) {
const params = new URLSearchParams(location.search);
const id = idOrUrl !== undefined ? idOrUrl : params.get('avatar');
let url;
if (!id) {
url = '/avatars/default.glb';
} else if (/^https?:\/\/|^\//.test(id)) {
url = id;
} else {
url = await _resolveAvatarUrl(id);
}
_avatarGlbUrl = url;
_clearAvatar();
const loader = new GLTFLoader();
loader.setMeshoptDecoder(await getMeshoptDecoder());
const gltf = await loader.loadAsync(url);
const root = gltf.scene;
root.traverse((obj) => {
if (obj.isMesh) {
obj.castShadow = true;
obj.receiveShadow = true;
}
});
scene.add(root);
xrViewer.content = root;
// Frame camera to avatar bounds
const { Box3, Vector3: V3 } = await import('three');
const box = new Box3().setFromObject(root);
const size = box.getSize(new V3()).length();
const center = box.getCenter(new V3());
controls.target.copy(center).setY(center.y * 0.7);
camera.position.set(0, center.y + size * 0.2, size * 1.4);
controls.update();
// Animations — fetch manifest, load idle clip, attach to model
try {
const manifest = await fetch('/animations/manifest.json', { cache: 'force-cache' })
.then((r) => r.ok ? r.json() : []);
const idleDef = manifest.find((d) => d.name === 'idle') || manifest[0];
if (idleDef) {
animationManager.setAnimationDefs([idleDef]);
await animationManager.loadAll();
}
} catch {}
animationManager.attach(root);
xrViewer.mixer = animationManager.mixer;
if (animationManager._animationDefs.length) animationManager.play('idle');
// Re-apply half-body if the mode was active during a hot-swap
if (halfBodyActive) applyHalfBody();
// Avatar label: prefer picker-supplied name, fall back to URL param
const handle = params.get('handle');
if (avatarLabel) avatarLabel.textContent = handle || 'Your Avatar';
}
// ── Resize ───────────────────────────────────────────────────────────────────
window.addEventListener('resize', () => {
renderer.setSize(window.innerWidth, window.innerHeight, false);
camera.aspect = window.innerWidth / window.innerHeight;
camera.updateProjectionMatrix();
});
// ── Avatar picker ─────────────────────────────────────────────────────────────
const avatarPicker = createAvatarPicker({
onSelect: async ({ id, url, name }) => {
_currentAvatarId = id;
if (avatarLabel) avatarLabel.textContent = name;
setStatus('Loading avatar…', 'loading');
// Update URL so sharing reflects the chosen avatar
const sp = new URLSearchParams(location.search);
if (id) sp.set('avatar', id); else sp.delete('avatar');
history.replaceState(null, '', location.pathname + (sp.toString() ? '?' + sp : ''));
try {
await loadAvatar(url);
setStatus('Avatar loaded', 'idle');
} catch (err) {
setStatus('Failed to load avatar', 'error');
log.error('[xr] avatar swap failed:', err);
}
},
});
if (changeAvatarBtn) {
changeAvatarBtn.addEventListener('click', () => {
avatarPicker.open(_currentAvatarId);
});
}
// ── Init ─────────────────────────────────────────────────────────────────────
(async () => {
_startRAF();
setStatus('Loading avatar…', 'loading');
try {
await loadAvatar();
} catch (err) {
setStatus('Failed to load avatar', 'error');
log.error('[xr] avatar load failed:', err);
}
if (loadingEl) loadingEl.hidden = true;
// Body toggle: start on full body
fullBtn.classList.add('active');
// Composite share — same module /irl uses, so the two surfaces never drift.
// Flattens the camera passthrough (when AR is on) under the 3D canvas into one
// PNG; renderer is built with preserveDrawingBuffer:true for a non-blank readback.
wireShareButton(document.getElementById('xr-share-btn'), {
getCanvas: () => canvas,
getVideo: () => videoEl,
getIsAR: () => arCameraActive,
filename: 'three-ws-xr.png',
title: 'XR · three.ws',
});
await _detectARMode();
})();