Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

17.0 KB
Raw Blame
// Shared live-preview engine for the /animations gallery.
//
// One WebGL renderer + one preview avatar serve every card and the detail
// modal. The canvas is *moved* into whichever container is previewing (hover a
// card, open the modal) and a RAF loop runs only while something is mounted —
// the page never holds more than a single GL context, where the previous
// gallery booted a full embed-viewer iframe (renderer, avatar, clip fetch) per
// hovered card.
//
// Everything heavy (three.js, the retarget engine, the avatar GLB) loads
// lazily on the first preview, so the gallery's initial paint ships no 3D at
// all. Clip JSONs and retargeted clips are cached by name, so re-hovering a
// card replays instantly.

const MODEL_URL = '/avatars/cz.glb';

export class AnimationLivePreview {
	constructor() {
		this._bootPromise = null;
		this._three = null; // three module namespace
		this._renderer = null;
		this._scene = null;
		this._camera = null;
		this._model = null;
		this._mixer = null;
		this._bones = [];
		this._restPose = new Map();
		this._maps = null; // canonical maps captured at bind pose
		// Framing computed by _frameCamera, plus the viewer's own offsets on top
		// of it. Kept apart so a new clip can re-frame without throwing away a
		// turn or a zoom the viewer asked for.
		this._framing = null;
		this._yaw = 0;
		this._zoom = 1;
		this._retargetMod = null;

		this._clipJsonCache = new Map(); // clip name → raw clip JSON promise
		this._boundClipCache = new Map(); // clip name → retargeted AnimationClip

		this._container = null;
		this._action = null;
		this._activeDef = null;
		this._fading = []; // crossfade sources awaiting retirement
		this._raf = 0;
		this._lastT = 0;
		this._playToken = 0;
		this._paused = false;
		this._onFrame = null; // (timeSec, durationSec) => void — modal scrubber hook
	}

	/** True once the engine has booted (first preview finished loading). */
	get ready() {
		return !!this._model;
	}

	/** Currently previewing def (or null). */
	get active() {
		return this._activeDef;
	}

	async _boot() {
		if (this._bootPromise) return this._bootPromise;
		this._bootPromise = (async () => {
			const [THREE, { GLTFLoader }, { RoomEnvironment }, retarget, { getMeshoptDecoder }] =
				await Promise.all([
					import('three'),
					import('three/addons/loaders/GLTFLoader.js'),
					import('three/addons/environments/RoomEnvironment.js'),
					import('./animation-retarget.js'),
					import('./viewer/internal.js'),
				]);
			this._three = THREE;
			this._retargetMod = retarget;

			const renderer = new THREE.WebGLRenderer({ antialias: true, alpha: true });
			renderer.setPixelRatio(Math.min(window.devicePixelRatio || 1, 2));
			renderer.shadowMap.enabled = true;
			// PCFSoftShadowMap is deprecated and silently downgrades to hard
			// PCFShadowMap at runtime (console warning, worse shadow edges).
			// VSMShadowMap is the still-supported soft-shadow type.
			renderer.shadowMap.type = THREE.VSMShadowMap;
			renderer.toneMapping = THREE.ACESFilmicToneMapping;
			renderer.toneMappingExposure = 1.05;
			renderer.domElement.className = 'alp-canvas';
			renderer.domElement.setAttribute('aria-hidden', 'true');
			this._renderer = renderer;

			const scene = new THREE.Scene();
			const pmrem = new THREE.PMREMGenerator(renderer);
			scene.environment = pmrem.fromScene(new RoomEnvironment(), 0.04).texture;
			this._scene = scene;

			const key = new THREE.DirectionalLight(0xffffff, 1.6);
			key.position.set(2.2, 4.5, 3.2);
			key.castShadow = true;
			key.shadow.mapSize.set(1024, 1024);
			key.shadow.camera.left = -3;
			key.shadow.camera.right = 3;
			key.shadow.camera.top = 3;
			key.shadow.camera.bottom = -3;
			key.shadow.bias = -0.0004;
			scene.add(key);
			scene.add(key.target);
			this._keyLight = key;
			const rim = new THREE.DirectionalLight(0x99bbff, 0.7);
			rim.position.set(-2.5, 2.5, -2.5);
			scene.add(rim);

			const ground = new THREE.Mesh(
				new THREE.PlaneGeometry(40, 40),
				new THREE.ShadowMaterial({ opacity: 0.28 }),
			);
			ground.rotation.x = -Math.PI / 2;
			ground.receiveShadow = true;
			scene.add(ground);

			this._camera = new THREE.PerspectiveCamera(33, 3 / 4, 0.05, 100);

			const loader = new GLTFLoader();
			loader.setMeshoptDecoder(await getMeshoptDecoder());
			const gltf = await loader.loadAsync(MODEL_URL);
			const model = gltf.scene;
			model.traverse((o) => {
				if (o.isMesh || o.isSkinnedMesh) {
					o.castShadow = true;
					o.frustumCulled = false;
				}
				if (o.isBone) this._bones.push(o);
				this._restPose.set(o, {
					p: o.position.clone(),
					q: o.quaternion.clone(),
					s: o.scale.clone(),
				});
			});
			scene.add(model);
			model.updateMatrixWorld(true);
			this._model = model;
			// Same bind-pose capture recipe as AnimationManager.attach(), so previews
			// play exactly what the studio and embeds play.
			this._maps = {
				canonicalToNode: retarget.canonicalNodeMapFromObject(model),
				targetRest: retarget.canonicalRestMapFromObject(model),
				targetWorldRest: retarget.canonicalWorldRestMapFromObject(model),
				hipsParentWorldQuat: retarget.hipsParentWorldQuat(model),
				hipTargetLocalY: retarget.hipRestLocalHeight(model),
			};
			this._mixer = new THREE.AnimationMixer(model);
		})();
		return this._bootPromise;
	}

	async _fetchClipJson(def) {
		if (this._clipJsonCache.has(def.id)) return this._clipJsonCache.get(def.id);
		const promise = (async () => {
			if (def.source === 'community') {
				const res = await fetch(
					`/api/animations/clips/${encodeURIComponent(def.id)}?play=1`,
					{ credentials: 'include' },
				);
				if (!res.ok) throw new Error(`HTTP ${res.status}`);
				const data = await res.json();
				if (!data?.clip?.clip) throw new Error('clip payload missing');
				return data.clip.clip;
			}
			// Curated manifest clips carry a site-relative url; library clips carry an
			// absolute CDN url (or none — resolve via the library manifest by name).
			let url = def.url;
			if (!url) throw new Error('clip has no source url');
			const res = await fetch(url);
			if (!res.ok) throw new Error(`HTTP ${res.status}`);
			return res.json();
		})();
		this._clipJsonCache.set(def.id, promise);
		promise.catch(() => this._clipJsonCache.delete(def.id));
		return promise;
	}

	_restoreRestPose() {
		this._mixer.stopAllAction();
		for (const [node, t] of this._restPose) {
			node.position.copy(t.p);
			node.quaternion.copy(t.q);
			node.scale.copy(t.s);
		}
		this._model.updateMatrixWorld(true);
	}

	async _bindClip(def) {
		if (this._boundClipCache.has(def.id)) return this._boundClipCache.get(def.id);
		const json = await this._fetchClipJson(def);
		const r = this._retargetMod;
		this._restoreRestPose();
		const clip = r.parseClipJSON(json, def.id);
		let hipScale = 1;
		const baseline = r.clipHipBaselineY(clip);
		if (this._maps.hipTargetLocalY > 0.05 && baseline > 0.05) {
			hipScale = Math.min(200, Math.max(0.2, this._maps.hipTargetLocalY / baseline));
		}
		const { clip: bound } = r.retargetClip(clip, this._maps.canonicalToNode, {
			...this._maps,
			hipScale,
			minCoverage: 0.3,
		});
		if (!bound) throw new Error('clip does not fit the preview avatar');
		this._boundClipCache.set(def.id, bound);
		// A page-lifetime cache of every hovered clip would grow unbounded on a
		// 2,000-card grid; keep the most recent ~40.
		if (this._boundClipCache.size > 40) {
			const oldest = this._boundClipCache.keys().next().value;
			if (oldest !== def.id) this._boundClipCache.delete(oldest);
		}
		return bound;
	}

	// Frame the figure's full motion envelope: sample the clip at a few points,
	// union the bone boxes, and lock the camera for the whole loop so walks and
	// flips stay in frame without camera chase.
	//
	// Accepts several clips, which is what a choreography needs: framing each
	// step on its own would jog the camera at every cut, so a routine unions the
	// envelopes of all of its clips once and holds that frame for the whole
	// performance.
	_frameCamera(bound) {
		const THREE = this._three;
		const clips = Array.isArray(bound) ? bound.filter(Boolean) : [bound];
		if (!clips.length) return;
		const box = new THREE.Box3();
		const v = new THREE.Vector3();
		const samples = [0.05, 0.3, 0.55, 0.8];
		for (const clip of clips) {
			this._mixer.stopAllAction();
			const probe = this._mixer.clipAction(clip);
			probe.play();
			for (const s of samples) {
				this._mixer.setTime(clip.duration * s);
				this._model.updateMatrixWorld(true);
				for (const b of this._bones) box.expandByPoint(b.getWorldPosition(v));
			}
			probe.stop();
		}
		// Sampling drove the skeleton; put it back before the caller starts the
		// real action, so a clip that animates only the upper body does not
		// inherit a probe's legs.
		this._mixer.setTime(0);
		this._restoreRestPose();
		box.expandByVector(new THREE.Vector3(0.24, 0.15, 0.24));
		box.min.y = Math.min(box.min.y, 0);
		const center = box.getCenter(new THREE.Vector3());
		const size = box.getSize(new THREE.Vector3());
		const cam = this._camera;
		const fovV = THREE.MathUtils.degToRad(cam.fov);
		const fovH = 2 * Math.atan(Math.tan(fovV / 2) * cam.aspect);
		const distV = size.y / 2 / Math.tan(fovV / 2);
		const distH = Math.max(size.x, size.z) / 2 / Math.tan(fovH / 2);
		const dist = Math.max(distV, distH) * 1.15 + Math.max(size.z, size.x) / 2;
		this._framing = {
			center,
			dist,
			azimuth: THREE.MathUtils.degToRad(24),
			elevation: THREE.MathUtils.degToRad(11),
		};
		this._applyCamera();
	}

	/** Place the camera from the current framing plus the viewer's turn and zoom. */
	_applyCamera() {
		const f = this._framing;
		if (!f || !this._camera) return;
		const cam = this._camera;
		const azimuth = f.azimuth + this._yaw;
		const dist = f.dist / this._zoom;
		cam.position.set(
			f.center.x + dist * Math.cos(f.elevation) * Math.sin(azimuth),
			f.center.y + dist * Math.sin(f.elevation),
			f.center.z + dist * Math.cos(f.elevation) * Math.cos(azimuth),
		);
		cam.lookAt(f.center);
		this._keyLight.target.position.copy(f.center);
		this._keyLight.target.updateMatrixWorld();
	}

	/**
	 * Turn the camera around the avatar: an absolute angle in radians, measured
	 * from the framed shot. Absolute rather than additive on purpose, because a
	 * crossfaded clip change keeps the yaw it already had while a cold mount
	 * resets it; a caller re-applying its own angle after every cut would double
	 * it on the warm path and the shot would drift further with every gesture.
	 */
	setYaw(radians) {
		this._yaw = radians || 0;
		this._applyCamera();
	}

	/** Dolly in or out. 1 is the framed distance; clamped to a usable range. */
	setZoom(factor) {
		this._zoom = Math.max(0.6, Math.min(3, factor || 1));
		this._applyCamera();
	}

	/** Drop the viewer's turn and zoom, back to the framed shot. */
	resetView() {
		this._yaw = 0;
		this._zoom = 1;
		this._applyCamera();
	}

	/** Stop every crossfade source whose fade has elapsed. */
	_retireFaded(incoming) {
		const now = performance.now();
		this._fading = this._fading.filter((entry) => {
			if (entry.action === incoming || now < entry.until) return true;
			entry.action.stop();
			return false;
		});
	}

	_resizeToContainer() {
		if (!this._container || !this._renderer) return;
		const w = this._container.clientWidth || 300;
		const h = this._container.clientHeight || 400;
		this._renderer.setSize(w, h, false);
		this._renderer.domElement.style.width = '100%';
		this._renderer.domElement.style.height = '100%';
		this._camera.aspect = w / h;
		this._camera.updateProjectionMatrix();
	}

	_loop = () => {
		this._raf = requestAnimationFrame(this._loop);
		const now = performance.now();
		const dt = Math.min(0.1, (now - this._lastT) / 1000);
		this._lastT = now;
		if (!this._paused) this._mixer.update(dt);
		if (this._onFrame && this._action) {
			this._onFrame(this._action.time, this._action.getClip().duration);
		}
		this._renderer.render(this._scene, this._camera);
	};

	/**
	 * Bind clips ahead of time. A choreography calls this once for its whole
	 * step list so the first cut is not the first fetch: without it, step two
	 * arrives mid-performance and the stage stutters exactly where the routine
	 * was supposed to flow.
	 *
	 * Failures are swallowed per clip — one unbindable clip must not stop the
	 * routine from being previewable — and reported by the resolved array's
	 * `null` entries.
	 *
	 * @param {Array<{id:string, source:string, url?:string}>} defs
	 * @returns {Promise<Array<object|null>>} bound clips, index-aligned with defs
	 */
	async prepare(defs) {
		await this._boot();
		return Promise.all(
			(defs || []).map((def) => this._bindClip(def).catch(() => null)),
		);
	}

	/**
	 * Mount the shared canvas into `container` and play `def`'s clip.
	 * Any previous preview stops. Resolves once playing; rejects on load or
	 * retarget failure (the caller shows its fallback).
	 *
	 * `opts.crossfade` blends out of whatever is already playing instead of
	 * hard-cutting, which is what turns a list of clips into a performance.
	 * `opts.frameWith` locks the camera to the union of several clips' motion
	 * envelopes, so it does not jog at every cut.
	 *
	 * @param {HTMLElement} container
	 * @param {{id:string, source:string, url?:string, loop?:boolean}} def
	 * @param {{ speed?: number, crossfade?: number, frameWith?: Array, onFrame?: (t:number,d:number)=>void }} [opts]
	 */
	async play(container, def, opts = {}) {
		const token = ++this._playToken;
		await this._boot();
		const bound = await this._bindClip(def);
		if (token !== this._playToken) return; // superseded while loading

		const THREE = this._three;
		// A crossfade is only possible into a stage that is already rendering this
		// clip's neighbour; anything else (first play, a different container) is a
		// cold start and takes the full reset path.
		const blending =
			opts.crossfade > 0 && this._action && this._container === container && this._raf;
		const previous = blending ? this._action : null;

		if (!blending) {
			this.stop({ keepBoot: true });
			this._container = container;
			container.appendChild(this._renderer.domElement);
			this._resizeToContainer();
			this._restoreRestPose();
		} else {
			// A pending one-shot replay belongs to the outgoing clip.
			this._mixer.removeEventListener?.('finished', this._replay);
			this._replay = null;
		}

		// Frame before the incoming action starts: sampling drives the skeleton,
		// and a routine frames once for all of its clips rather than per step.
		if (!blending || opts.frameWith) {
			this._frameCamera(opts.frameWith?.length ? opts.frameWith : bound);
		}

		const action = this._mixer.clipAction(bound);
		action.reset();
		if (def.loop === false) {
			action.setLoop(THREE.LoopOnce, 0);
			action.clampWhenFinished = true;
			// One-shots replay on a beat so a hovering user sees the motion again.
			// A sequenced step does not: the routine decides when the next cut is.
			if (!opts.crossfade) {
				this._mixer.removeEventListener?.('finished', this._replay);
				this._replay = () => {
					action.reset().play();
				};
				this._mixer.addEventListener('finished', this._replay);
			}
		} else {
			action.setLoop(THREE.LoopRepeat, Infinity);
		}
		action.timeScale = opts.speed ?? 1;
		if (previous && previous !== action) {
			action.play();
			previous.crossFadeTo(action, opts.crossfade, false);
			// A faded-out action keeps being evaluated at weight 0 forever unless it
			// is stopped. Over a looping routine that is a slow leak of frame time,
			// so retire each one as soon as its fade is spent.
			this._retireFaded(action);
			this._fading.push({ action: previous, until: performance.now() + opts.crossfade * 1000 });
		} else {
			action.setEffectiveWeight(1);
			action.play();
		}
		this._action = action;
		this._activeDef = def;
		this._paused = false;
		this._onFrame = opts.onFrame || null;
		this._lastT = performance.now();
		if (!this._raf) this._raf = requestAnimationFrame(this._loop);
	}

	/** Detach the canvas and halt rendering. Cheap — the engine stays warm. */
	stop() {
		this._playToken++;
		cancelAnimationFrame(this._raf);
		this._raf = 0;
		if (this._mixer && this._replay) {
			this._mixer.removeEventListener('finished', this._replay);
			this._replay = null;
		}
		if (this._mixer) this._mixer.stopAllAction();
		this._action = null;
		this._activeDef = null;
		this._fading = [];
		this._onFrame = null;
		if (this._renderer?.domElement?.parentNode) {
			this._renderer.domElement.parentNode.removeChild(this._renderer.domElement);
		}
		this._container = null;
		this._yaw = 0;
		this._zoom = 1;
	}

	/** Modal transport controls. No-ops when nothing is playing. */
	setPaused(paused) {
		this._paused = !!paused;
	}
	isPaused() {
		return this._paused;
	}
	setSpeed(factor) {
		if (this._action) this._action.timeScale = factor;
	}
	/** Seek to a 0..1 position in the clip. */
	seek(t01) {
		if (!this._action) return;
		const d = this._action.getClip().duration;
		this._action.time = Math.max(0, Math.min(0.999, t01)) * d;
		this._mixer.update(0);
	}
	/** Re-fit renderer + camera after the container was resized (modal open). */
	refit() {
		this._resizeToContainer();
		if (this._action) this._frameCamera(this._action.getClip());
	}
}

/** Page-singleton accessor. */
let _instance = null;
export function getLivePreview() {
	if (!_instance) _instance = new AnimationLivePreview();
	return _instance;
}