Skip to content

Open-source platform for 3D AI agents. Turn text or a photo into a rigged, animated GLB avatar, give it an LLM brain, memory and a wallet, and embed it anywhere with one web component. Remote MCP server, x402 payments, Three.js, browser-native.

14.9 KB
Raw Blame
/**
 * TalkScene — minimal three.js renderer used during "Talk to avatar" mode.
 *
 * model-viewer 4.x doesn't expose its internal three.js scene as a stable API,
 * so we render the avatar ourselves whenever lipsync needs to drive morphs in
 * real time. The showcase view (rotation, environment lighting) is still
 * served by model-viewer when talk mode is inactive — this module mounts only
 * on demand and unmounts cleanly when talk mode ends.
 *
 * Public surface:
 *   const scene = new TalkScene();
 *   await scene.mount({ container, glbUrl });
 *   scene.attachMouthTarget(target);   // AvatarMouthTarget
 *   scene.playAnimation('Idle');       // optional, if the GLB has clips
 *   scene.unmount();
 */

import {
	Timer,
	WebGLRenderer,
	Scene,
	AmbientLight,
	DirectionalLight,
	PerspectiveCamera,
	AnimationMixer,
	Box3,
} from 'three';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { OrbitControls } from 'three/addons/controls/OrbitControls.js';
import { applyCinematicDefaults, loadEnvironment, detectQualityTier, updateGroundContactShadow } from '../shared/cinematic-render.js';
import { computeFraming, CAMERA_PRESETS } from './camera-presets.js';
import { TalkEmotes } from './talk-emotes.js';
import { getMeshoptDecoder } from '../viewer/internal.js';

export class TalkScene {
	constructor() {
		this.container = null;
		this.renderer = null;
		this.scene = null;
		this.camera = null;
		this.controls = null;
		this.gltf = null;
		this.root = null;
		this.mixer = null;
		this._rigPaused = false;
		this._clips = [];
		this._currentAction = null;
		this._clock = new Timer();
		this._rafId = 0;
		this._running = false;
		this._resizeObserver = null;
		this._mouthTarget = null;
		this._cameraPreset = 'full';
		this._emotes = null;
		// Optional render throttle. 0 = uncapped (rAF native, ~60fps). A positive
		// cap (e.g. 30) skips render frames that arrive sooner than the budget so
		// an embedded preview doesn't monopolise the main thread — the elapsed dt
		// still accumulates across skipped frames, so motion stays time-correct.
		this._minFrameMs = 0;
		this._lastFrameAt = 0;
		// External per-frame subscribers (idle animation, custom drivers).
		// Receive dt in seconds. Use addOnTick() to register, call the returned
		// dispose function to unsubscribe.
		this._onTickFns = new Set();
	}

	/**
	 * Register a callback invoked every frame inside this scene's render loop
	 * with the elapsed dt in seconds. Returns a function that removes the
	 * subscription. Callbacks fire after controls/mixer/emotes update and
	 * before the render call, so writes to bones/morphs take effect this frame.
	 * @param {(dt: number) => void} fn
	 * @returns {() => void}
	 */
	addOnTick(fn) {
		if (typeof fn !== 'function') return () => {};
		this._onTickFns.add(fn);
		return () => this._onTickFns.delete(fn);
	}

	async mount({ container, glbUrl, glbBlob, cameraPreset = 'full' }) {
		this._cameraPreset = CAMERA_PRESETS.includes(cameraPreset) ? cameraPreset : 'full';
		if (!container) throw new Error('TalkScene.mount: container required');
		if (!glbUrl && !glbBlob) throw new Error('TalkScene.mount: glbUrl or glbBlob required');
		this.container = container;

		// preserveDrawingBuffer keeps the WebGL framebuffer readable via
		// `canvas.toBlob` — required for the snapshot pipeline (see
		// avatar-snapshot.js) to capture a valid JPEG after a render. The perf
		// cost is small for avatar-scale scenes; large at fullscreen-game scale.
		this.renderer = new WebGLRenderer({
			antialias: true,
			alpha: true,
			preserveDrawingBuffer: true,
		});
		// Cinematic defaults (ACES tone mapping, sRGB output, soft shadows,
		// pixel-ratio cap) shared with every other viewer on the platform.
		const qualityTier = detectQualityTier();
		applyCinematicDefaults(this.renderer, { exposure: 1.0, tier: qualityTier });

		const { width, height } = sizeOf(container);
		this.renderer.setSize(width, height, false);
		this.renderer.domElement.style.cssText = 'width:100%;height:100%;display:block;';
		container.appendChild(this.renderer.domElement);

		this.scene = new Scene();
		this.scene.background = null; // transparent so the page bg shows through

		// Real HDRI image-based lighting (falls back to procedural RoomEnvironment
		// on mobile tier or fetch failure).
		loadEnvironment(this.renderer, this.scene, qualityTier === 'mobile' ? null : 'studio');

		// Fill + key + rim for visual depth.
		this.scene.add(new AmbientLight(0xffffff, 0.4));
		const key = new DirectionalLight(0xffffff, 1.4);
		key.position.set(2, 4, 3);
		key.castShadow = true;
		key.shadow.mapSize.set(1024, 1024);
		key.shadow.camera.near = 0.5;
		key.shadow.camera.far = 10;
		key.shadow.camera.left = -1.5;
		key.shadow.camera.right = 1.5;
		key.shadow.camera.top = 2.5;
		key.shadow.camera.bottom = -1.5;
		key.shadow.bias = -0.0015;
		this.scene.add(key);
		const rim = new DirectionalLight(0x90a0ff, 0.6);
		rim.position.set(-3, 2, -2);
		this.scene.add(rim);

		this.camera = new PerspectiveCamera(35, width / height, 0.05, 100);
		this.camera.position.set(0, 1.55, 2.0);

		this.controls = new OrbitControls(this.camera, this.renderer.domElement);
		this.controls.enableDamping = true;
		this.controls.dampingFactor = 0.08;
		this.controls.target.set(0, 1.4, 0);
		this.controls.minDistance = 0.5;
		this.controls.maxDistance = 6;
		this.controls.update();

		// Load the GLB. Baked avatars carry EXT_meshopt_compression; the
		// loader can only decode them once the meshopt decoder is wired.
		//
		// When a Blob is supplied we read it into an ArrayBuffer once and call
		// `loader.parse()` directly — no second-fetch hop via object URL. This
		// avoids the "Failed to fetch" class of errors when an unstaged blob
		// URL is invalidated mid-load (navigation away, page reload). For
		// remote URLs we still use `loader.load()`.
		const loader = new GLTFLoader();
		loader.setMeshoptDecoder(await getMeshoptDecoder());
		const gltf = await new Promise(async (resolve, reject) => {
			try {
				if (glbBlob) {
					const buf = await glbBlob.arrayBuffer();
					loader.parse(buf, '', resolve, reject);
				} else {
					loader.load(glbUrl, resolve, undefined, reject);
				}
			} catch (err) {
				reject(err);
			}
		});
		this.gltf = gltf;
		this.root = gltf.scene;
		this.scene.add(this.root);
		this.root.traverse((node) => {
			if (node.isMesh) node.castShadow = true;
		});
		this._setupGroundShadow();

		// Frame the avatar: aim the camera at its head (estimated as the top
		// 15% of the bounding box) and back off proportional to its height.
		this._frameAvatar();

		// If the GLB ships any animations, set up a mixer and remember the
		// clips so the caller can request one (e.g. 'Idle' for a base loop).
		if (gltf.animations?.length) {
			this.mixer = new AnimationMixer(this.root);
			this._clips = gltf.animations;
			// Auto-play an idle if one is present.
			const idle = gltf.animations.find((c) => /idle|breath/i.test(c.name));
			if (idle) this.playAnimation(idle.name);
		}

		// External emote library — loads /animations/manifest.json so callers
		// can play any baked clip (wave, dance, celebrate, …) on this avatar.
		// Tracks are filtered per-bone so unmatched rigs cleanly no-op rather
		// than throwing.
		this._emotes = new TalkEmotes();
		this._emotes.attach(this.root);
		// Manifest fetch is async — fire it but don't block mount on it. The
		// caller can `await scene.getEmoteController().loadManifest()` if it
		// needs the bar populated before showing UI.
		this._emotes.loadManifest().catch(() => {});

		this._installResizeObserver();
		this._start();
		return this.root;
	}

	/** Access the emote controller (for UI integration). */
	getEmoteController() {
		return this._emotes;
	}

	/**
	 * Clip names baked into the loaded GLB. Empty means the model shipped with
	 * no animation of its own and needs a retargeted clip from the shared
	 * library, or it stands in bind pose.
	 * @returns {string[]}
	 */
	get clipNames() {
		return this._clips.map((c) => c.name);
	}

	/**
	 * Name of the GLB clip currently playing on the model's own mixer, or null.
	 * A caller can read this to decide whether the rig still needs driving.
	 * @returns {string|null}
	 */
	get playingClip() {
		return this._currentAction?.getClip?.()?.name || null;
	}

	/** Convenience: play an external emote by name. */
	playEmote(name) {
		return this._emotes?.play(name) ?? Promise.resolve(false);
	}

	/**
	 * Convenience: play a one-shot emote, settling back into a looping clip
	 * (idle by default) when it finishes so the rig never freezes on the final
	 * frame. Use for non-looping clips (wave, celebrate, flex, …).
	 */
	playEmoteOnce(name, opts) {
		return this._emotes?.playOnce(name, opts) ?? Promise.resolve(false);
	}

	attachMouthTarget(target) {
		this._mouthTarget = target;
		if (this.root) target.attach(this.root);
	}

	/** Play a clip by exact or fuzzy name. Returns true if a clip was started. */
	playAnimation(nameOrHint) {
		if (!this.mixer || !this._clips.length) return false;
		const hint = String(nameOrHint).toLowerCase();
		const clip =
			this._clips.find((c) => c.name === nameOrHint) ||
			this._clips.find((c) => c.name.toLowerCase().includes(hint));
		if (!clip) return false;
		const next = this.mixer.clipAction(clip);
		next.reset();
		next.fadeIn(0.25).play();
		if (this._currentAction && this._currentAction !== next) {
			this._currentAction.fadeOut(0.25);
		}
		this._currentAction = next;
		return true;
	}

	unmount() {
		this._running = false;
		if (this._rafId) cancelAnimationFrame(this._rafId);
		this._rafId = 0;

		if (this._emotes) {
			this._emotes.detach();
			this._emotes = null;
		}

		if (this._resizeObserver) {
			this._resizeObserver.disconnect();
			this._resizeObserver = null;
		}

		if (this.controls) {
			this.controls.dispose();
			this.controls = null;
		}

		// Dispose GPU resources.
		if (this.root) {
			this.root.traverse((node) => {
				node.geometry?.dispose?.();
				const mats = node.material ? (Array.isArray(node.material) ? node.material : [node.material]) : [];
				for (const m of mats) {
					m.map?.dispose?.();
					m.normalMap?.dispose?.();
					m.roughnessMap?.dispose?.();
					m.metalnessMap?.dispose?.();
					m.emissiveMap?.dispose?.();
					m.dispose?.();
				}
			});
		}

		this._groundShadow?.geometry?.dispose?.();
		this._groundShadow?.material?.dispose?.();
		this._groundShadow = null;
		this.scene?.environment?.dispose?.();
		this.renderer?.dispose?.();

		if (this.renderer?.domElement && this.renderer.domElement.parentNode === this.container) {
			this.container.removeChild(this.renderer.domElement);
		}

		this.renderer = null;
		this.scene = null;
		this.camera = null;
		this.root = null;
		this.gltf = null;
		this.mixer = null;
		this._clips = [];
		this._currentAction = null;
		this._mouthTarget = null;
	}

	// ── internals ────────────────────────────────────────────────────────

	/**
	 * Switch to a different camera framing preset and reframe immediately.
	 * No-op if the avatar hasn't loaded yet — the next _frameAvatar() picks up
	 * the new preset.
	 */
	setCameraPreset(preset) {
		if (!CAMERA_PRESETS.includes(preset)) return;
		this._cameraPreset = preset;
		this._frameAvatar();
	}

	getCameraPreset() {
		return this._cameraPreset;
	}

	// A ShadowMaterial plane renders fully invisible except where a shadow
	// falls on it, so it adds a soft ground-contact shadow at the avatar's
	// feet without introducing a visible floor — this scene stays transparent
	// (see `this.scene.background = null` in mount()). Sized/positioned from
	// the loaded model's own bounding box, so it works for any avatar height.
	_setupGroundShadow() {
		if (!this.root) return;
		this._groundShadow = updateGroundContactShadow(this.scene, this.root, this._groundShadow);
	}

	_frameAvatar() {
		if (!this.root || !this.camera || !this.controls) return;
		const b = new Box3().setFromObject(this.root);
		const aspect = this.camera.aspect || 1;
		const framing = computeFraming({
			box: { min: { x: b.min.x, y: b.min.y, z: b.min.z }, max: { x: b.max.x, y: b.max.y, z: b.max.z } },
			preset: this._cameraPreset,
			aspectRatio: aspect,
		});
		this.controls.target.set(framing.target.x, framing.target.y, framing.target.z);
		this.camera.position.set(framing.position.x, framing.position.y, framing.position.z);
		if (this.camera.fov !== framing.fov) {
			this.camera.fov = framing.fov;
			this.camera.updateProjectionMatrix();
		}
		this.controls.update();
	}

	_installResizeObserver() {
		if (typeof ResizeObserver === 'undefined') return;
		this._resizeObserver = new ResizeObserver(() => {
			if (!this.renderer || !this.camera) return;
			const { width, height } = sizeOf(this.container);
			if (width === 0 || height === 0) return;
			this.renderer.setSize(width, height, false);
			this.camera.aspect = width / height;
			this.camera.updateProjectionMatrix();
			// Distance/target depend on aspect (T-pose width vs viewport), so a
			// resize from wide → narrow needs a reframe or the silhouette clips.
			this._frameAvatar();
		});
		this._resizeObserver.observe(this.container);
	}

	/**
	 * Cap the render loop to `fps` frames per second (0 disables the cap). Used by
	 * the editor's walk preview to hold the canvas at 30fps so sculpt sliders and
	 * the rest of the UI stay responsive while a full locomotion loop runs.
	 * @param {number} fps
	 */
	setFpsCap(fps) {
		this._minFrameMs = fps && fps > 0 ? 1000 / fps : 0;
		this._lastFrameAt = 0;
	}

	/**
	 * Freeze or resume clip playback. Paused means the mixer and the emote
	 * controller stop advancing, so the skinned mesh holds the pose it is in;
	 * rendering, orbiting and everything else carry on.
	 * @param {boolean} paused
	 */
	setRigPaused(paused) {
		this._rigPaused = !!paused;
	}

	_start() {
		if (this._running) return;
		this._running = true;
		const tick = (now) => {
			if (!this._running) return;
			this._rafId = requestAnimationFrame(tick);
			// Honour an optional fps cap: bail before any work (and before the
			// clock ticks) so the accumulated dt lands on the frame we do render.
			if (this._minFrameMs) {
				const t = now || 0;
				if (this._lastFrameAt && t - this._lastFrameAt < this._minFrameMs) return;
				this._lastFrameAt = t;
			}
			this._clock.update();
			const dt = this._clock.getDelta();
			this.controls?.update();
			// A paused rig still renders and still orbits; it just stops being
			// re-posed. The free-sculpt brush needs a mesh that holds still
			// between pointer-down and pointer-up, and freezing the clip stack
			// is cheaper and more honest than trying to sculpt a moving target.
			if (!this._rigPaused) {
				this.mixer?.update(dt);
				this._emotes?.update(dt);
			}
			for (const fn of this._onTickFns) fn(dt);
			this.renderer?.render(this.scene, this.camera);
		};
		this._rafId = requestAnimationFrame(tick);
	}
}

function sizeOf(el) {
	const rect = el.getBoundingClientRect();
	return {
		width: Math.max(1, Math.floor(rect.width)),
		height: Math.max(1, Math.floor(rect.height)),
	};
}