Turn one illustration into an animated 2D mesh avatar with a coding agent and a local editor
import { LightingControls, LightHandle, lightingText } from '../lighting/Controls';
import { colorHex, LIVE_EXPRESSIONS, type LiveExpression, type MeshAvatar } from 'mesh-avatar';
import { loadLighting, saveLighting } from '../lighting/storage';
import { loadLightingPresets, sameLighting, saveLightingPresets } from '../lighting/presets';
import { useEffect, useRef, useState } from 'react';
import { useI18n } from '../editor/i18n';
import { createAvatarView } from './avatar-view';
import { viewSettings, streamUrl, backgroundColor, loadOverlays, saveOverlays } from './settings';
import { FacePose, type TrackingOptions } from './tracking';
import { CameraCapture, MicrophoneCapture, type CameraState, type MicState, type BackgroundTracking } from './media';
import { liveText } from './i18n';
import { Icon } from '../editor/Icon';
import { createLiveSender, receiveExpression, receiveLightingPreset, sendLighting, sendOverlays, watchDuplicateLivePages } from './relay';
import { EXPRESSION_TOKEN_PATH, LIGHTING_PRESET_PATH } from './expression-protocol';
export function LiveApp() {
const { language, setLanguage } = useI18n(), t = liveText[language];
const [settings, setSettings] = useState(() => { const view = viewSettings(location.search); return { ...view, lighting: view.lighting ?? loadLighting(view.project), overlays: view.overlays ?? loadOverlays(view.project) }; });
// Overlay layers the loaded avatar has (built/overlays), listed once it has loaded.
const [availableOverlays, setAvailableOverlays] = useState<string[]>([]);
const [support, setSupport] = useState<{ legs: boolean; arms: boolean } | null>(null);
const supportRef = useRef(support); supportRef.current = support;
const [lightingOpen, setLightingOpen] = useState(false);
const [presets, setPresets] = useState(() => loadLightingPresets(settings.project));
const presetsRef = useRef(presets); presetsRef.current = presets;
const [duplicatePage, setDuplicatePage] = useState(false);
const [expression, setExpression] = useState<LiveExpression>('neutral');
const expressionRef = useRef(expression); expressionRef.current = expression;
const [remote, setRemote] = useState<{ url: string; token: string } | null>(null), [remoteCopied, setRemoteCopied] = useState<'url' | 'token' | null>(null);
const [options, setOptions] = useState<TrackingOptions>({ mirror: true, sensitivity: 1, smoothing: 0.35 });
const [cameraState, setCameraState] = useState<CameraState>('stopped');
const [micState, setMicState] = useState<MicState>('micOff');
const [tracking, setTracking] = useState(false), [showCamera, setShowCamera] = useState(true);
const [devices, setDevices] = useState<MediaDeviceInfo[]>([]), [cameraId, setCameraId] = useState(''), [micId, setMicId] = useState('');
const [gain, setGain] = useState(1), [calibrated, setCalibrated] = useState(false);
const [copyState, setCopyState] = useState<'copied' | 'copyError' | null>(null);
const [viewState, setViewState] = useState<'loading' | 'ready' | 'projectError'>('loading');
const [backgroundStatus, setBackgroundStatus] = useState<BackgroundTracking>(null);
const canvas = useRef<HTMLCanvasElement>(null), video = useRef<HTMLVideoElement>(null);
const camera = useRef<CameraCapture | null>(null), microphone = useRef<MicrophoneCapture | null>(null);
const avatarRef = useRef<MeshAvatar | null>(null);
const lightRef = useRef(settings.lighting); lightRef.current = settings.lighting;
const overlaysRef = useRef(settings.overlays); overlaysRef.current = settings.overlays;
const pose = useRef(new FacePose());
const controls = useRef({ options, gain, micState, cameraState }); controls.current = { options, gain, micState, cameraState };
const refreshDevices = () => { void navigator.mediaDevices?.enumerateDevices().then(setDevices).catch(() => undefined); };
useEffect(() => {
const capture = new CameraCapture(video.current!, (result, now) => pose.current.update(result, now), state => { setCameraState(state); if (state !== 'running') setTracking(false); });
const mic = new MicrophoneCapture(setMicState);
camera.current = capture; microphone.current = mic;
refreshDevices(); navigator.mediaDevices?.addEventListener('devicechange', refreshDevices);
const stop = () => { capture.stop(); mic.stop(); pose.current.reset(); };
window.addEventListener('pagehide', stop);
return () => { stop(); navigator.mediaDevices?.removeEventListener('devicechange', refreshDevices); window.removeEventListener('pagehide', stop); };
}, []);
useEffect(() => {
let cancelled = false, view: Awaited<ReturnType<typeof createAvatarView>> | undefined;
const clock = new Worker(new URL('./clock-worker.ts', import.meta.url), { type: 'module' });
clock.onmessage = () => {
const now = performance.now();
view?.updateIfStalled(now);
setBackgroundStatus(camera.current?.backgroundStatus(document.visibilityState === 'hidden' || !!view?.paintPaused(now), now) ?? null);
};
const send = createLiveSender(settings.project);
setViewState('loading'); setSupport(null); supportRef.current = null;
void createAvatarView(canvas.current!, settings, (avatar, now, dt) => {
const control = controls.current, sampled = pose.current.sample(now, dt, { ...control.options, armsTracking: supportRef.current?.arms ?? false });
setTracking(sampled.tracking);
avatar.setAutoIdle(!sampled.tracking); avatar.setAutoMotion(!sampled.tracking);
avatar.setParameters(sampled.params, sampled.weight, sampled.parameterWeights);
const micOn = control.micState === 'micOn';
avatar.setSpeaking(micOn); avatar.setVoiceLevel(micOn ? microphone.current?.level(control.gain) ?? 0 : 0);
}, (avatar, now) => {
// An expression alone also drives the stream view, including its fade back to neutral.
if (controls.current.cameraState === 'running' || controls.current.micState === 'micOn' || avatar.getExpression().active) send(avatar.getParameters(), now);
}).then(value => { if (cancelled) value.destroy(); else { view = value; avatarRef.current = value.avatar; value.avatar.setLighting(lightRef.current); value.avatar.setExpression(expressionRef.current); value.avatar.setOverlays(overlaysRef.current); setAvailableOverlays(value.overlays); supportRef.current = value.support; setSupport(value.support); setViewState('ready'); } }).catch(() => { if (!cancelled) setViewState('projectError'); });
return () => { cancelled = true; clock.terminate(); view?.destroy(); avatarRef.current = null; };
}, [settings.project, settings.fit]);
useEffect(() => {
saveLighting(settings.project, settings.lighting);
avatarRef.current?.setLighting(settings.lighting);
}, [settings.project, settings.lighting]);
useEffect(() => {
let sent: typeof settings.lighting | undefined;
// Throttle continuous drags and deliver the final position even after dragging stops.
const timer = setInterval(() => {
if (sent === lightRef.current) return;
sent = lightRef.current; sendLighting(settings.project, sent);
}, 34);
return () => clearInterval(timer);
}, [settings.project]);
const changeLighting = (lighting: typeof settings.lighting) => setSettings(current => ({ ...current, lighting }));
useEffect(() => {
saveOverlays(settings.project, settings.overlays);
avatarRef.current?.setOverlays(settings.overlays);
sendOverlays(settings.project, settings.overlays);
}, [settings.project, settings.overlays]);
const toggleOverlay = (name: string) => setSettings(current => ({ ...current,
overlays: current.overlays.includes(name) ? current.overlays.filter(item => item !== name) : [...current.overlays, name] }));
useEffect(() => { avatarRef.current?.setExpression(expression); }, [expression]);
useEffect(() => watchDuplicateLivePages(settings.project, setDuplicatePage), [settings.project]);
useEffect(() => receiveExpression(settings.project, setExpression), [settings.project]);
// Empty slots are ignored, so a shortcut for an unsaved preset changes nothing.
const applyPreset = (preset: number) => { const value = presetsRef.current[preset - 1]; if (value) changeLighting({ ...value }); };
const savePreset = (index: number) => {
const next = presets.map((value, i) => i === index ? { ...settings.lighting } : value);
setPresets(next); saveLightingPresets(settings.project, next);
};
const applyPresetRef = useRef(applyPreset); applyPresetRef.current = applyPreset;
useEffect(() => receiveLightingPreset(settings.project, preset => applyPresetRef.current(preset)), [settings.project]);
const loadRemote = (method: 'GET' | 'POST') => fetch(EXPRESSION_TOKEN_PATH, { method, headers: { 'x-studio-request': '1' } })
.then(response => response.ok ? response.json() : null)
.then((data: { token?: unknown; path?: unknown } | null) => setRemote(typeof data?.token === 'string' && typeof data.path === 'string' ? { url: `${location.origin}${data.path}`, token: data.token } : null))
.catch(() => setRemote(null));
useEffect(() => { void loadRemote('GET'); }, []);
const copyRemote = (kind: 'url' | 'token', value: string) => { void navigator.clipboard.writeText(value).then(() => { setRemoteCopied(kind); setTimeout(() => setRemoteCopied(null), 1500); }).catch(() => undefined); };
useEffect(() => {
const onKey = (event: KeyboardEvent) => {
const target = event.target as HTMLElement | null;
if (event.repeat || event.altKey || event.ctrlKey || event.metaKey || target?.closest('input, select, textarea, [contenteditable="true"]')) return;
// Shift with a digit applies a lighting preset. The physical key is used because Shift
// changes the typed character (Shift+1 is "!" on common layouts).
if (event.shiftKey) {
const preset = /^Digit([1-8])$/.exec(event.code);
if (preset) { event.preventDefault(); applyPresetRef.current(Number(preset[1])); }
return;
}
const name = LIVE_EXPRESSIONS[Number(event.key) - 1];
if (!/^[1-8]$/.test(event.key) || !name) return;
event.preventDefault(); setExpression(name);
};
window.addEventListener('keydown', onKey);
return () => window.removeEventListener('keydown', onKey);
}, []);
const cameraActive = cameraState === 'starting' || cameraState === 'running';
const micActive = micState === 'micStarting' || micState === 'micOn';
const url = streamUrl(settings, location.origin);
const followed = support ? [t.upperBody, ...(support.legs ? [t.legs] : []), ...(support.arms ? [t.arms] : [])] : [];
const missing = support ? [...(!support.legs ? [t.legs] : []), ...(!support.arms ? [t.arms] : [])] : [];
const status = cameraState === 'running' ? tracking ? 'tracking' : 'lost' : cameraState;
return <main className="live-app">
<header className="live-header"><div><a href="/">{t.back}</a><h1>Mesh Avatar Studio <span>{t.title}</span></h1></div>
<div className="live-languages">{(['en', 'ja', 'zh'] as const).map(lang => <button key={lang} aria-pressed={language === lang} onClick={() => setLanguage(lang)}>{({ en: 'English', ja: '日本語', zh: '简体中文' })[lang]}</button>)}</div>
</header>
<div className="live-layout"><section className="live-view"><div className="live-preview checkerboard lighting-preview" style={{ backgroundColor: settings.background, backgroundImage: settings.background === 'transparent' ? undefined : 'none' }}>
<canvas ref={canvas} data-testid="live-avatar" />
{lightingOpen && <LightHandle value={settings.lighting} onChange={changeLighting} language={language} />}
</div><p role="status" className={viewState === 'projectError' ? 'live-error' : ''}>{t[viewState]} · {settings.project}</p></section>
<aside className="live-controls">
{duplicatePage && <p role="alert" className="live-warning" data-testid="duplicate-live-page">{t.duplicatePage}</p>}
<section><h2>{t.camera}</h2><label>{t.device}<select aria-label={t.camera} value={cameraId} disabled={cameraActive} onChange={event => setCameraId(event.target.value)}><option value="">{t.defaultDevice}</option>{devices.filter(device => device.kind === 'videoinput' && device.deviceId).map((device, i) => <option key={device.deviceId} value={device.deviceId}>{device.label || `${t.camera} ${i + 1}`}</option>)}</select></label>
<div className="live-buttons"><button className="live-primary" disabled={!cameraActive && viewState !== 'ready'} onClick={() => {
setCalibrated(false); pose.current.reset();
if (cameraActive) camera.current?.stop(); else void camera.current?.start(cameraId).then(refreshDevices);
}}>{cameraActive ? t.stop : t.start}</button><button disabled={!tracking} onClick={() => setCalibrated(pose.current.calibrate(performance.now()))}>{t.calibrate}</button></div>
<p role="status" data-testid="tracking-status" data-state={status}>{t[status]}</p>
{backgroundStatus && <p role="alert" className="live-error" data-testid="background-status">{t[backgroundStatus]}</p>}
<small>{calibrated ? t.calibrated : t.calibrateHint}</small>
<label className="live-check"><input data-testid="body-tracking-toggle" type="checkbox" checked={!!options.bodyTracking} onChange={event => {
const enabled = event.target.checked; setOptions(current => ({ ...current, bodyTracking: enabled })); camera.current?.setBodyTracking(enabled);
}} />{t.bodyTracking}</label>
{support && <div className="live-body-support" data-testid="body-support">
<small data-testid="body-follows">{t.follows}: {followed.join(' · ')}</small>
{missing.length > 0 && <small data-testid="body-missing">{t.missingParts.replace('{parts}', missing.join(' · '))}</small>}
</div>}<small>{t.bodyTrackingHint}</small>
<label className="live-check"><input type="checkbox" checked={options.mirror} onChange={event => setOptions(current => ({ ...current, mirror: event.target.checked }))} />{t.mirror}</label>
<label>{t.sensitivity}<input type="range" min="0.25" max="2" step="0.05" value={options.sensitivity} onChange={event => setOptions(current => ({ ...current, sensitivity: Number(event.target.value) }))} /></label>
<label>{t.smoothing}<input type="range" min="0" max="1" step="0.05" value={options.smoothing} onChange={event => setOptions(current => ({ ...current, smoothing: Number(event.target.value) }))} /></label>
<label className="live-check"><input type="checkbox" checked={showCamera} onChange={event => setShowCamera(event.target.checked)} />{t.cameraPreview}</label>
<video ref={video} autoPlay muted playsInline className={showCamera ? 'camera-preview' : 'camera-preview camera-hidden'} style={{ transform: options.mirror ? 'scaleX(-1)' : undefined }} aria-label={t.cameraPreview} />
</section>
<section><h2>{t.microphone}</h2><label className="live-check"><input type="checkbox" checked={micActive} disabled={!micActive && viewState !== 'ready'} onChange={event => { if (event.target.checked) void microphone.current?.start(micId).then(refreshDevices); else microphone.current?.stop(); }} />{t.microphone}</label>
<select aria-label={t.microphone} value={micId} disabled={micActive} onChange={event => setMicId(event.target.value)}><option value="">{t.defaultDevice}</option>{devices.filter(device => device.kind === 'audioinput' && device.deviceId).map((device, i) => <option key={device.deviceId} value={device.deviceId}>{device.label || `${t.microphone} ${i + 1}`}</option>)}</select>
<label>{t.gain}<input type="range" min="0.25" max="5" step="0.05" value={gain} onChange={event => setGain(Number(event.target.value))} /></label><small role="status">{t[micState]}</small>
</section>
<section><h2>{t.background}</h2><select aria-label={t.background} value={settings.background} onChange={event => setSettings(current => ({ ...current, background: backgroundColor(event.target.value) }))}>
<option value="transparent">{t.transparent}</option><option value="#00ff00">{t.green}</option><option value="#0000ff">{t.blue}</option>{!['transparent', '#00ff00', '#0000ff'].includes(settings.background) && <option value={settings.background}>{t.custom}</option>}
</select><small className="background-hint" data-testid="background-hint">{settings.background === 'transparent' ? t.bgHintTransparent : ['#00ff00', '#0000ff'].includes(settings.background) ? t.bgHintChroma : t.bgHintCustom}</small>
<label>{t.custom}<input type="color" value={settings.background === 'transparent' ? '#ffffff' : settings.background} onChange={event => setSettings(current => ({ ...current, background: event.target.value }))} /></label>
<label>{t.fit}<select value={settings.fit} onChange={event => setSettings(current => ({ ...current, fit: event.target.value as 'contain' | 'cover' }))}><option value="contain">{t.contain}</option><option value="cover">{t.cover}</option></select></label>
<div className="live-buttons"><button className="live-primary" onClick={() => { void navigator.clipboard.writeText(url).then(() => setCopyState('copied')).catch(() => setCopyState('copyError')); }}>{t.obs}</button>
<a className="live-open" href={url} target="_blank" rel="noreferrer">{t.openStream}</a></div>
{copyState && <p role="status">{t[copyState]}</p>}<input className="obs-url" aria-label={t.obs} readOnly value={url} onFocus={event => event.target.select()} /><small>{t.obsHelp}</small>
</section>
<details className="live-expression" data-testid="expression-section">
<summary><Icon name="face" />{t.expression}{expression !== 'neutral' && <span className="expression-on">{t[expression]}</span>}</summary>
<div className="live-expression-body"><div className="expression-buttons" role="group" aria-label={t.expression}>
{LIVE_EXPRESSIONS.map((name, i) => <button key={name} type="button" aria-pressed={expression === name} aria-keyshortcuts={String(i + 1)} onClick={() => setExpression(name)}><kbd>{i + 1}</kbd>{t[name]}</button>)}
</div><small>{t.expressionHint}</small>
<h3>{t.remoteTitle}</h3><p className="remote-help">{t.remoteHelp}</p>
{remote ? <>
<label>{t.remoteUrl}<span className="remote-field"><input readOnly value={remote.url} onFocus={event => event.target.select()} /><button type="button" onClick={() => copyRemote('url', remote.url)}>{remoteCopied === 'url' ? t.copiedShort : t.copy}</button></span></label>
<label>{t.remoteToken}<span className="remote-field"><input readOnly value={remote.token} onFocus={event => event.target.select()} /><button type="button" onClick={() => copyRemote('token', remote.token)}>{remoteCopied === 'token' ? t.copiedShort : t.copy}</button></span></label>
<small>{t.remoteNames}</small>
<div className="remote-actions"><button type="button" onClick={() => { void loadRemote('POST'); }}>{t.regenerate}</button><small>{t.regenerateHelp}</small></div>
</> : <small>{t.remoteUnavailable}</small>}
<a href="https://github.com/shinshin86/mesh-avatar-studio/blob/main/docs/reference.md#switching-expressions-from-other-apps" target="_blank" rel="noreferrer">{t.remoteDocs}</a></div>
</details>
{availableOverlays.length > 0 && <details className="live-overlays" data-testid="overlay-section" open>
<summary><Icon name="eye" />{t.overlays}{settings.overlays.some(name => availableOverlays.includes(name)) && <span className="overlays-on">ON</span>}</summary>
<div className="overlay-buttons" role="group" aria-label={t.overlays}>
{availableOverlays.map(name => <button key={name} type="button" aria-pressed={settings.overlays.includes(name)} onClick={() => toggleOverlay(name)}>{name}</button>)}
</div><small>{t.overlaysHint}</small>
</details>}
<details className="live-lighting" data-testid="lighting-section" onToggle={event => setLightingOpen(event.currentTarget.open)}>
<summary><Icon name="light" />{lightingText[language].title}{settings.lighting.enabled && <span className="lighting-on">ON</span>}</summary>
<div className="lighting-presets"><h3>{t.presets}</h3>
<div className="preset-slots" role="group" aria-label={t.presets}>
{presets.map((value, i) => <div className="preset-slot" key={i}>
{/* The swatch shows the light and ambient colours; the slot number is in the key hint. */}
<button type="button" disabled={!value} aria-label={`${t.preset} ${i + 1}${value ? '' : ` (${t.presetEmpty})`}`} aria-pressed={!!value && sameLighting(value, settings.lighting)} aria-keyshortcuts={`Shift+${i + 1}`} onClick={() => applyPreset(i + 1)}>
<kbd>⇧{i + 1}</kbd>{value ? <span className={`preset-swatch${value.enabled ? '' : ' off'}`} style={value.enabled ? { background: `linear-gradient(135deg, ${colorHex(value.color)} 50%, ${colorHex(value.ambientColor)} 50%)` } : undefined} /> : t.presetEmpty}</button>
<button type="button" className="preset-save" aria-label={`${t.presetSave}: ${t.preset} ${i + 1}`} onClick={() => savePreset(i)}>{t.presetSave}</button>
</div>)}
</div><small>{t.presetHint}</small>
{remote && <label>{t.presetUrl}<span className="remote-field"><input readOnly value={`${location.origin}${LIGHTING_PRESET_PATH}`} onFocus={event => event.target.select()} /></span></label>}
</div>
<LightingControls value={settings.lighting} onChange={changeLighting} language={language} />
</details>
<p className="live-privacy">{t.privacy}</p>
</aside></div>
</main>;
}