Skip to content

Turn one illustration into an animated 2D mesh avatar with a coding agent and a local editor

21.2 KB
Raw Blame
import { LightingControls, LightHandle, lightingText } from '../lighting/Controls';
import { colorHex, LIVE_EXPRESSIONS, type LiveExpression, type MeshAvatar } from 'mesh-avatar';
import { loadLighting, saveLighting } from '../lighting/storage';
import { loadLightingPresets, sameLighting, saveLightingPresets } from '../lighting/presets';
import { useEffect, useRef, useState } from 'react';
import { useI18n } from '../editor/i18n';
import { createAvatarView } from './avatar-view';
import { viewSettings, streamUrl, backgroundColor, loadOverlays, saveOverlays } from './settings';
import { FacePose, type TrackingOptions } from './tracking';
import { CameraCapture, MicrophoneCapture, type CameraState, type MicState, type BackgroundTracking } from './media';
import { liveText } from './i18n';
import { Icon } from '../editor/Icon';
import { createLiveSender, receiveExpression, receiveLightingPreset, sendLighting, sendOverlays, watchDuplicateLivePages } from './relay';
import { EXPRESSION_TOKEN_PATH, LIGHTING_PRESET_PATH } from './expression-protocol';

export function LiveApp() {
  const { language, setLanguage } = useI18n(), t = liveText[language];
  const [settings, setSettings] = useState(() => { const view = viewSettings(location.search); return { ...view, lighting: view.lighting ?? loadLighting(view.project), overlays: view.overlays ?? loadOverlays(view.project) }; });
  // Overlay layers the loaded avatar has (built/overlays), listed once it has loaded.
  const [availableOverlays, setAvailableOverlays] = useState<string[]>([]);
  const [support, setSupport] = useState<{ legs: boolean; arms: boolean } | null>(null);
  const supportRef = useRef(support); supportRef.current = support;
  const [lightingOpen, setLightingOpen] = useState(false);
  const [presets, setPresets] = useState(() => loadLightingPresets(settings.project));
  const presetsRef = useRef(presets); presetsRef.current = presets;
  const [duplicatePage, setDuplicatePage] = useState(false);
  const [expression, setExpression] = useState<LiveExpression>('neutral');
  const expressionRef = useRef(expression); expressionRef.current = expression;
  const [remote, setRemote] = useState<{ url: string; token: string } | null>(null), [remoteCopied, setRemoteCopied] = useState<'url' | 'token' | null>(null);
  const [options, setOptions] = useState<TrackingOptions>({ mirror: true, sensitivity: 1, smoothing: 0.35 });
  const [cameraState, setCameraState] = useState<CameraState>('stopped');
  const [micState, setMicState] = useState<MicState>('micOff');
  const [tracking, setTracking] = useState(false), [showCamera, setShowCamera] = useState(true);
  const [devices, setDevices] = useState<MediaDeviceInfo[]>([]), [cameraId, setCameraId] = useState(''), [micId, setMicId] = useState('');
  const [gain, setGain] = useState(1), [calibrated, setCalibrated] = useState(false);
  const [copyState, setCopyState] = useState<'copied' | 'copyError' | null>(null);
  const [viewState, setViewState] = useState<'loading' | 'ready' | 'projectError'>('loading');
  const [backgroundStatus, setBackgroundStatus] = useState<BackgroundTracking>(null);
  const canvas = useRef<HTMLCanvasElement>(null), video = useRef<HTMLVideoElement>(null);
  const camera = useRef<CameraCapture | null>(null), microphone = useRef<MicrophoneCapture | null>(null);
  const avatarRef = useRef<MeshAvatar | null>(null);
  const lightRef = useRef(settings.lighting); lightRef.current = settings.lighting;
  const overlaysRef = useRef(settings.overlays); overlaysRef.current = settings.overlays;
  const pose = useRef(new FacePose());
  const controls = useRef({ options, gain, micState, cameraState }); controls.current = { options, gain, micState, cameraState };
  const refreshDevices = () => { void navigator.mediaDevices?.enumerateDevices().then(setDevices).catch(() => undefined); };
  useEffect(() => {
    const capture = new CameraCapture(video.current!, (result, now) => pose.current.update(result, now), state => { setCameraState(state); if (state !== 'running') setTracking(false); });
    const mic = new MicrophoneCapture(setMicState);
    camera.current = capture; microphone.current = mic;
    refreshDevices(); navigator.mediaDevices?.addEventListener('devicechange', refreshDevices);
    const stop = () => { capture.stop(); mic.stop(); pose.current.reset(); };
    window.addEventListener('pagehide', stop);
    return () => { stop(); navigator.mediaDevices?.removeEventListener('devicechange', refreshDevices); window.removeEventListener('pagehide', stop); };
  }, []);
  useEffect(() => {
    let cancelled = false, view: Awaited<ReturnType<typeof createAvatarView>> | undefined;
    const clock = new Worker(new URL('./clock-worker.ts', import.meta.url), { type: 'module' });
    clock.onmessage = () => {
      const now = performance.now();
      view?.updateIfStalled(now);
      setBackgroundStatus(camera.current?.backgroundStatus(document.visibilityState === 'hidden' || !!view?.paintPaused(now), now) ?? null);
    };
    const send = createLiveSender(settings.project);
    setViewState('loading'); setSupport(null); supportRef.current = null;
    void createAvatarView(canvas.current!, settings, (avatar, now, dt) => {
      const control = controls.current, sampled = pose.current.sample(now, dt, { ...control.options, armsTracking: supportRef.current?.arms ?? false });
      setTracking(sampled.tracking);
      avatar.setAutoIdle(!sampled.tracking); avatar.setAutoMotion(!sampled.tracking);
      avatar.setParameters(sampled.params, sampled.weight, sampled.parameterWeights);
      const micOn = control.micState === 'micOn';
      avatar.setSpeaking(micOn); avatar.setVoiceLevel(micOn ? microphone.current?.level(control.gain) ?? 0 : 0);
    }, (avatar, now) => {
      // An expression alone also drives the stream view, including its fade back to neutral.
      if (controls.current.cameraState === 'running' || controls.current.micState === 'micOn' || avatar.getExpression().active) send(avatar.getParameters(), now);
    }).then(value => { if (cancelled) value.destroy(); else { view = value; avatarRef.current = value.avatar; value.avatar.setLighting(lightRef.current); value.avatar.setExpression(expressionRef.current); value.avatar.setOverlays(overlaysRef.current); setAvailableOverlays(value.overlays); supportRef.current = value.support; setSupport(value.support); setViewState('ready'); } }).catch(() => { if (!cancelled) setViewState('projectError'); });
    return () => { cancelled = true; clock.terminate(); view?.destroy(); avatarRef.current = null; };
  }, [settings.project, settings.fit]);
  useEffect(() => {
    saveLighting(settings.project, settings.lighting);
    avatarRef.current?.setLighting(settings.lighting);
  }, [settings.project, settings.lighting]);
  useEffect(() => {
    let sent: typeof settings.lighting | undefined;
    // Throttle continuous drags and deliver the final position even after dragging stops.
    const timer = setInterval(() => {
      if (sent === lightRef.current) return;
      sent = lightRef.current; sendLighting(settings.project, sent);
    }, 34);
    return () => clearInterval(timer);
  }, [settings.project]);
  const changeLighting = (lighting: typeof settings.lighting) => setSettings(current => ({ ...current, lighting }));
  useEffect(() => {
    saveOverlays(settings.project, settings.overlays);
    avatarRef.current?.setOverlays(settings.overlays);
    sendOverlays(settings.project, settings.overlays);
  }, [settings.project, settings.overlays]);
  const toggleOverlay = (name: string) => setSettings(current => ({ ...current,
    overlays: current.overlays.includes(name) ? current.overlays.filter(item => item !== name) : [...current.overlays, name] }));
  useEffect(() => { avatarRef.current?.setExpression(expression); }, [expression]);
  useEffect(() => watchDuplicateLivePages(settings.project, setDuplicatePage), [settings.project]);
  useEffect(() => receiveExpression(settings.project, setExpression), [settings.project]);
  // Empty slots are ignored, so a shortcut for an unsaved preset changes nothing.
  const applyPreset = (preset: number) => { const value = presetsRef.current[preset - 1]; if (value) changeLighting({ ...value }); };
  const savePreset = (index: number) => {
    const next = presets.map((value, i) => i === index ? { ...settings.lighting } : value);
    setPresets(next); saveLightingPresets(settings.project, next);
  };
  const applyPresetRef = useRef(applyPreset); applyPresetRef.current = applyPreset;
  useEffect(() => receiveLightingPreset(settings.project, preset => applyPresetRef.current(preset)), [settings.project]);
  const loadRemote = (method: 'GET' | 'POST') => fetch(EXPRESSION_TOKEN_PATH, { method, headers: { 'x-studio-request': '1' } })
    .then(response => response.ok ? response.json() : null)
    .then((data: { token?: unknown; path?: unknown } | null) => setRemote(typeof data?.token === 'string' && typeof data.path === 'string' ? { url: `${location.origin}${data.path}`, token: data.token } : null))
    .catch(() => setRemote(null));
  useEffect(() => { void loadRemote('GET'); }, []);
  const copyRemote = (kind: 'url' | 'token', value: string) => { void navigator.clipboard.writeText(value).then(() => { setRemoteCopied(kind); setTimeout(() => setRemoteCopied(null), 1500); }).catch(() => undefined); };
  useEffect(() => {
    const onKey = (event: KeyboardEvent) => {
      const target = event.target as HTMLElement | null;
      if (event.repeat || event.altKey || event.ctrlKey || event.metaKey || target?.closest('input, select, textarea, [contenteditable="true"]')) return;
      // Shift with a digit applies a lighting preset. The physical key is used because Shift
      // changes the typed character (Shift+1 is "!" on common layouts).
      if (event.shiftKey) {
        const preset = /^Digit([1-8])$/.exec(event.code);
        if (preset) { event.preventDefault(); applyPresetRef.current(Number(preset[1])); }
        return;
      }
      const name = LIVE_EXPRESSIONS[Number(event.key) - 1];
      if (!/^[1-8]$/.test(event.key) || !name) return;
      event.preventDefault(); setExpression(name);
    };
    window.addEventListener('keydown', onKey);
    return () => window.removeEventListener('keydown', onKey);
  }, []);
  const cameraActive = cameraState === 'starting' || cameraState === 'running';
  const micActive = micState === 'micStarting' || micState === 'micOn';
  const url = streamUrl(settings, location.origin);
  const followed = support ? [t.upperBody, ...(support.legs ? [t.legs] : []), ...(support.arms ? [t.arms] : [])] : [];
  const missing = support ? [...(!support.legs ? [t.legs] : []), ...(!support.arms ? [t.arms] : [])] : [];
  const status = cameraState === 'running' ? tracking ? 'tracking' : 'lost' : cameraState;
  return <main className="live-app">
    <header className="live-header"><div><a href="/">{t.back}</a><h1>Mesh Avatar Studio <span>{t.title}</span></h1></div>
      <div className="live-languages">{(['en', 'ja', 'zh'] as const).map(lang => <button key={lang} aria-pressed={language === lang} onClick={() => setLanguage(lang)}>{({ en: 'English', ja: '日本語', zh: '简体中文' })[lang]}</button>)}</div>
    </header>
    <div className="live-layout"><section className="live-view"><div className="live-preview checkerboard lighting-preview" style={{ backgroundColor: settings.background, backgroundImage: settings.background === 'transparent' ? undefined : 'none' }}>
      <canvas ref={canvas} data-testid="live-avatar" />
      {lightingOpen && <LightHandle value={settings.lighting} onChange={changeLighting} language={language} />}
    </div><p role="status" className={viewState === 'projectError' ? 'live-error' : ''}>{t[viewState]} · {settings.project}</p></section>
    <aside className="live-controls">
      {duplicatePage && <p role="alert" className="live-warning" data-testid="duplicate-live-page">{t.duplicatePage}</p>}
      <section><h2>{t.camera}</h2><label>{t.device}<select aria-label={t.camera} value={cameraId} disabled={cameraActive} onChange={event => setCameraId(event.target.value)}><option value="">{t.defaultDevice}</option>{devices.filter(device => device.kind === 'videoinput' && device.deviceId).map((device, i) => <option key={device.deviceId} value={device.deviceId}>{device.label || `${t.camera} ${i + 1}`}</option>)}</select></label>
        <div className="live-buttons"><button className="live-primary" disabled={!cameraActive && viewState !== 'ready'} onClick={() => {
          setCalibrated(false); pose.current.reset();
          if (cameraActive) camera.current?.stop(); else void camera.current?.start(cameraId).then(refreshDevices);
        }}>{cameraActive ? t.stop : t.start}</button><button disabled={!tracking} onClick={() => setCalibrated(pose.current.calibrate(performance.now()))}>{t.calibrate}</button></div>
        <p role="status" data-testid="tracking-status" data-state={status}>{t[status]}</p>
        {backgroundStatus && <p role="alert" className="live-error" data-testid="background-status">{t[backgroundStatus]}</p>}
        <small>{calibrated ? t.calibrated : t.calibrateHint}</small>
        <label className="live-check"><input data-testid="body-tracking-toggle" type="checkbox" checked={!!options.bodyTracking} onChange={event => {
          const enabled = event.target.checked; setOptions(current => ({ ...current, bodyTracking: enabled })); camera.current?.setBodyTracking(enabled);
        }} />{t.bodyTracking}</label>
        {support && <div className="live-body-support" data-testid="body-support">
          <small data-testid="body-follows">{t.follows}: {followed.join(' · ')}</small>
          {missing.length > 0 && <small data-testid="body-missing">{t.missingParts.replace('{parts}', missing.join(' · '))}</small>}
        </div>}<small>{t.bodyTrackingHint}</small>
        <label className="live-check"><input type="checkbox" checked={options.mirror} onChange={event => setOptions(current => ({ ...current, mirror: event.target.checked }))} />{t.mirror}</label>
        <label>{t.sensitivity}<input type="range" min="0.25" max="2" step="0.05" value={options.sensitivity} onChange={event => setOptions(current => ({ ...current, sensitivity: Number(event.target.value) }))} /></label>
        <label>{t.smoothing}<input type="range" min="0" max="1" step="0.05" value={options.smoothing} onChange={event => setOptions(current => ({ ...current, smoothing: Number(event.target.value) }))} /></label>
        <label className="live-check"><input type="checkbox" checked={showCamera} onChange={event => setShowCamera(event.target.checked)} />{t.cameraPreview}</label>
        <video ref={video} autoPlay muted playsInline className={showCamera ? 'camera-preview' : 'camera-preview camera-hidden'} style={{ transform: options.mirror ? 'scaleX(-1)' : undefined }} aria-label={t.cameraPreview} />
      </section>
      <section><h2>{t.microphone}</h2><label className="live-check"><input type="checkbox" checked={micActive} disabled={!micActive && viewState !== 'ready'} onChange={event => { if (event.target.checked) void microphone.current?.start(micId).then(refreshDevices); else microphone.current?.stop(); }} />{t.microphone}</label>
        <select aria-label={t.microphone} value={micId} disabled={micActive} onChange={event => setMicId(event.target.value)}><option value="">{t.defaultDevice}</option>{devices.filter(device => device.kind === 'audioinput' && device.deviceId).map((device, i) => <option key={device.deviceId} value={device.deviceId}>{device.label || `${t.microphone} ${i + 1}`}</option>)}</select>
        <label>{t.gain}<input type="range" min="0.25" max="5" step="0.05" value={gain} onChange={event => setGain(Number(event.target.value))} /></label><small role="status">{t[micState]}</small>
      </section>
      <section><h2>{t.background}</h2><select aria-label={t.background} value={settings.background} onChange={event => setSettings(current => ({ ...current, background: backgroundColor(event.target.value) }))}>
        <option value="transparent">{t.transparent}</option><option value="#00ff00">{t.green}</option><option value="#0000ff">{t.blue}</option>{!['transparent', '#00ff00', '#0000ff'].includes(settings.background) && <option value={settings.background}>{t.custom}</option>}
      </select><small className="background-hint" data-testid="background-hint">{settings.background === 'transparent' ? t.bgHintTransparent : ['#00ff00', '#0000ff'].includes(settings.background) ? t.bgHintChroma : t.bgHintCustom}</small>
        <label>{t.custom}<input type="color" value={settings.background === 'transparent' ? '#ffffff' : settings.background} onChange={event => setSettings(current => ({ ...current, background: event.target.value }))} /></label>
        <label>{t.fit}<select value={settings.fit} onChange={event => setSettings(current => ({ ...current, fit: event.target.value as 'contain' | 'cover' }))}><option value="contain">{t.contain}</option><option value="cover">{t.cover}</option></select></label>
        <div className="live-buttons"><button className="live-primary" onClick={() => { void navigator.clipboard.writeText(url).then(() => setCopyState('copied')).catch(() => setCopyState('copyError')); }}>{t.obs}</button>
        <a className="live-open" href={url} target="_blank" rel="noreferrer">{t.openStream}</a></div>
        {copyState && <p role="status">{t[copyState]}</p>}<input className="obs-url" aria-label={t.obs} readOnly value={url} onFocus={event => event.target.select()} /><small>{t.obsHelp}</small>
      </section>
      <details className="live-expression" data-testid="expression-section">
        <summary><Icon name="face" />{t.expression}{expression !== 'neutral' && <span className="expression-on">{t[expression]}</span>}</summary>
        <div className="live-expression-body"><div className="expression-buttons" role="group" aria-label={t.expression}>
          {LIVE_EXPRESSIONS.map((name, i) => <button key={name} type="button" aria-pressed={expression === name} aria-keyshortcuts={String(i + 1)} onClick={() => setExpression(name)}><kbd>{i + 1}</kbd>{t[name]}</button>)}
        </div><small>{t.expressionHint}</small>
        <h3>{t.remoteTitle}</h3><p className="remote-help">{t.remoteHelp}</p>
        {remote ? <>
          <label>{t.remoteUrl}<span className="remote-field"><input readOnly value={remote.url} onFocus={event => event.target.select()} /><button type="button" onClick={() => copyRemote('url', remote.url)}>{remoteCopied === 'url' ? t.copiedShort : t.copy}</button></span></label>
          <label>{t.remoteToken}<span className="remote-field"><input readOnly value={remote.token} onFocus={event => event.target.select()} /><button type="button" onClick={() => copyRemote('token', remote.token)}>{remoteCopied === 'token' ? t.copiedShort : t.copy}</button></span></label>
          <small>{t.remoteNames}</small>
          <div className="remote-actions"><button type="button" onClick={() => { void loadRemote('POST'); }}>{t.regenerate}</button><small>{t.regenerateHelp}</small></div>
        </> : <small>{t.remoteUnavailable}</small>}
        <a href="https://github.com/shinshin86/mesh-avatar-studio/blob/main/docs/reference.md#switching-expressions-from-other-apps" target="_blank" rel="noreferrer">{t.remoteDocs}</a></div>
      </details>
      {availableOverlays.length > 0 && <details className="live-overlays" data-testid="overlay-section" open>
        <summary><Icon name="eye" />{t.overlays}{settings.overlays.some(name => availableOverlays.includes(name)) && <span className="overlays-on">ON</span>}</summary>
        <div className="overlay-buttons" role="group" aria-label={t.overlays}>
          {availableOverlays.map(name => <button key={name} type="button" aria-pressed={settings.overlays.includes(name)} onClick={() => toggleOverlay(name)}>{name}</button>)}
        </div><small>{t.overlaysHint}</small>
      </details>}
      <details className="live-lighting" data-testid="lighting-section" onToggle={event => setLightingOpen(event.currentTarget.open)}>
        <summary><Icon name="light" />{lightingText[language].title}{settings.lighting.enabled && <span className="lighting-on">ON</span>}</summary>
        <div className="lighting-presets"><h3>{t.presets}</h3>
          <div className="preset-slots" role="group" aria-label={t.presets}>
            {presets.map((value, i) => <div className="preset-slot" key={i}>
              {/* The swatch shows the light and ambient colours; the slot number is in the key hint. */}
              <button type="button" disabled={!value} aria-label={`${t.preset} ${i + 1}${value ? '' : ` (${t.presetEmpty})`}`} aria-pressed={!!value && sameLighting(value, settings.lighting)} aria-keyshortcuts={`Shift+${i + 1}`} onClick={() => applyPreset(i + 1)}>
                <kbd>⇧{i + 1}</kbd>{value ? <span className={`preset-swatch${value.enabled ? '' : ' off'}`} style={value.enabled ? { background: `linear-gradient(135deg, ${colorHex(value.color)} 50%, ${colorHex(value.ambientColor)} 50%)` } : undefined} /> : t.presetEmpty}</button>
              <button type="button" className="preset-save" aria-label={`${t.presetSave}: ${t.preset} ${i + 1}`} onClick={() => savePreset(i)}>{t.presetSave}</button>
            </div>)}
          </div><small>{t.presetHint}</small>
          {remote && <label>{t.presetUrl}<span className="remote-field"><input readOnly value={`${location.origin}${LIGHTING_PRESET_PATH}`} onFocus={event => event.target.select()} /></span></label>}
        </div>
        <LightingControls value={settings.lighting} onChange={changeLighting} language={language} />
      </details>
      <p className="live-privacy">{t.privacy}</p>
    </aside></div>
  </main>;
}