Skip to content

Type a motion, watch a rigged character perform it: fast video model + GVHMR + live Mixamo retargeting

JavaScript
40.5 KB
Raw Blame
// Prompt2Rig — prompt → H3 Max Turbo video → live pose (MediaPipe) → Mixamo rig,
// with an optional GVHMR "HQ" pass on the local GPU that swaps in when ready.
import * as THREE from 'three';
import { GLTFLoader } from 'three/addons/loaders/GLTFLoader.js';
import { FBXLoader } from 'three/addons/loaders/FBXLoader.js';
import { GLTFExporter } from 'three/addons/exporters/GLTFExporter.js';
import { FilesetResolver, PoseLandmarker, DrawingUtils } from '@mediapipe/tasks-vision';
import { createStage } from './stage.js';
import { HAND_LIMITS, Rig, OneEuro, Track, trackToClip, blendSnapshots, smplGlobals, aaToQuat } from './retarget.js';

const $ = (s) => document.querySelector(s);
// Mixamo-rigged characters from the three.js examples (loaded from the CDN, not redistributed here).
const THREE_MODELS = 'https://cdn.jsdelivr.net/gh/mrdoob/three.js@r169/examples/models/gltf/';
const CHARACTERS = {
  xbot: { name: 'X Bot', url: THREE_MODELS + 'Xbot.glb' },
  michelle: { name: 'Michelle', url: THREE_MODELS + 'Michelle.glb' },
  soldier: { name: 'Soldier', url: THREE_MODELS + 'Soldier.glb' },
};
const VIDEO_MODEL_LABEL = { h3: 'h3 max turbo', flux3: 'flux 3 max' };
const MP_WASM = 'https://cdn.jsdelivr.net/npm/@mediapipe/[email protected]/wasm';
const MP_MODEL = (v) => `https://storage.googleapis.com/mediapipe-models/pose_landmarker/pose_landmarker_${v}/float16/latest/pose_landmarker_${v}.task`;

const PERFORMER_PROMPT =
  'Full-body photograph of one young adult standing in a relaxed neutral pose facing the camera, arms relaxed at the sides, ' +
  'wearing a fitted plain light-grey t-shirt, black slim pants and white sneakers. The whole body is visible from head to feet ' +
  'with empty space above the head and below the feet, the person is centered. Plain seamless ' +
  'light-grey studio backdrop and floor, flat even soft lighting, sharp focus, 35mm photo.';

// Presets chosen to retarget cleanly: facing the camera, limbs clearly separated from the body.
// Avoid flips, spins, floor work and arms crossing/hiding behind the torso — pose estimators
// lose track of which limb is which, and the error shows on the rig.
const PRESETS = [
  'does an energetic hip hop dance with wide, open arm movements',
  'throws boxing jabs and hooks toward the camera',
  'does jumping jacks',
  'mimes a full golf swing',
  'shoots an imaginary basketball jump shot',
  'celebrates a victory, jumping and pumping both fists in the air',
  'does a stiff zombie walk in place with arms stretched forward',
  'does a karate kata with straight punches and front kicks',
  'waves hello with both arms, then takes a bow',
  'does a big superhero landing pose, then stands up proudly',
];

const IDLE_PROMPT = 'stands in place in a relaxed idle pose, breathing and gently shifting weight from foot to foot';
const MAX_IDLE_STREAK = 3;
// Stream mode = continue-from-last-frame + play only once HQ is ready + idle between prompts.
const chainOn = () => settings.chain || settings.stream;
const waitOn = () => settings.hq && hqOnline && (settings.waitHq || settings.stream);

// Shared staging that keeps clips easy to read for pose estimation.
const STAGING = 'They stay roughly in place facing the camera, with both arms clearly visible and held away from the torso, no turning around, no flips. ' +
  'Their entire body, including both feet, stays visible in frame the whole time. Static locked-off camera, no camera movement, no zoom, no cuts.';
const videoPrompt = (action, fromImage) => fromImage
  ? `The person ${action}. ${STAGING} Plain studio background.`
  : `Wide full-body shot of one person wearing a plain grey t-shirt and dark pants in an empty light-grey studio. The person ${action}. ${STAGING}`;

// ---- settings ------------------------------------------------------------------
const DEFAULTS = {
  stream: false, source: 'performer', chain: false, duration: 5, resolution: '480P', hq: true, waitHq: false,
  rootMotion: true, poseModel: 'heavy', smoothing: 0.5, showSkel: true, orientation: 'portrait', performerUrls: {},
  character: 'xbot', videoModel: 'h3', swap: false, cinematic: false, hands: true,
};
const settings = (() => { try { return { ...DEFAULTS, ...JSON.parse(localStorage.getItem('p2r') || '{}') }; } catch { return { ...DEFAULTS }; } })();
const save = () => { try { localStorage.setItem('p2r', JSON.stringify(settings)); } catch {} };

function bindSetting(id, kind = 'value', onChange) {
  const el = $('#' + id);
  if (kind === 'checked') el.checked = !!settings[id]; else el.value = settings[id];
  el.addEventListener('input', () => {
    settings[id] = kind === 'checked' ? el.checked : (el.type === 'range' ? Number(el.value) : el.value);
    save(); onChange?.(settings[id]);
  });
  onChange?.(settings[id]);
}

// ---- log / status -------------------------------------------------------------
function log(msg, cls = '') {
  const el = $('#log'); const line = document.createElement('div');
  if (cls) line.className = cls;
  line.textContent = `${new Date().toLocaleTimeString([], { hour12: false })}  ${msg}`;
  el.appendChild(line); el.scrollTop = el.scrollHeight;
}
function pill(id, text, state) { const p = $('#pill-' + id); p.textContent = text; p.className = 'pill ' + (state || ''); }
function banner(text) { const b = $('#banner'); b.textContent = text || ''; b.classList.toggle('on', !!text); }

async function api(path, body) {
  const res = await fetch(path, { method: body ? 'POST' : 'GET', headers: { 'Content-Type': 'application/json' }, body: body && JSON.stringify(body) });
  const json = await res.json().catch(() => ({}));
  if (!res.ok) throw new Error(json.error || `${path} → ${res.status}`);
  return json;
}

// ---- three.js stage -------------------------------------------------------------
const stage = createStage($('#stage'));
const { scene } = stage;
// Keep the characters centered in the part of the stage the video panel doesn't cover.
new ResizeObserver(() => { const v = $('#vidwrap'); stage.setInset(v.offsetLeft + v.offsetWidth); }).observe($('#vidwrap'));

// ---- character --------------------------------------------------------------------
let model = null, rig = null;

/** Load a Mixamo character: scaled to meters, facing +Z, centered with feet on y=0. */
async function loadModel(src, name) {
  const isFbx = /\.fbx$/i.test(name);
  const obj = await (isFbx ? new FBXLoader() : new GLTFLoader()).loadAsync(src);
  const root = isFbx ? obj : obj.scene;
  if (isFbx) root.scale.setScalar(0.01);         // Mixamo FBX is in centimeters
  root.updateMatrixWorld(true);
  // Retargeting assumes the character faces +Z (toward the camera); some exports face -Z.
  const armX = (side) => {
    let b; root.traverse((o) => { if (!b && o.isBone && o.name.replace(/^mixamorig\d*[:_]?/i, '') === side + 'Arm') b = o; });
    return b?.getWorldPosition(new THREE.Vector3()).x;
  };
  const lx = armX('Left'), rx = armX('Right');
  if (lx != null && rx != null && lx < rx) { root.rotation.y += Math.PI; root.updateMatrixWorld(true); }
  const box = new THREE.Box3().setFromObject(root);
  root.position.y -= box.min.y;
  root.position.x -= (box.min.x + box.max.x) / 2;
  root.position.z -= (box.min.z + box.max.z) / 2;
  root.traverse((o) => { if (o.isMesh) { o.castShadow = true; o.frustumCulled = false; } });
  root.updateMatrixWorld(true);
  return root;
}

async function loadCharacter(src, name = 'X Bot') {
  pill('rig', 'loading ' + name.toLowerCase(), 'busy');
  const root = await loadModel(src, name);
  root.name = 'Character';
  const newRig = new Rig(root);
  if (model) { scene.remove(model); stage.removeContactShadow(model); }
  scene.add(root); stage.contactShadow(root); model = root; rig = newRig;
  // Takes store bone-order-specific data; re-bake HQ data for the new skeleton.
  for (const t of takes) { t.live = new Track(rig.order.length); t.liveDone = false; if (t.hqRaw) t.hq = bakeSmpl(t.hqRaw); }
  liveState ||= { hip0: null, baseY: null, mpp: null, scale: null };
  liveSnap = null; switchFrom = null;
  pill('rig', name.toLowerCase(), 'ok');
  log(`character: ${name} (${rig.order.length} bones, ${rig.height.toFixed(2)} m)`);
  resetCast();
}

// ---- character swap: one character at a time, the next one takes over on every loop -------------
// The selected character stays the hidden "driver" that takes are baked for; whichever character
// is on stage copies its pose each frame, so swapping never needs a re-bake.
const cast = {};        // key → { model, rig } for the non-driver characters, loaded on demand
let onStage = null;     // key of the character currently visible (null = the driver itself)

function resetCast() {
  for (const k of Object.keys(cast)) { scene.remove(cast[k].model); stage.removeContactShadow(cast[k].model); delete cast[k]; }
  onStage = null;
  if (model) model.visible = true;
}

async function swapCharacter() {
  const keys = Object.keys(CHARACTERS);
  const shown = onStage || settings.character;
  const next = keys[(keys.indexOf(shown) + 1) % keys.length];
  if (next === settings.character) { showOnStage(null); return; }
  if (!cast[next]) {
    const m = await loadModel(CHARACTERS[next].url, CHARACTERS[next].name);
    if (cast[next]) return;
    m.visible = false; scene.add(m);
    cast[next] = { model: m, rig: new Rig(m) };
  }
  showOnStage(next);
}

function showOnStage(key) {
  if (onStage && cast[onStage]) { cast[onStage].model.visible = false; stage.removeContactShadow(cast[onStage].model); }
  onStage = key;
  model.visible = !key;
  stage.removeContactShadow(model);
  if (key) { cast[key].model.visible = true; stage.contactShadow(cast[key].model); }
  else stage.contactShadow(model);
  pill('rig', (key ? CHARACTERS[key].name : (CHARACTERS[settings.character]?.name || 'character')).toLowerCase(), 'ok');
}

// ---- MediaPipe ---------------------------------------------------------------------
let landmarker = null, vision = null, mpTs = 0;
async function initPose(variant) {
  pill('pose', 'pose: loading ' + variant, 'busy');
  vision ||= await FilesetResolver.forVisionTasks(MP_WASM);
  const make = (delegate) => PoseLandmarker.createFromOptions(vision, {
    baseOptions: { modelAssetPath: MP_MODEL(variant), delegate },
    runningMode: 'VIDEO', numPoses: 1,
    minPoseDetectionConfidence: 0.5, minPosePresenceConfidence: 0.5, minTrackingConfidence: 0.5,
  });
  const old = landmarker; landmarker = null;
  try { landmarker = await make('GPU'); } catch (e) { log('GPU delegate failed, using CPU: ' + e.message, 'err'); landmarker = await make('CPU'); }
  old?.close();
  pill('pose', 'pose: ' + variant, 'ok');
}

// ---- takes --------------------------------------------------------------------------
let takeSeq = 0;
const takes = [];
class Take {
  constructor(prompt) {
    this.id = ++takeSeq; this.prompt = prompt;
    this.videoUrl = null; this.status = 'generating';
    this.live = null; this.liveDone = false;
    this.hq = null; this.hqRaw = null; this.hqStatus = 'off'; this.hqAt = 0;
    this.rootOffset = null; this.timings = {};
    this.ready = new Promise((r) => (this._resolveReady = r));
  }
  get best() { return this.hq || (this.liveDone ? this.live : null); }
}

function renderTakes() {
  const el = $('#takes'); el.innerHTML = '';
  for (const t of [...takes].reverse()) {
    const card = document.createElement('div');
    card.className = 'take' + (t === current ? ' playing' : '');
    const gen = t.status === 'generating' ? '<span class="pill busy">generating</span>'
      : t.status === 'error' ? '<span class="pill bad">failed</span>'
      : t.status === 'skipped' ? '<span class="pill">skipped</span>'
      : t.timings.gen ? `<span class="pill">${t.timings.gen.toFixed(1)}s gen</span>` : '';
    const live = t.liveDone ? '<span class="pill">live</span>' : '';
    const hq = { running: '<span class="pill busy">hq</span>', done: `<span class="pill ok">hq ${t.timings.hq?.toFixed(1)}s</span>`, error: '<span class="pill bad">hq failed</span>' }[t.hqStatus] || '';
    card.innerHTML = `<div class="p" title="${escapeHtml(t.prompt)}">#${t.id} · ${t.auto ? '<i style="color:var(--dim)">idle</i> · ' : ''}${escapeHtml(t.prompt)}</div>
      <div class="meta">${gen}${live}${hq}</div>
      <div class="actions"><button class="btn" data-a="play" ${t.videoUrl ? '' : 'disabled'}>▶ play</button>
      <button class="btn" data-a="glb" ${t.best ? '' : 'disabled'}>↓ glb</button></div>`;
    card.querySelector('[data-a=play]').onclick = () => playTake(t);
    card.querySelector('[data-a=glb]').onclick = () => exportTake(t);
    el.appendChild(card);
  }
}
const escapeHtml = (s) => s.replace(/[&<>"]/g, (c) => ({ '&': '&amp;', '<': '&lt;', '>': '&gt;', '"': '&quot;' }[c]));

// ---- generation ------------------------------------------------------------------------
let genChain = Promise.resolve();
let lastGenerated = null;

let idleStreak = 0;
function submitPrompt(prompt, { auto = false } = {}) {
  prompt = prompt.trim();
  if (!prompt) return;
  if (!auto) {
    idleStreak = 0;
    // A real prompt shouldn't wait behind an idle filler that hasn't started playing: drop it,
    // and chain from the clip before it instead.
    const idle = lastGenerated;
    if (settings.stream && idle?.auto && !idle.started && idle !== current && idle.status === 'ready' && !genBusy) {
      const qi = queue.indexOf(idle); if (qi >= 0) queue.splice(qi, 1);
      idle.status = 'skipped'; lastGenerated = idle.prevGenerated;
      log(`#${idle.id} idle filler skipped for your prompt`);
    }
  }
  const take = new Take(prompt);
  take.auto = auto;
  takes.push(take); renderTakes();
  // Serialized: "continue from last frame" needs the previous clip's final frame.
  genBusy++;
  genChain = genChain.then(() => generate(take)).catch(() => {}).finally(() => genBusy--);
}
let genBusy = 0;

/** Stream mode: keep one clip in the pipeline so the performer never freezes for long. */
function maybeIdle() {
  if (!settings.stream || idleStreak >= MAX_IDLE_STREAK) return;
  const ahead = queue.length + genBusy + takes.filter((t) => t.status === 'ready' && t.hqStatus === 'running' && !t.started).length;
  if (ahead === 0) { idleStreak++; submitPrompt(IDLE_PROMPT, { auto: true }); }
}

async function ensurePerformer(force = false) {
  const o = settings.orientation;
  if (settings.performerUrls[o] && !force) return settings.performerUrls[o];
  banner('creating performer…');
  const t0 = performance.now();
  const r = await api('/api/image', { prompt: PERFORMER_PROMPT, image_size: o === 'portrait' ? 'portrait_16_9' : 'landscape_16_9' });
  settings.performerUrls[o] = r.image_url; save();
  showPerformer();
  log(`performer image in ${((performance.now() - t0) / 1000).toFixed(1)}s`, 'ok');
  banner('');
  return r.image_url;
}

async function generate(take) {
  const t0 = performance.now();
  try {
    let image_url = null, from = 'text';
    if (chainOn() && lastGenerated?.videoUrl) {
      image_url = await (lastGenerated.lastFrame ||= lastFrameDataUrl(lastGenerated.videoUrl)); from = `take #${lastGenerated.id} last frame`;
    } else if (settings.source === 'performer') {
      image_url = await ensurePerformer(); from = 'performer';
    }
    banner(`generating #${take.id}…`);
    log(`#${take.id} → ${VIDEO_MODEL_LABEL[settings.videoModel]} (${from}, ${settings.duration}s ${settings.resolution})`);
    const r = await api('/api/video', {
      prompt: videoPrompt(take.prompt, !!image_url), image_url,
      model: settings.videoModel, duration: settings.duration, resolution: settings.resolution, aspect_ratio: settings.orientation === 'portrait' ? '9:16' : '16:9',
    });
    take.videoUrl = r.video_url; take.status = 'ready';
    take.timings.gen = (performance.now() - t0) / 1000;
    log(`#${take.id} video in ${take.timings.gen.toFixed(1)}s (gpu ${r.timings?.inference?.toFixed(2) ?? '?'}s)`, 'ok');
    take.prevGenerated = lastGenerated;
    lastGenerated = take;
    // Grab the final frame now (in parallel with HQ) so the next chained clip can start immediately.
    if (chainOn()) take.lastFrame = lastFrameDataUrl(take.videoUrl).catch(() => null);
    banner('');
    if (settings.hq && hqOnline) runHQ(take);
    renderTakes();
    if (waitOn() && !(settings.stream && held)) {
      // Normally play only once HQ is ready; if the player starves meanwhile, startStarved()
      // plays it early with the live preview and HQ cross-fades in when it lands.
      take.awaitingHq = true;
      if (!current) banner(`#${take.id}: waiting for hq motion…`);
      take.hqPromise.finally(() => {
        banner('');
        if (take.awaitingHq && take.status !== 'skipped') { take.awaitingHq = false; enqueue(take); }
      });
    } else enqueue(take);
  } catch (e) {
    take.status = 'error'; renderTakes(); banner('');
    log(`#${take.id} failed: ${e.message}`, 'err');
  }
}

/** Grab a clip's final frame as a JPEG data URI (first frame for the next clip). */
async function lastFrameDataUrl(url) {
  const v = document.createElement('video');
  v.crossOrigin = 'anonymous'; v.muted = true; v.preload = 'auto'; v.src = url;
  await new Promise((res, rej) => { v.onloadeddata = res; v.onerror = () => rej(new Error('cannot load video for last frame')); });
  v.currentTime = Math.max(0, v.duration - 0.05);
  await new Promise((res) => (v.onseeked = res));
  const c = document.createElement('canvas'); c.width = v.videoWidth; c.height = v.videoHeight;
  c.getContext('2d').drawImage(v, 0, 0);
  v.removeAttribute('src'); v.load();
  return c.toDataURL('image/jpeg', 0.92);
}

// ---- HQ (GVHMR) --------------------------------------------------------------------------
let hqOnline = false;
async function checkHQ() {
  try { const r = await api('/api/hq/health'); hqOnline = !!r.ok; pill('hq', r.ok ? `hq · ${(r.device || 'ready').replace(/^NVIDIA GeForce /, '').toLowerCase()}` : 'hq · ' + (r.error || 'not ready'), r.ok ? 'ok' : 'bad'); }
  catch { hqOnline = false; pill('hq', 'hq worker offline', 'bad'); }
}

function runHQ(take) {
  const t0 = performance.now();
  take.hqStatus = 'running'; renderTakes();
  take.hqPromise = api('/api/hq', { video_url: take.videoUrl, ...(take.handsMethod ? { hands: take.handsMethod } : {}) })
    .then((r) => {
      take.hqRaw = r; take.hq = bakeSmpl(r); take.hqAt = performance.now();
      take.hqStatus = 'done'; take.timings.hq = (performance.now() - t0) / 1000;
      log(`#${take.id} hq motion in ${take.timings.hq.toFixed(1)}s (${r.n} frames, worker ${r.elapsed?.toFixed(1)}s)`, 'hq');
    })
    .catch((e) => { take.hqStatus = 'error'; log(`#${take.id} hq failed: ${e.message}`, 'err'); })
    .finally(renderTakes);
}

// Finger tracking from the HQ worker: per hand, 21 wrist-relative keypoints per frame (GVHMR world axes).
const _kp = Array.from({ length: 21 }, () => new THREE.Vector3());
function applyHandsFrame(hands, f, yawFix) {
  // WiLoR's MANO hands are anatomically valid already; joint limits only help MediaPipe's noisy depth.
  HAND_LIMITS.enabled = hands?.method !== 'wilor';
  for (const [src, side] of [['R', 'Right'], ['L', 'Left']]) {
    const h = hands?.[src];
    if (settings.hands !== false && h?.valid[f]) {
      const o = f * 63;
      for (let j = 0; j < 21; j++) _kp[j].set(h.kp[o + j * 3], h.kp[o + j * 3 + 1], h.kp[o + j * 3 + 2]).applyQuaternion(yawFix);
      rig.applyHand(side, _kp);
    } else rig.relaxHand(side);
  }
}

/** GVHMR SMPL params → Track of Mixamo local rotations + grounded hips path. */
function bakeSmpl(r) {
  const { n, fps, global_orient: GO, body_pose: BP, transl: T, leg_length } = r;
  // Heading: turn GVHMR's world so the character faces the viewer the way the person faces the
  // camera (from the in-camera orientation). Chained clips share the camera, so headings line up.
  const fWorld = new THREE.Vector3(0, 0, 1).applyQuaternion(aaToQuat(GO[0], GO[1], GO[2]));
  let targetYaw = 0;
  if (r.incam_global_orient0) {
    const [a, b, c] = r.incam_global_orient0;
    const fc = new THREE.Vector3(0, 0, 1).applyQuaternion(aaToQuat(a, b, c)); // OpenCV camera axes
    targetYaw = Math.atan2(fc.x, -fc.z);                                        // → three.js (x, -y, -z)
  }
  const yawFix = new THREE.Quaternion().setFromAxisAngle(new THREE.Vector3(0, 1, 0), targetYaw - Math.atan2(fWorld.x, fWorld.z));
  const scale = rig.legLength / (leg_length || 0.85);
  const track = new Track(rig.order.length);
  const lows = [];
  const d = new THREE.Vector3();
  for (let f = 0; f < n; f++) {
    rig.reset();
    rig.applySmplGlobal(smplGlobals(GO.subarray ? GO.subarray(f * 3, f * 3 + 3) : GO.slice(f * 3, f * 3 + 3), BP.slice(f * 63, f * 63 + 63), yawFix));
    d.set(T[f * 3] - T[0], T[f * 3 + 1] - T[1], T[f * 3 + 2] - T[2]).applyQuaternion(yawFix).multiplyScalar(scale);
    const p = rig.rest.Hips.wp.clone().add(d);
    rig.setHipsWorld(p);
    applyHandsFrame(r.hands, f, yawFix);
    lows.push(rig.lowestFoot());
    track.push(f / fps, rig.snapshot());
  }
  // Ground: the typical lowest foot sits on the floor; never sink below it.
  const sorted = [...lows].sort((a, b) => a - b);
  const offset = rig.floorY - sorted[Math.floor(sorted.length * 0.08)];
  track.p.forEach((p, f) => { p[1] += offset + Math.max(0, rig.floorY - (lows[f] + offset)); });
  rig.reset();
  return track;
}

// ---- playback -----------------------------------------------------------------------------
const video = $('#video');
const overlay = $('#overlay');
const octx = overlay.getContext('2d');
let current = null;
const queue = [];
let liveSnap = null;
let liveState = null;
const worldFilter = new OneEuro(99);
const hipFilter = new OneEuro(2);
let lastRenderedHips = new THREE.Vector3();
let switchFrom = null, switchAt = 0;

function setSmoothing(s) {
  // s=0 → raw, s=1 → very smooth
  worldFilter.minCutoff = hipFilter.minCutoff = THREE.MathUtils.lerp(6, 0.4, s);
  worldFilter.beta = hipFilter.beta = THREE.MathUtils.lerp(1.2, 0.15, s);
}

let held = false; // stream mode: clip ended, nothing ready yet → hold the last pose
function enqueue(take) {
  queue.push(take);
  if (capturing) return; // the capture script decides what's on screen
  // Switch right away unless the current clip is still on its first (live) pass,
  // or we're chaining (then the next clip continues from this one's last frame).
  if (!current || held || (current.liveDone && !chainOn())) playNext();
}

/** Out of HQ-ready clips: start the next generated one now on the live pass instead of freezing. */
function startStarved() {
  const next = takes.find((t) => t.awaitingHq && t.status === 'ready' && !t.started);
  if (!next) return false;
  next.awaitingHq = false;
  log(`#${next.id} starting on live preview (hq still running)`);
  enqueue(next);
  return true;
}

function playNext() { const t = queue.shift(); if (t) playTake(t); }

function playTake(take) {
  if (!take.videoUrl) return;
  if (current && rig) { switchFrom = rig.snapshot(); switchAt = performance.now(); }
  held = false; take.started = true;
  if (take.hq) take.hqAt = -1e9; // already HQ: no live→HQ cross-fade
  const prev = current;
  current = take;
  const qi = queue.indexOf(take); if (qi >= 0) queue.splice(qi, 1);
  if (!take.rootOffset) {
    take.rootOffset = new THREE.Vector3();
    if (chainOn() && prev && rig) {
      take.rootOffset.set(lastRenderedHips.x - rig.rest.Hips.wp.x, 0, lastRenderedHips.z - rig.rest.Hips.wp.z);
      take.rootOffset.clampLength(0, 2.5);
    }
  }
  if (!take.liveDone) {
    take.live = new Track(rig.order.length);
    liveState = { hip0: null, baseY: null, mpp: null, scale: null };
    worldFilter.reset(); hipFilter.reset(); liveSnap = null;
  }
  $('#vid-empty').style.display = 'none';
  video.style.display = 'block';
  video.src = take.videoUrl;
  video.onloadedmetadata = () => $('#vidwrap').classList.toggle('portrait', video.videoHeight > video.videoWidth);
  lastDetectT = -1;
  video.play().catch(() => {});
  renderTakes();
  maybeIdle();
}

video.addEventListener('ended', () => {
  if (settings.swap && rig) swapCharacter().catch((e) => log('swap failed: ' + e.message, 'err'));
  if (current && !current.liveDone) {
    current.liveDone = current.live.length > 3;
    log(`#${current.id} live pass: ${current.live.length} frames`);
    renderTakes();
  }
  if (queue.length) playNext();
  else if (settings.stream) { held = true; if (!startStarved()) maybeIdle(); } // freeze on the last frame
  else { video.currentTime = 0; video.play().catch(() => {}); }
});

// Per decoded video frame: run pose, draw overlay, record the live track.
const stats = { frames: 0, detections: 0, world: 0, recorded: 0 };
let lastDetectT = -1;
// Called from the render loop: run pose once per new video frame (requestVideoFrameCallback
// stops firing after the source changes in Chromium, so we poll currentTime instead).
function detectIfNewFrame() {
  if (!landmarker || !rig || !current || video.readyState < 2 || video.seeking) return;
  const mediaTime = video.currentTime;
  if (mediaTime === lastDetectT) return;
  if (mediaTime < lastDetectT) liveLoopReset();
  lastDetectT = mediaTime;
  try {
    stats.frames++;
    const ts = Math.max(mpTs + 1, performance.now()); mpTs = ts;
    const res = landmarker.detectForVideo(video, ts);
    stats.detections += res.landmarks?.length ? 1 : 0;
    stats.world += res.worldLandmarks?.length ? 1 : 0;
    drawOverlay(res);
    if (!current.liveDone && res.worldLandmarks?.[0]) {
      liveSnap = solveLive(res, mediaTime);
      current.live.push(mediaTime, liveSnap); stats.recorded++;
    }
  } catch (e) { console.error(e); }
}
function liveLoopReset() { /* video looped or switched: keep filters from smearing across the jump */ worldFilter.reset(); hipFilter.reset(); }

let lastPose2d = null; // latest MediaPipe pose landmarks (normalized video coords) — used by the capture zoom
function drawOverlay(res) {
  if (res.landmarks?.[0]) lastPose2d = res.landmarks[0];
  overlay.width = video.videoWidth; overlay.height = video.videoHeight;
  octx.clearRect(0, 0, overlay.width, overlay.height);
  if (!settings.showSkel || !res.landmarks?.[0]) return;
  const du = new DrawingUtils(octx);
  du.drawConnectors(res.landmarks[0], PoseLandmarker.POSE_CONNECTIONS, { color: '#7cf29a', lineWidth: 3 });
  du.drawLandmarks(res.landmarks[0], { color: '#5ab8ff', fillColor: '#0e0f11', lineWidth: 2, radius: 3 });
}

const _lm = Array.from({ length: 33 }, () => new THREE.Vector3());
function solveLive(res, t) {
  const w = res.worldLandmarks[0], n2 = res.landmarks[0];
  const raw = new Float32Array(99);
  // MediaPipe world: x = image right (subject's left when facing camera), y down, z away from camera.
  w.forEach((p, i) => { raw[i * 3] = p.x; raw[i * 3 + 1] = -p.y; raw[i * 3 + 2] = -p.z; });
  const f = worldFilter.filter(raw, t);
  for (let i = 0; i < 33; i++) _lm[i].set(f[i * 3], f[i * 3 + 1], f[i * 3 + 2]);
  const vis = w.map((p) => p.visibility ?? 1);

  // Root motion from the 2D hip track, converted to meters via torso length.
  const W = video.videoWidth, H = video.videoHeight;
  const px = (i) => new THREE.Vector2(n2[i].x * W, n2[i].y * H);
  const hipPx = px(23).add(px(24)).multiplyScalar(0.5);
  const shPx = px(11).add(px(12)).multiplyScalar(0.5);
  const hf = hipFilter.filter([hipPx.x, hipPx.y], t);
  const midHip = _lm[23].clone().add(_lm[24]).multiplyScalar(0.5);
  const midSh = _lm[11].clone().add(_lm[12]).multiplyScalar(0.5);
  const mpp = midSh.distanceTo(midHip) / Math.max(1, shPx.distanceTo(hipPx));
  const leg = (_lm[23].distanceTo(_lm[25]) + _lm[25].distanceTo(_lm[27]) + _lm[24].distanceTo(_lm[26]) + _lm[26].distanceTo(_lm[28])) / 2;
  const S = liveState;
  S.mpp = S.mpp == null ? mpp : S.mpp + 0.05 * (mpp - S.mpp);
  S.scale = S.scale == null ? rig.legLength / leg : S.scale + 0.05 * (rig.legLength / leg - S.scale);
  if (!S.hip0) S.hip0 = [hf[0], hf[1]];

  rig.reset();
  rig.applyLandmarks(_lm, vis);
  const p = rig.rest.Hips.wp.clone();
  if (settings.rootMotion) {
    p.x += (hf[0] - S.hip0[0]) * S.mpp * S.scale;
    p.y += -(hf[1] - S.hip0[1]) * S.mpp * S.scale;
  }
  rig.setHipsWorld(p);
  if (S.baseY == null) S.baseY = rig.floorY - rig.lowestFoot();
  p.y += S.baseY; rig.setHipsWorld(p);
  rig.clampToFloor();
  return rig.snapshot();
}

// ---- render loop ----------------------------------------------------------------------------
const bufA = { q: null, p: new Float32Array(3) }, bufB = { q: null, p: new Float32Array(3) }, bufC = { q: null, p: new Float32Array(3) };

let lastRaf = 0;
let capturing = false; // frame-exact capture (tools/capture.mjs) drives tick() itself
function frame() {
  requestAnimationFrame(frame);
  lastRaf = performance.now();
  if (!capturing) tick();
}
// Browsers pause rAF in hidden/occluded views; keep pose + playback running from a timer then.
setInterval(() => { if (!capturing && performance.now() - lastRaf > 100) tick(); }, 33);

function tick() {
  detectIfNewFrame();
  if (rig && current) {
    const n = rig.order.length * 4;
    for (const b of [bufA, bufB, bufC]) if (!b.q || b.q.length !== n) b.q = new Float32Array(n);
    const t = video.currentTime;
    const live = current.liveDone ? current.live.sample(t, bufA) : liveSnap;
    const hq = current.hq ? current.hq.sample(t, bufB) : null;
    let snap = hq || live, src = hq ? 'hq · gvhmr' : live ? 'live · mediapipe' : '—';
    if (hq && live) {
      const w = Math.min(1, (performance.now() - current.hqAt) / 600);
      if (w < 1) { snap = blendSnapshots(live.q, live.p, hq.q, hq.p, w, bufC); src = 'live → hq'; }
    }
    if (snap && switchFrom) {
      const w = Math.min(1, (performance.now() - switchAt) / 250);
      if (w < 1) snap = blendSnapshots(switchFrom.q, switchFrom.p, snap.q, snap.p, w, snap === bufC ? bufA : bufC);
      else switchFrom = null;
    }
    if (snap) {
      rig.applySnapshot(snap);
      if (current.rootOffset?.lengthSq()) { const h = rig.hipsWorld().add(current.rootOffset); rig.setHipsWorld(h); }
      rig.hipsWorld(lastRenderedHips);
    }
    const hud = $('#hud-src'); hud.textContent = src; hud.className = 'tag ' + (hq ? 'src-hq' : 'src-live');
    const buf = settings.stream ? `  ·  ready ${queue.length}  gen ${genBusy}  hq ${takes.filter((x) => x.hqStatus === 'running').length}${held ? '  ·  HOLD' : ''}` : '';
    $('#hud-time').textContent = `#${current.id}  ${t.toFixed(2)}s${buf}`;
  }
  if (rig) {
    const shown = onStage ? cast[onStage].rig : rig;
    if (shown !== rig) shown.copyPoseFrom(rig);
    stage.placeContactShadow(shown.model, shown.hipsWorld(), shown.floorY, shown.height);
  }
  stage.update(rig ? lastRenderedHips : null);
}

// ---- export --------------------------------------------------------------------------------
async function exportTake(take) {
  const track = take.best;
  if (!track) return;
  const name = `take${take.id}_${take.prompt.replace(/[^a-z0-9]+/gi, '_').slice(0, 40)}${take.hq ? '_hq' : '_live'}`;
  const clip = trackToClip(rig, track, name);
  const glb = await new GLTFExporter().parseAsync(model, { binary: true, animations: [clip] });
  const a = document.createElement('a');
  a.href = URL.createObjectURL(new Blob([glb], { type: 'model/gltf-binary' }));
  a.download = name + '.glb'; a.click();
  // Also keep a copy in ./takes (with the source clip URL) for tools/render_blender.py.
  const q = new URLSearchParams({ name, prompt: take.prompt, ...(take.videoUrl?.startsWith('http') ? { source: take.videoUrl } : {}) });
  fetch('/api/takes?' + q, { method: 'POST', body: glb }).then((r) => r.ok && log(`saved takes/${name}.glb`, 'ok')).catch(() => {});
  setTimeout(() => URL.revokeObjectURL(a.href), 5000);
  log(`exported ${name}.glb`, 'ok');
}

// ---- UI wiring --------------------------------------------------------------------------------
bindSetting('stream', 'checked', (on) => {
  for (const id of ['chain', 'waitHq']) { const el = $('#' + id); el.disabled = on; el.closest('label').style.opacity = on ? 0.5 : 1; }
});
bindSetting('source'); bindSetting('chain', 'checked'); bindSetting('resolution');
bindSetting('videoModel', 'value', (m) => {
  const d = $('#duration'); d.min = m === 'flux3' ? 5 : 2;  // FLUX 3 Max: 5–20s
  if (settings.duration < d.min) { settings.duration = Number(d.min); d.value = d.min; $('#duration-v').textContent = d.min + 's'; save(); }
});
bindSetting('swap', 'checked', (on) => { if (!on && model) showOnStage(null); });
bindSetting('cinematic', 'checked', (on) => stage.setCinematic(on));
bindSetting('character', 'value', (c) => { if (rig && CHARACTERS[c]) loadCharacter(CHARACTERS[c].url, CHARACTERS[c].name).catch((e) => log('character failed: ' + e.message, 'err')); });
bindSetting('duration', 'value', (v) => ($('#duration-v').textContent = v + 's'));
bindSetting('hq', 'checked');
bindSetting('hands', 'checked', () => { if (rig) for (const t of takes) if (t.hqRaw) t.hq = bakeSmpl(t.hqRaw); }); bindSetting('waitHq', 'checked'); bindSetting('rootMotion', 'checked');
bindSetting('showSkel', 'checked');
bindSetting('smoothing', 'value', setSmoothing);
bindSetting('poseModel', 'value', (v) => { if (vision) initPose(v); });

const chips = $('#chips');
for (const p of PRESETS) {
  const c = document.createElement('span'); c.className = 'chip'; c.textContent = p;
  c.onclick = () => submitPrompt(p);
  chips.appendChild(c);
}
$('#go').onclick = () => submitPrompt($('#prompt').value);
$('#prompt').addEventListener('keydown', (e) => { if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { e.preventDefault(); submitPrompt($('#prompt').value); } });
$('#performer-new').onclick = () => ensurePerformer(true).catch((e) => { banner(''); log('performer failed: ' + e.message, 'err'); });
function showPerformer() {
  const img = $('#performer-img'), url = settings.performerUrls[settings.orientation];
  img.classList.toggle('portrait', settings.orientation === 'portrait');
  if (url) img.src = url; else img.removeAttribute('src');
  // Before any clip has loaded, shape the empty video panel like the clips that will come.
  if (!video.videoWidth) $('#vidwrap').classList.toggle('portrait', settings.orientation === 'portrait');
}
if (settings.performerUrl) { settings.performerUrls.landscape ||= settings.performerUrl; delete settings.performerUrl; save(); }
bindSetting('orientation', 'value', showPerformer);

$('#btn-char').onclick = () => $('#file-char').click();
$('#file-char').onchange = (e) => {
  const f = e.target.files[0]; if (!f) return;
  const url = URL.createObjectURL(f);
  loadCharacter(url, f.name).catch((err) => { log('character failed: ' + err.message, 'err'); pill('rig', 'character failed', 'bad'); });
};
$('#btn-vid').onclick = () => $('#file-vid').click();
$('#file-vid').onchange = (e) => {
  const f = e.target.files[0]; if (!f) return;
  const take = new Take(`(file) ${f.name}`);
  take.videoUrl = URL.createObjectURL(f); take.status = 'ready'; take.timings.gen = 0;
  takes.push(take); renderTakes(); enqueue(take);
};

// ---- capture zoom: glide the 3D camera onto a hand and zoom the video panel onto the same hand ----
// k ∈ [0,1] is the zoom amount (the capture script ramps it linearly in time); t is the output time.
// Eased with smootherstep, distance/scale interpolated in log space (constant perceived speed), the
// camera arcs around the hand on the way in and drifts slowly while holding.
let zoomBase = null, zoomTgt = null, zoomOrigin = null;
const smootherstep = (x) => x * x * x * (x * (x * 6 - 15) + 10);
function applyCaptureZoom(k, bone, t = 0) {
  const r = onStage ? cast[onStage].rig : rig;
  if (k <= 0.0005) {
    if (zoomBase) { stage.setFocusLock(null); stage.camera.position.copy(zoomBase.pos); stage.controls.target.copy(zoomBase.tgt); zoomBase = null; }
    for (const el of [video, overlay]) el.style.transform = '';
    overlay.style.opacity = ''; zoomTgt = zoomOrigin = null;
    return;
  }
  if (!zoomBase) {
    const off = stage.camera.position.clone().sub(stage.controls.target);
    zoomBase = { pos: stage.camera.position.clone(), tgt: stage.controls.target.clone(), dir: off.clone().normalize(), dist: off.length() };
  }
  const e = smootherstep(Math.min(1, Math.max(0, k)));
  // Target: palm centre, followed smoothly so the hand stays framed while it moves.
  const hand = r.bones[bone].getWorldPosition(new THREE.Vector3());
  const mid = r.bones[bone.replace('Hand', 'HandMiddle2')];
  const want = mid ? hand.lerp(mid.getWorldPosition(new THREE.Vector3()), 0.6) : hand;
  zoomTgt = zoomTgt ? zoomTgt.lerp(want, 0.3) : want.clone();
  const tgt = zoomBase.tgt.clone().lerp(zoomTgt, e);
  // Direction arcs from the wide shot's to a 3/4 close-up angle, plus a slow drift while held.
  // Nearly straight-on, like the performer's camera, so the close-up reads like the video panel.
  const closeDir = new THREE.Vector3(bone.startsWith('Right') ? 0.06 : -0.06, 0.04, 1).normalize();
  const qArc = new THREE.Quaternion().setFromUnitVectors(zoomBase.dir, closeDir);
  const dir = zoomBase.dir.clone().applyQuaternion(new THREE.Quaternion().slerp(qArc, e));
  dir.applyAxisAngle(new THREE.Vector3(0, 1, 0), 0.09 * Math.sin(t * 0.7) * e);
  const dist = Math.exp(Math.log(zoomBase.dist) + (Math.log(0.52) - Math.log(zoomBase.dist)) * e);
  stage.setFocusLock(tgt);
  stage.camera.position.copy(tgt).addScaledVector(dir, dist);
  stage.update(null);
  // Video panel: same easing, zooming on the performer's matching hand (pose landmarks right 16/18/20, left 15/17/19).
  const ids = bone.startsWith('Right') ? [16, 18, 20] : [15, 17, 19];
  if (lastPose2d) {
    // Wrist + knuckle landmarks, nudged past the knuckles toward the fingertips.
    const w = lastPose2d[ids[0]], k1 = lastPose2d[ids[1]], k2 = lastPose2d[ids[2]];
    const kx = (k1.x + k2.x) / 2, ky = (k1.y + k2.y) / 2;
    const c = [kx + 0.6 * (kx - w.x), ky + 0.6 * (ky - w.y)];
    zoomOrigin = zoomOrigin ? [zoomOrigin[0] + 0.3 * (c[0] - zoomOrigin[0]), zoomOrigin[1] + 0.3 * (c[1] - zoomOrigin[1])] : c;
  }
  const o = zoomOrigin || [0.5, 0.35];
  const scale = Math.exp(Math.log(3.2) * e);
  for (const el of [video, overlay]) { el.style.transformOrigin = `${o[0] * 100}% ${o[1] * 100}%`; el.style.transform = `scale(${scale})`; }
  overlay.style.opacity = String(1 - e); // the stick figure would only clutter the close-up
}

// Debug handle for the console.
window.p2r = {
  get current() { return current; }, get rig() { return rig; }, takes, video, stats,
  stage, HAND_LIMITS, rebake() { for (const t of takes) if (t.hqRaw) t.hq = bakeSmpl(t.hqRaw); },
  /** Debug: frame the camera on a bone (e.g. 'RightHand') at distance d. */
  focusBone(name, d = 0.6, dir = [0.15, 0.1, 1]) {
    const r = onStage ? cast[onStage].rig : rig;
    const p = r.bones[name].getWorldPosition(new THREE.Vector3());
    stage.setCinematic(false); stage.controls.target.copy(p);
    stage.camera.position.copy(p).add(new THREE.Vector3(...dir).normalize().multiplyScalar(d));
    stage.setFocusLock?.(p);
  },
  // ---- frame-exact capture: used by tools/capture.mjs ----
  submit(prompt, { duration } = {}) {
    if (duration) { settings.duration = duration; $('#duration').value = duration; $('#duration-v').textContent = duration + 's'; }
    submitPrompt(prompt); return takes[takes.length - 1].id;
  },
  take(id) { const t = takes.find((x) => x.id === id); return t && { id: t.id, status: t.status, hq: t.hqStatus, url: t.videoUrl, duration: t.hq?.duration ?? null }; },
  async beginCapture(id) {
    const t = takes.find((x) => x.id === id);
    if (!t?.hq) throw new Error('take has no HQ motion');
    capturing = true; video.loop = false;
    if (current !== t) playTake(t);
    video.pause();
    await new Promise((r) => (video.readyState >= 2 ? r() : video.addEventListener('loadeddata', r, { once: true })));
    renderTakes();
    return { duration: video.duration, frames: Math.floor(video.duration * 30) };
  },
  /** Show exactly media time t: seek, pose (GVHMR), overlay, camera, render. */
  async renderAt(t, zoom = null) {
    video.pause();
    if (Math.abs(video.currentTime - t) > 1e-4) {
      const seeked = new Promise((r) => video.addEventListener('seeked', r, { once: true }));
      video.currentTime = t; await seeked;
    }
    lastDetectT = -1; tick();
    if (zoom) applyCaptureZoom(zoom.k, zoom.bone || 'RightHand', zoom.t || 0);
    // Two frames so the compositor has painted (with a timeout: hidden tabs don't run rAF).
    await Promise.race([new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))), new Promise((r) => setTimeout(r, 80))]);
  },
  endCapture() { capturing = false; video.play().catch(() => {}); },
  playUrl(u, hq = true, hands, prompt) { const t = new Take(prompt || 'debug: ' + u.slice(-24)); t.videoUrl = u; t.status = 'ready'; t.handsMethod = hands; takes.push(t); if (hq && hqOnline) runHQ(t); enqueue(t); return { id: t.id }; },
};

// ---- boot --------------------------------------------------------------------------------------
frame();
const startChar = CHARACTERS[settings.character] || CHARACTERS.xbot;
loadCharacter(startChar.url, startChar.name).catch((e) => { log('character failed: ' + e.message, 'err'); pill('rig', 'character failed', 'bad'); });
initPose(settings.poseModel).catch((e) => { log('pose model failed: ' + e.message, 'err'); pill('pose', 'pose failed', 'bad'); });
checkHQ(); setInterval(checkHQ, 10000);
log('ready. type a motion and hit generate');