Type a motion, watch a rigged character perform it: fast video model + GVHMR + live Mixamo retargeting
JavaScript
Capture: cinematic hand zoom, replay passes, re-capture existing clips
blendi-remadebe23d85bab1f
diff --git a/.gitignore b/.gitignoreindex f5c7ee9..5a85fb3 100644--- a/.gitignore+++ b/.gitignore@@ -31,3 +31,4 @@ bench/clips/bench/out/*.taskcaptures_hands/+captures_hero/diff --git a/public/app.js b/public/app.jsindex 855ee17..8090a70 100644--- a/public/app.js+++ b/public/app.js@@ -508,7 +508,9 @@ function detectIfNewFrame() {}function liveLoopReset() { /* video looped or switched: keep filters from smearing across the jump */ worldFilter.reset(); hipFilter.reset(); }+let lastPose2d = null; // latest MediaPipe pose landmarks (normalized video coords) — used by the capture zoomfunction drawOverlay(res) {+ if (res.landmarks?.[0]) lastPose2d = res.landmarks[0];overlay.width = video.videoWidth; overlay.height = video.videoHeight;octx.clearRect(0, 0, overlay.width, overlay.height);if (!settings.showSkel || !res.landmarks?.[0]) return;@@ -673,6 +675,56 @@ $('#file-vid').onchange = (e) => {takes.push(take); renderTakes(); enqueue(take);};+// ---- capture zoom: glide the 3D camera onto a hand and zoom the video panel onto the same hand ----+// k ∈ [0,1] is the zoom amount (the capture script ramps it linearly in time); t is the output time.+// Eased with smootherstep, distance/scale interpolated in log space (constant perceived speed), the+// camera arcs around the hand on the way in and drifts slowly while holding.+let zoomBase = null, zoomTgt = null, zoomOrigin = null;+const smootherstep = (x) => x * x * x * (x * (x * 6 - 15) + 10);+function applyCaptureZoom(k, bone, t = 0) {+ const r = onStage ? cast[onStage].rig : rig;+ if (k <= 0.0005) {+ if (zoomBase) { stage.setFocusLock(null); stage.camera.position.copy(zoomBase.pos); stage.controls.target.copy(zoomBase.tgt); zoomBase = null; }+ for (const el of [video, overlay]) el.style.transform = '';+ overlay.style.opacity = ''; zoomTgt = zoomOrigin = null;+ return;+ }+ if (!zoomBase) {+ const off = stage.camera.position.clone().sub(stage.controls.target);+ zoomBase = { pos: stage.camera.position.clone(), tgt: stage.controls.target.clone(), dir: off.clone().normalize(), dist: off.length() };+ }+ const e = smootherstep(Math.min(1, Math.max(0, k)));+ // Target: palm centre, followed smoothly so the hand stays framed while it moves.+ const hand = r.bones[bone].getWorldPosition(new THREE.Vector3());+ const mid = r.bones[bone.replace('Hand', 'HandMiddle2')];+ const want = mid ? hand.lerp(mid.getWorldPosition(new THREE.Vector3()), 0.6) : hand;+ zoomTgt = zoomTgt ? zoomTgt.lerp(want, 0.3) : want.clone();+ const tgt = zoomBase.tgt.clone().lerp(zoomTgt, e);+ // Direction arcs from the wide shot's to a 3/4 close-up angle, plus a slow drift while held.+ // Nearly straight-on, like the performer's camera, so the close-up reads like the video panel.+ const closeDir = new THREE.Vector3(bone.startsWith('Right') ? 0.06 : -0.06, 0.04, 1).normalize();+ const qArc = new THREE.Quaternion().setFromUnitVectors(zoomBase.dir, closeDir);+ const dir = zoomBase.dir.clone().applyQuaternion(new THREE.Quaternion().slerp(qArc, e));+ dir.applyAxisAngle(new THREE.Vector3(0, 1, 0), 0.09 * Math.sin(t * 0.7) * e);+ const dist = Math.exp(Math.log(zoomBase.dist) + (Math.log(0.52) - Math.log(zoomBase.dist)) * e);+ stage.setFocusLock(tgt);+ stage.camera.position.copy(tgt).addScaledVector(dir, dist);+ stage.update(null);+ // Video panel: same easing, zooming on the performer's matching hand (pose landmarks right 16/18/20, left 15/17/19).+ const ids = bone.startsWith('Right') ? [16, 18, 20] : [15, 17, 19];+ if (lastPose2d) {+ // Wrist + knuckle landmarks, nudged past the knuckles toward the fingertips.+ const w = lastPose2d[ids[0]], k1 = lastPose2d[ids[1]], k2 = lastPose2d[ids[2]];+ const kx = (k1.x + k2.x) / 2, ky = (k1.y + k2.y) / 2;+ const c = [kx + 0.6 * (kx - w.x), ky + 0.6 * (ky - w.y)];+ zoomOrigin = zoomOrigin ? [zoomOrigin[0] + 0.3 * (c[0] - zoomOrigin[0]), zoomOrigin[1] + 0.3 * (c[1] - zoomOrigin[1])] : c;+ }+ const o = zoomOrigin || [0.5, 0.35];+ const scale = Math.exp(Math.log(3.2) * e);+ for (const el of [video, overlay]) { el.style.transformOrigin = `${o[0] * 100}% ${o[1] * 100}%`; el.style.transform = `scale(${scale})`; }+ overlay.style.opacity = String(1 - e); // the stick figure would only clutter the close-up+}+// Debug handle for the console.window.p2r = {get current() { return current; }, get rig() { return rig; }, takes, video, stats,@@ -702,18 +754,19 @@ window.p2r = {return { duration: video.duration, frames: Math.floor(video.duration * 30) };},/** Show exactly media time t: seek, pose (GVHMR), overlay, camera, render. */- async renderAt(t) {+ async renderAt(t, zoom = null) {video.pause();if (Math.abs(video.currentTime - t) > 1e-4) {const seeked = new Promise((r) => video.addEventListener('seeked', r, { once: true }));video.currentTime = t; await seeked;}lastDetectT = -1; tick();+ if (zoom) applyCaptureZoom(zoom.k, zoom.bone || 'RightHand', zoom.t || 0);// Two frames so the compositor has painted (with a timeout: hidden tabs don't run rAF).await Promise.race([new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))), new Promise((r) => setTimeout(r, 80))]);},endCapture() { capturing = false; video.play().catch(() => {}); },- playUrl(u, hq = true, hands) { const t = new Take('debug: ' + u.slice(-24)); t.videoUrl = u; t.status = 'ready'; t.handsMethod = hands; takes.push(t); if (hq && hqOnline) runHQ(t); enqueue(t); return { id: t.id }; },+ playUrl(u, hq = true, hands, prompt) { const t = new Take(prompt || 'debug: ' + u.slice(-24)); t.videoUrl = u; t.status = 'ready'; t.handsMethod = hands; takes.push(t); if (hq && hqOnline) runHQ(t); enqueue(t); return { id: t.id }; },};// ---- boot --------------------------------------------------------------------------------------diff --git a/tools/capture.mjs b/tools/capture.mjsindex f3757ed..7af88d8 100644--- a/tools/capture.mjs+++ b/tools/capture.mjs@@ -20,7 +20,10 @@ const OUT = path.resolve(args.out || 'captures');const URL = args.url || 'http://localhost:5177';const CHROME = args.chrome || 'C:/Program Files/Google/Chrome/Application/chrome.exe';// Lines may end in "| <seconds>" to set that clip's duration (FLUX 3 Max: 5–20s).-const prompts = fs.readFileSync(args.prompts || 'tools/prompts.txt', 'utf8').split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.startsWith('#'))+// --clip <video url> [--prompt "..."] re-captures an existing clip instead of generating from --prompts.+// --passes N plays it N times; --zoom a-b zooms onto --zoom-bone (default RightHand) between a and b+// seconds of the output, with --ramp seconds of ease in/out.+const prompts = args.clip ? [{ prompt: args.prompt || 'existing clip', duration: 5, url: args.clip }] : fs.readFileSync(args.prompts || 'tools/prompts.txt', 'utf8').split(/\r?\n/).map((l) => l.trim()).filter((l) => l && !l.startsWith('#')).map((l) => { const [p, d] = l.split('|').map((x) => x.trim()); return { prompt: p, duration: d ? Number(d) : Number(args.duration || 5) }; });// App settings for the recording: FLUX 3 Max, portrait, GVHMR only (hold until HQ), static camera.@@ -48,17 +51,27 @@ const results = [];try {const page = await browser.newPage();page.on('pageerror', (e) => log('page error:', e.message));- await page.evaluateOnNewDocument((s) => localStorage.setItem('p2r', JSON.stringify(s)), settings);+ await page.evaluateOnNewDocument((s) => { try { localStorage.setItem('p2r', JSON.stringify(s)); } catch {} }, settings); // sandboxed iframes can'tawait page.goto(URL, { waitUntil: 'networkidle2' });await page.waitForFunction(() => window.p2r?.rig && document.querySelector('#pill-hq').classList.contains('ok')- && document.querySelector('#pill-pose').classList.contains('ok'), { timeout: 120000 });+ && document.querySelector('#pill-pose').classList.contains('ok'), { timeout: 120000 }).catch(async (e) => {+ log('app not ready:', await page.evaluate(() => JSON.stringify({ p2r: !!window.p2r, rig: !!window.p2r?.rig,+ pills: [...document.querySelectorAll('.pill')].slice(0, 3).map((x) => x.className + ':' + x.textContent) })).catch(() => '?'));+ throw e;+ });log(`app ready (${settings.character}, ${settings.videoModel}); ${prompts.length} prompts`);- for (const [i, { prompt, duration: seconds }] of prompts.entries()) {+ const passes = Number(args.passes || 1);+ const [z0, z1] = args.zoom ? String(args.zoom).split('-').map(Number) : [Infinity, -Infinity];+ const ramp = Number(args.ramp || 1.1);+ const zoomAt = (t) => (t < z0 || t > z1 ? 0 : Math.min(1, (t - z0) / ramp, (z1 - t) / ramp));+ for (const [i, { prompt, duration: seconds, url: clipUrl }] of prompts.entries()) {const name = `${String(i + 1).padStart(2, '0')}_${slug(prompt)}`;log(`#${i + 1} generating (${seconds}s): ${prompt}`);await page.evaluate((p) => { document.querySelector('#prompt').value = p; }, prompt);- const id = await page.evaluate((p, d) => window.p2r.submit(p, { duration: d }), prompt, seconds);+ const id = clipUrl+ ? await page.evaluate((u, p) => window.p2r.playUrl(u, true, undefined, p).id, clipUrl, prompt)+ : await page.evaluate((p, d) => window.p2r.submit(p, { duration: d }), prompt, seconds);const take = await page.waitForFunction((tid) => {const t = window.p2r.take(tid);return t && (t.status === 'error' || t.hq === 'error' || t.hq === 'done') && t;@@ -72,8 +85,9 @@ try {const ff = spawn('ffmpeg', ['-y', '-loglevel', 'error', '-f', 'image2pipe', '-c:v', 'mjpeg', '-r', String(FPS), '-i', '-','-c:v', 'libx264', '-preset', 'medium', '-crf', '16', '-pix_fmt', 'yuv420p', '-movflags', '+faststart', file], { stdio: ['pipe', 'inherit', 'inherit'] });const t0 = Date.now();- for (let f = 0; f < frames; f++) {- await page.evaluate((t) => window.p2r.renderAt(t), f / FPS);+ for (let f = 0; f < frames * passes; f++) {+ const k = zoomAt(f / FPS);+ await page.evaluate((t, zk, bone, tt) => window.p2r.renderAt(t, { k: zk, bone, t: tt }), (f % frames) / FPS, k, args['zoom-bone'] || 'RightHand', f / FPS);const jpg = await page.screenshot({ type: 'jpeg', quality: 95, optimizeForSpeed: true });if (!ff.stdin.write(jpg)) await new Promise((r) => ff.stdin.once('drain', r));}