Skip to content

Type a motion, watch a rigged character perform it: fast video model + GVHMR + live Mixamo retargeting

JavaScript

Finger tracking: hand stage in the HQ worker + finger retargeting

blendi-remade130259dde0c2
diff --git a/.gitignore b/.gitignore
index 49ca04d..744aeb1 100644
--- a/.gitignore
+++ b/.gitignore
@@ -24,3 +24,9 @@ renders/
captures/
node_modules/
captures_xbot/
+
+# hand bench: environments, test clips, outputs, models
+bench/.venv/
+bench/clips/
+bench/out/
+*.task
diff --git a/bench/compose_review.py b/bench/compose_review.py
new file mode 100644
index 0000000..b965c30
--- /dev/null
+++ b/bench/compose_review.py
@@ -0,0 +1,60 @@
+"""Grid: source hand crop | character close-up (one column per method), per time and hand.
+
+ python bench/compose_review.py --clip bench/clips/h3.mp4 --name h3 --times 0.83,2.08 --methods mediapipe,wilor
+"""
+
+import argparse
+import os
+
+import cv2
+import numpy as np
+
+HERE = os.path.dirname(__file__)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--clip", required=True); ap.add_argument("--name", required=True)
+ ap.add_argument("--times", required=True); ap.add_argument("--methods", default="mediapipe")
+ ap.add_argument("--kp", default="mediapipe", help="bench npz used to locate the hand in the source")
+ ap.add_argument("--size", type=int, default=300)
+ a = ap.parse_args()
+ S = a.size
+ cap = cv2.VideoCapture(a.clip); fps = cap.get(cv2.CAP_PROP_FPS) or 24
+ frames = []
+ while True:
+ ok, f = cap.read()
+ if not ok:
+ break
+ frames.append(f)
+ kp = np.load(os.path.join(HERE, "out", f"{a.name}_{a.kp}.npz"))
+ methods = a.methods.split(",")
+ rows = []
+ for t in [float(x) for x in a.times.split(",")]:
+ fi = min(len(frames) - 1, int(round(t * fps)))
+ for side, h in (("Right", "R"), ("Left", "L")):
+ if kp[f"{h}_valid"][fi]:
+ p = kp[f"{h}_kp2d"][fi]; c = p.mean(0); half = max(60, np.abs(p - c).max() * 1.5)
+ else:
+ continue
+ src = cv2.copyMakeBorder(frames[fi], 600, 600, 600, 600, cv2.BORDER_CONSTANT)
+ x0, y0, x1, y1 = (int(c[0] - half) + 600, int(c[1] - half) + 600, int(c[0] + half) + 600, int(c[1] + half) + 600)
+ tiles = [cv2.resize(src[y0:y1, x0:x1], (S, S))]
+ cv2.putText(tiles[0], f"{a.name} {t:.2f}s {side}", (6, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (0, 200, 255), 2)
+ for m in methods:
+ path = os.path.join(HERE, "out", "review", f"{a.name}_{m}_{t:.2f}_{side}.png")
+ img = cv2.imread(path)
+ if img is None:
+ tiles.append(np.zeros((S, S, 3), np.uint8)); continue
+ hh, ww = img.shape[:2]; r = int(min(hh, ww) * 0.42)
+ cy, cx = hh // 2, ww // 2
+ tiles.append(cv2.resize(img[cy - r:cy + r, cx - r:cx + r], (S, S)))
+ cv2.putText(tiles[-1], m, (6, 20), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 2)
+ rows.append(np.hstack(tiles))
+ out = os.path.join(HERE, "out", f"{a.name}_review.png")
+ cv2.imwrite(out, np.vstack(rows))
+ print(out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/bench/hand_prompts.txt b/bench/hand_prompts.txt
new file mode 100644
index 0000000..0997f27
--- /dev/null
+++ b/bench/hand_prompts.txt
@@ -0,0 +1,6 @@
+gives a big thumbs up with both hands toward the camera, then two thumbs down
+makes a peace sign with both hands, then opens both hands wide with fingers spread
+counts from one to five on their raised right hand, finger by finger
+waves hello with an open hand, then makes tight fists and flexes
+points at the camera with the right index finger, then points up to the sky
+claps their hands several times, then does jazz hands with fingers spread
diff --git a/bench/hand_review.mjs b/bench/hand_review.mjs
new file mode 100644
index 0000000..87ba21e
--- /dev/null
+++ b/bench/hand_review.mjs
@@ -0,0 +1,47 @@
+// Render close-ups of the character's hands at given times, for side-by-side review with the source.
+// node bench/hand_review.mjs --url <clip url> --name h3 --times 0.8,2.1,3.3,4.6 [--character xbot] [--hands mediapipe|wilor|off]
+// Writes bench/out/review/<name>_<hands>_<t>_<side>.png (character's own Right/Left hand).
+import fs from 'node:fs';
+import os from 'node:os';
+import path from 'node:path';
+import puppeteer from 'puppeteer-core';
+
+const args = Object.fromEntries(process.argv.slice(2).reduce((acc, a, i, all) => {
+ if (a.startsWith('--')) acc.push([a.slice(2), all[i + 1] && !all[i + 1].startsWith('--') ? all[i + 1] : true]);
+ return acc;
+}, []));
+const OUT = path.resolve('bench/out/review');
+const times = String(args.times || '1,2,3,4').split(',').map(Number);
+const hands = args.hands || 'mediapipe';
+fs.mkdirSync(OUT, { recursive: true });
+
+const browser = await puppeteer.launch({
+ executablePath: 'C:/Program Files/Google/Chrome/Application/chrome.exe', headless: true,
+ userDataDir: fs.mkdtempSync(path.join(os.tmpdir(), 'p2r-review-')),
+ args: ['--use-angle=d3d11', '--enable-gpu', '--ignore-gpu-blocklist', '--mute-audio', '--window-size=1100,900'],
+ defaultViewport: { width: 1100, height: 900, deviceScaleFactor: 1 },
+});
+try {
+ const page = await browser.newPage();
+ page.on('pageerror', (e) => console.log('page error:', e.message));
+ await page.evaluateOnNewDocument((s) => localStorage.setItem('p2r', JSON.stringify(s)),
+ { character: args.character || 'xbot', hq: true, waitHq: true, swap: false, cinematic: false, stream: false, hands: hands !== 'off' });
+ await page.goto('http://localhost:5177', { waitUntil: 'networkidle2' });
+ await page.waitForFunction(() => window.p2r?.rig && document.querySelector('#pill-hq').classList.contains('ok'), { timeout: 120000 });
+ const id = await page.evaluate((u, h) => window.p2r.playUrl(u, true, h).id, args.url, hands);
+ await page.waitForFunction((tid) => ['done', 'error'].includes(window.p2r.take(tid)?.hq), { timeout: 180000, polling: 300 }, id);
+ await page.evaluate((tid) => window.p2r.beginCapture(tid), id);
+ await page.addStyleTag({ content: 'aside, footer, .vidwrap, .hud, .banner { display: none !important } main { position: fixed !important; inset: 0 !important }' });
+ await new Promise((r) => setTimeout(r, 500));
+ for (const t of times) {
+ for (const side of ['Right', 'Left']) {
+ await page.evaluate((tt) => window.p2r.renderAt(tt), t);
+ await page.evaluate((s, d) => window.p2r.focusBone(`${s}Hand`, d, [0.25, 0.1, 1]), side, Number(args.dist || 0.75));
+ await page.evaluate((tt) => window.p2r.renderAt(tt + 1e-3), t);
+ await page.screenshot({ path: path.join(OUT, `${args.name}_${hands}_${t.toFixed(2)}_${side}.png`) });
+ }
+ }
+ console.log('ok', times.length * 2, 'close-ups');
+} finally {
+ await browser.close();
+}
diff --git a/bench/hands_bench.py b/bench/hands_bench.py
new file mode 100644
index 0000000..ec41411
--- /dev/null
+++ b/bench/hands_bench.py
@@ -0,0 +1,170 @@
+"""Hand-pose bench: run a method over a clip and save per-frame keypoints for both hands.
+
+ python bench/hands_bench.py --method mediapipe|wilor|wilor_body --clip bench/clips/h1.mp4
+
+Saves bench/out/<clip>_<method>.npz with, per hand ("R" = subject's right, "L" = left):
+ kp2d (T,21,2) image px · kp3d (T,21,3) hand-local meters · valid (T,) · score (T,)
+and prints detection rate, 2D jitter and speed. Joint order (both methods): wrist, thumb 1-4,
+index 5-8, middle 9-12, ring 13-16, pinky 17-20.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import sys
+import time
+
+import cv2
+import numpy as np
+
+HANDS = ("R", "L")
+
+
+def read_frames(path):
+ cap = cv2.VideoCapture(path)
+ frames = []
+ while True:
+ ok, f = cap.read()
+ if not ok:
+ break
+ frames.append(f)
+ fps = cap.get(cv2.CAP_PROP_FPS) or 24
+ cap.release()
+ return frames, fps
+
+
+def empty(T):
+ return {h: dict(kp2d=np.zeros((T, 21, 2), np.float32), kp3d=np.zeros((T, 21, 3), np.float32),
+ valid=np.zeros(T, bool), score=np.zeros(T, np.float32)) for h in HANDS}
+
+
+# ---- methods ---------------------------------------------------------------------------------
+def run_mediapipe(frames, fps):
+ import mediapipe as mp
+
+ opts = mp.tasks.vision.HandLandmarkerOptions(
+ base_options=mp.tasks.BaseOptions(model_asset_path=os.path.join(os.path.dirname(__file__), "hand_landmarker.task")),
+ running_mode=mp.tasks.vision.RunningMode.VIDEO, num_hands=2,
+ min_hand_detection_confidence=0.3, min_hand_presence_confidence=0.3, min_tracking_confidence=0.3)
+ det = mp.tasks.vision.HandLandmarker.create_from_options(opts)
+ T = len(frames); out = empty(T)
+ h, w = frames[0].shape[:2]
+ for t, f in enumerate(frames):
+ img = mp.Image(image_format=mp.ImageFormat.SRGB, data=cv2.cvtColor(f, cv2.COLOR_BGR2RGB))
+ res = det.detect_for_video(img, int(t * 1000 / fps))
+ for lm, wl, hd in zip(res.hand_landmarks, res.hand_world_landmarks, res.handedness):
+ # MediaPipe assumes a mirrored (selfie) image: swap labels for a normal camera.
+ side = "L" if hd[0].category_name == "Right" else "R"
+ if out[side]["valid"][t] and out[side]["score"][t] >= hd[0].score:
+ continue
+ out[side]["kp2d"][t] = [(p.x * w, p.y * h) for p in lm]
+ out[side]["kp3d"][t] = [(p.x, p.y, p.z) for p in wl]
+ out[side]["valid"][t] = True
+ out[side]["score"][t] = hd[0].score
+ return out
+
+
+def wilor_pipe():
+ import torch
+ from wilor_mini.pipelines.wilor_hand_pose3d_estimation_pipeline import WiLorHandPose3dEstimationPipeline
+
+ root = os.path.join(os.path.dirname(__file__), "..", "hq", "weights", "wilor")
+ return WiLorHandPose3dEstimationPipeline(device=torch.device("cuda"), dtype=torch.float16, verbose=False,
+ wilor_pretrained_dir=root, WILOR_MINI_REPO_ID="__offline__")
+
+
+def run_wilor(frames, fps):
+ pipe = wilor_pipe()
+ T = len(frames); out = empty(T)
+ for t, f in enumerate(frames):
+ rets = pipe.predict(cv2.cvtColor(f, cv2.COLOR_BGR2RGB))
+ for r in rets:
+ if "wilor_preds" not in r:
+ continue
+ side = "R" if r["is_right"] > 0.5 else "L"
+ p = r["wilor_preds"]
+ out[side]["kp2d"][t] = p["pred_keypoints_2d"][0]
+ out[side]["kp3d"][t] = p["pred_keypoints_3d"][0]
+ out[side]["valid"][t] = True
+ out[side]["score"][t] = 1.0
+ return out
+
+
+METHODS = {"mediapipe": run_mediapipe, "wilor": run_wilor}
+
+
+# ---- left/right from the body: models' own handedness flips; body wrists don't ---------------
+def body_wrists(clip, frames):
+ """Per-frame (T,2,2) image px of the subject's [right, left] wrist from GVHMR's ViTPose cache."""
+ import torch
+
+ keys = dict(l.split()[:2] for l in open(os.path.join(os.path.dirname(clip), "keys.txt")))
+ key = keys[os.path.splitext(os.path.basename(clip))[0]]
+ kp = torch.load(os.path.join(os.path.dirname(__file__), "..", "hq", "cache", key, "preprocess", "vitpose.pt"), weights_only=False)
+ kp = np.asarray(kp)[..., :2] # (Tv,17,2) on the 854px / 30fps working copy
+ h, w = frames[0].shape[:2]
+ s = max(h, w) / 854.0
+ Tv, T = len(kp), len(frames)
+ idx = np.clip(np.round(np.arange(T) * (Tv - 1) / max(1, T - 1)).astype(int), 0, Tv - 1)
+ return np.stack([kp[idx, 10], kp[idx, 9]], 1) * s # COCO: 10 = right wrist, 9 = left wrist
+
+
+def relabel(out, wrists):
+ T = len(wrists)
+ hands = [(out[h]["kp2d"][t], out[h]["kp3d"][t], out[h]["score"][t]) if out[h]["valid"][t] else None
+ for t in range(T) for h in HANDS]
+ new = empty(T)
+ for t in range(T):
+ cands = [c for c in hands[2 * t:2 * t + 2] if c is not None]
+ if not cands:
+ continue
+ d = lambda c, i: np.linalg.norm(c[0][0] - wrists[t, i]) # noqa: E731
+ if len(cands) == 2:
+ a, b = cands
+ pairs = [(a, b)] if d(a, 0) + d(b, 1) <= d(a, 1) + d(b, 0) else [(b, a)]
+ assign = list(zip(HANDS, pairs[0]))
+ else:
+ c = cands[0]
+ assign = [(HANDS[0] if d(c, 0) <= d(c, 1) else HANDS[1], c)]
+ for side, (k2, k3, sc) in assign:
+ new[side]["kp2d"][t], new[side]["kp3d"][t], new[side]["score"][t] = k2, k3, sc
+ new[side]["valid"][t] = True
+ return new
+
+
+# ---- metrics -----------------------------------------------------------------------------------
+def metrics(out):
+ rows = []
+ for h in HANDS:
+ v = out[h]["valid"]; k = out[h]["kp2d"]
+ size = np.linalg.norm(k[:, 9] - k[:, 0], axis=-1) + 1e-6 # wrist → middle MCP
+ ok = v[2:] & v[1:-1] & v[:-2]
+ acc = np.linalg.norm(k[2:] - 2 * k[1:-1] + k[:-2], axis=-1).mean(-1) / size[1:-1]
+ jit = float(np.median(acc[ok])) if ok.any() else float("nan")
+ rows.append(f"{h}: detected {v.mean() * 100:5.1f}% jitter {jit:.3f}")
+ return " | ".join(rows)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--method", required=True, choices=METHODS)
+ ap.add_argument("--clip", required=True)
+ a = ap.parse_args()
+ frames, fps = read_frames(a.clip)
+ t0 = time.perf_counter()
+ out = METHODS[a.method](frames, fps)
+ dt = time.perf_counter() - t0
+ try:
+ out = relabel(out, body_wrists(a.clip, frames))
+ except Exception as e: # noqa: BLE001 — no body track: keep the model's own labels
+ print("relabel skipped:", e)
+ os.makedirs(os.path.join(os.path.dirname(__file__), "out"), exist_ok=True)
+ name = os.path.splitext(os.path.basename(a.clip))[0]
+ np.savez_compressed(os.path.join(os.path.dirname(__file__), "out", f"{name}_{a.method}.npz"),
+ **{f"{h}_{k}": v for h in HANDS for k, v in out[h].items()}, fps=fps)
+ print(f"{name} {a.method:10s} {len(frames)} frames in {dt:5.1f}s ({len(frames) / dt:5.1f} fps) {metrics(out)}")
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/bench/viz.py b/bench/viz.py
new file mode 100644
index 0000000..28810fa
--- /dev/null
+++ b/bench/viz.py
@@ -0,0 +1,78 @@
+"""Side-by-side hand crops with each method's 2D skeleton drawn on top.
+
+ python bench/viz.py bench/clips/h1.mp4 mediapipe wilor [--frames 20,50,80,110] [--out x.png]
+
+One row per (frame, hand); columns: plain crop, then one per method.
+"""
+
+import argparse
+import os
+
+import cv2
+import numpy as np
+
+BONES = [(0, 1), (1, 2), (2, 3), (3, 4), (0, 5), (5, 6), (6, 7), (7, 8), (0, 9), (9, 10), (10, 11), (11, 12),
+ (0, 13), (13, 14), (14, 15), (15, 16), (0, 17), (17, 18), (18, 19), (19, 20), (5, 9), (9, 13), (13, 17)]
+FINGER_COLORS = [(255, 255, 255), (0, 200, 255), (0, 255, 0), (255, 200, 0), (255, 0, 255), (0, 0, 255)]
+
+
+def color(a, b):
+ j = max(a, b)
+ return FINGER_COLORS[0 if j == 0 else (j - 1) // 4 + 1]
+
+
+def draw(img, kp, scale, off):
+ for a, b in BONES:
+ pa = tuple(int(v) for v in (kp[a] - off) * scale); pb = tuple(int(v) for v in (kp[b] - off) * scale)
+ cv2.line(img, pa, pb, color(a, b), 2, cv2.LINE_AA)
+ for p in kp:
+ cv2.circle(img, tuple(int(v) for v in (p - off) * scale), 3, (20, 20, 20), -1, cv2.LINE_AA)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("clip"); ap.add_argument("methods", nargs="+")
+ ap.add_argument("--frames", default="20,50,80,110"); ap.add_argument("--out"); ap.add_argument("--size", type=int, default=220)
+ a = ap.parse_args()
+ name = os.path.splitext(os.path.basename(a.clip))[0]
+ data = {m: np.load(os.path.join(os.path.dirname(__file__), "out", f"{name}_{m}.npz")) for m in a.methods}
+ cap = cv2.VideoCapture(a.clip)
+ frames = []
+ while True:
+ ok, f = cap.read()
+ if not ok:
+ break
+ frames.append(f)
+ S = a.size
+ rows = []
+ for t in [int(x) for x in a.frames.split(",")]:
+ for h in ("R", "L"):
+ pts = [data[m][f"{h}_kp2d"][t] for m in a.methods if data[m][f"{h}_valid"][t]]
+ if not pts:
+ continue
+ allp = np.concatenate(pts)
+ c = allp.mean(0); half = max(40.0, np.abs(allp - c).max() * 1.35)
+ off = c - half; scale = S / (2 * half)
+ x0, y0 = int(off[0]), int(off[1]); x1, y1 = int(c[0] + half), int(c[1] + half)
+ src = cv2.copyMakeBorder(frames[t], 400, 400, 400, 400, cv2.BORDER_CONSTANT)
+ crop = cv2.resize(src[y0 + 400:y1 + 400, x0 + 400:x1 + 400], (S, S))
+ tiles = [crop.copy()]
+ for m in a.methods:
+ tile = crop.copy()
+ if data[m][f"{h}_valid"][t]:
+ draw(tile, data[m][f"{h}_kp2d"][t], scale, off)
+ else:
+ cv2.putText(tile, "missed", (8, S // 2), cv2.FONT_HERSHEY_SIMPLEX, 0.7, (0, 0, 255), 2)
+ tiles.append(tile)
+ cv2.putText(tiles[0], f"{name} t{t} {h}", (6, 18), cv2.FONT_HERSHEY_SIMPLEX, 0.5, (0, 255, 255), 1)
+ rows.append(np.hstack(tiles))
+ header = np.zeros((26, rows[0].shape[1], 3), np.uint8)
+ for i, label in enumerate(["frame"] + a.methods):
+ cv2.putText(header, label, (i * S + 6, 18), cv2.FONT_HERSHEY_SIMPLEX, 0.55, (255, 255, 255), 1)
+ out = a.out or os.path.join(os.path.dirname(__file__), "out", f"{name}_viz.png")
+ cv2.imwrite(out, np.vstack([header] + rows))
+ print(out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/hq/hands.py b/hq/hands.py
new file mode 100644
index 0000000..4425f81
--- /dev/null
+++ b/hq/hands.py
@@ -0,0 +1,211 @@
+"""Hand / finger tracking for the HQ pass.
+
+Runs a hand model on the full-resolution source clip, assigns hands to the subject's wrists from
+GVHMR's body keypoints (model handedness flips; the body doesn't), rotates keypoints from camera
+into GVHMR's world frame, resamples to the motion frame rate, fills short gaps and smooths.
+
+Output per hand ("R" / "L", the subject's own): 21 wrist-relative keypoints per frame in world axes
+(Y up), plus a validity mask. Joint order: wrist, thumb 1-4, index 5-8, middle 9-12, ring 13-16,
+pinky 17-20 (MediaPipe / OpenPose order — WiLoR uses the same).
+
+Backends: "wilor" (WiLoR, needs MANO_RIGHT.pkl) and "mediapipe" (MediaPipe Hand Landmarker).
+"""
+
+from __future__ import annotations
+
+import os
+from pathlib import Path
+
+import cv2
+import numpy as np
+
+HQ = Path(__file__).resolve().parent
+WEIGHTS = HQ / "weights"
+SIDES = ("R", "L")
+MAX_GAP_S = 0.4 # interpolate missing stretches up to this long
+SMOOTH_SIGMA = 1.2 # temporal Gaussian (frames at 30fps)
+
+
+def read_frames(path):
+ cap = cv2.VideoCapture(str(path))
+ frames = []
+ while True:
+ ok, f = cap.read()
+ if not ok:
+ break
+ frames.append(f)
+ fps = cap.get(cv2.CAP_PROP_FPS) or 24.0
+ cap.release()
+ return frames, fps
+
+
+# ---- backends: frame list → per-frame list of (kp2d (21,2) px, kp3d (21,3) cam-axes meters, score) ----
+class MediaPipeHands:
+ name = "mediapipe"
+
+ def __init__(self):
+ import mediapipe as mp
+
+ self.mp = mp
+ self.opts = mp.tasks.vision.HandLandmarkerOptions(
+ base_options=mp.tasks.BaseOptions(model_asset_path=str(WEIGHTS / "hand_landmarker.task")),
+ running_mode=mp.tasks.vision.RunningMode.VIDEO, num_hands=2,
+ min_hand_detection_confidence=0.3, min_hand_presence_confidence=0.3, min_tracking_confidence=0.3)
+
+ def __call__(self, frames, fps):
+ det = self.mp.tasks.vision.HandLandmarker.create_from_options(self.opts) # fresh tracker per clip
+ h, w = frames[0].shape[:2]
+ out = []
+ for t, f in enumerate(frames):
+ img = self.mp.Image(image_format=self.mp.ImageFormat.SRGB, data=cv2.cvtColor(f, cv2.COLOR_BGR2RGB))
+ res = det.detect_for_video(img, int(t * 1000 / fps))
+ out.append([(np.array([(p.x * w, p.y * h) for p in lm], np.float32),
+ np.array([(p.x, p.y, p.z) for p in wl], np.float32), hd[0].score)
+ for lm, wl, hd in zip(res.hand_landmarks, res.hand_world_landmarks, res.handedness)])
+ det.close()
+ return out
+
+
+class WiLoRHands:
+ name = "wilor"
+
+ def __init__(self):
+ import torch
+ from wilor_mini.pipelines.wilor_hand_pose3d_estimation_pipeline import WiLorHandPose3dEstimationPipeline
+
+ root = WEIGHTS / "wilor"
+ if not (root / "pretrained_models" / "MANO_RIGHT.pkl").exists():
+ raise FileNotFoundError(f"WiLoR needs MANO_RIGHT.pkl in {root / 'pretrained_models'} (mano.is.tue.mpg.de)")
+ # Every file is present locally, so the pipeline never reaches for its mirror repo.
+ self.pipe = WiLorHandPose3dEstimationPipeline(device=torch.device("cuda"), dtype=torch.float16, verbose=False,
+ wilor_pretrained_dir=str(root), WILOR_MINI_REPO_ID="__offline__")
+
+ def __call__(self, frames, fps):
+ out = []
+ for f in frames:
+ rets = self.pipe.predict(cv2.cvtColor(f, cv2.COLOR_BGR2RGB), hand_conf=0.3)
+ out.append([(r["wilor_preds"]["pred_keypoints_2d"][0].astype(np.float32),
+ r["wilor_preds"]["pred_keypoints_3d"][0].astype(np.float32), 1.0)
+ for r in rets if "wilor_preds" in r])
+ return out
+
+
+_BACKENDS: dict = {}
+
+
+def backend(name: str):
+ if name not in _BACKENDS:
+ _BACKENDS[name] = {"mediapipe": MediaPipeHands, "wilor": WiLoRHands}[name]()
+ return _BACKENDS[name]
+
+
+# ---- post-processing ----------------------------------------------------------------------------
+def assign_to_wrists(dets, wrists):
+ """dets: per-frame candidates; wrists (T,2,2) px [right, left]. → {side: (kp3d (T,21,3), valid (T,))}."""
+ T = len(dets)
+ kp = {s: np.zeros((T, 21, 3), np.float32) for s in SIDES}
+ valid = {s: np.zeros(T, bool) for s in SIDES}
+ for t, cands in enumerate(dets):
+ cands = sorted(cands, key=lambda c: -c[2])[:2]
+ if not cands:
+ continue
+ dist = lambda c, i: float(np.linalg.norm(c[0][0] - wrists[t, i])) # noqa: E731
+ if len(cands) == 2:
+ a, b = cands
+ pairs = [(SIDES[0], a), (SIDES[1], b)] if dist(a, 0) + dist(b, 1) <= dist(a, 1) + dist(b, 0) \
+ else [(SIDES[0], b), (SIDES[1], a)]
+ else:
+ c = cands[0]
+ pairs = [(SIDES[0] if dist(c, 0) <= dist(c, 1) else SIDES[1], c)]
+ # Reject detections far from the wrist they were assigned to (another person, a stray blob).
+ forearm = np.linalg.norm(wrists[t, 0] - wrists[t, 1]) * 0.6 + 40
+ for side, (k2, k3, _) in pairs:
+ if np.linalg.norm(k2[0] - wrists[t, SIDES.index(side)]) < forearm:
+ kp[side][t], valid[side][t] = k3, True
+ return kp, valid
+
+
+def resample(kp, valid, src_fps, n, fps):
+ """Linear resample (T,21,3) from src_fps to n frames at fps; gaps longer than MAX_GAP_S stay invalid."""
+ T = len(kp)
+ t_src = np.arange(T) / src_fps
+ t_dst = np.arange(n) / fps
+ out = np.zeros((n, 21, 3), np.float32)
+ ok = np.zeros(n, bool)
+ vi = np.flatnonzero(valid)
+ if len(vi) < 2:
+ return out, ok
+ flat = kp[vi].reshape(len(vi), -1)
+ for j in range(flat.shape[1]):
+ out.reshape(n, -1)[:, j] = np.interp(t_dst, t_src[vi], flat[:, j])
+ # valid where the nearest valid source samples on both sides are close enough
+ nxt = np.searchsorted(t_src[vi], t_dst)
+ prev_t = t_src[vi][np.clip(nxt - 1, 0, len(vi) - 1)]
+ next_t = t_src[vi][np.clip(nxt, 0, len(vi) - 1)]
+ exact = np.isin(np.round(t_dst * src_fps).astype(int), vi)
+ ok = exact | ((next_t - prev_t) <= MAX_GAP_S) & (t_dst >= t_src[vi][0]) & (t_dst <= t_src[vi][-1])
+ return out, ok
+
+
+def smooth(kp, ok, sigma=SMOOTH_SIGMA):
+ """Gaussian smoothing within each valid run (no blending across gaps)."""
+ if sigma <= 0:
+ return kp
+ r = int(3 * sigma)
+ w = np.exp(-0.5 * (np.arange(-r, r + 1) / sigma) ** 2)
+ out = kp.copy()
+ n = len(kp)
+ i = 0
+ while i < n:
+ if not ok[i]:
+ i += 1
+ continue
+ j = i
+ while j < n and ok[j]:
+ j += 1
+ seg = kp[i:j]
+ for t in range(len(seg)):
+ lo, hi = max(0, t - r), min(len(seg), t + r + 1)
+ ww = w[lo - t + r:hi - t + r]
+ out[i + t] = (seg[lo:hi] * ww[:, None, None]).sum(0) / ww.sum()
+ i = j
+ return out
+
+
+def aa_to_mat(aa):
+ """(N,3) axis-angle → (N,3,3)."""
+ th = np.linalg.norm(aa, axis=-1, keepdims=True) + 1e-9
+ k = aa / th
+ K = np.zeros((len(aa), 3, 3), np.float32)
+ K[:, 0, 1], K[:, 0, 2], K[:, 1, 0] = -k[:, 2], k[:, 1], k[:, 2]
+ K[:, 1, 2], K[:, 2, 0], K[:, 2, 1] = -k[:, 0], -k[:, 1], k[:, 0]
+ s, c = np.sin(th)[..., None], np.cos(th)[..., None]
+ return np.eye(3, dtype=np.float32)[None] + s * K + (1 - c) * (K @ K)
+
+
+def detect(source_video: Path, method: str):
+ """Run the hand model over the full-resolution source clip (independent of GVHMR — run it in parallel)."""
+ frames, src_fps = read_frames(source_video)
+ return backend(method)(frames, src_fps), src_fps, frames[0].shape[:2]
+
+
+def finish(dets, src_fps, hw, vitpose_px, work_long, go_world, go_incam, n: int, fps: float):
+ """Assign to wrists, rotate camera → world, resample to the motion frames, smooth."""
+ T = len(dets)
+ s = max(hw) / float(work_long)
+ Tv = len(vitpose_px)
+ idx = np.clip(np.round(np.arange(T) / src_fps * fps).astype(int), 0, Tv - 1)
+ wrists = np.stack([vitpose_px[idx, 10, :2], vitpose_px[idx, 9, :2]], 1) * s # [right, left] px
+ kp_cam, valid = assign_to_wrists(dets, wrists)
+
+ # camera → GVHMR world, per motion frame: R_wc = R_world(pelvis) · R_cam(pelvis)^T
+ R_wc = aa_to_mat(np.asarray(go_world, np.float32)) @ np.transpose(aa_to_mat(np.asarray(go_incam, np.float32)), (0, 2, 1))
+ out = {}
+ for side in SIDES:
+ k, ok = resample(kp_cam[side], valid[side], src_fps, n, fps)
+ k = k - k[:, :1] # wrist-relative
+ k = np.einsum("tij,tkj->tki", R_wc[:n], k) # into world axes
+ k = smooth(k, ok)
+ out[side] = {"kp": k.reshape(-1).round(4).tolist(), "valid": ok.astype(int).tolist(),
+ "coverage": round(float(ok.mean()), 3)}
+ return out
diff --git a/hq/worker.py b/hq/worker.py
index 7a1798c..cfd31aa 100644
--- a/hq/worker.py
+++ b/hq/worker.py
@@ -219,6 +219,15 @@ def recover(video: Path, out_dir: Path):
_BODY: dict = {}
+def cv2_size(path) -> tuple[int, int]:
+ import cv2
+
+ cap = cv2.VideoCapture(str(path))
+ w, h = int(cap.get(cv2.CAP_PROP_FRAME_WIDTH)), int(cap.get(cv2.CAP_PROP_FRAME_HEIGHT))
+ cap.release()
+ return w, h
+
+
def leg_length(params) -> float:
"""Mean hip→ankle length (m) of the recovered body; models cached across calls.
@@ -244,6 +253,19 @@ def leg_length(params) -> float:
class ProcessIn(BaseModel):
video_url: str
+ hands: str | None = None # "wilor" | "mediapipe" | "off" (default: HQ_HANDS / auto)
+
+
+HANDS_EXEC = ThreadPoolExecutor(max_workers=1, thread_name_prefix="hands")
+
+
+def hands_method(requested: str | None) -> str | None:
+ m = (requested or os.environ.get("HQ_HANDS", "auto")).lower()
+ if m == "off":
+ return None
+ if m == "auto":
+ return "wilor" if (HQ / "weights" / "wilor" / "pretrained_models" / "MANO_RIGHT.pkl").exists() else "mediapipe"
+ return m
@app.get("/health")
@@ -277,6 +299,14 @@ def process(body: ProcessIn):
if not (CACHE / "_warmup.mp4").exists(): # reused to warm CUDA kernels on the next start
shutil.copyfile(video, CACHE / "_warmup.mp4")
+ # Hands only need the full-res clip: detect them while GVHMR runs, assign to wrists afterwards.
+ import hands as handmod
+
+ method = hands_method(body.hands)
+ src = out_dir / "source.mp4"
+ t_h0 = time.perf_counter()
+ hand_job = HANDS_EXEC.submit(handmod.detect, src if src.exists() else video, method) if method else None
+
to_np = lambda x: (x.detach().cpu().numpy() if hasattr(x, "detach") else np.asarray(x)).astype(np.float32) # noqa: E731
def run():
result, t = recover(video, out_dir)
@@ -284,9 +314,10 @@ def process(body: ProcessIn):
t_ll = time.perf_counter()
ll = leg_length(world)
t["leg_length"] = time.perf_counter() - t_ll
- return result, t, world, ll, to_np(result.smpl_params_camera["global_orient"])[0]
+ return result, t, world, ll, to_np(result.smpl_params_camera["global_orient"])
- result, t, world, ll, incam_go0 = GPU.submit(run).result()
+ result, t, world, ll, incam_go = GPU.submit(run).result()
+ incam_go0 = incam_go[0]
go, bp, tr = to_np(world["global_orient"]), to_np(world["body_pose"]), to_np(world["transl"])
n = int(go.shape[0])
@@ -294,9 +325,25 @@ def process(body: ProcessIn):
from motion2mixamorig.extract import playback_fps
fps = playback_fps(video, n, fallback=30.0)
+
+ hands_out, t_hands = None, None
+ if hand_job is not None:
+ try:
+ import torch
+
+ dets, src_fps, hw = hand_job.result()
+ vit = np.asarray(torch.load(out_dir / "preprocess" / "vitpose.pt", weights_only=False))
+ cap = cv2_size(video)
+ hands_out = handmod.finish(dets, src_fps, hw, vit, max(cap), go, incam_go, n, fps)
+ hands_out["method"] = method
+ t_hands = time.perf_counter() - t_h0
+ except Exception as e: # noqa: BLE001 — hands are a bonus; never fail the body motion
+ log(f"hands failed: {type(e).__name__}: {e}")
total = time.perf_counter() - t0
+ hand_note = (f" hands[{method}] {t_hands:.2f}s R {hands_out['R']['coverage']:.0%} L {hands_out['L']['coverage']:.0%}"
+ if hands_out else "")
log(f"{key}: {n} frames @ {fps:.1f}fps download {t_dl:.2f}s gvhmr {t['recover']:.2f}s "
- f"stabilize {t['stabilize']:.2f}s legs {t['leg_length']:.2f}s total {total:.2f}s")
+ f"stabilize {t['stabilize']:.2f}s legs {t['leg_length']:.2f}s{hand_note} total {total:.2f}s")
return {
"n": n, "fps": fps, "leg_length": ll,
"global_orient": go.reshape(-1).round(5).tolist(),
@@ -305,6 +352,8 @@ def process(body: ProcessIn):
# Frame-0 orientation relative to the camera (OpenCV axes): lets the client face the character
# the way the person faces the camera, so chained clips keep a consistent heading.
"incam_global_orient0": incam_go0.round(5).tolist(),
+ # Finger tracking: per hand 21 wrist-relative keypoints/frame in world axes (see hq/hands.py).
+ "hands": hands_out,
"elapsed": total, "timings": {"download": t_dl, **t},
}
diff --git a/public/app.js b/public/app.js
index bf91b88..3f8fb1a 100644
--- a/public/app.js
+++ b/public/app.js
@@ -59,7 +59,7 @@ const videoPrompt = (action, fromImage) => fromImage
const DEFAULTS = {
stream: false, source: 'performer', chain: false, duration: 5, resolution: '480P', hq: true, waitHq: false,
rootMotion: true, poseModel: 'heavy', smoothing: 0.5, showSkel: true, orientation: 'portrait', performerUrls: {},
- character: 'xbot', videoModel: 'h3', swap: false, cinematic: false,
+ character: 'xbot', videoModel: 'h3', swap: false, cinematic: false, hands: true,
};
const settings = (() => { try { return { ...DEFAULTS, ...JSON.parse(localStorage.getItem('p2r') || '{}') }; } catch { return { ...DEFAULTS }; } })();
const save = () => { try { localStorage.setItem('p2r', JSON.stringify(settings)); } catch {} };
@@ -340,7 +340,7 @@ async function checkHQ() {
function runHQ(take) {
const t0 = performance.now();
take.hqStatus = 'running'; renderTakes();
- take.hqPromise = api('/api/hq', { video_url: take.videoUrl })
+ take.hqPromise = api('/api/hq', { video_url: take.videoUrl, ...(take.handsMethod ? { hands: take.handsMethod } : {}) })
.then((r) => {
take.hqRaw = r; take.hq = bakeSmpl(r); take.hqAt = performance.now();
take.hqStatus = 'done'; take.timings.hq = (performance.now() - t0) / 1000;
@@ -350,6 +350,19 @@ function runHQ(take) {
.finally(renderTakes);
}
+// Finger tracking from the HQ worker: per hand, 21 wrist-relative keypoints per frame (GVHMR world axes).
+const _kp = Array.from({ length: 21 }, () => new THREE.Vector3());
+function applyHandsFrame(hands, f, yawFix) {
+ for (const [src, side] of [['R', 'Right'], ['L', 'Left']]) {
+ const h = hands?.[src];
+ if (settings.hands !== false && h?.valid[f]) {
+ const o = f * 63;
+ for (let j = 0; j < 21; j++) _kp[j].set(h.kp[o + j * 3], h.kp[o + j * 3 + 1], h.kp[o + j * 3 + 2]).applyQuaternion(yawFix);
+ rig.applyHand(side, _kp);
+ } else rig.relaxHand(side);
+ }
+}
+
/** GVHMR SMPL params → Track of Mixamo local rotations + grounded hips path. */
function bakeSmpl(r) {
const { n, fps, global_orient: GO, body_pose: BP, transl: T, leg_length } = r;
@@ -373,6 +386,7 @@ function bakeSmpl(r) {
d.set(T[f * 3] - T[0], T[f * 3 + 1] - T[1], T[f * 3 + 2] - T[2]).applyQuaternion(yawFix).multiplyScalar(scale);
const p = rig.rest.Hips.wp.clone().add(d);
rig.setHipsWorld(p);
+ applyHandsFrame(r.hands, f, yawFix);
lows.push(rig.lowestFoot());
track.push(f / fps, rig.snapshot());
}
@@ -618,7 +632,8 @@ bindSetting('swap', 'checked', (on) => { if (!on && model) showOnStage(null); })
bindSetting('cinematic', 'checked', (on) => stage.setCinematic(on));
bindSetting('character', 'value', (c) => { if (rig && CHARACTERS[c]) loadCharacter(CHARACTERS[c].url, CHARACTERS[c].name).catch((e) => log('character failed: ' + e.message, 'err')); });
bindSetting('duration', 'value', (v) => ($('#duration-v').textContent = v + 's'));
-bindSetting('hq', 'checked'); bindSetting('waitHq', 'checked'); bindSetting('rootMotion', 'checked');
+bindSetting('hq', 'checked');
+bindSetting('hands', 'checked', () => { if (rig) for (const t of takes) if (t.hqRaw) t.hq = bakeSmpl(t.hqRaw); }); bindSetting('waitHq', 'checked'); bindSetting('rootMotion', 'checked');
bindSetting('showSkel', 'checked');
bindSetting('smoothing', 'value', setSmoothing);
bindSetting('poseModel', 'value', (v) => { if (vision) initPose(v); });
@@ -659,6 +674,15 @@ $('#file-vid').onchange = (e) => {
// Debug handle for the console.
window.p2r = {
get current() { return current; }, get rig() { return rig; }, takes, video, stats,
+ stage,
+ /** Debug: frame the camera on a bone (e.g. 'RightHand') at distance d. */
+ focusBone(name, d = 0.6, dir = [0.15, 0.1, 1]) {
+ const r = onStage ? cast[onStage].rig : rig;
+ const p = r.bones[name].getWorldPosition(new THREE.Vector3());
+ stage.setCinematic(false); stage.controls.target.copy(p);
+ stage.camera.position.copy(p).add(new THREE.Vector3(...dir).normalize().multiplyScalar(d));
+ stage.setFocusLock?.(p);
+ },
// ---- frame-exact capture: used by tools/capture.mjs ----
submit(prompt, { duration } = {}) {
if (duration) { settings.duration = duration; $('#duration').value = duration; $('#duration-v').textContent = duration + 's'; }
@@ -683,10 +707,11 @@ window.p2r = {
video.currentTime = t; await seeked;
}
lastDetectT = -1; tick();
- await new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r)));
+ // Two frames so the compositor has painted (with a timeout: hidden tabs don't run rAF).
+ await Promise.race([new Promise((r) => requestAnimationFrame(() => requestAnimationFrame(r))), new Promise((r) => setTimeout(r, 80))]);
},
endCapture() { capturing = false; video.play().catch(() => {}); },
- playUrl(u, hq = true) { const t = new Take('debug: ' + u.slice(-24)); t.videoUrl = u; t.status = 'ready'; takes.push(t); if (hq && hqOnline) runHQ(t); enqueue(t); return t; },
+ playUrl(u, hq = true, hands) { const t = new Take('debug: ' + u.slice(-24)); t.videoUrl = u; t.status = 'ready'; t.handsMethod = hands; takes.push(t); if (hq && hqOnline) runHQ(t); enqueue(t); return { id: t.id }; },
};
// ---- boot --------------------------------------------------------------------------------------
diff --git a/public/index.html b/public/index.html
index 294b3f0..f6f0920 100644
--- a/public/index.html
+++ b/public/index.html
@@ -130,6 +130,7 @@
<label class="title">Motion</label>
<label class="check"><input type="checkbox" id="hq" /><span>HQ pass (GVHMR on local GPU)<small>Live MediaPipe preview immediately, swaps to GVHMR motion when ready.</small></span></label>
<label class="check"><input type="checkbox" id="waitHq" /><span>Hold playback until HQ is ready<small>Cleanest look; adds a few seconds before each clip.</small></span></label>
+ <label class="check"><input type="checkbox" id="hands" /><span>Track fingers<small>Hand model on the source clip → wrist + finger poses in the HQ motion. Untracked hands rest in a natural curl.</small></span></label>
<label class="check"><input type="checkbox" id="rootMotion" /><span>Root motion</span></label>
<div class="row"><span>Live pose model</span><select id="poseModel"><option value="heavy">Heavy</option><option value="full">Full</option></select></div>
<div class="row"><span>Live smoothing</span><input type="range" id="smoothing" min="0" max="1" step="0.05" /></div>
diff --git a/public/retarget.js b/public/retarget.js
index b133944..6a944eb 100644
--- a/public/retarget.js
+++ b/public/retarget.js
@@ -10,10 +10,13 @@ import * as THREE from 'three';
const V3 = THREE.Vector3;
const Q = THREE.Quaternion;
+const fingerBones = (side) => ['Thumb', 'Index', 'Middle', 'Ring', 'Pinky'].flatMap((f) => [1, 2, 3].map((k) => `${side}Hand${f}${k}`));
+
+// Parent-first, so world-space writes (copyPoseFrom) can go in order.
export const BONES = [
'Hips', 'Spine', 'Spine1', 'Spine2', 'Neck', 'Head',
- 'LeftShoulder', 'LeftArm', 'LeftForeArm', 'LeftHand',
- 'RightShoulder', 'RightArm', 'RightForeArm', 'RightHand',
+ 'LeftShoulder', 'LeftArm', 'LeftForeArm', 'LeftHand', ...fingerBones('Left'),
+ 'RightShoulder', 'RightArm', 'RightForeArm', 'RightHand', ...fingerBones('Right'),
'LeftUpLeg', 'LeftLeg', 'LeftFoot', 'LeftToeBase',
'RightUpLeg', 'RightLeg', 'RightFoot', 'RightToeBase',
];
@@ -35,6 +38,30 @@ const MP = {
lHeel: 29, rHeel: 30, lFoot: 31, rFoot: 32,
};
+// Hand keypoint chains (MediaPipe / OpenPose order) → Mixamo finger names.
+const HAND_CHAINS = [
+ ['Thumb', [1, 2, 3, 4]], ['Index', [5, 6, 7, 8]], ['Middle', [9, 10, 11, 12]],
+ ['Ring', [13, 14, 15, 16]], ['Pinky', [17, 18, 19, 20]],
+];
+
+/**
+ * Anatomical finger limits. Middle/end joints (PIP/DIP) are hinges: keep only the curl toward the palm,
+ * 0°–100°, no sideways or backward bending. Knuckles (MCP) keep their sideways spread but can't
+ * hyperextend more than ~20°. Removes most single-camera depth noise from tracked fingers.
+ */
+function hinge(dir, parentDir, palm, isKnuckle) {
+ const u = palm.clone().sub(parentDir.clone().multiplyScalar(palm.dot(parentDir))).normalize(); // palm side, ⟂ parent
+ const along = dir.dot(parentDir), curl = dir.dot(u);
+ if (isKnuckle) {
+ const lateral = dir.clone().sub(parentDir.clone().multiplyScalar(along)).sub(u.clone().multiplyScalar(curl));
+ const flex = Math.min(1.6, Math.max(-0.35, Math.atan2(curl, along)));
+ return parentDir.clone().multiplyScalar(Math.cos(flex)).add(u.clone().multiplyScalar(Math.sin(flex)))
+ .add(lateral.clampLength(0, 0.45)).normalize();
+ }
+ const flex = Math.min(1.75, Math.max(0, Math.atan2(curl, along)));
+ return parentDir.clone().multiplyScalar(Math.cos(flex)).add(u.multiplyScalar(Math.sin(flex))).normalize();
+}
+
const FOOT_POINTS = ['LeftToe_End', 'RightToe_End', 'LeftToeBase', 'RightToeBase', 'LeftFoot', 'RightFoot'];
const shortName = (n) => n.replace(/^mixamorig\d*[:_]?/i, '');
@@ -275,6 +302,64 @@ export class Rig {
if (weight < 1) { b.quaternion.copy(before.slerp(b.quaternion, weight)); b.updateMatrixWorld(true); }
}
+ // ---- hands / fingers ---------------------------------------------------------
+ /** Swing `name` so its bone (toward `childName`) points along world direction `dir`. */
+ aimTo(name, childName, dir) {
+ const b = this.bones[name], c = this.bones[childName];
+ if (!b || !c || dir.lengthSq() < 1e-10) return;
+ const cur = c.getWorldPosition(_v1).sub(b.getWorldPosition(_v2)).normalize();
+ const delta = new Q().setFromUnitVectors(cur, dir.clone().normalize());
+ this.setWorldQuat(name, delta.multiply(b.getWorldQuaternion(_q2)));
+ }
+
+ /**
+ * Pose a hand from 21 tracked keypoints (world axes, wrist-relative; MediaPipe/OpenPose order).
+ * `side` is 'Left' | 'Right' (the character's own). Palm orientation comes from the knuckles,
+ * each finger bone is aimed along its measured segment — directions only, so hand size doesn't matter.
+ */
+ applyHand(side, kp) {
+ const hand = `${side}Hand`;
+ if (!this.bones[hand]) return;
+ const along = kp[9].clone().sub(kp[0]).normalize();
+ const across = kp[5].clone().sub(kp[17]);
+ this.aimHand(hand, along, across, 1);
+ // Palm-side normal (mirror between hands): fingers may only curl toward it.
+ const palm = new V3().crossVectors(along, across).normalize();
+ if (side === 'Right') palm.negate();
+ for (const [finger, ids] of HAND_CHAINS) {
+ let parentDir = along;
+ for (let k = 1; k <= 3; k++) {
+ const name = `${side}Hand${finger}${k}`, child = `${side}Hand${finger}${k + 1}`;
+ if (!this.bones[name] || !this.bones[child]) break;
+ let dir = kp[ids[k]].clone().sub(kp[ids[k - 1]]).normalize();
+ if (finger !== 'Thumb') dir = hinge(dir, parentDir, palm, k === 1);
+ this.aimTo(name, child, dir);
+ parentDir = dir;
+ }
+ }
+ }
+
+ /** No tracking for this hand: a relaxed natural curl instead of a flat, rigid T-pose hand. */
+ relaxHand(side, amount = 1) {
+ const hand = this.bones[`${side}Hand`];
+ if (!hand) return;
+ // Palms face down in the Mixamo T-pose; carry that "palm side" along with the current hand pose.
+ const palm = new V3(0, -1, 0).applyQuaternion(hand.getWorldQuaternion(new Q()).multiply(_q3.copy(this.rest[`${side}Hand`].wq).invert()));
+ for (const [finger] of HAND_CHAINS) {
+ const curl = finger === 'Thumb' ? [0.12, 0.18, 0.15] : [0.32, 0.42, 0.3];
+ for (let k = 1; k <= 3; k++) {
+ const name = `${side}Hand${finger}${k}`, child = `${side}Hand${finger}${k + 1}`;
+ const b = this.bones[name], c = this.bones[child];
+ if (!b || !c) break;
+ const dir = c.getWorldPosition(_v1).sub(b.getWorldPosition(_v2)).normalize();
+ const axis = new V3().crossVectors(dir, palm).normalize();
+ if (axis.lengthSq() < 1e-6) continue;
+ const rot = new Q().setFromAxisAngle(axis, curl[k - 1] * amount);
+ this.setWorldQuat(name, rot.multiply(b.getWorldQuaternion(_q2)));
+ }
+ }
+ }
+
// ---- source B: SMPL (GVHMR) -------------------------------------------------
/** `G` = 22 global joint quaternions (three.js frame), relative to the SMPL T-pose. */
applySmplGlobal(G) {
diff --git a/public/stage.js b/public/stage.js
index 14777b3..a82762b 100644
--- a/public/stage.js
+++ b/public/stage.js
@@ -130,6 +130,7 @@ export function createStage(container) {
function fitFog() { const d = baseDistance(); scene.fog.near = d + 3; scene.fog.far = d + 20; }
function update(focus) {
+ if (state.lock) { controls.target.copy(state.lock); controls.update(); renderer.render(scene, camera); return; }
if (focus) state.focus.lerp(new THREE.Vector3(focus.x, 0.95, focus.z), 0.04);
const now = performance.now();
if (state.cinematic && now > state.pausedUntil) {
@@ -160,6 +161,8 @@ export function createStage(container) {
renderer, scene, camera, controls, update, contactShadow, removeContactShadow, placeContactShadow,
setCinematic(on) { state.cinematic = on; state.t0 = performance.now(); },
/** Width in meters of what should stay in frame (solo ≈ 1.4, crowd wider). */
+ /** Pin the camera target (debug close-ups); null to release. */
+ setFocusLock(p) { state.lock = p ? p.clone() : null; },
setFraming(widthMeters) { state.width = widthMeters; reframe(); fitFog(); },
/** Pixels on the left covered by the video panel. */
setInset(px) { if (Math.abs(px - state.inset) > 2) { state.inset = px; applyInset(); reframe(); fitFog(); } },
diff --git a/server.mjs b/server.mjs
index 2a94f4f..84260fb 100644
--- a/server.mjs
+++ b/server.mjs
@@ -137,7 +137,7 @@ const server = http.createServer(async (req, res) => {
const b = await readJson(req);
const r = await fetch(`${HQ_URL}/process`, {
method: 'POST', headers: { 'Content-Type': 'application/json' },
- body: JSON.stringify({ video_url: b.video_url }),
+ body: JSON.stringify({ video_url: b.video_url, ...(b.hands ? { hands: b.hands } : {}) }),
});
const text = await r.text();
res.writeHead(r.status, { 'Content-Type': 'application/json' });