Skip to content

Type a motion, watch a rigged character perform it: fast video model + GVHMR + live Mixamo retargeting

JavaScript

Portrait (9:16) clips by default, with an orientation setting

blendi-remade65ad1ad0b5a1
diff --git a/README.md b/README.md
index 17139fc..9308abd 100644
--- a/README.md
+++ b/README.md
@@ -54,6 +54,7 @@ hq\.venv\Scripts\python hq\worker.py # → 127.0.0.1:5178, loads once and sta
## Using it
- **Generate** a prompt or click a preset. By default every clip starts from the same flux-generated *performer* (one person, full body, plain backdrop, static camera — ideal pose-estimation input).
+- **Orientation** — portrait 9:16 (default) or landscape 16:9. Portrait gives the performer ~70% more pixels of height at 480P, which helps pose estimation.
- **Stream mode** — one continuous performance: each clip continues from the previous clip's last frame, plays once its HQ motion is ready while the next one generates, and short idle clips fill gaps between prompts. First motion ≈ 6s after the first prompt, continuous after that.
- **⬇ GLB** on any take exports the skinned character with the animation clip (Blender, Unity, three.js).
- **Load character…** swaps in any Mixamo-rigged `.glb`/`.fbx`; **Use a video file…** runs the live pass on your own footage.
diff --git a/hq/worker.py b/hq/worker.py
index 5a75143..f580fc5 100644
--- a/hq/worker.py
+++ b/hq/worker.py
@@ -170,6 +170,11 @@ def warmup() -> None:
shutil.rmtree(CACHE / "_warmup", ignore_errors=True) # cached stages would skip the warm-up
result, _ = recover(clip, CACHE / "_warmup")
leg_length(result.smpl_params_world)
+ if DETECTOR == "fast": # first YOLO call per input shape is slow: cover both orientations
+ det = FastSingleDetector.get()
+ for h, w in ((480, 832), (832, 480), (768, 1360), (1360, 768)):
+ for n in (det.batch, 27):
+ det.yolo.predict([np.zeros((h, w, 3), np.uint8)] * n, classes=[0], half=True, device=0, verbose=False)
log(f"warmup done in {time.perf_counter() - t0:.1f}s")
except Exception as e: # noqa: BLE001
log(f"warmup failed: {e}")
diff --git a/public/app.js b/public/app.js
index ee1c8c4..d73fe5c 100644
--- a/public/app.js
+++ b/public/app.js
@@ -16,7 +16,7 @@ const MP_MODEL = (v) => `https://storage.googleapis.com/mediapipe-models/pose_la
const PERFORMER_PROMPT =
'Full-body photograph of one young adult standing in a relaxed neutral pose facing the camera, arms relaxed at the sides, ' +
'wearing a fitted plain light-grey t-shirt, black slim pants and white sneakers. The whole body is visible from head to feet ' +
- 'with generous empty space around, the person is centered and occupies about half of the frame height. Plain seamless ' +
+ 'with empty space above the head and below the feet, the person is centered. Plain seamless ' +
'light-grey studio backdrop and floor, flat even soft lighting, sharp focus, 35mm photo.';
const PRESETS = [
@@ -43,7 +43,7 @@ const videoPrompt = (action, fromImage) => fromImage
// ---- settings ------------------------------------------------------------------
const DEFAULTS = {
stream: false, source: 'performer', chain: false, duration: 5, resolution: '480P', hq: true, waitHq: false,
- rootMotion: true, poseModel: 'heavy', smoothing: 0.5, showSkel: true, performerUrl: null,
+ rootMotion: true, poseModel: 'heavy', smoothing: 0.5, showSkel: true, orientation: 'portrait', performerUrls: {},
};
const settings = (() => { try { return { ...DEFAULTS, ...JSON.parse(localStorage.getItem('p2r') || '{}') }; } catch { return { ...DEFAULTS }; } })();
const save = () => { try { localStorage.setItem('p2r', JSON.stringify(settings)); } catch {} };
@@ -225,12 +225,13 @@ function maybeIdle() {
}
async function ensurePerformer(force = false) {
- if (settings.performerUrl && !force) return settings.performerUrl;
+ const o = settings.orientation;
+ if (settings.performerUrls[o] && !force) return settings.performerUrls[o];
banner('Creating performer…');
const t0 = performance.now();
- const r = await api('/api/image', { prompt: PERFORMER_PROMPT });
- settings.performerUrl = r.image_url; save();
- $('#performer-img').src = r.image_url;
+ const r = await api('/api/image', { prompt: PERFORMER_PROMPT, image_size: o === 'portrait' ? 'portrait_16_9' : 'landscape_16_9' });
+ settings.performerUrls[o] = r.image_url; save();
+ showPerformer();
log(`performer image in ${((performance.now() - t0) / 1000).toFixed(1)}s`, 'ok');
banner('');
return r.image_url;
@@ -249,7 +250,7 @@ async function generate(take) {
log(`#${take.id} → H3 Max Turbo (${from}, ${settings.duration}s ${settings.resolution})`);
const r = await api('/api/video', {
prompt: videoPrompt(take.prompt, !!image_url), image_url,
- duration: settings.duration, resolution: settings.resolution, aspect_ratio: '16:9',
+ duration: settings.duration, resolution: settings.resolution, aspect_ratio: settings.orientation === 'portrait' ? '9:16' : '16:9',
});
take.videoUrl = r.video_url; take.status = 'ready';
take.timings.gen = (performance.now() - t0) / 1000;
@@ -406,6 +407,7 @@ function playTake(take) {
$('#vid-empty').style.display = 'none';
video.style.display = 'block';
video.src = take.videoUrl;
+ video.onloadedmetadata = () => $('#vidwrap').classList.toggle('portrait', video.videoHeight > video.videoWidth);
lastDetectT = -1;
video.play().catch(() => {});
renderTakes();
@@ -579,7 +581,13 @@ for (const p of PRESETS) {
$('#go').onclick = () => submitPrompt($('#prompt').value);
$('#prompt').addEventListener('keydown', (e) => { if (e.key === 'Enter' && (e.ctrlKey || e.metaKey)) { e.preventDefault(); submitPrompt($('#prompt').value); } });
$('#performer-new').onclick = () => ensurePerformer(true).catch((e) => { banner(''); log('performer failed: ' + e.message, 'err'); });
-if (settings.performerUrl) $('#performer-img').src = settings.performerUrl;
+function showPerformer() {
+ const img = $('#performer-img'), url = settings.performerUrls[settings.orientation];
+ img.classList.toggle('portrait', settings.orientation === 'portrait');
+ if (url) img.src = url; else img.removeAttribute('src');
+}
+if (settings.performerUrl) { settings.performerUrls.landscape ||= settings.performerUrl; delete settings.performerUrl; save(); }
+bindSetting('orientation', 'value', showPerformer);
$('#btn-char').onclick = () => $('#file-char').click();
$('#file-char').onchange = (e) => {
diff --git a/public/index.html b/public/index.html
index 71710c3..432d7d6 100644
--- a/public/index.html
+++ b/public/index.html
@@ -44,6 +44,7 @@
.check small { display: block; color: var(--dim); }
.performer { display: flex; gap: 10px; align-items: center; }
.performer img { width: 120px; aspect-ratio: 16/9; object-fit: cover; border-radius: 6px; border: 1px solid var(--line); background: var(--panel2); }
+ .performer img.portrait { width: 62px; aspect-ratio: 9/16; }
.btn { padding: 5px 10px; border-radius: 6px; border: 1px solid var(--line); background: var(--panel2); cursor: pointer; }
.btn:hover { border-color: var(--accent2); }
.log { font: 11.5px/1.45 ui-monospace, Consolas, monospace; color: var(--dim); background: #0b0c0e; border: 1px solid var(--line); border-radius: 8px; padding: 8px; max-height: 160px; overflow-y: auto; white-space: pre-wrap; }
@@ -56,6 +57,8 @@
.vidwrap video, .vidwrap canvas { display: block; width: 100%; }
.vidwrap canvas { position: absolute; inset: 0; height: 100%; pointer-events: none; }
.vidwrap .empty { aspect-ratio: 16/9; display: grid; place-items: center; color: var(--dim); }
+ .vidwrap.portrait { width: auto; height: min(62%, 640px); aspect-ratio: 9/16; }
+ .vidwrap.portrait video { width: 100%; height: 100%; object-fit: contain; }
.hud { position: absolute; right: 14px; top: 14px; display: flex; flex-direction: column; gap: 6px; align-items: flex-end; pointer-events: none; }
.hud .tag { font: 12px ui-monospace, Consolas, monospace; background: rgba(14,15,17,.75); border: 1px solid var(--line); border-radius: 6px; padding: 4px 8px; }
.hud .src-live { color: var(--warn); } .hud .src-hq { color: var(--accent); }
@@ -104,6 +107,8 @@
<div class="section">
<label class="title">Generation</label>
<label class="check"><input type="checkbox" id="stream" /><span><b>Stream mode</b><small>One continuous performance: each clip continues from the last, plays once its HQ motion is ready, and idles between prompts.</small></span></label>
+ <div class="row"><span>Orientation</span>
+ <select id="orientation"><option value="portrait">Portrait 9:16</option><option value="landscape">Landscape 16:9</option></select></div>
<div class="row"><span>Source</span>
<select id="source"><option value="performer">Performer → video</option><option value="t2v">Text → video</option></select></div>
<label class="check"><input type="checkbox" id="chain" /><span>Continue from last frame<small>Each clip starts where the previous ended — a continuous stream you direct with prompts.</small></span></label>