Skip to content
Python

Added Steady Dancer support

DeepBeepMeep3feaa249cfcb
diff --git a/README.md b/README.md
index 6f4f6c6..44de59c 100644
--- a/README.md
+++ b/README.md
@@ -37,7 +37,7 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models
## 🔥 Latest Updates :
-### November 28 2025: WanGP v9.72, The Alpha & the Omega
+### December 2 2025: WanGP v9.72, The Alpha & the Omega ... and the Dancer
- **Flux 2**: the best ever open source *Image Generator* has just landed. It does everything very well: generate an *Image* based a *Text Prompt* or combine up to 10 *Images References*
@@ -47,12 +47,15 @@ Behold the WanGP Miracle ! Flux 2 wil work with only 8 GB of VRAM if you are hap
- **Z-Image**: a small model, very fast (8 steps), very low VRAM (optimized even more in WanGP for fun, just in case you want to generate 16 images at a time) that produces outstanding Image quality. Not yet the Flux 2 level, and no Image editing yet but a very good trade-off.
+- **Steady Dancer**: here is *Wan Steady Dancer* a very nice alternative to *Wan Animate*. You can transfer the motion of a Control video in a very smooth way. It will work best with Videos where the action happens center stage (hint: *dancing*). Use the *Lora accelerator* *Fusionix i2v 10 steps* for a fast generation. For higher quality you can set *Condition Guidance* to 2 or if you are very patient keep *Guidance* to a value greater than 1.
+
I have added a new Memory Profile *Profile 4+* that is sligthly slower than *Profile 4* but can save you up to 1GB of VRAM with Flux 2.
Also as we have now quite few models and Loras folders. *I have moved all the loras folder in the 'loras' folder*. There are also now unique subfolders for *Wan 5B* and *Wan 1.3B* models. A conversion script should have moved the loras in the right locations, but I advise that you check just in case.
*update 9.71* : added missing source file, have fun !
*update 9.72* : added Z-Image & Loras reorg
+*update 9.73* : added Steady Dancer
### November 24 2025: WanGP v9.62, The Return of the King
diff --git a/models/wan/any2video.py b/models/wan/any2video.py
index c7d9a42..231e36f 100644
--- a/models/wan/any2video.py
+++ b/models/wan/any2video.py
@@ -406,6 +406,7 @@ class WanAny2V:
audio_cfg_scale=None,
audio_proj=None,
audio_context_lens=None,
+ alt_guide_scale = 1.0,
overlapped_latents = None,
return_latent_slice = None,
overlap_noise = 0,
@@ -525,9 +526,10 @@ class WanAny2V:
lucy_edit= model_type in ["lucy_edit"]
animate= model_type in ["animate"]
chrono_edit = model_type in ["chrono_edit"]
- mocha = model_type in ["mocha"]
+ mocha = model_type in ["mocha"]
+ steadydancer = model_type in ["steadydancer"]
start_step_no = 0
- ref_images_count = 0
+ ref_images_count = inner_latent_frames = 0
trim_frames = 0
last_latent_preview = False
extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = latent_slice = freqs = None
@@ -638,9 +640,28 @@ class WanAny2V:
lat_y = self.vae.encode([input_video], VAE_tile_size)[0]
extended_overlapped_latents = lat_y[:, :overlapped_latents_frames_num].clone().unsqueeze(0)
- lat_y = input_video = None
+ lat_y = None
kwargs.update({ 'y': y})
+
+ # Steady Dancer
+ if steadydancer:
+ condition_guide_scale = alt_guide_scale # 2.0
+ # ref img_x
+ ref_x = self.vae.encode([input_video[:, :1]], VAE_tile_size)[0]
+ msk_ref = torch.ones(4, 1, lat_h, lat_w, device=self.device)
+ ref_x = torch.concat([ref_x, msk_ref, ref_x])
+ # ref img_c
+ ref_c = self.vae.encode([input_frames[:, :1]], VAE_tile_size)[0]
+ msk_c = torch.zeros(4, 1, lat_h, lat_w, device=self.device)
+ ref_c = torch.concat([ref_c, msk_c, ref_c])
+ kwargs.update({ 'steadydancer_ref_x': ref_x, 'steadydancer_ref_c': ref_c})
+ # conditions, w/o msk
+ conditions = self.vae.encode([input_frames])[0].unsqueeze(0)
+ # conditions_null, w/o msk
+ conditions_null = self.vae.encode([input_frames2])[0].unsqueeze(0)
+ inner_latent_frames = 2
+
# Chrono Edit
if chrono_edit:
if frame_num == 5:
@@ -690,6 +711,8 @@ class WanAny2V:
clip_context = self.clip.visual([clip_image_start[:, None, :, :]])
clip_image_start = clip_image_end = None
kwargs.update({'clip_fea': clip_context})
+ if steadydancer:
+ kwargs['steadydancer_clip_fea_c'] = self.clip.visual([input_frames[:, :1]])
# Recam Master & Lucy Edit
if recam or lucy_edit:
@@ -883,7 +906,7 @@ class WanAny2V:
shape[extended_input_dim-2] *= 2
freqs = get_rotary_pos_embed(shape, enable_RIFLEx= False)
else:
- freqs = get_rotary_pos_embed(target_shape[1:], enable_RIFLEx= enable_RIFLEx)
+ freqs = get_rotary_pos_embed( (target_shape[1]+ inner_latent_frames ,) + target_shape[2:] , enable_RIFLEx= enable_RIFLEx)
kwargs["freqs"] = freqs
@@ -1035,6 +1058,20 @@ class WanAny2V:
if model_type in ["lynx", "vace_lynx_14B"]:
gen_args["lynx_ref_buffer"] = [lynx_ref_buffer, lynx_ref_buffer_uncond]
+ elif steadydancer:
+ if guide_scale == 1:
+ gen_args = {
+ "x" : [latent_model_input, latent_model_input],
+ "context" : [context, context],
+ "steadydancer_condition": [conditions, conditions_null],
+ }
+ any_guidance = condition_guide_scale != 1
+ else:
+ gen_args = {
+ "x" : [latent_model_input, latent_model_input, latent_model_input],
+ "context" : [context, context_null, context],
+ "steadydancer_condition": [conditions, conditions, conditions_null],
+ }
elif multitalk and audio_proj != None:
if guide_scale == 1:
gen_args = {
@@ -1082,6 +1119,34 @@ class WanAny2V:
noise_pred_cond, noise_pred_noaudio, noise_pred_uncond = ret_values
noise_pred = noise_pred_uncond + guide_scale * (noise_pred_noaudio - noise_pred_uncond) + audio_cfg_scale * (noise_pred_cond - noise_pred_noaudio)
noise_pred_noaudio = None
+ elif steadydancer:
+ if apg_switch != 0 and False:
+ if guide_scale == 1:
+ noise_pred_cond, noise_pred_drop_audio = ret_values
+ noise_pred = noise_pred_cond + (audio_cfg_scale - 1)* adaptive_projected_guidance(noise_pred_cond - noise_pred_drop_audio,
+ noise_pred_cond,
+ momentum_buffer=audio_momentumbuffer,
+ norm_threshold=apg_norm_threshold)
+
+ else:
+ noise_pred_cond, noise_pred_drop_text, noise_pred_uncond = ret_values
+ noise_pred = noise_pred_cond + (guide_scale - 1) * adaptive_projected_guidance(noise_pred_cond - noise_pred_drop_text,
+ noise_pred_cond,
+ momentum_buffer=text_momentumbuffer,
+ norm_threshold=apg_norm_threshold) \
+ + (audio_cfg_scale - 1) * adaptive_projected_guidance(noise_pred_drop_text - noise_pred_uncond,
+ noise_pred_cond,
+ momentum_buffer=audio_momentumbuffer,
+ norm_threshold=apg_norm_threshold)
+ else:
+ if guide_scale == 1:
+ noise_pred_cond, noise_pred_uncond_condition = ret_values
+ noise_pred = noise_pred_uncond_condition + condition_guide_scale * (noise_pred_cond - noise_pred_uncond_condition)
+ else:
+ noise_pred_cond, noise_pred_uncond_context, noise_pred_uncond_condition = ret_values
+ noise_pred = noise_pred_uncond_context + guide_scale * (noise_pred_cond - noise_pred_uncond_context) + condition_guide_scale * (noise_pred_cond - noise_pred_uncond_condition)
+ noise_pred_uncond_context = noise_pred_cond = noise_pred_uncond_condition = None
+
elif multitalk and audio_proj != None:
if apg_switch != 0:
if guide_scale == 1:
diff --git a/models/wan/configs/steadydancer.json b/models/wan/configs/steadydancer.json
new file mode 100644
index 0000000..a00032f
--- /dev/null
+++ b/models/wan/configs/steadydancer.json
@@ -0,0 +1,15 @@
+{
+ "_class_name": "WanModel",
+ "_diffusers_version": "0.30.0",
+ "dim": 5120,
+ "eps": 1e-06,
+ "ffn_dim": 13824,
+ "freq_dim": 256,
+ "in_dim": 36,
+ "model_type": "i2v",
+ "num_heads": 40,
+ "num_layers": 40,
+ "out_dim": 16,
+ "text_len": 512,
+ "steadydancer": true
+}
diff --git a/models/wan/modules/model.py b/models/wan/modules/model.py
index 5a6ec58..5c4f86b 100644
--- a/models/wan/modules/model.py
+++ b/models/wan/modules/model.py
@@ -19,6 +19,8 @@ from ..multitalk.multitalk_utils import get_attn_map_with_target
from ..animate.motion_encoder import Generator
from ..animate.face_blocks import FaceAdapter, FaceEncoder
from ..animate.model_animate import after_patch_embedding
+from ..steadydancer.small_archs import FactorConv3d, PoseRefNetNoBNV3
+from ..steadydancer.mobilenetv2_dcd import DYModule
__all__ = ['WanModel']
@@ -1028,6 +1030,7 @@ class WanModel(ModelMixin, ConfigMixin):
standin= False,
motion_encoder_dim=0,
lynx=None,
+ steadydancer = False,
):
super().__init__()
@@ -1059,7 +1062,8 @@ class WanModel(ModelMixin, ConfigMixin):
self.vae_scale = vae_scale
multitalk = multitalk_output_dim > 0
- self.multitalk = multitalk
+ self.multitalk = multitalk
+ self.steadydancer = steadydancer
animate = motion_encoder_dim > 0
# embeddings
@@ -1181,6 +1185,32 @@ class WanModel(ModelMixin, ConfigMixin):
num_heads=4,
)
+ if steadydancer:
+ self.in_dim_c = 16
+
+ ############### Condition-Reconciliation Mechanism ###############
+ self.patch_embedding_fuse = nn.Conv3d( # x, fused pose, aligned pose
+ in_dim + self.in_dim_c + self.in_dim_c, dim, kernel_size=patch_size, stride=patch_size)
+ self.patch_embedding_ref_c = nn.Conv3d( # ref_c
+ self.in_dim_c, dim, kernel_size=patch_size, stride=patch_size)
+
+ ############### Synergistic Pose Modulation Modules ###############
+ # Spatial Structure Adaptive Extractor
+ self.condition_embedding_spatial = DYModule(inp=self.in_dim_c, oup=self.in_dim_c)
+ # Temporal Motion Coherence Module
+ self.condition_embedding_temporal = nn.Sequential(
+ FactorConv3d(in_channels=self.in_dim_c, out_channels=self.in_dim_c, kernel_size=(3, 3, 3), stride=1),
+ nn.SiLU(),
+ FactorConv3d(in_channels=self.in_dim_c, out_channels=self.in_dim_c, kernel_size=(3, 3, 3), stride=1),
+ nn.SiLU(),
+ FactorConv3d(in_channels=self.in_dim_c, out_channels=self.in_dim_c, kernel_size=(3, 3, 3), stride=1),
+ nn.SiLU()
+ )
+ # Frame-wise Attention Alignment Unit
+ self.condition_embedding_align = PoseRefNetNoBNV3(in_channels_x=16,
+ in_channels_c=16,
+ hidden_dim=128,
+ num_heads=8)
def adapt_modulation(self, block_name ='blocks'):
def move(v, param_name = "modulation"):
@@ -1414,8 +1444,11 @@ class WanModel(ModelMixin, ConfigMixin):
lynx_ip_scale = 0,
lynx_ref_scale = 0,
lynx_feature_extractor = False,
- lynx_ref_buffer = None,
-
+ lynx_ref_buffer = None,
+ steadydancer_condition = None,
+ steadydancer_ref_x = None,
+ steadydancer_ref_c = None,
+ steadydancer_clip_fea_c = None,
):
# patch_dtype = self.patch_embedding.weight.dtype
modulation_dtype = self.time_projection[1].weight.dtype
@@ -1430,11 +1463,15 @@ class WanModel(ModelMixin, ConfigMixin):
if chipmunk:
# from chipmunk.ops.voxel import voxel_chunk_no_padding, reverse_voxel_chunk_no_padding
voxel_shape = (4, 6, 8)
-
+ real_seq = 0
x_list = x
joint_pass = len(x_list) > 1
is_source_x = [ x.data_ptr() == x_list[0].data_ptr() and i > 0 for i, x in enumerate(x_list) ]
last_x_idx = 0
+ steadydancer = steadydancer_condition is not None
+ if steadydancer: # steady dancer
+ x_noise_clone = x_list[0].clone()
+
for i, (is_source, x) in enumerate(zip(is_source_x, x_list)):
if is_source:
x_list[i] = x_list[0].clone()
@@ -1447,11 +1484,32 @@ class WanModel(ModelMixin, ConfigMixin):
if bz > 1: y = y.expand(bz, -1, -1, -1, -1)
x = torch.cat([x, y], dim=1)
# embeddings
- x = self.patch_embedding(x).to(modulation_dtype)
- grid_sizes = x.shape[2:]
+ if not steadydancer:
+ x = self.patch_embedding(x).to(modulation_dtype)
+ grid_sizes = x.shape[2:]
x_list[i] = x
y = None
+ if steadydancer: # steady dancer
+ # Spatial Structure Adaptive Extractor.
+ time_steps = steadydancer_condition[0].shape[2]
+ for i, (x, condition) in enumerate(zip(x_list, steadydancer_condition)):
+ real_seq = x.shape[1]
+ # Temporal Motion Coherence Module.
+ condition_temporal =self.condition_embedding_temporal(condition)
+ condition_spatial = rearrange(self.condition_embedding_spatial(rearrange(condition, 'b c t h w -> (b t) c h w')), '(b t) c h w -> b c t h w', t=time_steps, b=1)
+ # Hierarchical Aggregation (1): condition, temporal condition, spatial condition
+ condition_fused = condition + condition_temporal + condition_spatial
+ # Frame-wise Attention Alignment Unit.
+ condition_aligned = self.condition_embedding_align(condition_fused, x_noise_clone)
+ # Condition Fusion/Injection, Hierarchical Aggregation (2): x, fused condition, aligned condition
+ x = self.patch_embedding_fuse(torch.cat([x, condition_fused, condition_aligned], 1))
+ x = torch.cat([x, self.patch_embedding(steadydancer_ref_x.unsqueeze(0)), self.patch_embedding_ref_c(steadydancer_ref_c[:16].unsqueeze(0))], dim=2)
+ grid_sizes = x.shape[2:]
+ x_list[i] = x
+ x = condition = condition_fused = condition_aligned = condition_temporal = condition_spatial = None
+ x_noise_clone = x = None
+
motion_vec_list = []
if face_pixel_values is None: face_pixel_values = [None] * len(x_list)
for i, (x, one_face_pixel_values) in enumerate(zip(x_list, face_pixel_values)):
@@ -1470,6 +1528,7 @@ class WanModel(ModelMixin, ConfigMixin):
x = None
+
block_mask = None
if causal_attention and causal_block_size > 0 and False: # NEVER WORKED
frame_num = grid_sizes[0]
@@ -1545,6 +1604,9 @@ class WanModel(ModelMixin, ConfigMixin):
if clip_fea is not None:
context_clip = self.img_emb(clip_fea) # bs x 257 x dim
+ if steadydancer_clip_fea_c is not None:
+ context_clip += self.img_emb(steadydancer_clip_fea_c) # bs x 257 x dim
+
context_list = []
for one_context in context:
if len(one_context) != len(context_clip):
@@ -1721,6 +1783,8 @@ class WanModel(ModelMixin, ConfigMixin):
# unpatchify
x_list[i] = self.unpatchify(x, grid_sizes)
+ if real_seq > 0:
+ x = x[:, :real_seq]
del x
return [x.float() for x in x_list]
diff --git a/models/wan/steadydancer/mobilenetv2_dcd.py b/models/wan/steadydancer/mobilenetv2_dcd.py
new file mode 100644
index 0000000..4fa3e55
--- /dev/null
+++ b/models/wan/steadydancer/mobilenetv2_dcd.py
@@ -0,0 +1,102 @@
+# Modify from https://github.com/liyunsheng13/dcd/blob/main/models/imagenet/mobilenetv2_dcd.py
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+
+class Hsigmoid(nn.Module):
+ def __init__(self, inplace=True):
+ super(Hsigmoid, self).__init__()
+ self.inplace = inplace
+
+ def forward(self, x):
+ return F.relu6(x + 3., inplace=self.inplace) / 3.
+
+
+class DYModule(nn.Module):
+ def __init__(self, inp, oup, fc_squeeze=8):
+ super(DYModule, self).__init__()
+ self.conv = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)
+ if inp < oup:
+ self.mul = 4
+ reduction = 8
+ self.avg_pool = nn.AdaptiveAvgPool2d(2)
+ else:
+ self.mul = 1
+ reduction = 2
+ self.avg_pool = nn.AdaptiveAvgPool2d(1)
+
+ self.dim = min((inp * self.mul) // reduction, oup // reduction)
+ while self.dim ** 2 > inp * self.mul * 2:
+ reduction *= 2
+ self.dim = min((inp * self.mul) // reduction, oup // reduction)
+ if self.dim < 4:
+ self.dim = 4
+
+ squeeze = max(inp * self.mul, self.dim ** 2) // fc_squeeze
+ if squeeze < 4:
+ squeeze = 4
+ self.conv_q = nn.Conv2d(inp, self.dim, 1, 1, 0, bias=False)
+
+ self.fc = nn.Sequential(
+ nn.Linear(inp * self.mul, squeeze, bias=False),
+ SEModule_small(squeeze),
+ )
+ self.fc_phi = nn.Linear(squeeze, self.dim ** 2, bias=False)
+ self.fc_scale = nn.Linear(squeeze, oup, bias=False)
+ self.hs = Hsigmoid()
+ self.conv_p = nn.Conv2d(self.dim, oup, 1, 1, 0, bias=False)
+ # self.bn1 = nn.BatchNorm2d(self.dim)
+ self.bn1 = nn.GroupNorm(num_groups=4, num_channels=self.dim)
+ # self.bn2 = nn.BatchNorm1d(self.dim)
+ self.bn2 = nn.GroupNorm(num_groups=4, num_channels=self.dim)
+
+ def forward(self, x):
+ r = self.conv(x)
+
+ b, c, h, w = x.size()
+ y = self.avg_pool(x).view(b, c * self.mul)
+ y = self.fc(y)
+ dy_phi = self.fc_phi(y).view(b, self.dim, self.dim)
+ dy_scale = self.hs(self.fc_scale(y)).view(b, -1, 1, 1)
+ r = dy_scale.expand_as(r) * r
+
+ x = self.conv_q(x)
+ x = self.bn1(x)
+ x = x.view(b, -1, h * w)
+ x = self.bn2(torch.matmul(dy_phi, x)) + x
+ x = x.view(b, -1, h, w)
+ x = self.conv_p(x)
+ return x + r
+
+
+class SEModule_small(nn.Module):
+ def __init__(self, channel):
+ super(SEModule_small, self).__init__()
+ self.fc = nn.Sequential(
+ nn.Linear(channel, channel, bias=False),
+ Hsigmoid()
+ )
+
+ def forward(self, x):
+ y = self.fc(x)
+ return x * y
+
+
+class SEModule(nn.Module):
+ def __init__(self, channel, reduction=4):
+ super(SEModule, self).__init__()
+ self.avg_pool = nn.AdaptiveAvgPool2d(1)
+ self.fc = nn.Sequential(
+ nn.Linear(channel, channel // reduction, bias=False),
+ nn.ReLU(inplace=True),
+ nn.Linear(channel // reduction, channel, bias=False),
+ Hsigmoid()
+ )
+
+ def forward(self, x):
+ b, c, _, _ = x.size()
+ y = self.avg_pool(x).view(b, c)
+ y = self.fc(y).view(b, c, 1, 1)
+ return x * y.expand_as(x)
diff --git a/models/wan/steadydancer/pose_align.py b/models/wan/steadydancer/pose_align.py
new file mode 100644
index 0000000..9d6804b
--- /dev/null
+++ b/models/wan/steadydancer/pose_align.py
@@ -0,0 +1,904 @@
+import argparse
+import os
+import sys
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Dict, List, Tuple, Union
+import itertools
+import time
+
+import cv2
+import imageio.v2 as imageio
+import numpy as np
+import torch
+from PIL import Image
+
+from preprocessing.dwpose.pose import convert_to_numpy, draw_pose
+from preprocessing.dwpose.wholebody import HWC3, Wholebody, resize_image
+from shared.utils import files_locator as fl
+from shared.utils.utils import process_images_multithread
+from shared.utils.utils import convert_tensor_to_image, convert_image_to_tensor
+
+ArrayImage = Union[np.ndarray, Image.Image, torch.Tensor]
+
+
+def _to_bgr_image(image: ArrayImage) -> np.ndarray:
+ """Convert supported image types to uint8 BGR HWC."""
+ if isinstance(image, torch.Tensor):
+ # Expect CHW or CTHW in [-1,1] or [0,1] or 0-255
+ if image.dim() == 4:
+ image = image[:, 0] # take first frame
+ img = image.detach().cpu()
+ if img.shape[0] in (1, 3, 4):
+ pass
+ elif img.shape[-1] in (1, 3, 4):
+ img = img.permute(2, 0, 1)
+ # normalize to uint8
+ if img.min() < 0:
+ img = (img + 1.0) * 127.5
+ elif img.max() <= 1.0:
+ img = img * 255.0
+ arr = img.clamp(0, 255).byte().permute(1, 2, 0).numpy()
+ if arr.shape[2] == 1:
+ arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
+ elif arr.shape[2] == 3:
+ arr = cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)
+ elif arr.shape[2] == 4:
+ arr = cv2.cvtColor(arr, cv2.COLOR_RGBA2BGR)
+ return HWC3(arr)
+ was_pil = isinstance(image, Image.Image)
+ arr = convert_to_numpy(image)
+
+ # Handle CHW tensors
+ if arr.ndim == 3 and arr.shape[0] in (1, 3, 4) and arr.shape[0] != arr.shape[1]:
+ arr = np.transpose(arr, (1, 2, 0))
+
+ if arr.ndim == 2:
+ arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)
+ elif arr.shape[2] == 4:
+ arr = cv2.cvtColor(arr, cv2.COLOR_RGBA2BGR)
+ elif was_pil:
+ arr = cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)
+
+ if arr.dtype != np.uint8:
+ if arr.max() <= 1.0:
+ arr = arr * 255.0
+ arr = np.clip(arr, 0, 255).astype(np.uint8)
+
+ return HWC3(arr)
+
+
+def _resize_to_height(img: np.ndarray, target_h: int) -> np.ndarray:
+ """Resize while preserving aspect ratio to a specific height."""
+ h, w = img.shape[:2]
+ if h == target_h:
+ return img
+ new_w = int(round(w * target_h / float(h)))
+ return cv2.resize(img, (new_w, target_h), interpolation=cv2.INTER_CUBIC)
+
+
+def _frames_to_tensor(frames: List[np.ndarray]) -> torch.Tensor:
+ """Convert a list of RGB uint8 frames to a tensor in [-1, 1] with shape 3,F,H,W."""
+ if not frames:
+ return torch.empty(0)
+ arr = np.stack(frames).astype(np.float32) / 127.5 - 1.0
+ return torch.from_numpy(arr).permute(3, 0, 1, 2)
+
+
+def _tensor_to_frames(tensor: torch.Tensor) -> List[np.ndarray]:
+ """Convert a tensor in [-1, 1] shaped 3,F,H,W back to RGB uint8 frames."""
+ if tensor.numel() == 0:
+ return []
+ arr = tensor.permute(1, 2, 3, 0).cpu().numpy()
+ arr = ((arr + 1.0) * 127.5).clip(0, 255).astype(np.uint8)
+ return [frame for frame in arr]
+
+
+def _ensure_xyz(arr: np.ndarray) -> np.ndarray:
+ """Ensure last dimension is 3 by padding zeros if needed."""
+ if arr.shape[-1] >= 3:
+ return arr
+ pad_width = [(0, 0)] * arr.ndim
+ pad_width[-1] = (0, 3 - arr.shape[-1])
+ return np.pad(arr, pad_width, mode="constant", constant_values=0)
+
+
+def _xy_only(arr: np.ndarray) -> np.ndarray:
+ """Return only x,y channels to satisfy drawing utils."""
+ if arr.shape[-1] <= 2:
+ return arr
+ return arr[..., :2]
+
+
+def _mask_to_float01(mask: Union[np.ndarray, Image.Image, torch.Tensor]) -> np.ndarray:
+ """Normalize any mask type to float32 single-channel in [0,1] with 0.5 threshold."""
+ if isinstance(mask, Image.Image):
+ mask = np.array(mask)
+ elif isinstance(mask, torch.Tensor):
+ m = mask.detach().cpu()
+ if m.dim() == 4:
+ m = m[0] # assume 1,C,H,W or C,H,W
+ if m.shape[0] in (1, 3, 4):
+ m = m.permute(1, 2, 0)
+ mask = m.numpy()
+ mask = np.asarray(mask)
+ if mask.ndim == 3 and mask.shape[2] > 1:
+ mask = mask.mean(axis=2)
+ mask = mask.astype(np.float32)
+ mask_min, mask_max = float(mask.min()), float(mask.max())
+ if mask_max > 1.0:
+ mask = mask / 255.0
+ elif mask_min < 0.0:
+ mask = (mask + 1.0) / 2.0
+ mask = np.clip(mask, 0.0, 1.0)
+ mask = (mask > 0.5).astype(np.float32)
+ return mask
+
+
+def _safe_ratio(num: float, den: float) -> float:
+ if den == 0 or not np.isfinite(den):
+ return 1.0
+ val = num / den
+ return float(val) if np.isfinite(val) else 1.0
+
+
+def _nan_to_one(val: float) -> float:
+ return 1.0 if not np.isfinite(val) else float(val)
+
+
+def _augment_pose(
+ pose: Dict,
+ offset_x: Tuple[float, float],
+ offset_y: Tuple[float, float],
+ scale_range: Tuple[float, float],
+ aspect_ratio_range: Tuple[float, float],
+ fixed_params: Tuple[float, float, float, float] | None = None,
+) -> Dict:
+ """Lightweight pose jitter used by the diff-aug variant."""
+ pose_aug = {
+ "bodies": {"candidate": pose["bodies"]["candidate"].copy(), "subset": pose["bodies"]["subset"].copy()},
+ "hands": pose["hands"].copy(),
+ "faces": pose["faces"].copy(),
+ }
+
+ if fixed_params is None:
+ sx = np.random.uniform(scale_range[0], scale_range[1])
+ aspect = np.random.uniform(aspect_ratio_range[0], aspect_ratio_range[1])
+ scale_x = sx * aspect
+ scale_y = sx / max(aspect, 1e-6)
+ dx = np.random.uniform(offset_x[0], offset_x[1])
+ dy = np.random.uniform(offset_y[0], offset_y[1])
+ else:
+ dx, dy, scale_x, scale_y = fixed_params
+
+ def _apply(arr: np.ndarray) -> np.ndarray:
+ arr = arr.copy()
+ mask = arr[..., 0] >= 0
+ arr[..., 0] = np.where(mask, arr[..., 0] * scale_x + dx, arr[..., 0])
+ arr[..., 1] = np.where(mask, arr[..., 1] * scale_y + dy, arr[..., 1])
+ return arr
+
+ pose_aug["bodies"]["candidate"] = _apply(pose_aug["bodies"]["candidate"])
+ pose_aug["hands"] = _apply(pose_aug["hands"])
+ pose_aug["faces"] = _apply(pose_aug["faces"])
+ return pose_aug
+
+
+def _save_video(path: str, frames: List[np.ndarray], fps: float) -> None:
+ if not frames:
+ return
+ os.makedirs(os.path.dirname(path), exist_ok=True)
+ with imageio.get_writer(path, fps=fps, codec="libx264") as writer:
+ for frame in frames:
+ writer.append_data(frame)
+
+
+def align_img(img: np.ndarray, pose_ori: Dict, scales: Dict[str, float]) -> Dict:
+ """Align a single pose dictionary using pre-computed scale factors."""
+ body_pose = pose_ori["bodies"]["candidate"].copy()
+ hands = pose_ori["hands"].copy()
+ faces = pose_ori["faces"].copy()
+
+ H_in, W_in, _ = img.shape
+ video_ratio = W_in / H_in
+ body_pose[:, 0] *= video_ratio
+ hands[:, :, 0] *= video_ratio
+ faces[:, :, 0] *= video_ratio
+
+ scale_neck = scales["scale_neck"]
+ scale_face_left = scales["scale_face_left"]
+ scale_face_right = scales["scale_face_right"]
+ scale_shoulder = scales["scale_shoulder"]
+ scale_arm_upper = scales["scale_arm_upper"]
+ scale_arm_lower = scales["scale_arm_lower"]
+ scale_hand = scales["scale_hand"]
+ scale_body_len = scales["scale_body_len"]
+ scale_leg_upper = scales["scale_leg_upper"]
+ scale_leg_lower = scales["scale_leg_lower"]
+
+ scale_list = [
+ scale_neck,
+ scale_face_left,
+ scale_face_right,
+ scale_shoulder,
+ scale_arm_upper,
+ scale_arm_lower,
+ scale_hand,
+ scale_body_len,
+ scale_leg_upper,
+ scale_leg_lower,
+ ]
+ finite_vals = [v for v in scale_list if np.isfinite(v)]
+ mean_scale = np.mean(finite_vals) if finite_vals else 1.0
+ scale_list = [mean_scale if np.isinf(v) else v for v in scale_list]
+ (
+ scale_neck,
+ scale_face_left,
+ scale_face_right,
+ scale_shoulder,
+ scale_arm_upper,
+ scale_arm_lower,
+ scale_hand,
+ scale_body_len,
+ scale_leg_upper,
+ scale_leg_lower,
+ ) = scale_list
+
+ offset = {
+ "14_16_to_0": body_pose[[14, 16], :] - body_pose[[0], :],
+ "15_17_to_0": body_pose[[15, 17], :] - body_pose[[0], :],
+ "3_to_2": body_pose[[3], :] - body_pose[[2], :],
+ "4_to_3": body_pose[[4], :] - body_pose[[3], :],
+ "6_to_5": body_pose[[6], :] - body_pose[[5], :],
+ "7_to_6": body_pose[[7], :] - body_pose[[6], :],
+ "9_to_8": body_pose[[9], :] - body_pose[[8], :],
+ "10_to_9": body_pose[[10], :] - body_pose[[9], :],
+ "12_to_11": body_pose[[12], :] - body_pose[[11], :],
+ "13_to_12": body_pose[[13], :] - body_pose[[12], :],
+ "hand_left_to_4": hands[1, :, :] - body_pose[[4], :],
+ "hand_right_to_7": hands[0, :, :] - body_pose[[7], :],
+ }
+
+ def _warp(target: np.ndarray, center: np.ndarray, scale: float) -> np.ndarray:
+ M = cv2.getRotationMatrix2D((center[0], center[1]), 0, scale)
+ return warpAffine_kps(target, M)
+
+ body_pose[[0], :] = _warp(body_pose[[0], :], body_pose[1], scale_neck)
+ body_pose[[14, 16], :] = _warp(offset["14_16_to_0"] + body_pose[[0], :], body_pose[0], scale_face_left)
+ body_pose[[15, 17], :] = _warp(offset["15_17_to_0"] + body_pose[[0], :], body_pose[0], scale_face_right)
+ body_pose[[2, 5], :] = _warp(body_pose[[2, 5], :], body_pose[1], scale_shoulder)
+
+ body_pose[[3], :] = _warp(offset["3_to_2"] + body_pose[[2], :], body_pose[2], scale_arm_upper)
+ body_pose[[4], :] = _warp(offset["4_to_3"] + body_pose[[3], :], body_pose[3], scale_arm_lower)
+ hands[1, :, :] = _warp(offset["hand_left_to_4"] + body_pose[[4], :], body_pose[4], scale_hand)
+
+ body_pose[[6], :] = _warp(offset["6_to_5"] + body_pose[[5], :], body_pose[5], scale_arm_upper)
+ body_pose[[7], :] = _warp(offset["7_to_6"] + body_pose[[6], :], body_pose[6], scale_arm_lower)
+ hands[0, :, :] = _warp(offset["hand_right_to_7"] + body_pose[[7], :], body_pose[7], scale_hand)
+
+ body_pose[[8, 11], :] = _warp(body_pose[[8, 11], :], body_pose[1], scale_body_len)
+ body_pose[[9], :] = _warp(offset["9_to_8"] + body_pose[[8], :], body_pose[8], scale_leg_upper)
+ body_pose[[10], :] = _warp(offset["10_to_9"] + body_pose[[9], :], body_pose[9], scale_leg_lower)
+ body_pose[[12], :] = _warp(offset["12_to_11"] + body_pose[[11], :], body_pose[11], scale_leg_upper)
+ body_pose[[13], :] = _warp(offset["13_to_12"] + body_pose[[12], :], body_pose[12], scale_leg_lower)
+
+ body_pose_none = pose_ori["bodies"]["candidate"] == -1.0
+ hands_none = pose_ori["hands"] == -1.0
+ faces_none = pose_ori["faces"] == -1.0
+
+ body_pose[body_pose_none] = -1.0
+ hands[hands_none] = -1.0
+ faces[faces_none] = -1.0
+
+ body_pose = np.nan_to_num(body_pose, nan=-1.0)
+ hands = np.nan_to_num(hands, nan=-1.0)
+ faces = np.nan_to_num(faces, nan=-1.0)
+
+ pose_align = dict(
+ bodies={"candidate": body_pose, "subset": pose_ori["bodies"]["subset"]},
+ hands=hands,
+ faces=faces,
+ )
+ return pose_align
+
+
+def warpAffine_kps(kps: np.ndarray, M: np.ndarray) -> np.ndarray:
+ kps_t = kps.copy()
+ kps_t[..., 0] = kps[..., 0] * M[0, 0] + kps[..., 1] * M[0, 1] + M[0, 2]
+ kps_t[..., 1] = kps[..., 0] * M[1, 0] + kps[..., 1] * M[1, 1] + M[1, 2]
+ return kps_t
+
+
+@dataclass
+class PoseDetection:
+ pose: Dict
+ pose_map_rgb: np.ndarray
+ frame_rgb: np.ndarray
+ orig_hw: Tuple[int, int]
+
+
+class PoseAligner:
+ def __init__(self, detect_resolution: int = 1024, device: str = None, detection_workers: int = 2) -> None:
+ det_model = fl.locate_file("pose/yolox_l.onnx")
+ pose_model = fl.locate_file("pose/dw-ll_ucoco_384.onnx")
+ resolved_device = device or ("cuda:0" if torch.cuda.is_available() else "cpu")
+ self.det_model = det_model
+ self.pose_model = pose_model
+ self.device = resolved_device
+ self.pose_estimation = Wholebody(det_model, pose_model, device=resolved_device)
+ self.detection_workers = max(1, int(detection_workers))
+ self.detect_resolution = detect_resolution
+
+ def _detect_pose_session(self, image: ArrayImage, pose_estimation: Wholebody) -> PoseDetection | None:
+ bgr_orig = _to_bgr_image(image)
+ resized = resize_image(bgr_orig, self.detect_resolution)
+ H, W, _ = resized.shape
+ candidate, subset, _ = pose_estimation(resized)
+ if len(candidate) == 0:
+ return None
+
+ nums, keys, locs = candidate.shape
+ if keys < 18:
+ return None # not enough keypoints detected
+
+ candidate = _ensure_xyz(candidate.copy())
+ subset = subset.copy()
+
+ # Normalize coordinates to [0, 1]
+ candidate[..., 0] /= float(W)
+ candidate[..., 1] /= float(H)
+
+ locs = candidate.shape[2]
+
+ # Extract first person's body keypoints (18 keypoints)
+ body = candidate[0, :18].copy() # shape (18, locs)
+ body_scores = subset[0, :18].copy() # shape (18,)
+
+ # Build score array and mask invisible keypoints
+ score = np.zeros((1, 18), dtype=np.float64)
+ for j in range(18):
+ if body_scores[j] <= 0.3:
+ body[j] = -1
+ score[0, j] = -1
+ else:
+ score[0, j] = j # Index into body array
+
+ # Extract faces (keypoints 24-92)
+ faces = candidate[0:1, 24:92].copy() # shape (1, 68, locs)
+ if subset.shape[1] > 24:
+ face_scores = subset[0, 24:92]
+ for j in range(faces.shape[1]):
+ if j < len(face_scores) and face_scores[j] <= 0.3:
+ faces[0, j] = -1
+ faces = _ensure_xyz(faces)
+
+ # Extract hands (keypoints 92-113 for right hand, 113-134 for left hand)
+ # hands array should be shape (2, 21, locs) - [right_hand, left_hand]
+ right_hand = candidate[0, 92:113].copy() if candidate.shape[1] > 92 else np.zeros((21, locs))
+ left_hand = candidate[0, 113:134].copy() if candidate.shape[1] > 113 else np.zeros((21, locs))
+
+ # Mask invalid hand keypoints
+ if subset.shape[1] > 92:
+ hand_scores = subset[0, 92:134]
+ for j in range(21):
+ if j < len(hand_scores) and hand_scores[j] <= 0.3:
+ right_hand[j] = -1
+ if j + 21 < len(hand_scores) and hand_scores[j + 21] <= 0.3:
+ left_hand[j] = -1
+
+ hands = _ensure_xyz(np.stack([right_hand, left_hand])) # shape (2, 21, locs)
+
+ pose = {"bodies": {"candidate": body, "subset": score}, "hands": hands, "faces": faces}
+
+ # Draw expects xy only
+ pose_for_draw = {
+ "bodies": {"candidate": _xy_only(body), "subset": score},
+ "hands": _xy_only(hands),
+ "faces": _xy_only(faces),
+ }
+ pose_map = draw_pose(pose_for_draw, H, W, use_body=True, use_hand=True, use_face=False)
+ pose_map_rgb = cv2.cvtColor(pose_map, cv2.COLOR_BGR2RGB)
+ frame_rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)
+ return PoseDetection(pose=pose, pose_map_rgb=pose_map_rgb, frame_rgb=frame_rgb, orig_hw=bgr_orig.shape[:2])
+
+ def _detect_pose(self, image: ArrayImage) -> PoseDetection | None:
+ return self._detect_pose_session(image, self.pose_estimation)
+
+ def _compute_alignment(self, ref_pose: Dict, first_pose: Dict, ref_ratio: float, video_ratio: float) -> Tuple[Dict, np.ndarray]:
+ body_ref = ref_pose["bodies"]["candidate"].copy()
+ hands_ref = ref_pose["hands"].copy()
+ faces_ref = ref_pose["faces"].copy()
+
+ body_first = first_pose["bodies"]["candidate"].copy()
+ hands_first = first_pose["hands"].copy()
+ faces_first = first_pose["faces"].copy()
+
+ body_ref[:, 0] *= ref_ratio
+ hands_ref[:, :, 0] *= ref_ratio
+ faces_ref[:, :, 0] *= ref_ratio
+
+ body_first[:, 0] *= video_ratio
+ hands_first[:, :, 0] *= video_ratio
+ faces_first[:, :, 0] *= video_ratio
+
+ def dist(body: np.ndarray, a: int, b: int) -> float:
+ pa, pb = body[a, :2], body[b, :2]
+ if (pa < 0).any() or (pb < 0).any():
+ return np.nan
+ return float(np.linalg.norm(pa - pb))
+
+ def hand_dist(hand: np.ndarray, idx_a: int, idx_b: int) -> float:
+ pa, pb = hand[idx_a, :2], hand[idx_b, :2]
+ if (pa < 0).any() or (pb < 0).any():
+ return np.nan
+ return float(np.linalg.norm(pa - pb))
+
+ align_args = {
+ "scale_neck": _safe_ratio(dist(body_ref, 0, 1), dist(body_first, 0, 1)),
+ "scale_face_left": _safe_ratio(
+ dist(body_ref, 16, 14) + dist(body_ref, 14, 0),
+ dist(body_first, 16, 14) + dist(body_first, 14, 0),
+ ),
+ "scale_face_right": _safe_ratio(
+ dist(body_ref, 17, 15) + dist(body_ref, 15, 0),
+ dist(body_first, 17, 15) + dist(body_first, 15, 0),
+ ),
+ "scale_shoulder": _safe_ratio(dist(body_ref, 2, 5), dist(body_first, 2, 5)),
+ "scale_arm_upper": np.nanmean(
+ [
+ _safe_ratio(dist(body_ref, 2, 3), dist(body_first, 2, 3)),
+ _safe_ratio(dist(body_ref, 5, 6), dist(body_first, 5, 6)),
+ ]
+ ),
+ "scale_arm_lower": np.nanmean(
+ [
+ _safe_ratio(dist(body_ref, 3, 4), dist(body_first, 3, 4)),
+ _safe_ratio(dist(body_ref, 6, 7), dist(body_first, 6, 7)),
+ ]
+ ),
+ "scale_body_len": _safe_ratio(
+ dist(body_ref, 1, 8) if not np.isnan(dist(body_ref, 1, 8)) else dist(body_ref, 1, 11),
+ dist(body_first, 1, 8) if not np.isnan(dist(body_first, 1, 8)) else dist(body_first, 1, 11),
+ ),
+ "scale_leg_upper": np.nanmean(
+ [
+ _safe_ratio(dist(body_ref, 8, 9), dist(body_first, 8, 9)),
+ _safe_ratio(dist(body_ref, 11, 12), dist(body_first, 11, 12)),
+ ]
+ ),
+ "scale_leg_lower": np.nanmean(
+ [
+ _safe_ratio(dist(body_ref, 9, 10), dist(body_first, 9, 10)),
+ _safe_ratio(dist(body_ref, 12, 13), dist(body_first, 12, 13)),
+ ]
+ ),
+ }
+
+ hand_pairs = [(0, 1), (0, 5), (0, 9), (0, 13), (0, 17)]
+ hand_ratios = []
+ for idx_a, idx_b in hand_pairs:
+ hand_ratios.append(
+ _safe_ratio(
+ hand_dist(hands_ref[0], idx_a, idx_b),
+ hand_dist(hands_first[0], idx_a, idx_b),
+ )
+ )
+ hand_ratios.append(
+ _safe_ratio(
+ hand_dist(hands_ref[1], idx_a, idx_b),
+ hand_dist(hands_first[1], idx_a, idx_b),
+ )
+ )
+ hand_ratios = [v for v in hand_ratios if np.isfinite(v)]
+ align_args["scale_hand"] = np.mean(hand_ratios) if hand_ratios else (
+ align_args["scale_arm_upper"] + align_args["scale_arm_lower"]
+ ) / 2
+
+ align_args = {k: _nan_to_one(v) for k, v in align_args.items()}
+ offset = np.array(
+ [body_ref[1, 0] - body_first[1, 0], body_ref[1, 1] - body_first[1, 1], 0.0],
+ dtype=np.float32,
+ )
+ return align_args, offset
+
+ def align(
+ self,
+ ref_video_frames: List[ArrayImage],
+ ref_image: ArrayImage,
+ ref_video_mask: List[ArrayImage] | None = None,
+ align_frame: int = 0,
+ max_frames: int | None = None,
+ include_composite: bool = False,
+ augment: bool = True,
+ augment_mode: str = "per_frame", # "per_frame" (original jitter), "fixed"
+ augment_params: Dict | None = None,
+ cpu_resize_workers: int = None,
+ resize_ref_video: bool = False,
+ detection_chunk_size: int = 8,
+ expand_scale = 0,
+ verbose: int = 0,
+ ) -> Dict[str, torch.Tensor]:
+ t0 = time.perf_counter()
+ ref_detection = self._detect_pose(ref_image)
+ if ref_detection is None:
+ raise ValueError("Unable to detect pose in the reference image.")
+
+ target_hw = ref_detection.frame_rgb.shape[:2]
+
+ def _tensor_video_to_list(t: torch.Tensor) -> List[np.ndarray]:
+ if t.dim() != 4 or t.shape[0] not in (1, 3, 4):
+ raise ValueError("Video tensor must be 4D CTHW or BGRHWC list")
+ vid = t.detach().cpu()
+ if vid.min() < 0:
+ vid = (vid + 1.0) * 127.5
+ elif vid.max() <= 1.0:
+ vid = vid * 255.0
+ vid = vid.clamp(0, 255).byte()
+ # Return BGR to match cv2.VideoCapture format
+ frames = []
+ for i in range(vid.shape[1]):
+ rgb = vid[:, i].permute(1, 2, 0).numpy()
+ bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)
+ frames.append(bgr)
+ return frames
+
+ if isinstance(ref_video_frames, torch.Tensor):
+ ref_video_frames = _tensor_video_to_list(ref_video_frames)
+ def _preprocess_item(idx_frame_mask):
+ idx, frame, mask = idx_frame_mask
+ # normalize mask here to leverage multithreading
+ if mask is not None:
+ mask = _mask_to_float01(mask)
+ if resize_ref_video and frame.shape[:2] != target_hw:
+ frame = cv2.resize(frame, (target_hw[1], target_hw[0]), interpolation=cv2.INTER_LANCZOS4)
+
+ if mask is not None:
+ mask_f = mask
+ if mask_f.shape != frame.shape[:2]:
+ mask_f = cv2.resize(mask_f, (frame.shape[1], frame.shape[0]), interpolation=cv2.INTER_LINEAR)
+ if expand_scale != 0:
+ kernel_size = abs(expand_scale)
+ kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
+ op_expand = cv2.dilate if expand_scale > 0 else cv2.erode
+ mask = op_expand(mask, kernel, iterations=3)
+ frame = (frame.astype(np.float32) * mask_f[..., None]).clip(0, 255).astype(np.uint8)
+ return idx, frame
+
+ # prepare frames (mask + optional resize) using CPU multithreading
+ items = []
+ for idx, frame in enumerate(ref_video_frames):
+ mask = None
+ if ref_video_mask is not None:
+ mask = ref_video_mask[:, idx] if torch.is_tensor(ref_video_mask) else ref_video_mask[idx]
+ items.append((idx, frame, mask))
+ if cpu_resize_workers is None:
+ cpu_resize_workers = max(1, int(os.cpu_count() / 2))
+ preprocessed = process_images_multithread(
+ _preprocess_item,
+ items,
+ process_type="prephase",
+ wrap_in_list=False,
+ max_workers=cpu_resize_workers,
+ in_place=False,
+ )
+ idx_to_frame = {idx: frame for idx, frame in preprocessed}
+ t_preprocess = time.perf_counter() - t0
+ if verbose:
+ print(f"[pose_align] preprocess done in {t_preprocess:.2f}s")
+
+ ref_ratio = ref_detection.frame_rgb.shape[1] / ref_detection.frame_rgb.shape[0]
+ pose_list: List[Dict] = []
+ video_frames: List[np.ndarray] = [] if include_composite else []
+ video_pose_frames: List[np.ndarray] = [] if include_composite else []
+
+ align_args = None
+ offset = None
+
+ # instantiate additional detectors for parallel processing
+ detectors = [self.pose_estimation]
+ for _ in range(max(1, self.detection_workers) - 1):
+ detectors.append(Wholebody(self.det_model, self.pose_model, device=self.device))
+
+ sequential_done = False
+ pending = []
+ t_first_stage = 0.0
+
+ for idx in sorted(idx_to_frame.keys()):
+ if idx < align_frame:
+ continue
+ if max_frames is not None and len(pose_list) >= max_frames:
+ break
+
+ frame = idx_to_frame[idx]
+ t_detect_start = time.perf_counter()
+ detection = self._detect_pose(frame) if not sequential_done else None
+ if detection is None:
+ # queue for parallel path once we have align_args
+ pending.append((idx, frame))
+ continue
+
+ if align_args is None:
+ video_ratio = detection.frame_rgb.shape[1] / detection.frame_rgb.shape[0]
+ align_args, offset = self._compute_alignment(ref_detection.pose, detection.pose, ref_ratio, video_ratio)
+
+ pose_aligned = align_img(detection.frame_rgb, detection.pose, align_args)
+ mask_body = pose_aligned["bodies"]["candidate"][..., 0] < 0
+ mask_hands = pose_aligned["hands"][..., 0] < 0
+ mask_faces = pose_aligned["faces"][..., 0] < 0
+
+ pose_aligned["bodies"]["candidate"] = _ensure_xyz(pose_aligned["bodies"]["candidate"]) + offset
+ pose_aligned["hands"] = _ensure_xyz(pose_aligned["hands"]) + offset
+ pose_aligned["faces"] = _ensure_xyz(pose_aligned["faces"]) + offset
+
+ pose_aligned["bodies"]["candidate"][mask_body] = -1
+ pose_aligned["hands"][mask_hands] = -1
+ pose_aligned["faces"][mask_faces] = -1
+
+ pose_aligned["bodies"]["candidate"][:, 0] /= ref_ratio
+ pose_aligned["hands"][:, :, 0] /= ref_ratio
+ pose_aligned["faces"][:, :, 0] /= ref_ratio
+
+ pose_list.append(pose_aligned)
+ if include_composite:
+ video_frames.append(detection.frame_rgb)
+ video_pose_frames.append(detection.pose_map_rgb)
+
+ sequential_done = True
+ start_idx = idx + 1
+ pending.extend([(j, idx_to_frame[j]) for j in sorted(idx_to_frame.keys()) if j >= start_idx])
+ t_first_stage = time.perf_counter() - t_detect_start
+ if verbose:
+ print(f"[pose_align] first frame detection+align done in {t_first_stage:.2f}s")
+ break
+
+ if align_args is None:
+ return {"composite": torch.empty(0), "pose_only": torch.empty(0), "pose_aug": torch.empty(0)}
+
+ # Parallel processing for remaining frames using a single persistent executor
+ def _process_pending(item):
+ idx, frame, det_idx = item
+ det = self._detect_pose_session(frame, detectors[det_idx])
+ if det is None:
+ return idx, None, None, None
+
+ pa = align_img(det.frame_rgb, det.pose, align_args)
+ mb = pa["bodies"]["candidate"][..., 0] < 0
+ mh = pa["hands"][..., 0] < 0
+ mf = pa["faces"][..., 0] < 0
+
+ pa["bodies"]["candidate"] = _ensure_xyz(pa["bodies"]["candidate"]) + offset
+ pa["hands"] = _ensure_xyz(pa["hands"]) + offset
+ pa["faces"] = _ensure_xyz(pa["faces"]) + offset
+
+ pa["bodies"]["candidate"][mb] = -1
+ pa["hands"][mh] = -1
+ pa["faces"][mf] = -1
+
+ pa["bodies"]["candidate"][:, 0] /= ref_ratio
+ pa["hands"][:, :, 0] /= ref_ratio
+ pa["faces"][:, :, 0] /= ref_ratio
+ return idx, pa, det.frame_rgb, det.pose_map_rgb
+
+ if pending:
+ from concurrent.futures import ThreadPoolExecutor, as_completed
+
+ results = []
+ det_cycle = itertools.cycle(range(self.detection_workers))
+ with ThreadPoolExecutor(max_workers=self.detection_workers) as executor:
+ for start in range(0, len(pending), detection_chunk_size * self.detection_workers):
+ chunk = pending[start : start + detection_chunk_size * self.detection_workers]
+ tasks = []
+ for idx, frame in chunk:
+ tasks.append((idx, frame, next(det_cycle)))
+ futures = {executor.submit(_process_pending, t): t[0] for t in tasks}
+ for future in as_completed(futures):
+ res = future.result()
+ if res[1] is None:
+ continue
+ results.append(res)
+ for idx, pa, fr, pm in sorted(results, key=lambda x: x[0]):
+ if max_frames is not None and len(pose_list) >= max_frames:
+ break
+ pose_list.append(pa)
+ if include_composite:
+ video_frames.append(fr)
+ video_pose_frames.append(pm)
+ t_parallel = time.perf_counter() - t0 - t_preprocess - t_first_stage
+ if verbose:
+ print(f"[pose_align] parallel detection done in {t_parallel:.2f}s")
+ else:
+ t_parallel = 0.0
+
+ if not pose_list:
+ return {"composite": torch.empty(0), "pose_only": torch.empty(0), "pose_aug": torch.empty(0)}
+
+ body_seq = [_ensure_xyz(pose["bodies"]["candidate"][:18]) for pose in pose_list]
+ body_seq_subset = [pose["bodies"]["subset"][:1] for pose in pose_list]
+ hands_seq = [_ensure_xyz(pose["hands"][:2]) for pose in pose_list]
+ faces_seq = [_ensure_xyz(pose["faces"][:1]) for pose in pose_list]
+
+ ref_H, ref_W = ref_detection.frame_rgb.shape[:2]
+ ref_target_H, ref_target_W = ref_detection.orig_hw if hasattr(ref_detection, "orig_hw") else (ref_H, ref_W)
+ composite_frames: List[np.ndarray] = [] if include_composite else []
+ pose_only_frames: List[np.ndarray] = []
+ pose_aug_frames: List[np.ndarray] = []
+
+ aug_cfg = augment_params or {}
+ offset_x = aug_cfg.get("offset_x", (-0.2, 0.2))
+ offset_y = aug_cfg.get("offset_y", (-0.2, 0.2))
+ scale_rng = aug_cfg.get("scale", (0.7, 1.3))
+ aspect_rng = aug_cfg.get("aspect_ratio_range", (0.6, 1.4))
+ fixed_aug = None
+ if augment and augment_mode == "fixed":
+ sx = np.random.uniform(scale_rng[0], scale_rng[1])
+ aspect = np.random.uniform(aspect_rng[0], aspect_rng[1])
+ scale_x = sx * aspect
+ scale_y = sx / max(aspect, 1e-6)
+ dx = np.random.uniform(offset_x[0], offset_x[1])
+ dy = np.random.uniform(offset_y[0], offset_y[1])
+ fixed_aug = (dx, dy, scale_x, scale_y)
+
+ for i in range(len(body_seq)):
+ pose_t = {
+ "bodies": {"candidate": body_seq[i], "subset": body_seq_subset[i]},
+ "hands": hands_seq[i],
+ "faces": faces_seq[i],
+ }
+
+ pose_for_draw = {
+ "bodies": {"candidate": _xy_only(body_seq[i]), "subset": body_seq_subset[i]},
+ "hands": _xy_only(hands_seq[i]),
+ "faces": _xy_only(faces_seq[i]),
+ }
+
+ aligned_pose = draw_pose(pose_for_draw, ref_H, ref_W, use_body=True, use_hand=True, use_face=False)
+ aligned_pose = cv2.cvtColor(aligned_pose, cv2.COLOR_BGR2RGB)
+ pose_only_frames.append(aligned_pose)
+
+ aug_frame = None
+ if augment:
+ params = None if augment_mode == "per_frame" else fixed_aug
+ pose_t_aug = _augment_pose(pose_t, offset_x, offset_y, scale_rng, aspect_rng, fixed_params=params)
+ pose_t_aug_draw = {
+ "bodies": {"candidate": _xy_only(pose_t_aug["bodies"]["candidate"]), "subset": pose_t_aug["bodies"]["subset"]},
+ "hands": _xy_only(pose_t_aug["hands"]),
+ "faces": _xy_only(pose_t_aug["faces"]),
+ }
+ aug_pose = draw_pose(pose_t_aug_draw, ref_H, ref_W, use_body=True, use_hand=True, use_face=False)
+ aug_frame = cv2.cvtColor(aug_pose, cv2.COLOR_BGR2RGB)
+ pose_aug_frames.append(aug_frame)
+
+ if include_composite:
+ video_frame = _resize_to_height(video_frames[i], ref_H)
+ video_pose = _resize_to_height(video_pose_frames[i], ref_H)
+ ref_pose_resized = _resize_to_height(ref_detection.pose_map_rgb, ref_H)
+ ref_img_resized = _resize_to_height(ref_detection.frame_rgb, ref_H)
+ parts = [
+ ref_img_resized,
+ ref_pose_resized,
+ aligned_pose,
+ ]
+ if augment and aug_frame is not None:
+ parts.append(aug_frame)
+ parts.extend([video_frame, video_pose])
+ comp = np.concatenate(parts, axis=1)
+ composite_frames.append(comp)
+
+ # Resize outputs to reference original size if needed
+ if (ref_target_H, ref_target_W) != (ref_H, ref_W):
+ def _resize_list(frames: List[np.ndarray]) -> List[np.ndarray]:
+ return [cv2.resize(f, (ref_target_W, ref_target_H), interpolation=cv2.INTER_CUBIC) for f in frames]
+ pose_only_frames = _resize_list(pose_only_frames)
+ pose_aug_frames = _resize_list(pose_aug_frames)
+ t_render = time.perf_counter() - t0 - t_preprocess - t_first_stage - t_parallel
+ t_total = time.perf_counter() - t0
+ if verbose:
+ print(f"[pose_align] render done in {t_render:.2f}s; total {t_total:.2f}s")
+
+ return {
+ "composite": _frames_to_tensor(composite_frames) if include_composite else torch.empty(0),
+ "pose_only": _frames_to_tensor(pose_only_frames),
+ "pose_aug": _frames_to_tensor(pose_aug_frames) if augment else torch.empty(0),
+ }
+
+
+def load_video_frames(path: str) -> Tuple[List[np.ndarray], float]:
+ cap = cv2.VideoCapture(path)
+ fps = cap.get(cv2.CAP_PROP_FPS) or 24.0
+ frames = []
+ while cap.isOpened():
+ ret, frame = cap.read()
+ if not ret:
+ break
+ frames.append(frame)
+ cap.release()
+ cv2.destroyAllWindows()
+ return frames, fps
+
+
+def load_mask_frames(path: str) -> List[np.ndarray]:
+ frames, _ = load_video_frames(path)
+ return frames
+
+
+def demo(
+ ref_image_path: str,
+ ref_video_path: str,
+ ref_video_mask_path: str | None,
+ output_dir: str,
+ include_composite: bool = True,
+ augment: bool = True,
+ augment_mode: str = "per_frame",
+ align_frame: int = 0,
+ max_frames: int | None = None,
+ detection_workers: int = 2,
+ cpu_resize_workers: int = None,
+ resize_ref_video: bool = False,
+ detection_chunk_size: int = 8,
+ verbose: int = 0,
+) -> None:
+ frames, fps = load_video_frames(ref_video_path)
+ mask_frames = load_mask_frames(ref_video_mask_path) if ref_video_mask_path else None
+ aligner = PoseAligner(detection_workers=detection_workers)
+ outputs = aligner.align(
+ frames,
+ Image.open(ref_image_path),
+ ref_video_mask=mask_frames,
+ align_frame=align_frame,
+ max_frames=max_frames,
+ include_composite=include_composite,
+ augment=augment,
+ augment_mode=augment_mode,
+ cpu_resize_workers=cpu_resize_workers,
+ resize_ref_video=resize_ref_video,
+ detection_chunk_size=detection_chunk_size,
+ verbose=verbose,
+ )
+
+ composite_frames = _tensor_to_frames(outputs["composite"])
+ pose_frames = _tensor_to_frames(outputs["pose_only"])
+ pose_aug_frames = _tensor_to_frames(outputs["pose_aug"])
+
+ if include_composite and composite_frames:
+ _save_video(os.path.join(output_dir, "composite.mp4"), composite_frames, fps)
+ if pose_frames:
+ _save_video(os.path.join(output_dir, "pose_only.mp4"), pose_frames, fps)
+ if pose_aug_frames:
+ _save_video(os.path.join(output_dir, "pose_only_aug.mp4"), pose_aug_frames, fps)
+
+
+if __name__ == "__main__":
+ parser = argparse.ArgumentParser(description="Pose alignment demo")
+ parser.add_argument("--ref_image", type=str, default=r"e:/steffete.png", help="Path to the reference image")
+ parser.add_argument("--ref_video", type=str, default=r"e:\stage.mp4", help="Path to the reference pose video (mp4)")
+ parser.add_argument("--ref_video_mask", type=str, default='e:/stagemasked_alpha.mp4', help="Optional path to a mask video (same length as ref video)")
+ # parser.add_argument("--ref_video_mask", type=str, default=None, help="Optional path to a mask video (same length as ref video)")
+ parser.add_argument("--output_dir", type=str, default="pose_align_demo", help="Directory to store demo mp4s")
+ parser.add_argument("--no_composite", action="store_true", help="Skip saving comparison/composite video")
+ parser.add_argument("--no_augment", action="store_true", help="Disable augmented pose output")
+ parser.add_argument("--augment_mode", type=str, default="per_frame", choices=["fixed", "per_frame"], help="Augmentation mode (fixed per video or per frame jitter)")
+ parser.add_argument("--align_frame", type=int, default=0, help="Frame index to start alignment from")
+ parser.add_argument("--max_frames", type=int, default=48, help="Maximum frames to process")
+ parser.add_argument("--detection_workers", type=int, default=2, help="Number of parallel detection workers (GPU-backed)")
+ parser.add_argument("--cpu_resize_workers", type=int, default=None, help="CPU workers for pre-mask/resize")
+ parser.add_argument("--resize_ref_video", action="store_true", help="Resize ref video to ref image size (Lanczos)")
+ parser.add_argument("--detection_chunk_size", type=int, default=8, help="Chunk size for parallel detection batches")
+ parser.add_argument("--verbose", type=int, default=1, help="Verbosity level for timing/debug")
+ args = parser.parse_args()
+
+ demo(
+ args.ref_image,
+ args.ref_video,
+ args.ref_video_mask,
+ args.output_dir,
+ include_composite=not args.no_composite,
+ augment=not args.no_augment,
+ augment_mode=args.augment_mode,
+ align_frame=args.align_frame,
+ max_frames=args.max_frames,
+ detection_workers=args.detection_workers,
+ cpu_resize_workers=args.cpu_resize_workers,
+ resize_ref_video=args.resize_ref_video,
+ detection_chunk_size=args.detection_chunk_size,
+ verbose=args.verbose,
+ )
diff --git a/models/wan/steadydancer/small_archs.py b/models/wan/steadydancer/small_archs.py
new file mode 100644
index 0000000..cdd74a6
--- /dev/null
+++ b/models/wan/steadydancer/small_archs.py
@@ -0,0 +1,138 @@
+import torch
+import torch.nn as nn
+
+
+class FactorConv3d(nn.Module):
+ """
+ (2+1)D 分解 3D 卷积:1×H×W 空间卷积 → Swish → T×1×1 时间卷积
+ """
+ def __init__(self,
+ in_channels: int,
+ out_channels: int,
+ kernel_size,
+ stride: int = 1,
+ dilation: int = 1):
+ super().__init__()
+
+ if isinstance(kernel_size, int):
+ k_t, k_h, k_w = kernel_size, kernel_size, kernel_size
+ else:
+ k_t, k_h, k_w = kernel_size
+
+ pad_t = (k_t - 1) * dilation // 2
+ pad_hw = (k_h - 1) * dilation // 2
+
+ self.spatial = nn.Conv3d(
+ in_channels, in_channels,
+ kernel_size=(1, k_h, k_w),
+ stride=(1, stride, stride),
+ padding=(0, pad_hw, pad_hw),
+ dilation=(1, dilation, dilation),
+ groups=in_channels,
+ bias=False
+ )
+
+ self.temporal = nn.Conv3d(
+ in_channels, out_channels,
+ kernel_size=(k_t, 1, 1),
+ stride=(stride, 1, 1),
+ padding=(pad_t, 0, 0),
+ dilation=(dilation, 1, 1),
+ bias=True
+ )
+
+ self.act = nn.SiLU()
+
+ def forward(self, x):
+ x = self.spatial(x)
+ x = self.act(x)
+ x = self.temporal(x)
+ return x
+
+
+class LayerNorm2D(nn.Module):
+ """
+ LayerNorm over C for a 4-D tensor (B, C, H, W)
+ """
+ def __init__(self, num_channels, eps=1e-5, affine=True):
+ super().__init__()
+ self.num_channels = num_channels
+ self.eps = eps
+ self.affine = affine
+ if affine:
+ self.weight = nn.Parameter(torch.ones(1, num_channels, 1, 1))
+ self.bias = nn.Parameter(torch.zeros(1, num_channels, 1, 1))
+
+ def forward(self, x):
+ # x: (B, C, H, W)
+ mean = x.mean(dim=1, keepdim=True) # (B, 1, H, W)
+ var = x.var (dim=1, keepdim=True, unbiased=False)
+ x = (x - mean) / torch.sqrt(var + self.eps)
+ if self.affine:
+ x = x * self.weight + self.bias
+ return x
+
+
+class PoseRefNetNoBNV3(nn.Module):
+ def __init__(self,
+ in_channels_c: int,
+ in_channels_x: int,
+ hidden_dim: int = 256,
+ num_heads: int = 8,
+ dropout: float = 0.1):
+ super().__init__()
+ self.d_model = hidden_dim
+ self.nhead = num_heads
+
+ self.proj_p = nn.Conv2d(in_channels_c, hidden_dim, kernel_size=1)
+ self.proj_r = nn.Conv2d(in_channels_x, hidden_dim, kernel_size=1)
+
+ self.proj_p_back = nn.Conv2d(hidden_dim, in_channels_c, kernel_size=1)
+
+ self.cross_attn = nn.MultiheadAttention(hidden_dim,
+ num_heads=num_heads,
+ dropout=dropout)
+
+ self.ffn_pose = nn.Sequential(
+ nn.Conv2d(hidden_dim, hidden_dim, kernel_size=1),
+ nn.SiLU(),
+ nn.Conv2d(hidden_dim, hidden_dim, kernel_size=1)
+ )
+
+ self.norm1 = LayerNorm2D(hidden_dim)
+ self.norm2 = LayerNorm2D(hidden_dim)
+
+ def forward(self, pose, ref, mask=None):
+ """
+ pose : (B, C1, T, H, W)
+ ref : (B, C2, T, H, W)
+ mask : (B, T*H*W) 可选 key_padding_mask
+ return: (B, d_model, T, H, W)
+ """
+ B, _, T, H, W = pose.shape
+ L = H * W
+
+ p_trans = pose.permute(0, 2, 1, 3, 4).contiguous().flatten(0, 1)
+ r_trans = ref.permute(0, 2, 1, 3, 4).contiguous().flatten(0, 1)
+
+ p_trans = self.proj_p(p_trans)
+ r_trans = self.proj_r(r_trans)
+
+ p_trans = p_trans.flatten(2).transpose(1, 2)
+ r_trans = r_trans.flatten(2).transpose(1, 2)
+
+ out = self.cross_attn(query=r_trans,
+ key=p_trans,
+ value=p_trans,
+ key_padding_mask=mask)[0]
+
+ out = out.transpose(1, 2).contiguous().view(B*T, -1, H, W)
+ out = self.norm1(out)
+
+ ffn_out = self.ffn_pose(out)
+ out = out + ffn_out
+ out = self.norm2(out)
+ out = self.proj_p_back(out)
+ out = out.view(B, T, -1, H, W).contiguous().transpose(1, 2)
+
+ return out
diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.py
index d0d6772..21b371b 100644
--- a/models/wan/wan_handler.py
+++ b/models/wan/wan_handler.py
@@ -3,13 +3,15 @@ import os
import torch
import numpy as np
import gradio as gr
+import cv2
+from PIL import Image
from shared.utils import files_locator as fl
def test_vace(base_model_type):
return base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B", "vace_ditto_14B"]
def test_class_i2v(base_model_type):
- return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate", "chrono_edit" ]
+ return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate", "chrono_edit", "steadydancer" ]
def test_class_t2v(base_model_type):
return base_model_type in ["t2v", "t2v_2_2", "alpha", "lynx"]
@@ -41,7 +43,7 @@ class family_handler():
return ["multitalk", "infinitetalk", "fantasy", "vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B",
"t2v_1.3B", "standin", "lynx_lite", "lynx", "t2v", "t2v_2_2", "vace_1.3B", "vace_ditto_14B", "phantom_1.3B", "phantom_14B",
"recam_1.3B", "animate", "alpha", "alpha_lynx", "chrono_edit",
- "i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "mocha"]
+ "i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "mocha", "steadydancer"]
@staticmethod
@@ -299,6 +301,24 @@ class family_handler():
}
extra_model_def["v2i_switch_supported"] = True
+
+ if base_model_type in ["steadydancer"]:
+ extra_model_def["guide_custom_choices"] = {
+ "choices":[
+ ("Use Control Video Poses to Animate Person in Start Image", "V"),
+ ("Use Control Video Poses filterd with Mask Video to Animate Person in Start Image", "VA"),
+ ],
+ "default": "PVB",
+ "letters_filter": "PVBA",
+ "label": "Type of Process",
+ "scale": 3,
+ "show_label" : False,
+ }
+ extra_model_def["custom_preprocessor"] = "Extracting Pose Information"
+ extra_model_def["alt_guidance"] = "Condition Guidance"
+ extra_model_def["no_guide2_refresh"] = True
+ extra_model_def["no_mask_refresh"] = True
+
if base_model_type in ["infinitetalk"]:
extra_model_def["no_background_removal"] = True
@@ -531,7 +551,7 @@ class family_handler():
image_prompt_types_allowed = "TSVL"
elif base_model_type in ["lucy_edit"]:
image_prompt_types_allowed = "TVL"
- elif multitalk or base_model_type in ["fantasy"]:
+ elif multitalk or base_model_type in ["fantasy", "steadydancer"]:
image_prompt_types_allowed = "SVL"
elif i2v:
image_prompt_types_allowed = "SEVL"
@@ -598,6 +618,25 @@ class family_handler():
return download_def
+ @staticmethod
+ def custom_preprocess(base_model_type, video_guide, video_mask, pre_video_guide=None, max_workers = 1, expand_scale = 0, **kwargs):
+ from .steadydancer.pose_align import PoseAligner
+ from shared.utils.utils import convert_tensor_to_image
+
+ ref_image = convert_tensor_to_image(pre_video_guide[:, 0])
+ frames = video_guide
+ mask_frames = None
+ mask_frames = None if video_mask is None else video_mask
+ aligner = PoseAligner()
+ outputs = aligner.align( frames, ref_image, ref_video_mask=mask_frames, align_frame=0, max_frames=None, augment=True, include_composite= True, cpu_resize_workers= max_workers, expand_scale = expand_scale )
+
+ video_guide_processed, video_guide_processed2 = outputs["pose_only"], outputs["pose_aug"]
+ if video_guide_processed.numel() == 0: return None, None, None
+
+ video_mask_processed = video_mask_processed2 = None
+
+ return video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2
+
@staticmethod
def load_model(model_filename, model_type, base_model_type, model_def, quantizeTransformer = False, text_encoder_quantization = None, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized= False, submodel_no_list = None, override_text_encoder = None, VAE_upsampling = None, **kwargs):
@@ -806,6 +845,14 @@ class family_handler():
"audio_prompt_type": "R",
"force_fps": "control",
})
+ elif base_model_type in ["steadydancer"]:
+ ui_defaults.update({
+ "video_prompt_type": "VA",
+ "image_prompt_type": "S",
+ "audio_prompt_type": "R",
+ "force_fps": "control",
+ "alt_guidance_scale" : 2.0,
+ })
if base_model_type in ["i2v_2_2"]:
diff --git a/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json b/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json
index 1fe4c14..32be54a 100644
--- a/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json
+++ b/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json
@@ -6,5 +6,7 @@
"loras_multipliers": "1",
"num_inference_steps": 10,
"guidance_phases": 1,
- "guidance_scale": 1
+ "guidance_scale": 1,
+ "alt_guidance_scale": 1,
+ "audio_guidance_scale": 1
}
diff --git a/wgp.py b/wgp.py
index 9a40485..fd76bff 100644
--- a/wgp.py
+++ b/wgp.py
@@ -83,7 +83,7 @@ global_queue_ref = []
AUTOSAVE_FILENAME = "queue.zip"
PROMPT_VARS_MAX = 10
target_mmgp_version = "3.6.9"
-WanGP_version = "9.72"
+WanGP_version = "9.73"
settings_version = 2.41
max_source_video_frames = 3000
prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None
@@ -288,6 +288,7 @@ def edit_task_in_queue(
switch_threshold2,
guidance_phases,
model_switch_phase,
+ alt_guidance_scale,
audio_guidance_scale,
flow_shift,
sample_solver,
@@ -3811,6 +3812,7 @@ def select_video(state, current_gallery_tab, input_file_list, file_selected, aud
video_guidance2_scale = configs.get("guidance2_scale", None)
video_guidance3_scale = configs.get("guidance3_scale", None)
video_audio_guidance_scale = configs.get("audio_guidance_scale", None)
+ video_alt_guidance_scale = configs.get("alt_guidance_scale", None)
video_switch_threshold = configs.get("switch_threshold", 0)
video_switch_threshold2 = configs.get("switch_threshold2", 0)
video_model_switch_phase = configs.get("model_switch_phase", 1)
@@ -3898,8 +3900,14 @@ def select_video(state, current_gallery_tab, input_file_list, file_selected, aud
if model_def.get("sample_solvers", None) is not None and len(video_sample_solver) > 0 :
values += [video_sample_solver]
labels += ["Sampler Solver"]
- values += [video_resolution, video_length_summary, video_seed, video_guidance_scale, video_audio_guidance_scale, video_flow_shift, video_num_inference_steps]
- labels += [ "Resolution", video_length_label, "Seed", video_guidance_label, "Audio Guidance Scale", "Shift Scale", "Num Inference steps"]
+ values += [video_resolution, video_length_summary, video_seed, video_guidance_scale, video_audio_guidance_scale]
+ labels += ["Resolution", video_length_label, "Seed", video_guidance_label, "Audio Guidance Scale"]
+ alt_guidance_type = model_def.get("alt_guidance", None)
+ if alt_guidance_type is not None and video_alt_guidance_scale is not None:
+ values += [video_alt_guidance_scale]
+ labels += [alt_guidance_type]
+ values += [video_flow_shift, video_num_inference_steps]
+ labels += ["Shift Scale", "Num Inference steps"]
video_negative_prompt = configs.get("negative_prompt", "")
if len(video_negative_prompt) > 0:
values += [video_negative_prompt]
@@ -4870,7 +4878,36 @@ def truncate_audio(generated_audio, trim_video_frames_beginning, trim_video_fram
start = int(trim_video_frames_beginning * samples_per_frame)
end = len(generated_audio) - int(trim_video_frames_end * samples_per_frame)
return generated_audio[start:end if end > 0 else None]
+
+def custom_preprocess_video_with_mask(model_handler, base_model_type, pre_video_guide, video_guide, video_mask, height, width, max_frames, start_frame, fit_canvas, fit_crop, target_fps, block_size, expand_scale):
+ pad_frames = 0
+ if start_frame < 0:
+ pad_frames= -start_frame
+ max_frames += start_frame
+ start_frame = 0
+
+ max_workers = get_default_workers()
+
+ if not video_guide or max_frames <= 0:
+ return None, None
+ video_guide = get_resampled_video(video_guide, start_frame, max_frames, target_fps).permute(-1, 0, 1, 2)
+ video_guide = video_guide / 127.5 - 1.
+ any_mask = video_mask is not None
+ if video_mask is not None:
+ video_mask = get_resampled_video(video_mask, start_frame, max_frames, target_fps).permute(-1, 0, 1, 2)
+ video_mask = video_mask[:1] / 255.
+
+ if len(video_guide) == 0 or any_mask and len(video_mask) == 0:
+ return None, None
+ video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2 = model_handler.custom_preprocess(base_model_type = base_model_type, pre_video_guide = pre_video_guide, video_guide = video_guide, video_mask = video_mask, height = height, width = width, fit_canvas = fit_canvas , fit_crop = fit_crop, target_fps = target_fps, block_size = block_size, max_workers = max_workers, expand_scale = expand_scale)
+
+ # if pad_frames > 0:
+ # masked_frames = masked_frames[0] * pad_frames + masked_frames
+ # if any_mask: masked_frames = masks[0] * pad_frames + masks
+
+ return video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2
+
def generate_video(
task,
send_cmd,
@@ -4890,6 +4927,7 @@ def generate_video(
switch_threshold2,
guidance_phases,
model_switch_phase,
+ alt_guidance_scale,
audio_guidance_scale,
flow_shift,
sample_solver,
@@ -5471,18 +5509,25 @@ def generate_video(
preprocess_type = process_map_video_guide.get(process_letter, "raw")
else:
preprocess_type2 = process_map_video_guide.get(process_letter, None)
- status_info = "Extracting " + processes_names[preprocess_type]
- extra_process_list = ([] if preprocess_type2==None else [preprocess_type2]) + ([] if process_outside_mask==None or process_outside_mask == preprocess_type else [process_outside_mask])
- if len(extra_process_list) == 1:
- status_info += " and " + processes_names[extra_process_list[0]]
- elif len(extra_process_list) == 2:
- status_info += ", " + processes_names[extra_process_list[0]] + " and " + processes_names[extra_process_list[1]]
- context_scale = [control_net_weight /2, control_net_weight2 /2] if preprocess_type2 is not None else [control_net_weight]
- if not (preprocess_type == "identity" and preprocess_type2 is None and video_mask is None):send_cmd("progress", [0, get_latest_status(state, status_info)])
- inpaint_color = 0 if preprocess_type=="pose" and process_outside_mask == "inpaint" else guide_inpaint_color
- video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size, to_bbox = "H" in video_prompt_type )
- if preprocess_type2 != None:
- video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size, to_bbox = "H" in video_prompt_type )
+ custom_preprocessor = model_def.get("custom_preprocessor", None)
+ if custom_preprocessor is not None:
+ status_info = custom_preprocessor
+ send_cmd("progress", [0, get_latest_status(state, status_info)])
+ video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2 = custom_preprocess_video_with_mask(model_handler, base_model_type, pre_video_guide, video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, block_size = block_size, expand_scale = mask_expand)
+ else:
+ status_info = "Extracting " + processes_names[preprocess_type]
+ extra_process_list = ([] if preprocess_type2==None else [preprocess_type2]) + ([] if process_outside_mask==None or process_outside_mask == preprocess_type else [process_outside_mask])
+ if len(extra_process_list) == 1:
+ status_info += " and " + processes_names[extra_process_list[0]]
+ elif len(extra_process_list) == 2:
+ status_info += ", " + processes_names[extra_process_list[0]] + " and " + processes_names[extra_process_list[1]]
+ context_scale = [control_net_weight /2, control_net_weight2 /2] if preprocess_type2 is not None else [control_net_weight]
+
+ if not (preprocess_type == "identity" and preprocess_type2 is None and video_mask is None):send_cmd("progress", [0, get_latest_status(state, status_info)])
+ inpaint_color = 0 if preprocess_type=="pose" and process_outside_mask == "inpaint" else guide_inpaint_color
+ video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size, to_bbox = "H" in video_prompt_type )
+ if preprocess_type2 != None:
+ video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size, to_bbox = "H" in video_prompt_type )
if video_guide_processed is not None and sample_fit_canvas is not None:
image_size = video_guide_processed.shape[-2:]
@@ -5573,9 +5618,9 @@ def generate_video(
preview_frame_no = 0 if extract_guide_from_window_start or model_def.get("dont_cat_preguide", False) or sparse_video_image is not None else (guide_start_frame - window_start_frame)
preview_frame_no = min(src_video.shape[1] -1, preview_frame_no)
refresh_preview["video_guide"] = convert_tensor_to_image(src_video, preview_frame_no)
- if src_video2 is not None:
+ if src_video2 is not None and not model_def.get("no_guide2_refresh", False):
refresh_preview["video_guide"] = [refresh_preview["video_guide"], convert_tensor_to_image(src_video2, preview_frame_no)]
- if src_mask is not None and video_mask is not None:
+ if src_mask is not None and video_mask is not None and not model_def.get("no_mask_refresh", False):
refresh_preview["video_mask"] = convert_tensor_to_image(src_mask, preview_frame_no, mask_levels = True)
if src_ref_images is not None or nb_frames_positions:
@@ -5664,6 +5709,7 @@ def generate_video(
apg_switch = apg_switch,
cfg_star_switch = cfg_star_switch,
cfg_zero_step = cfg_zero_step,
+ alt_guide_scale= alt_guidance_scale,
audio_cfg_scale= audio_guidance_scale,
audio_guide=audio_guide,
audio_guide2=audio_guide2,
@@ -6836,6 +6882,10 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = None
if not model_def.get("embedded_guidance", False):
pop += ["embedded_guidance_scale"]
+ if model_def.get("alt_guidance", None) is None:
+ pop += ["alt_guidance_scale"]
+
+
if not (model_def.get("tea_cache", False) or model_def.get("mag_cache", False)) :
pop += ["skip_steps_cache_type", "skip_steps_multiplier", "skip_steps_start_step_perc"]
@@ -7396,6 +7446,7 @@ def save_inputs(
switch_threshold2,
guidance_phases,
model_switch_phase,
+ alt_guidance_scale,
audio_guidance_scale,
flow_shift,
sample_solver,
@@ -8719,9 +8770,12 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
any_audio_guidance = model_def.get("audio_guidance", False)
any_embedded_guidance = model_def.get("embedded_guidance", False)
- with gr.Row(visible =any_embedded_guidance or any_audio_guidance) as embedded_guidance_row:
+ alt_guidance_type = model_def.get("alt_guidance", None)
+ any_alt_guidance = alt_guidance_type is not None
+ with gr.Row(visible =any_embedded_guidance or any_audio_guidance or any_alt_guidance) as embedded_guidance_row:
audio_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("audio_guidance_scale", 4), step=0.5, label="Audio Guidance", visible= any_audio_guidance, show_reset_button= False )
embedded_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("embedded_guidance_scale", 6.0), step=0.5, label="Embedded Guidance Scale", visible=any_embedded_guidance, show_reset_button= False )
+ alt_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("alt_guidance_scale", 6.0), step=0.5, label= alt_guidance_type if any_alt_guidance else "" , visible=any_alt_guidance, show_reset_button= False )
with gr.Row(visible=audio_only) as temperature_row:
temperature = gr.Slider( 0.1, 1.5, value=ui_defaults.get("temperature", 0.8), step=0.01, label="Temperature", show_reset_button= False)