Python
Added Steady Dancer support
DeepBeepMeep3feaa249cfcb
diff --git a/README.md b/README.mdindex 6f4f6c6..44de59c 100644--- a/README.md+++ b/README.md@@ -37,7 +37,7 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models## 🔥 Latest Updates :-### November 28 2025: WanGP v9.72, The Alpha & the Omega+### December 2 2025: WanGP v9.72, The Alpha & the Omega ... and the Dancer- **Flux 2**: the best ever open source *Image Generator* has just landed. It does everything very well: generate an *Image* based a *Text Prompt* or combine up to 10 *Images References*@@ -47,12 +47,15 @@ Behold the WanGP Miracle ! Flux 2 wil work with only 8 GB of VRAM if you are hap- **Z-Image**: a small model, very fast (8 steps), very low VRAM (optimized even more in WanGP for fun, just in case you want to generate 16 images at a time) that produces outstanding Image quality. Not yet the Flux 2 level, and no Image editing yet but a very good trade-off.+- **Steady Dancer**: here is *Wan Steady Dancer* a very nice alternative to *Wan Animate*. You can transfer the motion of a Control video in a very smooth way. It will work best with Videos where the action happens center stage (hint: *dancing*). Use the *Lora accelerator* *Fusionix i2v 10 steps* for a fast generation. For higher quality you can set *Condition Guidance* to 2 or if you are very patient keep *Guidance* to a value greater than 1.+I have added a new Memory Profile *Profile 4+* that is sligthly slower than *Profile 4* but can save you up to 1GB of VRAM with Flux 2.Also as we have now quite few models and Loras folders. *I have moved all the loras folder in the 'loras' folder*. There are also now unique subfolders for *Wan 5B* and *Wan 1.3B* models. A conversion script should have moved the loras in the right locations, but I advise that you check just in case.*update 9.71* : added missing source file, have fun !*update 9.72* : added Z-Image & Loras reorg+*update 9.73* : added Steady Dancer### November 24 2025: WanGP v9.62, The Return of the Kingdiff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex c7d9a42..231e36f 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -406,6 +406,7 @@ class WanAny2V:audio_cfg_scale=None,audio_proj=None,audio_context_lens=None,+ alt_guide_scale = 1.0,overlapped_latents = None,return_latent_slice = None,overlap_noise = 0,@@ -525,9 +526,10 @@ class WanAny2V:lucy_edit= model_type in ["lucy_edit"]animate= model_type in ["animate"]chrono_edit = model_type in ["chrono_edit"]- mocha = model_type in ["mocha"]+ mocha = model_type in ["mocha"]+ steadydancer = model_type in ["steadydancer"]start_step_no = 0- ref_images_count = 0+ ref_images_count = inner_latent_frames = 0trim_frames = 0last_latent_preview = Falseextended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = latent_slice = freqs = None@@ -638,9 +640,28 @@ class WanAny2V:lat_y = self.vae.encode([input_video], VAE_tile_size)[0]extended_overlapped_latents = lat_y[:, :overlapped_latents_frames_num].clone().unsqueeze(0)- lat_y = input_video = None+ lat_y = Nonekwargs.update({ 'y': y})++ # Steady Dancer+ if steadydancer:+ condition_guide_scale = alt_guide_scale # 2.0+ # ref img_x+ ref_x = self.vae.encode([input_video[:, :1]], VAE_tile_size)[0]+ msk_ref = torch.ones(4, 1, lat_h, lat_w, device=self.device)+ ref_x = torch.concat([ref_x, msk_ref, ref_x])+ # ref img_c+ ref_c = self.vae.encode([input_frames[:, :1]], VAE_tile_size)[0]+ msk_c = torch.zeros(4, 1, lat_h, lat_w, device=self.device)+ ref_c = torch.concat([ref_c, msk_c, ref_c])+ kwargs.update({ 'steadydancer_ref_x': ref_x, 'steadydancer_ref_c': ref_c})+ # conditions, w/o msk+ conditions = self.vae.encode([input_frames])[0].unsqueeze(0)+ # conditions_null, w/o msk+ conditions_null = self.vae.encode([input_frames2])[0].unsqueeze(0)+ inner_latent_frames = 2+# Chrono Editif chrono_edit:if frame_num == 5:@@ -690,6 +711,8 @@ class WanAny2V:clip_context = self.clip.visual([clip_image_start[:, None, :, :]])clip_image_start = clip_image_end = Nonekwargs.update({'clip_fea': clip_context})+ if steadydancer:+ kwargs['steadydancer_clip_fea_c'] = self.clip.visual([input_frames[:, :1]])# Recam Master & Lucy Editif recam or lucy_edit:@@ -883,7 +906,7 @@ class WanAny2V:shape[extended_input_dim-2] *= 2freqs = get_rotary_pos_embed(shape, enable_RIFLEx= False)else:- freqs = get_rotary_pos_embed(target_shape[1:], enable_RIFLEx= enable_RIFLEx)+ freqs = get_rotary_pos_embed( (target_shape[1]+ inner_latent_frames ,) + target_shape[2:] , enable_RIFLEx= enable_RIFLEx)kwargs["freqs"] = freqs@@ -1035,6 +1058,20 @@ class WanAny2V:if model_type in ["lynx", "vace_lynx_14B"]:gen_args["lynx_ref_buffer"] = [lynx_ref_buffer, lynx_ref_buffer_uncond]+ elif steadydancer:+ if guide_scale == 1:+ gen_args = {+ "x" : [latent_model_input, latent_model_input],+ "context" : [context, context],+ "steadydancer_condition": [conditions, conditions_null],+ }+ any_guidance = condition_guide_scale != 1+ else:+ gen_args = {+ "x" : [latent_model_input, latent_model_input, latent_model_input],+ "context" : [context, context_null, context],+ "steadydancer_condition": [conditions, conditions, conditions_null],+ }elif multitalk and audio_proj != None:if guide_scale == 1:gen_args = {@@ -1082,6 +1119,34 @@ class WanAny2V:noise_pred_cond, noise_pred_noaudio, noise_pred_uncond = ret_valuesnoise_pred = noise_pred_uncond + guide_scale * (noise_pred_noaudio - noise_pred_uncond) + audio_cfg_scale * (noise_pred_cond - noise_pred_noaudio)noise_pred_noaudio = None+ elif steadydancer:+ if apg_switch != 0 and False:+ if guide_scale == 1:+ noise_pred_cond, noise_pred_drop_audio = ret_values+ noise_pred = noise_pred_cond + (audio_cfg_scale - 1)* adaptive_projected_guidance(noise_pred_cond - noise_pred_drop_audio,+ noise_pred_cond,+ momentum_buffer=audio_momentumbuffer,+ norm_threshold=apg_norm_threshold)++ else:+ noise_pred_cond, noise_pred_drop_text, noise_pred_uncond = ret_values+ noise_pred = noise_pred_cond + (guide_scale - 1) * adaptive_projected_guidance(noise_pred_cond - noise_pred_drop_text,+ noise_pred_cond,+ momentum_buffer=text_momentumbuffer,+ norm_threshold=apg_norm_threshold) \+ + (audio_cfg_scale - 1) * adaptive_projected_guidance(noise_pred_drop_text - noise_pred_uncond,+ noise_pred_cond,+ momentum_buffer=audio_momentumbuffer,+ norm_threshold=apg_norm_threshold)+ else:+ if guide_scale == 1:+ noise_pred_cond, noise_pred_uncond_condition = ret_values+ noise_pred = noise_pred_uncond_condition + condition_guide_scale * (noise_pred_cond - noise_pred_uncond_condition)+ else:+ noise_pred_cond, noise_pred_uncond_context, noise_pred_uncond_condition = ret_values+ noise_pred = noise_pred_uncond_context + guide_scale * (noise_pred_cond - noise_pred_uncond_context) + condition_guide_scale * (noise_pred_cond - noise_pred_uncond_condition)+ noise_pred_uncond_context = noise_pred_cond = noise_pred_uncond_condition = None+elif multitalk and audio_proj != None:if apg_switch != 0:if guide_scale == 1:diff --git a/models/wan/configs/steadydancer.json b/models/wan/configs/steadydancer.jsonnew file mode 100644index 0000000..a00032f--- /dev/null+++ b/models/wan/configs/steadydancer.json@@ -0,0 +1,15 @@+{+ "_class_name": "WanModel",+ "_diffusers_version": "0.30.0",+ "dim": 5120,+ "eps": 1e-06,+ "ffn_dim": 13824,+ "freq_dim": 256,+ "in_dim": 36,+ "model_type": "i2v",+ "num_heads": 40,+ "num_layers": 40,+ "out_dim": 16,+ "text_len": 512,+ "steadydancer": true+}diff --git a/models/wan/modules/model.py b/models/wan/modules/model.pyindex 5a6ec58..5c4f86b 100644--- a/models/wan/modules/model.py+++ b/models/wan/modules/model.py@@ -19,6 +19,8 @@ from ..multitalk.multitalk_utils import get_attn_map_with_targetfrom ..animate.motion_encoder import Generatorfrom ..animate.face_blocks import FaceAdapter, FaceEncoderfrom ..animate.model_animate import after_patch_embedding+from ..steadydancer.small_archs import FactorConv3d, PoseRefNetNoBNV3+from ..steadydancer.mobilenetv2_dcd import DYModule__all__ = ['WanModel']@@ -1028,6 +1030,7 @@ class WanModel(ModelMixin, ConfigMixin):standin= False,motion_encoder_dim=0,lynx=None,+ steadydancer = False,):super().__init__()@@ -1059,7 +1062,8 @@ class WanModel(ModelMixin, ConfigMixin):self.vae_scale = vae_scalemultitalk = multitalk_output_dim > 0- self.multitalk = multitalk+ self.multitalk = multitalk+ self.steadydancer = steadydanceranimate = motion_encoder_dim > 0# embeddings@@ -1181,6 +1185,32 @@ class WanModel(ModelMixin, ConfigMixin):num_heads=4,)+ if steadydancer:+ self.in_dim_c = 16++ ############### Condition-Reconciliation Mechanism ###############+ self.patch_embedding_fuse = nn.Conv3d( # x, fused pose, aligned pose+ in_dim + self.in_dim_c + self.in_dim_c, dim, kernel_size=patch_size, stride=patch_size)+ self.patch_embedding_ref_c = nn.Conv3d( # ref_c+ self.in_dim_c, dim, kernel_size=patch_size, stride=patch_size)++ ############### Synergistic Pose Modulation Modules ###############+ # Spatial Structure Adaptive Extractor+ self.condition_embedding_spatial = DYModule(inp=self.in_dim_c, oup=self.in_dim_c)+ # Temporal Motion Coherence Module+ self.condition_embedding_temporal = nn.Sequential(+ FactorConv3d(in_channels=self.in_dim_c, out_channels=self.in_dim_c, kernel_size=(3, 3, 3), stride=1),+ nn.SiLU(),+ FactorConv3d(in_channels=self.in_dim_c, out_channels=self.in_dim_c, kernel_size=(3, 3, 3), stride=1),+ nn.SiLU(),+ FactorConv3d(in_channels=self.in_dim_c, out_channels=self.in_dim_c, kernel_size=(3, 3, 3), stride=1),+ nn.SiLU()+ )+ # Frame-wise Attention Alignment Unit+ self.condition_embedding_align = PoseRefNetNoBNV3(in_channels_x=16,+ in_channels_c=16,+ hidden_dim=128,+ num_heads=8)def adapt_modulation(self, block_name ='blocks'):def move(v, param_name = "modulation"):@@ -1414,8 +1444,11 @@ class WanModel(ModelMixin, ConfigMixin):lynx_ip_scale = 0,lynx_ref_scale = 0,lynx_feature_extractor = False,- lynx_ref_buffer = None,-+ lynx_ref_buffer = None,+ steadydancer_condition = None,+ steadydancer_ref_x = None,+ steadydancer_ref_c = None,+ steadydancer_clip_fea_c = None,):# patch_dtype = self.patch_embedding.weight.dtypemodulation_dtype = self.time_projection[1].weight.dtype@@ -1430,11 +1463,15 @@ class WanModel(ModelMixin, ConfigMixin):if chipmunk:# from chipmunk.ops.voxel import voxel_chunk_no_padding, reverse_voxel_chunk_no_paddingvoxel_shape = (4, 6, 8)-+ real_seq = 0x_list = xjoint_pass = len(x_list) > 1is_source_x = [ x.data_ptr() == x_list[0].data_ptr() and i > 0 for i, x in enumerate(x_list) ]last_x_idx = 0+ steadydancer = steadydancer_condition is not None+ if steadydancer: # steady dancer+ x_noise_clone = x_list[0].clone()+for i, (is_source, x) in enumerate(zip(is_source_x, x_list)):if is_source:x_list[i] = x_list[0].clone()@@ -1447,11 +1484,32 @@ class WanModel(ModelMixin, ConfigMixin):if bz > 1: y = y.expand(bz, -1, -1, -1, -1)x = torch.cat([x, y], dim=1)# embeddings- x = self.patch_embedding(x).to(modulation_dtype)- grid_sizes = x.shape[2:]+ if not steadydancer:+ x = self.patch_embedding(x).to(modulation_dtype)+ grid_sizes = x.shape[2:]x_list[i] = xy = None+ if steadydancer: # steady dancer+ # Spatial Structure Adaptive Extractor.+ time_steps = steadydancer_condition[0].shape[2]+ for i, (x, condition) in enumerate(zip(x_list, steadydancer_condition)):+ real_seq = x.shape[1]+ # Temporal Motion Coherence Module.+ condition_temporal =self.condition_embedding_temporal(condition)+ condition_spatial = rearrange(self.condition_embedding_spatial(rearrange(condition, 'b c t h w -> (b t) c h w')), '(b t) c h w -> b c t h w', t=time_steps, b=1)+ # Hierarchical Aggregation (1): condition, temporal condition, spatial condition+ condition_fused = condition + condition_temporal + condition_spatial+ # Frame-wise Attention Alignment Unit.+ condition_aligned = self.condition_embedding_align(condition_fused, x_noise_clone)+ # Condition Fusion/Injection, Hierarchical Aggregation (2): x, fused condition, aligned condition+ x = self.patch_embedding_fuse(torch.cat([x, condition_fused, condition_aligned], 1))+ x = torch.cat([x, self.patch_embedding(steadydancer_ref_x.unsqueeze(0)), self.patch_embedding_ref_c(steadydancer_ref_c[:16].unsqueeze(0))], dim=2)+ grid_sizes = x.shape[2:]+ x_list[i] = x+ x = condition = condition_fused = condition_aligned = condition_temporal = condition_spatial = None+ x_noise_clone = x = None+motion_vec_list = []if face_pixel_values is None: face_pixel_values = [None] * len(x_list)for i, (x, one_face_pixel_values) in enumerate(zip(x_list, face_pixel_values)):@@ -1470,6 +1528,7 @@ class WanModel(ModelMixin, ConfigMixin):x = None+block_mask = Noneif causal_attention and causal_block_size > 0 and False: # NEVER WORKEDframe_num = grid_sizes[0]@@ -1545,6 +1604,9 @@ class WanModel(ModelMixin, ConfigMixin):if clip_fea is not None:context_clip = self.img_emb(clip_fea) # bs x 257 x dim+ if steadydancer_clip_fea_c is not None:+ context_clip += self.img_emb(steadydancer_clip_fea_c) # bs x 257 x dim+context_list = []for one_context in context:if len(one_context) != len(context_clip):@@ -1721,6 +1783,8 @@ class WanModel(ModelMixin, ConfigMixin):# unpatchifyx_list[i] = self.unpatchify(x, grid_sizes)+ if real_seq > 0:+ x = x[:, :real_seq]del xreturn [x.float() for x in x_list]diff --git a/models/wan/steadydancer/mobilenetv2_dcd.py b/models/wan/steadydancer/mobilenetv2_dcd.pynew file mode 100644index 0000000..4fa3e55--- /dev/null+++ b/models/wan/steadydancer/mobilenetv2_dcd.py@@ -0,0 +1,102 @@+# Modify from https://github.com/liyunsheng13/dcd/blob/main/models/imagenet/mobilenetv2_dcd.py++import torch+import torch.nn as nn+import torch.nn.functional as F+++class Hsigmoid(nn.Module):+ def __init__(self, inplace=True):+ super(Hsigmoid, self).__init__()+ self.inplace = inplace++ def forward(self, x):+ return F.relu6(x + 3., inplace=self.inplace) / 3.+++class DYModule(nn.Module):+ def __init__(self, inp, oup, fc_squeeze=8):+ super(DYModule, self).__init__()+ self.conv = nn.Conv2d(inp, oup, 1, 1, 0, bias=False)+ if inp < oup:+ self.mul = 4+ reduction = 8+ self.avg_pool = nn.AdaptiveAvgPool2d(2)+ else:+ self.mul = 1+ reduction = 2+ self.avg_pool = nn.AdaptiveAvgPool2d(1)++ self.dim = min((inp * self.mul) // reduction, oup // reduction)+ while self.dim ** 2 > inp * self.mul * 2:+ reduction *= 2+ self.dim = min((inp * self.mul) // reduction, oup // reduction)+ if self.dim < 4:+ self.dim = 4++ squeeze = max(inp * self.mul, self.dim ** 2) // fc_squeeze+ if squeeze < 4:+ squeeze = 4+ self.conv_q = nn.Conv2d(inp, self.dim, 1, 1, 0, bias=False)++ self.fc = nn.Sequential(+ nn.Linear(inp * self.mul, squeeze, bias=False),+ SEModule_small(squeeze),+ )+ self.fc_phi = nn.Linear(squeeze, self.dim ** 2, bias=False)+ self.fc_scale = nn.Linear(squeeze, oup, bias=False)+ self.hs = Hsigmoid()+ self.conv_p = nn.Conv2d(self.dim, oup, 1, 1, 0, bias=False)+ # self.bn1 = nn.BatchNorm2d(self.dim)+ self.bn1 = nn.GroupNorm(num_groups=4, num_channels=self.dim)+ # self.bn2 = nn.BatchNorm1d(self.dim)+ self.bn2 = nn.GroupNorm(num_groups=4, num_channels=self.dim)++ def forward(self, x):+ r = self.conv(x)++ b, c, h, w = x.size()+ y = self.avg_pool(x).view(b, c * self.mul)+ y = self.fc(y)+ dy_phi = self.fc_phi(y).view(b, self.dim, self.dim)+ dy_scale = self.hs(self.fc_scale(y)).view(b, -1, 1, 1)+ r = dy_scale.expand_as(r) * r++ x = self.conv_q(x)+ x = self.bn1(x)+ x = x.view(b, -1, h * w)+ x = self.bn2(torch.matmul(dy_phi, x)) + x+ x = x.view(b, -1, h, w)+ x = self.conv_p(x)+ return x + r+++class SEModule_small(nn.Module):+ def __init__(self, channel):+ super(SEModule_small, self).__init__()+ self.fc = nn.Sequential(+ nn.Linear(channel, channel, bias=False),+ Hsigmoid()+ )++ def forward(self, x):+ y = self.fc(x)+ return x * y+++class SEModule(nn.Module):+ def __init__(self, channel, reduction=4):+ super(SEModule, self).__init__()+ self.avg_pool = nn.AdaptiveAvgPool2d(1)+ self.fc = nn.Sequential(+ nn.Linear(channel, channel // reduction, bias=False),+ nn.ReLU(inplace=True),+ nn.Linear(channel // reduction, channel, bias=False),+ Hsigmoid()+ )++ def forward(self, x):+ b, c, _, _ = x.size()+ y = self.avg_pool(x).view(b, c)+ y = self.fc(y).view(b, c, 1, 1)+ return x * y.expand_as(x)diff --git a/models/wan/steadydancer/pose_align.py b/models/wan/steadydancer/pose_align.pynew file mode 100644index 0000000..9d6804b--- /dev/null+++ b/models/wan/steadydancer/pose_align.py@@ -0,0 +1,904 @@+import argparse+import os+import sys+from dataclasses import dataclass+from pathlib import Path+from typing import Dict, List, Tuple, Union+import itertools+import time++import cv2+import imageio.v2 as imageio+import numpy as np+import torch+from PIL import Image++from preprocessing.dwpose.pose import convert_to_numpy, draw_pose+from preprocessing.dwpose.wholebody import HWC3, Wholebody, resize_image+from shared.utils import files_locator as fl+from shared.utils.utils import process_images_multithread+from shared.utils.utils import convert_tensor_to_image, convert_image_to_tensor++ArrayImage = Union[np.ndarray, Image.Image, torch.Tensor]+++def _to_bgr_image(image: ArrayImage) -> np.ndarray:+ """Convert supported image types to uint8 BGR HWC."""+ if isinstance(image, torch.Tensor):+ # Expect CHW or CTHW in [-1,1] or [0,1] or 0-255+ if image.dim() == 4:+ image = image[:, 0] # take first frame+ img = image.detach().cpu()+ if img.shape[0] in (1, 3, 4):+ pass+ elif img.shape[-1] in (1, 3, 4):+ img = img.permute(2, 0, 1)+ # normalize to uint8+ if img.min() < 0:+ img = (img + 1.0) * 127.5+ elif img.max() <= 1.0:+ img = img * 255.0+ arr = img.clamp(0, 255).byte().permute(1, 2, 0).numpy()+ if arr.shape[2] == 1:+ arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)+ elif arr.shape[2] == 3:+ arr = cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)+ elif arr.shape[2] == 4:+ arr = cv2.cvtColor(arr, cv2.COLOR_RGBA2BGR)+ return HWC3(arr)+ was_pil = isinstance(image, Image.Image)+ arr = convert_to_numpy(image)++ # Handle CHW tensors+ if arr.ndim == 3 and arr.shape[0] in (1, 3, 4) and arr.shape[0] != arr.shape[1]:+ arr = np.transpose(arr, (1, 2, 0))++ if arr.ndim == 2:+ arr = cv2.cvtColor(arr, cv2.COLOR_GRAY2BGR)+ elif arr.shape[2] == 4:+ arr = cv2.cvtColor(arr, cv2.COLOR_RGBA2BGR)+ elif was_pil:+ arr = cv2.cvtColor(arr, cv2.COLOR_RGB2BGR)++ if arr.dtype != np.uint8:+ if arr.max() <= 1.0:+ arr = arr * 255.0+ arr = np.clip(arr, 0, 255).astype(np.uint8)++ return HWC3(arr)+++def _resize_to_height(img: np.ndarray, target_h: int) -> np.ndarray:+ """Resize while preserving aspect ratio to a specific height."""+ h, w = img.shape[:2]+ if h == target_h:+ return img+ new_w = int(round(w * target_h / float(h)))+ return cv2.resize(img, (new_w, target_h), interpolation=cv2.INTER_CUBIC)+++def _frames_to_tensor(frames: List[np.ndarray]) -> torch.Tensor:+ """Convert a list of RGB uint8 frames to a tensor in [-1, 1] with shape 3,F,H,W."""+ if not frames:+ return torch.empty(0)+ arr = np.stack(frames).astype(np.float32) / 127.5 - 1.0+ return torch.from_numpy(arr).permute(3, 0, 1, 2)+++def _tensor_to_frames(tensor: torch.Tensor) -> List[np.ndarray]:+ """Convert a tensor in [-1, 1] shaped 3,F,H,W back to RGB uint8 frames."""+ if tensor.numel() == 0:+ return []+ arr = tensor.permute(1, 2, 3, 0).cpu().numpy()+ arr = ((arr + 1.0) * 127.5).clip(0, 255).astype(np.uint8)+ return [frame for frame in arr]+++def _ensure_xyz(arr: np.ndarray) -> np.ndarray:+ """Ensure last dimension is 3 by padding zeros if needed."""+ if arr.shape[-1] >= 3:+ return arr+ pad_width = [(0, 0)] * arr.ndim+ pad_width[-1] = (0, 3 - arr.shape[-1])+ return np.pad(arr, pad_width, mode="constant", constant_values=0)+++def _xy_only(arr: np.ndarray) -> np.ndarray:+ """Return only x,y channels to satisfy drawing utils."""+ if arr.shape[-1] <= 2:+ return arr+ return arr[..., :2]+++def _mask_to_float01(mask: Union[np.ndarray, Image.Image, torch.Tensor]) -> np.ndarray:+ """Normalize any mask type to float32 single-channel in [0,1] with 0.5 threshold."""+ if isinstance(mask, Image.Image):+ mask = np.array(mask)+ elif isinstance(mask, torch.Tensor):+ m = mask.detach().cpu()+ if m.dim() == 4:+ m = m[0] # assume 1,C,H,W or C,H,W+ if m.shape[0] in (1, 3, 4):+ m = m.permute(1, 2, 0)+ mask = m.numpy()+ mask = np.asarray(mask)+ if mask.ndim == 3 and mask.shape[2] > 1:+ mask = mask.mean(axis=2)+ mask = mask.astype(np.float32)+ mask_min, mask_max = float(mask.min()), float(mask.max())+ if mask_max > 1.0:+ mask = mask / 255.0+ elif mask_min < 0.0:+ mask = (mask + 1.0) / 2.0+ mask = np.clip(mask, 0.0, 1.0)+ mask = (mask > 0.5).astype(np.float32)+ return mask+++def _safe_ratio(num: float, den: float) -> float:+ if den == 0 or not np.isfinite(den):+ return 1.0+ val = num / den+ return float(val) if np.isfinite(val) else 1.0+++def _nan_to_one(val: float) -> float:+ return 1.0 if not np.isfinite(val) else float(val)+++def _augment_pose(+ pose: Dict,+ offset_x: Tuple[float, float],+ offset_y: Tuple[float, float],+ scale_range: Tuple[float, float],+ aspect_ratio_range: Tuple[float, float],+ fixed_params: Tuple[float, float, float, float] | None = None,+) -> Dict:+ """Lightweight pose jitter used by the diff-aug variant."""+ pose_aug = {+ "bodies": {"candidate": pose["bodies"]["candidate"].copy(), "subset": pose["bodies"]["subset"].copy()},+ "hands": pose["hands"].copy(),+ "faces": pose["faces"].copy(),+ }++ if fixed_params is None:+ sx = np.random.uniform(scale_range[0], scale_range[1])+ aspect = np.random.uniform(aspect_ratio_range[0], aspect_ratio_range[1])+ scale_x = sx * aspect+ scale_y = sx / max(aspect, 1e-6)+ dx = np.random.uniform(offset_x[0], offset_x[1])+ dy = np.random.uniform(offset_y[0], offset_y[1])+ else:+ dx, dy, scale_x, scale_y = fixed_params++ def _apply(arr: np.ndarray) -> np.ndarray:+ arr = arr.copy()+ mask = arr[..., 0] >= 0+ arr[..., 0] = np.where(mask, arr[..., 0] * scale_x + dx, arr[..., 0])+ arr[..., 1] = np.where(mask, arr[..., 1] * scale_y + dy, arr[..., 1])+ return arr++ pose_aug["bodies"]["candidate"] = _apply(pose_aug["bodies"]["candidate"])+ pose_aug["hands"] = _apply(pose_aug["hands"])+ pose_aug["faces"] = _apply(pose_aug["faces"])+ return pose_aug+++def _save_video(path: str, frames: List[np.ndarray], fps: float) -> None:+ if not frames:+ return+ os.makedirs(os.path.dirname(path), exist_ok=True)+ with imageio.get_writer(path, fps=fps, codec="libx264") as writer:+ for frame in frames:+ writer.append_data(frame)+++def align_img(img: np.ndarray, pose_ori: Dict, scales: Dict[str, float]) -> Dict:+ """Align a single pose dictionary using pre-computed scale factors."""+ body_pose = pose_ori["bodies"]["candidate"].copy()+ hands = pose_ori["hands"].copy()+ faces = pose_ori["faces"].copy()++ H_in, W_in, _ = img.shape+ video_ratio = W_in / H_in+ body_pose[:, 0] *= video_ratio+ hands[:, :, 0] *= video_ratio+ faces[:, :, 0] *= video_ratio++ scale_neck = scales["scale_neck"]+ scale_face_left = scales["scale_face_left"]+ scale_face_right = scales["scale_face_right"]+ scale_shoulder = scales["scale_shoulder"]+ scale_arm_upper = scales["scale_arm_upper"]+ scale_arm_lower = scales["scale_arm_lower"]+ scale_hand = scales["scale_hand"]+ scale_body_len = scales["scale_body_len"]+ scale_leg_upper = scales["scale_leg_upper"]+ scale_leg_lower = scales["scale_leg_lower"]++ scale_list = [+ scale_neck,+ scale_face_left,+ scale_face_right,+ scale_shoulder,+ scale_arm_upper,+ scale_arm_lower,+ scale_hand,+ scale_body_len,+ scale_leg_upper,+ scale_leg_lower,+ ]+ finite_vals = [v for v in scale_list if np.isfinite(v)]+ mean_scale = np.mean(finite_vals) if finite_vals else 1.0+ scale_list = [mean_scale if np.isinf(v) else v for v in scale_list]+ (+ scale_neck,+ scale_face_left,+ scale_face_right,+ scale_shoulder,+ scale_arm_upper,+ scale_arm_lower,+ scale_hand,+ scale_body_len,+ scale_leg_upper,+ scale_leg_lower,+ ) = scale_list++ offset = {+ "14_16_to_0": body_pose[[14, 16], :] - body_pose[[0], :],+ "15_17_to_0": body_pose[[15, 17], :] - body_pose[[0], :],+ "3_to_2": body_pose[[3], :] - body_pose[[2], :],+ "4_to_3": body_pose[[4], :] - body_pose[[3], :],+ "6_to_5": body_pose[[6], :] - body_pose[[5], :],+ "7_to_6": body_pose[[7], :] - body_pose[[6], :],+ "9_to_8": body_pose[[9], :] - body_pose[[8], :],+ "10_to_9": body_pose[[10], :] - body_pose[[9], :],+ "12_to_11": body_pose[[12], :] - body_pose[[11], :],+ "13_to_12": body_pose[[13], :] - body_pose[[12], :],+ "hand_left_to_4": hands[1, :, :] - body_pose[[4], :],+ "hand_right_to_7": hands[0, :, :] - body_pose[[7], :],+ }++ def _warp(target: np.ndarray, center: np.ndarray, scale: float) -> np.ndarray:+ M = cv2.getRotationMatrix2D((center[0], center[1]), 0, scale)+ return warpAffine_kps(target, M)++ body_pose[[0], :] = _warp(body_pose[[0], :], body_pose[1], scale_neck)+ body_pose[[14, 16], :] = _warp(offset["14_16_to_0"] + body_pose[[0], :], body_pose[0], scale_face_left)+ body_pose[[15, 17], :] = _warp(offset["15_17_to_0"] + body_pose[[0], :], body_pose[0], scale_face_right)+ body_pose[[2, 5], :] = _warp(body_pose[[2, 5], :], body_pose[1], scale_shoulder)++ body_pose[[3], :] = _warp(offset["3_to_2"] + body_pose[[2], :], body_pose[2], scale_arm_upper)+ body_pose[[4], :] = _warp(offset["4_to_3"] + body_pose[[3], :], body_pose[3], scale_arm_lower)+ hands[1, :, :] = _warp(offset["hand_left_to_4"] + body_pose[[4], :], body_pose[4], scale_hand)++ body_pose[[6], :] = _warp(offset["6_to_5"] + body_pose[[5], :], body_pose[5], scale_arm_upper)+ body_pose[[7], :] = _warp(offset["7_to_6"] + body_pose[[6], :], body_pose[6], scale_arm_lower)+ hands[0, :, :] = _warp(offset["hand_right_to_7"] + body_pose[[7], :], body_pose[7], scale_hand)++ body_pose[[8, 11], :] = _warp(body_pose[[8, 11], :], body_pose[1], scale_body_len)+ body_pose[[9], :] = _warp(offset["9_to_8"] + body_pose[[8], :], body_pose[8], scale_leg_upper)+ body_pose[[10], :] = _warp(offset["10_to_9"] + body_pose[[9], :], body_pose[9], scale_leg_lower)+ body_pose[[12], :] = _warp(offset["12_to_11"] + body_pose[[11], :], body_pose[11], scale_leg_upper)+ body_pose[[13], :] = _warp(offset["13_to_12"] + body_pose[[12], :], body_pose[12], scale_leg_lower)++ body_pose_none = pose_ori["bodies"]["candidate"] == -1.0+ hands_none = pose_ori["hands"] == -1.0+ faces_none = pose_ori["faces"] == -1.0++ body_pose[body_pose_none] = -1.0+ hands[hands_none] = -1.0+ faces[faces_none] = -1.0++ body_pose = np.nan_to_num(body_pose, nan=-1.0)+ hands = np.nan_to_num(hands, nan=-1.0)+ faces = np.nan_to_num(faces, nan=-1.0)++ pose_align = dict(+ bodies={"candidate": body_pose, "subset": pose_ori["bodies"]["subset"]},+ hands=hands,+ faces=faces,+ )+ return pose_align+++def warpAffine_kps(kps: np.ndarray, M: np.ndarray) -> np.ndarray:+ kps_t = kps.copy()+ kps_t[..., 0] = kps[..., 0] * M[0, 0] + kps[..., 1] * M[0, 1] + M[0, 2]+ kps_t[..., 1] = kps[..., 0] * M[1, 0] + kps[..., 1] * M[1, 1] + M[1, 2]+ return kps_t+++@dataclass+class PoseDetection:+ pose: Dict+ pose_map_rgb: np.ndarray+ frame_rgb: np.ndarray+ orig_hw: Tuple[int, int]+++class PoseAligner:+ def __init__(self, detect_resolution: int = 1024, device: str = None, detection_workers: int = 2) -> None:+ det_model = fl.locate_file("pose/yolox_l.onnx")+ pose_model = fl.locate_file("pose/dw-ll_ucoco_384.onnx")+ resolved_device = device or ("cuda:0" if torch.cuda.is_available() else "cpu")+ self.det_model = det_model+ self.pose_model = pose_model+ self.device = resolved_device+ self.pose_estimation = Wholebody(det_model, pose_model, device=resolved_device)+ self.detection_workers = max(1, int(detection_workers))+ self.detect_resolution = detect_resolution++ def _detect_pose_session(self, image: ArrayImage, pose_estimation: Wholebody) -> PoseDetection | None:+ bgr_orig = _to_bgr_image(image)+ resized = resize_image(bgr_orig, self.detect_resolution)+ H, W, _ = resized.shape+ candidate, subset, _ = pose_estimation(resized)+ if len(candidate) == 0:+ return None++ nums, keys, locs = candidate.shape+ if keys < 18:+ return None # not enough keypoints detected++ candidate = _ensure_xyz(candidate.copy())+ subset = subset.copy()++ # Normalize coordinates to [0, 1]+ candidate[..., 0] /= float(W)+ candidate[..., 1] /= float(H)++ locs = candidate.shape[2]++ # Extract first person's body keypoints (18 keypoints)+ body = candidate[0, :18].copy() # shape (18, locs)+ body_scores = subset[0, :18].copy() # shape (18,)++ # Build score array and mask invisible keypoints+ score = np.zeros((1, 18), dtype=np.float64)+ for j in range(18):+ if body_scores[j] <= 0.3:+ body[j] = -1+ score[0, j] = -1+ else:+ score[0, j] = j # Index into body array++ # Extract faces (keypoints 24-92)+ faces = candidate[0:1, 24:92].copy() # shape (1, 68, locs)+ if subset.shape[1] > 24:+ face_scores = subset[0, 24:92]+ for j in range(faces.shape[1]):+ if j < len(face_scores) and face_scores[j] <= 0.3:+ faces[0, j] = -1+ faces = _ensure_xyz(faces)++ # Extract hands (keypoints 92-113 for right hand, 113-134 for left hand)+ # hands array should be shape (2, 21, locs) - [right_hand, left_hand]+ right_hand = candidate[0, 92:113].copy() if candidate.shape[1] > 92 else np.zeros((21, locs))+ left_hand = candidate[0, 113:134].copy() if candidate.shape[1] > 113 else np.zeros((21, locs))++ # Mask invalid hand keypoints+ if subset.shape[1] > 92:+ hand_scores = subset[0, 92:134]+ for j in range(21):+ if j < len(hand_scores) and hand_scores[j] <= 0.3:+ right_hand[j] = -1+ if j + 21 < len(hand_scores) and hand_scores[j + 21] <= 0.3:+ left_hand[j] = -1++ hands = _ensure_xyz(np.stack([right_hand, left_hand])) # shape (2, 21, locs)++ pose = {"bodies": {"candidate": body, "subset": score}, "hands": hands, "faces": faces}++ # Draw expects xy only+ pose_for_draw = {+ "bodies": {"candidate": _xy_only(body), "subset": score},+ "hands": _xy_only(hands),+ "faces": _xy_only(faces),+ }+ pose_map = draw_pose(pose_for_draw, H, W, use_body=True, use_hand=True, use_face=False)+ pose_map_rgb = cv2.cvtColor(pose_map, cv2.COLOR_BGR2RGB)+ frame_rgb = cv2.cvtColor(resized, cv2.COLOR_BGR2RGB)+ return PoseDetection(pose=pose, pose_map_rgb=pose_map_rgb, frame_rgb=frame_rgb, orig_hw=bgr_orig.shape[:2])++ def _detect_pose(self, image: ArrayImage) -> PoseDetection | None:+ return self._detect_pose_session(image, self.pose_estimation)++ def _compute_alignment(self, ref_pose: Dict, first_pose: Dict, ref_ratio: float, video_ratio: float) -> Tuple[Dict, np.ndarray]:+ body_ref = ref_pose["bodies"]["candidate"].copy()+ hands_ref = ref_pose["hands"].copy()+ faces_ref = ref_pose["faces"].copy()++ body_first = first_pose["bodies"]["candidate"].copy()+ hands_first = first_pose["hands"].copy()+ faces_first = first_pose["faces"].copy()++ body_ref[:, 0] *= ref_ratio+ hands_ref[:, :, 0] *= ref_ratio+ faces_ref[:, :, 0] *= ref_ratio++ body_first[:, 0] *= video_ratio+ hands_first[:, :, 0] *= video_ratio+ faces_first[:, :, 0] *= video_ratio++ def dist(body: np.ndarray, a: int, b: int) -> float:+ pa, pb = body[a, :2], body[b, :2]+ if (pa < 0).any() or (pb < 0).any():+ return np.nan+ return float(np.linalg.norm(pa - pb))++ def hand_dist(hand: np.ndarray, idx_a: int, idx_b: int) -> float:+ pa, pb = hand[idx_a, :2], hand[idx_b, :2]+ if (pa < 0).any() or (pb < 0).any():+ return np.nan+ return float(np.linalg.norm(pa - pb))++ align_args = {+ "scale_neck": _safe_ratio(dist(body_ref, 0, 1), dist(body_first, 0, 1)),+ "scale_face_left": _safe_ratio(+ dist(body_ref, 16, 14) + dist(body_ref, 14, 0),+ dist(body_first, 16, 14) + dist(body_first, 14, 0),+ ),+ "scale_face_right": _safe_ratio(+ dist(body_ref, 17, 15) + dist(body_ref, 15, 0),+ dist(body_first, 17, 15) + dist(body_first, 15, 0),+ ),+ "scale_shoulder": _safe_ratio(dist(body_ref, 2, 5), dist(body_first, 2, 5)),+ "scale_arm_upper": np.nanmean(+ [+ _safe_ratio(dist(body_ref, 2, 3), dist(body_first, 2, 3)),+ _safe_ratio(dist(body_ref, 5, 6), dist(body_first, 5, 6)),+ ]+ ),+ "scale_arm_lower": np.nanmean(+ [+ _safe_ratio(dist(body_ref, 3, 4), dist(body_first, 3, 4)),+ _safe_ratio(dist(body_ref, 6, 7), dist(body_first, 6, 7)),+ ]+ ),+ "scale_body_len": _safe_ratio(+ dist(body_ref, 1, 8) if not np.isnan(dist(body_ref, 1, 8)) else dist(body_ref, 1, 11),+ dist(body_first, 1, 8) if not np.isnan(dist(body_first, 1, 8)) else dist(body_first, 1, 11),+ ),+ "scale_leg_upper": np.nanmean(+ [+ _safe_ratio(dist(body_ref, 8, 9), dist(body_first, 8, 9)),+ _safe_ratio(dist(body_ref, 11, 12), dist(body_first, 11, 12)),+ ]+ ),+ "scale_leg_lower": np.nanmean(+ [+ _safe_ratio(dist(body_ref, 9, 10), dist(body_first, 9, 10)),+ _safe_ratio(dist(body_ref, 12, 13), dist(body_first, 12, 13)),+ ]+ ),+ }++ hand_pairs = [(0, 1), (0, 5), (0, 9), (0, 13), (0, 17)]+ hand_ratios = []+ for idx_a, idx_b in hand_pairs:+ hand_ratios.append(+ _safe_ratio(+ hand_dist(hands_ref[0], idx_a, idx_b),+ hand_dist(hands_first[0], idx_a, idx_b),+ )+ )+ hand_ratios.append(+ _safe_ratio(+ hand_dist(hands_ref[1], idx_a, idx_b),+ hand_dist(hands_first[1], idx_a, idx_b),+ )+ )+ hand_ratios = [v for v in hand_ratios if np.isfinite(v)]+ align_args["scale_hand"] = np.mean(hand_ratios) if hand_ratios else (+ align_args["scale_arm_upper"] + align_args["scale_arm_lower"]+ ) / 2++ align_args = {k: _nan_to_one(v) for k, v in align_args.items()}+ offset = np.array(+ [body_ref[1, 0] - body_first[1, 0], body_ref[1, 1] - body_first[1, 1], 0.0],+ dtype=np.float32,+ )+ return align_args, offset++ def align(+ self,+ ref_video_frames: List[ArrayImage],+ ref_image: ArrayImage,+ ref_video_mask: List[ArrayImage] | None = None,+ align_frame: int = 0,+ max_frames: int | None = None,+ include_composite: bool = False,+ augment: bool = True,+ augment_mode: str = "per_frame", # "per_frame" (original jitter), "fixed"+ augment_params: Dict | None = None,+ cpu_resize_workers: int = None,+ resize_ref_video: bool = False,+ detection_chunk_size: int = 8,+ expand_scale = 0,+ verbose: int = 0,+ ) -> Dict[str, torch.Tensor]:+ t0 = time.perf_counter()+ ref_detection = self._detect_pose(ref_image)+ if ref_detection is None:+ raise ValueError("Unable to detect pose in the reference image.")++ target_hw = ref_detection.frame_rgb.shape[:2]++ def _tensor_video_to_list(t: torch.Tensor) -> List[np.ndarray]:+ if t.dim() != 4 or t.shape[0] not in (1, 3, 4):+ raise ValueError("Video tensor must be 4D CTHW or BGRHWC list")+ vid = t.detach().cpu()+ if vid.min() < 0:+ vid = (vid + 1.0) * 127.5+ elif vid.max() <= 1.0:+ vid = vid * 255.0+ vid = vid.clamp(0, 255).byte()+ # Return BGR to match cv2.VideoCapture format+ frames = []+ for i in range(vid.shape[1]):+ rgb = vid[:, i].permute(1, 2, 0).numpy()+ bgr = cv2.cvtColor(rgb, cv2.COLOR_RGB2BGR)+ frames.append(bgr)+ return frames++ if isinstance(ref_video_frames, torch.Tensor):+ ref_video_frames = _tensor_video_to_list(ref_video_frames)+ def _preprocess_item(idx_frame_mask):+ idx, frame, mask = idx_frame_mask+ # normalize mask here to leverage multithreading+ if mask is not None:+ mask = _mask_to_float01(mask)+ if resize_ref_video and frame.shape[:2] != target_hw:+ frame = cv2.resize(frame, (target_hw[1], target_hw[0]), interpolation=cv2.INTER_LANCZOS4)++ if mask is not None:+ mask_f = mask+ if mask_f.shape != frame.shape[:2]:+ mask_f = cv2.resize(mask_f, (frame.shape[1], frame.shape[0]), interpolation=cv2.INTER_LINEAR)+ if expand_scale != 0:+ kernel_size = abs(expand_scale)+ kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))+ op_expand = cv2.dilate if expand_scale > 0 else cv2.erode+ mask = op_expand(mask, kernel, iterations=3)+ frame = (frame.astype(np.float32) * mask_f[..., None]).clip(0, 255).astype(np.uint8)+ return idx, frame++ # prepare frames (mask + optional resize) using CPU multithreading+ items = []+ for idx, frame in enumerate(ref_video_frames):+ mask = None+ if ref_video_mask is not None:+ mask = ref_video_mask[:, idx] if torch.is_tensor(ref_video_mask) else ref_video_mask[idx]+ items.append((idx, frame, mask))+ if cpu_resize_workers is None:+ cpu_resize_workers = max(1, int(os.cpu_count() / 2))+ preprocessed = process_images_multithread(+ _preprocess_item,+ items,+ process_type="prephase",+ wrap_in_list=False,+ max_workers=cpu_resize_workers,+ in_place=False,+ )+ idx_to_frame = {idx: frame for idx, frame in preprocessed}+ t_preprocess = time.perf_counter() - t0+ if verbose:+ print(f"[pose_align] preprocess done in {t_preprocess:.2f}s")++ ref_ratio = ref_detection.frame_rgb.shape[1] / ref_detection.frame_rgb.shape[0]+ pose_list: List[Dict] = []+ video_frames: List[np.ndarray] = [] if include_composite else []+ video_pose_frames: List[np.ndarray] = [] if include_composite else []++ align_args = None+ offset = None++ # instantiate additional detectors for parallel processing+ detectors = [self.pose_estimation]+ for _ in range(max(1, self.detection_workers) - 1):+ detectors.append(Wholebody(self.det_model, self.pose_model, device=self.device))++ sequential_done = False+ pending = []+ t_first_stage = 0.0++ for idx in sorted(idx_to_frame.keys()):+ if idx < align_frame:+ continue+ if max_frames is not None and len(pose_list) >= max_frames:+ break++ frame = idx_to_frame[idx]+ t_detect_start = time.perf_counter()+ detection = self._detect_pose(frame) if not sequential_done else None+ if detection is None:+ # queue for parallel path once we have align_args+ pending.append((idx, frame))+ continue++ if align_args is None:+ video_ratio = detection.frame_rgb.shape[1] / detection.frame_rgb.shape[0]+ align_args, offset = self._compute_alignment(ref_detection.pose, detection.pose, ref_ratio, video_ratio)++ pose_aligned = align_img(detection.frame_rgb, detection.pose, align_args)+ mask_body = pose_aligned["bodies"]["candidate"][..., 0] < 0+ mask_hands = pose_aligned["hands"][..., 0] < 0+ mask_faces = pose_aligned["faces"][..., 0] < 0++ pose_aligned["bodies"]["candidate"] = _ensure_xyz(pose_aligned["bodies"]["candidate"]) + offset+ pose_aligned["hands"] = _ensure_xyz(pose_aligned["hands"]) + offset+ pose_aligned["faces"] = _ensure_xyz(pose_aligned["faces"]) + offset++ pose_aligned["bodies"]["candidate"][mask_body] = -1+ pose_aligned["hands"][mask_hands] = -1+ pose_aligned["faces"][mask_faces] = -1++ pose_aligned["bodies"]["candidate"][:, 0] /= ref_ratio+ pose_aligned["hands"][:, :, 0] /= ref_ratio+ pose_aligned["faces"][:, :, 0] /= ref_ratio++ pose_list.append(pose_aligned)+ if include_composite:+ video_frames.append(detection.frame_rgb)+ video_pose_frames.append(detection.pose_map_rgb)++ sequential_done = True+ start_idx = idx + 1+ pending.extend([(j, idx_to_frame[j]) for j in sorted(idx_to_frame.keys()) if j >= start_idx])+ t_first_stage = time.perf_counter() - t_detect_start+ if verbose:+ print(f"[pose_align] first frame detection+align done in {t_first_stage:.2f}s")+ break++ if align_args is None:+ return {"composite": torch.empty(0), "pose_only": torch.empty(0), "pose_aug": torch.empty(0)}++ # Parallel processing for remaining frames using a single persistent executor+ def _process_pending(item):+ idx, frame, det_idx = item+ det = self._detect_pose_session(frame, detectors[det_idx])+ if det is None:+ return idx, None, None, None++ pa = align_img(det.frame_rgb, det.pose, align_args)+ mb = pa["bodies"]["candidate"][..., 0] < 0+ mh = pa["hands"][..., 0] < 0+ mf = pa["faces"][..., 0] < 0++ pa["bodies"]["candidate"] = _ensure_xyz(pa["bodies"]["candidate"]) + offset+ pa["hands"] = _ensure_xyz(pa["hands"]) + offset+ pa["faces"] = _ensure_xyz(pa["faces"]) + offset++ pa["bodies"]["candidate"][mb] = -1+ pa["hands"][mh] = -1+ pa["faces"][mf] = -1++ pa["bodies"]["candidate"][:, 0] /= ref_ratio+ pa["hands"][:, :, 0] /= ref_ratio+ pa["faces"][:, :, 0] /= ref_ratio+ return idx, pa, det.frame_rgb, det.pose_map_rgb++ if pending:+ from concurrent.futures import ThreadPoolExecutor, as_completed++ results = []+ det_cycle = itertools.cycle(range(self.detection_workers))+ with ThreadPoolExecutor(max_workers=self.detection_workers) as executor:+ for start in range(0, len(pending), detection_chunk_size * self.detection_workers):+ chunk = pending[start : start + detection_chunk_size * self.detection_workers]+ tasks = []+ for idx, frame in chunk:+ tasks.append((idx, frame, next(det_cycle)))+ futures = {executor.submit(_process_pending, t): t[0] for t in tasks}+ for future in as_completed(futures):+ res = future.result()+ if res[1] is None:+ continue+ results.append(res)+ for idx, pa, fr, pm in sorted(results, key=lambda x: x[0]):+ if max_frames is not None and len(pose_list) >= max_frames:+ break+ pose_list.append(pa)+ if include_composite:+ video_frames.append(fr)+ video_pose_frames.append(pm)+ t_parallel = time.perf_counter() - t0 - t_preprocess - t_first_stage+ if verbose:+ print(f"[pose_align] parallel detection done in {t_parallel:.2f}s")+ else:+ t_parallel = 0.0++ if not pose_list:+ return {"composite": torch.empty(0), "pose_only": torch.empty(0), "pose_aug": torch.empty(0)}++ body_seq = [_ensure_xyz(pose["bodies"]["candidate"][:18]) for pose in pose_list]+ body_seq_subset = [pose["bodies"]["subset"][:1] for pose in pose_list]+ hands_seq = [_ensure_xyz(pose["hands"][:2]) for pose in pose_list]+ faces_seq = [_ensure_xyz(pose["faces"][:1]) for pose in pose_list]++ ref_H, ref_W = ref_detection.frame_rgb.shape[:2]+ ref_target_H, ref_target_W = ref_detection.orig_hw if hasattr(ref_detection, "orig_hw") else (ref_H, ref_W)+ composite_frames: List[np.ndarray] = [] if include_composite else []+ pose_only_frames: List[np.ndarray] = []+ pose_aug_frames: List[np.ndarray] = []++ aug_cfg = augment_params or {}+ offset_x = aug_cfg.get("offset_x", (-0.2, 0.2))+ offset_y = aug_cfg.get("offset_y", (-0.2, 0.2))+ scale_rng = aug_cfg.get("scale", (0.7, 1.3))+ aspect_rng = aug_cfg.get("aspect_ratio_range", (0.6, 1.4))+ fixed_aug = None+ if augment and augment_mode == "fixed":+ sx = np.random.uniform(scale_rng[0], scale_rng[1])+ aspect = np.random.uniform(aspect_rng[0], aspect_rng[1])+ scale_x = sx * aspect+ scale_y = sx / max(aspect, 1e-6)+ dx = np.random.uniform(offset_x[0], offset_x[1])+ dy = np.random.uniform(offset_y[0], offset_y[1])+ fixed_aug = (dx, dy, scale_x, scale_y)++ for i in range(len(body_seq)):+ pose_t = {+ "bodies": {"candidate": body_seq[i], "subset": body_seq_subset[i]},+ "hands": hands_seq[i],+ "faces": faces_seq[i],+ }++ pose_for_draw = {+ "bodies": {"candidate": _xy_only(body_seq[i]), "subset": body_seq_subset[i]},+ "hands": _xy_only(hands_seq[i]),+ "faces": _xy_only(faces_seq[i]),+ }++ aligned_pose = draw_pose(pose_for_draw, ref_H, ref_W, use_body=True, use_hand=True, use_face=False)+ aligned_pose = cv2.cvtColor(aligned_pose, cv2.COLOR_BGR2RGB)+ pose_only_frames.append(aligned_pose)++ aug_frame = None+ if augment:+ params = None if augment_mode == "per_frame" else fixed_aug+ pose_t_aug = _augment_pose(pose_t, offset_x, offset_y, scale_rng, aspect_rng, fixed_params=params)+ pose_t_aug_draw = {+ "bodies": {"candidate": _xy_only(pose_t_aug["bodies"]["candidate"]), "subset": pose_t_aug["bodies"]["subset"]},+ "hands": _xy_only(pose_t_aug["hands"]),+ "faces": _xy_only(pose_t_aug["faces"]),+ }+ aug_pose = draw_pose(pose_t_aug_draw, ref_H, ref_W, use_body=True, use_hand=True, use_face=False)+ aug_frame = cv2.cvtColor(aug_pose, cv2.COLOR_BGR2RGB)+ pose_aug_frames.append(aug_frame)++ if include_composite:+ video_frame = _resize_to_height(video_frames[i], ref_H)+ video_pose = _resize_to_height(video_pose_frames[i], ref_H)+ ref_pose_resized = _resize_to_height(ref_detection.pose_map_rgb, ref_H)+ ref_img_resized = _resize_to_height(ref_detection.frame_rgb, ref_H)+ parts = [+ ref_img_resized,+ ref_pose_resized,+ aligned_pose,+ ]+ if augment and aug_frame is not None:+ parts.append(aug_frame)+ parts.extend([video_frame, video_pose])+ comp = np.concatenate(parts, axis=1)+ composite_frames.append(comp)++ # Resize outputs to reference original size if needed+ if (ref_target_H, ref_target_W) != (ref_H, ref_W):+ def _resize_list(frames: List[np.ndarray]) -> List[np.ndarray]:+ return [cv2.resize(f, (ref_target_W, ref_target_H), interpolation=cv2.INTER_CUBIC) for f in frames]+ pose_only_frames = _resize_list(pose_only_frames)+ pose_aug_frames = _resize_list(pose_aug_frames)+ t_render = time.perf_counter() - t0 - t_preprocess - t_first_stage - t_parallel+ t_total = time.perf_counter() - t0+ if verbose:+ print(f"[pose_align] render done in {t_render:.2f}s; total {t_total:.2f}s")++ return {+ "composite": _frames_to_tensor(composite_frames) if include_composite else torch.empty(0),+ "pose_only": _frames_to_tensor(pose_only_frames),+ "pose_aug": _frames_to_tensor(pose_aug_frames) if augment else torch.empty(0),+ }+++def load_video_frames(path: str) -> Tuple[List[np.ndarray], float]:+ cap = cv2.VideoCapture(path)+ fps = cap.get(cv2.CAP_PROP_FPS) or 24.0+ frames = []+ while cap.isOpened():+ ret, frame = cap.read()+ if not ret:+ break+ frames.append(frame)+ cap.release()+ cv2.destroyAllWindows()+ return frames, fps+++def load_mask_frames(path: str) -> List[np.ndarray]:+ frames, _ = load_video_frames(path)+ return frames+++def demo(+ ref_image_path: str,+ ref_video_path: str,+ ref_video_mask_path: str | None,+ output_dir: str,+ include_composite: bool = True,+ augment: bool = True,+ augment_mode: str = "per_frame",+ align_frame: int = 0,+ max_frames: int | None = None,+ detection_workers: int = 2,+ cpu_resize_workers: int = None,+ resize_ref_video: bool = False,+ detection_chunk_size: int = 8,+ verbose: int = 0,+) -> None:+ frames, fps = load_video_frames(ref_video_path)+ mask_frames = load_mask_frames(ref_video_mask_path) if ref_video_mask_path else None+ aligner = PoseAligner(detection_workers=detection_workers)+ outputs = aligner.align(+ frames,+ Image.open(ref_image_path),+ ref_video_mask=mask_frames,+ align_frame=align_frame,+ max_frames=max_frames,+ include_composite=include_composite,+ augment=augment,+ augment_mode=augment_mode,+ cpu_resize_workers=cpu_resize_workers,+ resize_ref_video=resize_ref_video,+ detection_chunk_size=detection_chunk_size,+ verbose=verbose,+ )++ composite_frames = _tensor_to_frames(outputs["composite"])+ pose_frames = _tensor_to_frames(outputs["pose_only"])+ pose_aug_frames = _tensor_to_frames(outputs["pose_aug"])++ if include_composite and composite_frames:+ _save_video(os.path.join(output_dir, "composite.mp4"), composite_frames, fps)+ if pose_frames:+ _save_video(os.path.join(output_dir, "pose_only.mp4"), pose_frames, fps)+ if pose_aug_frames:+ _save_video(os.path.join(output_dir, "pose_only_aug.mp4"), pose_aug_frames, fps)+++if __name__ == "__main__":+ parser = argparse.ArgumentParser(description="Pose alignment demo")+ parser.add_argument("--ref_image", type=str, default=r"e:/steffete.png", help="Path to the reference image")+ parser.add_argument("--ref_video", type=str, default=r"e:\stage.mp4", help="Path to the reference pose video (mp4)")+ parser.add_argument("--ref_video_mask", type=str, default='e:/stagemasked_alpha.mp4', help="Optional path to a mask video (same length as ref video)")+ # parser.add_argument("--ref_video_mask", type=str, default=None, help="Optional path to a mask video (same length as ref video)")+ parser.add_argument("--output_dir", type=str, default="pose_align_demo", help="Directory to store demo mp4s")+ parser.add_argument("--no_composite", action="store_true", help="Skip saving comparison/composite video")+ parser.add_argument("--no_augment", action="store_true", help="Disable augmented pose output")+ parser.add_argument("--augment_mode", type=str, default="per_frame", choices=["fixed", "per_frame"], help="Augmentation mode (fixed per video or per frame jitter)")+ parser.add_argument("--align_frame", type=int, default=0, help="Frame index to start alignment from")+ parser.add_argument("--max_frames", type=int, default=48, help="Maximum frames to process")+ parser.add_argument("--detection_workers", type=int, default=2, help="Number of parallel detection workers (GPU-backed)")+ parser.add_argument("--cpu_resize_workers", type=int, default=None, help="CPU workers for pre-mask/resize")+ parser.add_argument("--resize_ref_video", action="store_true", help="Resize ref video to ref image size (Lanczos)")+ parser.add_argument("--detection_chunk_size", type=int, default=8, help="Chunk size for parallel detection batches")+ parser.add_argument("--verbose", type=int, default=1, help="Verbosity level for timing/debug")+ args = parser.parse_args()++ demo(+ args.ref_image,+ args.ref_video,+ args.ref_video_mask,+ args.output_dir,+ include_composite=not args.no_composite,+ augment=not args.no_augment,+ augment_mode=args.augment_mode,+ align_frame=args.align_frame,+ max_frames=args.max_frames,+ detection_workers=args.detection_workers,+ cpu_resize_workers=args.cpu_resize_workers,+ resize_ref_video=args.resize_ref_video,+ detection_chunk_size=args.detection_chunk_size,+ verbose=args.verbose,+ )diff --git a/models/wan/steadydancer/small_archs.py b/models/wan/steadydancer/small_archs.pynew file mode 100644index 0000000..cdd74a6--- /dev/null+++ b/models/wan/steadydancer/small_archs.py@@ -0,0 +1,138 @@+import torch+import torch.nn as nn+++class FactorConv3d(nn.Module):+ """+ (2+1)D 分解 3D 卷积:1×H×W 空间卷积 → Swish → T×1×1 时间卷积+ """+ def __init__(self,+ in_channels: int,+ out_channels: int,+ kernel_size,+ stride: int = 1,+ dilation: int = 1):+ super().__init__()++ if isinstance(kernel_size, int):+ k_t, k_h, k_w = kernel_size, kernel_size, kernel_size+ else:+ k_t, k_h, k_w = kernel_size++ pad_t = (k_t - 1) * dilation // 2+ pad_hw = (k_h - 1) * dilation // 2++ self.spatial = nn.Conv3d(+ in_channels, in_channels,+ kernel_size=(1, k_h, k_w),+ stride=(1, stride, stride),+ padding=(0, pad_hw, pad_hw),+ dilation=(1, dilation, dilation),+ groups=in_channels,+ bias=False+ )++ self.temporal = nn.Conv3d(+ in_channels, out_channels,+ kernel_size=(k_t, 1, 1),+ stride=(stride, 1, 1),+ padding=(pad_t, 0, 0),+ dilation=(dilation, 1, 1),+ bias=True+ )++ self.act = nn.SiLU()++ def forward(self, x):+ x = self.spatial(x)+ x = self.act(x)+ x = self.temporal(x)+ return x+++class LayerNorm2D(nn.Module):+ """+ LayerNorm over C for a 4-D tensor (B, C, H, W)+ """+ def __init__(self, num_channels, eps=1e-5, affine=True):+ super().__init__()+ self.num_channels = num_channels+ self.eps = eps+ self.affine = affine+ if affine:+ self.weight = nn.Parameter(torch.ones(1, num_channels, 1, 1))+ self.bias = nn.Parameter(torch.zeros(1, num_channels, 1, 1))++ def forward(self, x):+ # x: (B, C, H, W)+ mean = x.mean(dim=1, keepdim=True) # (B, 1, H, W)+ var = x.var (dim=1, keepdim=True, unbiased=False)+ x = (x - mean) / torch.sqrt(var + self.eps)+ if self.affine:+ x = x * self.weight + self.bias+ return x+++class PoseRefNetNoBNV3(nn.Module):+ def __init__(self,+ in_channels_c: int,+ in_channels_x: int,+ hidden_dim: int = 256,+ num_heads: int = 8,+ dropout: float = 0.1):+ super().__init__()+ self.d_model = hidden_dim+ self.nhead = num_heads++ self.proj_p = nn.Conv2d(in_channels_c, hidden_dim, kernel_size=1)+ self.proj_r = nn.Conv2d(in_channels_x, hidden_dim, kernel_size=1)++ self.proj_p_back = nn.Conv2d(hidden_dim, in_channels_c, kernel_size=1)++ self.cross_attn = nn.MultiheadAttention(hidden_dim,+ num_heads=num_heads,+ dropout=dropout)++ self.ffn_pose = nn.Sequential(+ nn.Conv2d(hidden_dim, hidden_dim, kernel_size=1),+ nn.SiLU(),+ nn.Conv2d(hidden_dim, hidden_dim, kernel_size=1)+ )++ self.norm1 = LayerNorm2D(hidden_dim)+ self.norm2 = LayerNorm2D(hidden_dim)++ def forward(self, pose, ref, mask=None):+ """+ pose : (B, C1, T, H, W)+ ref : (B, C2, T, H, W)+ mask : (B, T*H*W) 可选 key_padding_mask+ return: (B, d_model, T, H, W)+ """+ B, _, T, H, W = pose.shape+ L = H * W++ p_trans = pose.permute(0, 2, 1, 3, 4).contiguous().flatten(0, 1)+ r_trans = ref.permute(0, 2, 1, 3, 4).contiguous().flatten(0, 1)++ p_trans = self.proj_p(p_trans)+ r_trans = self.proj_r(r_trans)++ p_trans = p_trans.flatten(2).transpose(1, 2)+ r_trans = r_trans.flatten(2).transpose(1, 2)++ out = self.cross_attn(query=r_trans,+ key=p_trans,+ value=p_trans,+ key_padding_mask=mask)[0]++ out = out.transpose(1, 2).contiguous().view(B*T, -1, H, W)+ out = self.norm1(out)++ ffn_out = self.ffn_pose(out)+ out = out + ffn_out+ out = self.norm2(out)+ out = self.proj_p_back(out)+ out = out.view(B, T, -1, H, W).contiguous().transpose(1, 2)++ return outdiff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex d0d6772..21b371b 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -3,13 +3,15 @@ import osimport torchimport numpy as npimport gradio as gr+import cv2+from PIL import Imagefrom shared.utils import files_locator as fldef test_vace(base_model_type):return base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B", "vace_ditto_14B"]def test_class_i2v(base_model_type):- return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate", "chrono_edit" ]+ return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate", "chrono_edit", "steadydancer" ]def test_class_t2v(base_model_type):return base_model_type in ["t2v", "t2v_2_2", "alpha", "lynx"]@@ -41,7 +43,7 @@ class family_handler():return ["multitalk", "infinitetalk", "fantasy", "vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B","t2v_1.3B", "standin", "lynx_lite", "lynx", "t2v", "t2v_2_2", "vace_1.3B", "vace_ditto_14B", "phantom_1.3B", "phantom_14B","recam_1.3B", "animate", "alpha", "alpha_lynx", "chrono_edit",- "i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "mocha"]+ "i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp", "mocha", "steadydancer"]@staticmethod@@ -299,6 +301,24 @@ class family_handler():}extra_model_def["v2i_switch_supported"] = True++ if base_model_type in ["steadydancer"]:+ extra_model_def["guide_custom_choices"] = {+ "choices":[+ ("Use Control Video Poses to Animate Person in Start Image", "V"),+ ("Use Control Video Poses filterd with Mask Video to Animate Person in Start Image", "VA"),+ ],+ "default": "PVB",+ "letters_filter": "PVBA",+ "label": "Type of Process",+ "scale": 3,+ "show_label" : False,+ }+ extra_model_def["custom_preprocessor"] = "Extracting Pose Information"+ extra_model_def["alt_guidance"] = "Condition Guidance"+ extra_model_def["no_guide2_refresh"] = True+ extra_model_def["no_mask_refresh"] = True+if base_model_type in ["infinitetalk"]:extra_model_def["no_background_removal"] = True@@ -531,7 +551,7 @@ class family_handler():image_prompt_types_allowed = "TSVL"elif base_model_type in ["lucy_edit"]:image_prompt_types_allowed = "TVL"- elif multitalk or base_model_type in ["fantasy"]:+ elif multitalk or base_model_type in ["fantasy", "steadydancer"]:image_prompt_types_allowed = "SVL"elif i2v:image_prompt_types_allowed = "SEVL"@@ -598,6 +618,25 @@ class family_handler():return download_def+ @staticmethod+ def custom_preprocess(base_model_type, video_guide, video_mask, pre_video_guide=None, max_workers = 1, expand_scale = 0, **kwargs):+ from .steadydancer.pose_align import PoseAligner+ from shared.utils.utils import convert_tensor_to_image++ ref_image = convert_tensor_to_image(pre_video_guide[:, 0])+ frames = video_guide+ mask_frames = None+ mask_frames = None if video_mask is None else video_mask+ aligner = PoseAligner()+ outputs = aligner.align( frames, ref_image, ref_video_mask=mask_frames, align_frame=0, max_frames=None, augment=True, include_composite= True, cpu_resize_workers= max_workers, expand_scale = expand_scale )++ video_guide_processed, video_guide_processed2 = outputs["pose_only"], outputs["pose_aug"]+ if video_guide_processed.numel() == 0: return None, None, None++ video_mask_processed = video_mask_processed2 = None++ return video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2+@staticmethoddef load_model(model_filename, model_type, base_model_type, model_def, quantizeTransformer = False, text_encoder_quantization = None, dtype = torch.bfloat16, VAE_dtype = torch.float32, mixed_precision_transformer = False, save_quantized= False, submodel_no_list = None, override_text_encoder = None, VAE_upsampling = None, **kwargs):@@ -806,6 +845,14 @@ class family_handler():"audio_prompt_type": "R","force_fps": "control",})+ elif base_model_type in ["steadydancer"]:+ ui_defaults.update({+ "video_prompt_type": "VA",+ "image_prompt_type": "S",+ "audio_prompt_type": "R",+ "force_fps": "control",+ "alt_guidance_scale" : 2.0,+ })if base_model_type in ["i2v_2_2"]:diff --git a/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json b/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.jsonindex 1fe4c14..32be54a 100644--- a/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json+++ b/profiles/wan_i2v/Image2Video FusioniX - 10 Steps.json@@ -6,5 +6,7 @@"loras_multipliers": "1","num_inference_steps": 10,"guidance_phases": 1,- "guidance_scale": 1+ "guidance_scale": 1,+ "alt_guidance_scale": 1,+ "audio_guidance_scale": 1}diff --git a/wgp.py b/wgp.pyindex 9a40485..fd76bff 100644--- a/wgp.py+++ b/wgp.py@@ -83,7 +83,7 @@ global_queue_ref = []AUTOSAVE_FILENAME = "queue.zip"PROMPT_VARS_MAX = 10target_mmgp_version = "3.6.9"-WanGP_version = "9.72"+WanGP_version = "9.73"settings_version = 2.41max_source_video_frames = 3000prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None@@ -288,6 +288,7 @@ def edit_task_in_queue(switch_threshold2,guidance_phases,model_switch_phase,+ alt_guidance_scale,audio_guidance_scale,flow_shift,sample_solver,@@ -3811,6 +3812,7 @@ def select_video(state, current_gallery_tab, input_file_list, file_selected, audvideo_guidance2_scale = configs.get("guidance2_scale", None)video_guidance3_scale = configs.get("guidance3_scale", None)video_audio_guidance_scale = configs.get("audio_guidance_scale", None)+ video_alt_guidance_scale = configs.get("alt_guidance_scale", None)video_switch_threshold = configs.get("switch_threshold", 0)video_switch_threshold2 = configs.get("switch_threshold2", 0)video_model_switch_phase = configs.get("model_switch_phase", 1)@@ -3898,8 +3900,14 @@ def select_video(state, current_gallery_tab, input_file_list, file_selected, audif model_def.get("sample_solvers", None) is not None and len(video_sample_solver) > 0 :values += [video_sample_solver]labels += ["Sampler Solver"]- values += [video_resolution, video_length_summary, video_seed, video_guidance_scale, video_audio_guidance_scale, video_flow_shift, video_num_inference_steps]- labels += [ "Resolution", video_length_label, "Seed", video_guidance_label, "Audio Guidance Scale", "Shift Scale", "Num Inference steps"]+ values += [video_resolution, video_length_summary, video_seed, video_guidance_scale, video_audio_guidance_scale]+ labels += ["Resolution", video_length_label, "Seed", video_guidance_label, "Audio Guidance Scale"]+ alt_guidance_type = model_def.get("alt_guidance", None)+ if alt_guidance_type is not None and video_alt_guidance_scale is not None:+ values += [video_alt_guidance_scale]+ labels += [alt_guidance_type]+ values += [video_flow_shift, video_num_inference_steps]+ labels += ["Shift Scale", "Num Inference steps"]video_negative_prompt = configs.get("negative_prompt", "")if len(video_negative_prompt) > 0:values += [video_negative_prompt]@@ -4870,7 +4878,36 @@ def truncate_audio(generated_audio, trim_video_frames_beginning, trim_video_framstart = int(trim_video_frames_beginning * samples_per_frame)end = len(generated_audio) - int(trim_video_frames_end * samples_per_frame)return generated_audio[start:end if end > 0 else None]++def custom_preprocess_video_with_mask(model_handler, base_model_type, pre_video_guide, video_guide, video_mask, height, width, max_frames, start_frame, fit_canvas, fit_crop, target_fps, block_size, expand_scale):+ pad_frames = 0+ if start_frame < 0:+ pad_frames= -start_frame+ max_frames += start_frame+ start_frame = 0++ max_workers = get_default_workers()++ if not video_guide or max_frames <= 0:+ return None, None+ video_guide = get_resampled_video(video_guide, start_frame, max_frames, target_fps).permute(-1, 0, 1, 2)+ video_guide = video_guide / 127.5 - 1.+ any_mask = video_mask is not None+ if video_mask is not None:+ video_mask = get_resampled_video(video_mask, start_frame, max_frames, target_fps).permute(-1, 0, 1, 2)+ video_mask = video_mask[:1] / 255.++ if len(video_guide) == 0 or any_mask and len(video_mask) == 0:+ return None, None+ video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2 = model_handler.custom_preprocess(base_model_type = base_model_type, pre_video_guide = pre_video_guide, video_guide = video_guide, video_mask = video_mask, height = height, width = width, fit_canvas = fit_canvas , fit_crop = fit_crop, target_fps = target_fps, block_size = block_size, max_workers = max_workers, expand_scale = expand_scale)++ # if pad_frames > 0:+ # masked_frames = masked_frames[0] * pad_frames + masked_frames+ # if any_mask: masked_frames = masks[0] * pad_frames + masks++ return video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2+def generate_video(task,send_cmd,@@ -4890,6 +4927,7 @@ def generate_video(switch_threshold2,guidance_phases,model_switch_phase,+ alt_guidance_scale,audio_guidance_scale,flow_shift,sample_solver,@@ -5471,18 +5509,25 @@ def generate_video(preprocess_type = process_map_video_guide.get(process_letter, "raw")else:preprocess_type2 = process_map_video_guide.get(process_letter, None)- status_info = "Extracting " + processes_names[preprocess_type]- extra_process_list = ([] if preprocess_type2==None else [preprocess_type2]) + ([] if process_outside_mask==None or process_outside_mask == preprocess_type else [process_outside_mask])- if len(extra_process_list) == 1:- status_info += " and " + processes_names[extra_process_list[0]]- elif len(extra_process_list) == 2:- status_info += ", " + processes_names[extra_process_list[0]] + " and " + processes_names[extra_process_list[1]]- context_scale = [control_net_weight /2, control_net_weight2 /2] if preprocess_type2 is not None else [control_net_weight]- if not (preprocess_type == "identity" and preprocess_type2 is None and video_mask is None):send_cmd("progress", [0, get_latest_status(state, status_info)])- inpaint_color = 0 if preprocess_type=="pose" and process_outside_mask == "inpaint" else guide_inpaint_color- video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size, to_bbox = "H" in video_prompt_type )- if preprocess_type2 != None:- video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size, to_bbox = "H" in video_prompt_type )+ custom_preprocessor = model_def.get("custom_preprocessor", None)+ if custom_preprocessor is not None:+ status_info = custom_preprocessor+ send_cmd("progress", [0, get_latest_status(state, status_info)])+ video_guide_processed, video_guide_processed2, video_mask_processed, video_mask_processed2 = custom_preprocess_video_with_mask(model_handler, base_model_type, pre_video_guide, video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, block_size = block_size, expand_scale = mask_expand)+ else:+ status_info = "Extracting " + processes_names[preprocess_type]+ extra_process_list = ([] if preprocess_type2==None else [preprocess_type2]) + ([] if process_outside_mask==None or process_outside_mask == preprocess_type else [process_outside_mask])+ if len(extra_process_list) == 1:+ status_info += " and " + processes_names[extra_process_list[0]]+ elif len(extra_process_list) == 2:+ status_info += ", " + processes_names[extra_process_list[0]] + " and " + processes_names[extra_process_list[1]]+ context_scale = [control_net_weight /2, control_net_weight2 /2] if preprocess_type2 is not None else [control_net_weight]++ if not (preprocess_type == "identity" and preprocess_type2 is None and video_mask is None):send_cmd("progress", [0, get_latest_status(state, status_info)])+ inpaint_color = 0 if preprocess_type=="pose" and process_outside_mask == "inpaint" else guide_inpaint_color+ video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size, to_bbox = "H" in video_prompt_type )+ if preprocess_type2 != None:+ video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size, to_bbox = "H" in video_prompt_type )if video_guide_processed is not None and sample_fit_canvas is not None:image_size = video_guide_processed.shape[-2:]@@ -5573,9 +5618,9 @@ def generate_video(preview_frame_no = 0 if extract_guide_from_window_start or model_def.get("dont_cat_preguide", False) or sparse_video_image is not None else (guide_start_frame - window_start_frame)preview_frame_no = min(src_video.shape[1] -1, preview_frame_no)refresh_preview["video_guide"] = convert_tensor_to_image(src_video, preview_frame_no)- if src_video2 is not None:+ if src_video2 is not None and not model_def.get("no_guide2_refresh", False):refresh_preview["video_guide"] = [refresh_preview["video_guide"], convert_tensor_to_image(src_video2, preview_frame_no)]- if src_mask is not None and video_mask is not None:+ if src_mask is not None and video_mask is not None and not model_def.get("no_mask_refresh", False):refresh_preview["video_mask"] = convert_tensor_to_image(src_mask, preview_frame_no, mask_levels = True)if src_ref_images is not None or nb_frames_positions:@@ -5664,6 +5709,7 @@ def generate_video(apg_switch = apg_switch,cfg_star_switch = cfg_star_switch,cfg_zero_step = cfg_zero_step,+ alt_guide_scale= alt_guidance_scale,audio_cfg_scale= audio_guidance_scale,audio_guide=audio_guide,audio_guide2=audio_guide2,@@ -6836,6 +6882,10 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Noneif not model_def.get("embedded_guidance", False):pop += ["embedded_guidance_scale"]+ if model_def.get("alt_guidance", None) is None:+ pop += ["alt_guidance_scale"]++if not (model_def.get("tea_cache", False) or model_def.get("mag_cache", False)) :pop += ["skip_steps_cache_type", "skip_steps_multiplier", "skip_steps_start_step_perc"]@@ -7396,6 +7446,7 @@ def save_inputs(switch_threshold2,guidance_phases,model_switch_phase,+ alt_guidance_scale,audio_guidance_scale,flow_shift,sample_solver,@@ -8719,9 +8770,12 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonany_audio_guidance = model_def.get("audio_guidance", False)any_embedded_guidance = model_def.get("embedded_guidance", False)- with gr.Row(visible =any_embedded_guidance or any_audio_guidance) as embedded_guidance_row:+ alt_guidance_type = model_def.get("alt_guidance", None)+ any_alt_guidance = alt_guidance_type is not None+ with gr.Row(visible =any_embedded_guidance or any_audio_guidance or any_alt_guidance) as embedded_guidance_row:audio_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("audio_guidance_scale", 4), step=0.5, label="Audio Guidance", visible= any_audio_guidance, show_reset_button= False )embedded_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("embedded_guidance_scale", 6.0), step=0.5, label="Embedded Guidance Scale", visible=any_embedded_guidance, show_reset_button= False )+ alt_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("alt_guidance_scale", 6.0), step=0.5, label= alt_guidance_type if any_alt_guidance else "" , visible=any_alt_guidance, show_reset_button= False )with gr.Row(visible=audio_only) as temperature_row:temperature = gr.Slider( 0.1, 1.5, value=ui_defaults.get("temperature", 0.8), step=0.01, label="Temperature", show_reset_button= False)