Python
Vace Contenders are in Town
DeepBeepMeepe28c95ae912a
diff --git a/README.md b/README.mdindex 8c5e436..6c338ac 100644--- a/README.md+++ b/README.md@@ -20,6 +20,14 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models**Follow DeepBeepMeep on Twitter/X to get the Latest News**: https://x.com/deepbeepmeep## 🔥 Latest Updates :+### September 23 2025: WanGP v8.7 - Here Are Two New Contenders in the Vace Arena !++So in today's release you will find two Wannabe Vace that covers each only a subset of Vace features but offers some interesting advantages:+- **Wan 2.2 Animate**: this model is specialized in *Body Motion* and *Facial Motion tranfers*. It does that very well. You can use this model to either *Replace* a person in an in Video or *Animate* the person of your choice using an existing *Pose Video* (remember *Animate Anyone* ?). By default it will keep the original soundtrack. *Wan 2.2 Animate* seems to be under the hood a derived i2v model and should support the corresponding Loras Accelerators (for instance *FusioniX t2v*). Also as a WanGP exclusivity, you will find support for *Outpainting*.++- **Lucy Edit**: this one claims to be a *Nano Banana* for Videos. Give it a video and asks it to change it (it is specialized in clothes changing) and voila ! The nice thing about it is that is it based on the *Wan 2.2 5B* model and therefore is very fast especially if you the *FastWan* finetune that is also part of the package.++### September 15 2025: WanGP v8.6 - Attack of the Clones- The long awaited **Vace for Wan 2.2** is at last here or maybe not: it has been released by the *Fun Team* of *Alibaba* and it is not official. You can play with the vanilla version (**Vace Fun**) or with the one accelerated with Loras (**Vace Fan Cocktail**)diff --git a/defaults/lucy_edit.json b/defaults/lucy_edit.jsonindex 6344dff..a8f67ad 100644--- a/defaults/lucy_edit.json+++ b/defaults/lucy_edit.json@@ -2,7 +2,7 @@"model": {"name": "Wan2.2 Lucy Edit 5B","architecture": "lucy_edit",- "description": "Lucy Edit Dev is a video editing model that performs instruction-guided edits on videos using free-text prompts \u2014 it supports a variety of edits, such as clothing & accessory changes, character changes, object insertions, and scene replacements while preserving the motion and composition perfectly.",+ "description": "Lucy Edit is a video editing model that performs instruction-guided edits on videos using free-text prompts. It supports a variety of edits, such as clothing & accessory changes, character changes, object insertions, and scene replacements while preserving the motion and composition perfectly.","URLs": ["https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan2.2_lucy_edit_mbf16.safetensors","https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan2.2_lucy_edit_quanto_mbf16_int8.safetensors",@@ -10,6 +10,7 @@],"group": "wan2_2"},+ "prompt": "change the clothes to red","video_length": 81,"guidance_scale": 5,"flow_shift": 5,diff --git a/defaults/lucy_edit_fastwan.json b/defaults/lucy_edit_fastwan.jsonnew file mode 100644index 0000000..c67c795--- /dev/null+++ b/defaults/lucy_edit_fastwan.json@@ -0,0 +1,16 @@+{+ "model": {+ "name": "Wan2.2 FastWan Lucy Edit 5B",+ "architecture": "lucy_edit",+ "description": "Lucy Edit is a video editing model that performs instruction-guided edits on videos using free-text prompts. It supports a variety of edits, such as clothing & accessory changes, character changes, object insertions, and scene replacements while preserving the motion and composition perfectly. This is the FastWan version for faster generation.",+ "URLs": "lucy_edit",+ "group": "wan2_2",+ "loras": "ti2v_2_2"+ },+ "prompt": "change the clothes to red",+ "video_length": 81,+ "guidance_scale": 1,+ "flow_shift": 3,+ "num_inference_steps": 5,+ "resolution": "1280x720"+}\ No newline at end of filediff --git a/models/flux/flux_handler.py b/models/flux/flux_handler.pyindex d168881..83de7c3 100644--- a/models/flux/flux_handler.py+++ b/models/flux/flux_handler.py@@ -56,7 +56,7 @@ class family_handler():}- extra_model_def["lock_image_refs_ratios"] = True+ extra_model_def["fit_into_canvas_image_refs"] = 0return extra_model_defdiff --git a/models/flux/flux_main.py b/models/flux/flux_main.pyindex 6863711..6746a23 100644--- a/models/flux/flux_main.py+++ b/models/flux/flux_main.py@@ -142,8 +142,8 @@ class model_factory:n_prompt: str = None,sampling_steps: int = 20,input_ref_images = None,- image_guide= None,- image_mask= None,+ input_frames= None,+ input_masks= None,width= 832,height=480,embedded_guidance_scale: float = 2.5,@@ -197,10 +197,12 @@ class model_factory:for new_img in input_ref_images[1:]:stiched = stitch_images(stiched, new_img)input_ref_images = [stiched]- elif image_guide is not None:- input_ref_images = [image_guide]+ elif input_frames is not None:+ input_ref_images = [convert_tensor_to_image(input_frames) ]else:input_ref_images = None+ image_mask = None if input_masks is None else convert_tensor_to_image(input_masks, mask_levels= True)+if self.name in ['flux-dev-uso', 'flux-dev-umo'] :inp, height, width = prepare_multi_ip(@@ -253,8 +255,8 @@ class model_factory:if image_mask is not None:from shared.utils.utils import convert_image_to_tensorimg_msk_rebuilt = inp["img_msk_rebuilt"]- img= convert_image_to_tensor(image_guide)- x = img.squeeze(2) * (1 - img_msk_rebuilt) + x.to(img) * img_msk_rebuilt+ img= input_frames.squeeze(1).unsqueeze(0) # convert_image_to_tensor(image_guide)+ x = img * (1 - img_msk_rebuilt) + x.to(img) * img_msk_rebuiltx = x.clamp(-1, 1)x = x.transpose(0, 1)diff --git a/models/hyvideo/hunyuan.py b/models/hyvideo/hunyuan.pyindex 181a9a7..aa6c3b3 100644--- a/models/hyvideo/hunyuan.py+++ b/models/hyvideo/hunyuan.py@@ -865,7 +865,7 @@ class HunyuanVideoSampler(Inference):freqs_cos, freqs_sin = self.get_rotary_pos_embed_new(129, target_height, target_width, concat_dict)else:if input_frames != None:- target_height, target_width = input_frames.shape[-3:-1]+ target_height, target_width = input_frames.shape[-2:]elif input_video != None:target_height, target_width = input_video.shape[-2:]@@ -894,9 +894,10 @@ class HunyuanVideoSampler(Inference):pixel_value_bg = input_video.unsqueeze(0)pixel_value_mask = torch.zeros_like(input_video).unsqueeze(0)if input_frames != None:- pixel_value_video_bg = input_frames.permute(-1,0,1,2).unsqueeze(0).float()- pixel_value_video_mask = input_masks.unsqueeze(-1).repeat(1,1,1,3).permute(-1,0,1,2).unsqueeze(0).float()- pixel_value_video_bg = pixel_value_video_bg.div_(127.5).add_(-1.)+ pixel_value_video_bg = input_frames.unsqueeze(0) #.permute(-1,0,1,2).unsqueeze(0).float()+ # pixel_value_video_bg = pixel_value_video_bg.div_(127.5).add_(-1.)+ # pixel_value_video_mask = input_masks.unsqueeze(-1).repeat(1,1,1,3).permute(-1,0,1,2).unsqueeze(0).float()+ pixel_value_video_mask = input_masks.repeat(3,1,1,1).unsqueeze(0)if input_video != None:pixel_value_bg = torch.cat([pixel_value_bg, pixel_value_video_bg], dim=2)pixel_value_mask = torch.cat([ pixel_value_mask, pixel_value_video_mask], dim=2)@@ -908,10 +909,11 @@ class HunyuanVideoSampler(Inference):if pixel_value_bg.shape[2] < frame_num:padding_shape = list(pixel_value_bg.shape[0:2]) + [frame_num-pixel_value_bg.shape[2]] + list(pixel_value_bg.shape[3:])pixel_value_bg = torch.cat([pixel_value_bg, torch.full(padding_shape, -1, dtype=pixel_value_bg.dtype, device= pixel_value_bg.device ) ], dim=2)- pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 255, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)+ # pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 255, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)+ pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 1, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)bg_latents = self.vae.encode(pixel_value_bg).latent_dist.sample()- pixel_value_mask = pixel_value_mask.div_(127.5).add_(-1.)+ pixel_value_mask = pixel_value_mask.mul_(2).add_(-1.) # unmasked pixels is -1 (no 0 as usual) and masked is 1mask_latents = self.vae.encode(pixel_value_mask).latent_dist.sample()bg_latents = torch.cat([bg_latents, mask_latents], dim=1)bg_latents.mul_(self.vae.config.scaling_factor)diff --git a/models/ltx_video/ltxv_handler.py b/models/ltx_video/ltxv_handler.pyindex 2845fdb..8c322e1 100644--- a/models/ltx_video/ltxv_handler.py+++ b/models/ltx_video/ltxv_handler.py@@ -35,6 +35,8 @@ class family_handler():"selection": ["", "A", "NA", "XA", "XNA"],}+ extra_model_def["extra_control_frames"] = 1+ extra_model_def["dont_cat_preguide"]= Truereturn extra_model_def@staticmethoddiff --git a/models/qwen/qwen_handler.py b/models/qwen/qwen_handler.pyindex 010298e..cc6a764 100644--- a/models/qwen/qwen_handler.py+++ b/models/qwen/qwen_handler.py@@ -17,7 +17,7 @@ class family_handler():("Default", "default"),("Lightning", "lightning")],"guidance_max_phases" : 1,- "lock_image_refs_ratios": True,+ "fit_into_canvas_image_refs": 0,}if base_model_type in ["qwen_image_edit_20B"]:diff --git a/models/qwen/qwen_main.py b/models/qwen/qwen_main.pyindex cec0b8e..abd5c5c 100644--- a/models/qwen/qwen_main.py+++ b/models/qwen/qwen_main.py@@ -17,7 +17,7 @@ from .autoencoder_kl_qwenimage import AutoencoderKLQwenImagefrom diffusers import FlowMatchEulerDiscreteSchedulerfrom .pipeline_qwenimage import QwenImagePipelinefrom PIL import Image-from shared.utils.utils import calculate_new_dimensions+from shared.utils.utils import calculate_new_dimensions, convert_tensor_to_imagedef stitch_images(img1, img2):# Resize img2 to match img1's height@@ -103,8 +103,8 @@ class model_factory():n_prompt = None,sampling_steps: int = 20,input_ref_images = None,- image_guide= None,- image_mask= None,+ input_frames= None,+ input_masks= None,width= 832,height=480,guide_scale: float = 4,@@ -179,8 +179,10 @@ class model_factory():if n_prompt is None or len(n_prompt) == 0:n_prompt= "text, watermark, copyright, blurry, low resolution"- if image_guide is not None:- input_ref_images = [image_guide]++ image_mask = None if input_masks is None else convert_tensor_to_image(input_masks, mask_levels= True)+ if input_frames is not None:+ input_ref_images = [convert_tensor_to_image(input_frames) ]elif input_ref_images is not None:# image stiching methodstiched = input_ref_images[0]diff --git a/models/wan/animate/animate_utils.py b/models/wan/animate/animate_utils.pynew file mode 100644index 0000000..9474dce--- /dev/null+++ b/models/wan/animate/animate_utils.py@@ -0,0 +1,143 @@+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.+import torch+import numbers+from peft import LoraConfig+++def get_loraconfig(transformer, rank=128, alpha=128, init_lora_weights="gaussian"):+ target_modules = []+ for name, module in transformer.named_modules():+ if "blocks" in name and "face" not in name and "modulation" not in name and isinstance(module, torch.nn.Linear):+ target_modules.append(name)++ transformer_lora_config = LoraConfig(+ r=rank,+ lora_alpha=alpha,+ init_lora_weights=init_lora_weights,+ target_modules=target_modules,+ )+ return transformer_lora_config++++class TensorList(object):++ def __init__(self, tensors):+ """+ tensors: a list of torch.Tensor objects. No need to have uniform shape.+ """+ assert isinstance(tensors, (list, tuple))+ assert all(isinstance(u, torch.Tensor) for u in tensors)+ assert len(set([u.ndim for u in tensors])) == 1+ assert len(set([u.dtype for u in tensors])) == 1+ assert len(set([u.device for u in tensors])) == 1+ self.tensors = tensors++ def to(self, *args, **kwargs):+ return TensorList([u.to(*args, **kwargs) for u in self.tensors])++ def size(self, dim):+ assert dim == 0, 'only support get the 0th size'+ return len(self.tensors)++ def pow(self, *args, **kwargs):+ return TensorList([u.pow(*args, **kwargs) for u in self.tensors])++ def squeeze(self, dim):+ assert dim != 0+ if dim > 0:+ dim -= 1+ return TensorList([u.squeeze(dim) for u in self.tensors])++ def type(self, *args, **kwargs):+ return TensorList([u.type(*args, **kwargs) for u in self.tensors])++ def type_as(self, other):+ assert isinstance(other, (torch.Tensor, TensorList))+ if isinstance(other, torch.Tensor):+ return TensorList([u.type_as(other) for u in self.tensors])+ else:+ return TensorList([u.type(other.dtype) for u in self.tensors])++ @property+ def dtype(self):+ return self.tensors[0].dtype++ @property+ def device(self):+ return self.tensors[0].device++ @property+ def ndim(self):+ return 1 + self.tensors[0].ndim++ def __getitem__(self, index):+ return self.tensors[index]++ def __len__(self):+ return len(self.tensors)++ def __add__(self, other):+ return self._apply(other, lambda u, v: u + v)++ def __radd__(self, other):+ return self._apply(other, lambda u, v: v + u)++ def __sub__(self, other):+ return self._apply(other, lambda u, v: u - v)++ def __rsub__(self, other):+ return self._apply(other, lambda u, v: v - u)++ def __mul__(self, other):+ return self._apply(other, lambda u, v: u * v)++ def __rmul__(self, other):+ return self._apply(other, lambda u, v: v * u)++ def __floordiv__(self, other):+ return self._apply(other, lambda u, v: u // v)++ def __truediv__(self, other):+ return self._apply(other, lambda u, v: u / v)++ def __rfloordiv__(self, other):+ return self._apply(other, lambda u, v: v // u)++ def __rtruediv__(self, other):+ return self._apply(other, lambda u, v: v / u)++ def __pow__(self, other):+ return self._apply(other, lambda u, v: u ** v)++ def __rpow__(self, other):+ return self._apply(other, lambda u, v: v ** u)++ def __neg__(self):+ return TensorList([-u for u in self.tensors])++ def __iter__(self):+ for tensor in self.tensors:+ yield tensor++ def __repr__(self):+ return 'TensorList: \n' + repr(self.tensors)++ def _apply(self, other, op):+ if isinstance(other, (list, tuple, TensorList)) or (+ isinstance(other, torch.Tensor) and (+ other.numel() > 1 or other.ndim > 1+ )+ ):+ assert len(other) == len(self.tensors)+ return TensorList([op(u, v) for u, v in zip(self.tensors, other)])+ elif isinstance(other, numbers.Number) or (+ isinstance(other, torch.Tensor) and (+ other.numel() == 1 and other.ndim <= 1+ )+ ):+ return TensorList([op(u, other) for u in self.tensors])+ else:+ raise TypeError(+ f'unsupported operand for *: "TensorList" and "{type(other)}"'+ )\ No newline at end of filediff --git a/models/wan/animate/face_blocks.py b/models/wan/animate/face_blocks.pynew file mode 100644index 0000000..8ddb829--- /dev/null+++ b/models/wan/animate/face_blocks.py@@ -0,0 +1,382 @@+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.+from torch import nn+import torch+from typing import Tuple, Optional+from einops import rearrange+import torch.nn.functional as F+import math+from shared.attention import pay_attention++MEMORY_LAYOUT = {+ "flash": (+ lambda x: x.view(x.shape[0] * x.shape[1], *x.shape[2:]),+ lambda x: x,+ ),+ "torch": (+ lambda x: x.transpose(1, 2),+ lambda x: x.transpose(1, 2),+ ),+ "vanilla": (+ lambda x: x.transpose(1, 2),+ lambda x: x.transpose(1, 2),+ ),+}+++def attention(+ q,+ k,+ v,+ mode="torch",+ drop_rate=0,+ attn_mask=None,+ causal=False,+ max_seqlen_q=None,+ batch_size=1,+):+ """+ Perform QKV self attention.++ Args:+ q (torch.Tensor): Query tensor with shape [b, s, a, d], where a is the number of heads.+ k (torch.Tensor): Key tensor with shape [b, s1, a, d]+ v (torch.Tensor): Value tensor with shape [b, s1, a, d]+ mode (str): Attention mode. Choose from 'self_flash', 'cross_flash', 'torch', and 'vanilla'.+ drop_rate (float): Dropout rate in attention map. (default: 0)+ attn_mask (torch.Tensor): Attention mask with shape [b, s1] (cross_attn), or [b, a, s, s1] (torch or vanilla).+ (default: None)+ causal (bool): Whether to use causal attention. (default: False)+ cu_seqlens_q (torch.Tensor): dtype torch.int32. The cumulative sequence lengths of the sequences in the batch,+ used to index into q.+ cu_seqlens_kv (torch.Tensor): dtype torch.int32. The cumulative sequence lengths of the sequences in the batch,+ used to index into kv.+ max_seqlen_q (int): The maximum sequence length in the batch of q.+ max_seqlen_kv (int): The maximum sequence length in the batch of k and v.++ Returns:+ torch.Tensor: Output tensor after self attention with shape [b, s, ad]+ """+ pre_attn_layout, post_attn_layout = MEMORY_LAYOUT[mode]++ if mode == "torch":+ if attn_mask is not None and attn_mask.dtype != torch.bool:+ attn_mask = attn_mask.to(q.dtype)+ x = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask, dropout_p=drop_rate, is_causal=causal)++ elif mode == "flash":+ x = flash_attn_func(+ q,+ k,+ v,+ )+ x = x.view(batch_size, max_seqlen_q, x.shape[-2], x.shape[-1]) # reshape x to [b, s, a, d]+ elif mode == "vanilla":+ scale_factor = 1 / math.sqrt(q.size(-1))++ b, a, s, _ = q.shape+ s1 = k.size(2)+ attn_bias = torch.zeros(b, a, s, s1, dtype=q.dtype, device=q.device)+ if causal:+ # Only applied to self attention+ assert attn_mask is None, "Causal mask and attn_mask cannot be used together"+ temp_mask = torch.ones(b, a, s, s, dtype=torch.bool, device=q.device).tril(diagonal=0)+ attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf"))+ attn_bias.to(q.dtype)++ if attn_mask is not None:+ if attn_mask.dtype == torch.bool:+ attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf"))+ else:+ attn_bias += attn_mask++ attn = (q @ k.transpose(-2, -1)) * scale_factor+ attn += attn_bias+ attn = attn.softmax(dim=-1)+ attn = torch.dropout(attn, p=drop_rate, train=True)+ x = attn @ v+ else:+ raise NotImplementedError(f"Unsupported attention mode: {mode}")++ x = post_attn_layout(x)+ b, s, a, d = x.shape+ out = x.reshape(b, s, -1)+ return out+++class CausalConv1d(nn.Module):++ def __init__(self, chan_in, chan_out, kernel_size=3, stride=1, dilation=1, pad_mode="replicate", **kwargs):+ super().__init__()++ self.pad_mode = pad_mode+ padding = (kernel_size - 1, 0) # T+ self.time_causal_padding = padding++ self.conv = nn.Conv1d(chan_in, chan_out, kernel_size, stride=stride, dilation=dilation, **kwargs)++ def forward(self, x):+ x = F.pad(x, self.time_causal_padding, mode=self.pad_mode)+ return self.conv(x)++++class FaceEncoder(nn.Module):+ def __init__(self, in_dim: int, hidden_dim: int, num_heads=int, dtype=None, device=None):+ factory_kwargs = {"dtype": dtype, "device": device}+ super().__init__()++ self.num_heads = num_heads+ self.conv1_local = CausalConv1d(in_dim, 1024 * num_heads, 3, stride=1)+ self.norm1 = nn.LayerNorm(hidden_dim // 8, elementwise_affine=False, eps=1e-6, **factory_kwargs)+ self.act = nn.SiLU()+ self.conv2 = CausalConv1d(1024, 1024, 3, stride=2)+ self.conv3 = CausalConv1d(1024, 1024, 3, stride=2)++ self.out_proj = nn.Linear(1024, hidden_dim)+ self.norm1 = nn.LayerNorm(1024, elementwise_affine=False, eps=1e-6, **factory_kwargs)++ self.norm2 = nn.LayerNorm(1024, elementwise_affine=False, eps=1e-6, **factory_kwargs)++ self.norm3 = nn.LayerNorm(1024, elementwise_affine=False, eps=1e-6, **factory_kwargs)++ self.padding_tokens = nn.Parameter(torch.zeros(1, 1, 1, hidden_dim))++ def forward(self, x):++ x = rearrange(x, "b t c -> b c t")+ b, c, t = x.shape++ x = self.conv1_local(x)+ x = rearrange(x, "b (n c) t -> (b n) t c", n=self.num_heads)++ x = self.norm1(x)+ x = self.act(x)+ x = rearrange(x, "b t c -> b c t")+ x = self.conv2(x)+ x = rearrange(x, "b c t -> b t c")+ x = self.norm2(x)+ x = self.act(x)+ x = rearrange(x, "b t c -> b c t")+ x = self.conv3(x)+ x = rearrange(x, "b c t -> b t c")+ x = self.norm3(x)+ x = self.act(x)+ x = self.out_proj(x)+ x = rearrange(x, "(b n) t c -> b t n c", b=b)+ padding = self.padding_tokens.repeat(b, x.shape[1], 1, 1)+ x = torch.cat([x, padding], dim=-2)+ x_local = x.clone()++ return x_local++++class RMSNorm(nn.Module):+ def __init__(+ self,+ dim: int,+ elementwise_affine=True,+ eps: float = 1e-6,+ device=None,+ dtype=None,+ ):+ """+ Initialize the RMSNorm normalization layer.++ Args:+ dim (int): The dimension of the input tensor.+ eps (float, optional): A small value added to the denominator for numerical stability. Default is 1e-6.++ Attributes:+ eps (float): A small value added to the denominator for numerical stability.+ weight (nn.Parameter): Learnable scaling parameter.++ """+ factory_kwargs = {"device": device, "dtype": dtype}+ super().__init__()+ self.eps = eps+ if elementwise_affine:+ self.weight = nn.Parameter(torch.ones(dim, **factory_kwargs))++ def _norm(self, x):+ """+ Apply the RMSNorm normalization to the input tensor.++ Args:+ x (torch.Tensor): The input tensor.++ Returns:+ torch.Tensor: The normalized tensor.++ """+ return x * torch.rsqrt(x.pow(2).mean(-1, keepdim=True) + self.eps)++ def forward(self, x):+ """+ Forward pass through the RMSNorm layer.++ Args:+ x (torch.Tensor): The input tensor.++ Returns:+ torch.Tensor: The output tensor after applying RMSNorm.++ """+ output = self._norm(x.float()).type_as(x)+ if hasattr(self, "weight"):+ output = output * self.weight+ return output+++def get_norm_layer(norm_layer):+ """+ Get the normalization layer.++ Args:+ norm_layer (str): The type of normalization layer.++ Returns:+ norm_layer (nn.Module): The normalization layer.+ """+ if norm_layer == "layer":+ return nn.LayerNorm+ elif norm_layer == "rms":+ return RMSNorm+ else:+ raise NotImplementedError(f"Norm layer {norm_layer} is not implemented")+++class FaceAdapter(nn.Module):+ def __init__(+ self,+ hidden_dim: int,+ heads_num: int,+ qk_norm: bool = True,+ qk_norm_type: str = "rms",+ num_adapter_layers: int = 1,+ dtype=None,+ device=None,+ ):++ factory_kwargs = {"dtype": dtype, "device": device}+ super().__init__()+ self.hidden_size = hidden_dim+ self.heads_num = heads_num+ self.fuser_blocks = nn.ModuleList(+ [+ FaceBlock(+ self.hidden_size,+ self.heads_num,+ qk_norm=qk_norm,+ qk_norm_type=qk_norm_type,+ **factory_kwargs,+ )+ for _ in range(num_adapter_layers)+ ]+ )++ def forward(+ self,+ x: torch.Tensor,+ motion_embed: torch.Tensor,+ idx: int,+ freqs_cis_q: Tuple[torch.Tensor, torch.Tensor] = None,+ freqs_cis_k: Tuple[torch.Tensor, torch.Tensor] = None,+ ) -> torch.Tensor:++ return self.fuser_blocks[idx](x, motion_embed, freqs_cis_q, freqs_cis_k)++++class FaceBlock(nn.Module):+ def __init__(+ self,+ hidden_size: int,+ heads_num: int,+ qk_norm: bool = True,+ qk_norm_type: str = "rms",+ qk_scale: float = None,+ dtype: Optional[torch.dtype] = None,+ device: Optional[torch.device] = None,+ ):+ factory_kwargs = {"device": device, "dtype": dtype}+ super().__init__()++ self.deterministic = False+ self.hidden_size = hidden_size+ self.heads_num = heads_num+ head_dim = hidden_size // heads_num+ self.scale = qk_scale or head_dim**-0.5++ self.linear1_kv = nn.Linear(hidden_size, hidden_size * 2, **factory_kwargs)+ self.linear1_q = nn.Linear(hidden_size, hidden_size, **factory_kwargs)++ self.linear2 = nn.Linear(hidden_size, hidden_size, **factory_kwargs)++ qk_norm_layer = get_norm_layer(qk_norm_type)+ self.q_norm = (+ qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) if qk_norm else nn.Identity()+ )+ self.k_norm = (+ qk_norm_layer(head_dim, elementwise_affine=True, eps=1e-6, **factory_kwargs) if qk_norm else nn.Identity()+ )++ self.pre_norm_feat = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs)++ self.pre_norm_motion = nn.LayerNorm(hidden_size, elementwise_affine=False, eps=1e-6, **factory_kwargs)++ def forward(+ self,+ x: torch.Tensor,+ motion_vec: torch.Tensor,+ motion_mask: Optional[torch.Tensor] = None,+ use_context_parallel=False,+ ) -> torch.Tensor:++ B, T, N, C = motion_vec.shape+ T_comp = T++ x_motion = self.pre_norm_motion(motion_vec)+ x_feat = self.pre_norm_feat(x)++ kv = self.linear1_kv(x_motion)+ q = self.linear1_q(x_feat)++ k, v = rearrange(kv, "B L N (K H D) -> K B L N H D", K=2, H=self.heads_num)+ q = rearrange(q, "B S (H D) -> B S H D", H=self.heads_num)++ # Apply QK-Norm if needed.+ q = self.q_norm(q).to(v)+ k = self.k_norm(k).to(v)++ k = rearrange(k, "B L N H D -> (B L) N H D")+ v = rearrange(v, "B L N H D -> (B L) N H D")++ if use_context_parallel:+ q = gather_forward(q, dim=1)++ q = rearrange(q, "B (L S) H D -> (B L) S H D", L=T_comp)+ # Compute attention.+ # Size([batches, tokens, heads, head_features])+ qkv_list = [q, k, v]+ del q,k,v+ attn = pay_attention(qkv_list)+ # attn = attention(+ # q,+ # k,+ # v,+ # max_seqlen_q=q.shape[1],+ # batch_size=q.shape[0],+ # )++ attn = attn.reshape(*attn.shape[:2], -1)+ attn = rearrange(attn, "(B L) S C -> B (L S) C", L=T_comp)+ # if use_context_parallel:+ # attn = torch.chunk(attn, get_world_size(), dim=1)[get_rank()]++ output = self.linear2(attn)++ if motion_mask is not None:+ output = output * rearrange(motion_mask, "B T H W -> B (T H W)").unsqueeze(-1)++ return output\ No newline at end of filediff --git a/models/wan/animate/model_animate.py b/models/wan/animate/model_animate.pynew file mode 100644index 0000000..d07f762--- /dev/null+++ b/models/wan/animate/model_animate.py@@ -0,0 +1,31 @@+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.+import math+import types+from copy import deepcopy+from einops import rearrange+from typing import List+import numpy as np+import torch+import torch.cuda.amp as amp+import torch.nn as nn++def after_patch_embedding(self, x: List[torch.Tensor], pose_latents, face_pixel_values):+ pose_latents = self.pose_patch_embedding(pose_latents)+ x[:, :, 1:] += pose_latents++ b,c,T,h,w = face_pixel_values.shape+ face_pixel_values = rearrange(face_pixel_values, "b c t h w -> (b t) c h w")+ encode_bs = 8+ face_pixel_values_tmp = []+ for i in range(math.ceil(face_pixel_values.shape[0]/encode_bs)):+ face_pixel_values_tmp.append(self.motion_encoder.get_motion(face_pixel_values[i*encode_bs:(i+1)*encode_bs]))++ motion_vec = torch.cat(face_pixel_values_tmp)++ motion_vec = rearrange(motion_vec, "(b t) c -> b t c", t=T)+ motion_vec = self.face_encoder(motion_vec)++ B, L, H, C = motion_vec.shape+ pad_face = torch.zeros(B, 1, H, C).type_as(motion_vec)+ motion_vec = torch.cat([pad_face, motion_vec], dim=1)+ return x, motion_vecdiff --git a/models/wan/animate/motion_encoder.py b/models/wan/animate/motion_encoder.pynew file mode 100644index 0000000..02b0040--- /dev/null+++ b/models/wan/animate/motion_encoder.py@@ -0,0 +1,308 @@+# Modified from ``https://github.com/wyhsirius/LIA``+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.+import torch+import torch.nn as nn+from torch.nn import functional as F+import math++def custom_qr(input_tensor):+ original_dtype = input_tensor.dtype+ if original_dtype in [torch.bfloat16, torch.float16]:+ q, r = torch.linalg.qr(input_tensor.to(torch.float32))+ return q.to(original_dtype), r.to(original_dtype)+ return torch.linalg.qr(input_tensor)++def fused_leaky_relu(input, bias, negative_slope=0.2, scale=2 ** 0.5):+ return F.leaky_relu(input + bias, negative_slope) * scale+++def upfirdn2d_native(input, kernel, up_x, up_y, down_x, down_y, pad_x0, pad_x1, pad_y0, pad_y1):+ _, minor, in_h, in_w = input.shape+ kernel_h, kernel_w = kernel.shape++ out = input.view(-1, minor, in_h, 1, in_w, 1)+ out = F.pad(out, [0, up_x - 1, 0, 0, 0, up_y - 1, 0, 0])+ out = out.view(-1, minor, in_h * up_y, in_w * up_x)++ out = F.pad(out, [max(pad_x0, 0), max(pad_x1, 0), max(pad_y0, 0), max(pad_y1, 0)])+ out = out[:, :, max(-pad_y0, 0): out.shape[2] - max(-pad_y1, 0),+ max(-pad_x0, 0): out.shape[3] - max(-pad_x1, 0), ]++ out = out.reshape([-1, 1, in_h * up_y + pad_y0 + pad_y1, in_w * up_x + pad_x0 + pad_x1])+ w = torch.flip(kernel, [0, 1]).view(1, 1, kernel_h, kernel_w)+ out = F.conv2d(out, w)+ out = out.reshape(-1, minor, in_h * up_y + pad_y0 + pad_y1 - kernel_h + 1,+ in_w * up_x + pad_x0 + pad_x1 - kernel_w + 1, )+ return out[:, :, ::down_y, ::down_x]+++def upfirdn2d(input, kernel, up=1, down=1, pad=(0, 0)):+ return upfirdn2d_native(input, kernel, up, up, down, down, pad[0], pad[1], pad[0], pad[1])+++def make_kernel(k):+ k = torch.tensor(k, dtype=torch.float32)+ if k.ndim == 1:+ k = k[None, :] * k[:, None]+ k /= k.sum()+ return k+++class FusedLeakyReLU(nn.Module):+ def __init__(self, channel, negative_slope=0.2, scale=2 ** 0.5):+ super().__init__()+ self.bias = nn.Parameter(torch.zeros(1, channel, 1, 1))+ self.negative_slope = negative_slope+ self.scale = scale++ def forward(self, input):+ out = fused_leaky_relu(input, self.bias, self.negative_slope, self.scale)+ return out+++class Blur(nn.Module):+ def __init__(self, kernel, pad, upsample_factor=1):+ super().__init__()++ kernel = make_kernel(kernel)++ if upsample_factor > 1:+ kernel = kernel * (upsample_factor ** 2)++ self.register_buffer('kernel', kernel)++ self.pad = pad++ def forward(self, input):+ return upfirdn2d(input, self.kernel, pad=self.pad)+++class ScaledLeakyReLU(nn.Module):+ def __init__(self, negative_slope=0.2):+ super().__init__()++ self.negative_slope = negative_slope++ def forward(self, input):+ return F.leaky_relu(input, negative_slope=self.negative_slope)+++class EqualConv2d(nn.Module):+ def __init__(self, in_channel, out_channel, kernel_size, stride=1, padding=0, bias=True):+ super().__init__()++ self.weight = nn.Parameter(torch.randn(out_channel, in_channel, kernel_size, kernel_size))+ self.scale = 1 / math.sqrt(in_channel * kernel_size ** 2)++ self.stride = stride+ self.padding = padding++ if bias:+ self.bias = nn.Parameter(torch.zeros(out_channel))+ else:+ self.bias = None++ def forward(self, input):++ return F.conv2d(input, self.weight * self.scale, bias=self.bias, stride=self.stride, padding=self.padding)++ def __repr__(self):+ return (+ f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]},'+ f' {self.weight.shape[2]}, stride={self.stride}, padding={self.padding})'+ )+++class EqualLinear(nn.Module):+ def __init__(self, in_dim, out_dim, bias=True, bias_init=0, lr_mul=1, activation=None):+ super().__init__()++ self.weight = nn.Parameter(torch.randn(out_dim, in_dim).div_(lr_mul))++ if bias:+ self.bias = nn.Parameter(torch.zeros(out_dim).fill_(bias_init))+ else:+ self.bias = None++ self.activation = activation++ self.scale = (1 / math.sqrt(in_dim)) * lr_mul+ self.lr_mul = lr_mul++ def forward(self, input):++ if self.activation:+ out = F.linear(input, self.weight * self.scale)+ out = fused_leaky_relu(out, self.bias * self.lr_mul)+ else:+ out = F.linear(input, self.weight * self.scale, bias=self.bias * self.lr_mul)++ return out++ def __repr__(self):+ return (f'{self.__class__.__name__}({self.weight.shape[1]}, {self.weight.shape[0]})')+++class ConvLayer(nn.Sequential):+ def __init__(+ self,+ in_channel,+ out_channel,+ kernel_size,+ downsample=False,+ blur_kernel=[1, 3, 3, 1],+ bias=True,+ activate=True,+ ):+ layers = []++ if downsample:+ factor = 2+ p = (len(blur_kernel) - factor) + (kernel_size - 1)+ pad0 = (p + 1) // 2+ pad1 = p // 2++ layers.append(Blur(blur_kernel, pad=(pad0, pad1)))++ stride = 2+ self.padding = 0++ else:+ stride = 1+ self.padding = kernel_size // 2++ layers.append(EqualConv2d(in_channel, out_channel, kernel_size, padding=self.padding, stride=stride,+ bias=bias and not activate))++ if activate:+ if bias:+ layers.append(FusedLeakyReLU(out_channel))+ else:+ layers.append(ScaledLeakyReLU(0.2))++ super().__init__(*layers)+++class ResBlock(nn.Module):+ def __init__(self, in_channel, out_channel, blur_kernel=[1, 3, 3, 1]):+ super().__init__()++ self.conv1 = ConvLayer(in_channel, in_channel, 3)+ self.conv2 = ConvLayer(in_channel, out_channel, 3, downsample=True)++ self.skip = ConvLayer(in_channel, out_channel, 1, downsample=True, activate=False, bias=False)++ def forward(self, input):+ out = self.conv1(input)+ out = self.conv2(out)++ skip = self.skip(input)+ out = (out + skip) / math.sqrt(2)++ return out+++class EncoderApp(nn.Module):+ def __init__(self, size, w_dim=512):+ super(EncoderApp, self).__init__()++ channels = {+ 4: 512,+ 8: 512,+ 16: 512,+ 32: 512,+ 64: 256,+ 128: 128,+ 256: 64,+ 512: 32,+ 1024: 16+ }++ self.w_dim = w_dim+ log_size = int(math.log(size, 2))++ self.convs = nn.ModuleList()+ self.convs.append(ConvLayer(3, channels[size], 1))++ in_channel = channels[size]+ for i in range(log_size, 2, -1):+ out_channel = channels[2 ** (i - 1)]+ self.convs.append(ResBlock(in_channel, out_channel))+ in_channel = out_channel++ self.convs.append(EqualConv2d(in_channel, self.w_dim, 4, padding=0, bias=False))++ def forward(self, x):++ res = []+ h = x+ for conv in self.convs:+ h = conv(h)+ res.append(h)++ return res[-1].squeeze(-1).squeeze(-1), res[::-1][2:]+++class Encoder(nn.Module):+ def __init__(self, size, dim=512, dim_motion=20):+ super(Encoder, self).__init__()++ # appearance netmork+ self.net_app = EncoderApp(size, dim)++ # motion network+ fc = [EqualLinear(dim, dim)]+ for i in range(3):+ fc.append(EqualLinear(dim, dim))++ fc.append(EqualLinear(dim, dim_motion))+ self.fc = nn.Sequential(*fc)++ def enc_app(self, x):+ h_source = self.net_app(x)+ return h_source++ def enc_motion(self, x):+ h, _ = self.net_app(x)+ h_motion = self.fc(h)+ return h_motion+++class Direction(nn.Module):+ def __init__(self, motion_dim):+ super(Direction, self).__init__()+ self.weight = nn.Parameter(torch.randn(512, motion_dim))++ def forward(self, input):++ weight = self.weight + 1e-8+ Q, R = custom_qr(weight)+ if input is None:+ return Q+ else:+ input_diag = torch.diag_embed(input) # alpha, diagonal matrix+ out = torch.matmul(input_diag, Q.T)+ out = torch.sum(out, dim=1)+ return out+++class Synthesis(nn.Module):+ def __init__(self, motion_dim):+ super(Synthesis, self).__init__()+ self.direction = Direction(motion_dim)+++class Generator(nn.Module):+ def __init__(self, size, style_dim=512, motion_dim=20):+ super().__init__()++ self.enc = Encoder(size, style_dim, motion_dim)+ self.dec = Synthesis(motion_dim)++ def get_motion(self, img):+ #motion_feat = self.enc.enc_motion(img)+ # motion_feat = torch.utils.checkpoint.checkpoint((self.enc.enc_motion), img, use_reentrant=True)+ with torch.cuda.amp.autocast(dtype=torch.float32):+ motion_feat = self.enc.enc_motion(img)+ motion = self.dec.direction(motion_feat)+ return motion\ No newline at end of filediff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex c03752b..41d6d63 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -203,10 +203,7 @@ class WanAny2V:self.use_timestep_transform = Truedef vace_encode_frames(self, frames, ref_images, masks=None, tile_size = 0, overlapped_latents = None):- if ref_images is None:- ref_images = [None] * len(frames)- else:- assert len(frames) == len(ref_images)+ ref_images = [ref_images] * len(frames)if masks is None:latents = self.vae.encode(frames, tile_size = tile_size)@@ -238,11 +235,7 @@ class WanAny2V:return cat_latentsdef vace_encode_masks(self, masks, ref_images=None):- if ref_images is None:- ref_images = [None] * len(masks)- else:- assert len(masks) == len(ref_images)-+ ref_images = [ref_images] * len(masks)result_masks = []for mask, refs in zip(masks, ref_images):c, depth, height, width = mask.shape@@ -270,79 +263,6 @@ class WanAny2V:result_masks.append(mask)return result_masks- def vace_latent(self, z, m):- return [torch.cat([zz, mm], dim=0) for zz, mm in zip(z, m)]--- def prepare_source(self, src_video, src_mask, src_ref_images, total_frames, image_size, device, keep_video_guide_frames= [], pre_src_video = None, inject_frames = [], outpainting_dims = None, any_background_ref = False):- image_sizes = []- trim_video_guide = len(keep_video_guide_frames)- def conv_tensor(t, device):- return t.float().div_(127.5).add_(-1).permute(3, 0, 1, 2).to(device)-- for i, (sub_src_video, sub_src_mask, sub_pre_src_video) in enumerate(zip(src_video, src_mask,pre_src_video)):- prepend_count = 0 if sub_pre_src_video == None else sub_pre_src_video.shape[1]- num_frames = total_frames - prepend_count- num_frames = min(num_frames, trim_video_guide) if trim_video_guide > 0 and sub_src_video != None else num_frames- if sub_src_mask is not None and sub_src_video is not None:- src_video[i] = conv_tensor(sub_src_video[:num_frames], device)- src_mask[i] = conv_tensor(sub_src_mask[:num_frames], device)- # src_video is [-1, 1] (at this function output), 0 = inpainting area (in fact 127 in [0, 255])- # src_mask is [-1, 1] (at this function output), 0 = preserve original video (in fact 127 in [0, 255]) and 1 = Inpainting (in fact 255 in [0, 255])- if prepend_count > 0:- src_video[i] = torch.cat( [sub_pre_src_video, src_video[i]], dim=1)- src_mask[i] = torch.cat( [torch.full_like(sub_pre_src_video, -1.0), src_mask[i]] ,1)- src_video_shape = src_video[i].shape- if src_video_shape[1] != total_frames:- src_video[i] = torch.cat( [src_video[i], src_video[i].new_zeros(src_video_shape[0], total_frames -src_video_shape[1], *src_video_shape[-2:])], dim=1)- src_mask[i] = torch.cat( [src_mask[i], src_mask[i].new_ones(src_video_shape[0], total_frames -src_video_shape[1], *src_video_shape[-2:])], dim=1)- src_mask[i] = torch.clamp((src_mask[i][:, :, :, :] + 1) / 2, min=0, max=1)- image_sizes.append(src_video[i].shape[2:])- elif sub_src_video is None:- if prepend_count > 0:- src_video[i] = torch.cat( [sub_pre_src_video, torch.zeros((3, num_frames, image_size[0], image_size[1]), device=device)], dim=1)- src_mask[i] = torch.cat( [torch.zeros_like(sub_pre_src_video), torch.ones((3, num_frames, image_size[0], image_size[1]), device=device)] ,1)- else:- src_video[i] = torch.zeros((3, total_frames, image_size[0], image_size[1]), device=device)- src_mask[i] = torch.ones_like(src_video[i], device=device)- image_sizes.append(image_size)- else:- src_video[i] = conv_tensor(sub_src_video[:num_frames], device)- src_mask[i] = torch.ones_like(src_video[i], device=device)- if prepend_count > 0:- src_video[i] = torch.cat( [sub_pre_src_video, src_video[i]], dim=1)- src_mask[i] = torch.cat( [torch.zeros_like(sub_pre_src_video), src_mask[i]] ,1)- src_video_shape = src_video[i].shape- if src_video_shape[1] != total_frames:- src_video[i] = torch.cat( [src_video[i], src_video[i].new_zeros(src_video_shape[0], total_frames -src_video_shape[1], *src_video_shape[-2:])], dim=1)- src_mask[i] = torch.cat( [src_mask[i], src_mask[i].new_ones(src_video_shape[0], total_frames -src_video_shape[1], *src_video_shape[-2:])], dim=1)- image_sizes.append(src_video[i].shape[2:])- for k, keep in enumerate(keep_video_guide_frames):- if not keep:- pos = prepend_count + k- src_video[i][:, pos:pos+1] = 0- src_mask[i][:, pos:pos+1] = 1-- for k, frame in enumerate(inject_frames):- if frame != None:- pos = prepend_count + k- src_video[i][:, pos:pos+1], src_mask[i][:, pos:pos+1] = fit_image_into_canvas(frame, image_size, 0, device, True, outpainting_dims, return_mask= True)--- self.background_mask = None- for i, ref_images in enumerate(src_ref_images):- if ref_images is not None:- image_size = image_sizes[i]- for j, ref_img in enumerate(ref_images):- if ref_img is not None and not torch.is_tensor(ref_img):- if j==0 and any_background_ref:- if self.background_mask == None: self.background_mask = [None] * len(src_ref_images)- src_ref_images[i][j], self.background_mask[i] = fit_image_into_canvas(ref_img, image_size, 0, device, True, outpainting_dims, return_mask= True)- else:- src_ref_images[i][j], _ = fit_image_into_canvas(ref_img, image_size, 1, device)- if self.background_mask != None:- self.background_mask = [ item if item != None else self.background_mask[0] for item in self.background_mask ] # deplicate background mask with double control net since first controlnet image ref modifed by ref- return src_video, src_mask, src_ref_imagesdef get_vae_latents(self, ref_images, device, tile_size= 0):ref_vae_latents = []@@ -369,7 +289,9 @@ class WanAny2V:def generate(self,input_prompt,input_frames= None,+ input_frames2= None,input_masks = None,+ input_masks2 = None,input_ref_images = None,input_ref_masks = None,input_faces = None,@@ -615,21 +537,22 @@ class WanAny2V:pose_pixels = input_frames * input_masksinput_masks = 1. - input_maskspose_pixels -= input_masks- save_video(pose_pixels, "pose.mp4")pose_latents = self.vae.encode([pose_pixels], VAE_tile_size)[0].unsqueeze(0)input_frames = input_frames * input_masksif not "X" in video_prompt_type: input_frames += input_masks - 1 # masked area should black (-1) in background framesif prefix_frames_count > 0:input_frames[:, :prefix_frames_count] = input_videoinput_masks[:, :prefix_frames_count] = 1- save_video(input_frames, "input_frames.mp4")- save_video(input_masks, "input_masks.mp4", value_range=(0,1))+ # save_video(pose_pixels, "pose.mp4")+ # save_video(input_frames, "input_frames.mp4")+ # save_video(input_masks, "input_masks.mp4", value_range=(0,1))lat_h, lat_w = height // self.vae_stride[1], width // self.vae_stride[2]msk_ref = self.get_i2v_mask(lat_h, lat_w, nb_frames_unchanged=1,lat_t=1, device=self.device)msk_control = self.get_i2v_mask(lat_h, lat_w, nb_frames_unchanged=0, mask_pixel_values=input_masks, device=self.device)msk = torch.concat([msk_ref, msk_control], dim=1)- clip_image_start = image_ref = convert_image_to_tensor(input_ref_images[0]).to(self.device)- lat_y = torch.concat(self.vae.encode([image_ref.unsqueeze(1).to(self.device), input_frames.to(self.device)], VAE_tile_size), dim=1)+ image_ref = input_ref_images[0].to(self.device)+ clip_image_start = image_ref.squeeze(1)+ lat_y = torch.concat(self.vae.encode([image_ref, input_frames.to(self.device)], VAE_tile_size), dim=1)y = torch.concat([msk, lat_y])kwargs.update({ 'y': y, 'pose_latents': pose_latents, 'face_pixel_values' : input_faces.unsqueeze(0)})lat_y = msk = msk_control = msk_ref = pose_pixels = None@@ -701,12 +624,11 @@ class WanAny2V:# Phantomif phantom:- input_ref_images_neg = None- if input_ref_images != None: # Phantom Ref images- input_ref_images = self.get_vae_latents(input_ref_images, self.device)- input_ref_images_neg = torch.zeros_like(input_ref_images)- ref_images_count = input_ref_images.shape[1] if input_ref_images != None else 0- trim_frames = input_ref_images.shape[1]+ lat_input_ref_images_neg = None+ if input_ref_images is not None: # Phantom Ref images+ lat_input_ref_images = self.get_vae_latents(input_ref_images, self.device)+ lat_input_ref_images_neg = torch.zeros_like(lat_input_ref_images)+ ref_images_count = trim_frames = lat_input_ref_images.shape[1]if ti2v:if input_video is None:@@ -721,25 +643,23 @@ class WanAny2V:# Vaceif vace :# vace context encode- input_frames = [u.to(self.device) for u in input_frames]- input_ref_images = [ None if u == None else [v.to(self.device) for v in u] for u in input_ref_images]- input_masks = [u.to(self.device) for u in input_masks]+ input_frames = [input_frames.to(self.device)] +([] if input_frames2 is None else [input_frames2.to(self.device)])+ input_masks = [input_masks.to(self.device)] + ([] if input_masks2 is None else [input_masks2.to(self.device)])+ input_ref_images = None if input_ref_images is None else [ u.to(self.device) for u in input_ref_images]+ input_ref_masks = None if input_ref_masks is None else [ None if u is None else u.to(self.device) for u in input_ref_masks]ref_images_before = True- if self.background_mask != None: self.background_mask = [m.to(self.device) for m in self.background_mask]z0 = self.vace_encode_frames(input_frames, input_ref_images, masks=input_masks, tile_size = VAE_tile_size, overlapped_latents = overlapped_latents )m0 = self.vace_encode_masks(input_masks, input_ref_images)- if self.background_mask != None:- color_reference_frame = input_ref_images[0][0].clone()- zbg = self.vace_encode_frames([ref_img[0] for ref_img in input_ref_images], None, masks=self.background_mask, tile_size = VAE_tile_size )- mbg = self.vace_encode_masks(self.background_mask, None)+ if input_ref_masks is not None and len(input_ref_masks) > 0 and input_ref_masks[0] is not None:+ color_reference_frame = input_ref_images[0].clone()+ zbg = self.vace_encode_frames( input_ref_images[:1] * len(input_frames), None, masks=input_ref_masks[0], tile_size = VAE_tile_size )+ mbg = self.vace_encode_masks(input_ref_masks[:1] * len(input_frames), None)for zz0, mm0, zzbg, mmbg in zip(z0, m0, zbg, mbg):zz0[:, 0:1] = zzbgmm0[:, 0:1] = mmbg-- self.background_mask = zz0 = mm0 = zzbg = mmbg = None- z = self.vace_latent(z0, m0)-- ref_images_count = len(input_ref_images[0]) if input_ref_images != None and input_ref_images[0] != None else 0+ zz0 = mm0 = zzbg = mmbg = None+ z = [torch.cat([zz, mm], dim=0) for zz, mm in zip(z0, m0)]+ ref_images_count = len(input_ref_images) if input_ref_images is not None and input_ref_images is not None else 0context_scale = context_scale if context_scale != None else [1.0] * len(z)kwargs.update({'vace_context' : z, 'vace_context_scale' : context_scale, "ref_images_count": ref_images_count })if overlapped_latents != None :@@ -747,15 +667,8 @@ class WanAny2V:extended_overlapped_latents = z[0][:16, :overlapped_latents_size + ref_images_count].clone().unsqueeze(0)if prefix_frames_count > 0:color_reference_frame = input_frames[0][:, prefix_frames_count -1:prefix_frames_count].clone()-- target_shape = list(z0[0].shape)- target_shape[0] = int(target_shape[0] / 2)- lat_h, lat_w = target_shape[-2:]- height = self.vae_stride[1] * lat_h- width = self.vae_stride[2] * lat_w-- else:- target_shape = (self.vae.model.z_dim, lat_frames + ref_images_count, height // self.vae_stride[1], width // self.vae_stride[2])+ lat_h, lat_w = height // self.vae_stride[1], width // self.vae_stride[2]+ target_shape = (self.vae.model.z_dim, lat_frames + ref_images_count, lat_h, lat_w)if multitalk:if audio_proj is None:@@ -860,7 +773,9 @@ class WanAny2V:apg_norm_threshold = 55text_momentumbuffer = MomentumBuffer(apg_momentum)audio_momentumbuffer = MomentumBuffer(apg_momentum)-+ input_frames = input_frames2 = input_masks =input_masks2 = input_video = input_ref_images = input_ref_masks = pre_video_frame = None+ gc.collect()+ torch.cuda.empty_cache()# denoisingtrans = self.model@@ -878,7 +793,7 @@ class WanAny2V:kwargs.update({"t": timestep, "current_step": start_step_no + i})kwargs["slg_layers"] = slg_layers if int(slg_start * sampling_steps) <= i < int(slg_end * sampling_steps) else None- if denoising_strength < 1 and input_frames != None and i <= injection_denoising_step:+ if denoising_strength < 1 and i <= injection_denoising_step:sigma = t / 1000noise = torch.randn(batch_size, *target_shape, dtype=torch.float32, device=self.device, generator=seed_g)if inject_from_start:@@ -912,8 +827,8 @@ class WanAny2V:any_guidance = guide_scale != 1if phantom:gen_args = {- "x" : ([ torch.cat([latent_model_input[:,:, :-ref_images_count], input_ref_images.unsqueeze(0).expand(*expand_shape)], dim=2) ] * 2 +- [ torch.cat([latent_model_input[:,:, :-ref_images_count], input_ref_images_neg.unsqueeze(0).expand(*expand_shape)], dim=2)]),+ "x" : ([ torch.cat([latent_model_input[:,:, :-ref_images_count], lat_input_ref_images.unsqueeze(0).expand(*expand_shape)], dim=2) ] * 2 ++ [ torch.cat([latent_model_input[:,:, :-ref_images_count], lat_input_ref_images_neg.unsqueeze(0).expand(*expand_shape)], dim=2)]),"context": [context, context_null, context_null] ,}elif fantasy:diff --git a/models/wan/df_handler.py b/models/wan/df_handler.pyindex 7a5d3ea..31d5da6 100644--- a/models/wan/df_handler.py+++ b/models/wan/df_handler.py@@ -21,6 +21,7 @@ class family_handler():extra_model_def["fps"] =fpsextra_model_def["frames_minimum"] = 17extra_model_def["frames_steps"] = 20+ extra_model_def["latent_size"] = 4extra_model_def["sliding_window"] = Trueextra_model_def["skip_layer_guidance"] = Trueextra_model_def["tea_cache"] = Truediff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex ff1570f..f2a8fb4 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -114,7 +114,6 @@ class family_handler():"tea_cache" : not (base_model_type in ["i2v_2_2"] or test_wan_5B(base_model_type) or multiple_submodels),"mag_cache" : True,"keep_frames_video_guide_not_supported": base_model_type in ["infinitetalk"],- "convert_image_guide_to_video" : True,"sample_solvers":[("unipc", "unipc"),("euler", "euler"),@@ -175,6 +174,8 @@ class family_handler():extra_model_def["forced_guide_mask_inputs"] = Trueextra_model_def["background_removal_label"]= "Remove Backgrounds behind People (Animate Mode Only)"extra_model_def["background_ref_outpainted"] = False+ extra_model_def["return_image_refs_tensor"] = True+ extra_model_def["guide_inpaint_color"] = 0@@ -196,15 +197,15 @@ class family_handler():"letters_filter": "KFI",}- extra_model_def["lock_image_refs_ratios"] = Trueextra_model_def["background_removal_label"]= "Remove Backgrounds behind People / Objects, keep it for Landscape or Positioned Frames"extra_model_def["video_guide_outpainting"] = [0,1]extra_model_def["pad_guide_video"] = Trueextra_model_def["guide_inpaint_color"] = 127.5extra_model_def["forced_guide_mask_inputs"] = True+ extra_model_def["return_image_refs_tensor"] = Trueif base_model_type in ["standin"]:- extra_model_def["lock_image_refs_ratios"] = True+ extra_model_def["fit_into_canvas_image_refs"] = 0extra_model_def["image_ref_choices"] = {"choices": [("No Reference Image", ""),@@ -480,6 +481,7 @@ class family_handler():ui_defaults.update({"video_prompt_type": "PVBXAKI","mask_expand": 20,+ "audio_prompt_type_value": "R",})if text_oneframe_overlap(base_model_type):diff --git a/shared/utils/utils.py b/shared/utils/utils.pyindex 6e8a98b..bb2d5ff 100644--- a/shared/utils/utils.py+++ b/shared/utils/utils.py@@ -32,6 +32,14 @@ def seed_everything(seed: int):if torch.backends.mps.is_available():torch.mps.manual_seed(seed)+def has_video_file_extension(filename):+ extension = os.path.splitext(filename)[-1].lower()+ return extension in [".mp4"]++def has_image_file_extension(filename):+ extension = os.path.splitext(filename)[-1].lower()+ return extension in [".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp", ".tif", ".tiff", ".jfif", ".pjpeg"]+def resample(video_fps, video_frames_count, max_target_frames_count, target_fps, start_target_frame ):import math@@ -94,7 +102,7 @@ def get_video_info(video_path):return fps, width, height, frame_count-def get_video_frame(file_name: str, frame_no: int, return_last_if_missing: bool = False, return_PIL = True) -> torch.Tensor:+def get_video_frame(file_name: str, frame_no: int, return_last_if_missing: bool = False, target_fps = None, return_PIL = True) -> torch.Tensor:"""Extract nth frame from video as PyTorch tensor normalized to [-1, 1]."""cap = cv2.VideoCapture(file_name)@@ -102,7 +110,10 @@ def get_video_frame(file_name: str, frame_no: int, return_last_if_missing: boolraise ValueError(f"Cannot open video: {file_name}")total_frames = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))-+ fps = round(cap.get(cv2.CAP_PROP_FPS))+ if target_fps is not None:+ frame_no = round(target_fps * frame_no /fps)+# Handle out of boundsif frame_no >= total_frames or frame_no < 0:if return_last_if_missing:@@ -175,10 +186,15 @@ def remove_background(img, session=None):def convert_image_to_tensor(image):return torch.from_numpy(np.array(image).astype(np.float32)).div_(127.5).sub_(1.).movedim(-1, 0)-def convert_tensor_to_image(t, frame_no = 0):+def convert_tensor_to_image(t, frame_no = 0, mask_levels = False):if len(t.shape) == 4:t = t[:, frame_no]- return Image.fromarray(t.clone().add_(1.).mul_(127.5).permute(1,2,0).to(torch.uint8).cpu().numpy())+ if t.shape[0]== 1:+ t = t.expand(3,-1,-1)+ if mask_levels:+ return Image.fromarray(t.clone().mul_(255).permute(1,2,0).to(torch.uint8).cpu().numpy())+ else:+ return Image.fromarray(t.clone().add_(1.).mul_(127.5).permute(1,2,0).to(torch.uint8).cpu().numpy())def save_image(tensor_image, name, frame_no = -1):convert_tensor_to_image(tensor_image, frame_no).save(name)@@ -257,7 +273,7 @@ def calculate_dimensions_and_resize_image(image, canvas_height, canvas_width, fiimage = image.resize((new_width, new_height), resample=Image.Resampling.LANCZOS)return image, new_height, new_width-def resize_and_remove_background(img_list, budget_width, budget_height, rm_background, any_background_ref, fit_into_canvas = 0, block_size= 16, outpainting_dims = None, background_ref_outpainted = True, inpaint_color = 127.5 ):+def resize_and_remove_background(img_list, budget_width, budget_height, rm_background, any_background_ref, fit_into_canvas = 0, block_size= 16, outpainting_dims = None, background_ref_outpainted = True, inpaint_color = 127.5, return_tensor = False ):if rm_background:session = new_session()@@ -266,7 +282,7 @@ def resize_and_remove_background(img_list, budget_width, budget_height, rm_backgfor i, img in enumerate(img_list):width, height = img.sizeresized_mask = None- if fit_into_canvas == None or any_background_ref == 1 and i==0 or any_background_ref == 2:+ if any_background_ref == 1 and i==0 or any_background_ref == 2:if outpainting_dims is not None and background_ref_outpainted:resized_image, resized_mask = fit_image_into_canvas(img, (budget_height, budget_width), inpaint_color, full_frame = True, outpainting_dims = outpainting_dims, return_mask= True, return_image= True)elif img.size != (budget_width, budget_height):@@ -291,7 +307,10 @@ def resize_and_remove_background(img_list, budget_width, budget_height, rm_backgif rm_background and not (any_background_ref and i==0 or any_background_ref == 2) :# resized_image = remove(resized_image, session=session, alpha_matting_erode_size = 1,alpha_matting_background_threshold = 70, alpha_foreground_background_threshold = 100, alpha_matting = True, bgcolor=[255, 255, 255, 0]).convert('RGB')resized_image = remove(resized_image, session=session, alpha_matting_erode_size = 1, alpha_matting = True, bgcolor=[255, 255, 255, 0]).convert('RGB')- output_list.append(resized_image) #alpha_matting_background_threshold = 30, alpha_foreground_background_threshold = 200,+ if return_tensor:+ output_list.append(convert_image_to_tensor(resized_image).unsqueeze(1))+ else:+ output_list.append(resized_image)output_mask_list.append(resized_mask)return output_list, output_mask_list@@ -346,47 +365,46 @@ def fit_image_into_canvas(ref_img, image_size, canvas_tf_bg =127.5, device ="cpureturn ref_img.to(device), canvas-def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, image_size, current_video_length = 81, latent_size = 4, any_mask = False, any_guide_padding = False, guide_inpaint_color = 127.5, extract_guide_from_window_start = False, keep_video_guide_frames = [], inject_frames = [], outpainting_dims = None ):+def prepare_video_guide_and_mask( video_guides, video_masks, pre_video_guide, image_size, current_video_length = 81, latent_size = 4, any_mask = False, any_guide_padding = False, guide_inpaint_color = 127.5, keep_video_guide_frames = [], inject_frames = [], outpainting_dims = None, device ="cpu"):src_videos, src_masks = [], []- inpaint_color = guide_inpaint_color/127.5 - 1- prepend_count = pre_video_guide.shape[1] if not extract_guide_from_window_start and pre_video_guide is not None else 0+ inpaint_color_compressed = guide_inpaint_color/127.5 - 1+ prepend_count = pre_video_guide.shape[1] if pre_video_guide is not None else 0for guide_no, (cur_video_guide, cur_video_mask) in enumerate(zip(video_guides, video_masks)):- src_video = src_mask = None- if cur_video_guide is not None:- src_video = cur_video_guide.permute(3, 0, 1, 2).float().div_(127.5).sub_(1.) # c, f, h, w- if cur_video_mask is not None and any_mask:- src_mask = cur_video_mask.permute(3, 0, 1, 2).float().div_(255)[0:1] # c, f, h, w- if pre_video_guide is not None and not extract_guide_from_window_start:+ src_video, src_mask = cur_video_guide, cur_video_mask+ if pre_video_guide is not None:src_video = pre_video_guide if src_video is None else torch.cat( [pre_video_guide, src_video], dim=1)if any_mask:src_mask = torch.zeros_like(pre_video_guide[0:1]) if src_mask is None else torch.cat( [torch.zeros_like(pre_video_guide[0:1]), src_mask], dim=1)- if src_video is None:- if any_guide_padding:- src_video = torch.full( (3, current_video_length, *image_size ), inpaint_color, dtype = torch.float, device= "cpu")- if any_mask:- src_mask = torch.zeros_like(src_video[0:1])- elif src_video.shape[1] < current_video_length:- if any_guide_padding:- src_video = torch.cat([src_video, torch.full( (3, current_video_length - src_video.shape[1], *src_video.shape[-2:] ), inpaint_color, dtype = src_video.dtype, device= src_video.device) ], dim=1)- if cur_video_mask is not None and any_mask:- src_mask = torch.cat([src_mask, torch.full( (1, current_video_length - src_mask.shape[1], *src_mask.shape[-2:] ), 1, dtype = src_video.dtype, device= src_video.device) ], dim=1)- else:- new_num_frames = (src_video.shape[1] - 1) // latent_size * latent_size + 1- src_video = src_video[:, :new_num_frames]- if any_mask:- src_mask = src_mask[:, :new_num_frames]-- for k, keep in enumerate(keep_video_guide_frames):- if not keep:- pos = prepend_count + k- src_video[:, pos:pos+1] = inpaint_color- src_mask[:, pos:pos+1] = 1-- for k, frame in enumerate(inject_frames):- if frame != None:- pos = prepend_count + k- src_video[:, pos:pos+1], src_mask[:, pos:pos+1] = fit_image_into_canvas(frame, image_size, inpaint_color, device, True, outpainting_dims, return_mask= True)+ if any_guide_padding:+ if src_video is None:+ src_video = torch.full( (3, current_video_length, *image_size ), inpaint_color_compressed, dtype = torch.float, device= device)+ elif src_video.shape[1] < current_video_length:+ src_video = torch.cat([src_video, torch.full( (3, current_video_length - src_video.shape[1], *src_video.shape[-2:] ), inpaint_color_compressed, dtype = src_video.dtype, device= src_video.device) ], dim=1)+ elif src_video is not None:+ new_num_frames = (src_video.shape[1] - 1) // latent_size * latent_size + 1+ src_video = src_video[:, :new_num_frames]++ if any_mask and src_video is not None:+ if src_mask is None:+ src_mask = torch.ones_like(src_video[:1])+ elif src_mask.shape[1] < src_video.shape[1]:+ src_mask = torch.cat([src_mask, torch.full( (1, src_video.shape[1]- src_mask.shape[1], *src_mask.shape[-2:] ), 1, dtype = src_video.dtype, device= src_video.device) ], dim=1)+ else:+ src_mask = src_mask[:, :src_video.shape[1]]++ if src_video is not None :+ for k, keep in enumerate(keep_video_guide_frames):+ if not keep:+ pos = prepend_count + k+ src_video[:, pos:pos+1] = inpaint_color_compressed+ if any_mask: src_mask[:, pos:pos+1] = 1++ for k, frame in enumerate(inject_frames):+ if frame != None:+ pos = prepend_count + k+ src_video[:, pos:pos+1], msk = fit_image_into_canvas(frame, image_size, guide_inpaint_color, device, True, outpainting_dims, return_mask= any_mask)+ if any_mask: src_mask[:, pos:pos+1] = msksrc_videos.append(src_video)src_masks.append(src_mask)return src_videos, src_masksdiff --git a/wgp.py b/wgp.pyindex 0e6a0b2..6a7bca1 100644--- a/wgp.py+++ b/wgp.py@@ -24,6 +24,7 @@ from shared.utils import notification_soundfrom shared.utils.loras_mutipliers import preparse_loras_multipliers, parse_loras_multipliersfrom shared.utils.utils import convert_tensor_to_image, save_image, get_video_info, get_file_creation_date, convert_image_to_video, calculate_new_dimensions, convert_image_to_tensor, calculate_dimensions_and_resize_image, rescale_and_crop, get_video_frame, resize_and_remove_background, rgb_bw_to_rgba_maskfrom shared.utils.utils import calculate_new_dimensions, get_outpainting_frame_location, get_outpainting_full_area_dimensions+from shared.utils.utils import has_video_file_extension, has_image_file_extensionfrom shared.utils.audio_video import extract_audio_tracks, combine_video_with_audio_tracks, combine_and_concatenate_video_with_audio_tracks, cleanup_temp_audio_files, save_video, save_imagefrom shared.utils.audio_video import save_image_metadata, read_image_metadatafrom shared.match_archi import match_nvidia_architecture@@ -62,7 +63,7 @@ AUTOSAVE_FILENAME = "queue.zip"PROMPT_VARS_MAX = 10target_mmgp_version = "3.6.0"-WanGP_version = "8.61"+WanGP_version = "8.7"settings_version = 2.35max_source_video_frames = 3000prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None@@ -1942,7 +1943,8 @@ def get_model_min_frames_and_step(model_type):mode_def = get_model_def(model_type)frames_minimum = mode_def.get("frames_minimum", 5)frames_steps = mode_def.get("frames_steps", 4)- return frames_minimum, frames_steps+ latent_size = mode_def.get("latent_size", frames_steps)+ return frames_minimum, frames_steps, latent_sizedef get_model_fps(model_type):mode_def = get_model_def(model_type)@@ -3459,7 +3461,7 @@ def select_video(state, input_file_list, event_data: gr.EventData):if len(video_other_prompts) >0 :values += [video_other_prompts]labels += ["Other Prompts"]- if len(video_outpainting) >0 and any_letters(video_image_prompt_type, "VFK"):+ if len(video_outpainting) >0:values += [video_outpainting]labels += ["Outpainting"]video_sample_solver = configs.get("sample_solver", "")@@ -3532,6 +3534,11 @@ def convert_image(image):return cast(Image, ImageOps.exif_transpose(image))def get_resampled_video(video_in, start_frame, max_frames, target_fps, bridge='torch'):+ if isinstance(video_in, str) and has_image_file_extension(video_in):+ video_in = Image.open(video_in)+ if isinstance(video_in, Image.Image):+ return torch.from_numpy(np.array(video_in).astype(np.uint8)).unsqueeze(0)+from shared.utils.utils import resampleimport decord@@ -3653,19 +3660,22 @@ def get_preprocessor(process_type, inpaint_color):def process_images_multithread(image_processor, items, process_type, wrap_in_list = True, max_workers: int = os.cpu_count()/ 2) :if not items:return []- max_workers = 11+import concurrent.futuresstart_time = time.time()# print(f"Preprocessus:{process_type} started")if process_type in ["prephase", "upsample"]:if wrap_in_list :items = [ [img] for img in items]- with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:- futures = {executor.submit(image_processor, img): idx for idx, img in enumerate(items)}- results = [None] * len(items)- for future in concurrent.futures.as_completed(futures):- idx = futures[future]- results[idx] = future.result()+ if max_workers == 1:+ results = [image_processor(img) for img in items]+ else:+ with concurrent.futures.ThreadPoolExecutor(max_workers=max_workers) as executor:+ futures = {executor.submit(image_processor, img): idx for idx, img in enumerate(items)}+ results = [None] * len(items)+ for future in concurrent.futures.as_completed(futures):+ idx = futures[future]+ results[idx] = future.result()if wrap_in_list:results = [ img[0] for img in results]@@ -3677,55 +3687,6 @@ def process_images_multithread(image_processor, items, process_type, wrap_in_lisreturn results-def preprocess_image_with_mask(input_image, input_mask, height, width, fit_canvas = False, fit_crop = False, block_size= 16, expand_scale = 2, outpainting_dims = None, inpaint_color = 127):- frame_width, frame_height = input_image.size-- if fit_crop:- input_image = rescale_and_crop(input_image, width, height)- if input_mask is not None:- input_mask = rescale_and_crop(input_mask, width, height)- return input_image, input_mask-- if outpainting_dims != None:- if fit_canvas != None:- frame_height, frame_width = get_outpainting_full_area_dimensions(frame_height,frame_width, outpainting_dims)- else:- frame_height, frame_width = height, width-- if fit_canvas != None:- height, width = calculate_new_dimensions(height, width, frame_height, frame_width, fit_into_canvas = fit_canvas, block_size = block_size)-- if outpainting_dims != None:- final_height, final_width = height, width- height, width, margin_top, margin_left = get_outpainting_frame_location(final_height, final_width, outpainting_dims, 1)-- if fit_canvas != None or outpainting_dims != None:- input_image = input_image.resize((width, height), resample=Image.Resampling.LANCZOS)- if input_mask is not None:- input_mask = input_mask.resize((width, height), resample=Image.Resampling.LANCZOS)-- if expand_scale != 0 and input_mask is not None:- kernel_size = abs(expand_scale)- kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))- op_expand = cv2.dilate if expand_scale > 0 else cv2.erode- input_mask = np.array(input_mask)- input_mask = op_expand(input_mask, kernel, iterations=3)- input_mask = Image.fromarray(input_mask)-- if outpainting_dims != None:- inpaint_color = inpaint_color / 127.5-1- image = convert_image_to_tensor(input_image)- full_frame= torch.full( (image.shape[0], final_height, final_width), inpaint_color, dtype= torch.float, device= image.device)- full_frame[:, margin_top:margin_top+height, margin_left:margin_left+width] = image- input_image = convert_tensor_to_image(full_frame)-- if input_mask is not None:- mask = convert_image_to_tensor(input_mask)- full_frame= torch.full( (mask.shape[0], final_height, final_width), 1, dtype= torch.float, device= mask.device)- full_frame[:, margin_top:margin_top+height, margin_left:margin_left+width] = mask- input_mask = convert_tensor_to_image(full_frame)-- return input_image, input_maskdef extract_faces_from_video_with_mask(input_video_path, input_mask_path, max_frames, start_frame, target_fps, size = 512):if not input_video_path or max_frames <= 0:return None, None@@ -3780,6 +3741,8 @@ def extract_faces_from_video_with_mask(input_video_path, input_mask_path, max_frsave_one_video(f"faces.mp4", saved_faces_frames, fps=target_fps, quality=8, macro_block_size=None)return face_tensor+def get_default_workers():+ return os.cpu_count()/ 2def preprocess_video_with_mask(input_video_path, input_mask_path, height, width, max_frames, start_frame=0, fit_canvas = None, fit_crop = False, target_fps = 16, block_size= 16, expand_scale = 2, process_type = "inpaint", process_type2 = None, to_bbox = False, RGB_Mask = False, negate_mask = False, process_outside_mask = None, inpaint_color = 127, outpainting_dims = None, proc_no = 1):@@ -3906,8 +3869,8 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,return (target_frame, frame, mask)else:return (target_frame, None, None)-- proc_lists = process_images_multithread(prep_prephase, [frame_idx for frame_idx in range(num_frames)], "prephase", wrap_in_list= False)+ max_workers = get_default_workers()+ proc_lists = process_images_multithread(prep_prephase, [frame_idx for frame_idx in range(num_frames)], "prephase", wrap_in_list= False, max_workers=max_workers)proc_list, proc_list_outside, proc_mask = [None] * len(proc_lists), [None] * len(proc_lists), [None] * len(proc_lists)for frame_idx, frame_group in enumerate(proc_lists):proc_list[frame_idx], proc_list_outside[frame_idx], proc_mask[frame_idx] = frame_group@@ -3916,11 +3879,11 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,mask_video = Noneif preproc2 != None:- proc_list2 = process_images_multithread(preproc2, proc_list, process_type2)+ proc_list2 = process_images_multithread(preproc2, proc_list, process_type2, max_workers=max_workers)#### to be finished ...or not- proc_list = process_images_multithread(preproc, proc_list, process_type)+ proc_list = process_images_multithread(preproc, proc_list, process_type, max_workers=max_workers)if any_mask:- proc_list_outside = process_images_multithread(preproc_outside, proc_list_outside, process_outside_mask)+ proc_list_outside = process_images_multithread(preproc_outside, proc_list_outside, process_outside_mask, max_workers=max_workers)else:proc_list_outside = proc_mask = len(proc_list) * [None]@@ -3938,7 +3901,7 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,full_frame= torch.full( (final_height, final_width, mask.shape[-1]), 255, dtype= torch.uint8, device= mask.device)full_frame[margin_top:margin_top+height, margin_left:margin_left+width] = maskmask = full_frame- masks.append(mask)+ masks.append(mask[:, :, 0:1].clone())else:masked_frame = processed_img@@ -3958,13 +3921,13 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,proc_list[frame_no] = proc_list_outside[frame_no] = proc_mask[frame_no] = None- if args.save_masks:- from preprocessing.dwpose.pose import save_one_video- saved_masked_frames = [mask.cpu().numpy() for mask in masked_frames ]- save_one_video(f"masked_frames{'' if proc_no==1 else str(proc_no)}.mp4", saved_masked_frames, fps=target_fps, quality=8, macro_block_size=None)- if any_mask:- saved_masks = [mask.cpu().numpy() for mask in masks ]- save_one_video("masks.mp4", saved_masks, fps=target_fps, quality=8, macro_block_size=None)+ # if args.save_masks:+ # from preprocessing.dwpose.pose import save_one_video+ # saved_masked_frames = [mask.cpu().numpy() for mask in masked_frames ]+ # save_one_video(f"masked_frames{'' if proc_no==1 else str(proc_no)}.mp4", saved_masked_frames, fps=target_fps, quality=8, macro_block_size=None)+ # if any_mask:+ # saved_masks = [mask.cpu().numpy() for mask in masks ]+ # save_one_video("masks.mp4", saved_masks, fps=target_fps, quality=8, macro_block_size=None)preproc = Nonepreproc_outside = Nonegc.collect()@@ -3972,8 +3935,10 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,if pad_frames > 0:masked_frames = masked_frames[0] * pad_frames + masked_framesif any_mask: masked_frames = masks[0] * pad_frames + masks+ masked_frames = torch.stack(masked_frames).permute(-1,0,1,2).float().div_(127.5).sub_(1.)+ masks = torch.stack(masks).permute(-1,0,1,2).float().div_(255) if any_mask else None- return torch.stack(masked_frames), torch.stack(masks) if any_mask else None+ return masked_frames, masksdef preprocess_video(height, width, video_in, max_frames, start_frame=0, fit_canvas = None, fit_crop = False, target_fps = 16, block_size = 16):@@ -4102,7 +4067,7 @@ def perform_spatial_upsampling(sample, spatial_upsampling):frames_to_upsample = [sample[:, i] for i in range( sample.shape[1]) ]def upsample_frames(frame):return resize_lanczos(frame, h, w).unsqueeze(1)- sample = torch.cat(process_images_multithread(upsample_frames, frames_to_upsample, "upsample", wrap_in_list = False), dim=1)+ sample = torch.cat(process_images_multithread(upsample_frames, frames_to_upsample, "upsample", wrap_in_list = False, max_workers=get_default_workers()), dim=1)frames_to_upsample = Nonereturn sample@@ -4609,17 +4574,13 @@ def generate_video(batch_size = 1temp_filenames_list = []- convert_image_guide_to_video = model_def.get("convert_image_guide_to_video", False)- if convert_image_guide_to_video:- if image_guide is not None and isinstance(image_guide, Image.Image):- video_guide = convert_image_to_video(image_guide)- temp_filenames_list.append(video_guide)- image_guide = None+ if image_guide is not None and isinstance(image_guide, Image.Image):+ video_guide = image_guide+ image_guide = None- if image_mask is not None and isinstance(image_mask, Image.Image):- video_mask = convert_image_to_video(image_mask)- temp_filenames_list.append(video_mask)- image_mask = None+ if image_mask is not None and isinstance(image_mask, Image.Image):+ video_mask = image_mask+ image_mask = Noneif model_def.get("no_background_removal", False): remove_background_images_ref = 0@@ -4711,22 +4672,12 @@ def generate_video(device_mem_capacity = torch.cuda.get_device_properties(None).total_memory / 1048576guide_inpaint_color = model_def.get("guide_inpaint_color", 127.5)extract_guide_from_window_start = model_def.get("extract_guide_from_window_start", False)- i2v = test_class_i2v(model_type)- diffusion_forcing = "diffusion_forcing" in model_filename- t2v = base_model_type in ["t2v"]- ltxv = "ltxv" in model_filename- vace = test_vace_module(base_model_type)- hunyuan_t2v = "hunyuan_video_720" in model_filename- hunyuan_i2v = "hunyuan_video_i2v" in model_filenamehunyuan_custom = "hunyuan_video_custom" in model_filenamehunyuan_custom_audio = hunyuan_custom and "audio" in model_filenamehunyuan_custom_edit = hunyuan_custom and "edit" in model_filenamehunyuan_avatar = "hunyuan_video_avatar" in model_filenamefantasy = base_model_type in ["fantasy"]multitalk = model_def.get("multitalk_class", False)- standin = model_def.get("standin_class", False)- infinitetalk = base_model_type in ["infinitetalk"]- animate = base_model_type in ["animate"]if "B" in audio_prompt_type or "X" in audio_prompt_type:from models.wan.multitalk.multitalk import parse_speakers_locations@@ -4763,9 +4714,9 @@ def generate_video(sliding_window_size = current_video_lengthreuse_frames = 0- _, latent_size = get_model_min_frames_and_step(model_type)- if diffusion_forcing: latent_size = 4+ _, _, latent_size = get_model_min_frames_and_step(model_type)original_image_refs = image_refs+ image_refs = None if image_refs is None else [] + image_refs # work on a copy as it is going to be modified# image_refs = None# nb_frames_positions= 0# Output Video Ratio Priorities:@@ -4889,6 +4840,7 @@ def generate_video(initial_total_windows = 0discard_last_frames = sliding_window_discard_last_framesdefault_requested_frames_to_generate = current_video_length+ nb_frames_positions = 0if sliding_window:initial_total_windows= compute_sliding_window_no(default_requested_frames_to_generate, sliding_window_size, discard_last_frames, reuse_frames)current_video_length = sliding_window_size@@ -4907,7 +4859,7 @@ def generate_video(if repeat_no >= total_generation: breakrepeat_no +=1gen["repeat_no"] = repeat_no- src_video = src_mask = src_ref_images = new_image_guide = new_image_mask = src_faces = None+ src_video = src_video2 = src_mask = src_mask2 = src_faces = src_ref_images = src_ref_masks = Noneprefix_video = pre_video_frame = Nonesource_video_overlap_frames_count = 0 # number of frames overalapped in source video for first windowsource_video_frames_count = 0 # number of frames to use in source video (processing starts source_video_overlap_frames_count frames before )@@ -4963,7 +4915,6 @@ def generate_video(return_latent_slice = slice(-(reuse_frames - 1 + discard_last_frames ) // latent_size - 1, None if discard_last_frames == 0 else -(discard_last_frames // latent_size) )refresh_preview = {"image_guide" : image_guide, "image_mask" : image_mask} if image_mode >= 1 else {}- src_ref_images, src_ref_masks = image_refs, Noneimage_start_tensor = image_end_tensor = Noneif window_no == 1 and (video_source is not None or image_start is not None):if image_start is not None:@@ -5020,7 +4971,7 @@ def generate_video(if len(pos) > 0:if pos in ["L", "l"]:cur_end_pos += sliding_window_size if project_window_no > 1 else current_video_length- if cur_end_pos >= last_frame_no and not joker_used:+ if cur_end_pos >= last_frame_no-1 and not joker_used:joker_used = Truecur_end_pos = last_frame_no -1project_window_no += 1@@ -5036,141 +4987,53 @@ def generate_video(frames_to_inject[pos] = image_refs[i]+ video_guide_processed = video_mask_processed = video_guide_processed2 = video_mask_processed2 = Noneif video_guide is not None:keep_frames_parsed_full, error = parse_keep_frames_video_guide(keep_frames_video_guide, source_video_frames_count -source_video_overlap_frames_count + requested_frames_to_generate)if len(error) > 0:raise gr.Error(f"invalid keep frames {keep_frames_video_guide}")guide_frames_extract_start = aligned_window_start_frame if extract_guide_from_window_start else aligned_guide_start_frame+ extra_control_frames = model_def.get("extra_control_frames", 0)+ if extra_control_frames > 0 and aligned_guide_start_frame >= extra_control_frames: guide_frames_extract_start -= extra_control_frames+keep_frames_parsed = [True] * -guide_frames_extract_start if guide_frames_extract_start <0 else []keep_frames_parsed += keep_frames_parsed_full[max(0, guide_frames_extract_start): aligned_guide_end_frame ]guide_frames_extract_count = len(keep_frames_parsed)+ # Extract Faces to videoif "B" in video_prompt_type:send_cmd("progress", [0, get_latest_status(state, "Extracting Face Movements")])src_faces = extract_faces_from_video_with_mask(video_guide, video_mask, max_frames= guide_frames_extract_count, start_frame= guide_frames_extract_start, size= 512, target_fps = fps)if src_faces is not None and src_faces.shape[1] < current_video_length:src_faces = torch.cat([src_faces, torch.full( (3, current_video_length - src_faces.shape[1], 512, 512 ), -1, dtype = src_faces.dtype, device= src_faces.device) ], dim=1)- if vace or animate:- video_guide_processed = video_mask_processed = video_guide_processed2 = video_mask_processed2 = None- context_scale = [ control_net_weight]- if "V" in video_prompt_type:- process_outside_mask = process_map_outside_mask.get(filter_letters(video_prompt_type, "YWX"), None)- preprocess_type, preprocess_type2 = "raw", None- for process_num, process_letter in enumerate( filter_letters(video_prompt_type, "PEDSLCMU")):- if process_num == 0:- preprocess_type = process_map_video_guide.get(process_letter, "raw")- else:- preprocess_type2 = process_map_video_guide.get(process_letter, None)- status_info = "Extracting " + processes_names[preprocess_type]- extra_process_list = ([] if preprocess_type2==None else [preprocess_type2]) + ([] if process_outside_mask==None or process_outside_mask == preprocess_type else [process_outside_mask])- if len(extra_process_list) == 1:- status_info += " and " + processes_names[extra_process_list[0]]- elif len(extra_process_list) == 2:- status_info += ", " + processes_names[extra_process_list[0]] + " and " + processes_names[extra_process_list[1]]- if preprocess_type2 is not None:- context_scale = [ control_net_weight /2, control_net_weight2 /2]- send_cmd("progress", [0, get_latest_status(state, status_info)])- inpaint_color = 0 if preprocess_type=="pose" else guide_inpaint_color- video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color )- if preprocess_type2 != None:- video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2 )-- if video_guide_processed != None:- if sample_fit_canvas != None:- image_size = video_guide_processed.shape[-3: -1]- sample_fit_canvas = None- refresh_preview["video_guide"] = Image.fromarray(video_guide_processed[0].cpu().numpy())- if video_guide_processed2 != None:- refresh_preview["video_guide"] = [refresh_preview["video_guide"], Image.fromarray(video_guide_processed2[0].cpu().numpy())]- if video_mask_processed != None:- refresh_preview["video_mask"] = Image.fromarray(video_mask_processed[0].cpu().numpy())-- frames_to_inject_parsed = frames_to_inject[ window_start_frame if extract_guide_from_window_start else guide_start_frame: guide_end_frame]-- if not vace and (any_letters(video_prompt_type ,"FV") or model_def.get("forced_guide_mask_inputs", False)):- any_mask = True- any_guide_padding = model_def.get("pad_guide_video", False)- from shared.utils.utils import prepare_video_guide_and_mask- src_videos, src_masks = prepare_video_guide_and_mask( [video_guide_processed, video_guide_processed2],- [video_mask_processed, video_mask_processed2],- pre_video_guide, image_size, current_video_length, latent_size,- any_mask, any_guide_padding, guide_inpaint_color, extract_guide_from_window_start,- keep_frames_parsed, frames_to_inject_parsed , outpainting_dims)-- src_video, src_video2 = src_videos- src_mask, src_mask2 = src_masks- if src_video is None:- abort = True- break- if src_faces is not None:- if src_faces.shape[1] < src_video.shape[1]:- src_faces = torch.concat( [src_faces, src_faces[:, -1:].repeat(1, src_video.shape[1] - src_faces.shape[1], 1,1)], dim =1)- else:- src_faces = src_faces[:, :src_video.shape[1]]- if args.save_masks:- save_video( src_video, "masked_frames.mp4", fps)- if src_video2 is not None:- save_video( src_video2, "masked_frames2.mp4", fps)- if any_mask:- save_video( src_mask, "masks.mp4", fps, value_range=(0, 1))-- elif ltxv:- preprocess_type = process_map_video_guide.get(filter_letters(video_prompt_type, "PED"), "raw")- status_info = "Extracting " + processes_names[preprocess_type]- send_cmd("progress", [0, get_latest_status(state, status_info)])- # start one frame ealier to facilitate latents merging later- src_video, _ = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= len(keep_frames_parsed) + (0 if aligned_guide_start_frame == 0 else 1), start_frame = aligned_guide_start_frame - (0 if aligned_guide_start_frame == 0 else 1), fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, inpaint_color = 0, proc_no =1, negate_mask = "N" in video_prompt_type, process_outside_mask = "inpaint" if "X" in video_prompt_type else "identity", block_size =block_size )- if src_video != None:- src_video = src_video[ :(len(src_video)-1)// latent_size * latent_size +1 ]- refresh_preview["video_guide"] = Image.fromarray(src_video[0].cpu().numpy())- refresh_preview["video_mask"] = None- src_video = src_video.permute(3, 0, 1, 2)- src_video = src_video.float().div_(127.5).sub_(1.) # c, f, h, w- if sample_fit_canvas != None:- image_size = src_video.shape[-2:]- sample_fit_canvas = None-- elif hunyuan_custom_edit:- if "P" in video_prompt_type:- progress_args = [0, get_latest_status(state,"Extracting Open Pose Information and Expanding Mask")]- else:- progress_args = [0, get_latest_status(state,"Extracting Video and Mask")]-- send_cmd("progress", progress_args)- src_video, src_mask = preprocess_video_with_mask(video_guide, video_mask, height=height, width = width, max_frames= current_video_length if window_no == 1 else current_video_length - reuse_frames, start_frame = guide_start_frame, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type= "pose" if "P" in video_prompt_type else "inpaint", negate_mask = "N" in video_prompt_type, inpaint_color =0)- refresh_preview["video_guide"] = Image.fromarray(src_video[0].cpu().numpy())- if src_mask != None:- refresh_preview["video_mask"] = Image.fromarray(src_mask[0].cpu().numpy())-- elif "R" in video_prompt_type: # sparse video to video- src_image = get_video_frame(video_guide, aligned_guide_start_frame, return_last_if_missing = True, return_PIL = True)- src_image, _, _ = calculate_dimensions_and_resize_image(src_image, image_size[0], image_size[1 ], sample_fit_canvas, fit_crop, block_size = block_size)- refresh_preview["video_guide"] = src_image- src_video = convert_image_to_tensor(src_image).unsqueeze(1)- if sample_fit_canvas != None:- image_size = src_video.shape[-2:]- sample_fit_canvas = None-- else: # video to video- video_guide_processed = preprocess_video(width = image_size[1], height=image_size[0], video_in=video_guide, max_frames= len(keep_frames_parsed), start_frame = aligned_guide_start_frame, fit_canvas= sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, block_size= block_size)- if video_guide_processed is None:- src_video = pre_video_guide+ # Sparse Video to Video+ sparse_video_image = get_video_frame(video_guide, aligned_guide_start_frame, return_last_if_missing = True, target_fps = fps, return_PIL = True) if "R" in video_prompt_type else None++ # Generic Video Preprocessing+ process_outside_mask = process_map_outside_mask.get(filter_letters(video_prompt_type, "YWX"), None)+ preprocess_type, preprocess_type2 = "raw", None+ for process_num, process_letter in enumerate( filter_letters(video_prompt_type, "PEDSLCMU")):+ if process_num == 0:+ preprocess_type = process_map_video_guide.get(process_letter, "raw")else:- if sample_fit_canvas != None:- image_size = video_guide_processed.shape[-3: -1]- sample_fit_canvas = None- src_video = video_guide_processed.float().div_(127.5).sub_(1.).permute(-1,0,1,2)- if pre_video_guide != None:- src_video = torch.cat( [pre_video_guide, src_video], dim=1)- elif image_guide is not None:- new_image_guide, new_image_mask = preprocess_image_with_mask(image_guide, image_mask, image_size[0], image_size[1], fit_canvas = sample_fit_canvas, fit_crop= fit_crop, block_size= block_size, expand_scale = mask_expand, outpainting_dims=outpainting_dims)- if sample_fit_canvas is not None:- image_size = (new_image_guide.size[1], new_image_guide.size[0])+ preprocess_type2 = process_map_video_guide.get(process_letter, None)+ status_info = "Extracting " + processes_names[preprocess_type]+ extra_process_list = ([] if preprocess_type2==None else [preprocess_type2]) + ([] if process_outside_mask==None or process_outside_mask == preprocess_type else [process_outside_mask])+ if len(extra_process_list) == 1:+ status_info += " and " + processes_names[extra_process_list[0]]+ elif len(extra_process_list) == 2:+ status_info += ", " + processes_names[extra_process_list[0]] + " and " + processes_names[extra_process_list[1]]+ context_scale = [control_net_weight /2, control_net_weight2 /2] if preprocess_type2 is not None else [control_net_weight]+ if not (preprocess_type == "identity" and preprocess_type2 is None and video_mask is None):send_cmd("progress", [0, get_latest_status(state, status_info)])+ inpaint_color = 0 if preprocess_type=="pose" and process_outside_mask == "inpaint" else guide_inpaint_color+ video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size )+ if preprocess_type2 != None:+ video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size )++ if video_guide_processed is not None and sample_fit_canvas is not None:+ image_size = video_guide_processed.shape[-2:]sample_fit_canvas = None- refresh_preview["image_guide"] = new_image_guide- if new_image_mask is not None:- refresh_preview["image_mask"] = new_image_maskif window_no == 1 and image_refs is not None and len(image_refs) > 0:if sample_fit_canvas is not None and (nb_frames_positions > 0 or "K" in video_prompt_type) :@@ -5192,45 +5055,68 @@ def generate_video(image_refs[i] = rescale_and_crop(img, default_image_size[1], default_image_size[0])refresh_preview["image_refs"] = image_refs- if len(image_refs) > nb_frames_positions:+ if len(image_refs) > nb_frames_positions:+ src_ref_images = image_refs[nb_frames_positions:]if remove_background_images_ref > 0:send_cmd("progress", [0, get_latest_status(state, "Removing Images References Background")])- # keep image ratios if there is a background image ref (we will let the model preprocessor decide what to do) but remove bg if requested- image_refs[nb_frames_positions:], src_ref_masks = resize_and_remove_background(image_refs[nb_frames_positions:] , image_size[1], image_size[0],++ src_ref_images, src_ref_masks = resize_and_remove_background(src_ref_images , image_size[1], image_size[0],remove_background_images_ref > 0, any_background_ref,- fit_into_canvas= 0 if (any_background_ref > 0 or model_def.get("lock_image_refs_ratios", False)) else 1,+ fit_into_canvas= model_def.get("fit_into_canvas_image_refs", 1),block_size=block_size,outpainting_dims =outpainting_dims,- background_ref_outpainted = model_def.get("background_ref_outpainted", True) )- refresh_preview["image_refs"] = image_refs-+ background_ref_outpainted = model_def.get("background_ref_outpainted", True),+ return_tensor= model_def.get("return_image_refs_tensor", False) )+- if vace :- image_refs_copy = image_refs[nb_frames_positions:].copy() if image_refs != None and len(image_refs) > nb_frames_positions else None # required since prepare_source do inplace modifications-- src_video, src_mask, src_ref_images = wan_model.prepare_source([video_guide_processed] if video_guide_processed2 == None else [video_guide_processed, video_guide_processed2],- [video_mask_processed] if video_guide_processed2 == None else [video_mask_processed, video_mask_processed2],- [image_refs_copy] if video_guide_processed2 == None else [image_refs_copy, image_refs_copy],- current_video_length, image_size = image_size, device ="cpu",- keep_video_guide_frames=keep_frames_parsed,- pre_src_video = [pre_video_guide] if video_guide_processed2 == None else [pre_video_guide, pre_video_guide],- inject_frames= frames_to_inject_parsed,- outpainting_dims = outpainting_dims,- any_background_ref = any_background_ref- )- if len(frames_to_inject_parsed) or any_background_ref:- new_image_refs = [convert_tensor_to_image(src_video[0], frame_no + 0 if extract_guide_from_window_start else (aligned_guide_start_frame - aligned_window_start_frame) ) for frame_no, inject in enumerate(frames_to_inject_parsed) if inject]- if any_background_ref:- new_image_refs += [convert_tensor_to_image(image_refs_copy[0], 0)] + image_refs[nb_frames_positions+1:]+ frames_to_inject_parsed = frames_to_inject[ window_start_frame if extract_guide_from_window_start else guide_start_frame: guide_end_frame]+ if video_guide is not None or len(frames_to_inject_parsed) > 0 or model_def.get("forced_guide_mask_inputs", False):+ any_mask = video_mask is not None or model_def.get("forced_guide_mask_inputs", False)+ any_guide_padding = model_def.get("pad_guide_video", False)+ from shared.utils.utils import prepare_video_guide_and_mask+ src_videos, src_masks = prepare_video_guide_and_mask( [video_guide_processed] + ([] if video_guide_processed2 is None else [video_guide_processed2]),+ [video_mask_processed] + ([] if video_mask_processed2 is None else [video_mask_processed2]),+ None if extract_guide_from_window_start or model_def.get("dont_cat_preguide", False) or sparse_video_image is not None else pre_video_guide,+ image_size, current_video_length, latent_size,+ any_mask, any_guide_padding, guide_inpaint_color,+ keep_frames_parsed, frames_to_inject_parsed , outpainting_dims)+ video_guide_processed = video_guide_processed2 = video_mask_processed = video_mask_processed2 = None+ if len(src_videos) == 1:+ src_video, src_video2, src_mask, src_mask2 = src_videos[0], None, src_masks[0], None+ else:+ src_video, src_video2 = src_videos+ src_mask, src_mask2 = src_masks+ src_videos = src_masks = None+ if src_video is None:+ abort = True+ break+ if src_faces is not None:+ if src_faces.shape[1] < src_video.shape[1]:+ src_faces = torch.concat( [src_faces, src_faces[:, -1:].repeat(1, src_video.shape[1] - src_faces.shape[1], 1,1)], dim =1)else:- new_image_refs += image_refs[nb_frames_positions:]- refresh_preview["image_refs"] = new_image_refs- new_image_refs = None-- if sample_fit_canvas != None:- image_size = src_video[0].shape[-2:]- sample_fit_canvas = None-+ src_faces = src_faces[:, :src_video.shape[1]]+ if video_guide is not None or len(frames_to_inject_parsed) > 0:+ if args.save_masks:+ if src_video is not None: save_video( src_video, "masked_frames.mp4", fps)+ if src_video2 is not None: save_video( src_video2, "masked_frames2.mp4", fps)+ if any_mask: save_video( src_mask, "masks.mp4", fps, value_range=(0, 1))+ if video_guide is not None:+ preview_frame_no = 0 if extract_guide_from_window_start or model_def.get("dont_cat_preguide", False) or sparse_video_image is not None else (guide_start_frame - window_start_frame)+ refresh_preview["video_guide"] = convert_tensor_to_image(src_video, preview_frame_no)+ if src_video2 is not None:+ refresh_preview["video_guide"] = [refresh_preview["video_guide"], convert_tensor_to_image(src_video2, preview_frame_no)]+ if src_mask is not None and video_mask is not None:+ refresh_preview["video_mask"] = convert_tensor_to_image(src_mask, preview_frame_no, mask_levels = True)++ if src_ref_images is not None or nb_frames_positions:+ if len(frames_to_inject_parsed):+ new_image_refs = [convert_tensor_to_image(src_video, frame_no + (0 if extract_guide_from_window_start else (aligned_guide_start_frame - aligned_window_start_frame)) ) for frame_no, inject in enumerate(frames_to_inject_parsed) if inject]+ else:+ new_image_refs = []+ if src_ref_images is not None:+ new_image_refs += [convert_tensor_to_image(img) if torch.is_tensor(img) else img for img in src_ref_images ]+ refresh_preview["image_refs"] = new_image_refs+ new_image_refs = Noneif len(refresh_preview) > 0:new_inputs= locals()@@ -5339,8 +5225,6 @@ def generate_video(pre_video_frame = pre_video_frame,original_input_ref_images = original_image_refs[nb_frames_positions:] if original_image_refs is not None else [],image_refs_relative_size = image_refs_relative_size,- image_guide= new_image_guide,- image_mask= new_image_mask,outpainting_dims = outpainting_dims,)except Exception as e:@@ -6320,8 +6204,11 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Nonepop += ["image_refs_relative_size"]if not vace:- pop += ["frames_positions", "video_guide_outpainting", "control_net_weight", "control_net_weight2"]+ pop += ["frames_positions", "control_net_weight", "control_net_weight2"]+ if model_def.get("video_guide_outpainting", None) is None:+ pop += ["video_guide_outpainting"]+if not (vace or t2v):pop += ["min_frames_if_references"]@@ -6506,13 +6393,6 @@ def eject_video_from_gallery(state, input_file_list, choice):choice = min(choice, len(file_list))return gr.Gallery(value = file_list, selected_index= choice), gr.update() if len(file_list) >0 else get_default_video_info(), gr.Row(visible= len(file_list) > 0)-def has_video_file_extension(filename):- extension = os.path.splitext(filename)[-1].lower()- return extension in [".mp4"]--def has_image_file_extension(filename):- extension = os.path.splitext(filename)[-1].lower()- return extension in [".png", ".jpg", ".jpeg", ".bmp", ".gif", ".webp", ".tif", ".tiff", ".jfif", ".pjpeg"]def add_videos_to_gallery(state, input_file_list, choice, files_to_load):gen = get_gen_info(state)if files_to_load == None:@@ -7881,7 +7761,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonelif recammaster:video_length = gr.Slider(5, 193, value=ui_defaults.get("video_length", get_max_frames(81)), step=4, label="Number of frames (16 = 1s), locked", interactive= False, visible = True)else:- min_frames, frames_step = get_model_min_frames_and_step(base_model_type)+ min_frames, frames_step, _ = get_model_min_frames_and_step(base_model_type)current_video_length = ui_defaults.get("video_length", 81 if get_model_family(base_model_type)=="wan" else 97)@@ -8059,7 +7939,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = NonMMAudio_neg_prompt = gr.Text(ui_defaults.get("MMAudio_neg_prompt", ""), label="Negative Prompt (1 or 2 keywords)")- with gr.Column(visible = (t2v or vace) and not fantasy) as audio_prompt_type_remux_row:+ with gr.Column(visible = any_control_video) as audio_prompt_type_remux_row:gr.Markdown("<B>You may transfer the existing audio tracks of a Control Video</B>")audio_prompt_type_remux = gr.Dropdown(choices=[@@ -8284,16 +8164,16 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonvideo_info = gr.HTML(visible=True, min_height=100, value=get_default_video_info())with gr.Row(**default_visibility) as video_buttons_row:video_info_extract_settings_btn = gr.Button("Extract Settings", min_width= 1, size ="sm")- video_info_to_control_video_btn = gr.Button("To Control Video", min_width= 1, size ="sm", visible = any_control_video )video_info_to_video_source_btn = gr.Button("To Video Source", min_width= 1, size ="sm", visible = any_video_source)+ video_info_to_control_video_btn = gr.Button("To Control Video", min_width= 1, size ="sm", visible = any_control_video )video_info_eject_video_btn = gr.Button("Eject Video", min_width= 1, size ="sm")with gr.Row(**default_visibility) as image_buttons_row:video_info_extract_image_settings_btn = gr.Button("Extract Settings", min_width= 1, size ="sm")video_info_to_start_image_btn = gr.Button("To Start Image", size ="sm", min_width= 1, visible = any_start_image )video_info_to_end_image_btn = gr.Button("To End Image", size ="sm", min_width= 1, visible = any_end_image)- video_info_to_image_guide_btn = gr.Button("To Control Image", min_width= 1, size ="sm", visible = any_control_image )video_info_to_image_mask_btn = gr.Button("To Mask Image", min_width= 1, size ="sm", visible = any_image_mask and False)video_info_to_reference_image_btn = gr.Button("To Reference Image", min_width= 1, size ="sm", visible = any_reference_image)+ video_info_to_image_guide_btn = gr.Button("To Control Image", min_width= 1, size ="sm", visible = any_control_image )video_info_eject_image_btn = gr.Button("Eject Image", min_width= 1, size ="sm")with gr.Tab("Post Processing", id= "post_processing", visible = True) as video_postprocessing_tab:with gr.Group(elem_classes= "postprocess"):