Skip to content
Python

Animate upgraded

DeepBeepMeep63db8448f00a
diff --git a/models/wan/any2video.py b/models/wan/any2video.py
index 7dc4b19..b0fc246 100644
--- a/models/wan/any2video.py
+++ b/models/wan/any2video.py
@@ -354,6 +354,7 @@ class WanAny2V:
pre_video_frame = None,
video_prompt_type= "",
original_input_ref_images = [],
+ face_arc_embeds = None,
**bbargs
):
@@ -428,6 +429,7 @@ class WanAny2V:
multitalk = model_type in ["multitalk", "infinitetalk", "vace_multitalk_14B", "i2v_2_2_multitalk"]
infinitetalk = model_type in ["infinitetalk"]
standin = model_type in ["standin", "vace_standin_14B"]
+ lynx = model_type in ["lynx_lite", "lynx_full", "vace_lynx_lite_14B", "vace_lynx_full_14B"]
recam = model_type in ["recam_1.3B"]
ti2v = model_type in ["ti2v_2_2", "lucy_edit"]
lucy_edit= model_type in ["lucy_edit"]
@@ -535,6 +537,7 @@ class WanAny2V:
pose_latents = self.vae.encode([pose_pixels], VAE_tile_size)[0].unsqueeze(0)
input_frames = input_frames * input_masks
if not "X" in video_prompt_type: input_frames += input_masks - 1 # masked area should black (-1) in background frames
+ # input_frames = input_frames[:, :1].expand(-1, input_frames.shape[1], -1, -1)
if prefix_frames_count > 0:
input_frames[:, :prefix_frames_count] = input_video
input_masks[:, :prefix_frames_count] = 1
@@ -549,7 +552,8 @@ class WanAny2V:
clip_image_start = image_ref.squeeze(1)
lat_y = torch.concat(self.vae.encode([image_ref, input_frames.to(self.device)], VAE_tile_size), dim=1)
y = torch.concat([msk, lat_y])
- kwargs.update({ 'y': y, 'pose_latents': pose_latents, 'face_pixel_values' : input_faces.unsqueeze(0)})
+ kwargs.update({ 'y': y, 'pose_latents': pose_latents})
+ face_pixel_values = input_faces.unsqueeze(0)
lat_y = msk = msk_control = msk_ref = pose_pixels = None
ref_images_before = True
ref_images_count = 1
@@ -699,6 +703,23 @@ class WanAny2V:
kwargs["freqs"] = freqs
+ # Lynx
+ if lynx:
+ from .lynx.resampler import Resampler
+ from accelerate import init_empty_weights
+ lynx_lite = model_type in ["lynx_lite", "vace_lynx_lite_14B"]
+ with init_empty_weights():
+ arc_resampler = Resampler( depth=4, dim=1280, dim_head=64, embedding_dim=512, ff_mult=4, heads=20, num_queries=16, output_dim=2048 if lynx_lite else 5120 )
+ offload.load_model_data(arc_resampler, os.path.join("ckpts", "wan2.1_lynx_lite_arc_resampler.safetensors" if lynx_lite else "wan2.1_lynx_full_arc_resampler.safetensors"))
+ arc_resampler.to(self.device)
+ arcface_embed = face_arc_embeds[None,None,:].to(device=self.device, dtype=torch.float)
+ ip_hidden_states = arc_resampler(arcface_embed).to(self.dtype)
+ ip_hidden_states_uncond = arc_resampler(torch.zeros_like(arcface_embed)).to(self.dtype)
+ arc_resampler = None
+ gc.collect()
+ torch.cuda.empty_cache()
+ kwargs["lynx_ip_scale"] = 1.
+
#Standin
if standin:
from preprocessing.face_preprocessor import FaceProcessor
@@ -848,6 +869,18 @@ class WanAny2V:
"context" : [context, context_null, context_null],
"audio_scale": [audio_scale, None, None ]
}
+ elif animate:
+ gen_args = {
+ "x" : [latent_model_input, latent_model_input],
+ "context" : [context, context_null],
+ "face_pixel_values": [face_pixel_values, None]
+ }
+ elif lynx:
+ gen_args = {
+ "x" : [latent_model_input, latent_model_input],
+ "context" : [context, context_null],
+ "lynx_ip_embeds": [ip_hidden_states, ip_hidden_states_uncond]
+ }
elif multitalk and audio_proj != None:
if guide_scale == 1:
gen_args = {
diff --git a/models/wan/lynx/__init__.py b/models/wan/lynx/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/models/wan/lynx/flash_attention.py b/models/wan/lynx/flash_attention.py
new file mode 100644
index 0000000..3bb807d
--- /dev/null
+++ b/models/wan/lynx/flash_attention.py
@@ -0,0 +1,60 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0
+
+import torch
+try:
+ from flash_attn_interface import flash_attn_varlen_func # flash attn 3
+except:
+ from flash_attn.flash_attn_interface import flash_attn_varlen_func # flash attn 2
+
+def flash_attention(query, key, value, q_lens, kv_lens, causal=False):
+ """
+ Args:
+ query, key, value: [B, H, T, D_h]
+ q_lens: list[int] per sequence query length
+ kv_lens: list[int] per sequence key/value length
+ causal: whether to use causal mask
+
+ Returns:
+ output: [B, H, T_q, D_h]
+ """
+ B, H, T_q, D_h = query.shape
+ T_k = key.shape[2]
+ device = query.device
+
+ # Flatten: [B, H, T, D] -> [total_tokens, H, D]
+ q = query.permute(0, 2, 1, 3).reshape(B * T_q, H, D_h)
+ k = key.permute(0, 2, 1, 3).reshape(B * T_k, H, D_h)
+ v = value.permute(0, 2, 1, 3).reshape(B * T_k, H, D_h)
+
+ # Prepare cu_seqlens: prefix sum
+ q_lens_tensor = torch.tensor(q_lens, device=device, dtype=torch.int32)
+ kv_lens_tensor = torch.tensor(kv_lens, device=device, dtype=torch.int32)
+
+ cu_seqlens_q = torch.zeros(len(q_lens_tensor) + 1, device=device, dtype=torch.int32)
+ cu_seqlens_k = torch.zeros(len(kv_lens_tensor) + 1, device=device, dtype=torch.int32)
+
+ cu_seqlens_q[1:] = torch.cumsum(q_lens_tensor, dim=0)
+ cu_seqlens_k[1:] = torch.cumsum(kv_lens_tensor, dim=0)
+
+ max_seqlen_q = int(q_lens_tensor.max().item())
+ max_seqlen_k = int(kv_lens_tensor.max().item())
+
+ # Call FlashAttention varlen kernel
+ out = flash_attn_varlen_func(
+ q,
+ k,
+ v,
+ cu_seqlens_q,
+ cu_seqlens_k,
+ max_seqlen_q,
+ max_seqlen_k,
+ causal=causal
+ )
+ if not torch.is_tensor(out): # flash attn 3
+ out = out[0]
+
+ # Restore shape: [total_q, H, D_h] -> [B, H, T_q, D_h]
+ out = out.view(B, T_q, H, D_h).permute(0, 2, 1, 3).contiguous()
+
+ return out
\ No newline at end of file
diff --git a/models/wan/lynx/full_attention_processor.py b/models/wan/lynx/full_attention_processor.py
new file mode 100644
index 0000000..fd0edb7
--- /dev/null
+++ b/models/wan/lynx/full_attention_processor.py
@@ -0,0 +1,305 @@
+# Copyright (c) 2025 The Wan Team and The HuggingFace Team.
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+#
+# This file has been modified by Bytedance Ltd. and/or its affiliates on September 15, 2025.
+#
+# Original file was released under Apache License 2.0, with the full license text
+# available at https://github.com/huggingface/diffusers/blob/v0.30.3/LICENSE and https://github.com/Wan-Video/Wan2.1/blob/main/LICENSE.txt.
+#
+# This modified file is released under the same license.
+
+
+from typing import Optional, List
+import torch
+import torch.nn as nn
+from diffusers.models.attention_processor import Attention
+from modules.common.flash_attention import flash_attention
+from modules.common.navit_utils import vector_to_list, list_to_vector, merge_token_lists
+
+def new_forward(
+ self,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ **cross_attention_kwargs,
+) -> torch.Tensor:
+ return self.processor(
+ self,
+ hidden_states,
+ encoder_hidden_states=encoder_hidden_states,
+ attention_mask=attention_mask,
+ **cross_attention_kwargs,
+ )
+Attention.forward = new_forward
+
+class WanIPAttnProcessor(nn.Module):
+ def __init__(
+ self,
+ cross_attention_dim: int,
+ dim: int,
+ n_registers: int,
+ bias: bool,
+ ):
+ super().__init__()
+ self.to_k_ip = nn.Linear(cross_attention_dim, dim, bias=bias)
+ self.to_v_ip = nn.Linear(cross_attention_dim, dim, bias=bias)
+ if n_registers > 0:
+ self.registers = nn.Parameter(torch.randn(1, n_registers, cross_attention_dim) / dim**0.5)
+ else:
+ self.registers = None
+
+ def forward(
+ self,
+ attn: Attention,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: Optional[torch.Tensor] = None,
+ ip_hidden_states: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ rotary_emb: Optional[torch.Tensor] = None,
+ q_lens: List[int] = None,
+ kv_lens: List[int] = None,
+ ip_lens: List[int] = None,
+ ip_scale: float = 1.0,
+ ):
+ encoder_hidden_states_img = None
+ if attn.add_k_proj is not None:
+ encoder_hidden_states_img = encoder_hidden_states[:, :257]
+ encoder_hidden_states = encoder_hidden_states[:, 257:]
+ if encoder_hidden_states is None:
+ encoder_hidden_states = hidden_states
+
+ query = attn.to_q(hidden_states)
+ key = attn.to_k(encoder_hidden_states)
+ value = attn.to_v(encoder_hidden_states)
+
+ if ip_hidden_states is not None:
+
+ if self.registers is not None:
+ ip_hidden_states_list = vector_to_list(ip_hidden_states, ip_lens, 1)
+ ip_hidden_states_list = merge_token_lists(ip_hidden_states_list, [self.registers] * len(ip_hidden_states_list), 1)
+ ip_hidden_states, ip_lens = list_to_vector(ip_hidden_states_list, 1)
+
+ ip_query = query
+ ip_key = self.to_k_ip(ip_hidden_states)
+ ip_value = self.to_v_ip(ip_hidden_states)
+ if attn.norm_q is not None:
+ ip_query = attn.norm_q(ip_query)
+ if attn.norm_k is not None:
+ ip_key = attn.norm_k(ip_key)
+ ip_query = ip_query.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ ip_key = ip_key.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ ip_value = ip_value.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ ip_hidden_states = flash_attention(
+ ip_query, ip_key, ip_value,
+ q_lens=q_lens,
+ kv_lens=ip_lens
+ )
+ ip_hidden_states = ip_hidden_states.transpose(1, 2).flatten(2, 3)
+ ip_hidden_states = ip_hidden_states.type_as(ip_query)
+
+
+ if attn.norm_q is not None:
+ query = attn.norm_q(query)
+ if attn.norm_k is not None:
+ key = attn.norm_k(key)
+
+ query = query.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ key = key.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ value = value.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+
+ if rotary_emb is not None:
+
+ def apply_rotary_emb(hidden_states: torch.Tensor, freqs: torch.Tensor):
+ x_rotated = torch.view_as_complex(hidden_states.to(torch.float64).unflatten(3, (-1, 2)))
+ x_out = torch.view_as_real(x_rotated * freqs).flatten(3, 4)
+ return x_out.type_as(hidden_states)
+
+ query = apply_rotary_emb(query, rotary_emb)
+ key = apply_rotary_emb(key, rotary_emb)
+
+ hidden_states = flash_attention(
+ query, key, value,
+ q_lens=q_lens,
+ kv_lens=kv_lens,
+ )
+
+ hidden_states = hidden_states.transpose(1, 2).flatten(2, 3)
+ hidden_states = hidden_states.type_as(query)
+
+ if ip_hidden_states is not None:
+ hidden_states = hidden_states + ip_scale * ip_hidden_states
+
+ hidden_states = attn.to_out[0](hidden_states)
+ hidden_states = attn.to_out[1](hidden_states)
+ return hidden_states
+
+def register_ip_adapter(
+ model,
+ cross_attention_dim=None,
+ n_registers=0,
+ init_method="zero",
+ dtype=torch.float32,
+):
+ attn_procs = {}
+ transformer_sd = model.state_dict()
+ for layer_idx, block in enumerate(model.blocks):
+ name = f"blocks.{layer_idx}.attn2.processor"
+ layer_name = name.split(".processor")[0]
+ dim = transformer_sd[layer_name + ".to_k.weight"].shape[1]
+ attn_procs[name] = WanIPAttnProcessor(
+ cross_attention_dim=dim if cross_attention_dim is None else cross_attention_dim,
+ dim=dim,
+ n_registers=n_registers,
+ bias=True,
+ )
+ if init_method == "zero":
+ torch.nn.init.zeros_(attn_procs[name].to_k_ip.weight)
+ torch.nn.init.zeros_(attn_procs[name].to_k_ip.bias)
+ torch.nn.init.zeros_(attn_procs[name].to_v_ip.weight)
+ torch.nn.init.zeros_(attn_procs[name].to_v_ip.bias)
+ elif init_method == "clone":
+ layer_name = name.split(".processor")[0]
+ weights = {
+ "to_k_ip.weight": transformer_sd[layer_name + ".to_k.weight"],
+ "to_k_ip.bias": transformer_sd[layer_name + ".to_k.bias"],
+ "to_v_ip.weight": transformer_sd[layer_name + ".to_v.weight"],
+ "to_v_ip.bias": transformer_sd[layer_name + ".to_v.bias"],
+ }
+ attn_procs[name].load_state_dict(weights)
+ elif init_method == "random":
+ pass
+ else:
+ raise ValueError(f"{init_method} is not supported.")
+ block.attn2.processor = attn_procs[name]
+ ip_layers = torch.nn.ModuleList(attn_procs.values())
+ ip_layers.to(device=model.device, dtype=dtype)
+ return model, ip_layers
+
+
+class WanRefAttnProcessor(nn.Module):
+ def __init__(
+ self,
+ dim: int,
+ bias: bool,
+ ):
+ super().__init__()
+ self.to_k_ref = nn.Linear(dim, dim, bias=bias)
+ self.to_v_ref = nn.Linear(dim, dim, bias=bias)
+
+ def forward(
+ self,
+ attn: Attention,
+ hidden_states: torch.Tensor,
+ encoder_hidden_states: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ rotary_emb: Optional[torch.Tensor] = None,
+ q_lens: List[int] = None,
+ kv_lens: List[int] = None,
+ ref_feature: Optional[tuple] = None,
+ ref_scale: float = 1.0,
+ ):
+ encoder_hidden_states_img = None
+ if attn.add_k_proj is not None:
+ encoder_hidden_states_img = encoder_hidden_states[:, :257]
+ encoder_hidden_states = encoder_hidden_states[:, 257:]
+ if encoder_hidden_states is None:
+ encoder_hidden_states = hidden_states
+
+ query = attn.to_q(hidden_states)
+ key = attn.to_k(encoder_hidden_states)
+ value = attn.to_v(encoder_hidden_states)
+
+ if ref_feature is None:
+ ref_hidden_states = None
+ else:
+ ref_hidden_states, ref_lens = ref_feature
+ ref_query = query
+ ref_key = self.to_k_ref(ref_hidden_states)
+ ref_value = self.to_v_ref(ref_hidden_states)
+ if attn.norm_q is not None:
+ ref_query = attn.norm_q(ref_query)
+ if attn.norm_k is not None:
+ ref_key = attn.norm_k(ref_key)
+ ref_query = ref_query.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ ref_key = ref_key.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ ref_value = ref_value.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ ref_hidden_states = flash_attention(
+ ref_query, ref_key, ref_value,
+ q_lens=q_lens,
+ kv_lens=ref_lens
+ )
+ ref_hidden_states = ref_hidden_states.transpose(1, 2).flatten(2, 3)
+ ref_hidden_states = ref_hidden_states.type_as(ref_query)
+
+ if attn.norm_q is not None:
+ query = attn.norm_q(query)
+ if attn.norm_k is not None:
+ key = attn.norm_k(key)
+
+ query = query.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ key = key.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ value = value.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+
+ if rotary_emb is not None:
+
+ def apply_rotary_emb(hidden_states: torch.Tensor, freqs: torch.Tensor):
+ x_rotated = torch.view_as_complex(hidden_states.to(torch.float64).unflatten(3, (-1, 2)))
+ x_out = torch.view_as_real(x_rotated * freqs).flatten(3, 4)
+ return x_out.type_as(hidden_states)
+
+ query = apply_rotary_emb(query, rotary_emb)
+ key = apply_rotary_emb(key, rotary_emb)
+
+ hidden_states = flash_attention(
+ query, key, value,
+ q_lens=q_lens,
+ kv_lens=kv_lens,
+ )
+
+ hidden_states = hidden_states.transpose(1, 2).flatten(2, 3)
+ hidden_states = hidden_states.type_as(query)
+
+ if ref_hidden_states is not None:
+ hidden_states = hidden_states + ref_scale * ref_hidden_states
+
+ hidden_states = attn.to_out[0](hidden_states)
+ hidden_states = attn.to_out[1](hidden_states)
+ return hidden_states
+
+
+def register_ref_adapter(
+ model,
+ init_method="zero",
+ dtype=torch.float32,
+):
+ attn_procs = {}
+ transformer_sd = model.state_dict()
+ for layer_idx, block in enumerate(model.blocks):
+ name = f"blocks.{layer_idx}.attn1.processor"
+ layer_name = name.split(".processor")[0]
+ dim = transformer_sd[layer_name + ".to_k.weight"].shape[0]
+ attn_procs[name] = WanRefAttnProcessor(
+ dim=dim,
+ bias=True,
+ )
+ if init_method == "zero":
+ torch.nn.init.zeros_(attn_procs[name].to_k_ref.weight)
+ torch.nn.init.zeros_(attn_procs[name].to_k_ref.bias)
+ torch.nn.init.zeros_(attn_procs[name].to_v_ref.weight)
+ torch.nn.init.zeros_(attn_procs[name].to_v_ref.bias)
+ elif init_method == "clone":
+ weights = {
+ "to_k_ref.weight": transformer_sd[layer_name + ".to_k.weight"],
+ "to_k_ref.bias": transformer_sd[layer_name + ".to_k.bias"],
+ "to_v_ref.weight": transformer_sd[layer_name + ".to_v.weight"],
+ "to_v_ref.bias": transformer_sd[layer_name + ".to_v.bias"],
+ }
+ attn_procs[name].load_state_dict(weights)
+ elif init_method == "random":
+ pass
+ else:
+ raise ValueError(f"{init_method} is not supported.")
+ block.attn1.processor = attn_procs[name]
+ ref_layers = torch.nn.ModuleList(attn_procs.values())
+ ref_layers.to(device=model.device, dtype=dtype)
+ return model, ref_layers
diff --git a/models/wan/lynx/inference_utils.py b/models/wan/lynx/inference_utils.py
new file mode 100644
index 0000000..c1129f3
--- /dev/null
+++ b/models/wan/lynx/inference_utils.py
@@ -0,0 +1,39 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0
+
+from typing import Optional, Union
+
+import torch
+import numpy as np
+
+from PIL import Image
+from dataclasses import dataclass, field
+
+
+dtype_mapping = {
+ "bf16": torch.bfloat16,
+ "fp16": torch.float16,
+ "fp32": torch.float32
+}
+
+
+@dataclass
+class SubjectInfo:
+ name: str = ""
+ image_pil: Optional[Image.Image] = None
+ landmarks: Optional[Union[np.ndarray, torch.Tensor]] = None
+ face_embeds: Optional[Union[np.ndarray, torch.Tensor]] = None
+
+
+@dataclass
+class VideoStyleInfo: # key names should match those used in style.yaml file
+ style_name: str = 'none'
+ num_frames: int = 81
+ seed: int = -1
+ guidance_scale: float = 5.0
+ guidance_scale_i: float = 2.0
+ num_inference_steps: int = 50
+ width: int = 832
+ height: int = 480
+ prompt: str = ''
+ negative_prompt: str = ''
\ No newline at end of file
diff --git a/models/wan/lynx/light_attention_processor.py b/models/wan/lynx/light_attention_processor.py
new file mode 100644
index 0000000..a3c7516
--- /dev/null
+++ b/models/wan/lynx/light_attention_processor.py
@@ -0,0 +1,194 @@
+# Copyright (c) 2025 The Wan Team and The HuggingFace Team.
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+#
+# This file has been modified by Bytedance Ltd. and/or its affiliates on September 15, 2025.
+#
+# Original file was released under Apache License 2.0, with the full license text
+# available at https://github.com/huggingface/diffusers/blob/v0.30.3/LICENSE and https://github.com/Wan-Video/Wan2.1/blob/main/LICENSE.txt.
+#
+# This modified file is released under the same license.
+
+from typing import Tuple, Union, Optional
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from diffusers.models.normalization import RMSNorm
+
+
+def register_ip_adapter_wan(
+ model,
+ hidden_size=5120,
+ cross_attention_dim=2048,
+ dtype=torch.float32,
+ init_method="zero",
+ layers=None
+):
+ attn_procs = {}
+ transformer_sd = model.state_dict()
+
+ if layers is None:
+ layers = list(range(0, len(model.blocks)))
+ elif isinstance(layers, int): # Only interval provided
+ layers = list(range(0, len(model.blocks), layers))
+
+ for i, block in enumerate(model.blocks):
+ if i not in layers:
+ continue
+
+ name = f"blocks.{i}.attn2.processor"
+ attn_procs[name] = IPAWanAttnProcessor2_0(
+ hidden_size=hidden_size,
+ cross_attention_dim=cross_attention_dim
+ )
+
+ if init_method == "zero":
+ torch.nn.init.zeros_(attn_procs[name].to_k_ip.weight)
+ torch.nn.init.zeros_(attn_procs[name].to_v_ip.weight)
+ elif init_method == "clone":
+ layer_name = name.split(".processor")[0]
+ weights = {
+ "to_k_ip.weight": transformer_sd[layer_name + ".to_k.weight"],
+ "to_v_ip.weight": transformer_sd[layer_name + ".to_v.weight"],
+ }
+ attn_procs[name].load_state_dict(weights)
+ else:
+ raise ValueError(f"{init_method} is not supported.")
+
+ block.attn2.processor = attn_procs[name]
+
+ ip_layers = torch.nn.ModuleList(attn_procs.values())
+ ip_layers.to(model.device, dtype=dtype)
+
+ return model, ip_layers
+
+
+class IPAWanAttnProcessor2_0(torch.nn.Module):
+ def __init__(
+ self,
+ hidden_size,
+ cross_attention_dim=None,
+ scale=1.0,
+ bias=False,
+ ):
+ super().__init__()
+
+ if not hasattr(F, "scaled_dot_product_attention"):
+ raise ImportError("IPAWanAttnProcessor requires PyTorch 2.0, to use it, please upgrade PyTorch to 2.0.")
+
+ self.hidden_size = hidden_size
+ self.cross_attention_dim = cross_attention_dim
+ self.scale = scale
+
+ self.to_k_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=bias)
+ self.to_v_ip = nn.Linear(cross_attention_dim or hidden_size, hidden_size, bias=bias)
+
+ torch.nn.init.zeros_(self.to_k_ip.weight)
+ torch.nn.init.zeros_(self.to_v_ip.weight)
+ if bias:
+ torch.nn.init.zeros_(self.to_k_ip.bias)
+ torch.nn.init.zeros_(self.to_v_ip.bias)
+
+ self.norm_rms_k = RMSNorm(hidden_size, eps=1e-5, elementwise_affine=False)
+
+
+ def __call__(
+ self,
+ attn,
+ hidden_states: torch.Tensor,
+ image_embed: torch.Tensor = None,
+ ip_scale: float = None,
+ encoder_hidden_states: Optional[torch.Tensor] = None,
+ attention_mask: Optional[torch.Tensor] = None,
+ rotary_emb: Optional[torch.Tensor] = None,
+ ) -> torch.Tensor:
+ encoder_hidden_states_img = None
+ if attn.add_k_proj is not None:
+ encoder_hidden_states_img = encoder_hidden_states[:, :257]
+ encoder_hidden_states = encoder_hidden_states[:, 257:]
+ if encoder_hidden_states is None:
+ encoder_hidden_states = hidden_states
+
+ query = attn.to_q(hidden_states)
+ key = attn.to_k(encoder_hidden_states)
+ value = attn.to_v(encoder_hidden_states)
+
+ # =============================================================
+ batch_size = image_embed.size(0)
+
+ ip_hidden_states = image_embed
+ ip_query = query # attn.to_q(hidden_states.clone())
+ ip_key = self.to_k_ip(ip_hidden_states)
+ ip_value = self.to_v_ip(ip_hidden_states)
+
+ if attn.norm_q is not None:
+ ip_query = attn.norm_q(ip_query)
+ ip_key = self.norm_rms_k(ip_key)
+
+ ip_inner_dim = ip_key.shape[-1]
+ ip_head_dim = ip_inner_dim // attn.heads
+
+ ip_query = ip_query.view(batch_size, -1, attn.heads, ip_head_dim).transpose(1, 2)
+ ip_key = ip_key.view(batch_size, -1, attn.heads, ip_head_dim).transpose(1, 2)
+ ip_value = ip_value.view(batch_size, -1, attn.heads, ip_head_dim).transpose(1, 2)
+
+ ip_hidden_states = F.scaled_dot_product_attention(
+ ip_query, ip_key, ip_value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+ )
+
+ ip_hidden_states = ip_hidden_states.transpose(1, 2).reshape(batch_size, -1, attn.heads * ip_head_dim)
+ ip_hidden_states = ip_hidden_states.to(ip_query.dtype)
+ # ===========================================================================
+
+ if attn.norm_q is not None:
+ query = attn.norm_q(query)
+ if attn.norm_k is not None:
+ key = attn.norm_k(key)
+
+ query = query.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ key = key.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ value = value.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+
+ if rotary_emb is not None:
+
+ def apply_rotary_emb_inner(hidden_states: torch.Tensor, freqs: torch.Tensor):
+ x_rotated = torch.view_as_complex(hidden_states.to(torch.float64).unflatten(3, (-1, 2)))
+ x_out = torch.view_as_real(x_rotated * freqs).flatten(3, 4)
+ return x_out.type_as(hidden_states)
+
+ query = apply_rotary_emb_inner(query, rotary_emb)
+ key = apply_rotary_emb_inner(key, rotary_emb)
+
+ # I2V task
+ hidden_states_img = None
+ if encoder_hidden_states_img is not None:
+ key_img = attn.add_k_proj(encoder_hidden_states_img)
+ key_img = attn.norm_added_k(key_img)
+ value_img = attn.add_v_proj(encoder_hidden_states_img)
+
+ key_img = key_img.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+ value_img = value_img.unflatten(2, (attn.heads, -1)).transpose(1, 2)
+
+ hidden_states_img = F.scaled_dot_product_attention(
+ query, key_img, value_img, attn_mask=None, dropout_p=0.0, is_causal=False
+ )
+ hidden_states_img = hidden_states_img.transpose(1, 2).flatten(2, 3)
+ hidden_states_img = hidden_states_img.type_as(query)
+ hidden_states = F.scaled_dot_product_attention(
+ query, key, value, attn_mask=attention_mask, dropout_p=0.0, is_causal=False
+ )
+ hidden_states = hidden_states.transpose(1, 2).flatten(2, 3)
+ hidden_states = hidden_states.type_as(query)
+
+ if hidden_states_img is not None:
+ hidden_states = hidden_states + hidden_states_img
+
+ # Add IPA residual
+ ip_scale = ip_scale or self.scale
+ hidden_states = hidden_states + ip_scale * ip_hidden_states
+
+ hidden_states = attn.to_out[0](hidden_states)
+ hidden_states = attn.to_out[1](hidden_states)
+
+ return hidden_states
\ No newline at end of file
diff --git a/models/wan/lynx/model_utils_wan.py b/models/wan/lynx/model_utils_wan.py
new file mode 100644
index 0000000..2fd1170
--- /dev/null
+++ b/models/wan/lynx/model_utils_wan.py
@@ -0,0 +1,264 @@
+# Copyright (c) 2025 The Wan Team and The HuggingFace Team.
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0
+#
+# This file has been modified by Bytedance Ltd. and/or its affiliates on September 15, 2025.
+#
+# Original file was released under Apache License 2.0, with the full license text
+# available at https://github.com/huggingface/diffusers/blob/v0.30.3/LICENSE and https://github.com/Wan-Video/Wan2.1/blob/main/LICENSE.txt.
+#
+# This modified file is released under the same license.
+
+import os
+import math
+import torch
+
+from typing import List, Optional, Tuple, Union
+
+from diffusers.models.embeddings import get_3d_rotary_pos_embed
+from diffusers.pipelines.cogvideo.pipeline_cogvideox import get_resize_crop_region_for_grid
+
+from diffusers import FlowMatchEulerDiscreteScheduler
+
+""" Utility functions for Wan2.1-T2I model """
+
+
+def cal_mean_and_std(vae, device, dtype):
+ # https://github.com/Wan-Video/Wan2.1/blob/main/wan/modules/vae.py#L483
+
+ # latents_mean
+ latents_mean = torch.tensor(vae.config.latents_mean).view(
+ 1, vae.config.z_dim, 1, 1, 1).to(device, dtype)
+
+ # latents_std
+ latents_std = 1.0 / torch.tensor(vae.config.latents_std).view(
+ 1, vae.config.z_dim, 1, 1, 1).to(device, dtype)
+
+ return latents_mean, latents_std
+
+
+def encode_video(vae, video):
+ # Input video is formatted as [B, F, C, H, W]
+ video = video.to(vae.device, dtype=vae.dtype)
+ video = video.unsqueeze(0) if len(video.shape) == 4 else video
+
+ # VAE takes [B, C, F, H, W]
+ video = video.permute(0, 2, 1, 3, 4)
+ latent_dist = vae.encode(video).latent_dist
+
+ mean, std = cal_mean_and_std(vae, vae.device, vae.dtype)
+
+ # NOTE: this implementation is weird but I still borrow it from the official
+ # codes. A standard normailization should be (x - mean) / std, but the std
+ # here is computed by `1 / vae.config.latents_std` from cal_mean_and_std().
+
+ # Sample latent with scaling
+ video_latent = (latent_dist.sample() - mean) * std
+
+ # Return as [B, C, F, H, W] as required by Wan
+ return video_latent
+
+
+def decode_latent(vae, latent):
+ # Input latent is formatted as [B, C, F, H, W]
+ latent = latent.to(vae.device, dtype=vae.dtype)
+ latent = latent.unsqueeze(0) if len(latent.shape) == 4 else latent
+
+ mean, std = cal_mean_and_std(vae, latent.device, latent.dtype)
+ latent = latent / std + mean
+
+ # VAE takes [B, C, F, H, W]
+ frames = vae.decode(latent).sample
+
+ # Return as [B, F, C, H, W]
+ frames = frames.permute(0, 2, 1, 3, 4).float()
+
+ return frames
+
+
+def encode_prompt(
+ tokenizer,
+ text_encoder,
+ prompt: Union[str, List[str]],
+ num_videos_per_prompt: int = 1,
+ max_sequence_length: int = 226,
+ device: Optional[torch.device] = None,
+ dtype: Optional[torch.dtype] = None,
+ text_input_ids=None,
+ prompt_attention_mask=None,
+):
+ prompt = [prompt] if isinstance(prompt, str) else prompt
+ batch_size = len(prompt)
+
+ if tokenizer is not None:
+ text_inputs = tokenizer(
+ prompt,
+ padding="max_length",
+ max_length=max_sequence_length,
+ truncation=True,
+ add_special_tokens=True,
+ return_tensors="pt",
+ )
+ text_input_ids = text_inputs.input_ids
+ prompt_attention_mask = text_inputs.attention_mask
+ else:
+ if text_input_ids is None:
+ raise ValueError("`text_input_ids` must be provided when the tokenizer is not specified.")
+
+ seq_lens = prompt_attention_mask.gt(0).sum(dim=1).long()
+ prompt_embeds = text_encoder(text_input_ids.to(device), prompt_attention_mask.to(device)).last_hidden_state
+
+ prompt_embeds = prompt_embeds.to(dtype=dtype, device=device)
+ prompt_embeds = [u[:v] for u, v in zip(prompt_embeds, seq_lens)]
+ prompt_embeds = torch.stack(
+ [torch.cat([u, u.new_zeros(max_sequence_length - u.size(0), u.size(1))]) for u in prompt_embeds], dim=0
+ )
+
+ # duplicate text embeddings for each generation per prompt, using mps friendly method
+ _, seq_len, _ = prompt_embeds.shape
+ prompt_embeds = prompt_embeds.repeat(1, num_videos_per_prompt, 1)
+ prompt_embeds = prompt_embeds.view(batch_size * num_videos_per_prompt, seq_len, -1)
+
+ return prompt_embeds, prompt_attention_mask
+
+
+def compute_prompt_embeddings(
+ tokenizer, text_encoder, prompt, max_sequence_length, device, dtype, requires_grad: bool = False
+):
+ if requires_grad:
+ prompt_embeds = encode_prompt(
+ tokenizer,
+ text_encoder,
+ prompt,
+ num_videos_per_prompt=1,
+ max_sequence_length=max_sequence_length,
+ device=device,
+ dtype=dtype,
+ )
+ else:
+ with torch.no_grad():
+ prompt_embeds = encode_prompt(
+ tokenizer,
+ text_encoder,
+ prompt,
+ num_videos_per_prompt=1,
+ max_sequence_length=max_sequence_length,
+ device=device,
+ dtype=dtype,
+ )
+ return prompt_embeds[0]
+
+
+def prepare_rotary_positional_embeddings(
+ height: int,
+ width: int,
+ num_frames: int,
+ vae_scale_factor_spatial: int = 8,
+ patch_size: int = 2,
+ attention_head_dim: int = 64,
+ device: Optional[torch.device] = None,
+ base_height: int = 480,
+ base_width: int = 720,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ grid_height = height // (vae_scale_factor_spatial * patch_size)
+ grid_width = width // (vae_scale_factor_spatial * patch_size)
+ base_size_width = base_width // (vae_scale_factor_spatial * patch_size)
+ base_size_height = base_height // (vae_scale_factor_spatial * patch_size)
+
+ grid_crops_coords = get_resize_crop_region_for_grid((grid_height, grid_width), base_size_width, base_size_height)
+ freqs_cos, freqs_sin = get_3d_rotary_pos_embed(
+ embed_dim=attention_head_dim,
+ crops_coords=grid_crops_coords,
+ grid_size=(grid_height, grid_width),
+ temporal_size=num_frames,
+ )
+
+ freqs_cos = freqs_cos.to(device=device)
+ freqs_sin = freqs_sin.to(device=device)
+ return freqs_cos, freqs_sin
+
+
+def prepare_sigmas(
+ scheduler: FlowMatchEulerDiscreteScheduler,
+ sigmas: torch.Tensor,
+ batch_size: int,
+ num_train_timesteps: int,
+ flow_weighting_scheme: str = "none",
+ flow_logit_mean: float = 0.0,
+ flow_logit_std: float = 1.0,
+ flow_mode_scale: float = 1.29,
+ device: torch.device = torch.device("cpu"),
+ generator: Optional[torch.Generator] = None,
+) -> torch.Tensor:
+ if isinstance(scheduler, FlowMatchEulerDiscreteScheduler):
+ weights = compute_density_for_timestep_sampling(
+ weighting_scheme=flow_weighting_scheme,
+ batch_size=batch_size,
+ logit_mean=flow_logit_mean,
+ logit_std=flow_logit_std,
+ mode_scale=flow_mode_scale,
+ device=device,
+ generator=generator,
+ )
+ indices = (weights * num_train_timesteps).long()
+ else:
+ raise ValueError(f"Unsupported scheduler type {type(scheduler)}")
+
+ return sigmas[indices]
+
+
+def compute_density_for_timestep_sampling(
+ weighting_scheme: str,
+ batch_size: int,
+ logit_mean: float = None,
+ logit_std: float = None,
+ mode_scale: float = None,
+ device: torch.device = torch.device("cpu"),
+ generator: Optional[torch.Generator] = None,
+) -> torch.Tensor:
+ r"""
+ Compute the density for sampling the timesteps when doing SD3 training.
+
+ Courtesy: This was contributed by Rafie Walker in https://github.com/huggingface/diffusers/pull/8528.
+
+ SD3 paper reference: https://arxiv.org/abs/2403.03206v1.
+ """
+ if weighting_scheme == "logit_normal":
+ # See 3.1 in the SD3 paper ($rf/lognorm(0.00,1.00)$).
+ u = torch.normal(mean=logit_mean, std=logit_std, size=(batch_size,), device=device, generator=generator)
+ u = torch.nn.functional.sigmoid(u)
+ elif weighting_scheme == "mode":
+ u = torch.rand(size=(batch_size,), device=device, generator=generator)
+ u = 1 - u - mode_scale * (torch.cos(math.pi * u / 2) ** 2 - 1 + u)
+ else:
+ u = torch.rand(size=(batch_size,), device=device, generator=generator)
+ return u
+
+
+def expand_tensor_dims(tensor: torch.Tensor, ndim: int) -> torch.Tensor:
+ assert len(tensor.shape) <= ndim
+ return tensor.reshape(tensor.shape + (1,) * (ndim - len(tensor.shape)))
+
+
+def flow_match_xt(x0: torch.Tensor, n: torch.Tensor, t: torch.Tensor) -> torch.Tensor:
+ r"""Forward process of flow matching."""
+ return (1.0 - t) * x0 + t * n
+
+
+def flow_match_target(n: torch.Tensor, x0: torch.Tensor) -> torch.Tensor:
+ r"""Loss target for flow matching."""
+ return n - x0
+
+
+def prepare_loss_weights(
+ scheduler: FlowMatchEulerDiscreteScheduler,
+ sigmas: Optional[torch.Tensor] = None,
+ flow_weighting_scheme: str = "none",
+
+) -> torch.Tensor:
+
+ assert isinstance(scheduler, FlowMatchEulerDiscreteScheduler)
+
+ from diffusers.training_utils import compute_loss_weighting_for_sd3
+
+ return compute_loss_weighting_for_sd3(sigmas=sigmas, weighting_scheme=flow_weighting_scheme)
\ No newline at end of file
diff --git a/models/wan/lynx/navit_utils.py b/models/wan/lynx/navit_utils.py
new file mode 100644
index 0000000..593e7f0
--- /dev/null
+++ b/models/wan/lynx/navit_utils.py
@@ -0,0 +1,16 @@
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0
+
+import torch
+
+def vector_to_list(tensor, lens, dim):
+ return list(torch.split(tensor, lens, dim=dim))
+
+def list_to_vector(tensor_list, dim):
+ lens = [tensor.shape[dim] for tensor in tensor_list]
+ tensor = torch.cat(tensor_list, dim)
+ return tensor, lens
+
+def merge_token_lists(list1, list2, dim):
+ assert(len(list1) == len(list2))
+ return [torch.cat((t1, t2), dim) for t1, t2 in zip(list1, list2)]
\ No newline at end of file
diff --git a/models/wan/lynx/resampler.py b/models/wan/lynx/resampler.py
new file mode 100644
index 0000000..e225a06
--- /dev/null
+++ b/models/wan/lynx/resampler.py
@@ -0,0 +1,185 @@
+
+# Copyright (c) 2022 Phil Wang
+# Copyright (c) 2023 Anas Awadalla, Irena Gao, Joshua Gardner, Jack Hessel, Yusuf Hanafy, Wanrong Zhu, Kalyani Marathe, Yonatan Bitton, Samir Gadre, Jenia Jitsev, Simon Kornblith, Pang Wei Koh, Gabriel Ilharco, Mitchell Wortsman, Ludwig Schmidt.
+# Copyright (c) 2023 Tencent AI Lab
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+
+# This file has been modified by Bytedance Ltd. and/or its affiliates on September 15, 2025.
+# SPDX-License-Identifier: Apache-2.0
+
+# Original file (IP-Adapter) was released under Apache License 2.0, with the full license text
+# available at https://github.com/tencent-ailab/IP-Adapter/blob/main/LICENSE.
+
+# Original file (open_flamingo and flamingo-pytorch) was released under MIT License:
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+
+import math
+
+import torch
+import torch.nn as nn
+
+
+# FFN
+def FeedForward(dim, mult=4):
+ inner_dim = int(dim * mult)
+ return nn.Sequential(
+ nn.LayerNorm(dim),
+ nn.Linear(dim, inner_dim, bias=False),
+ nn.GELU(),
+ nn.Linear(inner_dim, dim, bias=False),
+ )
+
+
+def reshape_tensor(x, heads):
+ bs, length, width = x.shape
+ #(bs, length, width) --> (bs, length, n_heads, dim_per_head)
+ x = x.view(bs, length, heads, -1)
+ # (bs, length, n_heads, dim_per_head) --> (bs, n_heads, length, dim_per_head)
+ x = x.transpose(1, 2)
+ # (bs, n_heads, length, dim_per_head) --> (bs*n_heads, length, dim_per_head)
+ x = x.reshape(bs, heads, length, -1)
+ return x
+
+
+class PerceiverAttention(nn.Module):
+ def __init__(self, *, dim, dim_head=64, heads=8):
+ super().__init__()
+ self.scale = dim_head**-0.5
+ self.dim_head = dim_head
+ self.heads = heads
+ inner_dim = dim_head * heads
+
+ self.norm1 = nn.LayerNorm(dim)
+ self.norm2 = nn.LayerNorm(dim)
+
+ self.to_q = nn.Linear(dim, inner_dim, bias=False)
+ self.to_kv = nn.Linear(dim, inner_dim * 2, bias=False)
+ self.to_out = nn.Linear(inner_dim, dim, bias=False)
+
+
+ def forward(self, x, latents):
+ """
+ Args:
+ x (torch.Tensor): image features
+ shape (b, n1, D)
+ latent (torch.Tensor): latent features
+ shape (b, n2, D)
+ """
+ x = self.norm1(x)
+ latents = self.norm2(latents)
+
+ b, l, _ = latents.shape
+
+ q = self.to_q(latents)
+ kv_input = torch.cat((x, latents), dim=-2)
+ k, v = self.to_kv(kv_input).chunk(2, dim=-1)
+
+ q = reshape_tensor(q, self.heads)
+ k = reshape_tensor(k, self.heads)
+ v = reshape_tensor(v, self.heads)
+
+ # attention
+ scale = 1 / math.sqrt(math.sqrt(self.dim_head))
+ weight = (q * scale) @ (k * scale).transpose(-2, -1) # More stable with f16 than dividing afterwards
+ weight = torch.softmax(weight.float(), dim=-1).type(weight.dtype)
+ out = weight @ v
+
+ out = out.permute(0, 2, 1, 3).reshape(b, l, -1)
+
+ return self.to_out(out)
+
+
+class Resampler(nn.Module):
+ def __init__(
+ self,
+ dim=1024,
+ depth=8,
+ dim_head=64,
+ heads=16,
+ num_queries=8,
+ embedding_dim=768,
+ output_dim=1024,
+ ff_mult=4,
+ max_seq_len: int = 257, # CLIP tokens + CLS token
+ apply_pos_emb: bool = False,
+ num_latents_mean_pooled: int = 0, # number of latents derived from mean pooled representation of the sequence
+ **kwargs,
+ ):
+ super().__init__()
+ self.pos_emb = nn.Embedding(max_seq_len, embedding_dim) if apply_pos_emb else None
+
+ self.latents = nn.Parameter(torch.randn(1, num_queries, dim) / dim**0.5)
+
+ self.proj_in = nn.Linear(embedding_dim, dim)
+
+ self.proj_out = nn.Linear(dim, output_dim)
+ self.norm_out = nn.LayerNorm(output_dim)
+
+ self.to_latents_from_mean_pooled_seq = (
+ nn.Sequential(
+ nn.LayerNorm(dim),
+ nn.Linear(dim, dim * num_latents_mean_pooled),
+ Rearrange("b (n d) -> b n d", n=num_latents_mean_pooled),
+ )
+ if num_latents_mean_pooled > 0
+ else None
+ )
+
+ self.layers = nn.ModuleList([])
+ for _ in range(depth):
+ self.layers.append(
+ nn.ModuleList(
+ [
+ PerceiverAttention(dim=dim, dim_head=dim_head, heads=heads),
+ FeedForward(dim=dim, mult=ff_mult),
+ ]
+ )
+ )
+
+ self.pretrained = kwargs.pop("pretrained", None)
+ self.freeze = kwargs.pop("freeze", False)
+ self.skip_last_layer = kwargs.pop("skip_last_layer", False)
+
+ if self.freeze and self.pretrained is None:
+ raise ValueError("Trying to freeze the model but no pretrained provided!")
+
+
+ def forward(self, x):
+ if self.pos_emb is not None:
+ n, device = x.shape[1], x.device
+ pos_emb = self.pos_emb(torch.arange(n, device=device))
+ x = x + pos_emb
+
+ latents = self.latents.repeat(x.size(0), 1, 1)
+
+ x = self.proj_in(x)
+
+ if self.to_latents_from_mean_pooled_seq:
+ meanpooled_seq = masked_mean(x, dim=1, mask=torch.ones(x.shape[:2], device=x.device, dtype=torch.bool))
+ meanpooled_latents = self.to_latents_from_mean_pooled_seq(meanpooled_seq)
+ latents = torch.cat((meanpooled_latents, latents), dim=-2)
+
+ for attn, ff in self.layers:
+ latents = attn(x, latents) + latents
+ latents = ff(latents) + latents
+
+ latents = self.proj_out(latents)
+ return self.norm_out(latents)
diff --git a/models/wan/modules/model.py b/models/wan/modules/model.py
index 82a313c..42af877 100644
--- a/models/wan/modules/model.py
+++ b/models/wan/modules/model.py
@@ -355,13 +355,38 @@ class WanSelfAttention(nn.Module):
class WanT2VCrossAttention(WanSelfAttention):
- def forward(self, xlist, context, grid_sizes, *args, **kwargs):
+ def forward(self, xlist, context, grid_sizes, lynx_ip_embeds = None, lynx_ip_scale = 0, *args, **kwargs):
r"""
Args:
x(Tensor): Shape [B, L1, C]
context(Tensor): Shape [B, L2, C]
"""
- x, _ = self.text_cross_attention( xlist, context)
+ x, q = self.text_cross_attention( xlist, context, return_q=lynx_ip_embeds is not None)
+ if lynx_ip_embeds is not None and self.to_k_ip is not None:
+ if self.registers is not None:
+ from ..lynx.navit_utils import vector_to_list, merge_token_lists, list_to_vector
+ ip_hidden_states_list = vector_to_list(lynx_ip_embeds, lynx_ip_embeds.shape[1], 1)
+ ip_hidden_states_list = merge_token_lists(ip_hidden_states_list, [self.registers] * len(ip_hidden_states_list), 1)
+ lynx_ip_embeds, ip_lens = list_to_vector(ip_hidden_states_list, 1)
+ ip_hidden_states_list = None
+ ip_key = self.to_k_ip(lynx_ip_embeds)
+ ip_value = self.to_v_ip(lynx_ip_embeds)
+ # if self.norm_q is not None: ip_query = self.norm_q(q)
+ if self.norm_rms_k is None:
+ ip_key = self.norm_k(ip_key)
+ else:
+ ip_key = self.norm_rms_k(ip_key)
+ ip_inner_dim = ip_key.shape[-1]
+ ip_head_dim = ip_inner_dim // self.num_heads
+ batch_size = q.shape[0]
+ # ip_query = ip_query.view(batch_size, -1, attn.heads, ip_head_dim)
+ ip_key = ip_key.view(batch_size, -1, self.num_heads, ip_head_dim)
+ ip_value = ip_value.view(batch_size, -1, self.num_heads, ip_head_dim)
+ qkv_list = [q, ip_key, ip_value]
+ del q, ip_key, ip_value
+ ip_hidden_states = pay_attention(qkv_list).reshape(*x.shape)
+ x.add_(ip_hidden_states, alpha= lynx_ip_scale)
+
x = self.o(x)
return x
@@ -382,7 +407,7 @@ class WanI2VCrossAttention(WanSelfAttention):
# self.alpha = nn.Parameter(torch.zeros((1, )))
self.norm_k_img = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity()
- def forward(self, xlist, context, grid_sizes, audio_proj, audio_scale, audio_context_lens ):
+ def forward(self, xlist, context, grid_sizes, audio_proj, audio_scale, audio_context_lens, *args, **kwargs ):
r"""
Args:
x(Tensor): Shape [B, L1, C]
@@ -510,6 +535,8 @@ class WanAttentionBlock(nn.Module):
ref_images_count=0,
standin_phase=-1,
motion_vec = None,
+ lynx_ip_embeds = None,
+ lynx_ip_scale = 0,
):
r"""
Args:
@@ -568,7 +595,7 @@ class WanAttentionBlock(nn.Module):
y = y.to(attention_dtype)
ylist= [y]
del y
- x += self.cross_attn(ylist, context, grid_sizes, audio_proj, audio_scale, audio_context_lens).to(dtype)
+ x += self.cross_attn(ylist, context, grid_sizes, audio_proj = audio_proj, audio_scale = audio_scale, audio_context_lens = audio_context_lens, lynx_ip_embeds=lynx_ip_embeds, lynx_ip_scale=lynx_ip_scale).to(dtype)
if multitalk_audio != None:
# cross attn of multitalk audio
@@ -914,6 +941,7 @@ class WanModel(ModelMixin, ConfigMixin):
norm_output_audio=True,
standin= False,
motion_encoder_dim=0,
+ lynx=None,
):
super().__init__()
@@ -1044,6 +1072,32 @@ class WanModel(ModelMixin, ConfigMixin):
block.self_attn.k_loras = LoRALinearLayer(dim, dim, rank=128)
block.self_attn.v_loras = LoRALinearLayer(dim, dim, rank=128)
+ if lynx is not None:
+ from ..lynx.light_attention_processor import IPAWanAttnProcessor2_0
+ lynx_full = lynx=="full"
+ if lynx_full:
+ lynx_cross_dim = 5120
+ lynx_layers = len(self.blocks)
+ else:
+ lynx_cross_dim = 2048
+ lynx_layers = 20
+ for i, block in enumerate(self.blocks):
+ if i < lynx_layers:
+ attn = IPAWanAttnProcessor2_0(hidden_size=dim, cross_attention_dim = lynx_cross_dim,)
+ block.cross_attn.to_k_ip = attn.to_k_ip
+ block.cross_attn.to_v_ip = attn.to_v_ip
+ else:
+ block.cross_attn.to_k_ip = None
+ block.cross_attn.to_v_ip = None
+ if lynx_full:
+ block.cross_attn.registers = nn.Parameter(torch.randn(1, 16, lynx_cross_dim) / dim**0.5)
+ block.cross_attn.norm_rms_k = None
+ else:
+ block.cross_attn.registers = None
+ block.cross_attn.norm_rms_k = attn.norm_rms_k
+
+
+
if animate:
self.pose_patch_embedding = nn.Conv3d(
16, dim, kernel_size=patch_size, stride=patch_size
@@ -1247,6 +1301,8 @@ class WanModel(ModelMixin, ConfigMixin):
standin_ref = None,
pose_latents=None,
face_pixel_values=None,
+ lynx_ip_embeds = None,
+ lynx_ip_scale = 0,
):
# patch_dtype = self.patch_embedding.weight.dtype
@@ -1286,11 +1342,11 @@ class WanModel(ModelMixin, ConfigMixin):
y = None
motion_vec_list = []
- for i, x in enumerate(x_list):
+ for i, (x, one_face_pixel_values) in enumerate(zip(x_list, face_pixel_values)):
# animate embeddings
motion_vec = None
if pose_latents is not None:
- x, motion_vec = after_patch_embedding(self, x, pose_latents, face_pixel_values)
+ x, motion_vec = after_patch_embedding(self, x, pose_latents, torch.zeros_like(one_face_pixel_values) if one_face_pixel_values is None else one_face_pixel_values)
motion_vec_list.append(motion_vec)
if chipmunk:
x = x.unsqueeze(-1)
@@ -1330,6 +1386,7 @@ class WanModel(ModelMixin, ConfigMixin):
audio_proj=audio_proj,
audio_context_lens=audio_context_lens,
ref_images_count=ref_images_count,
+ lynx_ip_scale= lynx_ip_scale,
)
_flag_df = t.dim() == 2
@@ -1348,6 +1405,11 @@ class WanModel(ModelMixin, ConfigMixin):
standin_e = self.time_embedding( sinusoidal_embedding_1d(self.freq_dim, torch.zeros_like(t)).to(modulation_dtype) )
standin_e0 = self.time_projection(standin_e).unflatten(1, (6, self.dim)).to(e.dtype)
standin_e = standin_ref = None
+
+ if lynx_ip_embeds is None:
+ lynx_ip_embeds_list = [None] * len(x_list)
+ else:
+ lynx_ip_embeds_list = lynx_ip_embeds
if self.inject_sample_info and fps!=None:
fps = torch.tensor(fps, dtype=torch.long, device=device)
@@ -1498,9 +1560,9 @@ class WanModel(ModelMixin, ConfigMixin):
continue
x_list[0] = block(x_list[0], context = context_list[0], audio_scale= audio_scale_list[0], e= e0, **kwargs)
else:
- for i, (x, context, hints, audio_scale, multitalk_audio, multitalk_masks, should_calc, motion_vec) in enumerate(zip(x_list, context_list, hints_list, audio_scale_list, multitalk_audio_list, multitalk_masks_list, x_should_calc,motion_vec_list)):
+ for i, (x, context, hints, audio_scale, multitalk_audio, multitalk_masks, should_calc, motion_vec, lynx_ip_embeds) in enumerate(zip(x_list, context_list, hints_list, audio_scale_list, multitalk_audio_list, multitalk_masks_list, x_should_calc,motion_vec_list, lynx_ip_embeds_list)):
if should_calc:
- x_list[i] = block(x, context = context, hints= hints, audio_scale= audio_scale, multitalk_audio = multitalk_audio, multitalk_masks =multitalk_masks, e= e0, motion_vec = motion_vec,**kwargs)
+ x_list[i] = block(x, context = context, hints= hints, audio_scale= audio_scale, multitalk_audio = multitalk_audio, multitalk_masks =multitalk_masks, e= e0, motion_vec = motion_vec, lynx_ip_embeds= lynx_ip_embeds, **kwargs)
del x
context = hints = audio_embedding = None
diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.py
index 1a30fff..e1829f0 100644
--- a/models/wan/wan_handler.py
+++ b/models/wan/wan_handler.py
@@ -17,6 +17,9 @@ def test_multitalk(base_model_type):
def test_standin(base_model_type):
return base_model_type in ["standin", "vace_standin_14B"]
+def test_lynx(base_model_type):
+ return base_model_type in ["lynx_lite", "vace_lynx_lite_14B", "lynx_full", "vace_lynx_full_14B"]
+
def test_wan_5B(base_model_type):
return base_model_type in ["ti2v_2_2", "lucy_edit"]
class family_handler():
@@ -82,6 +85,7 @@ class family_handler():
extra_model_def["i2v_class"] = i2v
extra_model_def["multitalk_class"] = test_multitalk(base_model_type)
extra_model_def["standin_class"] = test_standin(base_model_type)
+ extra_model_def["lynx_class"] = test_lynx(base_model_type)
vace_class = base_model_type in ["vace_14B", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B"]
extra_model_def["vace_class"] = vace_class
@@ -170,15 +174,26 @@ class family_handler():
"choices":[
("Animate Person in Reference Image using Motion of Whole Control Video", "PVBKI"),
("Animate Person in Reference Image using Motion of Targeted Person in Control Video", "PVBXAKI"),
- ("Replace Person in Control Video by Person in Reference Image", "PVBAI"),
- ("Replace Person in Control Video by Person in Reference Image and Apply Relighting Process", "PVBAI1"),
+ ("Replace Person in Control Video by Person in Ref Image", "PVBAIH#"),
+ ("Replace Person in Control Video by Person in Ref Image. See Through Mask", "PVBAI#"),
],
"default": "PVBKI",
- "letters_filter": "PVBXAKI1",
+ "letters_filter": "PVBXAKIH#",
"label": "Type of Process",
+ "scale": 3,
"show_label" : False,
}
+ extra_model_def["custom_video_selection"] = {
+ "choices":[
+ ("None", ""),
+ ("Apply Relighting", "1"),
+ ],
+ "label": "Custom Process",
+ "show_label" : False,
+ "scale": 1,
+ }
+
extra_model_def["mask_preprocessing"] = {
"selection":[ "", "A", "XA"],
"visible": False
@@ -229,7 +244,7 @@ class family_handler():
extra_model_def["forced_guide_mask_inputs"] = True
extra_model_def["return_image_refs_tensor"] = True
- if base_model_type in ["standin"]:
+ if base_model_type in ["standin", "lynx_lite", "lynx_full"]:
extra_model_def["fit_into_canvas_image_refs"] = 0
extra_model_def["image_ref_choices"] = {
"choices": [
@@ -298,7 +313,7 @@ class family_handler():
@staticmethod
def query_supported_types():
return ["multitalk", "infinitetalk", "fantasy", "vace_14B", "vace_multitalk_14B", "vace_standin_14B",
- "t2v_1.3B", "standin", "t2v", "vace_1.3B", "phantom_1.3B", "phantom_14B",
+ "t2v_1.3B", "standin", "lynx_lite", "lynx_full", "t2v", "vace_1.3B", "phantom_1.3B", "phantom_14B",
"recam_1.3B", "animate",
"i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp"]
@@ -312,8 +327,8 @@ class family_handler():
}
models_comp_map = {
- "vace_14B" : [ "vace_multitalk_14B", "vace_standin_14B"],
- "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "t2v_1.3B", "phantom_1.3B","phantom_14B", "standin"],
+ "vace_14B" : [ "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_full_14B"],
+ "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "t2v_1.3B", "phantom_1.3B","phantom_14B", "standin", "lynx_lite", "lynx_full"],
"i2v" : [ "fantasy", "multitalk", "flf2v_720p" ],
"i2v_2_2" : ["i2v_2_2_multitalk"],
"fantasy": ["multitalk"],
@@ -423,7 +438,7 @@ class family_handler():
ui_defaults["video_prompt_type"] = video_prompt_type
if settings_version < 2.31:
- if base_model_type in "recam_1.3B":
+ if base_model_type in ["recam_1.3B"]:
video_prompt_type = ui_defaults.get("video_prompt_type", "")
if not "V" in video_prompt_type:
video_prompt_type += "UV"
@@ -438,6 +453,14 @@ class family_handler():
if test_class_i2v(base_model_type) and len(image_prompt_type) == 0:
ui_defaults["image_prompt_type"] = "S"
+
+ if settings_version < 2.37:
+ if base_model_type in ["animate"]:
+ video_prompt_type = ui_defaults.get("video_prompt_type", "")
+ if "1" in video_prompt_type:
+ video_prompt_type = video_prompt_type.replace("1", "#1")
+ ui_defaults["video_prompt_type"] = video_prompt_type
+
@staticmethod
def update_default_settings(base_model_type, model_def, ui_defaults):
ui_defaults.update({
@@ -481,6 +504,15 @@ class family_handler():
"video_prompt_type": "I",
"remove_background_images_ref" : 1,
})
+ elif base_model_type in ["lynx_lite", "lynx_full"]:
+ ui_defaults.update({
+ "guidance_scale": 5.0,
+ "flow_shift": 7, # 11 for 720p
+ "sliding_window_overlap" : 9,
+ "video_prompt_type": "QI",
+ "remove_background_images_ref" : 1,
+ })
+
elif base_model_type in ["phantom_1.3B", "phantom_14B"]:
ui_defaults.update({
"guidance_scale": 7.5,
diff --git a/preprocessing/arc/face_encoder.py b/preprocessing/arc/face_encoder.py
new file mode 100644
index 0000000..0fb9162
--- /dev/null
+++ b/preprocessing/arc/face_encoder.py
@@ -0,0 +1,97 @@
+# Copyright 2025 Bytedance Ltd. and/or its affiliates
+# SPDX-License-Identifier: Apache-2.0
+
+import torch
+import torchvision.transforms as T
+
+import numpy as np
+
+from insightface.utils import face_align
+from insightface.app import FaceAnalysis
+from facexlib.recognition import init_recognition_model
+
+
+__all__ = [
+ "FaceEncoderArcFace",
+ "get_landmarks_from_image",
+]
+
+
+detector = None
+
+def get_landmarks_from_image(image):
+ """
+ Detect landmarks with insightface.
+
+ Args:
+ image (np.ndarray or PIL.Image):
+ The input image in RGB format.
+
+ Returns:
+ 5 2D keypoints, only one face will be returned.
+ """
+ global detector
+ if detector is None:
+ detector = FaceAnalysis()
+ detector.prepare(ctx_id=0, det_size=(640, 640))
+
+ in_image = np.array(image).copy()
+
+ faces = detector.get(in_image)
+ if len(faces) == 0:
+ raise ValueError("No face detected in the image")
+
+ # Get the largest face
+ face = max(faces, key=lambda x: (x.bbox[2] - x.bbox[0]) * (x.bbox[3] - x.bbox[1]))
+
+ # Return the 5 keypoints directly
+ keypoints = face.kps # 5 x 2
+
+ return keypoints
+
+
+class FaceEncoderArcFace():
+ """ Official ArcFace, no_grad-only """
+
+ def __repr__(self):
+ return "ArcFace"
+
+
+ def init_encoder_model(self, device, eval_mode=True):
+ self.device = device
+ self.encoder_model = init_recognition_model('arcface', device=device)
+
+ if eval_mode:
+ self.encoder_model.eval()
+
+
+ @torch.no_grad()
+ def input_preprocessing(self, in_image, landmarks, image_size=112):
+ assert landmarks is not None, "landmarks are not provided!"
+
+ in_image = np.array(in_image)
+ landmark = np.array(landmarks)
+
+ face_aligned = face_align.norm_crop(in_image, landmark=landmark, image_size=image_size)
+
+ image_transform = T.Compose([
+ T.ToTensor(),
+ T.Normalize([0.5], [0.5]),
+ ])
+ face_aligned = image_transform(face_aligned).unsqueeze(0).to(self.device)
+
+ return face_aligned
+
+
+ @torch.no_grad()
+ def __call__(self, in_image, need_proc=False, landmarks=None, image_size=112):
+
+ if need_proc:
+ in_image = self.input_preprocessing(in_image, landmarks, image_size)
+ else:
+ assert isinstance(in_image, torch.Tensor)
+
+ in_image = in_image[:, [2, 1, 0], :, :].contiguous()
+ image_embeds = self.encoder_model(in_image) # [B, 512], normalized
+
+ return image_embeds
\ No newline at end of file
diff --git a/preprocessing/arc/face_utils.py b/preprocessing/arc/face_utils.py
new file mode 100644
index 0000000..41a4545
--- /dev/null
+++ b/preprocessing/arc/face_utils.py
@@ -0,0 +1,63 @@
+# Copyright (c) 2022 Insightface Team
+# Copyright (c) 2025 Bytedance Ltd. and/or its affiliates
+
+# This file has been modified by Bytedance Ltd. and/or its affiliates on September 15, 2025.
+# SPDX-License-Identifier: Apache-2.0
+
+# Original file (insightface) was released under MIT License:
+
+# Permission is hereby granted, free of charge, to any person obtaining a copy
+# of this software and associated documentation files (the "Software"), to deal
+# in the Software without restriction, including without limitation the rights
+# to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+# copies of the Software, and to permit persons to whom the Software is
+# furnished to do so, subject to the following conditions:
+
+# The above copyright notice and this permission notice shall be included in all
+# copies or substantial portions of the Software.
+
+# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+# SOFTWARE.
+
+import numpy as np
+import cv2
+from PIL import Image
+
+def estimate_norm(lmk, image_size=112, arcface_dst=None):
+ from skimage import transform as trans
+ assert lmk.shape == (5, 2)
+ assert image_size%112==0 or image_size%128==0
+ if image_size%112==0:
+ ratio = float(image_size)/112.0
+ diff_x = 0
+ else:
+ ratio = float(image_size)/128.0
+ diff_x = 8.0*ratio
+ dst = arcface_dst * ratio
+ dst[:,0] += diff_x
+ tform = trans.SimilarityTransform()
+ tform.estimate(lmk, dst)
+ M = tform.params[0:2, :]
+ return M
+
+def get_arcface_dst(extend_face_crop=False, extend_ratio=0.8):
+ arcface_dst = np.array(
+ [[38.2946, 51.6963], [73.5318, 51.5014], [56.0252, 71.7366],
+ [41.5493, 92.3655], [70.7299, 92.2041]], dtype=np.float32)
+ if extend_face_crop:
+ arcface_dst[:,1] = arcface_dst[:,1] + 10
+ arcface_dst = (arcface_dst - 112/2) * extend_ratio + 112/2
+ return arcface_dst
+
+def align_face(image_pil, face_kpts, extend_face_crop=False, extend_ratio=0.8, face_size=112):
+ arcface_dst = get_arcface_dst(extend_face_crop, extend_ratio)
+ M = estimate_norm(face_kpts, face_size, arcface_dst)
+ image_cv2 = cv2.cvtColor(np.array(image_pil), cv2.COLOR_RGB2BGR)
+ face_image_cv2 = cv2.warpAffine(image_cv2, M, (face_size, face_size), borderValue=0.0)
+ face_image = Image.fromarray(cv2.cvtColor(face_image_cv2, cv2.COLOR_BGR2RGB))
+ return face_image
\ No newline at end of file
diff --git a/requirements.txt b/requirements.txt
index d6f75f7..e567f3a 100644
--- a/requirements.txt
+++ b/requirements.txt
@@ -35,6 +35,8 @@ rembg[gpu]==2.0.65
onnxruntime-gpu
decord
timm
+# insightface==0.7.3
+# facexlib==0.3.0
# Config & orchestration
omegaconf
diff --git a/wgp.py b/wgp.py
index 6a257e8..9143689 100644
--- a/wgp.py
+++ b/wgp.py
@@ -63,8 +63,8 @@ AUTOSAVE_FILENAME = "queue.zip"
PROMPT_VARS_MAX = 10
target_mmgp_version = "3.6.0"
-WanGP_version = "8.75"
-settings_version = 2.36
+WanGP_version = "8.76"
+settings_version = 2.37
max_source_video_frames = 3000
prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, None
@@ -3736,7 +3736,7 @@ def extract_faces_from_video_with_mask(input_video_path, input_mask_path, max_fr
alpha_mask[mask > 127] = 1
frame = frame * alpha_mask
frame = Image.fromarray(frame)
- face = face_processor.process(frame, resize_to=size, face_crop_scale = 1)
+ face = face_processor.process(frame, resize_to=size, face_crop_scale = 1.2,)
face_list.append(face)
face_processor = None
@@ -3863,7 +3863,7 @@ def preprocess_video_with_mask(input_video_path, input_mask_path, height, width,
mask = op_expand(mask, kernel, iterations=3)
_, mask = cv2.threshold(mask, 127.5, 255, cv2.THRESH_BINARY)
- if to_bbox and np.sum(mask == 255) > 0:
+ if to_bbox and np.sum(mask == 255) > 0 : #or True
x0, y0, x1, y1 = mask_to_xyxy_box(mask)
mask = mask * 0
mask[y0:y1, x0:x1] = 255
@@ -4766,7 +4766,7 @@ def generate_video(
raise Exception(f"unknown cache type {skip_steps_cache_type}")
trans.cache = skip_steps_cache
if trans2 is not None: trans2.cache = skip_steps_cache
-
+ face_arc_embeds = None
output_new_audio_data = None
output_new_audio_filepath = None
original_audio_guide = audio_guide
@@ -5039,9 +5039,9 @@ def generate_video(
context_scale = [control_net_weight /2, control_net_weight2 /2] if preprocess_type2 is not None else [control_net_weight]
if not (preprocess_type == "identity" and preprocess_type2 is None and video_mask is None):send_cmd("progress", [0, get_latest_status(state, status_info)])
inpaint_color = 0 if preprocess_type=="pose" and process_outside_mask == "inpaint" else guide_inpaint_color
- video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size )
+ video_guide_processed, video_mask_processed = preprocess_video_with_mask(video_guide if sparse_video_image is None else sparse_video_image, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =1, inpaint_color =inpaint_color, block_size = block_size, to_bbox = "H" in video_prompt_type )
if preprocess_type2 != None:
- video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size )
+ video_guide_processed2, video_mask_processed2 = preprocess_video_with_mask(video_guide, video_mask, height=image_size[0], width = image_size[1], max_frames= guide_frames_extract_count, start_frame = guide_frames_extract_start, fit_canvas = sample_fit_canvas, fit_crop = fit_crop, target_fps = fps, process_type = preprocess_type2, expand_scale = mask_expand, RGB_Mask = True, negate_mask = "N" in video_prompt_type, process_outside_mask = process_outside_mask, outpainting_dims = outpainting_dims, proc_no =2, block_size = block_size, to_bbox = "H" in video_prompt_type )
if video_guide_processed is not None and sample_fit_canvas is not None:
image_size = video_guide_processed.shape[-2:]
@@ -5069,6 +5069,17 @@ def generate_video(
if len(image_refs) > nb_frames_positions:
src_ref_images = image_refs[nb_frames_positions:]
+ if "Q" in video_prompt_type:
+ from preprocessing.arc.face_encoder import FaceEncoderArcFace, get_landmarks_from_image
+ image_pil = src_ref_images[-1]
+ face_encoder = FaceEncoderArcFace()
+ face_encoder.init_encoder_model(processing_device)
+ face_arc_embeds = face_encoder(image_pil, need_proc=True, landmarks=get_landmarks_from_image(image_pil))
+ face_arc_embeds = face_arc_embeds.squeeze(0).cpu()
+ face_encoder = image_pil = None
+ gc.collect()
+ torch.cuda.empty_cache()
+
if remove_background_images_ref > 0:
send_cmd("progress", [0, get_latest_status(state, "Removing Images References Background")])
@@ -5079,7 +5090,6 @@ def generate_video(
outpainting_dims =outpainting_dims,
background_ref_outpainted = model_def.get("background_ref_outpainted", True),
return_tensor= model_def.get("return_image_refs_tensor", False) )
-
frames_to_inject_parsed = frames_to_inject[ window_start_frame if extract_guide_from_window_start else guide_start_frame: guide_end_frame]
if video_guide is not None or len(frames_to_inject_parsed) > 0 or model_def.get("forced_guide_mask_inputs", False):
@@ -5241,6 +5251,7 @@ def generate_video(
original_input_ref_images = original_image_refs[nb_frames_positions:] if original_image_refs is not None else [],
image_refs_relative_size = image_refs_relative_size,
outpainting_dims = outpainting_dims,
+ face_arc_embeds = face_arc_embeds,
)
except Exception as e:
if len(control_audio_tracks) > 0 or len(source_audio_tracks) > 0:
@@ -6980,7 +6991,7 @@ def refresh_video_prompt_type_alignment(state, video_prompt_type, video_prompt_t
video_prompt_type = add_to_sequence(video_prompt_type, video_prompt_type_video_guide)
return video_prompt_type
-all_guide_processes ="PDESLCMUVB"
+all_guide_processes ="PDESLCMUVBH"
def refresh_video_prompt_type_video_guide(state, filter_type, video_prompt_type, video_prompt_type_video_guide, image_mode, old_image_mask_guide_value, old_image_guide_value, old_image_mask_value ):
model_type = state["model_type"]
@@ -7006,7 +7017,34 @@ def refresh_video_prompt_type_video_guide(state, filter_type, video_prompt_type,
else:
mask_selector_visible = True
ref_images_visible = "I" in video_prompt_type
- return video_prompt_type, gr.update(visible = visible and not image_outputs), image_guide, gr.update(visible = keep_frames_video_guide_visible), gr.update(visible = visible and "G" in video_prompt_type), gr.update(visible= (visible or "F" in video_prompt_type or "K" in video_prompt_type) and any_outpainting), gr.update(visible= visible and mask_selector_visible and not "U" in video_prompt_type ) , gr.update(visible= mask_visible and not image_outputs), image_mask, image_mask_guide, gr.update(visible= mask_visible), gr.update(visible = ref_images_visible )
+ custom_options = custom_checkbox= False
+ if "#" in video_prompt_type:
+ custom_options = True
+ custom_video_choices = model_def.get("custom_video_selection", None)
+ if custom_video_choices is not None:
+ custom_checkbox = len(custom_video_choices.get("choices", [])) <= 2
+
+ return video_prompt_type, gr.update(visible = visible and not image_outputs), image_guide, gr.update(visible = keep_frames_video_guide_visible), gr.update(visible = visible and "G" in video_prompt_type), gr.update(visible= (visible or "F" in video_prompt_type or "K" in video_prompt_type) and any_outpainting), gr.update(visible= visible and mask_selector_visible and not "U" in video_prompt_type ) , gr.update(visible= mask_visible and not image_outputs), image_mask, image_mask_guide, gr.update(visible= mask_visible), gr.update(visible = ref_images_visible ), gr.update(visible= custom_options and not custom_checkbox ), gr.update(visible= custom_options and custom_checkbox )
+
+def refresh_video_prompt_type_video_custom_dropbox(state, video_prompt_type, video_prompt_type_video_custom_dropbox):
+ model_type = state["model_type"]
+ model_def = get_model_def(model_type)
+ custom_video_choices = model_def.get("custom_video_selection", None)
+ if not "#" in video_prompt_type or custom_video_choices is None: return gr.update()
+ video_prompt_type = del_in_sequence(video_prompt_type, "0123456789")
+ video_prompt_type = add_to_sequence(video_prompt_type, video_prompt_type_video_custom_dropbox)
+ return video_prompt_type
+
+def refresh_video_prompt_type_video_custom_checkbox(state, video_prompt_type, video_prompt_type_video_custom_checkbox):
+ model_type = state["model_type"]
+ model_def = get_model_def(model_type)
+ custom_video_choices = model_def.get("custom_video_selection", None)
+ if not "#" in video_prompt_type or custom_video_choices is None: return gr.update()
+ video_prompt_type = del_in_sequence(video_prompt_type, "0123456789")
+ if video_prompt_type_video_custom_checkbox:
+ video_prompt_type = add_to_sequence(video_prompt_type, custom_video_choices["choices"][1][1])
+ return video_prompt_type
+
# def refresh_video_prompt_type_video_guide_alt(state, video_prompt_type, video_prompt_type_video_guide_alt, image_mode, old_image_mask_guide_value, old_image_guide_value, old_image_mask_value ):
@@ -7474,7 +7512,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
else:
image_prompt_type_radio = gr.Radio(choices=[("", "")], value="", visible= False)
if "E" in image_prompt_types_allowed:
- image_prompt_type_endcheckbox = gr.Checkbox( value ="E" in image_prompt_type_value, label="End Image(s)", show_label= False, visible= any_letters(image_prompt_type_radio_value, "SVL") and not image_outputs , scale= 1)
+ image_prompt_type_endcheckbox = gr.Checkbox( value ="E" in image_prompt_type_value, label="End Image(s)", show_label= False, visible= any_letters(image_prompt_type_radio_value, "SVL") and not image_outputs , scale= 1, elem_classes="cbx_centered")
any_end_image = True
else:
image_prompt_type_endcheckbox = gr.Checkbox( value =False, show_label= False, visible= False , scale= 1)
@@ -7539,14 +7577,14 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
video_prompt_type_video_guide = gr.Dropdown(
guide_preprocessing_choices,
value=filter_letters(video_prompt_type_value, all_guide_processes, guide_preprocessing.get("default", "") ),
- label= video_prompt_type_video_guide_label , scale = 2, visible= guide_preprocessing.get("visible", True) , show_label= True,
+ label= video_prompt_type_video_guide_label , scale = 1, visible= guide_preprocessing.get("visible", True) , show_label= True,
)
any_control_video = True
any_control_image = image_outputs
# Alternate Control Video Preprocessing / Options
if guide_custom_choices is None:
- video_prompt_type_video_guide_alt = gr.Dropdown(choices=[("","")], value="", label="Control Video", visible= False, scale = 2 )
+ video_prompt_type_video_guide_alt = gr.Dropdown(choices=[("","")], value="", label="Control Video", visible= False, scale = 1 )
else:
video_prompt_type_video_guide_alt_label = guide_custom_choices.get("label", "Control Video Process")
if image_outputs: video_prompt_type_video_guide_alt_label = video_prompt_type_video_guide_alt_label.replace("Video", "Image")
@@ -7557,14 +7595,37 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
# value=filter_letters(video_prompt_type_value, guide_custom_choices["letters_filter"], guide_custom_choices.get("default", "") ),
value=guide_custom_choices_value,
visible = guide_custom_choices.get("visible", True),
- label= video_prompt_type_video_guide_alt_label, show_label= guide_custom_choices.get("show_label", True), scale = 2
+ label= video_prompt_type_video_guide_alt_label, show_label= guide_custom_choices.get("show_label", True), scale = guide_custom_choices.get("scale", 1),
)
any_control_video = True
any_control_image = image_outputs
+ # Custom dropdown box & checkbox
+ custom_video_selection = model_def.get("custom_video_selection", None)
+ custom_checkbox= False
+ if custom_video_selection is None:
+ video_prompt_type_video_custom_dropbox = gr.Dropdown(choices=[("","")], value="", label="Custom Dropdown", scale = 1, visible= False, show_label= True, )
+ video_prompt_type_video_custom_checkbox = gr.Checkbox(value=False, label="Custom Checkbbox", scale = 1, visible= False, show_label= True, )
+
+ else:
+ custom_choices = "#" in video_prompt_type_value
+ custom_video_choices = custom_video_selection["choices"]
+ custom_checkbox = len(custom_video_choices) <= 2
+
+ video_prompt_type_video_custom_label = custom_video_selection.get("label", "Custom Choices")
+
+ video_prompt_type_video_custom_dropbox = gr.Dropdown(
+ custom_video_choices,
+ value=filter_letters(video_prompt_type_value, "0123456789", custom_video_selection.get("default", "")),
+ scale = custom_video_selection.get("scale", 1),
+ label= video_prompt_type_video_custom_label , visible= not custom_checkbox and custom_choices,
+ show_label= custom_video_selection.get("show_label", True),
+ )
+ video_prompt_type_video_custom_checkbox = gr.Checkbox(value= custom_video_choices[1][1] in video_prompt_type_value , label=custom_video_choices[1][0] , scale = custom_video_selection.get("scale", 1), visible=custom_checkbox and custom_choices, show_label= True, elem_classes="cbx_centered" )
+
# Control Mask Preprocessing
if mask_preprocessing is None:
- video_prompt_type_video_mask = gr.Dropdown(choices=[("","")], value="", label="Video Mask", scale = 2, visible= False, show_label= True, )
+ video_prompt_type_video_mask = gr.Dropdown(choices=[("","")], value="", label="Video Mask", scale = 1, visible= False, show_label= True, )
any_image_mask = image_outputs
else:
mask_preprocessing_labels_all = {
@@ -7592,7 +7653,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
video_prompt_type_video_mask = gr.Dropdown(
mask_preprocessing_choices,
value=filter_letters(video_prompt_type_value, "XYZWNA", mask_preprocessing.get("default", "")),
- label= video_prompt_type_video_mask_label , scale = 2, visible= "V" in video_prompt_type_value and not "U" in video_prompt_type_value and mask_preprocessing.get("visible", True),
+ label= video_prompt_type_video_mask_label , scale = 1, visible= "V" in video_prompt_type_value and not "U" in video_prompt_type_value and mask_preprocessing.get("visible", True),
show_label= True,
)
@@ -7604,7 +7665,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
choices=[ ("None", ""),],
value=filter_letters(video_prompt_type_value, ""),
visible = False,
- label="Start / Reference Images", scale = 2
+ label="Start / Reference Images", scale = 1
)
any_reference_image = False
else:
@@ -7613,7 +7674,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
choices= image_ref_choices["choices"],
value=filter_letters(video_prompt_type_value, image_ref_choices["letters_filter"]),
visible = image_ref_choices.get("visible", True),
- label=image_ref_choices.get("label", "Inject Reference Images"), show_label= True, scale = 2
+ label=image_ref_choices.get("label", "Inject Reference Images"), show_label= True, scale = 1
)
image_guide = gr.Image(label= "Control Image", height = 800, type ="pil", visible= image_mode_value==1 and "V" in video_prompt_type_value and ("U" in video_prompt_type_value or not "A" in video_prompt_type_value ) , value= ui_defaults.get("image_guide", None))
@@ -8287,7 +8348,8 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
extra_inputs = prompt_vars + [wizard_prompt, wizard_variables_var, wizard_prompt_activated_var, video_prompt_column, image_prompt_column, image_prompt_type_group, image_prompt_type_radio, image_prompt_type_endcheckbox,
prompt_column_advanced, prompt_column_wizard_vars, prompt_column_wizard, lset_name, save_lset_prompt_drop, advanced_row, speed_tab, audio_tab, mmaudio_col, quality_tab,
sliding_window_tab, misc_tab, prompt_enhancer_row, inference_steps_row, skip_layer_guidance_row, audio_guide_row, RIFLEx_setting_col,
- video_prompt_type_video_guide, video_prompt_type_video_guide_alt, video_prompt_type_video_mask, video_prompt_type_image_refs, apg_col, audio_prompt_type_sources, audio_prompt_type_remux, audio_prompt_type_remux_row,
+ video_prompt_type_video_guide, video_prompt_type_video_guide_alt, video_prompt_type_video_mask, video_prompt_type_image_refs, video_prompt_type_video_custom_dropbox, video_prompt_type_video_custom_checkbox,
+ apg_col, audio_prompt_type_sources, audio_prompt_type_remux, audio_prompt_type_remux_row,
video_guide_outpainting_col,video_guide_outpainting_top, video_guide_outpainting_bottom, video_guide_outpainting_left, video_guide_outpainting_right,
video_guide_outpainting_checkbox, video_guide_outpainting_row, show_advanced, video_info_to_control_video_btn, video_info_to_video_source_btn, sample_solver_row,
video_buttons_row, image_buttons_row, video_postprocessing_tab, audio_remuxing_tab, PP_MMAudio_row, PP_custom_audio_row,
@@ -8316,9 +8378,12 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
image_prompt_type_radio.change(fn=refresh_image_prompt_type_radio, inputs=[state, image_prompt_type, image_prompt_type_radio], outputs=[image_prompt_type, image_start_row, image_end_row, video_source, keep_frames_video_source, image_prompt_type_endcheckbox], show_progress="hidden" )
image_prompt_type_endcheckbox.change(fn=refresh_image_prompt_type_endcheckbox, inputs=[state, image_prompt_type, image_prompt_type_radio, image_prompt_type_endcheckbox], outputs=[image_prompt_type, image_end_row] )
video_prompt_type_image_refs.input(fn=refresh_video_prompt_type_image_refs, inputs = [state, video_prompt_type, video_prompt_type_image_refs,image_mode], outputs = [video_prompt_type, image_refs_row, remove_background_images_ref, image_refs_relative_size, frames_positions,video_guide_outpainting_col], show_progress="hidden")
- video_prompt_type_video_guide.input(fn=refresh_video_prompt_type_video_guide, inputs = [state, gr.State(""), video_prompt_type, video_prompt_type_video_guide, image_mode, image_mask_guide, image_guide, image_mask], outputs = [video_prompt_type, video_guide, image_guide, keep_frames_video_guide, denoising_strength, video_guide_outpainting_col, video_prompt_type_video_mask, video_mask, image_mask, image_mask_guide, mask_expand, image_refs_row], show_progress="hidden")
- video_prompt_type_video_guide_alt.input(fn=refresh_video_prompt_type_video_guide, inputs = [state, gr.State("alt"),video_prompt_type, video_prompt_type_video_guide_alt, image_mode, image_mask_guide, image_guide, image_mask], outputs = [video_prompt_type, video_guide, image_guide, keep_frames_video_guide, denoising_strength, video_guide_outpainting_col, video_prompt_type_video_mask, video_mask, image_mask, image_mask_guide, mask_expand, image_refs_row], show_progress="hidden")
+ video_prompt_type_video_guide.input(fn=refresh_video_prompt_type_video_guide, inputs = [state, gr.State(""), video_prompt_type, video_prompt_type_video_guide, image_mode, image_mask_guide, image_guide, image_mask], outputs = [video_prompt_type, video_guide, image_guide, keep_frames_video_guide, denoising_strength, video_guide_outpainting_col, video_prompt_type_video_mask, video_mask, image_mask, image_mask_guide, mask_expand, image_refs_row, video_prompt_type_video_custom_dropbox, video_prompt_type_video_custom_checkbox], show_progress="hidden")
+ video_prompt_type_video_guide_alt.input(fn=refresh_video_prompt_type_video_guide, inputs = [state, gr.State("alt"),video_prompt_type, video_prompt_type_video_guide_alt, image_mode, image_mask_guide, image_guide, image_mask], outputs = [video_prompt_type, video_guide, image_guide, keep_frames_video_guide, denoising_strength, video_guide_outpainting_col, video_prompt_type_video_mask, video_mask, image_mask, image_mask_guide, mask_expand, image_refs_row, video_prompt_type_video_custom_dropbox, video_prompt_type_video_custom_checkbox], show_progress="hidden")
# video_prompt_type_video_guide_alt.input(fn=refresh_video_prompt_type_video_guide_alt, inputs = [state, video_prompt_type, video_prompt_type_video_guide_alt, image_mode, image_mask_guide, image_guide, image_mask], outputs = [video_prompt_type, video_guide, image_guide, image_refs_row, denoising_strength, video_mask, mask_expand, image_mask_guide, image_guide, image_mask, keep_frames_video_guide ], show_progress="hidden")
+ video_prompt_type_video_custom_dropbox.input(fn= refresh_video_prompt_type_video_custom_dropbox, inputs=[state, video_prompt_type, video_prompt_type_video_custom_dropbox], outputs = video_prompt_type)
+ video_prompt_type_video_custom_checkbox.input(fn= refresh_video_prompt_type_video_custom_checkbox, inputs=[state, video_prompt_type, video_prompt_type_video_custom_checkbox], outputs = video_prompt_type)
+
video_prompt_type_video_mask.input(fn=refresh_video_prompt_type_video_mask, inputs = [state, video_prompt_type, video_prompt_type_video_mask, image_mode, image_mask_guide, image_guide, image_mask], outputs = [video_prompt_type, video_mask, image_mask_guide, image_guide, image_mask, mask_expand], show_progress="hidden")
video_prompt_type_alignment.input(fn=refresh_video_prompt_type_alignment, inputs = [state, video_prompt_type, video_prompt_type_alignment], outputs = [video_prompt_type])
multi_prompts_gen_type.select(fn=refresh_prompt_labels, inputs=[multi_prompts_gen_type, image_mode], outputs=[prompt, wizard_prompt, image_end], show_progress="hidden")
@@ -9587,6 +9652,7 @@ def create_ui():
transition: visibility 0s linear 1s, opacity 0.3s linear 1s; /* 1s delay before showing */
}
.btn_centered {margin-top:10px; text-wrap-mode: nowrap;}
+ .cbx_centered label {margin-top:8px; text-wrap-mode: nowrap;}
"""
UI_theme = server_config.get("UI_theme", "default")
UI_theme = args.theme if len(args.theme) > 0 else UI_theme