Python
ovi support - intermediate commit
deepbeepmeep40a00a152856
diff --git a/defaults/ovi.json b/defaults/ovi.jsonnew file mode 100644index 0000000..0bfbef8--- /dev/null+++ b/defaults/ovi.json@@ -0,0 +1,19 @@+{+ "model": {+ "name": "Wan2.2 Ovi 10B",+ "architecture": "ovi",+ "description": "Ovi will generate an Audio soundtrack with the Video. It is specialized in speaking characters. Use the tags <S> and <E> to delimit the speaker words and <AUDCAP> and <ENDAUDCAP> to set the background noise.",+ "URLs": [+ "https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan2.2_ovi_video_10B_bf16.safetensors",+ "https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan2.2_ovi_video_10B_quanto_bf16_int8.safetensors"+ ],+ "URLs2": [+ "https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan2.2_ovi_audio_10B_bf16.safetensors",+ "https://huggingface.co/DeepBeepMeep/Wan2.2/resolve/main/wan2.2_ovi_audio_10B_quanto_bf16_int8.safetensors"+ ]+ },+ "resolution": "1280x720",+ "num_inference_steps": 30,+ "prompt" : "A singer in a glittering jacket grips the microphone, sweat shining on his brow, and shouts, <S>The end is night<E>. The crowd roars in response, fists in the air. Behind him, a guitarist steps to the mic and adds to say <S>We must all find a bunker where to hide.<E>. The energy peaks as the lights flare brighter.. <AUDCAP>Electric guitar riffs, cheering crowd, shouted male voices.<ENDAUDCAP>"++}diff --git a/models/wan/__init__.py b/models/wan/__init__.pyindex fe3be71..893b061 100644--- a/models/wan/__init__.py+++ b/models/wan/__init__.py@@ -1,4 +1,4 @@from . import configs, distributed, modulesfrom .any2video import WanAny2Vfrom .diffusion_forcing import DTT2V-from . import wan_handler, df_handler+from . import wan_handler, df_handler, ovi_handlerdiff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex 6484f41..ac5a2eb 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -137,7 +137,7 @@ class WanAny2V:self.patch_size = config.patch_sizeself.vae = vae( vae_pth=fl.locate_file(vae_checkpoint), dtype= VAE_dtype, device="cpu")- self.vae.device = self.device+ self.vae.device = self.device # need to set to cuda so that vae buffers are properly moved (although the rest will stay in the CPU)self.vae2 = Noneif vae_checkpoint2 is not None:self.vae2 = vae( vae_pth=fl.locate_file(vae_checkpoint2), dtype= VAE_dtype, device="cpu")diff --git a/models/wan/modules/model.py b/models/wan/modules/model.pyindex 7485c01..fda9bf2 100644--- a/models/wan/modules/model.py+++ b/models/wan/modules/model.py@@ -338,9 +338,9 @@ class WanSelfAttention(nn.Module):self.norm_k(k)q,k,v = q.view(b, s, n, d), k.view(b, s, n, d), v.view(b, s, n, d)del x+qklist = [q,k]del q,k-q,k = apply_rotary_emb(qklist, freqs, head_first=False)if standin_phase >= 1:diff --git a/models/wan/modules/posemb_layers.py b/models/wan/modules/posemb_layers.pyindex 083587a..8a99893 100644--- a/models/wan/modules/posemb_layers.py+++ b/models/wan/modules/posemb_layers.py@@ -228,6 +228,28 @@ def rotate_half(x):) # [B, S, H, D//2]return torch.stack([-x_imag, x_real], dim=-1).flatten(3)+def apply_rotary_emb_single( qklist,+ freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],+ head_first: bool = False,+):+ xq = qklist[0]+ qklist.clear()+ xk_out = None+ cos, sin = reshape_for_broadcast(freqs_cis, xq, head_first) # [S, D]+ cos, sin = cos.to(xq.device), sin.to(xq.device)+ # real * cos - imag * sin+ # imag * cos + real * sin+ xq_dtype = xq.dtype+ xq_out = xq.to(torch.float)+ xq = None+ xq_rot = rotate_half(xq_out)+ xq_out *= cos+ xq_rot *= sin+ xq_out += xq_rot+ del xq_rot+ xq_out = xq_out.to(xq_dtype)+ return xq_out+def apply_rotary_emb( qklist,freqs_cis: Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]],@@ -308,7 +330,9 @@ def get_nd_rotary_pos_embed(interpolation_factor: Union[float, List[float]] = 1.0,k = 6,L_test = 66,- enable_riflex = False+ enable_riflex = False,+ rope_dim_list = [44, 42, 42],+ head_dim = 128,):"""This is a n-d version of precompute_freqs_cis, which is a RoPE for tokens with n-d structure.@@ -328,8 +352,6 @@ def get_nd_rotary_pos_embed(Returns:pos_embed (torch.Tensor): [HW, D/2]"""- head_dim = 128- rope_dim_list = [44, 42, 42]if rope_dim_list is None:rope_dim_list = [head_dim // target_ndim for _ in range(target_ndim)]assert (diff --git a/models/wan/ovi/__init__.py b/models/wan/ovi/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/models/wan/ovi/configs/audio.json b/models/wan/ovi/configs/audio.jsonnew file mode 100644index 0000000..869d375--- /dev/null+++ b/models/wan/ovi/configs/audio.json@@ -0,0 +1,17 @@+{+ "patch_size": [1],+ "model_type": "t2a",+ "dim": 3072,+ "ffn_dim": 14336,+ "freq_dim": 256,+ "num_heads": 24,+ "num_layers": 30,+ "in_dim": 20,+ "out_dim": 20,+ "text_len": 512,+ "window_size": [-1, -1],+ "qk_norm": true,+ "cross_attn_norm": true,+ "eps": 1e-6,+ "temporal_rope_scaling_factor": 0.19676+}\ No newline at end of filediff --git a/models/wan/ovi/configs/inference_fusion.yaml b/models/wan/ovi/configs/inference_fusion.yamlnew file mode 100644index 0000000..dab7eb9--- /dev/null+++ b/models/wan/ovi/configs/inference_fusion.yaml@@ -0,0 +1,18 @@+ckpt_dir: ./ckpts+output_dir: ./outputs+num_steps: 50+solver_name: unipc+shift: 5.0+sp_size: 1+audio_guidance_scale: 3.0+video_guidance_scale: 4.0+mode: "i2v" # ["t2v", "i2v", "t2i2v"] all comes with audio+fp8: False+cpu_offload: False+seed: 103+video_negative_prompt: "jitter, bad hands, blur, distortion" # Artifacts to avoid in video+audio_negative_prompt: "robotic, muffled, echo, distorted" # Artifacts to avoid in audio+video_frame_height_width: [512, 992] # only useful if mode = t2v or t2i2v, recommended values: [512, 992], [992, 512], [960, 512], [512, 960], [720, 720], [448, 1120]+text_prompt: example_prompts/gpt_examples_i2v.csv+slg_layer: 11+each_example_n_times: 2\ No newline at end of filediff --git a/models/wan/ovi/configs/video.json b/models/wan/ovi/configs/video.jsonnew file mode 100644index 0000000..11cc4b3--- /dev/null+++ b/models/wan/ovi/configs/video.json@@ -0,0 +1,16 @@+{+ "patch_size": [1, 2, 2],+ "model_type": "ti2v",+ "dim": 3072,+ "ffn_dim": 14336,+ "freq_dim": 256,+ "num_heads": 24,+ "num_layers": 30,+ "in_dim": 48,+ "out_dim": 48,+ "text_len": 512,+ "window_size": [-1, -1],+ "qk_norm": true,+ "cross_attn_norm": true,+ "eps": 1e-6+}\ No newline at end of filediff --git a/models/wan/ovi/modules/__init__.py b/models/wan/ovi/modules/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/models/wan/ovi/modules/attention.py b/models/wan/ovi/modules/attention.pynew file mode 100644index 0000000..7377c1e--- /dev/null+++ b/models/wan/ovi/modules/attention.py@@ -0,0 +1,163 @@+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.+import torch++++def attention_with_weights(+ q,+ k,+ v,+ q_lens=None,+ k_lens=None,+ softmax_scale=None,+ q_scale=None,+ causal=False,+ average_for_q=False,+ total_video_latent_frames = 21+):+ """+ Compute attention with explicit attention weights for visualization.+ Returns both output and attention weights.+ """+ out_dtype = q.dtype++ # Handle sequence lengths+ b, lq, lk = q.size(0), q.size(1), k.size(1)++ if q_lens is None:+ q_lens = torch.tensor([lq] * b, dtype=torch.int32, device=q.device)+ else:+ # Ensure q_lens is on the same device as q+ q_lens = q_lens.to(q.device)++ if k_lens is None:+ k_lens = torch.tensor([lk] * b, dtype=torch.int32, device=k.device)+ else:+ # Ensure k_lens is on the same device as k+ k_lens = k_lens.to(k.device)++ # Apply q_scale if provided+ if q_scale is not None:+ q = q * q_scale++ # Compute attention weights manually+ # q: [B, Lq, Nq, C], k: [B, Lk, Nk, C]+ scale = softmax_scale if softmax_scale is not None else (q.size(-1) ** -0.5)++ # Compute scores: [B, Nq, Lq, Lk]+ scores = torch.einsum('blhd,bshd->bhls', q, k) * scale++ # Apply causal mask if needed+ if causal:+ mask = torch.triu(torch.ones(lq, lk, device=q.device, dtype=torch.bool), diagonal=1)+ scores.masked_fill_(mask.unsqueeze(0).unsqueeze(0), float('-inf'))++ # Mask for k_lens (columns)+ k_mask = torch.arange(lk, device=k.device).unsqueeze(0) >= k_lens.unsqueeze(1) # [B, Lk]+ scores.masked_fill_(k_mask.unsqueeze(1).unsqueeze(2), float('-inf')) # [B, 1, 1, Lk]++ # Mask for q_lens (rows)+ q_mask = torch.arange(lq, device=q.device).unsqueeze(0) >= q_lens.unsqueeze(1) # [B, Lq]+ scores.masked_fill_(q_mask.unsqueeze(1).unsqueeze(3), float('-inf')) # [B, 1, Lq, 1]++ # Compute attention weights+ attn_weights = torch.softmax(scores, dim=-1) # [B, Nq, Lq, Lk]+ assert attn_weights.shape[0] == 1, "Batch size > 1 not supported for attention visualization."++ # Average attention weights to reduce memory usage before returning+ # Average across batch dimension (should be 1) and query heads and query sequence length+ # This gives us attention weight per video token: [Lk]+ if average_for_q:+ #avg_attn_weights = torch.mean(attn_weights, dim=(0, 1, 3)) # [Lq]+ avg_attn_weights = torch.max(attn_weights, dim=3)[0].mean(dim=(0, 1)) # [Lq]+ else:+ if 0:+ avg_attn_weights = torch.mean(attn_weights, dim=(0, 1, 2)) # [Lk]+ elif 1:+ B, H, Lq, Lk = attn_weights.shape # [1, H, Lq, Lk]+ per_frame_seq_len = Lk // total_video_latent_frames+ per_frame_aud_len = Lq // total_video_latent_frames++ avg_attn_weights = torch.zeros((Lk,), device=attn_weights.device, dtype=attn_weights.dtype)++ eps = 1e-8 # numerical stability+ for i in range(total_video_latent_frames):+ start_idx_v = i * per_frame_seq_len+ end_idx_v = (i + 1) * per_frame_seq_len++ start_idx_a = i * per_frame_aud_len+ end_idx_a = (i + 1) * per_frame_aud_len++ # attn_chunk: [H, La, Lv]+ attn_chunk = attn_weights[0, :, start_idx_a:end_idx_a, start_idx_v:end_idx_v]++ # ---- Head informativeness via (low) entropy over Lv ----+ # Normalize within the Lv slice per (head, query) to make a proper distribution+ p = attn_chunk / (attn_chunk.sum(dim=-1, keepdim=True) + eps) # [H, La, Lv]+ entropy = -(p * (p + eps).log()).sum(dim=-1).mean(dim=1) # [H]++ # Convert to positive head weights (lower entropy -> larger weight)+ saliency = 1.0 / (entropy + 1e-6) # [H]+ head_w = saliency / (saliency.sum() + eps) # [H], sum=1++ # Reduce across audio queries first (pick strong responses), then weight heads+ per_head = torch.amax(attn_chunk, dim=1) # [H, Lv]+ weighted = (per_head * head_w[:, None]).sum(dim=0) # [Lv]++ avg_attn_weights[start_idx_v:end_idx_v] = weighted+ else:+ avg_attn_weights = torch.mean(attn_weights, dim=(0, 2)).max(dim=(0))[0] # [Lk]++ # Compute output: [B, Lq, Nq, C]+ out = torch.einsum('bhls,bshd->blhd', attn_weights, v)++ return out.to(out_dtype), avg_attn_weights.to(out_dtype)+++def attention(+ q,+ k,+ v,+ q_lens=None,+ k_lens=None,+ dropout_p=0.,+ softmax_scale=None,+ q_scale=None,+ causal=False,+ window_size=(-1, -1),+ deterministic=False,+ dtype=torch.bfloat16,+ fa_version=None,+):+ if FLASH_ATTN_2_AVAILABLE or FLASH_ATTN_3_AVAILABLE:+ return flash_attention(+ q=q,+ k=k,+ v=v,+ q_lens=q_lens,+ k_lens=k_lens,+ dropout_p=dropout_p,+ softmax_scale=softmax_scale,+ q_scale=q_scale,+ causal=causal,+ window_size=window_size,+ deterministic=deterministic,+ dtype=dtype,+ version=fa_version,+ )+ else:+ if q_lens is not None or k_lens is not None:+ warnings.warn(+ 'Padding mask is disabled when using scaled_dot_product_attention. It can have a significant impact on performance.'+ )+ attn_mask = None++ q = q.transpose(1, 2).to(dtype)+ k = k.transpose(1, 2).to(dtype)+ v = v.transpose(1, 2).to(dtype)++ out = torch.nn.functional.scaled_dot_product_attention(+ q, k, v, attn_mask=attn_mask, is_causal=causal, dropout_p=dropout_p)++ out = out.transpose(1, 2).contiguous()+ return outdiff --git a/models/wan/ovi/modules/fusion.py b/models/wan/ovi/modules/fusion.pynew file mode 100644index 0000000..95f593b--- /dev/null+++ b/models/wan/ovi/modules/fusion.py@@ -0,0 +1,376 @@++import torch+import torch.nn as nn+from .model import WanLayerNorm, WanModel, WanRMSNorm, rope_apply+from shared.attention import pay_attention+##### Enjoy this spagheti VRAM optimizations done by DeepBeepMeep !+# I am sure you are a nice person and as you copy this code, you will give me officially proper credits:+# Please link to https://github.com/deepbeepmeep/Wan2GP and @deepbeepmeep on twitter++def reshape_latent(latent, latent_frames):+ return latent.reshape(latent.shape[0], latent_frames, -1, latent.shape[-1] )++def restore_latent_shape(latent):+ return latent.reshape(latent.shape[0], -1, latent.shape[-1] )++class FusionModel(nn.Module):+ def __init__(self, video_config=None, audio_config=None):+ super().__init__()+ has_video = True+ has_audio = True+ if video_config is not None:+ self.video_model = WanModel(**video_config)+ else:+ has_video = False+ self.video_model = None+ print("Warning: No video model is provided!")++ if audio_config is not None:+ self.audio_model = WanModel(**audio_config)+ else:+ has_audio = False+ self.audio_model = None+ print("Warning: No audio model is provided!")++ if has_video and has_audio:+ assert len(self.video_model.blocks) == len(self.audio_model.blocks)+ self.num_blocks = len(self.video_model.blocks)++ self.use_sp = False+ self.inject_cross_attention_kv_projections()++ self.init_weights()++ def inject_cross_attention_kv_projections(self):+ for vid_block in self.video_model.blocks:+ vid_block.cross_attn.k_fusion = nn.Linear(vid_block.dim, vid_block.dim)+ vid_block.cross_attn.v_fusion = nn.Linear(vid_block.dim, vid_block.dim)+ vid_block.cross_attn.pre_attn_norm_fusion = WanLayerNorm(vid_block.dim, elementwise_affine=True)+ vid_block.cross_attn.norm_k_fusion = WanRMSNorm(vid_block.dim, eps=1e-6) if vid_block.qk_norm else nn.Identity()+++ for audio_block in self.audio_model.blocks:+ audio_block.cross_attn.k_fusion = nn.Linear(audio_block.dim, audio_block.dim)+ audio_block.cross_attn.v_fusion = nn.Linear(audio_block.dim, audio_block.dim)+ audio_block.cross_attn.pre_attn_norm_fusion = WanLayerNorm(audio_block.dim, elementwise_affine=True)+ audio_block.cross_attn.norm_k_fusion = WanRMSNorm(audio_block.dim, eps=1e-6) if audio_block.qk_norm else nn.Identity()+++ def merge_kwargs(self, vid_kwargs, audio_kwargs):+ """+ keys in each kwarg:+ e+ seq_lens+ grid_sizes+ freqs+ context+ context_lens+ """+ merged_kwargs = {}+ for key in vid_kwargs:+ merged_kwargs[f"vid_{key}"] = vid_kwargs[key]+ for key in audio_kwargs:+ merged_kwargs[f"audio_{key}"] = audio_kwargs[key]+ return merged_kwargs++ def single_fusion_cross_attention_forward(self,+ cross_attn_block,+ src_seq,+ src_grid_sizes,+ src_freqs,+ target_seq,+ target_seq_lens,+ target_grid_sizes,+ target_freqs,+ context,+ context_lens+ ):++ b, n, d = src_seq.size(0), cross_attn_block.num_heads, cross_attn_block.head_dim+ if hasattr(cross_attn_block, "k_img"):+ ## means is i2v block+ q, k, v, k_img, v_img = cross_attn_block.qkv_fn(src_seq, context)+ else:+ ## means is t2v block+ q, k, v = cross_attn_block.qkv_fn(src_seq, context)+ k_img = v_img = None+++ qkv_list =[q, k, v]+ del k, v+ x = pay_attention(qkv_list)++ if k_img is not None:+ qkv_list =[q, k_img, v_img]+ del k_img, v_img+ img_x = pay_attention(qkv_list)++ # img_x = flash_attention(q, k_img, v_img, k_lens=None)+ x += img_x++ is_vid = src_grid_sizes.shape[1] > 1+ # compute target attention+ target_seq = cross_attn_block.pre_attn_norm_fusion(target_seq)+ k_target = cross_attn_block.norm_k_fusion(cross_attn_block.k_fusion(target_seq)).view(b, -1, n, d)+ v_target = cross_attn_block.v_fusion(target_seq).view(b, -1, n, d)++ q = rope_apply(q, src_grid_sizes, src_freqs)+ k_target = rope_apply(k_target, target_grid_sizes, target_freqs)++ qkv_list =[q, k_target, v_target]+ del q, k_target, v_target+ target_x = pay_attention(qkv_list)++ # target_x = flash_attention(q, k_target, v_target, k_lens=target_seq_lens)++ x += target_x++ x = x.flatten(2) # [B, L/P, C]++ x = cross_attn_block.o(x)+ return x++ def single_fusion_cross_attention_ffn_forward(self,+ attn_block,+ src_seq,+ src_grid_sizes,+ src_freqs,+ target_seq,+ target_seq_lens,+ target_grid_sizes,+ target_freqs,+ context,+ context_lens,+ src_e):++ src_seq += self.single_fusion_cross_attention_forward(attn_block.cross_attn,+ attn_block.norm3(src_seq),+ src_grid_sizes=src_grid_sizes,+ src_freqs=src_freqs,+ target_seq=target_seq,+ target_seq_lens=target_seq_lens,+ target_grid_sizes=target_grid_sizes,+ target_freqs=target_freqs,+ context=context,+ context_lens=context_lens+ )++ latent_frames = src_e[0].shape[0]++ y = attn_block.norm2(src_seq).to(torch.bfloat16)+ y = reshape_latent(y , latent_frames)+ y *= (1 + src_e[4].squeeze(2))+ y += src_e[3].squeeze(2)+ y = restore_latent_shape(y)+ # y = attn_block.ffn(y)+++ ffn = attn_block.ffn[0]+ gelu = attn_block.ffn[1]+ ffn2= attn_block.ffn[2]++ y_shape = y.shape+ y = y.view(-1, y_shape[-1])+ chunk_size = int(y.shape[0]/2.7)+ chunks =torch.split(y, chunk_size)+ for y_chunk in chunks:+ mlp_chunk = ffn(y_chunk)+ mlp_chunk = gelu(mlp_chunk)+ y_chunk[...] = ffn2(mlp_chunk)+ del mlp_chunk+ y = y.view(y_shape)++ src_seq, y = reshape_latent(src_seq , latent_frames), reshape_latent(y , latent_frames)+ src_seq.addcmul_(y, src_e[5].squeeze(2))+ src_seq = restore_latent_shape(src_seq)+ del y++ # # y = attn_block.ffn(attn_block.norm2(src_seq).bfloat16() * (1 + src_e[4].squeeze(2)) + src_e[3].squeeze(2))+ # with torch.amp.autocast('cuda', dtype=torch.bfloat16):+ # src_seq = src_seq + y * src_e[5].squeeze(2)+ return src_seq++ def single_fusion_block_forward(self,+ vid_block,+ audio_block,+ vid,+ audio,+ vid_e,+ vid_seq_lens,+ vid_grid_sizes,+ vid_freqs,+ vid_context,+ vid_context_lens,+ audio_e,+ audio_seq_lens,+ audio_grid_sizes,+ audio_freqs,+ audio_context,+ audio_context_lens,+ ):+ ## audio modulation+ audio_e = audio_block.modulation(audio_e).chunk(6, dim=1)++ # audio self-attention+ audio_y = audio_block.norm1(audio).to(torch.bfloat16)+ audio_y *= (1 + audio_e[1].squeeze(2))+ audio_y += audio_e[0].squeeze(2)+ audio_y = audio_block.self_attn(audio_y, audio_seq_lens, audio_grid_sizes, audio_freqs)++ audio.addcmul_(audio_y, audio_e[2].squeeze(2))+ del audio_y++ latent_frames = vid_e.shape[0]++ ## video modulation+ vid_e = vid_block.modulation(vid_e).chunk(6, dim=1)+++ # video self-attention+ vid_y = vid_block.norm1(vid).to(torch.bfloat16)+ vid_y = reshape_latent(vid_y , latent_frames)+ vid_y *= (1 + vid_e[1].squeeze(2))+ vid_y += vid_e[0].squeeze(2)+ vid_y = restore_latent_shape(vid_y)+ vid_y = vid_block.self_attn(vid_y, vid_seq_lens, vid_grid_sizes, vid_freqs)+++ vid, vid_y = reshape_latent(vid , latent_frames), reshape_latent(vid_y , latent_frames)+ vid.addcmul_(vid_y, vid_e[2].squeeze(2))+ vid = restore_latent_shape(vid)+ del vid_y++ # og_audio = audio++ # audio cross-attention+ audio = self.single_fusion_cross_attention_ffn_forward(+ audio_block,+ audio,+ audio_grid_sizes,+ audio_freqs,+ vid,+ vid_seq_lens,+ vid_grid_sizes,+ vid_freqs,+ audio_context,+ audio_context_lens,+ audio_e,+ )+ if audio is None:+ return None, None+ # if torch.equal(og_audio, audio):+ # print("Audio should be changed after cross-attention!")+ # assert not torch.equal(og_audio, audio), "Audio should be changed after cross-attention!"++ # video cross-attention+ vid = self.single_fusion_cross_attention_ffn_forward(+ vid_block,+ vid,+ vid_grid_sizes,+ vid_freqs,+ audio,+ audio_seq_lens,+ audio_grid_sizes,+ audio_freqs,+ vid_context,+ vid_context_lens,+ vid_e,+ )+ if vid is None:+ return None, None++ return vid, audio++ def forward(+ self,+ vid,+ audio,+ t,+ vid_context,+ audio_context,+ vid_seq_len,+ audio_seq_len,+ clip_fea=None,+ clip_fea_audio=None,+ y=None,+ first_frame_is_clean=False,+ computed_slg_layers=None,+ callback = None,+ pipeline= None,+ x_id_list = 0,+ video_freqs = None,+ audio_freqs = None,+ ):+ vid_list = []+ vid_e_list = []+ audio_list = []+ audio_e_list = []+ kwargs_list = []+ for one_vid_context, one_audio_context in zip(vid_context, audio_context):+ one_vid, one_vid_e, vid_kwargs = self.video_model.prepare_transformer_block_kwargs(+ x=[vid], t=t, context=[one_vid_context], seq_len=vid_seq_len, clip_fea=clip_fea, y=y, first_frame_is_clean=first_frame_is_clean, freqs=video_freqs+ )+ vid_list.append(one_vid)+ vid_e_list.append(one_vid_e)+ one_vid = one_vid_e = None+ one_audio, one_audio_e, audio_kwargs = self.audio_model.prepare_transformer_block_kwargs(+ x=[audio], t=t, context=[one_audio_context], seq_len=audio_seq_len, clip_fea=clip_fea_audio, y=None, first_frame_is_clean=False, freqs=audio_freqs+ )+ audio_list.append(one_audio)+ audio_e_list.append(one_audio_e)+ one_audio = one_audio_e = None++ kwargs_list.append(self.merge_kwargs(vid_kwargs, audio_kwargs))++ for i in range(self.num_blocks):+ """+ 1 fusion block refers to 1 audio block with 1 video block.+ """+ if callback != None:+ callback(-1, None, False, True)+ vid_block = self.video_model.blocks[i]+ audio_block = self.audio_model.blocks[i]+ for x_id, one_vid, one_vid_e, one_audio, one_audio_e, one_kwargs in zip(x_id_list, vid_list, vid_e_list, audio_list, audio_e_list, kwargs_list):+ if pipeline._interrupt:+ return None, None+ if x_id == 1 and computed_slg_layers is not None and i in computed_slg_layers:+ continue+ # one_vid[...], one_audio[...] = self.single_fusion_block_forward(+ a, b = self.single_fusion_block_forward(+ vid_block=vid_block,+ audio_block=audio_block,+ vid=one_vid,+ audio=one_audio,+ **one_kwargs+ )+ one_vid[...], one_audio[...] = a, b+++ for i, (x_id, one_vid, one_vid_e, one_audio, one_audio_e) in enumerate(zip(x_id_list, vid_list, vid_e_list, audio_list, audio_e_list)):+ one_vid = self.video_model.post_transformer_block_out(one_vid, vid_kwargs['grid_sizes'], one_vid_e)+ vid_list[i] = one_vid+ one_vid = None+ one_audio = self.audio_model.post_transformer_block_out(one_audio, audio_kwargs['grid_sizes'], one_audio_e)+ audio_list[i] = one_audio+ one_audio = None++ if len(vid_list) == 1:+ return vid_list[0], audio_list[0]+ return vid_list, audio_list++ def init_weights(self):+ if self.audio_model is not None:+ self.audio_model.init_weights()++ if self.video_model is not None:+ self.video_model.init_weights()++ for name, mod in self.video_model.named_modules():+ if "fusion" in name and isinstance(mod, nn.Linear):+ with torch.no_grad():+ mod.weight.div_(10.0)+++ def set_rope_params(self):+ self.video_model.set_rope_params()+ self.audio_model.set_rope_params()diff --git a/models/wan/ovi/modules/mmaudio/__init__.py b/models/wan/ovi/modules/mmaudio/__init__.pynew file mode 100644index 0000000..e5601dd--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/__init__.py@@ -0,0 +1 @@+# MMAudio package\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/__init__.py b/models/wan/ovi/modules/mmaudio/ext/__init__.pynew file mode 100644index 0000000..8b13789--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/__init__.py@@ -0,0 +1 @@+diff --git a/models/wan/ovi/modules/mmaudio/ext/autoencoder/__init__.py b/models/wan/ovi/modules/mmaudio/ext/autoencoder/__init__.pynew file mode 100644index 0000000..b69d4b7--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/autoencoder/__init__.py@@ -0,0 +1 @@+from .autoencoder import AutoEncoderModule\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/autoencoder/autoencoder.py b/models/wan/ovi/modules/mmaudio/ext/autoencoder/autoencoder.pynew file mode 100644index 0000000..ceb57ec--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/autoencoder/autoencoder.py@@ -0,0 +1,55 @@+from typing import Literal, Optional++import torch+import torch.nn as nn+import numpy as np++from .vae import VAE, get_my_vae+from .distributions import DiagonalGaussianDistribution+from ..bigvgan import BigVGAN+from ..bigvgan_v2.bigvgan import BigVGAN as BigVGANv2+++++class AutoEncoderModule(nn.Module):++ def __init__(self,+ *,+ vae_ckpt_path,+ vocoder_ckpt_path: Optional[str] = None,+ mode: Literal['16k', '44k'],+ need_vae_encoder: bool = True):+ super().__init__()+ self.vae: VAE = get_my_vae(mode).eval()+ vae_state_dict = torch.load(vae_ckpt_path, weights_only=True, map_location='cpu')+ self.vae.load_state_dict(vae_state_dict)+ self.vae.remove_weight_norm()++ if mode == '16k':+ assert vocoder_ckpt_path is not None+ self.vocoder = BigVGAN(vocoder_ckpt_path).eval()+ elif mode == '44k':+ self.vocoder = BigVGANv2.from_pretrained('nvidia/bigvgan_v2_44khz_128band_512x',+ use_cuda_kernel=False)+ self.vocoder.remove_weight_norm()+ else:+ raise ValueError(f'Unknown mode: {mode}')++ for param in self.parameters():+ param.requires_grad = False++ if not need_vae_encoder:+ del self.vae.encoder++ @torch.inference_mode()+ def encode(self, x: torch.Tensor) -> DiagonalGaussianDistribution:+ return self.vae.encode(x)++ @torch.inference_mode()+ def decode(self, z: torch.Tensor) -> torch.Tensor:+ return self.vae.decode(z)++ @torch.inference_mode()+ def vocode(self, spec: torch.Tensor) -> torch.Tensor:+ return self.vocoder(spec)diff --git a/models/wan/ovi/modules/mmaudio/ext/autoencoder/distributions.py b/models/wan/ovi/modules/mmaudio/ext/autoencoder/distributions.pynew file mode 100644index 0000000..996f641--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/autoencoder/distributions.py@@ -0,0 +1,45 @@+from typing import Optional+import torch+import numpy as np+++class DiagonalGaussianDistribution:++ def __init__(self, parameters, deterministic=False):+ self.parameters = parameters+ self.mean, self.logvar = torch.chunk(parameters, 2, dim=1)+ self.logvar = torch.clamp(self.logvar, -30.0, 20.0)+ self.deterministic = deterministic+ self.std = torch.exp(0.5 * self.logvar)+ self.var = torch.exp(self.logvar)+ if self.deterministic:+ self.var = self.std = torch.zeros_like(self.mean).to(device=self.parameters.device)++ def sample(self, rng: Optional[torch.Generator] = None):+ # x = self.mean + self.std * torch.randn(self.mean.shape).to(device=self.parameters.device)++ r = torch.empty_like(self.mean).normal_(generator=rng)+ x = self.mean + self.std * r++ return x++ def kl(self, other=None):+ if self.deterministic:+ return torch.Tensor([0.])+ else:+ if other is None:++ return 0.5 * torch.pow(self.mean, 2) + self.var - 1.0 - self.logvar+ else:+ return 0.5 * (torch.pow(self.mean - other.mean, 2) / other.var ++ self.var / other.var - 1.0 - self.logvar + other.logvar)++ def nll(self, sample, dims=[1, 2, 3]):+ if self.deterministic:+ return torch.Tensor([0.])+ logtwopi = np.log(2.0 * np.pi)+ return 0.5 * torch.sum(logtwopi + self.logvar + torch.pow(sample - self.mean, 2) / self.var,+ dim=dims)++ def mode(self):+ return self.mean\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/autoencoder/edm2_utils.py b/models/wan/ovi/modules/mmaudio/ext/autoencoder/edm2_utils.pynew file mode 100644index 0000000..a18ffba--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/autoencoder/edm2_utils.py@@ -0,0 +1,168 @@+# Copyright (c) 2024, NVIDIA CORPORATION & AFFILIATES. All rights reserved.+#+# This work is licensed under a Creative Commons+# Attribution-NonCommercial-ShareAlike 4.0 International License.+# You should have received a copy of the license along with this+# work. If not, see http://creativecommons.org/licenses/by-nc-sa/4.0/+"""Improved diffusion model architecture proposed in the paper+"Analyzing and Improving the Training Dynamics of Diffusion Models"."""++import numpy as np+import torch++#----------------------------------------------------------------------------+# Variant of constant() that inherits dtype and device from the given+# reference tensor by default.++_constant_cache = dict()+++def constant(value, shape=None, dtype=None, device=None, memory_format=None):+ value = np.asarray(value)+ if shape is not None:+ shape = tuple(shape)+ if dtype is None:+ dtype = torch.get_default_dtype()+ if device is None:+ device = torch.device('cpu')+ if memory_format is None:+ memory_format = torch.contiguous_format++ key = (value.shape, value.dtype, value.tobytes(), shape, dtype, device, memory_format)+ tensor = _constant_cache.get(key, None)+ if tensor is None:+ tensor = torch.as_tensor(value.copy(), dtype=dtype, device=device)+ if shape is not None:+ tensor, _ = torch.broadcast_tensors(tensor, torch.empty(shape))+ tensor = tensor.contiguous(memory_format=memory_format)+ _constant_cache[key] = tensor+ return tensor+++def const_like(ref, value, shape=None, dtype=None, device=None, memory_format=None):+ if dtype is None:+ dtype = ref.dtype+ if device is None:+ device = ref.device+ return constant(value, shape=shape, dtype=dtype, device=device, memory_format=memory_format)+++#----------------------------------------------------------------------------+# Normalize given tensor to unit magnitude with respect to the given+# dimensions. Default = all dimensions except the first.+++def normalize(x, dim=None, eps=1e-4):+ if dim is None:+ dim = list(range(1, x.ndim))+ norm = torch.linalg.vector_norm(x, dim=dim, keepdim=True, dtype=torch.float32)+ norm = torch.add(eps, norm, alpha=np.sqrt(norm.numel() / x.numel()))+ return x / norm.to(x.dtype)+++class Normalize(torch.nn.Module):++ def __init__(self, dim=None, eps=1e-4):+ super().__init__()+ self.dim = dim+ self.eps = eps++ def forward(self, x):+ return normalize(x, dim=self.dim, eps=self.eps)+++#----------------------------------------------------------------------------+# Upsample or downsample the given tensor with the given filter,+# or keep it as is.+++def resample(x, f=[1, 1], mode='keep'):+ if mode == 'keep':+ return x+ f = np.float32(f)+ assert f.ndim == 1 and len(f) % 2 == 0+ pad = (len(f) - 1) // 2+ f = f / f.sum()+ f = np.outer(f, f)[np.newaxis, np.newaxis, :, :]+ f = const_like(x, f)+ c = x.shape[1]+ if mode == 'down':+ return torch.nn.functional.conv2d(x,+ f.tile([c, 1, 1, 1]),+ groups=c,+ stride=2,+ padding=(pad, ))+ assert mode == 'up'+ return torch.nn.functional.conv_transpose2d(x, (f * 4).tile([c, 1, 1, 1]),+ groups=c,+ stride=2,+ padding=(pad, ))+++#----------------------------------------------------------------------------+# Magnitude-preserving SiLU (Equation 81).+++def mp_silu(x):+ return torch.nn.functional.silu(x) / 0.596+++class MPSiLU(torch.nn.Module):++ def forward(self, x):+ return mp_silu(x)+++#----------------------------------------------------------------------------+# Magnitude-preserving sum (Equation 88).+++def mp_sum(a, b, t=0.5):+ return a.lerp(b, t) / np.sqrt((1 - t)**2 + t**2)+++#----------------------------------------------------------------------------+# Magnitude-preserving concatenation (Equation 103).+++def mp_cat(a, b, dim=1, t=0.5):+ Na = a.shape[dim]+ Nb = b.shape[dim]+ C = np.sqrt((Na + Nb) / ((1 - t)**2 + t**2))+ wa = C / np.sqrt(Na) * (1 - t)+ wb = C / np.sqrt(Nb) * t+ return torch.cat([wa * a, wb * b], dim=dim)+++#----------------------------------------------------------------------------+# Magnitude-preserving convolution or fully-connected layer (Equation 47)+# with force weight normalization (Equation 66).+++class MPConv1D(torch.nn.Module):++ def __init__(self, in_channels, out_channels, kernel_size):+ super().__init__()+ self.out_channels = out_channels+ self.weight = torch.nn.Parameter(torch.randn(out_channels, in_channels, kernel_size))++ self.weight_norm_removed = False++ def forward(self, x, gain=1):+ assert self.weight_norm_removed, 'call remove_weight_norm() before inference'++ w = self.weight * gain+ if w.ndim == 2:+ return x @ w.t()+ assert w.ndim == 3+ return torch.nn.functional.conv1d(x, w, padding=(w.shape[-1] // 2, ))++ def remove_weight_norm(self):+ w = self.weight.to(torch.float32)+ w = normalize(w) # traditional weight normalization+ w = w / np.sqrt(w[0].numel())+ w = w.to(self.weight.dtype)+ self.weight.data.copy_(w)++ self.weight_norm_removed = True+ return selfdiff --git a/models/wan/ovi/modules/mmaudio/ext/autoencoder/vae.py b/models/wan/ovi/modules/mmaudio/ext/autoencoder/vae.pynew file mode 100644index 0000000..d1b9254--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/autoencoder/vae.py@@ -0,0 +1,369 @@+import logging+from typing import Optional++import torch+import torch.nn as nn++from .edm2_utils import MPConv1D+from .vae_modules import (AttnBlock1D, Downsample1D, ResnetBlock1D,+ Upsample1D, nonlinearity)+from .distributions import DiagonalGaussianDistribution++log = logging.getLogger()++DATA_MEAN_80D = [+ -1.6058, -1.3676, -1.2520, -1.2453, -1.2078, -1.2224, -1.2419, -1.2439, -1.2922, -1.2927,+ -1.3170, -1.3543, -1.3401, -1.3836, -1.3907, -1.3912, -1.4313, -1.4152, -1.4527, -1.4728,+ -1.4568, -1.5101, -1.5051, -1.5172, -1.5623, -1.5373, -1.5746, -1.5687, -1.6032, -1.6131,+ -1.6081, -1.6331, -1.6489, -1.6489, -1.6700, -1.6738, -1.6953, -1.6969, -1.7048, -1.7280,+ -1.7361, -1.7495, -1.7658, -1.7814, -1.7889, -1.8064, -1.8221, -1.8377, -1.8417, -1.8643,+ -1.8857, -1.8929, -1.9173, -1.9379, -1.9531, -1.9673, -1.9824, -2.0042, -2.0215, -2.0436,+ -2.0766, -2.1064, -2.1418, -2.1855, -2.2319, -2.2767, -2.3161, -2.3572, -2.3954, -2.4282,+ -2.4659, -2.5072, -2.5552, -2.6074, -2.6584, -2.7107, -2.7634, -2.8266, -2.8981, -2.9673+]++DATA_STD_80D = [+ 1.0291, 1.0411, 1.0043, 0.9820, 0.9677, 0.9543, 0.9450, 0.9392, 0.9343, 0.9297, 0.9276, 0.9263,+ 0.9242, 0.9254, 0.9232, 0.9281, 0.9263, 0.9315, 0.9274, 0.9247, 0.9277, 0.9199, 0.9188, 0.9194,+ 0.9160, 0.9161, 0.9146, 0.9161, 0.9100, 0.9095, 0.9145, 0.9076, 0.9066, 0.9095, 0.9032, 0.9043,+ 0.9038, 0.9011, 0.9019, 0.9010, 0.8984, 0.8983, 0.8986, 0.8961, 0.8962, 0.8978, 0.8962, 0.8973,+ 0.8993, 0.8976, 0.8995, 0.9016, 0.8982, 0.8972, 0.8974, 0.8949, 0.8940, 0.8947, 0.8936, 0.8939,+ 0.8951, 0.8956, 0.9017, 0.9167, 0.9436, 0.9690, 1.0003, 1.0225, 1.0381, 1.0491, 1.0545, 1.0604,+ 1.0761, 1.0929, 1.1089, 1.1196, 1.1176, 1.1156, 1.1117, 1.1070+]++DATA_MEAN_128D = [+ -3.3462, -2.6723, -2.4893, -2.3143, -2.2664, -2.3317, -2.1802, -2.4006, -2.2357, -2.4597,+ -2.3717, -2.4690, -2.5142, -2.4919, -2.6610, -2.5047, -2.7483, -2.5926, -2.7462, -2.7033,+ -2.7386, -2.8112, -2.7502, -2.9594, -2.7473, -3.0035, -2.8891, -2.9922, -2.9856, -3.0157,+ -3.1191, -2.9893, -3.1718, -3.0745, -3.1879, -3.2310, -3.1424, -3.2296, -3.2791, -3.2782,+ -3.2756, -3.3134, -3.3509, -3.3750, -3.3951, -3.3698, -3.4505, -3.4509, -3.5089, -3.4647,+ -3.5536, -3.5788, -3.5867, -3.6036, -3.6400, -3.6747, -3.7072, -3.7279, -3.7283, -3.7795,+ -3.8259, -3.8447, -3.8663, -3.9182, -3.9605, -3.9861, -4.0105, -4.0373, -4.0762, -4.1121,+ -4.1488, -4.1874, -4.2461, -4.3170, -4.3639, -4.4452, -4.5282, -4.6297, -4.7019, -4.7960,+ -4.8700, -4.9507, -5.0303, -5.0866, -5.1634, -5.2342, -5.3242, -5.4053, -5.4927, -5.5712,+ -5.6464, -5.7052, -5.7619, -5.8410, -5.9188, -6.0103, -6.0955, -6.1673, -6.2362, -6.3120,+ -6.3926, -6.4797, -6.5565, -6.6511, -6.8130, -6.9961, -7.1275, -7.2457, -7.3576, -7.4663,+ -7.6136, -7.7469, -7.8815, -8.0132, -8.1515, -8.3071, -8.4722, -8.7418, -9.3975, -9.6628,+ -9.7671, -9.8863, -9.9992, -10.0860, -10.1709, -10.5418, -11.2795, -11.3861+]++DATA_STD_128D = [+ 2.3804, 2.4368, 2.3772, 2.3145, 2.2803, 2.2510, 2.2316, 2.2083, 2.1996, 2.1835, 2.1769, 2.1659,+ 2.1631, 2.1618, 2.1540, 2.1606, 2.1571, 2.1567, 2.1612, 2.1579, 2.1679, 2.1683, 2.1634, 2.1557,+ 2.1668, 2.1518, 2.1415, 2.1449, 2.1406, 2.1350, 2.1313, 2.1415, 2.1281, 2.1352, 2.1219, 2.1182,+ 2.1327, 2.1195, 2.1137, 2.1080, 2.1179, 2.1036, 2.1087, 2.1036, 2.1015, 2.1068, 2.0975, 2.0991,+ 2.0902, 2.1015, 2.0857, 2.0920, 2.0893, 2.0897, 2.0910, 2.0881, 2.0925, 2.0873, 2.0960, 2.0900,+ 2.0957, 2.0958, 2.0978, 2.0936, 2.0886, 2.0905, 2.0845, 2.0855, 2.0796, 2.0840, 2.0813, 2.0817,+ 2.0838, 2.0840, 2.0917, 2.1061, 2.1431, 2.1976, 2.2482, 2.3055, 2.3700, 2.4088, 2.4372, 2.4609,+ 2.4731, 2.4847, 2.5072, 2.5451, 2.5772, 2.6147, 2.6529, 2.6596, 2.6645, 2.6726, 2.6803, 2.6812,+ 2.6899, 2.6916, 2.6931, 2.6998, 2.7062, 2.7262, 2.7222, 2.7158, 2.7041, 2.7485, 2.7491, 2.7451,+ 2.7485, 2.7233, 2.7297, 2.7233, 2.7145, 2.6958, 2.6788, 2.6439, 2.6007, 2.4786, 2.2469, 2.1877,+ 2.1392, 2.0717, 2.0107, 1.9676, 1.9140, 1.7102, 0.9101, 0.7164+]+++class VAE(nn.Module):++ def __init__(+ self,+ *,+ data_dim: int,+ embed_dim: int,+ hidden_dim: int,+ ):+ super().__init__()++ if data_dim == 80:+ self.data_mean = nn.Buffer(torch.tensor(DATA_MEAN_80D, dtype=torch.float32))+ self.data_std = nn.Buffer(torch.tensor(DATA_STD_80D, dtype=torch.float32))+ elif data_dim == 128:+ self.data_mean = nn.Buffer(torch.tensor(DATA_MEAN_128D, dtype=torch.float32))+ self.data_std = nn.Buffer(torch.tensor(DATA_STD_128D, dtype=torch.float32))++ self.data_mean = self.data_mean.view(1, -1, 1)+ self.data_std = self.data_std.view(1, -1, 1)++ self.encoder = Encoder1D(+ dim=hidden_dim,+ ch_mult=(1, 2, 4),+ num_res_blocks=2,+ attn_layers=[3],+ down_layers=[0],+ in_dim=data_dim,+ embed_dim=embed_dim,+ )+ self.decoder = Decoder1D(+ dim=hidden_dim,+ ch_mult=(1, 2, 4),+ num_res_blocks=2,+ attn_layers=[3],+ down_layers=[0],+ in_dim=data_dim,+ out_dim=data_dim,+ embed_dim=embed_dim,+ )++ self.embed_dim = embed_dim+ # self.quant_conv = nn.Conv1d(2 * embed_dim, 2 * embed_dim, 1)+ # self.post_quant_conv = nn.Conv1d(embed_dim, embed_dim, 1)++ self.initialize_weights()++ def initialize_weights(self):+ pass++ def encode(self, x: torch.Tensor, normalize: bool = True) -> DiagonalGaussianDistribution:+ if normalize:+ x = self.normalize(x)+ moments = self.encoder(x)+ posterior = DiagonalGaussianDistribution(moments)+ return posterior++ def decode(self, z: torch.Tensor, unnormalize: bool = True) -> torch.Tensor:+ dec = self.decoder(z)+ if unnormalize:+ dec = self.unnormalize(dec)+ return dec++ def normalize(self, x: torch.Tensor) -> torch.Tensor:+ return (x - self.data_mean) / self.data_std++ def unnormalize(self, x: torch.Tensor) -> torch.Tensor:+ return x * self.data_std + self.data_mean++ def forward(+ self,+ x: torch.Tensor,+ sample_posterior: bool = True,+ rng: Optional[torch.Generator] = None,+ normalize: bool = True,+ unnormalize: bool = True,+ ) -> tuple[torch.Tensor, DiagonalGaussianDistribution]:++ posterior = self.encode(x, normalize=normalize)+ if sample_posterior:+ z = posterior.sample(rng)+ else:+ z = posterior.mode()+ dec = self.decode(z, unnormalize=unnormalize)+ return dec, posterior++ def load_weights(self, src_dict) -> None:+ self.load_state_dict(src_dict, strict=True)++ @property+ def device(self) -> torch.device:+ return next(self.parameters()).device++ def get_last_layer(self):+ return self.decoder.conv_out.weight++ def remove_weight_norm(self):+ for name, m in self.named_modules():+ if isinstance(m, MPConv1D):+ m.remove_weight_norm()+ log.debug(f"Removed weight norm from {name}")+ return self+++class Encoder1D(nn.Module):++ def __init__(self,+ *,+ dim: int,+ ch_mult: tuple[int] = (1, 2, 4, 8),+ num_res_blocks: int,+ attn_layers: list[int] = [],+ down_layers: list[int] = [],+ resamp_with_conv: bool = True,+ in_dim: int,+ embed_dim: int,+ double_z: bool = True,+ kernel_size: int = 3,+ clip_act: float = 256.0):+ super().__init__()+ self.dim = dim+ self.num_layers = len(ch_mult)+ self.num_res_blocks = num_res_blocks+ self.in_channels = in_dim+ self.clip_act = clip_act+ self.down_layers = down_layers+ self.attn_layers = attn_layers+ self.conv_in = MPConv1D(in_dim, self.dim, kernel_size=kernel_size)++ in_ch_mult = (1, ) + tuple(ch_mult)+ self.in_ch_mult = in_ch_mult+ # downsampling+ self.down = nn.ModuleList()+ for i_level in range(self.num_layers):+ block = nn.ModuleList()+ attn = nn.ModuleList()+ block_in = dim * in_ch_mult[i_level]+ block_out = dim * ch_mult[i_level]+ for i_block in range(self.num_res_blocks):+ block.append(+ ResnetBlock1D(in_dim=block_in,+ out_dim=block_out,+ kernel_size=kernel_size,+ use_norm=True))+ block_in = block_out+ if i_level in attn_layers:+ attn.append(AttnBlock1D(block_in))+ down = nn.Module()+ down.block = block+ down.attn = attn+ if i_level in down_layers:+ down.downsample = Downsample1D(block_in, resamp_with_conv)+ self.down.append(down)++ # middle+ self.mid = nn.Module()+ self.mid.block_1 = ResnetBlock1D(in_dim=block_in,+ out_dim=block_in,+ kernel_size=kernel_size,+ use_norm=True)+ self.mid.attn_1 = AttnBlock1D(block_in)+ self.mid.block_2 = ResnetBlock1D(in_dim=block_in,+ out_dim=block_in,+ kernel_size=kernel_size,+ use_norm=True)++ # end+ self.conv_out = MPConv1D(block_in,+ 2 * embed_dim if double_z else embed_dim,+ kernel_size=kernel_size)++ self.learnable_gain = nn.Parameter(torch.zeros([]))++ def forward(self, x):++ # downsampling+ hs = [self.conv_in(x)]+ for i_level in range(self.num_layers):+ for i_block in range(self.num_res_blocks):+ h = self.down[i_level].block[i_block](hs[-1])+ if len(self.down[i_level].attn) > 0:+ h = self.down[i_level].attn[i_block](h)+ h = h.clamp(-self.clip_act, self.clip_act)+ hs.append(h)+ if i_level in self.down_layers:+ hs.append(self.down[i_level].downsample(hs[-1]))++ # middle+ h = hs[-1]+ h = self.mid.block_1(h)+ h = self.mid.attn_1(h)+ h = self.mid.block_2(h)+ h = h.clamp(-self.clip_act, self.clip_act)++ # end+ h = nonlinearity(h)+ h = self.conv_out(h, gain=(self.learnable_gain + 1))+ return h+++class Decoder1D(nn.Module):++ def __init__(self,+ *,+ dim: int,+ out_dim: int,+ ch_mult: tuple[int] = (1, 2, 4, 8),+ num_res_blocks: int,+ attn_layers: list[int] = [],+ down_layers: list[int] = [],+ kernel_size: int = 3,+ resamp_with_conv: bool = True,+ in_dim: int,+ embed_dim: int,+ clip_act: float = 256.0):+ super().__init__()+ self.ch = dim+ self.num_layers = len(ch_mult)+ self.num_res_blocks = num_res_blocks+ self.in_channels = in_dim+ self.clip_act = clip_act+ self.down_layers = [i + 1 for i in down_layers] # each downlayer add one++ # compute in_ch_mult, block_in and curr_res at lowest res+ block_in = dim * ch_mult[self.num_layers - 1]++ # z to block_in+ self.conv_in = MPConv1D(embed_dim, block_in, kernel_size=kernel_size)++ # middle+ self.mid = nn.Module()+ self.mid.block_1 = ResnetBlock1D(in_dim=block_in, out_dim=block_in, use_norm=True)+ self.mid.attn_1 = AttnBlock1D(block_in)+ self.mid.block_2 = ResnetBlock1D(in_dim=block_in, out_dim=block_in, use_norm=True)++ # upsampling+ self.up = nn.ModuleList()+ for i_level in reversed(range(self.num_layers)):+ block = nn.ModuleList()+ attn = nn.ModuleList()+ block_out = dim * ch_mult[i_level]+ for i_block in range(self.num_res_blocks + 1):+ block.append(ResnetBlock1D(in_dim=block_in, out_dim=block_out, use_norm=True))+ block_in = block_out+ if i_level in attn_layers:+ attn.append(AttnBlock1D(block_in))+ up = nn.Module()+ up.block = block+ up.attn = attn+ if i_level in self.down_layers:+ up.upsample = Upsample1D(block_in, resamp_with_conv)+ self.up.insert(0, up) # prepend to get consistent order++ # end+ self.conv_out = MPConv1D(block_in, out_dim, kernel_size=kernel_size)+ self.learnable_gain = nn.Parameter(torch.zeros([]))++ def forward(self, z):+ # z to block_in+ h = self.conv_in(z)++ # middle+ h = self.mid.block_1(h)+ h = self.mid.attn_1(h)+ h = self.mid.block_2(h)+ h = h.clamp(-self.clip_act, self.clip_act)++ # upsampling+ for i_level in reversed(range(self.num_layers)):+ for i_block in range(self.num_res_blocks + 1):+ h = self.up[i_level].block[i_block](h)+ if len(self.up[i_level].attn) > 0:+ h = self.up[i_level].attn[i_block](h)+ h = h.clamp(-self.clip_act, self.clip_act)+ if i_level in self.down_layers:+ h = self.up[i_level].upsample(h)++ h = nonlinearity(h)+ h = self.conv_out(h, gain=(self.learnable_gain + 1))+ return h+++def VAE_16k(**kwargs) -> VAE:+ return VAE(data_dim=80, embed_dim=20, hidden_dim=384, **kwargs)+++def VAE_44k(**kwargs) -> VAE:+ return VAE(data_dim=128, embed_dim=40, hidden_dim=512, **kwargs)+++def get_my_vae(name: str, **kwargs) -> VAE:+ if name == '16k':+ return VAE_16k(**kwargs)+ if name == '44k':+ return VAE_44k(**kwargs)+ raise ValueError(f'Unknown model: {name}')+++if __name__ == '__main__':+ network = get_my_vae('standard')++ # print the number of parameters in terms of millions+ num_params = sum(p.numel() for p in network.parameters()) / 1e6+ print(f'Number of parameters: {num_params:.2f}M')diff --git a/models/wan/ovi/modules/mmaudio/ext/autoencoder/vae_modules.py b/models/wan/ovi/modules/mmaudio/ext/autoencoder/vae_modules.pynew file mode 100644index 0000000..f083a0e--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/autoencoder/vae_modules.py@@ -0,0 +1,117 @@+import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange++from .edm2_utils import (MPConv1D, mp_silu, mp_sum, normalize)+++def nonlinearity(x):+ # swish+ return mp_silu(x)+++class ResnetBlock1D(nn.Module):++ def __init__(self, *, in_dim, out_dim=None, conv_shortcut=False, kernel_size=3, use_norm=True):+ super().__init__()+ self.in_dim = in_dim+ out_dim = in_dim if out_dim is None else out_dim+ self.out_dim = out_dim+ self.use_conv_shortcut = conv_shortcut+ self.use_norm = use_norm++ self.conv1 = MPConv1D(in_dim, out_dim, kernel_size=kernel_size)+ self.conv2 = MPConv1D(out_dim, out_dim, kernel_size=kernel_size)+ if self.in_dim != self.out_dim:+ if self.use_conv_shortcut:+ self.conv_shortcut = MPConv1D(in_dim, out_dim, kernel_size=kernel_size)+ else:+ self.nin_shortcut = MPConv1D(in_dim, out_dim, kernel_size=1)++ def forward(self, x: torch.Tensor) -> torch.Tensor:++ # pixel norm+ if self.use_norm:+ x = normalize(x, dim=1)++ h = x+ h = nonlinearity(h)+ h = self.conv1(h)++ h = nonlinearity(h)+ h = self.conv2(h)++ if self.in_dim != self.out_dim:+ if self.use_conv_shortcut:+ x = self.conv_shortcut(x)+ else:+ x = self.nin_shortcut(x)++ return mp_sum(x, h, t=0.3)+++class AttnBlock1D(nn.Module):++ def __init__(self, in_channels, num_heads=1):+ super().__init__()+ self.in_channels = in_channels++ self.num_heads = num_heads+ self.qkv = MPConv1D(in_channels, in_channels * 3, kernel_size=1)+ self.proj_out = MPConv1D(in_channels, in_channels, kernel_size=1)++ def forward(self, x):+ h = x+ y = self.qkv(h)+ y = y.reshape(y.shape[0], self.num_heads, -1, 3, y.shape[-1])+ q, k, v = normalize(y, dim=2).unbind(3)++ q = rearrange(q, 'b h c l -> b h l c')+ k = rearrange(k, 'b h c l -> b h l c')+ v = rearrange(v, 'b h c l -> b h l c')++ h = F.scaled_dot_product_attention(q, k, v)+ h = rearrange(h, 'b h l c -> b (h c) l')++ h = self.proj_out(h)++ return mp_sum(x, h, t=0.3)+++class Upsample1D(nn.Module):++ def __init__(self, in_channels, with_conv):+ super().__init__()+ self.with_conv = with_conv+ if self.with_conv:+ self.conv = MPConv1D(in_channels, in_channels, kernel_size=3)++ def forward(self, x):+ x = F.interpolate(x, scale_factor=2.0, mode='nearest-exact') # support 3D tensor(B,C,T)+ if self.with_conv:+ x = self.conv(x)+ return x+++class Downsample1D(nn.Module):++ def __init__(self, in_channels, with_conv):+ super().__init__()+ self.with_conv = with_conv+ if self.with_conv:+ # no asymmetric padding in torch conv, must do it ourselves+ self.conv1 = MPConv1D(in_channels, in_channels, kernel_size=1)+ self.conv2 = MPConv1D(in_channels, in_channels, kernel_size=1)++ def forward(self, x):++ if self.with_conv:+ x = self.conv1(x)++ x = F.avg_pool1d(x, kernel_size=2, stride=2)++ if self.with_conv:+ x = self.conv2(x)++ return xdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/LICENSE b/models/wan/ovi/modules/mmaudio/ext/bigvgan/LICENSEnew file mode 100644index 0000000..e966359--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/LICENSE@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2022 NVIDIA CORPORATION.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/__init__.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/__init__.pynew file mode 100644index 0000000..00f13e9--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/__init__.py@@ -0,0 +1 @@+from .bigvgan import BigVGANdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/activations.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/activations.pynew file mode 100644index 0000000..61f2808--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/activations.py@@ -0,0 +1,120 @@+# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license.+# LICENSE is in incl_licenses directory.++import torch+from torch import nn, sin, pow+from torch.nn import Parameter+++class Snake(nn.Module):+ '''+ Implementation of a sine-based periodic activation function+ Shape:+ - Input: (B, C, T)+ - Output: (B, C, T), same shape as the input+ Parameters:+ - alpha - trainable parameter+ References:+ - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:+ https://arxiv.org/abs/2006.08195+ Examples:+ >>> a1 = snake(256)+ >>> x = torch.randn(256)+ >>> x = a1(x)+ '''+ def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False):+ '''+ Initialization.+ INPUT:+ - in_features: shape of the input+ - alpha: trainable parameter+ alpha is initialized to 1 by default, higher values = higher-frequency.+ alpha will be trained along with the rest of your model.+ '''+ super(Snake, self).__init__()+ self.in_features = in_features++ # initialize alpha+ self.alpha_logscale = alpha_logscale+ if self.alpha_logscale: # log scale alphas initialized to zeros+ self.alpha = Parameter(torch.zeros(in_features) * alpha)+ else: # linear scale alphas initialized to ones+ self.alpha = Parameter(torch.ones(in_features) * alpha)++ self.alpha.requires_grad = alpha_trainable++ self.no_div_by_zero = 0.000000001++ def forward(self, x):+ '''+ Forward pass of the function.+ Applies the function to the input elementwise.+ Snake ∶= x + 1/a * sin^2 (xa)+ '''+ alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T]+ if self.alpha_logscale:+ alpha = torch.exp(alpha)+ x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2)++ return x+++class SnakeBeta(nn.Module):+ '''+ A modified Snake function which uses separate parameters for the magnitude of the periodic components+ Shape:+ - Input: (B, C, T)+ - Output: (B, C, T), same shape as the input+ Parameters:+ - alpha - trainable parameter that controls frequency+ - beta - trainable parameter that controls magnitude+ References:+ - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:+ https://arxiv.org/abs/2006.08195+ Examples:+ >>> a1 = snakebeta(256)+ >>> x = torch.randn(256)+ >>> x = a1(x)+ '''+ def __init__(self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False):+ '''+ Initialization.+ INPUT:+ - in_features: shape of the input+ - alpha - trainable parameter that controls frequency+ - beta - trainable parameter that controls magnitude+ alpha is initialized to 1 by default, higher values = higher-frequency.+ beta is initialized to 1 by default, higher values = higher-magnitude.+ alpha will be trained along with the rest of your model.+ '''+ super(SnakeBeta, self).__init__()+ self.in_features = in_features++ # initialize alpha+ self.alpha_logscale = alpha_logscale+ if self.alpha_logscale: # log scale alphas initialized to zeros+ self.alpha = Parameter(torch.zeros(in_features) * alpha)+ self.beta = Parameter(torch.zeros(in_features) * alpha)+ else: # linear scale alphas initialized to ones+ self.alpha = Parameter(torch.ones(in_features) * alpha)+ self.beta = Parameter(torch.ones(in_features) * alpha)++ self.alpha.requires_grad = alpha_trainable+ self.beta.requires_grad = alpha_trainable++ self.no_div_by_zero = 0.000000001++ def forward(self, x):+ '''+ Forward pass of the function.+ Applies the function to the input elementwise.+ SnakeBeta ∶= x + 1/b * sin^2 (xa)+ '''+ alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # line up with x to [B, C, T]+ beta = self.beta.unsqueeze(0).unsqueeze(-1)+ if self.alpha_logscale:+ alpha = torch.exp(alpha)+ beta = torch.exp(beta)+ x = x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2)++ return x\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/__init__.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/__init__.pynew file mode 100644index 0000000..a2318b6--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/__init__.py@@ -0,0 +1,6 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++from .filter import *+from .resample import *+from .act import *\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/act.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/act.pynew file mode 100644index 0000000..028debd--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/act.py@@ -0,0 +1,28 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++import torch.nn as nn+from .resample import UpSample1d, DownSample1d+++class Activation1d(nn.Module):+ def __init__(self,+ activation,+ up_ratio: int = 2,+ down_ratio: int = 2,+ up_kernel_size: int = 12,+ down_kernel_size: int = 12):+ super().__init__()+ self.up_ratio = up_ratio+ self.down_ratio = down_ratio+ self.act = activation+ self.upsample = UpSample1d(up_ratio, up_kernel_size)+ self.downsample = DownSample1d(down_ratio, down_kernel_size)++ # x: [B,C,T]+ def forward(self, x):+ x = self.upsample(x)+ x = self.act(x)+ x = self.downsample(x)++ return x\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/filter.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/filter.pynew file mode 100644index 0000000..7ad6ea8--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/filter.py@@ -0,0 +1,95 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++import torch+import torch.nn as nn+import torch.nn.functional as F+import math++if 'sinc' in dir(torch):+ sinc = torch.sinc+else:+ # This code is adopted from adefossez's julius.core.sinc under the MIT License+ # https://adefossez.github.io/julius/julius/core.html+ # LICENSE is in incl_licenses directory.+ def sinc(x: torch.Tensor):+ """+ Implementation of sinc, i.e. sin(pi * x) / (pi * x)+ __Warning__: Different to julius.sinc, the input is multiplied by `pi`!+ """+ return torch.where(x == 0,+ torch.tensor(1., device=x.device, dtype=x.dtype),+ torch.sin(math.pi * x) / math.pi / x)+++# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License+# https://adefossez.github.io/julius/julius/lowpass.html+# LICENSE is in incl_licenses directory.+def kaiser_sinc_filter1d(cutoff, half_width, kernel_size): # return filter [1,1,kernel_size]+ even = (kernel_size % 2 == 0)+ half_size = kernel_size // 2++ #For kaiser window+ delta_f = 4 * half_width+ A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95+ if A > 50.:+ beta = 0.1102 * (A - 8.7)+ elif A >= 21.:+ beta = 0.5842 * (A - 21)**0.4 + 0.07886 * (A - 21.)+ else:+ beta = 0.+ window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)++ # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio+ if even:+ time = (torch.arange(-half_size, half_size) + 0.5)+ else:+ time = torch.arange(kernel_size) - half_size+ if cutoff == 0:+ filter_ = torch.zeros_like(time)+ else:+ filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)+ # Normalize filter to have sum = 1, otherwise we will have a small leakage+ # of the constant component in the input signal.+ filter_ /= filter_.sum()+ filter = filter_.view(1, 1, kernel_size)++ return filter+++class LowPassFilter1d(nn.Module):+ def __init__(self,+ cutoff=0.5,+ half_width=0.6,+ stride: int = 1,+ padding: bool = True,+ padding_mode: str = 'replicate',+ kernel_size: int = 12):+ # kernel_size should be even number for stylegan3 setup,+ # in this implementation, odd number is also possible.+ super().__init__()+ if cutoff < -0.:+ raise ValueError("Minimum cutoff must be larger than zero.")+ if cutoff > 0.5:+ raise ValueError("A cutoff above 0.5 does not make sense.")+ self.kernel_size = kernel_size+ self.even = (kernel_size % 2 == 0)+ self.pad_left = kernel_size // 2 - int(self.even)+ self.pad_right = kernel_size // 2+ self.stride = stride+ self.padding = padding+ self.padding_mode = padding_mode+ filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size)+ self.register_buffer("filter", filter)++ #input [B, C, T]+ def forward(self, x):+ _, C, _ = x.shape++ if self.padding:+ x = F.pad(x, (self.pad_left, self.pad_right),+ mode=self.padding_mode)+ out = F.conv1d(x, self.filter.expand(C, -1, -1),+ stride=self.stride, groups=C)++ return out\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/resample.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/resample.pynew file mode 100644index 0000000..750e6c3--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/alias_free_torch/resample.py@@ -0,0 +1,49 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++import torch.nn as nn+from torch.nn import functional as F+from .filter import LowPassFilter1d+from .filter import kaiser_sinc_filter1d+++class UpSample1d(nn.Module):+ def __init__(self, ratio=2, kernel_size=None):+ super().__init__()+ self.ratio = ratio+ self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size+ self.stride = ratio+ self.pad = self.kernel_size // ratio - 1+ self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2+ self.pad_right = self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2+ filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio,+ half_width=0.6 / ratio,+ kernel_size=self.kernel_size)+ self.register_buffer("filter", filter)++ # x: [B, C, T]+ def forward(self, x):+ _, C, _ = x.shape++ x = F.pad(x, (self.pad, self.pad), mode='replicate')+ x = self.ratio * F.conv_transpose1d(+ x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)+ x = x[..., self.pad_left:-self.pad_right]++ return x+++class DownSample1d(nn.Module):+ def __init__(self, ratio=2, kernel_size=None):+ super().__init__()+ self.ratio = ratio+ self.kernel_size = int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size+ self.lowpass = LowPassFilter1d(cutoff=0.5 / ratio,+ half_width=0.6 / ratio,+ stride=ratio,+ kernel_size=self.kernel_size)++ def forward(self, x):+ xx = self.lowpass(x)++ return xx\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/bigvgan.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/bigvgan.pynew file mode 100644index 0000000..4955f69--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/bigvgan.py@@ -0,0 +1,32 @@+from pathlib import Path++import torch+import torch.nn as nn+from omegaconf import OmegaConf++from .models import BigVGANVocoder++_bigvgan_vocoder_path = Path(__file__).parent / 'bigvgan_vocoder.yml'+++class BigVGAN(nn.Module):++ def __init__(self, ckpt_path, config_path=_bigvgan_vocoder_path):+ super().__init__()+ vocoder_cfg = OmegaConf.load(config_path)+ self.vocoder = BigVGANVocoder(vocoder_cfg).eval()+ vocoder_ckpt = torch.load(ckpt_path, map_location='cpu', weights_only=True)['generator']+ self.vocoder.load_state_dict(vocoder_ckpt)++ self.weight_norm_removed = False+ self.remove_weight_norm()++ @torch.inference_mode()+ def forward(self, x):+ assert self.weight_norm_removed, 'call remove_weight_norm() before inference'+ return self.vocoder(x)++ def remove_weight_norm(self):+ self.vocoder.remove_weight_norm()+ self.weight_norm_removed = True+ return selfdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/bigvgan_vocoder.yml b/models/wan/ovi/modules/mmaudio/ext/bigvgan/bigvgan_vocoder.ymlnew file mode 100644index 0000000..d4db31e--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/bigvgan_vocoder.yml@@ -0,0 +1,63 @@+resblock: '1'+num_gpus: 0+batch_size: 64+num_mels: 80+learning_rate: 0.0001+adam_b1: 0.8+adam_b2: 0.99+lr_decay: 0.999+seed: 1234+upsample_rates:+- 4+- 4+- 2+- 2+- 2+- 2+upsample_kernel_sizes:+- 8+- 8+- 4+- 4+- 4+- 4+upsample_initial_channel: 1536+resblock_kernel_sizes:+- 3+- 7+- 11+resblock_dilation_sizes:+- - 1+ - 3+ - 5+- - 1+ - 3+ - 5+- - 1+ - 3+ - 5+activation: snakebeta+snake_logscale: true+resolutions:+- - 1024+ - 120+ - 600+- - 2048+ - 240+ - 1200+- - 512+ - 50+ - 240+mpd_reshapes:+- 2+- 3+- 5+- 7+- 11+use_spectral_norm: false+discriminator_channel_mult: 1+num_workers: 4+dist_config:+ dist_backend: nccl+ dist_url: tcp://localhost:54341+ world_size: 1diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/env.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/env.pynew file mode 100644index 0000000..b8be238--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/env.py@@ -0,0 +1,18 @@+# Adapted from https://github.com/jik876/hifi-gan under the MIT license.+# LICENSE is in incl_licenses directory.++import os+import shutil+++class AttrDict(dict):+ def __init__(self, *args, **kwargs):+ super(AttrDict, self).__init__(*args, **kwargs)+ self.__dict__ = self+++def build_env(config, config_name, path):+ t_path = os.path.join(path, config_name)+ if config != t_path:+ os.makedirs(path, exist_ok=True)+ shutil.copyfile(config, os.path.join(path, config_name))\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_1 b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_1new file mode 100644index 0000000..5afae39--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_1@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2020 Jungil Kong++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_2 b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_2new file mode 100644index 0000000..322b758--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_2@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2020 Edward Dixon++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_3 b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_3new file mode 100644index 0000000..56ee3c8--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_3@@ -0,0 +1,201 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_4 b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_4new file mode 100644index 0000000..48fd1a1--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_4@@ -0,0 +1,29 @@+BSD 3-Clause License++Copyright (c) 2019, Seungwon Park 박승원+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its+ contributors may be used to endorse or promote products derived from+ this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_5 b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_5new file mode 100644index 0000000..01ae553--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/incl_licenses/LICENSE_5@@ -0,0 +1,16 @@+Copyright 2020 Alexandre Défossez++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and+associated documentation files (the "Software"), to deal in the Software without restriction,+including without limitation the rights to use, copy, modify, merge, publish, distribute,+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all copies or+substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/models.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/models.pynew file mode 100644index 0000000..b39e471--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/models.py@@ -0,0 +1,255 @@+# Copyright (c) 2022 NVIDIA CORPORATION.+# Licensed under the MIT license.++# Adapted from https://github.com/jik876/hifi-gan under the MIT license.+# LICENSE is in incl_licenses directory.++import torch+import torch.nn as nn+from torch.nn import Conv1d, ConvTranspose1d+from torch.nn.utils.parametrizations import weight_norm+from torch.nn.utils.parametrize import remove_parametrizations++from . import activations+from .alias_free_torch import *+from .utils import get_padding, init_weights++LRELU_SLOPE = 0.1+++class AMPBlock1(torch.nn.Module):++ def __init__(self, h, channels, kernel_size=3, dilation=(1, 3, 5), activation=None):+ super(AMPBlock1, self).__init__()+ self.h = h++ self.convs1 = nn.ModuleList([+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[0],+ padding=get_padding(kernel_size, dilation[0]))),+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[1],+ padding=get_padding(kernel_size, dilation[1]))),+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[2],+ padding=get_padding(kernel_size, dilation[2])))+ ])+ self.convs1.apply(init_weights)++ self.convs2 = nn.ModuleList([+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=1,+ padding=get_padding(kernel_size, 1))),+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=1,+ padding=get_padding(kernel_size, 1))),+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=1,+ padding=get_padding(kernel_size, 1)))+ ])+ self.convs2.apply(init_weights)++ self.num_layers = len(self.convs1) + len(self.convs2) # total number of conv layers++ if activation == 'snake': # periodic nonlinearity with snake function and anti-aliasing+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.Snake(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ elif activation == 'snakebeta': # periodic nonlinearity with snakebeta function and anti-aliasing+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.SnakeBeta(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ else:+ raise NotImplementedError(+ "activation incorrectly specified. check the config file and look for 'activation'."+ )++ def forward(self, x):+ acts1, acts2 = self.activations[::2], self.activations[1::2]+ for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):+ xt = a1(x)+ xt = c1(xt)+ xt = a2(xt)+ xt = c2(xt)+ x = xt + x++ return x++ def remove_weight_norm(self):+ for l in self.convs1:+ remove_parametrizations(l, 'weight')+ for l in self.convs2:+ remove_parametrizations(l, 'weight')+++class AMPBlock2(torch.nn.Module):++ def __init__(self, h, channels, kernel_size=3, dilation=(1, 3), activation=None):+ super(AMPBlock2, self).__init__()+ self.h = h++ self.convs = nn.ModuleList([+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[0],+ padding=get_padding(kernel_size, dilation[0]))),+ weight_norm(+ Conv1d(channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[1],+ padding=get_padding(kernel_size, dilation[1])))+ ])+ self.convs.apply(init_weights)++ self.num_layers = len(self.convs) # total number of conv layers++ if activation == 'snake': # periodic nonlinearity with snake function and anti-aliasing+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.Snake(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ elif activation == 'snakebeta': # periodic nonlinearity with snakebeta function and anti-aliasing+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.SnakeBeta(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ else:+ raise NotImplementedError(+ "activation incorrectly specified. check the config file and look for 'activation'."+ )++ def forward(self, x):+ for c, a in zip(self.convs, self.activations):+ xt = a(x)+ xt = c(xt)+ x = xt + x++ return x++ def remove_weight_norm(self):+ for l in self.convs:+ remove_parametrizations(l, 'weight')+++class BigVGANVocoder(torch.nn.Module):+ # this is our main BigVGAN model. Applies anti-aliased periodic activation for resblocks.+ def __init__(self, h):+ super().__init__()+ self.h = h++ self.num_kernels = len(h.resblock_kernel_sizes)+ self.num_upsamples = len(h.upsample_rates)++ # pre conv+ self.conv_pre = weight_norm(Conv1d(h.num_mels, h.upsample_initial_channel, 7, 1, padding=3))++ # define which AMPBlock to use. BigVGAN uses AMPBlock1 as default+ resblock = AMPBlock1 if h.resblock == '1' else AMPBlock2++ # transposed conv-based upsamplers. does not apply anti-aliasing+ self.ups = nn.ModuleList()+ for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):+ self.ups.append(+ nn.ModuleList([+ weight_norm(+ ConvTranspose1d(h.upsample_initial_channel // (2**i),+ h.upsample_initial_channel // (2**(i + 1)),+ k,+ u,+ padding=(k - u) // 2))+ ]))++ # residual blocks using anti-aliased multi-periodicity composition modules (AMP)+ self.resblocks = nn.ModuleList()+ for i in range(len(self.ups)):+ ch = h.upsample_initial_channel // (2**(i + 1))+ for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):+ self.resblocks.append(resblock(h, ch, k, d, activation=h.activation))++ # post conv+ if h.activation == "snake": # periodic nonlinearity with snake function and anti-aliasing+ activation_post = activations.Snake(ch, alpha_logscale=h.snake_logscale)+ self.activation_post = Activation1d(activation=activation_post)+ elif h.activation == "snakebeta": # periodic nonlinearity with snakebeta function and anti-aliasing+ activation_post = activations.SnakeBeta(ch, alpha_logscale=h.snake_logscale)+ self.activation_post = Activation1d(activation=activation_post)+ else:+ raise NotImplementedError(+ "activation incorrectly specified. check the config file and look for 'activation'."+ )++ self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3))++ # weight initialization+ for i in range(len(self.ups)):+ self.ups[i].apply(init_weights)+ self.conv_post.apply(init_weights)++ def forward(self, x):+ # pre conv+ x = self.conv_pre(x)++ for i in range(self.num_upsamples):+ # upsampling+ for i_up in range(len(self.ups[i])):+ x = self.ups[i][i_up](x)+ # AMP blocks+ xs = None+ for j in range(self.num_kernels):+ if xs is None:+ xs = self.resblocks[i * self.num_kernels + j](x)+ else:+ xs += self.resblocks[i * self.num_kernels + j](x)+ x = xs / self.num_kernels++ # post conv+ x = self.activation_post(x)+ x = self.conv_post(x)+ x = torch.tanh(x)++ return x++ def remove_weight_norm(self):+ print('Removing weight norm...')+ for l in self.ups:+ for l_i in l:+ remove_parametrizations(l_i, 'weight')+ for l in self.resblocks:+ l.remove_weight_norm()+ remove_parametrizations(self.conv_pre, 'weight')+ remove_parametrizations(self.conv_post, 'weight')diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan/utils.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan/utils.pynew file mode 100644index 0000000..aff7e65--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan/utils.py@@ -0,0 +1,31 @@+# Adapted from https://github.com/jik876/hifi-gan under the MIT license.+# LICENSE is in incl_licenses directory.++import os++import torch+from torch.nn.utils.parametrizations import weight_norm+++def init_weights(m, mean=0.0, std=0.01):+ classname = m.__class__.__name__+ if classname.find("Conv") != -1:+ m.weight.data.normal_(mean, std)+++def apply_weight_norm(m):+ classname = m.__class__.__name__+ if classname.find("Conv") != -1:+ weight_norm(m)+++def get_padding(kernel_size, dilation=1):+ return int((kernel_size * dilation - dilation) / 2)+++def load_checkpoint(filepath, device):+ assert os.path.isfile(filepath)+ print("Loading '{}'".format(filepath))+ checkpoint_dict = torch.load(filepath, map_location=device)+ print("Complete.")+ return checkpoint_dictdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/LICENSE b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/LICENSEnew file mode 100644index 0000000..4c78361--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/LICENSE@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2024 NVIDIA CORPORATION.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/__init__.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/activations.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/activations.pynew file mode 100644index 0000000..4f08dda--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/activations.py@@ -0,0 +1,126 @@+# Implementation adapted from https://github.com/EdwardDixon/snake under the MIT license.+# LICENSE is in incl_licenses directory.++import torch+from torch import nn, sin, pow+from torch.nn import Parameter+++class Snake(nn.Module):+ """+ Implementation of a sine-based periodic activation function+ Shape:+ - Input: (B, C, T)+ - Output: (B, C, T), same shape as the input+ Parameters:+ - alpha - trainable parameter+ References:+ - This activation function is from this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:+ https://arxiv.org/abs/2006.08195+ Examples:+ >>> a1 = snake(256)+ >>> x = torch.randn(256)+ >>> x = a1(x)+ """++ def __init__(+ self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False+ ):+ """+ Initialization.+ INPUT:+ - in_features: shape of the input+ - alpha: trainable parameter+ alpha is initialized to 1 by default, higher values = higher-frequency.+ alpha will be trained along with the rest of your model.+ """+ super(Snake, self).__init__()+ self.in_features = in_features++ # Initialize alpha+ self.alpha_logscale = alpha_logscale+ if self.alpha_logscale: # Log scale alphas initialized to zeros+ self.alpha = Parameter(torch.zeros(in_features) * alpha)+ else: # Linear scale alphas initialized to ones+ self.alpha = Parameter(torch.ones(in_features) * alpha)++ self.alpha.requires_grad = alpha_trainable++ self.no_div_by_zero = 0.000000001++ def forward(self, x):+ """+ Forward pass of the function.+ Applies the function to the input elementwise.+ Snake ∶= x + 1/a * sin^2 (xa)+ """+ alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # Line up with x to [B, C, T]+ if self.alpha_logscale:+ alpha = torch.exp(alpha)+ x = x + (1.0 / (alpha + self.no_div_by_zero)) * pow(sin(x * alpha), 2)++ return x+++class SnakeBeta(nn.Module):+ """+ A modified Snake function which uses separate parameters for the magnitude of the periodic components+ Shape:+ - Input: (B, C, T)+ - Output: (B, C, T), same shape as the input+ Parameters:+ - alpha - trainable parameter that controls frequency+ - beta - trainable parameter that controls magnitude+ References:+ - This activation function is a modified version based on this paper by Liu Ziyin, Tilman Hartwig, Masahito Ueda:+ https://arxiv.org/abs/2006.08195+ Examples:+ >>> a1 = snakebeta(256)+ >>> x = torch.randn(256)+ >>> x = a1(x)+ """++ def __init__(+ self, in_features, alpha=1.0, alpha_trainable=True, alpha_logscale=False+ ):+ """+ Initialization.+ INPUT:+ - in_features: shape of the input+ - alpha - trainable parameter that controls frequency+ - beta - trainable parameter that controls magnitude+ alpha is initialized to 1 by default, higher values = higher-frequency.+ beta is initialized to 1 by default, higher values = higher-magnitude.+ alpha will be trained along with the rest of your model.+ """+ super(SnakeBeta, self).__init__()+ self.in_features = in_features++ # Initialize alpha+ self.alpha_logscale = alpha_logscale+ if self.alpha_logscale: # Log scale alphas initialized to zeros+ self.alpha = Parameter(torch.zeros(in_features) * alpha)+ self.beta = Parameter(torch.zeros(in_features) * alpha)+ else: # Linear scale alphas initialized to ones+ self.alpha = Parameter(torch.ones(in_features) * alpha)+ self.beta = Parameter(torch.ones(in_features) * alpha)++ self.alpha.requires_grad = alpha_trainable+ self.beta.requires_grad = alpha_trainable++ self.no_div_by_zero = 0.000000001++ def forward(self, x):+ """+ Forward pass of the function.+ Applies the function to the input elementwise.+ SnakeBeta ∶= x + 1/b * sin^2 (xa)+ """+ alpha = self.alpha.unsqueeze(0).unsqueeze(-1) # Line up with x to [B, C, T]+ beta = self.beta.unsqueeze(0).unsqueeze(-1)+ if self.alpha_logscale:+ alpha = torch.exp(alpha)+ beta = torch.exp(beta)+ x = x + (1.0 / (beta + self.no_div_by_zero)) * pow(sin(x * alpha), 2)++ return xdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/__init__.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/__init__.pynew file mode 100644index 0000000..e69de29diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/activation1d.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/activation1d.pynew file mode 100644index 0000000..fbc0fd8--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/activation1d.py@@ -0,0 +1,77 @@+# Copyright (c) 2024 NVIDIA CORPORATION.+# Licensed under the MIT license.++import torch+import torch.nn as nn+from alias_free_activation.torch.resample import UpSample1d, DownSample1d++# load fused CUDA kernel: this enables importing anti_alias_activation_cuda+from alias_free_activation.cuda import load++anti_alias_activation_cuda = load.load()+++class FusedAntiAliasActivation(torch.autograd.Function):+ """+ Assumes filter size 12, replication padding on upsampling/downsampling, and logscale alpha/beta parameters as inputs.+ The hyperparameters are hard-coded in the kernel to maximize speed.+ NOTE: The fused kenrel is incorrect for Activation1d with different hyperparameters.+ """++ @staticmethod+ def forward(ctx, inputs, up_ftr, down_ftr, alpha, beta):+ activation_results = anti_alias_activation_cuda.forward(+ inputs, up_ftr, down_ftr, alpha, beta+ )++ return activation_results++ @staticmethod+ def backward(ctx, output_grads):+ raise NotImplementedError+ return output_grads, None, None+++class Activation1d(nn.Module):+ def __init__(+ self,+ activation,+ up_ratio: int = 2,+ down_ratio: int = 2,+ up_kernel_size: int = 12,+ down_kernel_size: int = 12,+ fused: bool = True,+ ):+ super().__init__()+ self.up_ratio = up_ratio+ self.down_ratio = down_ratio+ self.act = activation+ self.upsample = UpSample1d(up_ratio, up_kernel_size)+ self.downsample = DownSample1d(down_ratio, down_kernel_size)++ self.fused = fused # Whether to use fused CUDA kernel or not++ def forward(self, x):+ if not self.fused:+ x = self.upsample(x)+ x = self.act(x)+ x = self.downsample(x)+ return x+ else:+ if self.act.__class__.__name__ == "Snake":+ beta = self.act.alpha.data # Snake uses same params for alpha and beta+ else:+ beta = (+ self.act.beta.data+ ) # Snakebeta uses different params for alpha and beta+ alpha = self.act.alpha.data+ if (+ not self.act.alpha_logscale+ ): # Exp baked into cuda kernel, cancel it out with a log+ alpha = torch.log(alpha)+ beta = torch.log(beta)++ x = FusedAntiAliasActivation.apply(+ x, self.upsample.filter, self.downsample.lowpass.filter, alpha, beta+ )+ return xdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/anti_alias_activation.cpp b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/anti_alias_activation.cppnew file mode 100644index 0000000..c5651f7--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/anti_alias_activation.cpp@@ -0,0 +1,23 @@+/* coding=utf-8+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++ #include <torch/extension.h>++extern "C" torch::Tensor fwd_cuda(torch::Tensor const &input, torch::Tensor const &up_filter, torch::Tensor const &down_filter, torch::Tensor const &alpha, torch::Tensor const &beta);++PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {+ m.def("forward", &fwd_cuda, "Anti-Alias Activation forward (CUDA)");+}\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/anti_alias_activation_cuda.cu b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/anti_alias_activation_cuda.cunew file mode 100644index 0000000..8c44233--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/anti_alias_activation_cuda.cu@@ -0,0 +1,246 @@+/* coding=utf-8+ * Copyright (c) 2024, NVIDIA CORPORATION. All rights reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <ATen/ATen.h>+#include <cuda.h>+#include <cuda_runtime.h>+#include <cuda_fp16.h>+#include <cuda_profiler_api.h>+#include <ATen/cuda/CUDAContext.h>+#include <torch/extension.h>+#include "type_shim.h"+#include <assert.h>+#include <cfloat>+#include <limits>+#include <stdint.h>+#include <c10/macros/Macros.h>++namespace+{+ // Hard-coded hyperparameters+ // WARP_SIZE and WARP_BATCH must match the return values batches_per_warp and+ constexpr int ELEMENTS_PER_LDG_STG = 1; //(WARP_ITERATIONS < 4) ? 1 : 4;+ constexpr int BUFFER_SIZE = 32;+ constexpr int FILTER_SIZE = 12;+ constexpr int HALF_FILTER_SIZE = 6;+ constexpr int UPSAMPLE_REPLICATION_PAD = 5; // 5 on each side, matching torch impl+ constexpr int DOWNSAMPLE_REPLICATION_PAD_LEFT = 5; // matching torch impl+ constexpr int DOWNSAMPLE_REPLICATION_PAD_RIGHT = 6; // matching torch impl++ template <typename input_t, typename output_t, typename acc_t>+ __global__ void anti_alias_activation_forward(+ output_t *dst,+ const input_t *src,+ const input_t *up_ftr,+ const input_t *down_ftr,+ const input_t *alpha,+ const input_t *beta,+ int batch_size,+ int channels,+ int seq_len)+ {+ // Up and downsample filters+ input_t up_filter[FILTER_SIZE];+ input_t down_filter[FILTER_SIZE];++ // Load data from global memory including extra indices reserved for replication paddings+ input_t elements[2 * FILTER_SIZE + 2 * BUFFER_SIZE + 2 * UPSAMPLE_REPLICATION_PAD] = {0};+ input_t intermediates[2 * FILTER_SIZE + 2 * BUFFER_SIZE + DOWNSAMPLE_REPLICATION_PAD_LEFT + DOWNSAMPLE_REPLICATION_PAD_RIGHT] = {0};++ // Output stores downsampled output before writing to dst+ output_t output[BUFFER_SIZE];++ // blockDim/threadIdx = (128, 1, 1)+ // gridDim/blockIdx = (seq_blocks, channels, batches)+ int block_offset = (blockIdx.x * 128 * BUFFER_SIZE + seq_len * (blockIdx.y + gridDim.y * blockIdx.z));+ int local_offset = threadIdx.x * BUFFER_SIZE;+ int seq_offset = blockIdx.x * 128 * BUFFER_SIZE + local_offset;++ // intermediate have double the seq_len+ int intermediate_local_offset = threadIdx.x * BUFFER_SIZE * 2;+ int intermediate_seq_offset = blockIdx.x * 128 * BUFFER_SIZE * 2 + intermediate_local_offset;++ // Get values needed for replication padding before moving pointer+ const input_t *right_most_pntr = src + (seq_len * (blockIdx.y + gridDim.y * blockIdx.z));+ input_t seq_left_most_value = right_most_pntr[0];+ input_t seq_right_most_value = right_most_pntr[seq_len - 1];++ // Move src and dst pointers+ src += block_offset + local_offset;+ dst += block_offset + local_offset;++ // Alpha and beta values for snake activatons. Applies exp by default+ alpha = alpha + blockIdx.y;+ input_t alpha_val = expf(alpha[0]);+ beta = beta + blockIdx.y;+ input_t beta_val = expf(beta[0]);++ #pragma unroll+ for (int it = 0; it < FILTER_SIZE; it += 1)+ {+ up_filter[it] = up_ftr[it];+ down_filter[it] = down_ftr[it];+ }++ // Apply replication padding for upsampling, matching torch impl+ #pragma unroll+ for (int it = -HALF_FILTER_SIZE; it < BUFFER_SIZE + HALF_FILTER_SIZE; it += 1)+ {+ int element_index = seq_offset + it; // index for element+ if ((element_index < 0) && (element_index >= -UPSAMPLE_REPLICATION_PAD))+ {+ elements[2 * (HALF_FILTER_SIZE + it)] = 2 * seq_left_most_value;+ }+ if ((element_index >= seq_len) && (element_index < seq_len + UPSAMPLE_REPLICATION_PAD))+ {+ elements[2 * (HALF_FILTER_SIZE + it)] = 2 * seq_right_most_value;+ }+ if ((element_index >= 0) && (element_index < seq_len))+ {+ elements[2 * (HALF_FILTER_SIZE + it)] = 2 * src[it];+ }+ }++ // Apply upsampling strided convolution and write to intermediates. It reserves DOWNSAMPLE_REPLICATION_PAD_LEFT for replication padding of the downsampilng conv later+ #pragma unroll+ for (int it = 0; it < (2 * BUFFER_SIZE + 2 * FILTER_SIZE); it += 1)+ {+ input_t acc = 0.0;+ int element_index = intermediate_seq_offset + it; // index for intermediate+ #pragma unroll+ for (int f_idx = 0; f_idx < FILTER_SIZE; f_idx += 1)+ {+ if ((element_index + f_idx) >= 0)+ {+ acc += up_filter[f_idx] * elements[it + f_idx];+ }+ }+ intermediates[it + DOWNSAMPLE_REPLICATION_PAD_LEFT] = acc;+ }++ // Apply activation function. It reserves DOWNSAMPLE_REPLICATION_PAD_LEFT and DOWNSAMPLE_REPLICATION_PAD_RIGHT for replication padding of the downsampilng conv later+ double no_div_by_zero = 0.000000001;+ #pragma unroll+ for (int it = 0; it < 2 * BUFFER_SIZE + 2 * FILTER_SIZE; it += 1)+ {+ intermediates[it + DOWNSAMPLE_REPLICATION_PAD_LEFT] += (1.0 / (beta_val + no_div_by_zero)) * sinf(intermediates[it + DOWNSAMPLE_REPLICATION_PAD_LEFT] * alpha_val) * sinf(intermediates[it + DOWNSAMPLE_REPLICATION_PAD_LEFT] * alpha_val);+ }++ // Apply replication padding before downsampling conv from intermediates+ #pragma unroll+ for (int it = 0; it < DOWNSAMPLE_REPLICATION_PAD_LEFT; it += 1)+ {+ intermediates[it] = intermediates[DOWNSAMPLE_REPLICATION_PAD_LEFT];+ }+ #pragma unroll+ for (int it = DOWNSAMPLE_REPLICATION_PAD_LEFT + 2 * BUFFER_SIZE + 2 * FILTER_SIZE; it < DOWNSAMPLE_REPLICATION_PAD_LEFT + 2 * BUFFER_SIZE + 2 * FILTER_SIZE + DOWNSAMPLE_REPLICATION_PAD_RIGHT; it += 1)+ {+ intermediates[it] = intermediates[DOWNSAMPLE_REPLICATION_PAD_LEFT + 2 * BUFFER_SIZE + 2 * FILTER_SIZE - 1];+ }++ // Apply downsample strided convolution (assuming stride=2) from intermediates+ #pragma unroll+ for (int it = 0; it < BUFFER_SIZE; it += 1)+ {+ input_t acc = 0.0;+ #pragma unroll+ for (int f_idx = 0; f_idx < FILTER_SIZE; f_idx += 1)+ {+ // Add constant DOWNSAMPLE_REPLICATION_PAD_RIGHT to match torch implementation+ acc += down_filter[f_idx] * intermediates[it * 2 + f_idx + DOWNSAMPLE_REPLICATION_PAD_RIGHT];+ }+ output[it] = acc;+ }++ // Write output to dst+ #pragma unroll+ for (int it = 0; it < BUFFER_SIZE; it += ELEMENTS_PER_LDG_STG)+ {+ int element_index = seq_offset + it;+ if (element_index < seq_len)+ {+ dst[it] = output[it];+ }+ }++ }++ template <typename input_t, typename output_t, typename acc_t>+ void dispatch_anti_alias_activation_forward(+ output_t *dst,+ const input_t *src,+ const input_t *up_ftr,+ const input_t *down_ftr,+ const input_t *alpha,+ const input_t *beta,+ int batch_size,+ int channels,+ int seq_len)+ {+ if (seq_len == 0)+ {+ return;+ }+ else+ {+ // Use 128 threads per block to maximimize gpu utilization+ constexpr int threads_per_block = 128;+ constexpr int seq_len_per_block = 4096;+ int blocks_per_seq_len = (seq_len + seq_len_per_block - 1) / seq_len_per_block;+ dim3 blocks(blocks_per_seq_len, channels, batch_size);+ dim3 threads(threads_per_block, 1, 1);++ anti_alias_activation_forward<input_t, output_t, acc_t>+ <<<blocks, threads, 0, at::cuda::getCurrentCUDAStream()>>>(dst, src, up_ftr, down_ftr, alpha, beta, batch_size, channels, seq_len);+ }+ }+}++extern "C" torch::Tensor fwd_cuda(torch::Tensor const &input, torch::Tensor const &up_filter, torch::Tensor const &down_filter, torch::Tensor const &alpha, torch::Tensor const &beta)+{+ // Input is a 3d tensor with dimensions [batches, channels, seq_len]+ const int batches = input.size(0);+ const int channels = input.size(1);+ const int seq_len = input.size(2);++ // Output+ auto act_options = input.options().requires_grad(false);++ torch::Tensor anti_alias_activation_results =+ torch::empty({batches, channels, seq_len}, act_options);++ void *input_ptr = static_cast<void *>(input.data_ptr());+ void *up_filter_ptr = static_cast<void *>(up_filter.data_ptr());+ void *down_filter_ptr = static_cast<void *>(down_filter.data_ptr());+ void *alpha_ptr = static_cast<void *>(alpha.data_ptr());+ void *beta_ptr = static_cast<void *>(beta.data_ptr());+ void *anti_alias_activation_results_ptr = static_cast<void *>(anti_alias_activation_results.data_ptr());++ DISPATCH_FLOAT_HALF_AND_BFLOAT(+ input.scalar_type(),+ "dispatch anti alias activation_forward",+ dispatch_anti_alias_activation_forward<scalar_t, scalar_t, float>(+ reinterpret_cast<scalar_t *>(anti_alias_activation_results_ptr),+ reinterpret_cast<const scalar_t *>(input_ptr),+ reinterpret_cast<const scalar_t *>(up_filter_ptr),+ reinterpret_cast<const scalar_t *>(down_filter_ptr),+ reinterpret_cast<const scalar_t *>(alpha_ptr),+ reinterpret_cast<const scalar_t *>(beta_ptr),+ batches,+ channels,+ seq_len););+ return anti_alias_activation_results;+}\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/compat.h b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/compat.hnew file mode 100644index 0000000..25818b2--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/compat.h@@ -0,0 +1,29 @@+/* coding=utf-8+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++/*This code is copied fron NVIDIA apex:+ * https://github.com/NVIDIA/apex+ * with minor changes. */++#ifndef TORCH_CHECK+#define TORCH_CHECK AT_CHECK+#endif++#ifdef VERSION_GE_1_3+#define DATA_PTR data_ptr+#else+#define DATA_PTR data+#endifdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/load.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/load.pynew file mode 100644index 0000000..ca5d01d--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/load.py@@ -0,0 +1,86 @@+# Copyright (c) 2024 NVIDIA CORPORATION.+# Licensed under the MIT license.++import os+import pathlib+import subprocess++from torch.utils import cpp_extension++"""+Setting this param to a list has a problem of generating different compilation commands (with diferent order of architectures) and leading to recompilation of fused kernels.+Set it to empty stringo avoid recompilation and assign arch flags explicity in extra_cuda_cflags below+"""+os.environ["TORCH_CUDA_ARCH_LIST"] = ""+++def load():+ # Check if cuda 11 is installed for compute capability 8.0+ cc_flag = []+ _, bare_metal_major, _ = _get_cuda_bare_metal_version(cpp_extension.CUDA_HOME)+ if int(bare_metal_major) >= 11:+ cc_flag.append("-gencode")+ cc_flag.append("arch=compute_80,code=sm_80")++ # Build path+ srcpath = pathlib.Path(__file__).parent.absolute()+ buildpath = srcpath / "build"+ _create_build_dir(buildpath)++ # Helper function to build the kernels.+ def _cpp_extention_load_helper(name, sources, extra_cuda_flags):+ return cpp_extension.load(+ name=name,+ sources=sources,+ build_directory=buildpath,+ extra_cflags=[+ "-O3",+ ],+ extra_cuda_cflags=[+ "-O3",+ "-gencode",+ "arch=compute_70,code=sm_70",+ "--use_fast_math",+ ]+ + extra_cuda_flags+ + cc_flag,+ verbose=True,+ )++ extra_cuda_flags = [+ "-U__CUDA_NO_HALF_OPERATORS__",+ "-U__CUDA_NO_HALF_CONVERSIONS__",+ "--expt-relaxed-constexpr",+ "--expt-extended-lambda",+ ]++ sources = [+ srcpath / "anti_alias_activation.cpp",+ srcpath / "anti_alias_activation_cuda.cu",+ ]+ anti_alias_activation_cuda = _cpp_extention_load_helper(+ "anti_alias_activation_cuda", sources, extra_cuda_flags+ )++ return anti_alias_activation_cuda+++def _get_cuda_bare_metal_version(cuda_dir):+ raw_output = subprocess.check_output(+ [cuda_dir + "/bin/nvcc", "-V"], universal_newlines=True+ )+ output = raw_output.split()+ release_idx = output.index("release") + 1+ release = output[release_idx].split(".")+ bare_metal_major = release[0]+ bare_metal_minor = release[1][0]++ return raw_output, bare_metal_major, bare_metal_minor+++def _create_build_dir(buildpath):+ try:+ os.mkdir(buildpath)+ except OSError:+ if not os.path.isdir(buildpath):+ print(f"Creation of the build directory {buildpath} failed")diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/type_shim.h b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/type_shim.hnew file mode 100644index 0000000..5db7e8a--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/cuda/type_shim.h@@ -0,0 +1,92 @@+/* coding=utf-8+ * Copyright (c) 2020, NVIDIA CORPORATION. All rights reserved.+ *+ * Licensed under the Apache License, Version 2.0 (the "License");+ * you may not use this file except in compliance with the License.+ * You may obtain a copy of the License at+ *+ * http://www.apache.org/licenses/LICENSE-2.0+ *+ * Unless required by applicable law or agreed to in writing, software+ * distributed under the License is distributed on an "AS IS" BASIS,+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ * See the License for the specific language governing permissions and+ * limitations under the License.+ */++#include <ATen/ATen.h>+#include "compat.h"++#define DISPATCH_FLOAT_HALF_AND_BFLOAT(TYPE, NAME, ...) \+ switch (TYPE) \+ { \+ case at::ScalarType::Float: \+ { \+ using scalar_t = float; \+ __VA_ARGS__; \+ break; \+ } \+ case at::ScalarType::Half: \+ { \+ using scalar_t = at::Half; \+ __VA_ARGS__; \+ break; \+ } \+ case at::ScalarType::BFloat16: \+ { \+ using scalar_t = at::BFloat16; \+ __VA_ARGS__; \+ break; \+ } \+ default: \+ AT_ERROR(#NAME, " not implemented for '", toString(TYPE), "'"); \+ }++#define DISPATCH_FLOAT_HALF_AND_BFLOAT_INOUT_TYPES(TYPEIN, TYPEOUT, NAME, ...) \+ switch (TYPEIN) \+ { \+ case at::ScalarType::Float: \+ { \+ using scalar_t_in = float; \+ switch (TYPEOUT) \+ { \+ case at::ScalarType::Float: \+ { \+ using scalar_t_out = float; \+ __VA_ARGS__; \+ break; \+ } \+ case at::ScalarType::Half: \+ { \+ using scalar_t_out = at::Half; \+ __VA_ARGS__; \+ break; \+ } \+ case at::ScalarType::BFloat16: \+ { \+ using scalar_t_out = at::BFloat16; \+ __VA_ARGS__; \+ break; \+ } \+ default: \+ AT_ERROR(#NAME, " not implemented for '", toString(TYPEOUT), "'"); \+ } \+ break; \+ } \+ case at::ScalarType::Half: \+ { \+ using scalar_t_in = at::Half; \+ using scalar_t_out = at::Half; \+ __VA_ARGS__; \+ break; \+ } \+ case at::ScalarType::BFloat16: \+ { \+ using scalar_t_in = at::BFloat16; \+ using scalar_t_out = at::BFloat16; \+ __VA_ARGS__; \+ break; \+ } \+ default: \+ AT_ERROR(#NAME, " not implemented for '", toString(TYPEIN), "'"); \+ }diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/__init__.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/__init__.pynew file mode 100644index 0000000..8f756ed--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/__init__.py@@ -0,0 +1,6 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++from .filter import *+from .resample import *+from .act import *diff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/act.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/act.pynew file mode 100644index 0000000..f25dda1--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/act.py@@ -0,0 +1,32 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++import torch.nn as nn++from .resample import (DownSample1d, UpSample1d)+++class Activation1d(nn.Module):++ def __init__(+ self,+ activation,+ up_ratio: int = 2,+ down_ratio: int = 2,+ up_kernel_size: int = 12,+ down_kernel_size: int = 12,+ ):+ super().__init__()+ self.up_ratio = up_ratio+ self.down_ratio = down_ratio+ self.act = activation+ self.upsample = UpSample1d(up_ratio, up_kernel_size)+ self.downsample = DownSample1d(down_ratio, down_kernel_size)++ # x: [B,C,T]+ def forward(self, x):+ x = self.upsample(x)+ x = self.act(x)+ x = self.downsample(x)++ return xdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/filter.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/filter.pynew file mode 100644index 0000000..0fa35b0--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/filter.py@@ -0,0 +1,101 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++import torch+import torch.nn as nn+import torch.nn.functional as F+import math++if "sinc" in dir(torch):+ sinc = torch.sinc+else:+ # This code is adopted from adefossez's julius.core.sinc under the MIT License+ # https://adefossez.github.io/julius/julius/core.html+ # LICENSE is in incl_licenses directory.+ def sinc(x: torch.Tensor):+ """+ Implementation of sinc, i.e. sin(pi * x) / (pi * x)+ __Warning__: Different to julius.sinc, the input is multiplied by `pi`!+ """+ return torch.where(+ x == 0,+ torch.tensor(1.0, device=x.device, dtype=x.dtype),+ torch.sin(math.pi * x) / math.pi / x,+ )+++# This code is adopted from adefossez's julius.lowpass.LowPassFilters under the MIT License+# https://adefossez.github.io/julius/julius/lowpass.html+# LICENSE is in incl_licenses directory.+def kaiser_sinc_filter1d(+ cutoff, half_width, kernel_size+): # return filter [1,1,kernel_size]+ even = kernel_size % 2 == 0+ half_size = kernel_size // 2++ # For kaiser window+ delta_f = 4 * half_width+ A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95+ if A > 50.0:+ beta = 0.1102 * (A - 8.7)+ elif A >= 21.0:+ beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0)+ else:+ beta = 0.0+ window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)++ # ratio = 0.5/cutoff -> 2 * cutoff = 1 / ratio+ if even:+ time = torch.arange(-half_size, half_size) + 0.5+ else:+ time = torch.arange(kernel_size) - half_size+ if cutoff == 0:+ filter_ = torch.zeros_like(time)+ else:+ filter_ = 2 * cutoff * window * sinc(2 * cutoff * time)+ """+ Normalize filter to have sum = 1, otherwise we will have a small leakage of the constant component in the input signal.+ """+ filter_ /= filter_.sum()+ filter = filter_.view(1, 1, kernel_size)++ return filter+++class LowPassFilter1d(nn.Module):+ def __init__(+ self,+ cutoff=0.5,+ half_width=0.6,+ stride: int = 1,+ padding: bool = True,+ padding_mode: str = "replicate",+ kernel_size: int = 12,+ ):+ """+ kernel_size should be even number for stylegan3 setup, in this implementation, odd number is also possible.+ """+ super().__init__()+ if cutoff < -0.0:+ raise ValueError("Minimum cutoff must be larger than zero.")+ if cutoff > 0.5:+ raise ValueError("A cutoff above 0.5 does not make sense.")+ self.kernel_size = kernel_size+ self.even = kernel_size % 2 == 0+ self.pad_left = kernel_size // 2 - int(self.even)+ self.pad_right = kernel_size // 2+ self.stride = stride+ self.padding = padding+ self.padding_mode = padding_mode+ filter = kaiser_sinc_filter1d(cutoff, half_width, kernel_size)+ self.register_buffer("filter", filter)++ # Input [B, C, T]+ def forward(self, x):+ _, C, _ = x.shape++ if self.padding:+ x = F.pad(x, (self.pad_left, self.pad_right), mode=self.padding_mode)+ out = F.conv1d(x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)++ return outdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/resample.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/resample.pynew file mode 100644index 0000000..9ee1225--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/alias_free_activation/torch/resample.py@@ -0,0 +1,53 @@+# Adapted from https://github.com/junjun3518/alias-free-torch under the Apache License 2.0+# LICENSE is in incl_licenses directory.++import torch.nn as nn+from torch.nn import functional as F++from .filter import (LowPassFilter1d, kaiser_sinc_filter1d)+++class UpSample1d(nn.Module):++ def __init__(self, ratio=2, kernel_size=None):+ super().__init__()+ self.ratio = ratio+ self.kernel_size = (int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size)+ self.stride = ratio+ self.pad = self.kernel_size // ratio - 1+ self.pad_left = self.pad * self.stride + (self.kernel_size - self.stride) // 2+ self.pad_right = (self.pad * self.stride + (self.kernel_size - self.stride + 1) // 2)+ filter = kaiser_sinc_filter1d(cutoff=0.5 / ratio,+ half_width=0.6 / ratio,+ kernel_size=self.kernel_size)+ self.register_buffer("filter", filter)++ # x: [B, C, T]+ def forward(self, x):+ _, C, _ = x.shape++ x = F.pad(x, (self.pad, self.pad), mode="replicate")+ x = self.ratio * F.conv_transpose1d(+ x, self.filter.expand(C, -1, -1), stride=self.stride, groups=C)+ x = x[..., self.pad_left:-self.pad_right]++ return x+++class DownSample1d(nn.Module):++ def __init__(self, ratio=2, kernel_size=None):+ super().__init__()+ self.ratio = ratio+ self.kernel_size = (int(6 * ratio // 2) * 2 if kernel_size is None else kernel_size)+ self.lowpass = LowPassFilter1d(+ cutoff=0.5 / ratio,+ half_width=0.6 / ratio,+ stride=ratio,+ kernel_size=self.kernel_size,+ )++ def forward(self, x):+ xx = self.lowpass(x)++ return xxdiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/bigvgan.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/bigvgan.pynew file mode 100644index 0000000..cbcb21f--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/bigvgan.py@@ -0,0 +1,439 @@+# Copyright (c) 2024 NVIDIA CORPORATION.+# Licensed under the MIT license.++# Adapted from https://github.com/jik876/hifi-gan under the MIT license.+# LICENSE is in incl_licenses directory.++import json+import os+from pathlib import Path+from typing import Dict, Optional, Union++import torch+import torch.nn as nn+from huggingface_hub import PyTorchModelHubMixin, hf_hub_download+from torch.nn import Conv1d, ConvTranspose1d+from torch.nn.utils.parametrizations import weight_norm+from torch.nn.utils.parametrize import remove_parametrizations++from . import activations+from .alias_free_activation.torch.act import \+ Activation1d as TorchActivation1d+from .env import AttrDict+from .utils import get_padding, init_weights+++def load_hparams_from_json(path) -> AttrDict:+ with open(path) as f:+ data = f.read()+ return AttrDict(json.loads(data))+++class AMPBlock1(torch.nn.Module):+ """+ AMPBlock applies Snake / SnakeBeta activation functions with trainable parameters that control periodicity, defined for each layer.+ AMPBlock1 has additional self.convs2 that contains additional Conv1d layers with a fixed dilation=1 followed by each layer in self.convs1++ Args:+ h (AttrDict): Hyperparameters.+ channels (int): Number of convolution channels.+ kernel_size (int): Size of the convolution kernel. Default is 3.+ dilation (tuple): Dilation rates for the convolutions. Each dilation layer has two convolutions. Default is (1, 3, 5).+ activation (str): Activation function type. Should be either 'snake' or 'snakebeta'. Default is None.+ """++ def __init__(+ self,+ h: AttrDict,+ channels: int,+ kernel_size: int = 3,+ dilation: tuple = (1, 3, 5),+ activation: str = None,+ ):+ super().__init__()++ self.h = h++ self.convs1 = nn.ModuleList([+ weight_norm(+ Conv1d(+ channels,+ channels,+ kernel_size,+ stride=1,+ dilation=d,+ padding=get_padding(kernel_size, d),+ )) for d in dilation+ ])+ self.convs1.apply(init_weights)++ self.convs2 = nn.ModuleList([+ weight_norm(+ Conv1d(+ channels,+ channels,+ kernel_size,+ stride=1,+ dilation=1,+ padding=get_padding(kernel_size, 1),+ )) for _ in range(len(dilation))+ ])+ self.convs2.apply(init_weights)++ self.num_layers = len(self.convs1) + len(self.convs2) # Total number of conv layers++ # Select which Activation1d, lazy-load cuda version to ensure backward compatibility+ if self.h.get("use_cuda_kernel", False):+ from alias_free_activation.cuda.activation1d import \+ Activation1d as CudaActivation1d++ Activation1d = CudaActivation1d+ else:+ Activation1d = TorchActivation1d++ # Activation functions+ if activation == "snake":+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.Snake(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ elif activation == "snakebeta":+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.SnakeBeta(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ else:+ raise NotImplementedError(+ "activation incorrectly specified. check the config file and look for 'activation'."+ )++ def forward(self, x):+ acts1, acts2 = self.activations[::2], self.activations[1::2]+ for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):+ xt = a1(x)+ xt = c1(xt)+ xt = a2(xt)+ xt = c2(xt)+ x = xt + x++ return x++ def remove_weight_norm(self):+ for l in self.convs1:+ remove_parametrizations(l, 'weight')+ for l in self.convs2:+ remove_parametrizations(l, 'weight')+++class AMPBlock2(torch.nn.Module):+ """+ AMPBlock applies Snake / SnakeBeta activation functions with trainable parameters that control periodicity, defined for each layer.+ Unlike AMPBlock1, AMPBlock2 does not contain extra Conv1d layers with fixed dilation=1++ Args:+ h (AttrDict): Hyperparameters.+ channels (int): Number of convolution channels.+ kernel_size (int): Size of the convolution kernel. Default is 3.+ dilation (tuple): Dilation rates for the convolutions. Each dilation layer has two convolutions. Default is (1, 3, 5).+ activation (str): Activation function type. Should be either 'snake' or 'snakebeta'. Default is None.+ """++ def __init__(+ self,+ h: AttrDict,+ channels: int,+ kernel_size: int = 3,+ dilation: tuple = (1, 3, 5),+ activation: str = None,+ ):+ super().__init__()++ self.h = h++ self.convs = nn.ModuleList([+ weight_norm(+ Conv1d(+ channels,+ channels,+ kernel_size,+ stride=1,+ dilation=d,+ padding=get_padding(kernel_size, d),+ )) for d in dilation+ ])+ self.convs.apply(init_weights)++ self.num_layers = len(self.convs) # Total number of conv layers++ # Select which Activation1d, lazy-load cuda version to ensure backward compatibility+ if self.h.get("use_cuda_kernel", False):+ from alias_free_activation.cuda.activation1d import \+ Activation1d as CudaActivation1d++ Activation1d = CudaActivation1d+ else:+ Activation1d = TorchActivation1d++ # Activation functions+ if activation == "snake":+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.Snake(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ elif activation == "snakebeta":+ self.activations = nn.ModuleList([+ Activation1d(+ activation=activations.SnakeBeta(channels, alpha_logscale=h.snake_logscale))+ for _ in range(self.num_layers)+ ])+ else:+ raise NotImplementedError(+ "activation incorrectly specified. check the config file and look for 'activation'."+ )++ def forward(self, x):+ for c, a in zip(self.convs, self.activations):+ xt = a(x)+ xt = c(xt)+ x = xt + x+ return x++ def remove_weight_norm(self):+ for l in self.convs:+ remove_weight_norm(l)+++class BigVGAN(+ torch.nn.Module,+ PyTorchModelHubMixin,+ library_name="bigvgan",+ repo_url="https://github.com/NVIDIA/BigVGAN",+ docs_url="https://github.com/NVIDIA/BigVGAN/blob/main/README.md",+ pipeline_tag="audio-to-audio",+ license="mit",+ tags=["neural-vocoder", "audio-generation", "arxiv:2206.04658"],+):+ """+ BigVGAN is a neural vocoder model that applies anti-aliased periodic activation for residual blocks (resblocks).+ New in BigVGAN-v2: it can optionally use optimized CUDA kernels for AMP (anti-aliased multi-periodicity) blocks.++ Args:+ h (AttrDict): Hyperparameters.+ use_cuda_kernel (bool): If set to True, loads optimized CUDA kernels for AMP. This should be used for inference only, as training is not supported with CUDA kernels.++ Note:+ - The `use_cuda_kernel` parameter should be used for inference only, as training with CUDA kernels is not supported.+ - Ensure that the activation function is correctly specified in the hyperparameters (h.activation).+ """++ def __init__(self, h: AttrDict, use_cuda_kernel: bool = False):+ super().__init__()+ self.h = h+ self.h["use_cuda_kernel"] = use_cuda_kernel++ # Select which Activation1d, lazy-load cuda version to ensure backward compatibility+ if self.h.get("use_cuda_kernel", False):+ from alias_free_activation.cuda.activation1d import \+ Activation1d as CudaActivation1d++ Activation1d = CudaActivation1d+ else:+ Activation1d = TorchActivation1d++ self.num_kernels = len(h.resblock_kernel_sizes)+ self.num_upsamples = len(h.upsample_rates)++ # Pre-conv+ self.conv_pre = weight_norm(Conv1d(h.num_mels, h.upsample_initial_channel, 7, 1, padding=3))++ # Define which AMPBlock to use. BigVGAN uses AMPBlock1 as default+ if h.resblock == "1":+ resblock_class = AMPBlock1+ elif h.resblock == "2":+ resblock_class = AMPBlock2+ else:+ raise ValueError(+ f"Incorrect resblock class specified in hyperparameters. Got {h.resblock}")++ # Transposed conv-based upsamplers. does not apply anti-aliasing+ self.ups = nn.ModuleList()+ for i, (u, k) in enumerate(zip(h.upsample_rates, h.upsample_kernel_sizes)):+ self.ups.append(+ nn.ModuleList([+ weight_norm(+ ConvTranspose1d(+ h.upsample_initial_channel // (2**i),+ h.upsample_initial_channel // (2**(i + 1)),+ k,+ u,+ padding=(k - u) // 2,+ ))+ ]))++ # Residual blocks using anti-aliased multi-periodicity composition modules (AMP)+ self.resblocks = nn.ModuleList()+ for i in range(len(self.ups)):+ ch = h.upsample_initial_channel // (2**(i + 1))+ for j, (k, d) in enumerate(zip(h.resblock_kernel_sizes, h.resblock_dilation_sizes)):+ self.resblocks.append(resblock_class(h, ch, k, d, activation=h.activation))++ # Post-conv+ activation_post = (activations.Snake(ch, alpha_logscale=h.snake_logscale)+ if h.activation == "snake" else+ (activations.SnakeBeta(ch, alpha_logscale=h.snake_logscale)+ if h.activation == "snakebeta" else None))+ if activation_post is None:+ raise NotImplementedError(+ "activation incorrectly specified. check the config file and look for 'activation'."+ )++ self.activation_post = Activation1d(activation=activation_post)++ # Whether to use bias for the final conv_post. Default to True for backward compatibility+ self.use_bias_at_final = h.get("use_bias_at_final", True)+ self.conv_post = weight_norm(Conv1d(ch, 1, 7, 1, padding=3, bias=self.use_bias_at_final))++ # Weight initialization+ for i in range(len(self.ups)):+ self.ups[i].apply(init_weights)+ self.conv_post.apply(init_weights)++ # Final tanh activation. Defaults to True for backward compatibility+ self.use_tanh_at_final = h.get("use_tanh_at_final", True)++ def forward(self, x):+ # Pre-conv+ x = self.conv_pre(x)++ for i in range(self.num_upsamples):+ # Upsampling+ for i_up in range(len(self.ups[i])):+ x = self.ups[i][i_up](x)+ # AMP blocks+ xs = None+ for j in range(self.num_kernels):+ if xs is None:+ xs = self.resblocks[i * self.num_kernels + j](x)+ else:+ xs += self.resblocks[i * self.num_kernels + j](x)+ x = xs / self.num_kernels++ # Post-conv+ x = self.activation_post(x)+ x = self.conv_post(x)+ # Final tanh activation+ if self.use_tanh_at_final:+ x = torch.tanh(x)+ else:+ x = torch.clamp(x, min=-1.0, max=1.0) # Bound the output to [-1, 1]++ return x++ def remove_weight_norm(self):+ try:+ print("Removing weight norm...")+ for l in self.ups:+ for l_i in l:+ remove_parametrizations(l_i, 'weight')+ for l in self.resblocks:+ l.remove_weight_norm()+ remove_parametrizations(self.conv_pre, 'weight')+ remove_parametrizations(self.conv_post, 'weight')+ except ValueError:+ print("[INFO] Model already removed weight norm. Skipping!")+ pass++ # Additional methods for huggingface_hub support+ def _save_pretrained(self, save_directory: Path) -> None:+ """Save weights and config.json from a Pytorch model to a local directory."""++ model_path = save_directory / "bigvgan_generator.pt"+ torch.save({"generator": self.state_dict()}, model_path)++ config_path = save_directory / "config.json"+ with open(config_path, "w") as config_file:+ json.dump(self.h, config_file, indent=4)++ @classmethod+ def _from_pretrained(+ cls,+ *,+ model_id: str,+ revision: str,+ cache_dir: str,+ force_download: bool,+ proxies: Optional[Dict],+ resume_download: bool,+ local_files_only: bool,+ token: Union[str, bool, None],+ map_location: str = "cpu", # Additional argument+ strict: bool = False, # Additional argument+ use_cuda_kernel: bool = False,+ **model_kwargs,+ ):+ """Load Pytorch pretrained weights and return the loaded model."""++ # Download and load hyperparameters (h) used by BigVGAN+ if os.path.isdir(model_id):+ print("Loading config.json from local directory")+ config_file = os.path.join(model_id, "config.json")+ else:+ config_file = hf_hub_download(+ repo_id=model_id,+ filename="config.json",+ revision=revision,+ cache_dir=cache_dir,+ force_download=force_download,+ proxies=proxies,+ resume_download=resume_download,+ token=token,+ local_files_only=local_files_only,+ )+ h = load_hparams_from_json(config_file)++ # instantiate BigVGAN using h+ if use_cuda_kernel:+ print(+ f"[WARNING] You have specified use_cuda_kernel=True during BigVGAN.from_pretrained(). Only inference is supported (training is not implemented)!"+ )+ print(+ f"[WARNING] You need nvcc and ninja installed in your system that matches your PyTorch build is using to build the kernel. If not, the model will fail to initialize or generate incorrect waveform!"+ )+ print(+ f"[WARNING] For detail, see the official GitHub repository: https://github.com/NVIDIA/BigVGAN?tab=readme-ov-file#using-custom-cuda-kernel-for-synthesis"+ )+ model = cls(h, use_cuda_kernel=use_cuda_kernel)++ # Download and load pretrained generator weight+ if os.path.isdir(model_id):+ print("Loading weights from local directory")+ model_file = os.path.join(model_id, "bigvgan_generator.pt")+ else:+ print(f"Loading weights from {model_id}")+ model_file = hf_hub_download(+ repo_id=model_id,+ filename="bigvgan_generator.pt",+ revision=revision,+ cache_dir=cache_dir,+ force_download=force_download,+ proxies=proxies,+ resume_download=resume_download,+ token=token,+ local_files_only=local_files_only,+ )++ checkpoint_dict = torch.load(model_file, map_location=map_location, weights_only=True)++ try:+ model.load_state_dict(checkpoint_dict["generator"])+ except RuntimeError:+ print(+ f"[INFO] the pretrained checkpoint does not contain weight norm. Loading the checkpoint after removing weight norm!"+ )+ model.remove_weight_norm()+ model.load_state_dict(checkpoint_dict["generator"])++ return modeldiff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/env.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/env.pynew file mode 100644index 0000000..b8be238--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/env.py@@ -0,0 +1,18 @@+# Adapted from https://github.com/jik876/hifi-gan under the MIT license.+# LICENSE is in incl_licenses directory.++import os+import shutil+++class AttrDict(dict):+ def __init__(self, *args, **kwargs):+ super(AttrDict, self).__init__(*args, **kwargs)+ self.__dict__ = self+++def build_env(config, config_name, path):+ t_path = os.path.join(path, config_name)+ if config != t_path:+ os.makedirs(path, exist_ok=True)+ shutil.copyfile(config, os.path.join(path, config_name))\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_1 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_1new file mode 100644index 0000000..5afae39--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_1@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2020 Jungil Kong++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_2 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_2new file mode 100644index 0000000..322b758--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_2@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2020 Edward Dixon++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_3 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_3new file mode 100644index 0000000..56ee3c8--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_3@@ -0,0 +1,201 @@+ Apache License+ Version 2.0, January 2004+ http://www.apache.org/licenses/++ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION++ 1. Definitions.++ "License" shall mean the terms and conditions for use, reproduction,+ and distribution as defined by Sections 1 through 9 of this document.++ "Licensor" shall mean the copyright owner or entity authorized by+ the copyright owner that is granting the License.++ "Legal Entity" shall mean the union of the acting entity and all+ other entities that control, are controlled by, or are under common+ control with that entity. For the purposes of this definition,+ "control" means (i) the power, direct or indirect, to cause the+ direction or management of such entity, whether by contract or+ otherwise, or (ii) ownership of fifty percent (50%) or more of the+ outstanding shares, or (iii) beneficial ownership of such entity.++ "You" (or "Your") shall mean an individual or Legal Entity+ exercising permissions granted by this License.++ "Source" form shall mean the preferred form for making modifications,+ including but not limited to software source code, documentation+ source, and configuration files.++ "Object" form shall mean any form resulting from mechanical+ transformation or translation of a Source form, including but+ not limited to compiled object code, generated documentation,+ and conversions to other media types.++ "Work" shall mean the work of authorship, whether in Source or+ Object form, made available under the License, as indicated by a+ copyright notice that is included in or attached to the work+ (an example is provided in the Appendix below).++ "Derivative Works" shall mean any work, whether in Source or Object+ form, that is based on (or derived from) the Work and for which the+ editorial revisions, annotations, elaborations, or other modifications+ represent, as a whole, an original work of authorship. For the purposes+ of this License, Derivative Works shall not include works that remain+ separable from, or merely link (or bind by name) to the interfaces of,+ the Work and Derivative Works thereof.++ "Contribution" shall mean any work of authorship, including+ the original version of the Work and any modifications or additions+ to that Work or Derivative Works thereof, that is intentionally+ submitted to Licensor for inclusion in the Work by the copyright owner+ or by an individual or Legal Entity authorized to submit on behalf of+ the copyright owner. For the purposes of this definition, "submitted"+ means any form of electronic, verbal, or written communication sent+ to the Licensor or its representatives, including but not limited to+ communication on electronic mailing lists, source code control systems,+ and issue tracking systems that are managed by, or on behalf of, the+ Licensor for the purpose of discussing and improving the Work, but+ excluding communication that is conspicuously marked or otherwise+ designated in writing by the copyright owner as "Not a Contribution."++ "Contributor" shall mean Licensor and any individual or Legal Entity+ on behalf of whom a Contribution has been received by Licensor and+ subsequently incorporated within the Work.++ 2. Grant of Copyright License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ copyright license to reproduce, prepare Derivative Works of,+ publicly display, publicly perform, sublicense, and distribute the+ Work and such Derivative Works in Source or Object form.++ 3. Grant of Patent License. Subject to the terms and conditions of+ this License, each Contributor hereby grants to You a perpetual,+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable+ (except as stated in this section) patent license to make, have made,+ use, offer to sell, sell, import, and otherwise transfer the Work,+ where such license applies only to those patent claims licensable+ by such Contributor that are necessarily infringed by their+ Contribution(s) alone or by combination of their Contribution(s)+ with the Work to which such Contribution(s) was submitted. If You+ institute patent litigation against any entity (including a+ cross-claim or counterclaim in a lawsuit) alleging that the Work+ or a Contribution incorporated within the Work constitutes direct+ or contributory patent infringement, then any patent licenses+ granted to You under this License for that Work shall terminate+ as of the date such litigation is filed.++ 4. Redistribution. You may reproduce and distribute copies of the+ Work or Derivative Works thereof in any medium, with or without+ modifications, and in Source or Object form, provided that You+ meet the following conditions:++ (a) You must give any other recipients of the Work or+ Derivative Works a copy of this License; and++ (b) You must cause any modified files to carry prominent notices+ stating that You changed the files; and++ (c) You must retain, in the Source form of any Derivative Works+ that You distribute, all copyright, patent, trademark, and+ attribution notices from the Source form of the Work,+ excluding those notices that do not pertain to any part of+ the Derivative Works; and++ (d) If the Work includes a "NOTICE" text file as part of its+ distribution, then any Derivative Works that You distribute must+ include a readable copy of the attribution notices contained+ within such NOTICE file, excluding those notices that do not+ pertain to any part of the Derivative Works, in at least one+ of the following places: within a NOTICE text file distributed+ as part of the Derivative Works; within the Source form or+ documentation, if provided along with the Derivative Works; or,+ within a display generated by the Derivative Works, if and+ wherever such third-party notices normally appear. The contents+ of the NOTICE file are for informational purposes only and+ do not modify the License. You may add Your own attribution+ notices within Derivative Works that You distribute, alongside+ or as an addendum to the NOTICE text from the Work, provided+ that such additional attribution notices cannot be construed+ as modifying the License.++ You may add Your own copyright statement to Your modifications and+ may provide additional or different license terms and conditions+ for use, reproduction, or distribution of Your modifications, or+ for any such Derivative Works as a whole, provided Your use,+ reproduction, and distribution of the Work otherwise complies with+ the conditions stated in this License.++ 5. Submission of Contributions. Unless You explicitly state otherwise,+ any Contribution intentionally submitted for inclusion in the Work+ by You to the Licensor shall be under the terms and conditions of+ this License, without any additional terms or conditions.+ Notwithstanding the above, nothing herein shall supersede or modify+ the terms of any separate license agreement you may have executed+ with Licensor regarding such Contributions.++ 6. Trademarks. This License does not grant permission to use the trade+ names, trademarks, service marks, or product names of the Licensor,+ except as required for reasonable and customary use in describing the+ origin of the Work and reproducing the content of the NOTICE file.++ 7. Disclaimer of Warranty. Unless required by applicable law or+ agreed to in writing, Licensor provides the Work (and each+ Contributor provides its Contributions) on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or+ implied, including, without limitation, any warranties or conditions+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A+ PARTICULAR PURPOSE. You are solely responsible for determining the+ appropriateness of using or redistributing the Work and assume any+ risks associated with Your exercise of permissions under this License.++ 8. Limitation of Liability. In no event and under no legal theory,+ whether in tort (including negligence), contract, or otherwise,+ unless required by applicable law (such as deliberate and grossly+ negligent acts) or agreed to in writing, shall any Contributor be+ liable to You for damages, including any direct, indirect, special,+ incidental, or consequential damages of any character arising as a+ result of this License or out of the use or inability to use the+ Work (including but not limited to damages for loss of goodwill,+ work stoppage, computer failure or malfunction, or any and all+ other commercial damages or losses), even if such Contributor+ has been advised of the possibility of such damages.++ 9. Accepting Warranty or Additional Liability. While redistributing+ the Work or Derivative Works thereof, You may choose to offer,+ and charge a fee for, acceptance of support, warranty, indemnity,+ or other liability obligations and/or rights consistent with this+ License. However, in accepting such obligations, You may act only+ on Your own behalf and on Your sole responsibility, not on behalf+ of any other Contributor, and only if You agree to indemnify,+ defend, and hold each Contributor harmless for any liability+ incurred by, or claims asserted against, such Contributor by reason+ of your accepting any such warranty or additional liability.++ END OF TERMS AND CONDITIONS++ APPENDIX: How to apply the Apache License to your work.++ To apply the Apache License to your work, attach the following+ boilerplate notice, with the fields enclosed by brackets "[]"+ replaced with your own identifying information. (Don't include+ the brackets!) The text should be enclosed in the appropriate+ comment syntax for the file format. We also recommend that a+ file or class name and description of purpose be included on the+ same "printed page" as the copyright notice for easier+ identification within third-party archives.++ Copyright [yyyy] [name of copyright owner]++ Licensed under the Apache License, Version 2.0 (the "License");+ you may not use this file except in compliance with the License.+ You may obtain a copy of the License at++ http://www.apache.org/licenses/LICENSE-2.0++ Unless required by applicable law or agreed to in writing, software+ distributed under the License is distributed on an "AS IS" BASIS,+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.+ See the License for the specific language governing permissions and+ limitations under the License.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_4 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_4new file mode 100644index 0000000..48fd1a1--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_4@@ -0,0 +1,29 @@+BSD 3-Clause License++Copyright (c) 2019, Seungwon Park 박승원+All rights reserved.++Redistribution and use in source and binary forms, with or without+modification, are permitted provided that the following conditions are met:++1. Redistributions of source code must retain the above copyright notice, this+ list of conditions and the following disclaimer.++2. Redistributions in binary form must reproduce the above copyright notice,+ this list of conditions and the following disclaimer in the documentation+ and/or other materials provided with the distribution.++3. Neither the name of the copyright holder nor the names of its+ contributors may be used to endorse or promote products derived from+ this software without specific prior written permission.++THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS"+AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE+IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE+DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE+FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL+DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR+SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER+CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,+OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_5 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_5new file mode 100644index 0000000..01ae553--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_5@@ -0,0 +1,16 @@+Copyright 2020 Alexandre Défossez++Permission is hereby granted, free of charge, to any person obtaining a copy of this software and+associated documentation files (the "Software"), to deal in the Software without restriction,+including without limitation the rights to use, copy, modify, merge, publish, distribute,+sublicense, and/or sell copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all copies or+substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED, INCLUDING BUT+NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM,+DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_6 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_6new file mode 100644index 0000000..2569ec0--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_6@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2023-present, Descript++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_7 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_7new file mode 100644index 0000000..c37bdaf--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_7@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2023 Charactr Inc.++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_8 b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_8new file mode 100644index 0000000..ab3d7ff--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/incl_licenses/LICENSE_8@@ -0,0 +1,21 @@+MIT License++Copyright (c) 2023 Amphion++Permission is hereby granted, free of charge, to any person obtaining a copy+of this software and associated documentation files (the "Software"), to deal+in the Software without restriction, including without limitation the rights+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell+copies of the Software, and to permit persons to whom the Software is+furnished to do so, subject to the following conditions:++The above copyright notice and this permission notice shall be included in all+copies or substantial portions of the Software.++THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE+SOFTWARE.\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/utils.py b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/utils.pynew file mode 100644index 0000000..3b1d416--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/bigvgan_v2/utils.py@@ -0,0 +1,31 @@+# Adapted from https://github.com/jik876/hifi-gan under the MIT license.+# LICENSE is in incl_licenses directory.++import os++import torch+from torch.nn.utils import weight_norm+++def init_weights(m, mean=0.0, std=0.01):+ classname = m.__class__.__name__+ if classname.find("Conv") != -1:+ m.weight.data.normal_(mean, std)+++def apply_weight_norm(m):+ classname = m.__class__.__name__+ if classname.find("Conv") != -1:+ weight_norm(m)+++def get_padding(kernel_size, dilation=1):+ return int((kernel_size * dilation - dilation) / 2)+++def load_checkpoint(filepath, device):+ assert os.path.isfile(filepath)+ print(f"Loading '{filepath}'")+ checkpoint_dict = torch.load(filepath, map_location=device)+ print("Complete.")+ return checkpoint_dictdiff --git a/models/wan/ovi/modules/mmaudio/ext/mel_converter.py b/models/wan/ovi/modules/mmaudio/ext/mel_converter.pynew file mode 100644index 0000000..e75fb56--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/mel_converter.py@@ -0,0 +1,106 @@+# Reference: # https://github.com/bytedance/Make-An-Audio-2+from typing import Literal++import torch+import torch.nn as nn+from librosa.filters import mel as librosa_mel_fn+++def dynamic_range_compression_torch(x, C=1, clip_val=1e-5, *, norm_fn):+ return norm_fn(torch.clamp(x, min=clip_val) * C)+++def spectral_normalize_torch(magnitudes, norm_fn):+ output = dynamic_range_compression_torch(magnitudes, norm_fn=norm_fn)+ return output+++class MelConverter(nn.Module):++ def __init__(+ self,+ *,+ sampling_rate: float,+ n_fft: int,+ num_mels: int,+ hop_size: int,+ win_size: int,+ fmin: float,+ fmax: float,+ norm_fn,+ ):+ super().__init__()+ self.sampling_rate = sampling_rate+ self.n_fft = n_fft+ self.num_mels = num_mels+ self.hop_size = hop_size+ self.win_size = win_size+ self.fmin = fmin+ self.fmax = fmax+ self.norm_fn = norm_fn++ mel = librosa_mel_fn(sr=self.sampling_rate,+ n_fft=self.n_fft,+ n_mels=self.num_mels,+ fmin=self.fmin,+ fmax=self.fmax)+ mel_basis = torch.from_numpy(mel).float()+ hann_window = torch.hann_window(self.win_size)++ self.register_buffer('mel_basis', mel_basis)+ self.register_buffer('hann_window', hann_window)++ @property+ def device(self):+ return self.mel_basis.device++ def forward(self, waveform: torch.Tensor, center: bool = False) -> torch.Tensor:+ waveform = waveform.clamp(min=-1., max=1.).to(self.device)++ waveform = torch.nn.functional.pad(+ waveform.unsqueeze(1),+ [int((self.n_fft - self.hop_size) / 2),+ int((self.n_fft - self.hop_size) / 2)],+ mode='reflect')+ waveform = waveform.squeeze(1)++ spec = torch.stft(waveform,+ self.n_fft,+ hop_length=self.hop_size,+ win_length=self.win_size,+ window=self.hann_window,+ center=center,+ pad_mode='reflect',+ normalized=False,+ onesided=True,+ return_complex=True)++ spec = torch.view_as_real(spec)+ spec = torch.sqrt(spec.pow(2).sum(-1) + (1e-9)).float()+ spec = torch.matmul(self.mel_basis, spec)+ spec = spectral_normalize_torch(spec, self.norm_fn)++ return spec+++def get_mel_converter(mode: Literal['16k', '44k']) -> MelConverter:+ if mode == '16k':+ return MelConverter(sampling_rate=16_000,+ n_fft=1024,+ num_mels=80,+ hop_size=256,+ win_size=1024,+ fmin=0,+ fmax=8_000,+ norm_fn=torch.log10)+ elif mode == '44k':+ return MelConverter(sampling_rate=44_100,+ n_fft=2048,+ num_mels=128,+ hop_size=512,+ win_size=2048,+ fmin=0,+ fmax=44100 / 2,+ norm_fn=torch.log)+ else:+ raise ValueError(f'Unknown mode: {mode}')diff --git a/models/wan/ovi/modules/mmaudio/ext/rotary_embeddings.py b/models/wan/ovi/modules/mmaudio/ext/rotary_embeddings.pynew file mode 100644index 0000000..1ea9d56--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/rotary_embeddings.py@@ -0,0 +1,35 @@+from typing import Union++import torch+from einops import rearrange+from torch import Tensor++# Ref: https://github.com/black-forest-labs/flux/blob/main/src/flux/math.py+# Ref: https://github.com/lucidrains/rotary-embedding-torch+++def compute_rope_rotations(length: int,+ dim: int,+ theta: int,+ *,+ freq_scaling: float = 1.0,+ device: Union[torch.device, str] = 'cpu') -> Tensor:+ assert dim % 2 == 0++ with torch.amp.autocast(device_type='cuda', enabled=False):+ pos = torch.arange(length, dtype=torch.float32, device=device)+ freqs = 1.0 / (theta**(torch.arange(0, dim, 2, dtype=torch.float32, device=device) / dim))+ freqs *= freq_scaling++ rot = torch.einsum('..., f -> ... f', pos, freqs)+ rot = torch.stack([torch.cos(rot), -torch.sin(rot), torch.sin(rot), torch.cos(rot)], dim=-1)+ rot = rearrange(rot, 'n d (i j) -> 1 n d i j', i=2, j=2)+ return rot+++def apply_rope(x: Tensor, rot: Tensor) -> tuple[Tensor, Tensor]:+ with torch.amp.autocast(device_type='cuda', enabled=False):+ _x = x.float()+ _x = _x.view(*_x.shape[:-1], -1, 1, 2)+ x_out = rot[..., 0] * _x[..., 0] + rot[..., 1] * _x[..., 1]+ return x_out.reshape(*x.shape).to(dtype=x.dtype)diff --git a/models/wan/ovi/modules/mmaudio/ext/stft_converter.py b/models/wan/ovi/modules/mmaudio/ext/stft_converter.pynew file mode 100644index 0000000..6292206--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/stft_converter.py@@ -0,0 +1,183 @@+# Reference: # https://github.com/bytedance/Make-An-Audio-2++import torch+import torch.nn as nn+import torchaudio+from einops import rearrange+from librosa.filters import mel as librosa_mel_fn+++def dynamic_range_compression_torch(x, C=1, clip_val=1e-5, norm_fn=torch.log10):+ return norm_fn(torch.clamp(x, min=clip_val) * C)+++def spectral_normalize_torch(magnitudes, norm_fn):+ output = dynamic_range_compression_torch(magnitudes, norm_fn=norm_fn)+ return output+++class STFTConverter(nn.Module):++ def __init__(+ self,+ *,+ sampling_rate: float = 16_000,+ n_fft: int = 1024,+ num_mels: int = 128,+ hop_size: int = 256,+ win_size: int = 1024,+ fmin: float = 0,+ fmax: float = 8_000,+ norm_fn=torch.log,+ ):+ super().__init__()+ self.sampling_rate = sampling_rate+ self.n_fft = n_fft+ self.num_mels = num_mels+ self.hop_size = hop_size+ self.win_size = win_size+ self.fmin = fmin+ self.fmax = fmax+ self.norm_fn = norm_fn++ mel = librosa_mel_fn(sr=self.sampling_rate,+ n_fft=self.n_fft,+ n_mels=self.num_mels,+ fmin=self.fmin,+ fmax=self.fmax)+ mel_basis = torch.from_numpy(mel).float()+ hann_window = torch.hann_window(self.win_size)++ self.register_buffer('mel_basis', mel_basis)+ self.register_buffer('hann_window', hann_window)++ @property+ def device(self):+ return self.hann_window.device++ def forward(self, waveform: torch.Tensor) -> torch.Tensor:+ # input: batch_size * length+ bs = waveform.shape[0]+ waveform = waveform.clamp(min=-1., max=1.)++ spec = torch.stft(waveform,+ self.n_fft,+ hop_length=self.hop_size,+ win_length=self.win_size,+ window=self.hann_window,+ center=True,+ pad_mode='reflect',+ normalized=False,+ onesided=True,+ return_complex=True)++ spec = torch.view_as_real(spec)+ # print('After stft', spec.shape, spec.min(), spec.max(), spec.mean())++ power = spec.pow(2).sum(-1)+ angle = torch.atan2(spec[..., 1], spec[..., 0])++ print('power', power.shape, power.min(), power.max(), power.mean())+ print('angle', angle.shape, angle.min(), angle.max(), angle.mean())++ # print('mel', self.mel_basis.shape, self.mel_basis.min(), self.mel_basis.max(),+ # self.mel_basis.mean())++ # spec = rearrange(spec, 'b f t c -> (b c) f t')++ # spec = self.mel_transform(spec)++ # spec = torch.matmul(self.mel_basis, spec)++ # print('After mel', spec.shape, spec.min(), spec.max(), spec.mean())++ # spec = spectral_normalize_torch(spec, self.norm_fn)++ # print('After norm', spec.shape, spec.min(), spec.max(), spec.mean())++ # compute magnitude+ # magnitude = torch.sqrt((spec**2).sum(-1))+ # normalize by magnitude+ # scaled_magnitude = torch.log10(magnitude.clamp(min=1e-5)) * 10+ # spec = spec / magnitude.unsqueeze(-1) * scaled_magnitude.unsqueeze(-1)++ # power = torch.log10(power.clamp(min=1e-5)) * 10+ power = torch.log10(power.clamp(min=1e-5))++ print('After scaling', power.shape, power.min(), power.max(), power.mean())++ spec = torch.stack([power, angle], dim=-1)++ # spec = rearrange(spec, '(b c) f t -> b c f t', b=bs)+ spec = rearrange(spec, 'b f t c -> b c f t', b=bs)++ # spec[:, :, 400:] = 0++ return spec++ def invert(self, spec: torch.Tensor, length: int) -> torch.Tensor:+ bs = spec.shape[0]++ # spec = rearrange(spec, 'b c f t -> (b c) f t')+ # print(spec.shape, self.mel_basis.shape)+ # spec = torch.linalg.lstsq(self.mel_basis.unsqueeze(0), spec).solution+ # spec = torch.linalg.pinv(self.mel_basis.unsqueeze(0)) @ spec++ # spec = self.invmel_transform(spec)++ spec = rearrange(spec, 'b c f t -> b f t c', b=bs).contiguous()++ # spec[..., 0] = 10**(spec[..., 0] / 10)++ power = spec[..., 0]+ power = 10**power++ # print('After unscaling', spec[..., 0].shape, spec[..., 0].min(), spec[..., 0].max(),+ # spec[..., 0].mean())++ unit_vector = torch.stack([+ torch.cos(spec[..., 1]),+ torch.sin(spec[..., 1]),+ ], dim=-1)++ spec = torch.sqrt(power) * unit_vector++ # spec = rearrange(spec, '(b c) f t -> b f t c', b=bs).contiguous()+ spec = torch.view_as_complex(spec)++ waveform = torch.istft(+ spec,+ self.n_fft,+ length=length,+ hop_length=self.hop_size,+ win_length=self.win_size,+ window=self.hann_window,+ center=True,+ normalized=False,+ onesided=True,+ return_complex=False,+ )++ return waveform+++if __name__ == '__main__':++ converter = STFTConverter(sampling_rate=16000)++ signal = torchaudio.load('./output/ZZ6GRocWW38_000090.wav')[0]+ # resample signal at 44100 Hz+ # signal = torchaudio.transforms.Resample(16_000, 44_100)(signal)++ L = signal.shape[1]+ print('Input signal', signal.shape)+ spec = converter(signal)++ print('Final spec', spec.shape)++ signal_recon = converter.invert(spec, length=L)+ print('Output signal', signal_recon.shape, signal_recon.min(), signal_recon.max(),+ signal_recon.mean())++ print('MSE', torch.nn.functional.mse_loss(signal, signal_recon))+ torchaudio.save('./output/ZZ6GRocWW38_000090_recon.wav', signal_recon, 16000)diff --git a/models/wan/ovi/modules/mmaudio/ext/stft_converter_mel.py b/models/wan/ovi/modules/mmaudio/ext/stft_converter_mel.pynew file mode 100644index 0000000..f6b32d4--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/ext/stft_converter_mel.py@@ -0,0 +1,234 @@+# Reference: # https://github.com/bytedance/Make-An-Audio-2++import torch+import torch.nn as nn+import torchaudio+from einops import rearrange+from librosa.filters import mel as librosa_mel_fn+++def dynamic_range_compression_torch(x, C=1, clip_val=1e-5, norm_fn=torch.log10):+ return norm_fn(torch.clamp(x, min=clip_val) * C)+++def spectral_normalize_torch(magnitudes, norm_fn):+ output = dynamic_range_compression_torch(magnitudes, norm_fn=norm_fn)+ return output+++class STFTConverter(nn.Module):++ def __init__(+ self,+ *,+ sampling_rate: float = 16_000,+ n_fft: int = 1024,+ num_mels: int = 128,+ hop_size: int = 256,+ win_size: int = 1024,+ fmin: float = 0,+ fmax: float = 8_000,+ norm_fn=torch.log,+ ):+ super().__init__()+ self.sampling_rate = sampling_rate+ self.n_fft = n_fft+ self.num_mels = num_mels+ self.hop_size = hop_size+ self.win_size = win_size+ self.fmin = fmin+ self.fmax = fmax+ self.norm_fn = norm_fn++ mel = librosa_mel_fn(sr=self.sampling_rate,+ n_fft=self.n_fft,+ n_mels=self.num_mels,+ fmin=self.fmin,+ fmax=self.fmax)+ mel_basis = torch.from_numpy(mel).float()+ hann_window = torch.hann_window(self.win_size)++ self.register_buffer('mel_basis', mel_basis)+ self.register_buffer('hann_window', hann_window)++ @property+ def device(self):+ return self.hann_window.device++ def forward(self, waveform: torch.Tensor) -> torch.Tensor:+ # input: batch_size * length+ bs = waveform.shape[0]+ waveform = waveform.clamp(min=-1., max=1.)++ spec = torch.stft(waveform,+ self.n_fft,+ hop_length=self.hop_size,+ win_length=self.win_size,+ window=self.hann_window,+ center=True,+ pad_mode='reflect',+ normalized=False,+ onesided=True,+ return_complex=True)++ spec = torch.view_as_real(spec)+ # print('After stft', spec.shape, spec.min(), spec.max(), spec.mean())++ power = (spec.pow(2).sum(-1))**(0.5)+ angle = torch.atan2(spec[..., 1], spec[..., 0])++ print('power 1', power.shape, power.min(), power.max(), power.mean())+ print('angle 1', angle.shape, angle.min(), angle.max(), angle.mean(), angle[:, :2, :2])++ # print('mel', self.mel_basis.shape, self.mel_basis.min(), self.mel_basis.max(),+ # self.mel_basis.mean())++ # spec = self.mel_transform(spec)++ # power = torch.matmul(self.mel_basis, power)++ spec = rearrange(spec, 'b f t c -> (b c) f t')+ spec = self.mel_basis.unsqueeze(0) @ spec+ spec = rearrange(spec, '(b c) f t -> b f t c', b=bs)++ power = (spec.pow(2).sum(-1))**(0.5)+ angle = torch.atan2(spec[..., 1], spec[..., 0])++ print('power', power.shape, power.min(), power.max(), power.mean())+ print('angle', angle.shape, angle.min(), angle.max(), angle.mean(), angle[:, :2, :2])++ # print('After mel', spec.shape, spec.min(), spec.max(), spec.mean())++ # spec = spectral_normalize_torch(spec, self.norm_fn)++ # print('After norm', spec.shape, spec.min(), spec.max(), spec.mean())++ # compute magnitude+ # magnitude = torch.sqrt((spec**2).sum(-1))+ # normalize by magnitude+ # scaled_magnitude = torch.log10(magnitude.clamp(min=1e-5)) * 10+ # spec = spec / magnitude.unsqueeze(-1) * scaled_magnitude.unsqueeze(-1)++ # power = torch.log10(power.clamp(min=1e-5)) * 10+ power = torch.log10(power.clamp(min=1e-8))++ print('After scaling', power.shape, power.min(), power.max(), power.mean())++ # spec = torch.stack([power, angle], dim=-1)++ # spec = rearrange(spec, '(b c) f t -> b c f t', b=bs)+ # spec = rearrange(spec, 'b f t c -> b c f t', b=bs)++ # spec[:, :, 400:] = 0++ return power, angle+ # return spec[..., 0], spec[..., 1]++ def invert(self, spec: torch.Tensor, length: int) -> torch.Tensor:++ power, angle = spec++ bs = power.shape[0]++ # spec = rearrange(spec, 'b c f t -> (b c) f t')+ # print(spec.shape, self.mel_basis.shape)+ # spec = torch.linalg.lstsq(self.mel_basis.unsqueeze(0), spec).solution+ # spec = torch.linalg.pinv(self.mel_basis.unsqueeze(0)) @ spec++ # spec = self.invmel_transform(spec)++ # spec = rearrange(spec, 'b c f t -> b f t c', b=bs).contiguous()++ # spec[..., 0] = 10**(spec[..., 0] / 10)++ # power = spec[..., 0]+ power = 10**power++ # print('After unscaling', spec[..., 0].shape, spec[..., 0].min(), spec[..., 0].max(),+ # spec[..., 0].mean())++ unit_vector = torch.stack([+ torch.cos(angle),+ torch.sin(angle),+ ], dim=-1)++ spec = power.unsqueeze(-1) * unit_vector++ # power = torch.linalg.lstsq(self.mel_basis.unsqueeze(0), power).solution+ spec = rearrange(spec, 'b f t c -> (b c) f t')+ spec = torch.linalg.pinv(self.mel_basis.unsqueeze(0)) @ spec+ # spec = torch.linalg.lstsq(self.mel_basis.unsqueeze(0), spec).solution+ spec = rearrange(spec, '(b c) f t -> b f t c', b=bs).contiguous()++ power = (spec.pow(2).sum(-1))**(0.5)+ angle = torch.atan2(spec[..., 1], spec[..., 0])++ print('power 2', power.shape, power.min(), power.max(), power.mean())+ print('angle 2', angle.shape, angle.min(), angle.max(), angle.mean(), angle[:, :2, :2])++ # spec = rearrange(spec, '(b c) f t -> b f t c', b=bs).contiguous()+ spec = torch.view_as_complex(spec)++ waveform = torch.istft(+ spec,+ self.n_fft,+ length=length,+ hop_length=self.hop_size,+ win_length=self.win_size,+ window=self.hann_window,+ center=True,+ normalized=False,+ onesided=True,+ return_complex=False,+ )++ return waveform+++if __name__ == '__main__':++ converter = STFTConverter(sampling_rate=16000)++ signal = torchaudio.load('./output/ZZ6GRocWW38_000090.wav')[0]+ # resample signal at 44100 Hz+ # signal = torchaudio.transforms.Resample(16_000, 44_100)(signal)++ L = signal.shape[1]+ print('Input signal', signal.shape)+ spec = converter(signal)++ power, angle = spec++ # print(power.shape, angle.shape)+ # print(power, power.min(), power.max(), power.mean())+ # power = power.clamp(-1, 1)+ # angle = angle.clamp(-1, 1)++ import matplotlib.pyplot as plt++ # Visualize power+ plt.figure()+ plt.imshow(power[0].detach().numpy(), aspect='auto', origin='lower')+ plt.colorbar()+ plt.title('Power')+ plt.xlabel('Time')+ plt.ylabel('Frequency')+ plt.savefig('./output/power.png')++ # Visualize angle+ plt.figure()+ plt.imshow(angle[0].detach().numpy(), aspect='auto', origin='lower')+ plt.colorbar()+ plt.title('Angle')+ plt.xlabel('Time')+ plt.ylabel('Frequency')+ plt.savefig('./output/angle.png')++ # print('Final spec', spec.shape)++ signal_recon = converter.invert(spec, length=L)+ print('Output signal', signal_recon.shape, signal_recon.min(), signal_recon.max(),+ signal_recon.mean())++ print('MSE', torch.nn.functional.mse_loss(signal, signal_recon))+ torchaudio.save('./output/ZZ6GRocWW38_000090_recon.wav', signal_recon, 16000)diff --git a/models/wan/ovi/modules/mmaudio/features_utils.py b/models/wan/ovi/modules/mmaudio/features_utils.pynew file mode 100644index 0000000..7684dbc--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/features_utils.py@@ -0,0 +1,99 @@+from typing import Literal, Optional++import open_clip+import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange+from open_clip import create_model_from_pretrained+from torchvision.transforms import Normalize++from .ext.autoencoder import AutoEncoderModule+from .ext.autoencoder.distributions import DiagonalGaussianDistribution+from .ext.mel_converter import get_mel_converter+++def patch_clip(clip_model):+ # a hack to make it output last hidden states+ # https://github.com/mlfoundations/open_clip/blob/fc5a37b72d705f760ebbc7915b84729816ed471f/src/open_clip/model.py#L269+ def new_encode_text(self, text, normalize: bool = False):+ cast_dtype = self.transformer.get_cast_dtype()++ x = self.token_embedding(text).to(cast_dtype) # [batch_size, n_ctx, d_model]++ x = x + self.positional_embedding.to(cast_dtype)+ x = self.transformer(x, attn_mask=self.attn_mask)+ x = self.ln_final(x) # [batch_size, n_ctx, transformer.width]+ return F.normalize(x, dim=-1) if normalize else x++ clip_model.encode_text = new_encode_text.__get__(clip_model)+ return clip_model+++class FeaturesUtils(nn.Module):++ def __init__(+ self,+ *,+ tod_vae_ckpt: str,+ bigvgan_vocoder_ckpt: Optional[str] = None,+ mode=Literal['16k', '44k'],+ need_vae_encoder: bool = True,+ ):+ super().__init__()++ self.mel_converter = get_mel_converter(mode)+ self.tod = AutoEncoderModule(vae_ckpt_path=tod_vae_ckpt,+ vocoder_ckpt_path=bigvgan_vocoder_ckpt,+ mode=mode,+ need_vae_encoder=need_vae_encoder)++ def compile(self):++ self.decode = torch.compile(self.decode)+ self.vocode = torch.compile(self.vocode)++ def train(self, mode: bool) -> None:+ return super().train(False)++ @torch.inference_mode()+ def encode_audio(self, x) -> DiagonalGaussianDistribution:+ assert self.tod is not None, 'VAE is not loaded'+ # x: (B * L)+ mel = self.mel_converter(x)+ dist = self.tod.encode(mel)++ return dist++ @torch.inference_mode()+ def vocode(self, mel: torch.Tensor) -> torch.Tensor:+ assert self.tod is not None, 'VAE is not loaded'+ return self.tod.vocode(mel)++ @torch.inference_mode()+ def decode(self, z: torch.Tensor) -> torch.Tensor:+ assert self.tod is not None, 'VAE is not loaded'+ return self.tod.decode(z)++ @property+ def device(self):+ return next(self.parameters()).device++ @property+ def dtype(self):+ return next(self.parameters()).dtype++ @torch.no_grad()+ def wrapped_decode(self, z):+ with torch.amp.autocast('cuda', dtype=self.dtype):+ mel_decoded = self.decode(z)+ audio = self.vocode(mel_decoded)++ return audio++ @torch.no_grad()+ def wrapped_encode(self, audio):+ with torch.amp.autocast('cuda', dtype=self.dtype):+ dist = self.encode_audio(audio)++ return dist.mean\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/test_vae.py b/models/wan/ovi/modules/mmaudio/test_vae.pynew file mode 100644index 0000000..3c95377--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/test_vae.py@@ -0,0 +1,143 @@+#!/usr/bin/env python3++import os+import sys+import yaml+import torch+import random+import numpy as np+import soundfile as sf+from pathlib import Path+from datasets import load_dataset++# Add the current directory to path to import features_utils+sys.path.append(os.path.dirname(os.path.abspath(__file__)))+from features_utils import FeaturesUtils++def load_config(config_path):+ """Load configuration from yaml file"""+ with open(config_path, 'r') as f:+ return yaml.safe_load(f)++def main():+ # Set random seed for reproducibility+ random.seed(42)+ np.random.seed(42)+ torch.manual_seed(42)++ # Load configuration+ config_path = "/home/weiminwang/audio/veo3/multimodal-generation/models/wan/modules/mmaudio/test_vae_config.yaml"+ if not os.path.exists(config_path):+ print(f"Config file {config_path} not found!")+ return++ config = load_config(config_path)++ # Check if VAE checkpoints are specified in config+ vae_config = config.get('audio_vae', {})+ tod_vae_ckpt = vae_config.get('tod_vae_ckpt')+ bigvgan_vocoder_ckpt = vae_config.get('bigvgan_vocoder_ckpt')+ mode = vae_config.get('mode', '16k')+ need_vae_encoder = vae_config.get('need_vae_encoder', True)++ if not tod_vae_ckpt:+ print("tod_vae_ckpt not specified in config!")+ print("Please update vae_config.yaml with the path to your VAE checkpoint")+ return++ # Create output directory+ output_dir = Path("rec_results")+ output_dir.mkdir(exist_ok=True)++ print("Loading HuggingFace dataset...")+ # Load the NonverbalTTS dataset+ dataset = load_dataset("deepvk/NonverbalTTS")+ dev_split = dataset['dev']++ print(f"Dev split has {len(dev_split)} samples")++ # Select 10 random samples+ num_samples = min(10, len(dev_split))+ random_indices = random.sample(range(len(dev_split)), num_samples)++ print(f"Selected {num_samples} random samples: {random_indices}")++ # Initialize FeaturesUtils+ print("Initializing VAE...")+ try:+ features_utils = FeaturesUtils(+ tod_vae_ckpt=tod_vae_ckpt,+ bigvgan_vocoder_ckpt=bigvgan_vocoder_ckpt,+ mode=mode,+ need_vae_encoder=need_vae_encoder+ )++ # Move to GPU if available+ device = torch.device('cuda' if torch.cuda.is_available() else 'cpu')+ features_utils = features_utils.to(device)+ features_utils.eval()++ print(f"VAE initialized on device: {device}")++ except Exception as e:+ print(f"Error initializing VAE: {e}")+ print("Make sure the checkpoint paths in vae_config.yaml are correct")+ return++ # Process each selected sample+ for i, idx in enumerate(random_indices):+ try:+ print(f"Processing sample {i+1}/{num_samples} (index {idx})...")++ # Get audio data+ audio_data = dev_split[idx]['audio']+ audio_array = audio_data['array']+ sampling_rate = audio_data['sampling_rate']++ print(f" Original audio shape: {audio_array.shape}, SR: {sampling_rate}")++ # Convert to tensor and add batch dimension+ audio_tensor = torch.from_numpy(audio_array).float().unsqueeze(0).to(device)++ # Save original audio+ original_path = output_dir / f"{i}.wav"+ sf.write(original_path, audio_array, sampling_rate)++ # Encode audio+ print(f" Encoding... {audio_tensor.shape}")+ latent_features = features_utils.wrapped_encode(audio_tensor)+ print(f" Latent features shape: {latent_features.shape}")+ print(f" latent feature stats: {latent_features.mean().item()}, {latent_features.std().item()}, {latent_features.min().item()}, {latent_features.max().item()}")++ # Decode audio+ print(" Decoding...")+ reconstructed_audio = features_utils.wrapped_decode(latent_features)++ # Convert back to numpy+ reconstructed_np = reconstructed_audio.squeeze().cpu().numpy()+ print(f" Reconstructed audio shape: {reconstructed_np.shape}")++ # Save reconstructed audio+ reconstructed_path = output_dir / f"{i}_rec.wav"+ sf.write(reconstructed_path, reconstructed_np, sampling_rate)++ print(f" Saved: {original_path} and {reconstructed_path}")++ ## shape testing:+ dummy = torch.randn((12, 195584)).float().to(device) # Example shape (B, F, T)+ latent_features = features_utils.wrapped_encode(dummy)++ print(f"Shape testing: {dummy.shape} -> {latent_features.shape}")++ except Exception as e:+ print(f"Error processing sample {i} (index {idx}): {e}")+ continue++ print(f"\nProcessing complete! Results saved in {output_dir}/")+ print("Files saved:")+ for i in range(num_samples):+ print(f" {i}.wav - original audio")+ print(f" {i}_rec.wav - reconstructed audio")++if __name__ == "__main__":+ main()\ No newline at end of filediff --git a/models/wan/ovi/modules/mmaudio/test_vae_config.yaml b/models/wan/ovi/modules/mmaudio/test_vae_config.yamlnew file mode 100644index 0000000..731d726--- /dev/null+++ b/models/wan/ovi/modules/mmaudio/test_vae_config.yaml@@ -0,0 +1,5 @@+audio_vae:+ tod_vae_ckpt: /data/weiminwang/ckpts/mm_audio_ckpts/v1-16.pth+ bigvgan_vocoder_ckpt: /data/weiminwang/ckpts/mm_audio_ckpts/best_netG.pt+ mode: 16k+ need_vae_encoder: True\ No newline at end of filediff --git a/models/wan/ovi/modules/model.py b/models/wan/ovi/modules/model.pynew file mode 100644index 0000000..90427d8--- /dev/null+++ b/models/wan/ovi/modules/model.py@@ -0,0 +1,790 @@+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.+##### Enjoy this spagheti VRAM optimizations done by DeepBeepMeep !+# I am sure you are a nice person and as you copy this code, you will give me officially proper credits:+# Please link to https://github.com/deepbeepmeep/Wan2GP and @deepbeepmeep on twitter+import math+import torch+import torch.amp as amp+import torch.nn as nn+import torch.nn.functional as F++from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.models.modeling_utils import ModelMixin+from shared.attention import pay_attention+++def sinusoidal_embedding_1d(dim, position):+ # preprocess+ assert dim % 2 == 0+ half = dim // 2+ position = position.type(torch.float64)++ # calculation+ sinusoid = torch.outer(+ position, torch.pow(10000, -torch.arange(half).to(position).div(half)))+ x = torch.cat([torch.cos(sinusoid), torch.sin(sinusoid)], dim=1)+ return x++def reshape_latent(latent, latent_frames):+ return latent.reshape(latent.shape[0], latent_frames, -1, latent.shape[-1] )++def restore_latent_shape(latent):+ return latent.reshape(latent.shape[0], -1, latent.shape[-1] )++[email protected]('cuda', enabled=False)+def rope_params(max_seq_len, dim, theta=10000, freqs_scaling=1.0):+ assert dim % 2 == 0+ pos = torch.arange(max_seq_len)+ freqs = 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float64).div(dim))+ freqs = freqs_scaling * freqs+ freqs = torch.outer(pos, freqs)+ freqs = torch.polar(torch.ones_like(freqs), freqs)+ return freqs++# def rope_params(max_seq_len, dim, theta=10000, freqs_scaling=1.0, use_real = True):+# freqs = 1.0 / (+# theta ** (torch.arange(0, dim, 2)[: (dim // 2)].float() / dim)+# ) # [D/2]+# # assert interpolation_factor == 1.0, f"interpolation_factor: {interpolation_factor}"+# freqs = torch.outer(pos * interpolation_factor, freqs) # [S, D/2]+# if use_real:+# freqs_cos = freqs.cos().repeat_interleave(2, dim=1) # [S, D]+# freqs_sin = freqs.sin().repeat_interleave(2, dim=1) # [S, D]+# return freqs_cos, freqs_sin+# else:+# freqs_cis = torch.polar( torch.ones_like(freqs), freqs) # complex64 # [S, D/2]+# return freqs_cis+[email protected]('cuda', enabled=False)+def rope_apply_1d(x, grid_sizes, freqs):+ n, c = x.size(2), x.size(3) // 2 ## b l h d+ c_rope = freqs.shape[1] # number of complex dims to rotate+ assert c_rope <= c, "RoPE dimensions cannot exceed half of hidden size"++ # loop over samples+ output = []+ for i, (l, ) in enumerate(grid_sizes.tolist()):+ seq_len = l+ # precompute multipliers+ x_i = torch.view_as_complex(x[i, :seq_len].to(torch.float64).reshape(+ seq_len, n, -1, 2)) # [l n d//2]+ x_i_rope = x_i[:, :, :c_rope] * freqs[:seq_len, None, :] # [L, N, c_rope]+ x_i_passthrough = x_i[:, :, c_rope:] # untouched dims+ x_i = torch.cat([x_i_rope, x_i_passthrough], dim=2)++ # apply rotary embedding+ x_i = torch.view_as_real(x_i).flatten(2)+ x_i = torch.cat([x_i, x[i, seq_len:]])++ # append to collection+ output.append(x_i)+ return torch.stack(output).bfloat16()+[email protected]('cuda', enabled=False)+def rope_apply_3d(x, grid_sizes, freqs):+ n, c = x.size(2), x.size(3) // 2++ # split freqs+ freqs = freqs.split([c - 2 * (c // 3), c // 3, c // 3], dim=1)++ # loop over samples+ output = []+ for i, (f, h, w) in enumerate(grid_sizes.tolist()):+ seq_len = f * h * w++ # precompute multipliers+ x_i = torch.view_as_complex(x[i, :seq_len].to(torch.float64).reshape(+ seq_len, n, -1, 2))+ freqs_i = torch.cat([+ freqs[0][:f].view(f, 1, 1, -1).expand(f, h, w, -1),+ freqs[1][:h].view(1, h, 1, -1).expand(f, h, w, -1),+ freqs[2][:w].view(1, 1, w, -1).expand(f, h, w, -1)+ ],+ dim=-1).reshape(seq_len, 1, -1)++ # apply rotary embedding+ x_i = torch.view_as_real(x_i * freqs_i).flatten(2)+ x_i = torch.cat([x_i, x[i, seq_len:]])++ # append to collection+ output.append(x_i)+ return torch.stack(output).bfloat16()+[email protected]('cuda', enabled=False)+def rope_apply(x, grid_sizes, freqs):+ if isinstance(freqs, tuple):+ from ...modules.posemb_layers import apply_rotary_emb_single+ qklist = [x]+ del x+ return apply_rotary_emb_single(qklist, freqs, head_first=False)+ else:+ x_ndim = grid_sizes.shape[-1]+ if x_ndim == 3:+ return rope_apply_3d(x, grid_sizes, freqs)+ else:+ return rope_apply_1d(x, grid_sizes, freqs)++++class ChannelLastConv1d(nn.Conv1d):++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ x = x.permute(0, 2, 1)+ x = super().forward(x)+ x = x.permute(0, 2, 1)+ return x+++class ConvMLP(nn.Module):++ def __init__(+ self,+ dim: int,+ hidden_dim: int,+ multiple_of: int = 256,+ kernel_size: int = 3,+ padding: int = 1,+ ):+ """+ Initialize the FeedForward module.++ Args:+ dim (int): Input dimension.+ hidden_dim (int): Hidden dimension of the feedforward layer.+ multiple_of (int): Value to ensure hidden dimension is a multiple of this value.++ Attributes:+ w1 (ColumnParallelLinear): Linear transformation for the first layer.+ w2 (RowParallelLinear): Linear transformation for the second layer.+ w3 (ColumnParallelLinear): Linear transformation for the third layer.++ """+ super().__init__()+ hidden_dim = int(2 * hidden_dim / 3)+ hidden_dim = multiple_of * ((hidden_dim + multiple_of - 1) // multiple_of)++ self.w1 = ChannelLastConv1d(dim,+ hidden_dim,+ bias=False,+ kernel_size=kernel_size,+ padding=padding)+ self.w2 = ChannelLastConv1d(hidden_dim,+ dim,+ bias=False,+ kernel_size=kernel_size,+ padding=padding)+ self.w3 = ChannelLastConv1d(dim,+ hidden_dim,+ bias=False,+ kernel_size=kernel_size,+ padding=padding)++ def forward(self, x):+ return self.w2(F.silu(self.w1(x)) * self.w3(x))++class WanRMSNorm(nn.Module):++ def __init__(self, dim, eps=1e-5):+ super().__init__()+ self.dim = dim+ self.eps = eps+ self.weight = nn.Parameter(torch.ones(dim))++ def forward(self, x):+ r"""+ Args:+ x(Tensor): Shape [B, L, C]+ """+ return self._norm(x.bfloat16()).type_as(x) * self.weight.bfloat16()++ def _norm(self, x):+ return x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)+++class WanLayerNorm(nn.LayerNorm):++ def __init__(self, dim, eps=1e-6, elementwise_affine=False):+ super().__init__(dim, elementwise_affine=elementwise_affine, eps=eps)++ def forward(self, x):+ r"""+ Args:+ x(Tensor): Shape [B, L, C]+ """+ return super().forward(x.bfloat16()).type_as(x)+++class WanSelfAttention(nn.Module):++ def __init__(self,+ dim,+ num_heads,+ window_size=(-1, -1),+ qk_norm=True,+ eps=1e-6):+ assert dim % num_heads == 0+ super().__init__()+ self.dim = dim+ self.num_heads = num_heads+ self.head_dim = dim // num_heads+ self.window_size = window_size+ self.qk_norm = qk_norm+ self.eps = eps++ # layers+ self.q = nn.Linear(dim, dim)+ self.k = nn.Linear(dim, dim)+ self.v = nn.Linear(dim, dim)+ self.o = nn.Linear(dim, dim)+ self.norm_q = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity()+ self.norm_k = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity()++ # query, key, value function+ def qkv_fn(self, x):+ b, s, n, d = *x.shape[:2], self.num_heads, self.head_dim++ q = self.norm_q(self.q(x)).view(b, s, n, d)+ k = self.norm_k(self.k(x)).view(b, s, n, d)+ v = self.v(x).view(b, s, n, d)+ return q, k, v++ def forward(self, x, seq_lens, grid_sizes, freqs):+ r"""+ Args:+ x(Tensor): Shape [B, L, C]+ seq_lens(Tensor): Shape [B]+ grid_sizes(Tensor): Shape [B, 3], the second dimension contains (F, H, W)+ freqs(Tensor): Rope freqs, shape [1024, C / num_heads / 2]+ """+ q, k, v = self.qkv_fn(x)+ q=rope_apply(q, grid_sizes, freqs)+ k=rope_apply(k, grid_sizes, freqs)++ qkv_list =[q,k,v]+ del q,k,v+ x = pay_attention( qkv_list)+ x = x.flatten(2)+ x = self.o(x)+ return x+++class WanT2VCrossAttention(WanSelfAttention):+ def qkv_fn(self, x, context):+ b, n, d = x.size(0), self.num_heads, self.head_dim++ # compute query, key, value+ q = self.norm_q(self.q(x)).view(b, -1, n, d)+ k = self.norm_k(self.k(context)).view(b, -1, n, d)+ v = self.v(context).view(b, -1, n, d)++ return q, k, v++ def forward(self, x, context, context_lens):+ raise Exception("this code is never executed, you should not be here")++++class WanI2VCrossAttention(WanSelfAttention):++ def __init__(self,+ dim,+ num_heads,+ window_size=(-1, -1),+ qk_norm=True,+ eps=1e-6,+ additional_emb_length=None):+ super().__init__(dim, num_heads, window_size, qk_norm, eps)++ self.k_img = nn.Linear(dim, dim)+ self.v_img = nn.Linear(dim, dim)+ # self.alpha = nn.Parameter(torch.zeros((1, )))+ self.norm_k_img = WanRMSNorm(dim, eps=eps) if qk_norm else nn.Identity()+ self.additional_emb_length = additional_emb_length++ def qkv_fn(self, x, context):+ context_img = context[:, : self.additional_emb_length]+ context = context[:, self.additional_emb_length :]+ b, n, d = x.size(0), self.num_heads, self.head_dim++ # compute query, key, value+ q = self.norm_q(self.q(x)).view(b, -1, n, d)+ k = self.norm_k(self.k(context)).view(b, -1, n, d)+ v = self.v(context).view(b, -1, n, d)+ k_img = self.norm_k_img(self.k_img(context_img)).view(b, -1, n, d)+ v_img = self.v_img(context_img).view(b, -1, n, d)++ return q, k, v, k_img, v_img+++ def forward(self, x, context, context_lens):+ raise Exception("this code is never executed, you should not be here")++++WAN_CROSSATTENTION_CLASSES = {+ 't2v_cross_attn': WanT2VCrossAttention,+ 'i2v_cross_attn': WanI2VCrossAttention,+}++class ModulationAdd(nn.Module):+ def __init__(self, dim, num):+ super().__init__()+ self.modulation = nn.Parameter(torch.randn(1, num, dim) / dim**0.5)++ def forward(self, e):+ return self.modulation.bfloat16() + e.bfloat16()++class WanAttentionBlock(nn.Module):++ def __init__(self,+ cross_attn_type,+ dim,+ ffn_dim,+ num_heads,+ window_size=(-1, -1),+ qk_norm=True,+ cross_attn_norm=False,+ eps=1e-6,+ additional_emb_length=None):+ super().__init__()+ self.dim = dim+ self.ffn_dim = ffn_dim+ self.num_heads = num_heads+ self.window_size = window_size+ self.qk_norm = qk_norm+ self.cross_attn_norm = cross_attn_norm+ self.eps = eps++ # layers+ self.norm1 = WanLayerNorm(dim, eps)+ self.self_attn = WanSelfAttention(dim, num_heads, window_size, qk_norm,+ eps)+ self.norm3 = WanLayerNorm(+ dim, eps,+ elementwise_affine=True) if cross_attn_norm else nn.Identity()+ if cross_attn_type == 'i2v_cross_attn':+ assert additional_emb_length is not None, "additional_emb_length should be specified for i2v_cross_attn"+ self.cross_attn = WanI2VCrossAttention(dim,+ num_heads,+ (-1, -1),+ qk_norm,+ eps,+ additional_emb_length)+ else:+ assert additional_emb_length is None, "additional_emb_length should be None for t2v_cross_attn"+ self.cross_attn = WanT2VCrossAttention(dim,+ num_heads,+ (-1, -1),+ qk_norm,+ eps, )+ self.norm2 = WanLayerNorm(dim, eps)+ self.ffn = nn.Sequential(+ nn.Linear(dim, ffn_dim), nn.GELU(approximate='tanh'),+ nn.Linear(ffn_dim, dim))++ # modulation+ # self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)+ # self.modulation = nn.Parameter(torch.randn(1, 6, dim) / dim**0.5)+ self.modulation = ModulationAdd(dim, 6)+++ def forward(+ self,+ x,+ e,+ seq_lens,+ grid_sizes,+ freqs,+ context,+ context_lens,+ ):+ raise Exception("this code is never executed, you should not be here")++++class Head(nn.Module):++ def __init__(self, dim, out_dim, patch_size, eps=1e-6):+ super().__init__()+ self.dim = dim+ self.out_dim = out_dim+ self.patch_size = patch_size+ self.eps = eps++ # layers+ out_dim = math.prod(patch_size) * out_dim+ self.norm = WanLayerNorm(dim, eps)+ self.head = nn.Linear(dim, out_dim)++ # modulation+ self.modulation = nn.Parameter(torch.randn(1, 2, dim) / dim**0.5)++ def forward(self, x, e):+ r"""+ Args:+ x(Tensor): Shape [B, L1, C]+ e(Tensor): Shape [B, C]+ """+ # assert e.dtype == torch.float32+ dtype = x.dtype++ latent_frames = e.shape[0]+ e = (self.modulation + e.unsqueeze(1)).chunk(2, dim=1)+ x = self.norm(x).to(dtype)+ x = reshape_latent(x , latent_frames)+ x *= (1 + e[1])+ x += e[0]+ x = restore_latent_shape(x)+ x= x.to(self.head.weight.dtype)+ x = self.head(x)+ return x+++class MLPProj(torch.nn.Module):++ def __init__(self, in_dim, out_dim):+ super().__init__()++ self.proj = torch.nn.Sequential(+ torch.nn.LayerNorm(in_dim), torch.nn.Linear(in_dim, in_dim),+ torch.nn.GELU(), torch.nn.Linear(in_dim, out_dim),+ torch.nn.LayerNorm(out_dim))++ def forward(self, image_embeds):+ clip_extra_context_tokens = self.proj(image_embeds)+ return clip_extra_context_tokens+++class WanModel(ModelMixin, ConfigMixin):+ r"""+ Wan diffusion backbone supporting both text-to-video and image-to-video, text-to-audio.+ """++ ignore_for_config = [+ 'patch_size', 'cross_attn_norm', 'qk_norm', 'text_dim', 'window_size'+ ]+ _no_split_modules = ['WanAttentionBlock']++ @register_to_config+ def __init__(self,+ model_type='t2v',+ patch_size=(1, 2, 2),+ text_len=512,+ in_dim=16,+ dim=2048,+ ffn_dim=8192,+ freq_dim=256,+ text_dim=4096,+ additional_emb_dim=None,+ additional_emb_length=None,+ out_dim=16,+ num_heads=16,+ num_layers=32,+ window_size=(-1, -1),+ qk_norm=True,+ cross_attn_norm=True,+ gradient_checkpointing = False,+ temporal_rope_scaling_factor=1.0,+ eps=1e-6):+ r"""+ Initialize the diffusion model backbone.++ Args:+ model_type (`str`, *optional*, defaults to 't2v'):+ Model variant - 't2v' (text-to-video) or 'i2v' (image-to-video)+ patch_size (`tuple`, *optional*, defaults to (1, 2, 2)):+ 3D patch dimensions for video embedding (t_patch, h_patch, w_patch)+ text_len (`int`, *optional*, defaults to 512):+ Fixed length for text embeddings+ in_dim (`int`, *optional*, defaults to 16):+ Input video channels (C_in)+ dim (`int`, *optional*, defaults to 2048):+ Hidden dimension of the transformer+ ffn_dim (`int`, *optional*, defaults to 8192):+ Intermediate dimension in feed-forward network+ freq_dim (`int`, *optional*, defaults to 256):+ Dimension for sinusoidal time embeddings+ text_dim (`int`, *optional*, defaults to 4096):+ Input dimension for text embeddings+ out_dim (`int`, *optional*, defaults to 16):+ Output video channels (C_out)+ num_heads (`int`, *optional*, defaults to 16):+ Number of attention heads+ num_layers (`int`, *optional*, defaults to 32):+ Number of transformer blocks+ window_size (`tuple`, *optional*, defaults to (-1, -1)):+ Window size for local attention (-1 indicates global attention)+ qk_norm (`bool`, *optional*, defaults to True):+ Enable query/key normalization+ cross_attn_norm (`bool`, *optional*, defaults to False):+ Enable cross-attention normalization+ eps (`float`, *optional*, defaults to 1e-6):+ Epsilon value for normalization layers+ """++ super().__init__()++ assert model_type in ['t2v', 'i2v', 't2a', 'tt2a', 'ti2v'] ## tt2a means text transcript + text description to audio (to support both TTS and T2A+ self.model_type = model_type+ is_audio_type = "a" in self.model_type+ is_video_type = "v" in self.model_type+ assert is_audio_type ^ is_video_type, "Either audio or video model should be specified"+ if is_audio_type:+ ## audio model+ assert len(patch_size) == 1 and patch_size[0] == 1, "Audio model should only accept 1 dimensional input, and we dont do patchify"++ self.patch_size = patch_size+ self.text_len = text_len+ self.in_dim = in_dim+ self.dim = dim+ self.ffn_dim = ffn_dim+ self.freq_dim = freq_dim+ self.text_dim = text_dim+ self.out_dim = out_dim+ self.num_heads = num_heads+ self.num_layers = num_layers+ self.window_size = window_size+ self.qk_norm = qk_norm+ self.cross_attn_norm = cross_attn_norm+ self.eps = eps+ self.temporal_rope_scaling_factor = temporal_rope_scaling_factor+ self.is_audio_type = is_audio_type+ self.is_video_type = is_video_type+ # embeddings+ if is_audio_type:+ ## hardcoded to MMAudio+ self.patch_embedding = nn.Sequential(+ ChannelLastConv1d(in_dim, dim, kernel_size=7, padding=3),+ nn.SiLU(),+ ConvMLP(dim, dim * 4, kernel_size=7, padding=3),+ )+ else:+ self.patch_embedding = nn.Conv3d(+ in_dim, dim, kernel_size=patch_size, stride=patch_size)++ self.text_embedding = nn.Sequential(+ nn.Linear(text_dim, dim), nn.GELU(approximate='tanh'),+ nn.Linear(dim, dim))++ self.time_embedding = nn.Sequential(+ nn.Linear(freq_dim, dim), nn.SiLU(), nn.Linear(dim, dim))+ self.time_projection = nn.Sequential(nn.SiLU(), nn.Linear(dim, dim * 6))+ self.use_sp = False+ # blocks+ ## so i2v and tt2a share the same cross attention while t2v and t2a share the same cross attention+ cross_attn_type = 't2v_cross_attn' if model_type in ['t2v', 't2a', 'ti2v'] else 'i2v_cross_attn'++ if cross_attn_type == 't2v_cross_attn':+ assert additional_emb_dim is None and additional_emb_length is None, "additional_emb_length should be None for t2v and t2a model"+ else:+ assert additional_emb_dim is not None and additional_emb_length is not None, "additional_emb_length should be specified for i2v and tt2a model"++ self.blocks = nn.ModuleList([+ WanAttentionBlock(cross_attn_type, dim, ffn_dim, num_heads,+ window_size, qk_norm, cross_attn_norm, eps, additional_emb_length)+ for _ in range(num_layers)+ ])++ # head+ self.head = Head(dim, out_dim, patch_size, eps)++ # self.set_rope_params()++ if model_type in ['i2v', 'tt2a']:+ self.img_emb = MLPProj(additional_emb_dim, dim)++ # initialize weights+ self.init_weights()+++ def get_audio_rope_params(self):+ dim = self.dim+ num_heads = self.num_heads+ assert (dim % num_heads) == 0 and (dim // num_heads) % 2 == 0+ d = dim // num_heads+ return rope_params(1024, d - 4 * (d // 6), freqs_scaling=self.temporal_rope_scaling_factor)++ def set_rope_params(self):+ # buffers (don't use register_buffer otherwise dtype will be changed in to())+ dim = self.dim+ num_heads = self.num_heads+ assert (dim % num_heads) == 0 and (dim // num_heads) % 2 == 0+ d = dim // num_heads++ if self.is_audio_type:+ ## to be determined+ # self.freqs = rope_params(1024, d, freqs_scaling=temporal_rope_scaling_factor)+ self.freqs = rope_params(1024, d - 4 * (d // 6), freqs_scaling=self.temporal_rope_scaling_factor)+ else:+ self.freqs = torch.cat([+ rope_params(1024, d - 4 * (d // 6)),+ rope_params(1024, 2 * (d // 6)),+ rope_params(1024, 2 * (d // 6))+ ],+ dim=1)+++ def prepare_transformer_block_kwargs(+ self,+ x,+ t,+ context,+ seq_len,+ clip_fea=None,+ y=None,+ first_frame_is_clean=False,+ freqs=None,+ ):++ # params+ ## need to change!+ device = "cuda" # next(self.patch_embedding.parameters()).device+ if isinstance(freqs, tuple):+ freqs = tuple(item.to(device) for item in freqs)+ else:+ freqs = freqs.to(device)++ if y is not None:+ x = [torch.cat([u, v], dim=0) for u, v in zip(x, y)]++ # embeddings+ x = [self.patch_embedding(u.unsqueeze(0)) for u in x] ## x is list of [B L D] or [B C F H W]+ if self.is_audio_type:+ # [B, 1]+ grid_sizes = torch.stack(+ [torch.tensor(u.shape[1:2], dtype=torch.long) for u in x]+ )+ else:+ # [B, 3]+ grid_sizes = torch.stack(+ [torch.tensor(u.shape[2:], dtype=torch.long) for u in x])+ x = [u.flatten(2).transpose(1, 2) for u in x] # [B C F H W] -> [B (F H W) C] -> [B L C]++ seq_lens = torch.tensor([u.size(1) for u in x], dtype=torch.long)+ assert seq_lens.max() <= seq_len, f"Sequence length {seq_lens.max()} exceeds maximum {seq_len}."+ x = torch.cat([+ torch.cat([u, u.new_zeros(1, seq_len - u.size(1), u.size(2))],+ dim=1) for u in x+ ]) # single [B, L, C]++ modulation_dtype = torch.bfloat16+ # time embeddings+ if t.dim() == 1:+ if first_frame_is_clean:+ latent_frames = int(grid_sizes[0][0])+ t = torch.full((latent_frames,), int(t), dtype=torch.int64, device=t.device)+ t[:1] = 0+ e = self.time_embedding(+ sinusoidal_embedding_1d(self.freq_dim, t.flatten()).to(modulation_dtype) # self.patch_embedding.weight.dtype)+ ) # b, dim+ e0 = self.time_projection(e).unflatten(1, (6, self.dim)).to(e.dtype)++++ # context+ context_lens = None+ context = self.text_embedding(+ torch.stack([+ torch.cat(+ [u, u.new_zeros(self.text_len - u.size(0), u.size(1))])+ for u in context+ ]))++ if clip_fea is not None:+ context_clip = self.img_emb(clip_fea) # bs x 257 x dim+ context = torch.concat([context_clip, context], dim=1)++ # arguments+ kwargs = dict(+ e=e0,+ seq_lens=seq_lens,+ grid_sizes=grid_sizes,+ freqs=freqs, #self.freqs,+ context=context,+ context_lens=context_lens)++ return x, e, kwargs++ def post_transformer_block_out(self, x, grid_sizes, e):+ # head+ x = self.head(x, e)+ # unpatchify+ if self.is_audio_type:+ ## grid_sizes is [B 1] where 1 is L,+ # converting grid_sizes from [B 1] -> [B]+ grid_sizes = [gs[0] for gs in grid_sizes]+ assert len(x) == len(grid_sizes)+ x = [u[:gs] for u, gs in zip(x, grid_sizes)]+ else:+ ## grid_sizes is [B 3] where 3 is F H w+ x = self.unpatchify(x, grid_sizes)+ return x[0]+ # return [u.bfloat16() for u in x]+++ def forward(+ self,+ x,+ t,+ context,+ seq_len,+ clip_fea=None,+ y=None,+ first_frame_is_clean=False,+ ):++ raise Exception("this code is never executed, you should not be here")++ def unpatchify(self, x, grid_sizes):+ r"""+ Reconstruct video tensors from patch embeddings.++ Args:+ x (List[Tensor]):+ List of patchified features, each with shape [L, C_out * prod(patch_size)]+ grid_sizes (Tensor):+ Original spatial-temporal grid dimensions before patching,+ shape [B, 3] (3 dimensions correspond to F_patches, H_patches, W_patches)++ Returns:+ List[Tensor]:+ Reconstructed video tensors with shape [C_out, F, H / 8, W / 8]+ """++ c = self.out_dim+ out = []+ for u, v in zip(x, grid_sizes.tolist()):+ # v is [F H w] F * H * 80, 100, it was right padded by 20.+ u = u[:math.prod(v)].view(*v, *self.patch_size, c)+ u = torch.einsum('fhwpqrc->cfphqwr', u)+ u = u.reshape(c, *[i * j for i, j in zip(v, self.patch_size)])+ out.append(u)+ # out is list of [C F H W]+ return out++ def init_weights(self):+ r"""+ Initialize model parameters using Xavier initialization.+ """++ # basic init+ for m in self.modules():+ if isinstance(m, nn.Linear):+ nn.init.xavier_uniform_(m.weight)+ if m.bias is not None:+ nn.init.zeros_(m.bias)++ # init embeddings+ if self.is_video_type:+ assert isinstance(self.patch_embedding, nn.Conv3d), f"Patch embedding for video should be a Conv3d layer, got {type(self.patch_embedding)}"+ nn.init.xavier_uniform_(self.patch_embedding.weight.flatten(1))+ for m in self.text_embedding.modules():+ if isinstance(m, nn.Linear):+ nn.init.normal_(m.weight, std=.02)+ for m in self.time_embedding.modules():+ if isinstance(m, nn.Linear):+ nn.init.normal_(m.weight, std=.02)++ # init output layer+ nn.init.zeros_(self.head.head.weight)diff --git a/models/wan/ovi/utils/__init__.py b/models/wan/ovi/utils/__init__.pynew file mode 100644index 0000000..6e9a339--- /dev/null+++ b/models/wan/ovi/utils/__init__.py@@ -0,0 +1,8 @@+from .fm_solvers import (FlowDPMSolverMultistepScheduler, get_sampling_sigmas,+ retrieve_timesteps)+from .fm_solvers_unipc import FlowUniPCMultistepScheduler++__all__ = [+ 'HuggingfaceTokenizer', 'get_sampling_sigmas', 'retrieve_timesteps',+ 'FlowDPMSolverMultistepScheduler', 'FlowUniPCMultistepScheduler'+]diff --git a/models/wan/ovi/utils/fm_solvers.py b/models/wan/ovi/utils/fm_solvers.pynew file mode 100644index 0000000..70bdb9b--- /dev/null+++ b/models/wan/ovi/utils/fm_solvers.py@@ -0,0 +1,860 @@+# Copied from https://github.com/huggingface/diffusers/blob/main/src/diffusers/schedulers/scheduling_dpmsolver_multistep.py+# Convert dpm solver for flow matching+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.++import inspect+import math+from typing import List, Optional, Tuple, Union++import numpy as np+import torch+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.schedulers.scheduling_utils import (KarrasDiffusionSchedulers,+ SchedulerMixin,+ SchedulerOutput)+from diffusers.utils import deprecate, is_scipy_available+from diffusers.utils.torch_utils import randn_tensor++if is_scipy_available():+ pass+++def get_sampling_sigmas(sampling_steps, shift):+ sigma = np.linspace(1, 0, sampling_steps + 1)[:sampling_steps]+ sigma = (shift * sigma / (1 + (shift - 1) * sigma))++ return sigma+++def retrieve_timesteps(+ scheduler,+ num_inference_steps=None,+ device=None,+ timesteps=None,+ sigmas=None,+ **kwargs,+):+ if timesteps is not None and sigmas is not None:+ raise ValueError(+ "Only one of `timesteps` or `sigmas` can be passed. Please choose one to set custom values"+ )+ if timesteps is not None:+ accepts_timesteps = "timesteps" in set(+ inspect.signature(scheduler.set_timesteps).parameters.keys())+ if not accepts_timesteps:+ raise ValueError(+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"+ f" timestep schedules. Please check whether you are using the correct scheduler."+ )+ scheduler.set_timesteps(timesteps=timesteps, device=device, **kwargs)+ timesteps = scheduler.timesteps+ num_inference_steps = len(timesteps)+ elif sigmas is not None:+ accept_sigmas = "sigmas" in set(+ inspect.signature(scheduler.set_timesteps).parameters.keys())+ if not accept_sigmas:+ raise ValueError(+ f"The current scheduler class {scheduler.__class__}'s `set_timesteps` does not support custom"+ f" sigmas schedules. Please check whether you are using the correct scheduler."+ )+ scheduler.set_timesteps(sigmas=sigmas, device=device, **kwargs)+ timesteps = scheduler.timesteps+ num_inference_steps = len(timesteps)+ else:+ scheduler.set_timesteps(num_inference_steps, device=device, **kwargs)+ timesteps = scheduler.timesteps+ return timesteps, num_inference_steps+++class FlowDPMSolverMultistepScheduler(SchedulerMixin, ConfigMixin):+ """+ `FlowDPMSolverMultistepScheduler` is a fast dedicated high-order solver for diffusion ODEs.+ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic+ methods the library implements for all schedulers such as loading and saving.+ Args:+ num_train_timesteps (`int`, defaults to 1000):+ The number of diffusion steps to train the model. This determines the resolution of the diffusion process.+ solver_order (`int`, defaults to 2):+ The DPMSolver order which can be `1`, `2`, or `3`. It is recommended to use `solver_order=2` for guided+ sampling, and `solver_order=3` for unconditional sampling. This affects the number of model outputs stored+ and used in multistep updates.+ prediction_type (`str`, defaults to "flow_prediction"):+ Prediction type of the scheduler function; must be `flow_prediction` for this scheduler, which predicts+ the flow of the diffusion process.+ shift (`float`, *optional*, defaults to 1.0):+ A factor used to adjust the sigmas in the noise schedule. It modifies the step sizes during the sampling+ process.+ use_dynamic_shifting (`bool`, defaults to `False`):+ Whether to apply dynamic shifting to the timesteps based on image resolution. If `True`, the shifting is+ applied on the fly.+ thresholding (`bool`, defaults to `False`):+ Whether to use the "dynamic thresholding" method. This method adjusts the predicted sample to prevent+ saturation and improve photorealism.+ dynamic_thresholding_ratio (`float`, defaults to 0.995):+ The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.+ sample_max_value (`float`, defaults to 1.0):+ The threshold value for dynamic thresholding. Valid only when `thresholding=True` and+ `algorithm_type="dpmsolver++"`.+ algorithm_type (`str`, defaults to `dpmsolver++`):+ Algorithm type for the solver; can be `dpmsolver`, `dpmsolver++`, `sde-dpmsolver` or `sde-dpmsolver++`. The+ `dpmsolver` type implements the algorithms in the [DPMSolver](https://huggingface.co/papers/2206.00927)+ paper, and the `dpmsolver++` type implements the algorithms in the+ [DPMSolver++](https://huggingface.co/papers/2211.01095) paper. It is recommended to use `dpmsolver++` or+ `sde-dpmsolver++` with `solver_order=2` for guided sampling like in Stable Diffusion.+ solver_type (`str`, defaults to `midpoint`):+ Solver type for the second-order solver; can be `midpoint` or `heun`. The solver type slightly affects the+ sample quality, especially for a small number of steps. It is recommended to use `midpoint` solvers.+ lower_order_final (`bool`, defaults to `True`):+ Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can+ stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10.+ euler_at_final (`bool`, defaults to `False`):+ Whether to use Euler's method in the final step. It is a trade-off between numerical stability and detail+ richness. This can stabilize the sampling of the SDE variant of DPMSolver for small number of inference+ steps, but sometimes may result in blurring.+ final_sigmas_type (`str`, *optional*, defaults to "zero"):+ The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final+ sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0.+ lambda_min_clipped (`float`, defaults to `-inf`):+ Clipping threshold for the minimum value of `lambda(t)` for numerical stability. This is critical for the+ cosine (`squaredcos_cap_v2`) noise schedule.+ variance_type (`str`, *optional*):+ Set to "learned" or "learned_range" for diffusion models that predict variance. If set, the model's output+ contains the predicted Gaussian variance.+ """++ _compatibles = [e.name for e in KarrasDiffusionSchedulers]+ order = 1++ @register_to_config+ def __init__(+ self,+ num_train_timesteps: int = 1000,+ solver_order: int = 2,+ prediction_type: str = "flow_prediction",+ shift: Optional[float] = 1.0,+ use_dynamic_shifting=False,+ thresholding: bool = False,+ dynamic_thresholding_ratio: float = 0.995,+ sample_max_value: float = 1.0,+ algorithm_type: str = "dpmsolver++",+ solver_type: str = "midpoint",+ lower_order_final: bool = True,+ euler_at_final: bool = False,+ final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min"+ lambda_min_clipped: float = -float("inf"),+ variance_type: Optional[str] = None,+ invert_sigmas: bool = False,+ ):+ if algorithm_type in ["dpmsolver", "sde-dpmsolver"]:+ deprecation_message = f"algorithm_type {algorithm_type} is deprecated and will be removed in a future version. Choose from `dpmsolver++` or `sde-dpmsolver++` instead"+ deprecate("algorithm_types dpmsolver and sde-dpmsolver", "1.0.0",+ deprecation_message)++ # settings for DPM-Solver+ if algorithm_type not in [+ "dpmsolver", "dpmsolver++", "sde-dpmsolver", "sde-dpmsolver++"+ ]:+ if algorithm_type == "deis":+ self.register_to_config(algorithm_type="dpmsolver++")+ else:+ raise NotImplementedError(+ f"{algorithm_type} is not implemented for {self.__class__}")++ if solver_type not in ["midpoint", "heun"]:+ if solver_type in ["logrho", "bh1", "bh2"]:+ self.register_to_config(solver_type="midpoint")+ else:+ raise NotImplementedError(+ f"{solver_type} is not implemented for {self.__class__}")++ if algorithm_type not in ["dpmsolver++", "sde-dpmsolver++"+ ] and final_sigmas_type == "zero":+ raise ValueError(+ f"`final_sigmas_type` {final_sigmas_type} is not supported for `algorithm_type` {algorithm_type}. Please choose `sigma_min` instead."+ )++ # setable values+ self.num_inference_steps = None+ alphas = np.linspace(1, 1 / num_train_timesteps,+ num_train_timesteps)[::-1].copy()+ sigmas = 1.0 - alphas+ sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32)++ if not use_dynamic_shifting:+ # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution+ sigmas = shift * sigmas / (1 ++ (shift - 1) * sigmas) # pyright: ignore++ self.sigmas = sigmas+ self.timesteps = sigmas * num_train_timesteps++ self.model_outputs = [None] * solver_order+ self.lower_order_nums = 0+ self._step_index = None+ self._begin_index = None++ # self.sigmas = self.sigmas.to(+ # "cpu") # to avoid too much CPU/GPU communication+ self.sigma_min = self.sigmas[-1].item()+ self.sigma_max = self.sigmas[0].item()++ @property+ def step_index(self):+ """+ The index counter for current timestep. It will increase 1 after each scheduler step.+ """+ return self._step_index++ @property+ def begin_index(self):+ """+ The index for the first timestep. It should be set from pipeline with `set_begin_index` method.+ """+ return self._begin_index++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index+ def set_begin_index(self, begin_index: int = 0):+ """+ Sets the begin index for the scheduler. This function should be run from pipeline before the inference.+ Args:+ begin_index (`int`):+ The begin index for the scheduler.+ """+ self._begin_index = begin_index++ # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps+ def set_timesteps(+ self,+ num_inference_steps: Union[int, None] = None,+ device: Union[str, torch.device] = None,+ sigmas: Optional[List[float]] = None,+ mu: Optional[Union[float, None]] = None,+ shift: Optional[Union[float, None]] = None,+ ):+ """+ Sets the discrete timesteps used for the diffusion chain (to be run before inference).+ Args:+ num_inference_steps (`int`):+ Total number of the spacing of the time steps.+ device (`str` or `torch.device`, *optional*):+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.+ """++ if self.config.use_dynamic_shifting and mu is None:+ raise ValueError(+ " you have to pass a value for `mu` when `use_dynamic_shifting` is set to be `True`"+ )++ if sigmas is None:+ sigmas = np.linspace(self.sigma_max, self.sigma_min,+ num_inference_steps ++ 1).copy()[:-1] # pyright: ignore++ if self.config.use_dynamic_shifting:+ sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore+ else:+ if shift is None:+ shift = self.config.shift+ sigmas = shift * sigmas / (1 ++ (shift - 1) * sigmas) # pyright: ignore++ if self.config.final_sigmas_type == "sigma_min":+ sigma_last = ((1 - self.alphas_cumprod[0]) /+ self.alphas_cumprod[0])**0.5+ elif self.config.final_sigmas_type == "zero":+ sigma_last = 0+ else:+ raise ValueError(+ f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}"+ )++ timesteps = sigmas * self.config.num_train_timesteps+ sigmas = np.concatenate([sigmas, [sigma_last]+ ]).astype(np.float32) # pyright: ignore++ self.sigmas = torch.from_numpy(sigmas)+ self.timesteps = torch.from_numpy(timesteps).to(+ device=device, dtype=torch.int64)++ self.num_inference_steps = len(timesteps)++ self.model_outputs = [+ None,+ ] * self.config.solver_order+ self.lower_order_nums = 0++ self._step_index = None+ self._begin_index = None+ # self.sigmas = self.sigmas.to(+ # "cpu") # to avoid too much CPU/GPU communication++ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample+ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:+ """+ "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the+ prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by+ s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing+ pixels from saturation at each step. We find that dynamic thresholding results in significantly better+ photorealism as well as better image-text alignment, especially when using very large guidance weights."+ https://arxiv.org/abs/2205.11487+ """+ dtype = sample.dtype+ batch_size, channels, *remaining_dims = sample.shape++ if dtype not in (torch.float32, torch.float64):+ sample = sample.float(+ ) # upcast for quantile calculation, and clamp not implemented for cpu half++ # Flatten sample for doing quantile calculation along each image+ sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))++ abs_sample = sample.abs() # "a certain percentile absolute pixel value"++ s = torch.quantile(+ abs_sample, self.config.dynamic_thresholding_ratio, dim=1)+ s = torch.clamp(+ s, min=1, max=self.config.sample_max_value+ ) # When clamped to min=1, equivalent to standard clipping to [-1, 1]+ s = s.unsqueeze(+ 1) # (batch_size, 1) because clamp will broadcast along dim=0+ sample = torch.clamp(+ sample, -s, s+ ) / s # "we threshold xt0 to the range [-s, s] and then divide by s"++ sample = sample.reshape(batch_size, channels, *remaining_dims)+ sample = sample.to(dtype)++ return sample++ # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler._sigma_to_t+ def _sigma_to_t(self, sigma):+ return sigma * self.config.num_train_timesteps++ def _sigma_to_alpha_sigma_t(self, sigma):+ return 1 - sigma, sigma++ # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.set_timesteps+ def time_shift(self, mu: float, sigma: float, t: torch.Tensor):+ return math.exp(mu) / (math.exp(mu) + (1 / t - 1)**sigma)++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.convert_model_output+ def convert_model_output(+ self,+ model_output: torch.Tensor,+ *args,+ sample: torch.Tensor = None,+ **kwargs,+ ) -> torch.Tensor:+ """+ Convert the model output to the corresponding type the DPMSolver/DPMSolver++ algorithm needs. DPM-Solver is+ designed to discretize an integral of the noise prediction model, and DPM-Solver++ is designed to discretize an+ integral of the data prediction model.+ <Tip>+ The algorithm and model type are decoupled. You can use either DPMSolver or DPMSolver++ for both noise+ prediction and data prediction models.+ </Tip>+ Args:+ model_output (`torch.Tensor`):+ The direct output from the learned diffusion model.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.+ Returns:+ `torch.Tensor`:+ The converted model output.+ """+ timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None)+ if sample is None:+ if len(args) > 1:+ sample = args[1]+ else:+ raise ValueError(+ "missing `sample` as a required keyward argument")+ if timestep is not None:+ deprecate(+ "timesteps",+ "1.0.0",+ "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ # DPM-Solver++ needs to solve an integral of the data prediction model.+ if self.config.algorithm_type in ["dpmsolver++", "sde-dpmsolver++"]:+ if self.config.prediction_type == "flow_prediction":+ sigma_t = self.sigmas[self.step_index]+ x0_pred = sample - sigma_t * model_output+ else:+ raise ValueError(+ f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`,"+ " `v_prediction`, or `flow_prediction` for the FlowDPMSolverMultistepScheduler."+ )++ if self.config.thresholding:+ x0_pred = self._threshold_sample(x0_pred)++ return x0_pred++ # DPM-Solver needs to solve an integral of the noise prediction model.+ elif self.config.algorithm_type in ["dpmsolver", "sde-dpmsolver"]:+ if self.config.prediction_type == "flow_prediction":+ sigma_t = self.sigmas[self.step_index]+ epsilon = sample - (1 - sigma_t) * model_output+ else:+ raise ValueError(+ f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`,"+ " `v_prediction` or `flow_prediction` for the FlowDPMSolverMultistepScheduler."+ )++ if self.config.thresholding:+ sigma_t = self.sigmas[self.step_index]+ x0_pred = sample - sigma_t * model_output+ x0_pred = self._threshold_sample(x0_pred)+ epsilon = model_output + x0_pred++ return epsilon++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.dpm_solver_first_order_update+ def dpm_solver_first_order_update(+ self,+ model_output: torch.Tensor,+ *args,+ sample: torch.Tensor = None,+ noise: Optional[torch.Tensor] = None,+ **kwargs,+ ) -> torch.Tensor:+ """+ One step for the first-order DPMSolver (equivalent to DDIM).+ Args:+ model_output (`torch.Tensor`):+ The direct output from the learned diffusion model.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.+ Returns:+ `torch.Tensor`:+ The sample tensor at the previous timestep.+ """+ timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None)+ prev_timestep = args[1] if len(args) > 1 else kwargs.pop(+ "prev_timestep", None)+ if sample is None:+ if len(args) > 2:+ sample = args[2]+ else:+ raise ValueError(+ " missing `sample` as a required keyward argument")+ if timestep is not None:+ deprecate(+ "timesteps",+ "1.0.0",+ "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ if prev_timestep is not None:+ deprecate(+ "prev_timestep",+ "1.0.0",+ "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ sigma_t, sigma_s = self.sigmas[self.step_index + 1], self.sigmas[+ self.step_index] # pyright: ignore+ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t)+ alpha_s, sigma_s = self._sigma_to_alpha_sigma_t(sigma_s)+ lambda_t = torch.log(alpha_t) - torch.log(sigma_t)+ lambda_s = torch.log(alpha_s) - torch.log(sigma_s)++ h = lambda_t - lambda_s+ if self.config.algorithm_type == "dpmsolver++":+ x_t = (sigma_t /+ sigma_s) * sample - (alpha_t *+ (torch.exp(-h) - 1.0)) * model_output+ elif self.config.algorithm_type == "dpmsolver":+ x_t = (alpha_t /+ alpha_s) * sample - (sigma_t *+ (torch.exp(h) - 1.0)) * model_output+ elif self.config.algorithm_type == "sde-dpmsolver++":+ assert noise is not None+ x_t = ((sigma_t / sigma_s * torch.exp(-h)) * sample ++ (alpha_t * (1 - torch.exp(-2.0 * h))) * model_output ++ sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise)+ elif self.config.algorithm_type == "sde-dpmsolver":+ assert noise is not None+ x_t = ((alpha_t / alpha_s) * sample - 2.0 *+ (sigma_t * (torch.exp(h) - 1.0)) * model_output ++ sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise)+ return x_t # pyright: ignore++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_second_order_update+ def multistep_dpm_solver_second_order_update(+ self,+ model_output_list: List[torch.Tensor],+ *args,+ sample: torch.Tensor = None,+ noise: Optional[torch.Tensor] = None,+ **kwargs,+ ) -> torch.Tensor:+ """+ One step for the second-order multistep DPMSolver.+ Args:+ model_output_list (`List[torch.Tensor]`):+ The direct outputs from learned diffusion model at current and latter timesteps.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.+ Returns:+ `torch.Tensor`:+ The sample tensor at the previous timestep.+ """+ timestep_list = args[0] if len(args) > 0 else kwargs.pop(+ "timestep_list", None)+ prev_timestep = args[1] if len(args) > 1 else kwargs.pop(+ "prev_timestep", None)+ if sample is None:+ if len(args) > 2:+ sample = args[2]+ else:+ raise ValueError(+ " missing `sample` as a required keyward argument")+ if timestep_list is not None:+ deprecate(+ "timestep_list",+ "1.0.0",+ "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ if prev_timestep is not None:+ deprecate(+ "prev_timestep",+ "1.0.0",+ "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ sigma_t, sigma_s0, sigma_s1 = (+ self.sigmas[self.step_index + 1], # pyright: ignore+ self.sigmas[self.step_index],+ self.sigmas[self.step_index - 1], # pyright: ignore+ )++ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t)+ alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0)+ alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1)++ lambda_t = torch.log(alpha_t) - torch.log(sigma_t)+ lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0)+ lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1)++ m0, m1 = model_output_list[-1], model_output_list[-2]++ h, h_0 = lambda_t - lambda_s0, lambda_s0 - lambda_s1+ r0 = h_0 / h+ D0, D1 = m0, (1.0 / r0) * (m0 - m1)+ if self.config.algorithm_type == "dpmsolver++":+ # See https://arxiv.org/abs/2211.01095 for detailed derivations+ if self.config.solver_type == "midpoint":+ x_t = ((sigma_t / sigma_s0) * sample -+ (alpha_t * (torch.exp(-h) - 1.0)) * D0 - 0.5 *+ (alpha_t * (torch.exp(-h) - 1.0)) * D1)+ elif self.config.solver_type == "heun":+ x_t = ((sigma_t / sigma_s0) * sample -+ (alpha_t * (torch.exp(-h) - 1.0)) * D0 ++ (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1)+ elif self.config.algorithm_type == "dpmsolver":+ # See https://arxiv.org/abs/2206.00927 for detailed derivations+ if self.config.solver_type == "midpoint":+ x_t = ((alpha_t / alpha_s0) * sample -+ (sigma_t * (torch.exp(h) - 1.0)) * D0 - 0.5 *+ (sigma_t * (torch.exp(h) - 1.0)) * D1)+ elif self.config.solver_type == "heun":+ x_t = ((alpha_t / alpha_s0) * sample -+ (sigma_t * (torch.exp(h) - 1.0)) * D0 -+ (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1)+ elif self.config.algorithm_type == "sde-dpmsolver++":+ assert noise is not None+ if self.config.solver_type == "midpoint":+ x_t = ((sigma_t / sigma_s0 * torch.exp(-h)) * sample ++ (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 + 0.5 *+ (alpha_t * (1 - torch.exp(-2.0 * h))) * D1 ++ sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise)+ elif self.config.solver_type == "heun":+ x_t = ((sigma_t / sigma_s0 * torch.exp(-h)) * sample ++ (alpha_t * (1 - torch.exp(-2.0 * h))) * D0 ++ (alpha_t * ((1.0 - torch.exp(-2.0 * h)) /+ (-2.0 * h) + 1.0)) * D1 ++ sigma_t * torch.sqrt(1.0 - torch.exp(-2 * h)) * noise)+ elif self.config.algorithm_type == "sde-dpmsolver":+ assert noise is not None+ if self.config.solver_type == "midpoint":+ x_t = ((alpha_t / alpha_s0) * sample - 2.0 *+ (sigma_t * (torch.exp(h) - 1.0)) * D0 -+ (sigma_t * (torch.exp(h) - 1.0)) * D1 ++ sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise)+ elif self.config.solver_type == "heun":+ x_t = ((alpha_t / alpha_s0) * sample - 2.0 *+ (sigma_t * (torch.exp(h) - 1.0)) * D0 - 2.0 *+ (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 ++ sigma_t * torch.sqrt(torch.exp(2 * h) - 1.0) * noise)+ return x_t # pyright: ignore++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.multistep_dpm_solver_third_order_update+ def multistep_dpm_solver_third_order_update(+ self,+ model_output_list: List[torch.Tensor],+ *args,+ sample: torch.Tensor = None,+ **kwargs,+ ) -> torch.Tensor:+ """+ One step for the third-order multistep DPMSolver.+ Args:+ model_output_list (`List[torch.Tensor]`):+ The direct outputs from learned diffusion model at current and latter timesteps.+ sample (`torch.Tensor`):+ A current instance of a sample created by diffusion process.+ Returns:+ `torch.Tensor`:+ The sample tensor at the previous timestep.+ """++ timestep_list = args[0] if len(args) > 0 else kwargs.pop(+ "timestep_list", None)+ prev_timestep = args[1] if len(args) > 1 else kwargs.pop(+ "prev_timestep", None)+ if sample is None:+ if len(args) > 2:+ sample = args[2]+ else:+ raise ValueError(+ " missing`sample` as a required keyward argument")+ if timestep_list is not None:+ deprecate(+ "timestep_list",+ "1.0.0",+ "Passing `timestep_list` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ if prev_timestep is not None:+ deprecate(+ "prev_timestep",+ "1.0.0",+ "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ sigma_t, sigma_s0, sigma_s1, sigma_s2 = (+ self.sigmas[self.step_index + 1], # pyright: ignore+ self.sigmas[self.step_index],+ self.sigmas[self.step_index - 1], # pyright: ignore+ self.sigmas[self.step_index - 2], # pyright: ignore+ )++ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t)+ alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0)+ alpha_s1, sigma_s1 = self._sigma_to_alpha_sigma_t(sigma_s1)+ alpha_s2, sigma_s2 = self._sigma_to_alpha_sigma_t(sigma_s2)++ lambda_t = torch.log(alpha_t) - torch.log(sigma_t)+ lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0)+ lambda_s1 = torch.log(alpha_s1) - torch.log(sigma_s1)+ lambda_s2 = torch.log(alpha_s2) - torch.log(sigma_s2)++ m0, m1, m2 = model_output_list[-1], model_output_list[+ -2], model_output_list[-3]++ h, h_0, h_1 = lambda_t - lambda_s0, lambda_s0 - lambda_s1, lambda_s1 - lambda_s2+ r0, r1 = h_0 / h, h_1 / h+ D0 = m0+ D1_0, D1_1 = (1.0 / r0) * (m0 - m1), (1.0 / r1) * (m1 - m2)+ D1 = D1_0 + (r0 / (r0 + r1)) * (D1_0 - D1_1)+ D2 = (1.0 / (r0 + r1)) * (D1_0 - D1_1)+ if self.config.algorithm_type == "dpmsolver++":+ # See https://arxiv.org/abs/2206.00927 for detailed derivations+ x_t = ((sigma_t / sigma_s0) * sample -+ (alpha_t * (torch.exp(-h) - 1.0)) * D0 ++ (alpha_t * ((torch.exp(-h) - 1.0) / h + 1.0)) * D1 -+ (alpha_t * ((torch.exp(-h) - 1.0 + h) / h**2 - 0.5)) * D2)+ elif self.config.algorithm_type == "dpmsolver":+ # See https://arxiv.org/abs/2206.00927 for detailed derivations+ x_t = ((alpha_t / alpha_s0) * sample - (sigma_t *+ (torch.exp(h) - 1.0)) * D0 -+ (sigma_t * ((torch.exp(h) - 1.0) / h - 1.0)) * D1 -+ (sigma_t * ((torch.exp(h) - 1.0 - h) / h**2 - 0.5)) * D2)+ return x_t # pyright: ignore++ def index_for_timestep(self, timestep, schedule_timesteps=None):+ if schedule_timesteps is None:+ schedule_timesteps = self.timesteps++ indices = (schedule_timesteps == timestep).nonzero()++ # The sigma index that is taken for the **very** first `step`+ # is always the second index (or the last index if there is only 1)+ # This way we can ensure we don't accidentally skip a sigma in+ # case we start in the middle of the denoising schedule (e.g. for image-to-image)+ pos = 1 if len(indices) > 1 else 0++ return indices[pos].item()++ def _init_step_index(self, timestep):+ """+ Initialize the step_index counter for the scheduler.+ """++ if self.begin_index is None:+ if isinstance(timestep, torch.Tensor):+ timestep = timestep.to(self.timesteps.device)+ self._step_index = self.index_for_timestep(timestep)+ else:+ self._step_index = self._begin_index++ # Modified from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.step+ def step(+ self,+ model_output: torch.Tensor,+ timestep: Union[int, torch.Tensor],+ sample: torch.Tensor,+ generator=None,+ variance_noise: Optional[torch.Tensor] = None,+ return_dict: bool = True,+ ) -> Union[SchedulerOutput, Tuple]:+ """+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with+ the multistep DPMSolver.+ Args:+ model_output (`torch.Tensor`):+ The direct output from learned diffusion model.+ timestep (`int`):+ The current discrete timestep in the diffusion chain.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.+ generator (`torch.Generator`, *optional*):+ A random number generator.+ variance_noise (`torch.Tensor`):+ Alternative to generating noise with `generator` by directly providing the noise for the variance+ itself. Useful for methods such as [`LEdits++`].+ return_dict (`bool`):+ Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`.+ Returns:+ [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`:+ If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a+ tuple is returned where the first element is the sample tensor.+ """+ if self.num_inference_steps is None:+ raise ValueError(+ "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"+ )++ if self.step_index is None:+ self._init_step_index(timestep)++ # Improve numerical stability for small number of steps+ lower_order_final = (self.step_index == len(self.timesteps) - 1) and (+ self.config.euler_at_final or+ (self.config.lower_order_final and len(self.timesteps) < 15) or+ self.config.final_sigmas_type == "zero")+ lower_order_second = ((self.step_index == len(self.timesteps) - 2) and+ self.config.lower_order_final and+ len(self.timesteps) < 15)++ model_output = self.convert_model_output(model_output, sample=sample)+ for i in range(self.config.solver_order - 1):+ self.model_outputs[i] = self.model_outputs[i + 1]+ self.model_outputs[-1] = model_output++ # Upcast to avoid precision issues when computing prev_sample+ sample = sample.to(torch.float32)+ if self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"+ ] and variance_noise is None:+ noise = randn_tensor(+ model_output.shape,+ generator=generator,+ device=model_output.device,+ dtype=torch.float32)+ elif self.config.algorithm_type in ["sde-dpmsolver", "sde-dpmsolver++"]:+ noise = variance_noise.to(+ device=model_output.device,+ dtype=torch.float32) # pyright: ignore+ else:+ noise = None++ if self.config.solver_order == 1 or self.lower_order_nums < 1 or lower_order_final:+ prev_sample = self.dpm_solver_first_order_update(+ model_output, sample=sample, noise=noise)+ elif self.config.solver_order == 2 or self.lower_order_nums < 2 or lower_order_second:+ prev_sample = self.multistep_dpm_solver_second_order_update(+ self.model_outputs, sample=sample, noise=noise)+ else:+ prev_sample = self.multistep_dpm_solver_third_order_update(+ self.model_outputs, sample=sample)++ if self.lower_order_nums < self.config.solver_order:+ self.lower_order_nums += 1++ # Cast sample back to expected dtype+ prev_sample = prev_sample.to(model_output.dtype)++ # upon completion increase step index by one+ self._step_index += 1 # pyright: ignore++ if not return_dict:+ return (prev_sample,)++ return SchedulerOutput(prev_sample=prev_sample)++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.scale_model_input+ def scale_model_input(self, sample: torch.Tensor, *args,+ **kwargs) -> torch.Tensor:+ """+ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the+ current timestep.+ Args:+ sample (`torch.Tensor`):+ The input sample.+ Returns:+ `torch.Tensor`:+ A scaled input sample.+ """+ return sample++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.scale_model_input+ def add_noise(+ self,+ original_samples: torch.Tensor,+ noise: torch.Tensor,+ timesteps: torch.IntTensor,+ ) -> torch.Tensor:+ # Make sure sigmas and timesteps have the same device and dtype as original_samples+ sigmas = self.sigmas.to(+ device=original_samples.device, dtype=original_samples.dtype)+ if original_samples.device.type == "mps" and torch.is_floating_point(+ timesteps):+ # mps does not support float64+ schedule_timesteps = self.timesteps.to(+ original_samples.device, dtype=torch.float32)+ timesteps = timesteps.to(+ original_samples.device, dtype=torch.float32)+ else:+ schedule_timesteps = self.timesteps.to(original_samples.device)+ timesteps = timesteps.to(original_samples.device)++ # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index+ if self.begin_index is None:+ step_indices = [+ self.index_for_timestep(t, schedule_timesteps)+ for t in timesteps+ ]+ elif self.step_index is not None:+ # add_noise is called after first denoising step (for inpainting)+ step_indices = [self.step_index] * timesteps.shape[0]+ else:+ # add noise is called before first denoising step to create initial latent(img2img)+ step_indices = [self.begin_index] * timesteps.shape[0]++ sigma = sigmas[step_indices].flatten()+ while len(sigma.shape) < len(original_samples.shape):+ sigma = sigma.unsqueeze(-1)++ import pdb+ pdb.set_trace()++ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)+ noisy_samples = alpha_t * original_samples + sigma_t * noise+ return noisy_samples++ def __len__(self):+ return self.config.num_train_timestepsdiff --git a/models/wan/ovi/utils/fm_solvers_unipc.py b/models/wan/ovi/utils/fm_solvers_unipc.pynew file mode 100644index 0000000..57321ba--- /dev/null+++ b/models/wan/ovi/utils/fm_solvers_unipc.py@@ -0,0 +1,800 @@+# Copied from https://github.com/huggingface/diffusers/blob/v0.31.0/src/diffusers/schedulers/scheduling_unipc_multistep.py+# Convert unipc for flow matching+# Copyright 2024-2025 The Alibaba Wan Team Authors. All rights reserved.++import math+from typing import List, Optional, Tuple, Union++import numpy as np+import torch+from diffusers.configuration_utils import ConfigMixin, register_to_config+from diffusers.schedulers.scheduling_utils import (KarrasDiffusionSchedulers,+ SchedulerMixin,+ SchedulerOutput)+from diffusers.utils import deprecate, is_scipy_available++if is_scipy_available():+ import scipy.stats+++class FlowUniPCMultistepScheduler(SchedulerMixin, ConfigMixin):+ """+ `UniPCMultistepScheduler` is a training-free framework designed for the fast sampling of diffusion models.++ This model inherits from [`SchedulerMixin`] and [`ConfigMixin`]. Check the superclass documentation for the generic+ methods the library implements for all schedulers such as loading and saving.++ Args:+ num_train_timesteps (`int`, defaults to 1000):+ The number of diffusion steps to train the model.+ solver_order (`int`, default `2`):+ The UniPC order which can be any positive integer. The effective order of accuracy is `solver_order + 1`+ due to the UniC. It is recommended to use `solver_order=2` for guided sampling, and `solver_order=3` for+ unconditional sampling.+ prediction_type (`str`, defaults to "flow_prediction"):+ Prediction type of the scheduler function; must be `flow_prediction` for this scheduler, which predicts+ the flow of the diffusion process.+ thresholding (`bool`, defaults to `False`):+ Whether to use the "dynamic thresholding" method. This is unsuitable for latent-space diffusion models such+ as Stable Diffusion.+ dynamic_thresholding_ratio (`float`, defaults to 0.995):+ The ratio for the dynamic thresholding method. Valid only when `thresholding=True`.+ sample_max_value (`float`, defaults to 1.0):+ The threshold value for dynamic thresholding. Valid only when `thresholding=True` and `predict_x0=True`.+ predict_x0 (`bool`, defaults to `True`):+ Whether to use the updating algorithm on the predicted x0.+ solver_type (`str`, default `bh2`):+ Solver type for UniPC. It is recommended to use `bh1` for unconditional sampling when steps < 10, and `bh2`+ otherwise.+ lower_order_final (`bool`, default `True`):+ Whether to use lower-order solvers in the final steps. Only valid for < 15 inference steps. This can+ stabilize the sampling of DPMSolver for steps < 15, especially for steps <= 10.+ disable_corrector (`list`, default `[]`):+ Decides which step to disable the corrector to mitigate the misalignment between `epsilon_theta(x_t, c)`+ and `epsilon_theta(x_t^c, c)` which can influence convergence for a large guidance scale. Corrector is+ usually disabled during the first few steps.+ solver_p (`SchedulerMixin`, default `None`):+ Any other scheduler that if specified, the algorithm becomes `solver_p + UniC`.+ use_karras_sigmas (`bool`, *optional*, defaults to `False`):+ Whether to use Karras sigmas for step sizes in the noise schedule during the sampling process. If `True`,+ the sigmas are determined according to a sequence of noise levels {σi}.+ use_exponential_sigmas (`bool`, *optional*, defaults to `False`):+ Whether to use exponential sigmas for step sizes in the noise schedule during the sampling process.+ timestep_spacing (`str`, defaults to `"linspace"`):+ The way the timesteps should be scaled. Refer to Table 2 of the [Common Diffusion Noise Schedules and+ Sample Steps are Flawed](https://huggingface.co/papers/2305.08891) for more information.+ steps_offset (`int`, defaults to 0):+ An offset added to the inference steps, as required by some model families.+ final_sigmas_type (`str`, defaults to `"zero"`):+ The final `sigma` value for the noise schedule during the sampling process. If `"sigma_min"`, the final+ sigma is the same as the last sigma in the training schedule. If `zero`, the final sigma is set to 0.+ """++ _compatibles = [e.name for e in KarrasDiffusionSchedulers]+ order = 1++ @register_to_config+ def __init__(+ self,+ num_train_timesteps: int = 1000,+ solver_order: int = 2,+ prediction_type: str = "flow_prediction",+ shift: Optional[float] = 1.0,+ use_dynamic_shifting=False,+ thresholding: bool = False,+ dynamic_thresholding_ratio: float = 0.995,+ sample_max_value: float = 1.0,+ predict_x0: bool = True,+ solver_type: str = "bh2",+ lower_order_final: bool = True,+ disable_corrector: List[int] = [],+ solver_p: SchedulerMixin = None,+ timestep_spacing: str = "linspace",+ steps_offset: int = 0,+ final_sigmas_type: Optional[str] = "zero", # "zero", "sigma_min"+ ):++ if solver_type not in ["bh1", "bh2"]:+ if solver_type in ["midpoint", "heun", "logrho"]:+ self.register_to_config(solver_type="bh2")+ else:+ raise NotImplementedError(+ f"{solver_type} is not implemented for {self.__class__}")++ self.predict_x0 = predict_x0+ # setable values+ self.num_inference_steps = None+ alphas = np.linspace(1, 1 / num_train_timesteps,+ num_train_timesteps)[::-1].copy()+ sigmas = 1.0 - alphas+ sigmas = torch.from_numpy(sigmas).to(dtype=torch.float32)++ if not use_dynamic_shifting:+ # when use_dynamic_shifting is True, we apply the timestep shifting on the fly based on the image resolution+ sigmas = shift * sigmas / (1 ++ (shift - 1) * sigmas) # pyright: ignore++ self.sigmas = sigmas+ self.timesteps = sigmas * num_train_timesteps++ self.model_outputs = [None] * solver_order+ self.timestep_list = [None] * solver_order+ self.lower_order_nums = 0+ self.disable_corrector = disable_corrector+ self.solver_p = solver_p+ self.last_sample = None+ self._step_index = None+ self._begin_index = None++ self.sigmas = self.sigmas.to(+ "cpu") # to avoid too much CPU/GPU communication+ self.sigma_min = self.sigmas[-1].item()+ self.sigma_max = self.sigmas[0].item()++ @property+ def step_index(self):+ """+ The index counter for current timestep. It will increase 1 after each scheduler step.+ """+ return self._step_index++ @property+ def begin_index(self):+ """+ The index for the first timestep. It should be set from pipeline with `set_begin_index` method.+ """+ return self._begin_index++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.set_begin_index+ def set_begin_index(self, begin_index: int = 0):+ """+ Sets the begin index for the scheduler. This function should be run from pipeline before the inference.++ Args:+ begin_index (`int`):+ The begin index for the scheduler.+ """+ self._begin_index = begin_index++ # Modified from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler.set_timesteps+ def set_timesteps(+ self,+ num_inference_steps: Union[int, None] = None,+ device: Union[str, torch.device] = None,+ sigmas: Optional[List[float]] = None,+ mu: Optional[Union[float, None]] = None,+ shift: Optional[Union[float, None]] = None,+ ):+ """+ Sets the discrete timesteps used for the diffusion chain (to be run before inference).+ Args:+ num_inference_steps (`int`):+ Total number of the spacing of the time steps.+ device (`str` or `torch.device`, *optional*):+ The device to which the timesteps should be moved to. If `None`, the timesteps are not moved.+ """++ if self.config.use_dynamic_shifting and mu is None:+ raise ValueError(+ " you have to pass a value for `mu` when `use_dynamic_shifting` is set to be `True`"+ )++ if sigmas is None:+ sigmas = np.linspace(self.sigma_max, self.sigma_min,+ num_inference_steps ++ 1).copy()[:-1] # pyright: ignore++ if self.config.use_dynamic_shifting:+ sigmas = self.time_shift(mu, 1.0, sigmas) # pyright: ignore+ else:+ if shift is None:+ shift = self.config.shift+ sigmas = shift * sigmas / (1 ++ (shift - 1) * sigmas) # pyright: ignore++ if self.config.final_sigmas_type == "sigma_min":+ sigma_last = ((1 - self.alphas_cumprod[0]) /+ self.alphas_cumprod[0])**0.5+ elif self.config.final_sigmas_type == "zero":+ sigma_last = 0+ else:+ raise ValueError(+ f"`final_sigmas_type` must be one of 'zero', or 'sigma_min', but got {self.config.final_sigmas_type}"+ )++ timesteps = sigmas * self.config.num_train_timesteps+ sigmas = np.concatenate([sigmas, [sigma_last]+ ]).astype(np.float32) # pyright: ignore++ self.sigmas = torch.from_numpy(sigmas)+ self.timesteps = torch.from_numpy(timesteps).to(+ device=device, dtype=torch.int64)++ self.num_inference_steps = len(timesteps)++ self.model_outputs = [+ None,+ ] * self.config.solver_order+ self.lower_order_nums = 0+ self.last_sample = None+ if self.solver_p:+ self.solver_p.set_timesteps(self.num_inference_steps, device=device)++ # add an index counter for schedulers that allow duplicated timesteps+ self._step_index = None+ self._begin_index = None+ self.sigmas = self.sigmas.to(+ "cpu") # to avoid too much CPU/GPU communication++ # Copied from diffusers.schedulers.scheduling_ddpm.DDPMScheduler._threshold_sample+ def _threshold_sample(self, sample: torch.Tensor) -> torch.Tensor:+ """+ "Dynamic thresholding: At each sampling step we set s to a certain percentile absolute pixel value in xt0 (the+ prediction of x_0 at timestep t), and if s > 1, then we threshold xt0 to the range [-s, s] and then divide by+ s. Dynamic thresholding pushes saturated pixels (those near -1 and 1) inwards, thereby actively preventing+ pixels from saturation at each step. We find that dynamic thresholding results in significantly better+ photorealism as well as better image-text alignment, especially when using very large guidance weights."++ https://arxiv.org/abs/2205.11487+ """+ dtype = sample.dtype+ batch_size, channels, *remaining_dims = sample.shape++ if dtype not in (torch.float32, torch.float64):+ sample = sample.float(+ ) # upcast for quantile calculation, and clamp not implemented for cpu half++ # Flatten sample for doing quantile calculation along each image+ sample = sample.reshape(batch_size, channels * np.prod(remaining_dims))++ abs_sample = sample.abs() # "a certain percentile absolute pixel value"++ s = torch.quantile(+ abs_sample, self.config.dynamic_thresholding_ratio, dim=1)+ s = torch.clamp(+ s, min=1, max=self.config.sample_max_value+ ) # When clamped to min=1, equivalent to standard clipping to [-1, 1]+ s = s.unsqueeze(+ 1) # (batch_size, 1) because clamp will broadcast along dim=0+ sample = torch.clamp(+ sample, -s, s+ ) / s # "we threshold xt0 to the range [-s, s] and then divide by s"++ sample = sample.reshape(batch_size, channels, *remaining_dims)+ sample = sample.to(dtype)++ return sample++ # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.FlowMatchEulerDiscreteScheduler._sigma_to_t+ def _sigma_to_t(self, sigma):+ return sigma * self.config.num_train_timesteps++ def _sigma_to_alpha_sigma_t(self, sigma):+ return 1 - sigma, sigma++ # Copied from diffusers.schedulers.scheduling_flow_match_euler_discrete.set_timesteps+ def time_shift(self, mu: float, sigma: float, t: torch.Tensor):+ return math.exp(mu) / (math.exp(mu) + (1 / t - 1)**sigma)++ def convert_model_output(+ self,+ model_output: torch.Tensor,+ *args,+ sample: torch.Tensor = None,+ **kwargs,+ ) -> torch.Tensor:+ r"""+ Convert the model output to the corresponding type the UniPC algorithm needs.++ Args:+ model_output (`torch.Tensor`):+ The direct output from the learned diffusion model.+ timestep (`int`):+ The current discrete timestep in the diffusion chain.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.++ Returns:+ `torch.Tensor`:+ The converted model output.+ """+ timestep = args[0] if len(args) > 0 else kwargs.pop("timestep", None)+ if sample is None:+ if len(args) > 1:+ sample = args[1]+ else:+ raise ValueError(+ "missing `sample` as a required keyward argument")+ if timestep is not None:+ deprecate(+ "timesteps",+ "1.0.0",+ "Passing `timesteps` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ sigma = self.sigmas[self.step_index]+ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)++ if self.predict_x0:+ if self.config.prediction_type == "flow_prediction":+ sigma_t = self.sigmas[self.step_index]+ x0_pred = sample - sigma_t * model_output+ else:+ raise ValueError(+ f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`,"+ " `v_prediction` or `flow_prediction` for the UniPCMultistepScheduler."+ )++ if self.config.thresholding:+ x0_pred = self._threshold_sample(x0_pred)++ return x0_pred+ else:+ if self.config.prediction_type == "flow_prediction":+ sigma_t = self.sigmas[self.step_index]+ epsilon = sample - (1 - sigma_t) * model_output+ else:+ raise ValueError(+ f"prediction_type given as {self.config.prediction_type} must be one of `epsilon`, `sample`,"+ " `v_prediction` or `flow_prediction` for the UniPCMultistepScheduler."+ )++ if self.config.thresholding:+ sigma_t = self.sigmas[self.step_index]+ x0_pred = sample - sigma_t * model_output+ x0_pred = self._threshold_sample(x0_pred)+ epsilon = model_output + x0_pred++ return epsilon++ def multistep_uni_p_bh_update(+ self,+ model_output: torch.Tensor,+ *args,+ sample: torch.Tensor = None,+ order: int = None, # pyright: ignore+ **kwargs,+ ) -> torch.Tensor:+ """+ One step for the UniP (B(h) version). Alternatively, `self.solver_p` is used if is specified.++ Args:+ model_output (`torch.Tensor`):+ The direct output from the learned diffusion model at the current timestep.+ prev_timestep (`int`):+ The previous discrete timestep in the diffusion chain.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.+ order (`int`):+ The order of UniP at this timestep (corresponds to the *p* in UniPC-p).++ Returns:+ `torch.Tensor`:+ The sample tensor at the previous timestep.+ """+ prev_timestep = args[0] if len(args) > 0 else kwargs.pop(+ "prev_timestep", None)+ if sample is None:+ if len(args) > 1:+ sample = args[1]+ else:+ raise ValueError(+ " missing `sample` as a required keyward argument")+ if order is None:+ if len(args) > 2:+ order = args[2]+ else:+ raise ValueError(+ " missing `order` as a required keyward argument")+ if prev_timestep is not None:+ deprecate(+ "prev_timestep",+ "1.0.0",+ "Passing `prev_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )+ model_output_list = self.model_outputs++ s0 = self.timestep_list[-1]+ m0 = model_output_list[-1]+ x = sample++ if self.solver_p:+ x_t = self.solver_p.step(model_output, s0, x).prev_sample+ return x_t++ sigma_t, sigma_s0 = self.sigmas[self.step_index + 1], self.sigmas[+ self.step_index] # pyright: ignore+ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t)+ alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0)++ lambda_t = torch.log(alpha_t) - torch.log(sigma_t)+ lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0)++ h = lambda_t - lambda_s0+ device = sample.device++ rks = []+ D1s = []+ for i in range(1, order):+ si = self.step_index - i # pyright: ignore+ mi = model_output_list[-(i + 1)]+ alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si])+ lambda_si = torch.log(alpha_si) - torch.log(sigma_si)+ rk = (lambda_si - lambda_s0) / h+ rks.append(rk)+ D1s.append((mi - m0) / rk) # pyright: ignore++ rks.append(1.0)+ rks = torch.tensor(rks, device=device)++ R = []+ b = []++ hh = -h if self.predict_x0 else h+ h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1+ h_phi_k = h_phi_1 / hh - 1++ factorial_i = 1++ if self.config.solver_type == "bh1":+ B_h = hh+ elif self.config.solver_type == "bh2":+ B_h = torch.expm1(hh)+ else:+ raise NotImplementedError()++ for i in range(1, order + 1):+ R.append(torch.pow(rks, i - 1))+ b.append(h_phi_k * factorial_i / B_h)+ factorial_i *= i + 1+ h_phi_k = h_phi_k / hh - 1 / factorial_i++ R = torch.stack(R)+ b = torch.tensor(b, device=device)++ if len(D1s) > 0:+ D1s = torch.stack(D1s, dim=1) # (B, K)+ # for order 2, we use a simplified version+ if order == 2:+ rhos_p = torch.tensor([0.5], dtype=x.dtype, device=device)+ else:+ rhos_p = torch.linalg.solve(R[:-1, :-1],+ b[:-1]).to(device).to(x.dtype)+ else:+ D1s = None++ if self.predict_x0:+ x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0+ if D1s is not None:+ pred_res = torch.einsum("k,bkc...->bc...", rhos_p,+ D1s) # pyright: ignore+ else:+ pred_res = 0+ x_t = x_t_ - alpha_t * B_h * pred_res+ else:+ x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0+ if D1s is not None:+ pred_res = torch.einsum("k,bkc...->bc...", rhos_p,+ D1s) # pyright: ignore+ else:+ pred_res = 0+ x_t = x_t_ - sigma_t * B_h * pred_res++ x_t = x_t.to(x.dtype)+ return x_t++ def multistep_uni_c_bh_update(+ self,+ this_model_output: torch.Tensor,+ *args,+ last_sample: torch.Tensor = None,+ this_sample: torch.Tensor = None,+ order: int = None, # pyright: ignore+ **kwargs,+ ) -> torch.Tensor:+ """+ One step for the UniC (B(h) version).++ Args:+ this_model_output (`torch.Tensor`):+ The model outputs at `x_t`.+ this_timestep (`int`):+ The current timestep `t`.+ last_sample (`torch.Tensor`):+ The generated sample before the last predictor `x_{t-1}`.+ this_sample (`torch.Tensor`):+ The generated sample after the last predictor `x_{t}`.+ order (`int`):+ The `p` of UniC-p at this step. The effective order of accuracy should be `order + 1`.++ Returns:+ `torch.Tensor`:+ The corrected sample tensor at the current timestep.+ """+ this_timestep = args[0] if len(args) > 0 else kwargs.pop(+ "this_timestep", None)+ if last_sample is None:+ if len(args) > 1:+ last_sample = args[1]+ else:+ raise ValueError(+ " missing`last_sample` as a required keyward argument")+ if this_sample is None:+ if len(args) > 2:+ this_sample = args[2]+ else:+ raise ValueError(+ " missing`this_sample` as a required keyward argument")+ if order is None:+ if len(args) > 3:+ order = args[3]+ else:+ raise ValueError(+ " missing`order` as a required keyward argument")+ if this_timestep is not None:+ deprecate(+ "this_timestep",+ "1.0.0",+ "Passing `this_timestep` is deprecated and has no effect as model output conversion is now handled via an internal counter `self.step_index`",+ )++ model_output_list = self.model_outputs++ m0 = model_output_list[-1]+ x = last_sample+ x_t = this_sample+ model_t = this_model_output++ sigma_t, sigma_s0 = self.sigmas[self.step_index], self.sigmas[+ self.step_index - 1] # pyright: ignore+ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma_t)+ alpha_s0, sigma_s0 = self._sigma_to_alpha_sigma_t(sigma_s0)++ lambda_t = torch.log(alpha_t) - torch.log(sigma_t)+ lambda_s0 = torch.log(alpha_s0) - torch.log(sigma_s0)++ h = lambda_t - lambda_s0+ device = this_sample.device++ rks = []+ D1s = []+ for i in range(1, order):+ si = self.step_index - (i + 1) # pyright: ignore+ mi = model_output_list[-(i + 1)]+ alpha_si, sigma_si = self._sigma_to_alpha_sigma_t(self.sigmas[si])+ lambda_si = torch.log(alpha_si) - torch.log(sigma_si)+ rk = (lambda_si - lambda_s0) / h+ rks.append(rk)+ D1s.append((mi - m0) / rk) # pyright: ignore++ rks.append(1.0)+ rks = torch.tensor(rks, device=device)++ R = []+ b = []++ hh = -h if self.predict_x0 else h+ h_phi_1 = torch.expm1(hh) # h\phi_1(h) = e^h - 1+ h_phi_k = h_phi_1 / hh - 1++ factorial_i = 1++ if self.config.solver_type == "bh1":+ B_h = hh+ elif self.config.solver_type == "bh2":+ B_h = torch.expm1(hh)+ else:+ raise NotImplementedError()++ for i in range(1, order + 1):+ R.append(torch.pow(rks, i - 1))+ b.append(h_phi_k * factorial_i / B_h)+ factorial_i *= i + 1+ h_phi_k = h_phi_k / hh - 1 / factorial_i++ R = torch.stack(R)+ b = torch.tensor(b, device=device)++ if len(D1s) > 0:+ D1s = torch.stack(D1s, dim=1)+ else:+ D1s = None++ # for order 1, we use a simplified version+ if order == 1:+ rhos_c = torch.tensor([0.5], dtype=x.dtype, device=device)+ else:+ rhos_c = torch.linalg.solve(R, b).to(device).to(x.dtype)++ if self.predict_x0:+ x_t_ = sigma_t / sigma_s0 * x - alpha_t * h_phi_1 * m0+ if D1s is not None:+ corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s)+ else:+ corr_res = 0+ D1_t = model_t - m0+ x_t = x_t_ - alpha_t * B_h * (corr_res + rhos_c[-1] * D1_t)+ else:+ x_t_ = alpha_t / alpha_s0 * x - sigma_t * h_phi_1 * m0+ if D1s is not None:+ corr_res = torch.einsum("k,bkc...->bc...", rhos_c[:-1], D1s)+ else:+ corr_res = 0+ D1_t = model_t - m0+ x_t = x_t_ - sigma_t * B_h * (corr_res + rhos_c[-1] * D1_t)+ x_t = x_t.to(x.dtype)+ return x_t++ def index_for_timestep(self, timestep, schedule_timesteps=None):+ if schedule_timesteps is None:+ schedule_timesteps = self.timesteps++ indices = (schedule_timesteps == timestep).nonzero()++ # The sigma index that is taken for the **very** first `step`+ # is always the second index (or the last index if there is only 1)+ # This way we can ensure we don't accidentally skip a sigma in+ # case we start in the middle of the denoising schedule (e.g. for image-to-image)+ pos = 1 if len(indices) > 1 else 0++ return indices[pos].item()++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler._init_step_index+ def _init_step_index(self, timestep):+ """+ Initialize the step_index counter for the scheduler.+ """++ if self.begin_index is None:+ if isinstance(timestep, torch.Tensor):+ timestep = timestep.to(self.timesteps.device)+ self._step_index = self.index_for_timestep(timestep)+ else:+ self._step_index = self._begin_index++ def step(self,+ model_output: torch.Tensor,+ timestep: Union[int, torch.Tensor],+ sample: torch.Tensor,+ return_dict: bool = True,+ generator=None) -> Union[SchedulerOutput, Tuple]:+ """+ Predict the sample from the previous timestep by reversing the SDE. This function propagates the sample with+ the multistep UniPC.++ Args:+ model_output (`torch.Tensor`):+ The direct output from learned diffusion model.+ timestep (`int`):+ The current discrete timestep in the diffusion chain.+ sample (`torch.Tensor`):+ A current instance of a sample created by the diffusion process.+ return_dict (`bool`):+ Whether or not to return a [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`.++ Returns:+ [`~schedulers.scheduling_utils.SchedulerOutput`] or `tuple`:+ If return_dict is `True`, [`~schedulers.scheduling_utils.SchedulerOutput`] is returned, otherwise a+ tuple is returned where the first element is the sample tensor.++ """+ if self.num_inference_steps is None:+ raise ValueError(+ "Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"+ )++ if self.step_index is None:+ self._init_step_index(timestep)++ use_corrector = (+ self.step_index > 0 and+ self.step_index - 1 not in self.disable_corrector and+ self.last_sample is not None # pyright: ignore+ )++ model_output_convert = self.convert_model_output(+ model_output, sample=sample)+ if use_corrector:+ sample = self.multistep_uni_c_bh_update(+ this_model_output=model_output_convert,+ last_sample=self.last_sample,+ this_sample=sample,+ order=self.this_order,+ )++ for i in range(self.config.solver_order - 1):+ self.model_outputs[i] = self.model_outputs[i + 1]+ self.timestep_list[i] = self.timestep_list[i + 1]++ self.model_outputs[-1] = model_output_convert+ self.timestep_list[-1] = timestep # pyright: ignore++ if self.config.lower_order_final:+ this_order = min(self.config.solver_order,+ len(self.timesteps) -+ self.step_index) # pyright: ignore+ else:+ this_order = self.config.solver_order++ self.this_order = min(this_order,+ self.lower_order_nums + 1) # warmup for multistep+ assert self.this_order > 0++ self.last_sample = sample+ prev_sample = self.multistep_uni_p_bh_update(+ model_output=model_output, # pass the original non-converted model output, in case solver-p is used+ sample=sample,+ order=self.this_order,+ )++ if self.lower_order_nums < self.config.solver_order:+ self.lower_order_nums += 1++ # upon completion increase step index by one+ self._step_index += 1 # pyright: ignore++ if not return_dict:+ return (prev_sample,)++ return SchedulerOutput(prev_sample=prev_sample)++ def scale_model_input(self, sample: torch.Tensor, *args,+ **kwargs) -> torch.Tensor:+ """+ Ensures interchangeability with schedulers that need to scale the denoising model input depending on the+ current timestep.++ Args:+ sample (`torch.Tensor`):+ The input sample.++ Returns:+ `torch.Tensor`:+ A scaled input sample.+ """+ return sample++ # Copied from diffusers.schedulers.scheduling_dpmsolver_multistep.DPMSolverMultistepScheduler.add_noise+ def add_noise(+ self,+ original_samples: torch.Tensor,+ noise: torch.Tensor,+ timesteps: torch.IntTensor,+ ) -> torch.Tensor:+ # Make sure sigmas and timesteps have the same device and dtype as original_samples+ sigmas = self.sigmas.to(+ device=original_samples.device, dtype=original_samples.dtype)+ if original_samples.device.type == "mps" and torch.is_floating_point(+ timesteps):+ # mps does not support float64+ schedule_timesteps = self.timesteps.to(+ original_samples.device, dtype=torch.float32)+ timesteps = timesteps.to(+ original_samples.device, dtype=torch.float32)+ else:+ schedule_timesteps = self.timesteps.to(original_samples.device)+ timesteps = timesteps.to(original_samples.device)++ # begin_index is None when the scheduler is used for training or pipeline does not implement set_begin_index+ if self.begin_index is None:+ step_indices = [+ self.index_for_timestep(t, schedule_timesteps)+ for t in timesteps+ ]+ elif self.step_index is not None:+ # add_noise is called after first denoising step (for inpainting)+ step_indices = [self.step_index] * timesteps.shape[0]+ else:+ # add noise is called before first denoising step to create initial latent(img2img)+ step_indices = [self.begin_index] * timesteps.shape[0]++ sigma = sigmas[step_indices].flatten()+ while len(sigma.shape) < len(original_samples.shape):+ sigma = sigma.unsqueeze(-1)++ alpha_t, sigma_t = self._sigma_to_alpha_sigma_t(sigma)+ noisy_samples = alpha_t * original_samples + sigma_t * noise+ return noisy_samples++ def __len__(self):+ return self.config.num_train_timestepsdiff --git a/models/wan/ovi_fusion_engine.py b/models/wan/ovi_fusion_engine.pynew file mode 100644index 0000000..b3fddf2--- /dev/null+++ b/models/wan/ovi_fusion_engine.py@@ -0,0 +1,368 @@+import os++import torch+import logging+from textwrap import indent+import torch.nn as nn+from tqdm import tqdm+from .ovi.utils.fm_solvers_unipc import FlowUniPCMultistepScheduler+from diffusers import FlowMatchEulerDiscreteScheduler+from .ovi.utils.fm_solvers import (FlowDPMSolverMultistepScheduler,+ get_sampling_sigmas, retrieve_timesteps)+from shared.utils import files_locator as fl+from .modules.vae2_2 import Wan2_2_VAE+from .modules.t5 import T5EncoderModel+from .ovi.modules.mmaudio.features_utils import FeaturesUtils+from .ovi.modules.fusion import FusionModel+import json+from mmgp import offload+from shared.utils.loras_mutipliers import update_loras_slists, get_model_switch_steps++++def init_fusion_score_model_ovi(rank: int = 0, meta_init=True):+ config_root = os.path.join("models", "wan", "ovi", "configs")+ video_config_path = os.path.join(config_root , "video.json")+ audio_config_path = os.path.join(config_root , "audio.json")++ with open(video_config_path, encoding="utf-8") as f:+ video_config = json.load(f)++ with open(audio_config_path, encoding="utf-8") as f:+ audio_config = json.load(f)++ with torch.device("meta"):+ fusion_model = FusionModel(video_config, audio_config)++ return fusion_model, video_config, audio_config++def init_mmaudio_vae():+ tod_path = fl.locate_file( os.path.join("mmaudio", "v1-16.pth"))+ bigvgan_path = fl.locate_file(os.path.join("mmaudio", "best_netG.pt"))++ vae_config = {+ "mode": "16k",+ "need_vae_encoder": True,+ "tod_vae_ckpt": str(tod_path),+ "bigvgan_vocoder_ckpt": str(bigvgan_path),+ }++ return FeaturesUtils(**vae_config).to("cpu")++class OviFusionEngine:+ def __init__(self,+ device="cuda",+ model_filename = None,+ text_encoder_filename = None,+ VAE_dtype = torch.bfloat16,+ dtype = torch.bfloat16,+ model_def = None,+ **any):++ self.device = "cpu"+ self.dtype = dtype+ self.sr = 16000+ self.fps = model_def.get("fps", 24)+ self._interrupt = False+ self.last_audio = None++ # Load fusion model+ self.device = device+ self.target_dtype = torch.bfloat16 # dtype, wont work with torch.float16+ model, video_config, audio_config = init_fusion_score_model_ovi()+ offload.load_model_data(model.video_model, fl.locate_file(model_filename[0]))+ offload.load_model_data(model.audio_model, fl.locate_file(model_filename[1]))+ offload.change_dtype(model, dtype, True)+ model = model.eval()+ # model.set_rope_params()+ self.model = model+++ # offload.save_model(model.video_model, "wan2.2_ovi_video_10B_bf16.safetensors")+ # offload.save_model(model.video_model, "wan2.2_ovi_video_10B_quanto_fp16_int8.safetensors", do_quantize=True)++ # offload.save_model(model.audio_model, "wan2.2_ovi_audio_10B_bf16.safetensors")+ # offload.save_model(model.audio_model, "wan2.2_ovi_audio_10B_quanto_fp16_int8.safetensors", do_quantize=True)+++ self.vae_stride = (4, 16, 16)+ vae_checkpoint = "Wan2.2_VAE.safetensors"+ self.vae = Wan2_2_VAE( vae_pth=fl.locate_file(vae_checkpoint), dtype= VAE_dtype, device="cpu")+ self.vae.device = self.device # need to set to cuda so that vae buffers are properly moved (although the rest will stay in the CPU)+ self.vae.model.requires_grad_(False).eval()++ vae_model_audio = init_mmaudio_vae()+ vae_model_audio.requires_grad_(False).eval()+ self.audio_vae = vae_model_audio.bfloat16()+ # Load T5 text model+ self.text_encoder = T5EncoderModel(+ text_len=512,+ dtype=torch.bfloat16,+ device=torch.device('cpu'),+ checkpoint_path=text_encoder_filename,+ tokenizer_path=fl.locate_folder("umt5-xxl"),+ shard_fn= None)++++ ## Load t2i as part of pipeline+ self.image_model = None++ # if config.get("mode") == "t2i2v":+ # logging.info(f"Loading Flux Krea for first frame generation...")+ # self.image_model = FluxPipeline.from_pretrained("black-forest-labs/FLUX.1-Krea-dev", torch_dtype=torch.bfloat16)+ # self.image_model.enable_model_cpu_offload(gpu_id=self.device) #save some VRAM by offloading the model to CPU. Remove this if you have enough GPU VRAM++ # Fixed attributes, non-configurable+ self.audio_latent_channel = audio_config.get("in_dim")+ self.video_latent_channel = video_config.get("in_dim")+ self.audio_latent_length = 157+ self.video_latent_length = 31++ logging.info(f"OVI Fusion Engine initialized, GPU VRAM allocated: {torch.cuda.memory_allocated(device)/1e9:.2f} GB, reserved: {torch.cuda.memory_reserved(device)/1e9:.2f} GB")++++ @torch.inference_mode()+ def generate(self,+ input_prompt,+ image_start=None,+ input_video = None,+ width = 1280,+ height = 720,+ seed=100,+ solver_name="unipc",+ sampling_steps=50,+ shift=5.0,+ guide_scale=5.0,+ audio_cfg_scalecale=4.0,+ slg_layers=[11],+ slg_start = 0.0,+ slg_end = 1.0,+ n_prompt="",+ audio_negative_prompt="",+ loras_slists = None,+ callback = None,+ block_size = 0,+ VAE_tile_size = 0,+ joint_pass = False,+ **bbkwargs,+ ):++ if len(n_prompt) == 0:+ n_prompt = "jitter, bad hands, blur, distortion" # Artifacts to avoid in video+ if len(audio_negative_prompt) == 0:+ audio_negative_prompt= "robotic, muffled, echo, distorted" # Artifacts to avoid in audio++ slg_layer = None+ if isinstance(slg_layers, (list, tuple)) and slg_layers:+ slg_layer = int(slg_layers[0])+ elif isinstance(slg_layers, (int, float)):+ slg_layer = int(slg_layers)+ if slg_layer is None:+ slg_layer = 11++ video_frame_height_width=(height, width)++ scheduler_video, timesteps_video = self.get_scheduler_time_steps(+ sampling_steps=sampling_steps,+ device=self.device,+ solver_name=solver_name,+ shift=shift+ )+ scheduler_audio, timesteps_audio = self.get_scheduler_time_steps(+ sampling_steps=sampling_steps,+ device=self.device,+ solver_name=solver_name,+ shift=shift+ )++ if self._interrupt:+ return None+++ if input_video is not None:+ first_frame = input_video #image_start.unsqueeze(1) if is_i2v else None+ is_i2v = True+ else:+ first_frame = None+ is_i2v = False++ if callback != None:+ callback(-1, None, True)++ text_embeddings = self.text_encoder([input_prompt, n_prompt, audio_negative_prompt], device= self.device)+ text_embeddings = [emb.to(self.target_dtype).to(self.device) for emb in text_embeddings]+ # Split embeddings+ text_embeddings_audio_pos = text_embeddings[0]+ text_embeddings_video_pos = text_embeddings[0]++ text_embeddings_video_neg = text_embeddings[1]+ text_embeddings_audio_neg = text_embeddings[2]++ if is_i2v:+ with torch.no_grad():+ latents_images = self.vae.encode([first_frame], VAE_tile_size)[0].to(self.target_dtype) # c 1 h w+ latents_images = latents_images.to(self.target_dtype)+ video_latent_h, video_latent_w = latents_images.shape[2], latents_images.shape[3]+ else:+ video_h, video_w = video_frame_height_width+ video_latent_h, video_latent_w = video_h // 16, video_w // 16++ video_latent_length = 31+ audio_latent_length = 157++++ from .modules.posemb_layers import get_rotary_pos_embed, get_nd_rotary_pos_embed++ video_freqs = get_nd_rotary_pos_embed((0, 0, 0 ), (video_latent_length, video_latent_h//2, video_latent_w//2 ))+ # audio_freqs = get_nd_rotary_pos_embed((0,), (audio_latent_length, ), interpolation_factor= self.model.audio_model.temporal_rope_scaling_factor, rope_dim_list= [44])+ audio_freqs = self.model.audio_model.get_audio_rope_params()+ video_noise = torch.randn((self.video_latent_channel, video_latent_length, video_latent_h, video_latent_w), device=self.device, dtype=self.target_dtype, generator=torch.Generator(device=self.device).manual_seed(seed)) # c, f, h, w+ audio_noise = torch.randn((audio_latent_length, self.audio_latent_channel), device=self.device, dtype=self.target_dtype, generator=torch.Generator(device=self.device).manual_seed(seed)) # 1, l c -> l, c+ # self.model.audio_model.freqs= self.model.audio_model.freqs.to("cuda")+ # self.model.video_model.freqs= self.model.video_model.freqs.to("cuda")+ def ret():+ # self.model.audio_model.freqs= self.model.audio_model.freqs.to("cpu")+ # self.model.video_model.freqs= self.model.video_model.freqs.to("cpu")+ return None++ # Calculate sequence lengths from actual latents+ max_seq_len_audio = audio_noise.shape[0] # L dimension from latents_audios shape [1, L, D]+ _patch_size_h, _patch_size_w = self.model.video_model.patch_size[1], self.model.video_model.patch_size[2]+ max_seq_len_video = video_noise.shape[1] * video_noise.shape[2] * video_noise.shape[3] // (_patch_size_h*_patch_size_w) # f * h * w from [1, c, f, h, w]++ update_loras_slists(self.model.video_model, loras_slists, len(timesteps_video))+ kwargs = {+ 'vid_seq_len': max_seq_len_video,+ 'audio_seq_len': max_seq_len_audio,+ 'first_frame_is_clean': is_i2v,+ 'callback' : callback,+ 'pipeline': self,+ 'video_freqs': video_freqs,+ 'audio_freqs': audio_freqs,+ }++ # Sampling loop+ with torch.amp.autocast('cuda', enabled=self.target_dtype != torch.float32, dtype=self.target_dtype):+ for i, (t_v, t_a) in tqdm(enumerate(zip(timesteps_video, timesteps_audio))):+ timestep_input = torch.full((1,), t_v, device=self.device)+ kwargs.update({+ "vid": video_noise,+ "audio" : audio_noise,+ "t": timestep_input,+ })+ offload.set_step_no_for_lora(self.model.video_model, i)+ if is_i2v:+ video_noise[:, :1] = latents_images+ computed_slg_layers = slg_layers if int(slg_start * sampling_steps) <= i < int(slg_end * sampling_steps) else None+ any_guidance = not (guide_scale == 1 and audio_cfg_scalecale ==1)++ if any_guidance and not joint_pass:+ pred_vid_pos, pred_audio_pos = self.model(+ audio_context= [text_embeddings_audio_pos],+ vid_context= [text_embeddings_video_pos],+ x_id_list =[0],+ **kwargs+ )+ if pred_vid_pos is None:+ return ret()++ pred_vid_neg, pred_audio_neg = self.model(+ audio_context= [text_embeddings_audio_neg],+ vid_context =[text_embeddings_video_neg],+ x_id_list =[1],+ computed_slg_layers = computed_slg_layers,+ **kwargs+ )+ if pred_vid_neg is None:+ return ret()+ else:+ vid, audio = self.model(+ audio_context= [text_embeddings_audio_pos, text_embeddings_audio_neg],+ vid_context= [text_embeddings_video_pos, text_embeddings_video_neg],+ computed_slg_layers = computed_slg_layers,+ x_id_list =[0,1],+ **kwargs+ )+ if vid is None:+ return ret()+ pred_vid_pos, pred_vid_neg = vid+ pred_audio_pos, pred_audio_neg = audio+ vid = audio = None+ # Apply classifier-free guidance+ pred_video_guided = pred_vid_neg + guide_scale * (pred_vid_pos - pred_vid_neg)+ pred_audio_guided = pred_audio_neg + audio_cfg_scalecale * (pred_audio_pos - pred_audio_neg)+ pred_audio_neg = pred_audio_pos = pred_vid_neg = pred_vid_pos = None+ # Update noise using scheduler+ video_noise = scheduler_video.step(+ pred_video_guided.unsqueeze(0), t_v, video_noise.unsqueeze(0), return_dict=False+ )[0].squeeze(0)+ pred_video_guided = None+ audio_noise = scheduler_audio.step(+ pred_audio_guided.unsqueeze(0), t_a, audio_noise.unsqueeze(0), return_dict=False+ )[0].squeeze(0)+ pred_audio_guided = None+ if callback is not None:+ latents_preview = video_noise+ callback(i, latents_preview, False )+ latents_preview = None+ ret()++ if is_i2v:+ video_noise[:, :1] = latents_images++ # Decode audio+ audio_latents_for_vae = audio_noise.unsqueeze(0).transpose(1, 2) # 1, c, l+ generated_audio = self.audio_vae.wrapped_decode(audio_latents_for_vae)+ generated_audio = generated_audio.squeeze().cpu().float().numpy()++ # Decode video+ video_latents_for_vae = video_noise.unsqueeze(0) # 1, c, f, h, w+ generated_video = self.vae.decode(video_latents_for_vae, VAE_tile_size)[0]+ generated_video = generated_video.cpu().float() # c, f, h, w++ # self.last_audio = audio+ output = {"x": generated_video, "audio": generated_audio}+ return output++++ def get_scheduler_time_steps(self, sampling_steps, solver_name='unipc', device=0, shift=5.0):+ torch.manual_seed(4)++ if solver_name == 'unipc':+ sample_scheduler = FlowUniPCMultistepScheduler(+ num_train_timesteps=1000,+ shift=1,+ use_dynamic_shifting=False)+ sample_scheduler.set_timesteps(+ sampling_steps, device=device, shift=shift)+ timesteps = sample_scheduler.timesteps++ elif solver_name == 'dpm++':+ sample_scheduler = FlowDPMSolverMultistepScheduler(+ num_train_timesteps=1000,+ shift=1,+ use_dynamic_shifting=False)+ sampling_sigmas = get_sampling_sigmas(sampling_steps, shift=shift)+ timesteps, _ = retrieve_timesteps(+ sample_scheduler,+ device=device,+ sigmas=sampling_sigmas)++ elif solver_name == 'euler':+ sample_scheduler = FlowMatchEulerDiscreteScheduler(+ shift=shift+ )+ timesteps, sampling_steps = retrieve_timesteps(+ sample_scheduler,+ sampling_steps,+ device=device,+ )++ else:+ raise NotImplementedError("Unsupported solver.")++ return sample_scheduler, timestepsdiff --git a/models/wan/ovi_handler.py b/models/wan/ovi_handler.pynew file mode 100644index 0000000..4d3c857--- /dev/null+++ b/models/wan/ovi_handler.py@@ -0,0 +1,146 @@+import os+from pathlib import Path+from typing import Any, Dict, Tuple++import torch++from shared.utils import files_locator as fl+++class family_handler:+ @staticmethod+ def query_supported_types():+ return ["ovi"]++ @staticmethod+ def query_family_maps() -> Tuple[Dict[str, str], Dict[str, list]]:+ return {}, {}++ @staticmethod+ def query_model_family():+ return "wan"++ @staticmethod+ def query_family_infos():+ return {}++ @staticmethod+ def get_wan_text_encoder_filename(text_encoder_quantization):+ text_encoder_filename = "umt5-xxl/models_t5_umt5-xxl-enc-bf16.safetensors"+ if text_encoder_quantization =="int8":+ text_encoder_filename = text_encoder_filename.replace("bf16", "quanto_int8")+ return fl.locate_file(text_encoder_filename, True)++ @staticmethod+ def query_model_def(base_model_type: str, model_def: Dict[str, Any]):+ cfg = {+ "wan_5B_class": True,+ "profiles_dir": [""],+ "group": "wan2_2",+ "fps": 24,+ "frames_minimum": 121,+ "frames_steps": 120,+ "sliding_window": False,+ "multiple_submodels": False,+ "guidance_max_phases": 1,+ "skip_layer_guidance": True,+ "returns_audio": True,+ "sample_solvers": [+ ("unipc", "unipc"),+ ("dpm++", "dpm++"),+ ("euler", "euler"),+ ],+ "audio_guidance": True,+ "image_prompt_types_allowed" : "TSVL",+ "sliding_window": True,+ "sliding_window_size_locked": True,+ "sliding_window_defaults" : { "overlap_min" : 1, "overlap_max" : 1, "overlap_step": 0, "overlap_default": 1},+ }+ cfg.update(model_def)+ return cfg++ @staticmethod+ def query_model_files(computeList, base_model_type, model_filename, text_encoder_quantization):++ from .wan_handler import family_handler+ download_def = family_handler.query_model_files(computeList, "ti2v_2_2", model_filename, text_encoder_quantization)+ if not isinstance(download_def, list):+ download_def = [download_def]+ download_def += [{+ "repoId" : "DeepBeepMeep/Wan2.1",+ "sourceFolderList" : ["mmaudio", ],+ "fileList" : [ [ "v1-44.pth", "best_netG.pt"]]+ }]++ return download_def++ @staticmethod+ def load_model(+ model_filename,+ model_type,+ base_model_type,+ model_def,+ quantizeTransformer=False,+ text_encoder_quantization=None,+ dtype=torch.bfloat16,+ VAE_dtype=torch.float32,+ mixed_precision_transformer=False,+ save_quantized=False,+ submodel_no_list=None,+ override_text_encoder=None,+ ):+ from .ovi_fusion_engine import OviFusionEngine++ checkpoint_dir = "ckpts"++ ovi_model = OviFusionEngine(+ config=None,+ checkpoint_dir=checkpoint_dir,+ model_def=model_def,+ model_filename = model_filename,+ text_encoder_filename = family_handler.get_wan_text_encoder_filename(text_encoder_quantization),+ dtype=dtype,+ )++ pipe = {+ "transformer": ovi_model.model.video_model,+ "transformer2": ovi_model.model.audio_model,+ "text_encoder": ovi_model.text_encoder.model,+ "vae": ovi_model.vae.model,+ "vae2": ovi_model.audio_vae,+ }+ cotenants_map = {+ "transformer": ["transformer2"],+ "transformer2": ["transformer"],+ }++ dict = { "pipe": pipe, "coTenantsMap": cotenants_map}+ return ovi_model, dict++ @staticmethod+ def fix_settings(base_model_type, settings_version, model_def, ui_defaults):+ pass++ @staticmethod+ def update_default_settings(base_model_type, model_def, ui_defaults):+ ui_defaults.update({ "sample_solver": "unipc",+ "flow_shift": 5.0,+ "guidance_scale": 4.0,+ "audio_guidance_scale": 3.0,+ "num_inference_steps": 50,+ "slg_switch": 1,+ "sliding_window_size": 121,+ "video_length": 121,+ "slg_layers" : [11]+ })+++ @staticmethod+ def get_vae_block_size(base_model_type):+ return 32++ @staticmethod+ def get_rgb_factors(base_model_type):+ from shared.RGB_factors import get_rgb_factors++ return get_rgb_factors("wan", "ti2v_2_2")diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex 07a6ad1..011fe44 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -12,7 +12,7 @@ def test_class_i2v(base_model_type):def test_class_t2v(base_model_type):return base_model_type in ["t2v", "t2v_2_2", "alpha", "lynx"]-def text_oneframe_overlap(base_model_type):+def test_oneframe_overlap(base_model_type):return test_class_i2v(base_model_type) and not (test_multitalk(base_model_type) or base_model_type in ["animate"]) or test_wan_5B(base_model_type)def test_class_1_3B(base_model_type):@@ -419,7 +419,8 @@ class family_handler():"labels" : { "UV": "Control Video"},"visible" : False,}-+ extra_model_def["video_length_locked"] = 81+if vace_class or base_model_type in ["animate", "t2v", "t2v_2_2", "lynx"] :image_prompt_types_allowed = "TVL"elif base_model_type in ["infinitetalk"]:@@ -435,8 +436,10 @@ class family_handler():else:image_prompt_types_allowed = ""extra_model_def["image_prompt_types_allowed"] = image_prompt_types_allowed+ if base_model_type in ["fantasy"] or multitalk:+ extra_model_def["audio_guidance"] = True- if text_oneframe_overlap(base_model_type):+ if test_oneframe_overlap(base_model_type):extra_model_def["sliding_window_defaults"] = { "overlap_min" : 1, "overlap_max" : 1, "overlap_step": 0, "overlap_default": 1}# if base_model_type in ["phantom_1.3B", "phantom_14B"]:@@ -550,7 +553,7 @@ class family_handler():ui_defaults["video_prompt_type"] = video_prompt_typeui_defaults["image_prompt_type"] = ""- if text_oneframe_overlap(base_model_type):+ if test_oneframe_overlap(base_model_type):ui_defaults["sliding_window_overlap"] = 1if settings_version < 2.32:@@ -670,7 +673,7 @@ class family_handler():"video_prompt_type": "V",})- if text_oneframe_overlap(base_model_type):+ if test_oneframe_overlap(base_model_type):ui_defaults["sliding_window_overlap"] = 1ui_defaults["color_correction_strength"]= 0diff --git a/wgp.py b/wgp.pyindex 8a14112..72add81 100644--- a/wgp.py+++ b/wgp.py@@ -8,7 +8,14 @@ p = os.path.dirname(os.path.abspath(__file__))if p not in sys.path:sys.path.insert(0, p)import asyncio-if os.name == "nt": asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())+if os.name == "nt":+ asyncio.set_event_loop_policy(asyncio.WindowsSelectorEventLoopPolicy())+ import asyncio.proactor_events as _proactor+ _orig_call_lost = _proactor._ProactorBasePipeTransport._call_connection_lost+ def _quiet_call_lost(self, exc):+ if isinstance(exc, ConnectionResetError): exc = None+ return _orig_call_lost(self, exc)+ _proactor._ProactorBasePipeTransport._call_connection_lost = _quiet_call_lostimport timeimport threadingimport argparse@@ -58,6 +65,7 @@ import atexitimport shutilimport globimport cv2+import htmlfrom transformers.utils import logginglogging.set_verbosity_errorfrom tqdm import tqdm@@ -1585,8 +1593,8 @@ def generate_queue_html(queue):row_index = i - 1task_id = item['id']- truncated_prompt = (item['prompt'][:97] + '...') if len(item['prompt']) > 100 else item['prompt']- full_prompt = item['prompt'].replace('"', '"')+ full_prompt = html.escape(item['prompt'])+ truncated_prompt = (html.escape(item['prompt'][:97]) + '...') if len(item['prompt']) > 100 else full_promptprompt_cell = f'<div class="prompt-cell" title="{full_prompt}">{truncated_prompt}</div>'start_img_data = item.get('start_image_data_base64') or [None]@@ -2159,7 +2167,7 @@ for f, s in [(fl.locate_file("Florence2/modeling_florence2.py"), 127287)]:except: passmodels_def = {}-family_handlers = ["models.wan.wan_handler", "models.wan.df_handler", "models.hyvideo.hunyuan_handler", "models.ltx_video.ltxv_handler", "models.flux.flux_handler", "models.qwen.qwen_handler", "models.chatterbox.chatterbox_handler"]+family_handlers = ["models.wan.wan_handler", "models.wan.ovi_handler", "models.wan.df_handler", "models.hyvideo.hunyuan_handler", "models.ltx_video.ltxv_handler", "models.flux.flux_handler", "models.qwen.qwen_handler", "models.chatterbox.chatterbox_handler"]# only needed for imported old settings files@@ -3258,6 +3266,11 @@ def load_models(model_type, override_profile = -1):dtype = transformer_dtype, VAE_dtype = VAE_dtype, mixed_precision_transformer = mixed_precision_transformer, save_quantized = save_quantized, submodel_no_list = model_submodel_no_list, override_text_encoder = override_text_encoder )kwargs = {}+ if "pipe" in pipe:+ kwargs = pipe+ pipe = kwargs.pop("pipe")+ if "coTenantsMap" not in kwargs: kwargs["coTenantsMap"] = {}+profile = init_pipe(pipe, kwargs, override_profile)if server_config.get("enhancer_mode", 0) == 0:setup_prompt_enhancer(pipe, kwargs)@@ -3266,7 +3279,7 @@ def load_models(model_type, override_profile = -1):loras_transformer += ["transformer"]if "transformer2" in pipe:loras_transformer += ["transformer2"]- offloadobj = offload.profile(pipe, profile_no= profile, compile = compile, quantizeTransformer = False, loras = loras_transformer, coTenantsMap= {}, perc_reserved_mem_max = perc_reserved_mem_max , vram_safety_coefficient = vram_safety_coefficient , convertWeightsFloatTo = transformer_dtype, **kwargs)+ offloadobj = offload.profile(pipe, profile_no= profile, compile = compile, quantizeTransformer = False, loras = loras_transformer, perc_reserved_mem_max = perc_reserved_mem_max , vram_safety_coefficient = vram_safety_coefficient , convertWeightsFloatTo = transformer_dtype, **kwargs)if len(args.gpu) > 0:torch.set_default_device(args.gpu)transformer_type = model_type@@ -3472,6 +3485,7 @@ def refresh_gallery(state): #, msgif prompt.startswith("!enhanced!\n"):enhanced = Trueprompt = prompt[len("!enhanced!\n"):]+ prompt = html.escape(prompt)if "\n" in prompt :prompts = prompt.split("\n")window_no= gen.get("window_no",1)@@ -3515,8 +3529,8 @@ def refresh_gallery(state): #, msg"""if params.get("mode", None) in ['edit'] : onemorewindow_visible = Falsegen_buttons_visible = True- html = f"<TABLE WIDTH=100% ID=PINFO style='{table_style}'><TR style='height:140px'><TD width=100% style='{table_style}'>" + prompt + "</TD>" + thumbnails + "</TR></TABLE>"- html_output = gr.HTML(html, visible= True)+ html_content = f"<TABLE WIDTH=100% ID=PINFO style='{table_style}'><TR style='height:140px'><TD width=100% style='{table_style}'>" + prompt + "</TD>" + thumbnails + "</TR></TABLE>"+ html_output = gr.HTML(html_content, visible= True)if last_was_audio:audio_choice = max(0, audio_choice)else:@@ -3702,7 +3716,7 @@ def select_video(state, current_gallery_tab, input_file_list, file_selected, audvalues +=[video_creation_date]labels +=["Creation Date"]else:- video_prompt = configs.get("prompt", "")[:1024]+ video_prompt = html.escape(configs.get("prompt", "")[:1024])video_video_prompt_type = configs.get("video_prompt_type", "")video_image_prompt_type = configs.get("image_prompt_type", "")video_audio_prompt_type = configs.get("audio_prompt_type", "")@@ -3874,11 +3888,11 @@ def select_video(state, current_gallery_tab, input_file_list, file_selected, aud</STYLE>"""rows = [f"<TR><TD style='text-align: right;' WIDTH=1% NOWRAP VALIGN=TOP>{label}</TD><TD><B>{value}</B></TD></TR>" for label, value in zip(labels, values)]- html = f"{table_style}<TABLE ID=video_info WIDTH=100%>" + "".join(rows) + "</TABLE>"+ html_content = f"{table_style}<TABLE ID=video_info WIDTH=100%>" + "".join(rows) + "</TABLE>"else:- html = get_default_video_info()+ html_content = get_default_video_info()visible= len(files) > 0- return choice if source=="video" else gr.update(), html, gr.update(visible=visible and is_video) , gr.update(visible=visible and is_image), gr.update(visible=visible and is_audio), gr.update(visible=visible and is_video) , gr.update(visible=visible and is_video)+ return choice if source=="video" else gr.update(), html_content, gr.update(visible=visible and is_video) , gr.update(visible=visible and is_image), gr.update(visible=visible and is_audio), gr.update(visible=visible and is_video) , gr.update(visible=visible and is_video)def convert_image(image):@@ -4399,7 +4413,11 @@ def any_audio_track(model_type):if base_model_type in ["fantasy", "hunyuan_avatar", "hunyuan_custom_audio", "chatterbox"]:return Truemodel_def = get_model_def(model_type)- return model_def.get("multitalk_class", False) if model_def else False+ if not model_def:+ return False+ if model_def.get("returns_audio", False):+ return True+ return model_def.get("multitalk_class", False)def get_available_filename(target_path, video_source, suffix = "", force_extension = None):name, extension = os.path.splitext(os.path.basename(video_source))@@ -5492,6 +5510,7 @@ def generate_video(gen["header_text"] = txtsend_cmd("output")+ generated_audio = Nonetry:samples = wan_model.generate(input_prompt = prompt,@@ -5623,8 +5642,10 @@ def generate_video(if isinstance(samples, dict):overlapped_latents = samples.get("latent_slice", None)BGRA_frames = samples.get("BGRA_frames", None)- samples= samples["x"]- samples = samples.to("cpu")+ generated_audio = samples.get("audio", generated_audio)+ samples = samples.get("x", None)+ if samples is not None:+ samples = samples.to("cpu")clear_gen_cache()offloadobj.unload_all()gc.collect()@@ -5635,13 +5656,14 @@ def generate_video(# file_name = f"{time_flag}_seed{seed}_{sanitize_file_name(save_prompt[:50]).strip()}.mp4"# sample = samples.cpu()# cache_video( tensor=sample[None].clone(), save_file=os.path.join(save_path, file_name), fps=16, nrow=1, normalize=True, value_range=(-1, 1))-if samples == None:abort = Truestate["prompt"] = ""send_cmd("output")else:sample = samples.cpu()+ if generated_audio is not None:+ output_new_audio_data = generated_audioabort = not (is_image or audio_only) and sample.shape[1] < current_video_length# if True: # for testing# torch.save(sample, "output.pt")@@ -5727,17 +5749,17 @@ def generate_video(save_path_tmp = video_path.rsplit('.', 1)[0] + f"_tmp.{container}"save_video( tensor=sample[None], save_file=save_path_tmp, fps=output_fps, nrow=1, normalize=True, value_range=(-1, 1), codec_type = server_config.get("video_output_codec", None), container=container)output_new_audio_temp_filepath = None- new_audio_from_start = reset_control_aligment+ new_audio_added_from_audio_start = reset_control_aligment or generated_audio is not None # if not beginning of audio will be skippedsource_audio_duration = source_video_frames_count / fpsif any_mmaudio:send_cmd("progress", [0, get_latest_status(state,"MMAudio Soundtrack Generation")])from postprocessing.mmaudio.mmaudio import video_to_audiooutput_new_audio_filepath = output_new_audio_temp_filepath = get_available_filename(save_path, f"tmp{time_flag}.wav" )video_to_audio(save_path_tmp, prompt = MMAudio_prompt, negative_prompt = MMAudio_neg_prompt, seed = seed, num_steps = 25, cfg_strength = 4.5, duration= sample.shape[1] /fps, save_path = output_new_audio_filepath, persistent_models = server_config.get("mmaudio_enabled", 0) == 2, audio_file_only = True, verboseLevel = verbose_level)- new_audio_from_start = False+ new_audio_added_from_audio_start = Falseelif audio_source is not None:output_new_audio_filepath = audio_source- new_audio_from_start = True+ new_audio_added_from_audio_start = Trueelif output_new_audio_data is not None:import soundfile as sfoutput_new_audio_filepath = output_new_audio_temp_filepath = get_available_filename(save_path, f"tmp{time_flag}.wav" )@@ -5747,7 +5769,7 @@ def generate_video(else:new_audio_tracks = control_audio_tracks- combine_and_concatenate_video_with_audio_tracks(video_path, save_path_tmp, source_audio_tracks, new_audio_tracks, source_audio_duration, audio_sampling_rate, new_audio_from_start = new_audio_from_start, source_audio_metadata= source_audio_metadata, verbose = verbose_level>=2 )+ combine_and_concatenate_video_with_audio_tracks(video_path, save_path_tmp, source_audio_tracks, new_audio_tracks, source_audio_duration, audio_sampling_rate, new_audio_from_start = new_audio_added_from_audio_start, source_audio_metadata= source_audio_metadata, verbose = verbose_level>=2 )os.remove(save_path_tmp)if output_new_audio_temp_filepath is not None: os.remove(output_new_audio_temp_filepath)@@ -6609,10 +6631,6 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Nonevace = test_vace_module(base_model_type)t2v= test_class_t2v(base_model_type)ltxv = base_model_type in ["ltxv_13B"]- recammaster = base_model_type in ["recam_1.3B"]- phantom = base_model_type in ["phantom_1.3B", "phantom_14B"]- flux = model_family in ["flux"]- hunyuan_video_custom = base_model_type in ["hunyuan_custom", "hunyuan_custom_audio", "hunyuan_custom_edit"]if target == "settings":return inputs@@ -6628,9 +6646,6 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Noneif model_def.get("sample_solvers", None) is None:pop += ["sample_solver"]- # if not (test_class_i2v(base_model_type) or diffusion_forcing or ltxv or recammaster or vace):- # pop += ["image_prompt_type"]-if any_audio_track(base_model_type) or server_config.get("mmaudio_enabled", 0) == 0:pop += ["MMAudio_setting", "MMAudio_prompt", "MMAudio_neg_prompt"]@@ -6670,7 +6685,7 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Noneif not test_any_sliding_window( base_model_type):pop += ["sliding_window_size", "sliding_window_overlap", "sliding_window_overlap_noise", "sliding_window_discard_last_frames", "sliding_window_color_correction_strength"]- if not (base_model_type in ["fantasy"] or model_def.get("multitalk_class", False)):+ if not model_def.get("audio_guidance", False):pop += ["audio_guidance_scale", "speakers_locations"]if not model_def.get("embedded_guidance", False):@@ -7187,7 +7202,7 @@ def load_settings_from_file(state, file_path):# Extract and apply embedded source images from video filesextracted_images = 0if file_path.endswith('.mkv') or file_path.endswith('.mp4'):- extracted_images = extract_and_apply_source_images(file_path, state)+ extracted_images = extract_and_apply_source_images(file_path, configs)if any_audio:gr.Info(f"Settings Loaded from Audio file with prompt '{prompt[:100]}'")elif any_video_or_image_file:@@ -7730,7 +7745,7 @@ def get_image_end_label(multi_prompts_gen_type):def refresh_prompt_labels(state, multi_prompts_gen_type, image_mode):mode_def = get_model_def(state["model_type"])- prompt_label, wizard_prompt_label = get_prompt_labels(multi_prompts_gen_type, image_mode > 0, mode_def("audio_only", False))+ prompt_label, wizard_prompt_label = get_prompt_labels(multi_prompts_gen_type, image_mode > 0, mode_def.get("audio_only", False))return gr.update(label=prompt_label), gr.update(label = wizard_prompt_label), gr.update(label=get_image_end_label(multi_prompts_gen_type))def update_video_guide_outpainting(video_guide_outpainting_value, value, pos):@@ -7922,12 +7937,14 @@ def detect_auto_save_form(state, evt:gr.SelectData):else:return gr.update()-def compute_video_length_label(fps, current_video_length):+def compute_video_length_label(fps, current_video_length, video_length_locked = None):if fps is None:- return f"Number of frames"+ ret = f"Number of frames"else:- return f"Number of frames ({fps} frames = 1s), current duration: {(current_video_length / fps):.1f}s",-+ ret = f"Number of frames ({fps} frames = 1s), current duration: {(current_video_length / fps):.1f}s"+ if video_length_locked is not None:+ ret += ", locked"+ return retdef refresh_video_length_label(state, current_video_length, force_fps, video_guide, video_source):base_model_type = get_base_model_type(state["model_type"])computed_fps = get_computed_fps(force_fps, base_model_type , video_guide, video_source )@@ -8481,16 +8498,15 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonbatch_size = gr.Slider(1, 16, value=ui_defaults.get("batch_size", 1), step=1, label="Number of Images to Generate", visible = image_outputs)if image_outputs:video_length = gr.Slider(1, 9999, value=ui_defaults.get("video_length", 1), step=1, label="Number of frames", visible = False)- elif recammaster:- video_length = gr.Slider(5, 193, value=ui_defaults.get("video_length", get_max_frames(81)), step=4, label="Number of frames (16 = 1s), locked", interactive= False, visible = True)else:+ video_length_locked = model_def.get("video_length_locked", None)min_frames, frames_step, _ = get_model_min_frames_and_step(base_model_type)- current_video_length = ui_defaults.get("video_length", 81 if get_model_family(base_model_type)=="wan" else 97)+ current_video_length = video_length_locked if video_length_locked is not None else ui_defaults.get("video_length", 81 if get_model_family(base_model_type)=="wan" else 97)computed_fps = get_computed_fps(ui_defaults.get("force_fps",""), base_model_type , ui_defaults.get("video_guide", None), ui_defaults.get("video_source", None))video_length = gr.Slider(min_frames, get_max_frames(737 if test_any_sliding_window(base_model_type) else 337), value=current_video_length,- step=frames_step, label=compute_video_length_label(computed_fps, current_video_length) , visible = True, interactive= True)+ step=frames_step, label=compute_video_length_label(computed_fps, current_video_length, video_length_locked) , visible = True, interactive= video_length_locked is None)with gr.Row(visible = not lock_inference_steps) as inference_steps_row:num_inference_steps = gr.Slider(1, 100, value=ui_defaults.get("num_inference_steps",30), step=1, label="Number of Inference Steps", visible = True)@@ -8500,7 +8516,6 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonwith gr.Tabs(visible=advanced_ui) as advanced_row:guidance_max_phases = model_def.get("guidance_max_phases", 0)no_negative_prompt = model_def.get("no_negative_prompt", False)- any_audio_guidance = fantasy or multitalkwith gr.Tab("General"):with gr.Column():with gr.Row():@@ -8534,6 +8549,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonguidance2_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("guidance2_scale",5), step=0.5, label="Guidance2 (CFG)", visible= guidance_max_phases >=2 and guidance_phases_value >= 2)guidance3_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("guidance3_scale",5), step=0.5, label="Guidance3 (CFG)", visible= guidance_max_phases >=3 and guidance_phases_value >= 3)+ any_audio_guidance = model_def.get("audio_guidance", False)any_embedded_guidance = model_def.get("embedded_guidance", False)with gr.Row(visible =any_embedded_guidance or any_audio_guidance) as embedded_guidance_row:audio_guidance_scale = gr.Slider(1.0, 20.0, value=ui_defaults.get("audio_guidance_scale", 4), step=0.5, label="Audio Guidance", visible= any_audio_guidance )@@ -8790,7 +8806,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonsliding_window_discard_last_frames = gr.Slider(0, 20, value=ui_defaults.get("sliding_window_discard_last_frames", 0), step=4, label="Discard Last Frames of a Window (that may have bad quality)", visible = True)else: # Vace, Multitalksliding_window_defaults = model_def.get("sliding_window_defaults", {})- sliding_window_size = gr.Slider(5, get_max_frames(257), value=ui_defaults.get("sliding_window_size", 129), step=4, label="Sliding Window Size")+ sliding_window_size = gr.Slider(5, get_max_frames(257), value=ui_defaults.get("sliding_window_size", 129), step=4, label="Sliding Window Size", interactive=not model_def.get("sliding_window_size_locked"))sliding_window_overlap = gr.Slider(sliding_window_defaults.get("overlap_min", 1), sliding_window_defaults.get("overlap_max", 97), value=ui_defaults.get("sliding_window_overlap",sliding_window_defaults.get("overlap_default", 5)), step=sliding_window_defaults.get("overlap_step", 4), label="Windows Frames Overlap (needed to maintain continuity between windows, a higher value will require more windows)")sliding_window_color_correction_strength = gr.Slider(0, 1, value=ui_defaults.get("sliding_window_color_correction_strength",0), step=0.01, label="Color Correction Strength (match colors of new window with previous one, 0 = disabled)", visible = True)sliding_window_overlap_noise = gr.Slider(0, 150, value=ui_defaults.get("sliding_window_overlap_noise",20 if vace else 0), step=1, label="Noise to be added to overlapped frames to reduce blur effect" , visible = vace)