Python
Added Hunyuan Custom Audio and Edit support
DeepBeepMeep43aa414eafd4
diff --git a/README.md b/README.mdindex ac8705e..1eb93db 100644--- a/README.md+++ b/README.md@@ -20,11 +20,16 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models**Follow DeepBeepMeep on Twitter/X to get the Latest News**: https://x.com/deepbeepmeep## π₯ Latest Updates-### May 28 2025: WanGP v5.41+### June 11 2025: WanGP v5.5+π *Hunyuan Video Custom Audio*: it is similar to Hunyuan Video Avatar except there isn't any lower limit on the number of frames and you can use your reference images in a different context than the image itself\+*Hunyuan Video Custom Edit*: Hunyuan Video Controlnet, use it to do inpainting and replace a person in a video while still keeping his poses. Similar to Vace but less restricted than the Wan models in terms of content...+++### June 6 2025: WanGP v5.41π Bonus release: Support for **AccVideo** Lora to speed up x2 Video generations in Wan models. Check the Loras documentation to get the usage instructions of AccVideo.\You will need to do a *pip install -r requirements.txt*-### May 28 2025: WanGP v5.4+### June 6 2025: WanGP v5.4π World Exclusive : **Hunyuan Video Avatar** Support ! You won't need 80 GB of VRAM nor 32 GB oF VRAM, just 10 GB of VRAM will be sufficient to generate up to 15s of high quality speech / song driven Video at a high speed with no quality degradation. Support for TeaCache included.\Here is a link to the original repo where you will find some very interesting documentation and examples. https://github.com/Tencent-Hunyuan/HunyuanVideo-Avatar. Kudos to the Hunyuan Video Avatar team for the best model of its kind.\Also many thanks to Reevoy24 for his repackaging / completing the documentationdiff --git a/docs/CHANGELOG.md b/docs/CHANGELOG.mdindex c2f1488..dc15df2 100644--- a/docs/CHANGELOG.md+++ b/docs/CHANGELOG.md@@ -1,10 +1,14 @@# Changelog## π₯ Latest News-### May 28 2025: WanGP v5.41+### June 11 2025: WanGP v5.5+π *Hunyuan Video Custom Audio*: it is similar to Hunyuan Video Avatar excpet there isn't any lower limit on the number of frames and you can use your reference images in a different context than the image itself\+*Hunyuan Video Custom Edit*: Hunyuan Video Controlnet, use it to do inpainting and replace a person in a video while still keeping his poses. Similar to Vace but less restricted than the Wan models in terms of content...++### June 6 2025: WanGP v5.41π Bonus release: Support for **AccVideo** Lora to speed up x2 Video generations in Wan models. Check the Loras documentation to get the usage instructions of AccVideo.-### May 28 2025: WanGP v5.4+### June 6 2025: WanGP v5.4π World Exclusive : Hunyuan Video Avatar Support ! You won't need 80 GB of VRAM nor 32 GB oF VRAM, just 10 GB of VRAM will be sufficient to generate up to 15s of high quality speech / song driven Video at a high speed with no quality degradation. Support for TeaCache included.### May 26, 2025: WanGP v5.3diff --git a/hyvideo/data_kits/audio_preprocessor.py b/hyvideo/data_kits/audio_preprocessor.pyindex 89ac470..be4ce1b 100644--- a/hyvideo/data_kits/audio_preprocessor.py+++ b/hyvideo/data_kits/audio_preprocessor.py@@ -54,6 +54,10 @@ def encode_audio(wav2vec, audio_feats, fps, num_frames=129):elif fps == 12.5:start_ts = [0]step_ts = [2]+ else:+ start_ts = [0]+ step_ts = [1]+num_frames = min(num_frames, 400)audio_feats = wav2vec.encoder(audio_feats.unsqueeze(0)[:, :, :3000], output_hidden_states=True).hidden_statesaudio_feats = torch.stack(audio_feats, dim=2)diff --git a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.pyindex f7339e5..5ee7317 100644--- a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py+++ b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video.py@@ -774,7 +774,10 @@ class HunyuanVideoPipeline(DiffusionPipeline):# uncond_ref_latents: Optional[torch.Tensor] = None,pixel_value_llava: Optional[torch.Tensor] = None,uncond_pixel_value_llava: Optional[torch.Tensor] = None,+ bg_latents: Optional[torch.Tensor] = None,+ audio_prompts: Optional[torch.Tensor] = None,ip_cfg_scale: float = 0.0,+ audio_strength: float = 1.0,use_deepcache: int = 1,num_videos_per_prompt: Optional[int] = 1,eta: float = 0.0,@@ -922,6 +925,9 @@ class HunyuanVideoPipeline(DiffusionPipeline):# "Passing `callback_steps` as an input argument to `__call__` is deprecated, consider using `callback_on_step_end`",# )++ if self._interrupt:+ return [None]if isinstance(callback_on_step_end, (PipelineCallback, MultiPipelineCallbacks)):callback_on_step_end_tensor_inputs = callback_on_step_end.tensor_inputs@@ -968,7 +974,6 @@ class HunyuanVideoPipeline(DiffusionPipeline):self._guidance_rescale = guidance_rescaleself._clip_skip = clip_skipself._cross_attention_kwargs = cross_attention_kwargs- self._interrupt = False# 2. Define call parametersif prompt is not None and isinstance(prompt, str):@@ -1057,6 +1062,12 @@ class HunyuanVideoPipeline(DiffusionPipeline):prompt_mask[0] = torch.cat([torch.ones((1, prompt_mask[0].sum() - 575)).to(prompt_mask),torch.zeros((1, prompt_mask.shape[1] - prompt_mask[0].sum() + 575)).to(prompt_mask)], dim=1)+ if bg_latents is not None:+ bg_latents = torch.cat([bg_latents, bg_latents], dim=0)++ if audio_prompts is not None:+ audio_prompts = torch.cat([torch.zeros_like(audio_prompts), audio_prompts], dim=0)+if ip_cfg_scale>0:prompt_embeds = torch.cat([prompt_embeds, prompt_embeds[1:]])prompt_embeds_2 = torch.cat([prompt_embeds_2, prompt_embeds_2[1:]])@@ -1263,6 +1274,9 @@ class HunyuanVideoPipeline(DiffusionPipeline):pipeline=self,x_id=j,step_no=i,+ bg_latents=bg_latents[j].unsqueeze(0) if bg_latents!=None else None,+ audio_prompts=audio_prompts[j].unsqueeze(0) if audio_prompts!=None else None,+ audio_strength=audio_strength,callback = callback,)if self._interrupt:@@ -1292,6 +1306,9 @@ class HunyuanVideoPipeline(DiffusionPipeline):guidance=guidance_expand,pipeline=self,step_no=i,+ bg_latents=bg_latents,+ audio_prompts=audio_prompts,+ audio_strength=audio_strength,callback = callback,)if self._interrupt:diff --git a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video_audio.py b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video_audio.pyindex 8be3c47..80a450c 100644--- a/hyvideo/diffusion/pipelines/pipeline_hunyuan_video_audio.py+++ b/hyvideo/diffusion/pipelines/pipeline_hunyuan_video_audio.py@@ -908,6 +908,10 @@ class HunyuanVideoAudioPipeline(DiffusionPipeline):second element is a list of `bool`s indicating whether the corresponding generated image contains"not-safe-for-work" (nsfw) content."""++ if self._interrupt:+ return [None]+callback = kwargs.pop("callback", None)callback_steps = kwargs.pop("callback_steps", None)if callback_steps is not None:@@ -956,7 +960,6 @@ class HunyuanVideoAudioPipeline(DiffusionPipeline):self._guidance_rescale = guidance_rescaleself._clip_skip = clip_skipself._cross_attention_kwargs = cross_attention_kwargs- self._interrupt = False# 2. Define call parametersif prompt is not None and isinstance(prompt, str):diff --git a/hyvideo/hunyuan.py b/hyvideo/hunyuan.pyindex 19a147e..81bd77a 100644--- a/hyvideo/hunyuan.py+++ b/hyvideo/hunyuan.py@@ -235,13 +235,13 @@ def patched_llava_forward(image_hidden_states=image_features if pixel_values is not None else None,)-def adapt_avatar_model(model):+def adapt_model(model, audio_block_name):modules_dict= { k: m for k, m in model.named_modules()}for model_layer, avatar_layer in model.double_stream_map.items():- module = modules_dict[f"audio_adapter_blocks.{avatar_layer}"]+ module = modules_dict[f"{audio_block_name}.{avatar_layer}"]target = modules_dict[f"double_blocks.{model_layer}"]setattr(target, "audio_adapter", module )- delattr(model, "audio_adapter_blocks")+ delattr(model, audio_block_name)class DataPreprocess(object):def __init__(self):@@ -329,17 +329,25 @@ class Inference(object):precision = "bf16"vae_precision = "fp32" if VAE_dtype == torch.float32 else "bf16"embedded_cfg_scale = 6+ filepath = model_filepath[0]i2v_condition_type = None- i2v_mode = "i2v" in model_filepath[0]+ i2v_mode = "i2v" in filepathcustom = False+ custom_audio = Falseavatar = Falseif i2v_mode:model_id = "HYVideo-T/2"i2v_condition_type = "token_replace"- elif "custom" in model_filepath[0]:- model_id = "HYVideo-T/2-custom"+ elif "custom" in filepath:+ if "audio" in filepath:+ model_id = "HYVideo-T/2-custom-audio"+ custom_audio = True+ elif "edit" in filepath:+ model_id = "HYVideo-T/2-custom-edit"+ else:+ model_id = "HYVideo-T/2-custom"custom = True- elif "avatar" in model_filepath[0]:+ elif "avatar" in filepath :model_id = "HYVideo-T/2-avatar"text_len = 256avatar = True@@ -376,11 +384,11 @@ class Inference(object):from mmgp import offload# model = Inference.load_state_dict(args, model, model_filepath)- # model_filepath ="c:/temp/avatar/mp_rank_00_model_states.pt"+ # model_filepath ="c:/temp/hc/mp_rank_00_model_states_video.pt"offload.load_model_data(model, model_filepath, pinToMemory = pinToMemory, partialPinning = partialPinning)pass- # offload.save_model(model, "hunyuan_video_avatar_720_bf16.safetensors")- # offload.save_model(model, "hunyuan_video_avatar_720_quanto_bf16_int8.safetensors", do_quantize= True)+ # offload.save_model(model, "hunyuan_video_avatar_edit_720_bf16.safetensors")+ # offload.save_model(model, "hunyuan_video_avatar_edit_720_quanto_bf16_int8.safetensors", do_quantize= True)model.mixed_precision = mixed_precision_transformer@@ -472,15 +480,17 @@ class Inference(object):wav2vec = Nonealign_instance = None- if avatar:+ if avatar or custom_audio:feature_extractor = AutoFeatureExtractor.from_pretrained("ckpts/whisper-tiny/")wav2vec = WhisperModel.from_pretrained("ckpts/whisper-tiny/").to(device="cpu", dtype=torch.float32)wav2vec._model_dtype = torch.float32wav2vec.requires_grad_(False)+ if avatar:align_instance = AlignImage("cuda", det_path="ckpts/det_align/detface.pt")align_instance.facedet.model.to("cpu")-- adapt_avatar_model(model)+ adapt_model(model, "audio_adapter_blocks")+ elif custom_audio:+ adapt_model(model, "audio_models")return cls(i2v=i2v_mode,@@ -600,7 +610,7 @@ class HunyuanVideoSampler(Inference):return pipeline- def get_rotary_pos_embed_new(self, video_length, height, width, concat_dict={}):+ def get_rotary_pos_embed_new(self, video_length, height, width, concat_dict={}, enable_riflex = False):target_ndim = 3ndim = 5 - 2latents_size = [(video_length-1)//4+1 , height//8, width//8]@@ -628,7 +638,10 @@ class HunyuanVideoSampler(Inference):theta=256,use_real=True,theta_rescale_factor=1,- concat_dict=concat_dict)+ concat_dict=concat_dict,+ L_test = (video_length - 1) // 4 + 1,+ enable_riflex = enable_riflex+ )return freqs_cos, freqs_sindef get_rotary_pos_embed(self, video_length, height, width, enable_riflex = False):@@ -687,6 +700,9 @@ class HunyuanVideoSampler(Inference):input_prompt,input_ref_images = None,audio_guide = None,+ input_frames = None,+ input_masks = None,+ input_video = None,fps = 24,height=192,width=336,@@ -701,13 +717,14 @@ class HunyuanVideoSampler(Inference):num_videos_per_prompt=1,i2v_resolution="720p",image_start=None,- enable_riflex = False,+ enable_RIFLEx = False,i2v_condition_type: str = "token_replace",i2v_stability=True,VAE_tile_size = None,joint_pass = False,cfg_star_switch = False,fit_into_canvas = True,+ conditioning_latents_size = 0,**kwargs,):@@ -777,6 +794,7 @@ class HunyuanVideoSampler(Inference):target_height = align_to(height, 16)target_width = align_to(width, 16)target_frame_num = frame_num+ audio_strength = 1if input_ref_images != None:# ip_cfg_scale = 3.0@@ -862,7 +880,7 @@ class HunyuanVideoSampler(Inference):# ========================================================================if input_ref_images == None:- freqs_cos, freqs_sin = self.get_rotary_pos_embed(target_frame_num, target_height, target_width, enable_riflex)+ freqs_cos, freqs_sin = self.get_rotary_pos_embed(target_frame_num, target_height, target_width, enable_RIFLEx)else:if self.avatar:w, h = input_ref_images.size@@ -873,8 +891,13 @@ class HunyuanVideoSampler(Inference):concat_dict = {'mode': 'timecat', 'bias': -1}freqs_cos, freqs_sin = self.get_rotary_pos_embed_new(129, target_height, target_width, concat_dict)else:+ if input_frames != None:+ target_height, target_width = input_frames.shape[-3:-1]+ elif input_video != None:+ target_height, target_width = input_video.shape[-2:]+concat_dict = {'mode': 'timecat-w', 'bias': -1}- freqs_cos, freqs_sin = self.get_rotary_pos_embed_new(target_frame_num, target_height, target_width, concat_dict)+ freqs_cos, freqs_sin = self.get_rotary_pos_embed_new(target_frame_num, target_height, target_width, concat_dict, enable_RIFLEx)n_tokens = freqs_cos.shape[0]@@ -892,7 +915,35 @@ class HunyuanVideoSampler(Inference):ref_latents, uncond_audio_prompts, audio_prompts, face_masks, motion_exp, motion_pose = None, None, None, None, None, None- if audio_guide != None:++ bg_latents = None+ if input_video != None:+ pixel_value_bg = input_video.unsqueeze(0)+ pixel_value_mask = torch.zeros_like(input_video).unsqueeze(0)+ if input_frames != None:+ pixel_value_video_bg = input_frames.permute(-1,0,1,2).unsqueeze(0).float()+ pixel_value_video_mask = input_masks.unsqueeze(-1).repeat(1,1,1,3).permute(-1,0,1,2).unsqueeze(0).float()+ pixel_value_video_bg = pixel_value_video_bg.div_(127.5).add_(-1.)+ if input_video != None:+ pixel_value_bg = torch.cat([pixel_value_bg, pixel_value_video_bg], dim=2)+ pixel_value_mask = torch.cat([ pixel_value_mask, pixel_value_video_mask], dim=2)+ else:+ pixel_value_bg = pixel_value_video_bg+ pixel_value_mask = pixel_value_video_mask+ pixel_value_video_mask, pixel_value_video_bg = None, None+ if input_video != None or input_frames != None:+ if pixel_value_bg.shape[2] < frame_num:+ padding_shape = list(pixel_value_bg.shape[0:2]) + [frame_num-pixel_value_bg.shape[2]] + list(pixel_value_bg.shape[3:])+ pixel_value_bg = torch.cat([pixel_value_bg, torch.full(padding_shape, -1, dtype=pixel_value_bg.dtype, device= pixel_value_bg.device ) ], dim=2)+ pixel_value_mask = torch.cat([ pixel_value_mask, torch.full(padding_shape, 255, dtype=pixel_value_mask.dtype, device= pixel_value_mask.device ) ], dim=2)++ bg_latents = self.vae.encode(pixel_value_bg).latent_dist.sample()+ pixel_value_mask = pixel_value_mask.div_(127.5).add_(-1.)+ mask_latents = self.vae.encode(pixel_value_mask).latent_dist.sample()+ bg_latents = torch.cat([bg_latents, mask_latents], dim=1)+ bg_latents.mul_(self.vae.config.scaling_factor)++ if self.avatar:if n_prompt == None or len(n_prompt) == 0:n_prompt = "Aerial view, aerial view, overexposed, low quality, deformation, a poor composition, bad hands, bad teeth, bad eyes, bad limbs, distortion, blurring, Lens changes"@@ -930,28 +981,33 @@ class HunyuanVideoSampler(Inference):# from wan.utils.utils import cache_video# cache_video( tensor=image, save_file="decode.mp4", fps=25, nrow=1, normalize=True, value_range=(-1, 1))+ motion_pose = np.array([25] * 4)+ motion_exp = np.array([30] * 4)+ motion_pose = torch.from_numpy(motion_pose).unsqueeze(0)+ motion_exp = torch.from_numpy(motion_exp).unsqueeze(0)face_masks = torch.nn.functional.interpolate(face_masks.float().squeeze(2),(ref_latents.shape[-2],ref_latents.shape[-1]),mode="bilinear").unsqueeze(2).to(dtype=ref_latents.dtype)-+++ if audio_guide != None:audio_input, audio_len = get_audio_feature(self.feature_extractor, audio_guide, duration = frame_num/fps )audio_prompts = audio_input[0]weight_dtype = audio_prompts.dtype-- motion_pose = np.array([25] * 4)- motion_exp = np.array([30] * 4)- motion_pose = torch.from_numpy(motion_pose).unsqueeze(0)- motion_exp = torch.from_numpy(motion_exp).unsqueeze(0)+ if self.custom:+ audio_len = min(audio_len, frame_num)+ audio_input = audio_input[:, :audio_len]audio_prompts = encode_audio(self.wav2vec, audio_prompts.to(dtype=self.wav2vec.dtype), fps, num_frames=audio_len)audio_prompts = audio_prompts.to(self.model.dtype)- if audio_prompts.shape[1] <= 129:- audio_prompts = torch.cat([audio_prompts, torch.zeros_like(audio_prompts[:, :1]).repeat(1,129-audio_prompts.shape[1], 1, 1, 1)], dim=1)+ segment_size = 129 if self.avatar else frame_num+ if audio_prompts.shape[1] <= segment_size:+ audio_prompts = torch.cat([audio_prompts, torch.zeros_like(audio_prompts[:, :1]).repeat(1,segment_size-audio_prompts.shape[1], 1, 1, 1)], dim=1)else:audio_prompts = torch.cat([audio_prompts, torch.zeros_like(audio_prompts[:, :1]).repeat(1, 5, 1, 1, 1)], dim=1)uncond_audio_prompts = torch.zeros_like(audio_prompts[:,:129])- # target_frame_num = min(target_frame_num, audio_len)+samples = self.pipeline(prompt=input_prompt,height=target_height,@@ -976,6 +1032,9 @@ class HunyuanVideoSampler(Inference):motion_pose=motion_pose,fps= torch.from_numpy(np.array(fps)),+ bg_latents = bg_latents,+ audio_strength = audio_strength,+denoise_strength=denoise_strength,ip_cfg_scale=ip_cfg_scale,freqs_cis=(freqs_cos, freqs_sin),diff --git a/hyvideo/modules/models.py b/hyvideo/modules/models.pyindex 5e76a2a..65075cc 100644--- a/hyvideo/modules/models.py+++ b/hyvideo/modules/models.py@@ -591,7 +591,10 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):dtype: Optional[torch.dtype] = None,device: Optional[torch.device] = None,attention_mode: Optional[str] = "sdpa",+ video_condition: bool = False,+ audio_condition: bool = False,avatar = False,+ custom = False,):factory_kwargs = {"device": device, "dtype": dtype}super().__init__()@@ -606,7 +609,11 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):self.rope_dim_list = rope_dim_listself.i2v_condition_type = i2v_condition_typeself.attention_mode = attention_mode-+ self.video_condition = video_condition+ self.audio_condition = audio_condition+ self.avatar = avatar+ self.custom = custom+# Text projection. Default to linear projection.# Alternative: TokenRefiner. See more details (LI-DiT): http://arxiv.org/abs/2406.11831self.use_attention_mask = use_attention_mask@@ -710,44 +717,56 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):get_activation_layer("silu"),**factory_kwargs,)- avatar_audio = avatar- if avatar_audio:- self.ref_in = PatchEmbed(- self.patch_size, self.in_channels, self.hidden_size, **factory_kwargs- )- # -------------------- audio_proj_model --------------------- self.audio_proj = AudioProjNet2(seq_len=10, blocks=5, channels=384, intermediate_dim=1024, output_dim=3072, context_tokens=4)-- # -------------------- motion-embeder --------------------- self.motion_exp = TimestepEmbedder(- self.hidden_size // 4,- get_activation_layer("silu"),- **factory_kwargs- )- self.motion_pose = TimestepEmbedder(- self.hidden_size // 4,- get_activation_layer("silu"),- **factory_kwargs- )-- self.fps_proj = TimestepEmbedder(- self.hidden_size,- get_activation_layer("silu"),- **factory_kwargs- )-- self.before_proj = nn.Linear(self.hidden_size, self.hidden_size)+ if self.video_condition:+ self.bg_in = PatchEmbed(+ self.patch_size, self.in_channels * 2, self.hidden_size, **factory_kwargs+ )+ self.bg_proj = nn.Linear(self.hidden_size, self.hidden_size)++ if audio_condition:+ if avatar:+ self.ref_in = PatchEmbed(+ self.patch_size, self.in_channels, self.hidden_size, **factory_kwargs+ )++ # -------------------- audio_proj_model --------------------+ self.audio_proj = AudioProjNet2(seq_len=10, blocks=5, channels=384, intermediate_dim=1024, output_dim=3072, context_tokens=4)++ # -------------------- motion-embeder --------------------+ self.motion_exp = TimestepEmbedder(+ self.hidden_size // 4,+ get_activation_layer("silu"),+ **factory_kwargs+ )+ self.motion_pose = TimestepEmbedder(+ self.hidden_size // 4,+ get_activation_layer("silu"),+ **factory_kwargs+ )++ self.fps_proj = TimestepEmbedder(+ self.hidden_size,+ get_activation_layer("silu"),+ **factory_kwargs+ )++ self.before_proj = nn.Linear(self.hidden_size, self.hidden_size)++ # -------------------- audio_insert_model --------------------+ self.double_stream_list = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]+ audio_block_name = "audio_adapter_blocks"+ elif custom:+ self.audio_proj = AudioProjNet2(seq_len=10, blocks=5, channels=384, intermediate_dim=1024, output_dim=3072, context_tokens=4)+ self.double_stream_list = [1, 3, 5, 7, 9, 11]+ audio_block_name = "audio_models"- # -------------------- audio_insert_model --------------------- self.double_stream_list = [1, 3, 5, 7, 9, 11, 13, 15, 17, 19]- self.single_stream_list = []self.double_stream_map = {str(i): j for j, i in enumerate(self.double_stream_list)}+ self.single_stream_list = []self.single_stream_map = {str(i): j+len(self.double_stream_list) for j, i in enumerate(self.single_stream_list)}-- self.audio_adapter_blocks = nn.ModuleList([+ setattr(self, audio_block_name, nn.ModuleList([PerceiverAttentionCA(dim=3072, dim_head=1024, heads=33) for _ in range(len(self.double_stream_list) + len(self.single_stream_list))- ])+ ]))@@ -798,6 +817,8 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):motion_pose = None,fps = None,face_mask = None,+ audio_strength = None,+ bg_latents = None,) -> Union[torch.Tensor, Dict[str, torch.Tensor]]:img = x@@ -854,12 +875,16 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):# Embed image and text.img, shape_mask = self.img_in(img)- if audio_prompts != None:+ if self.avatar:ref_latents_first = ref_latents[:, :, :1].clone()ref_latents,_ = self.ref_in(ref_latents)ref_latents_first,_ = self.img_in(ref_latents_first)- elif ref_latents != None:- ref_latents, _ = self.img_in(ref_latents)+ elif self.custom:+ if ref_latents != None:+ ref_latents, _ = self.img_in(ref_latents)+ if bg_latents is not None and self.video_condition:+ bg_latents, _ = self.bg_in(bg_latents)+ img += self.bg_proj(bg_latents)if self.text_projection == "linear":txt = self.txt_in(txt)@@ -870,7 +895,7 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):f"Unsupported text_projection: {self.text_projection}")- if audio_prompts != None:+ if self.avatar:img += self.before_proj(ref_latents)ref_length = ref_latents_first.shape[-2] # [b s c]img = torch.cat([ref_latents_first, img], dim=-2) # t c@@ -956,7 +981,10 @@ class HYVideoDiffusionTransformer(ModelMixin, ConfigMixin):if audio_adapter != None:real_img = img[i:i+1,ref_length:].view(1, ot, -1, 3072)real_img = audio_adapter(audio_feature_all_insert[i:i+1], real_img).view(1, -1, 3072)- real_img *= face_mask[i:i+1]+ if face_mask != None:+ real_img *= face_mask[i:i+1]+ if audio_strength != None and audio_strength != 1:+ real_img *= audio_strengthimg[i:i+1, ref_length:] += real_imgreal_img = None@@ -1095,6 +1123,27 @@ HUNYUAN_VIDEO_CONFIG = {"hidden_size": 3072,"heads_num": 24,"mlp_width_ratio": 4,+ 'custom' : True+ },+ 'HYVideo-T/2-custom-audio': { # 9.0B / 12.5B+ "mm_double_blocks_depth": 20,+ "mm_single_blocks_depth": 40,+ "rope_dim_list": [16, 56, 56],+ "hidden_size": 3072,+ "heads_num": 24,+ "mlp_width_ratio": 4,+ 'custom' : True,+ 'audio_condition' : True,+ },+ 'HYVideo-T/2-custom-edit': { # 9.0B / 12.5B+ "mm_double_blocks_depth": 20,+ "mm_single_blocks_depth": 40,+ "rope_dim_list": [16, 56, 56],+ "hidden_size": 3072,+ "heads_num": 24,+ "mlp_width_ratio": 4,+ 'custom' : True,+ 'video_condition' : True,},'HYVideo-T/2-avatar': { # 9.0B / 12.5B'mm_double_blocks_depth': 20,@@ -1104,6 +1153,7 @@ HUNYUAN_VIDEO_CONFIG = {'heads_num': 24,'mlp_width_ratio': 4,'avatar': True,+ 'audio_condition' : True,},}\ No newline at end of filediff --git a/hyvideo/modules/posemb_layers.py b/hyvideo/modules/posemb_layers.pyindex 3428568..4d17a10 100644--- a/hyvideo/modules/posemb_layers.py+++ b/hyvideo/modules/posemb_layers.py@@ -295,7 +295,10 @@ def apply_rotary_emb( qklist,def get_nd_rotary_pos_embed_new(rope_dim_list, start, *args, theta=10000., use_real=False,theta_rescale_factor: Union[float, List[float]]=1.0,interpolation_factor: Union[float, List[float]]=1.0,- concat_dict={}+ concat_dict={},+ k = 4,+ L_test = 66,+ enable_riflex = True):grid = get_meshgrid_nd(start, *args, dim=len(rope_dim_list)) # [3, W, H, D] / [2, W, H]@@ -327,9 +330,17 @@ def get_nd_rotary_pos_embed_new(rope_dim_list, start, *args, theta=10000., use_r# use 1/ndim of dimensions to encode grid_axisembs = []for i in range(len(rope_dim_list)):- emb = get_1d_rotary_pos_embed(rope_dim_list[i], grid[i].reshape(-1), theta, use_real=use_real,- theta_rescale_factor=theta_rescale_factor[i],- interpolation_factor=interpolation_factor[i]) # 2 x [WHD, rope_dim_list[i]]+ # === RIFLEx modification start ===+ # apply RIFLEx for time dimension+ if i == 0 and enable_riflex:+ emb = get_1d_rotary_pos_embed_riflex(rope_dim_list[i], grid[i].reshape(-1), theta, use_real=True, k=k, L_test=L_test)+ # === RIFLEx modification end ===+ else:+ emb = get_1d_rotary_pos_embed(rope_dim_list[i], grid[i].reshape(-1), theta, use_real=True, theta_rescale_factor=theta_rescale_factor[i],interpolation_factor=interpolation_factor[i],)++ # emb = get_1d_rotary_pos_embed(rope_dim_list[i], grid[i].reshape(-1), theta, use_real=use_real,+ # theta_rescale_factor=theta_rescale_factor[i],+ # w interpolation_factor=interpolation_factor[i]) # 2 x [WHD, rope_dim_list[i]]embs.append(emb)diff --git a/preprocessing/matanyone/app.py b/preprocessing/matanyone/app.pyindex 4a5f988..c367101 100644--- a/preprocessing/matanyone/app.py+++ b/preprocessing/matanyone/app.py@@ -517,10 +517,14 @@ def export_image(image_refs, image_output):image_refs.append( image_output)return image_refs-def export_to_current_video_engine(foreground_video_output, alpha_video_output):+def export_to_current_video_engine(model_type, foreground_video_output, alpha_video_output):gr.Info("Masked Video Input and Full Mask transferred to Current Video Engine For Inpainting")# return "MV#" + str(time.time()), foreground_video_output, alpha_video_output- return foreground_video_output, alpha_video_output+ if "custom_edit" in model_type:+ return gr.update(), alpha_video_output+ else:+ return foreground_video_output, alpha_video_output+def teleport_to_video_tab():return gr.Tabs(selected="video_gen")@@ -675,7 +679,7 @@ def display(tabs, model_choice, vace_video_input, vace_video_mask, vace_image_reexport_to_vace_video_14B_btn.click( fn=teleport_to_vace_14B, inputs=[], outputs=[tabs, model_choice]).then(fn=export_to_current_video_engine, inputs= [foreground_video_output, alpha_video_output], outputs= [video_prompt_video_guide_trigger, vace_video_input, vace_video_mask])- export_to_current_video_engine_btn.click( fn=export_to_current_video_engine, inputs= [foreground_video_output, alpha_video_output], outputs= [vace_video_input, vace_video_mask]).then( #video_prompt_video_guide_trigger,+ export_to_current_video_engine_btn.click( fn=export_to_current_video_engine, inputs= [model_choice, foreground_video_output, alpha_video_output], outputs= [vace_video_input, vace_video_mask]).then( #video_prompt_video_guide_trigger,fn=teleport_to_video_tab, inputs= [], outputs= [tabs])diff --git a/requirements.txt b/requirements.txtindex 8324d48..c6f5045 100644--- a/requirements.txt+++ b/requirements.txt@@ -33,4 +33,5 @@ librosalogurusentencepieceav+opencv-python# rembg==2.0.65diff --git a/wan/utils/utils.py b/wan/utils/utils.pyindex 32bb79c..3b32d08 100644--- a/wan/utils/utils.py+++ b/wan/utils/utils.py@@ -46,7 +46,8 @@ def resample(video_fps, video_frames_count, max_target_frames_count, target_fps,while True:if max_target_frames_count != 0 and len(frame_ids) >= max_target_frames_count :break- add_frames_count = math.ceil( (target_time -cur_time) / video_frame_duration )+ diff = round( (target_time -cur_time) / video_frame_duration , 5)+ add_frames_count = math.ceil( diff)frame_no += add_frames_countif frame_no >= video_frames_count:breakdiff --git a/wgp.py b/wgp.pyindex 427444e..21e8a1f 100644--- a/wgp.py+++ b/wgp.py@@ -33,6 +33,7 @@ import tempfileimport atexitimport shutilimport glob+import cv2from transformers.utils import logginglogging.set_verbosity_error@@ -43,7 +44,7 @@ AUTOSAVE_FILENAME = "queue.zip"PROMPT_VARS_MAX = 10target_mmgp_version = "3.4.8"-WanGP_version = "5.41"+WanGP_version = "5.5"prompt_enhancer_image_caption_model, prompt_enhancer_image_caption_processor, prompt_enhancer_llm_model, prompt_enhancer_llm_tokenizer = None, None, None, Nonefrom importlib.metadata import version@@ -184,12 +185,18 @@ def process_prompt_and_add_tasks(state, model_choice):gr.Info("You must use the 14B model to generate videos with a resolution equivalent to 720P")return- if "diffusion_forcing" in model_filename or "ltxv" in model_filename or "Vace" in model_filename:+ if "diffusion_forcing" in model_filename or "ltxv" in model_filename or "Vace" in model_filename or "hunyuan_video_custom_edit" in model_filename:video_length = inputs["video_length"]sliding_window_size = inputs["sliding_window_size"]if video_length > sliding_window_size:gr.Info(f"The Number of Frames to generate ({video_length}) is greater than the Sliding Window Size ({sliding_window_size}) , multiple Windows will be generated")+ if "hunyuan_video_custom_edit" in model_filename:+ keep_frames_video_guide= inputs["keep_frames_video_guide"]+ if len(keep_frames_video_guide) > 0:+ gr.Info("Filtering Frames with this model is not supported")+ return+if "phantom" in model_filename or "hunyuan_video_custom" in model_filename or "hunyuan_video_avatar" in model_filename:image_refs = inputs["image_refs"]audio_guide = inputs["audio_guide"]@@ -1552,6 +1559,8 @@ ltxv_choices= ["ckpts/ltxv_0.9.7_13B_dev_bf16.safetensors", "ckpts/ltxv_0.9.7_13hunyuan_choices= ["ckpts/hunyuan_video_720_bf16.safetensors", "ckpts/hunyuan_video_720_quanto_int8.safetensors", "ckpts/hunyuan_video_i2v_720_bf16v2.safetensors", "ckpts/hunyuan_video_i2v_720_quanto_int8v2.safetensors","ckpts/hunyuan_video_custom_720_bf16.safetensors", "ckpts/hunyuan_video_custom_720_quanto_bf16_int8.safetensors",+ "ckpts/hunyuan_video_custom_audio_720_bf16.safetensors", "ckpts/hunyuan_video_custom_audio_720_quanto_bf16_int8.safetensors",+ "ckpts/hunyuan_video_custom_edit_720_bf16.safetensors", "ckpts/hunyuan_video_custom_edit_720_quanto_bf16_int8.safetensors","ckpts/hunyuan_video_avatar_720_bf16.safetensors", "ckpts/hunyuan_video_avatar_720_quanto_bf16_int8.safetensors",]@@ -1563,13 +1572,14 @@ def get_dependent_models(model_filename, quantization, dtype_policy ):return [get_model_filename("ltxv_13B", quantization, dtype_policy)]else:return []-model_types = [ "t2v_1.3B", "t2v", "i2v", "i2v_720p", "flf2v_720p", "vace_1.3B","vace_14B","moviigen", "phantom_1.3B", "phantom_14B", "fantasy", "fun_inp_1.3B", "fun_inp", "recam_1.3B", "sky_df_1.3B", "sky_df_14B", "sky_df_720p_14B", "ltxv_13B", "ltxv_13B_distilled", "hunyuan", "hunyuan_i2v", "hunyuan_custom", "hunyuan_avatar"]+model_types = [ "t2v_1.3B", "t2v", "i2v", "i2v_720p", "flf2v_720p", "vace_1.3B","vace_14B","moviigen", "phantom_1.3B", "phantom_14B", "fantasy", "fun_inp_1.3B", "fun_inp", "recam_1.3B", "sky_df_1.3B", "sky_df_14B", "sky_df_720p_14B", "ltxv_13B", "ltxv_13B_distilled", "hunyuan", "hunyuan_i2v", "hunyuan_custom", "hunyuan_custom_audio", "hunyuan_custom_edit", "hunyuan_avatar"]model_signatures = {"t2v": "text2video_14B", "t2v_1.3B" : "text2video_1.3B", "fun_inp_1.3B" : "Fun_InP_1.3B", "fun_inp" : "Fun_InP_14B","i2v" : "image2video_480p", "i2v_720p" : "image2video_720p" , "vace_1.3B" : "Vace_1.3B", "vace_14B" : "Vace_14B","recam_1.3B": "recammaster_1.3B","flf2v_720p" : "FLF2V_720p", "sky_df_1.3B" : "sky_reels2_diffusion_forcing_1.3B", "sky_df_14B" : "sky_reels2_diffusion_forcing_14B","sky_df_720p_14B" : "sky_reels2_diffusion_forcing_720p_14B", "moviigen" :"moviigen","phantom_1.3B" : "phantom_1.3B", "phantom_14B" : "phantom_14B", "fantasy" : "fantasy", "ltxv_13B" : "ltxv_0.9.7_13B_dev", "ltxv_13B_distilled" : "ltxv_0.9.7_13B_distilled",- "hunyuan" : "hunyuan_video_720", "hunyuan_i2v" : "hunyuan_video_i2v_720", "hunyuan_custom" : "hunyuan_video_custom", "hunyuan_avatar" : "hunyuan_video_avatar" }+ "hunyuan" : "hunyuan_video_720", "hunyuan_i2v" : "hunyuan_video_i2v_720", "hunyuan_custom" : "hunyuan_video_custom_720", "hunyuan_custom_audio" : "hunyuan_video_custom_audio", "hunyuan_custom_edit" : "hunyuan_video_custom_edit",+ "hunyuan_avatar" : "hunyuan_video_avatar" }def get_model_type(model_filename):@@ -1653,8 +1663,15 @@ def get_model_name(model_filename, description_container = [""]):model_name = "Hunyuan Video image2video 720p 13B"description = "A good looking image 2 video model, but not so good in prompt adherence."elif "hunyuan_video_custom" in model_filename:- model_name = "Hunyuan Video Custom 720p 13B"- description = "The Hunyuan Video Custom model is probably the best model to transfer people (only people for the momment) as it is quite good to keep their identity. However it is slow as to get good results, you need to generate 720p videos with 30 steps."+ if "audio" in model_filename:+ model_name = "Hunyuan Video Custom Audio 720p 13B"+ description = "The Hunyuan Video Custom Audio model can be used to generate scenes of a person speaking given a Reference Image and a Recorded Voice or Song. The reference image is not a start image and therefore one can represent the person in a different context.The video length can be anything up to 10s. It is also quite good to generate no sound Video based on a person."+ elif "edit" in model_filename:+ model_name = "Hunyuan Video Custom Edit 720p 13B"+ description = "The Hunyuan Video Custom Edit model can be used to do Video inpainting on a person (add accessories or completely replace the person). You will need in any case to define a Video Mask which will indicate which area of the Video should be edited."+ else:+ model_name = "Hunyuan Video Custom 720p 13B"+ description = "The Hunyuan Video Custom model is probably the best model to transfer people (only people for the momment) as it is quite good to keep their identity. However it is slow as to get good results, you need to generate 720p videos with 30 steps."elif "hunyuan_video_avatar" in model_filename:model_name = "Hunyuan Video Avatar 720p 13B"description = "With the Hunyuan Video Avatar model you can animate a person based on the content of an audio input. Please note that the video generator works by processing 128 frames segment at a time (even if you ask less). The good news is that it will concatenate multiple segments for long video generation (max 3 segments recommended as the quality will get worse)."@@ -1778,6 +1795,13 @@ def get_default_settings(filename):"flow_shift": 13,"resolution": "1280x720",})+ elif get_model_type(filename) in ("hunyuan_custom_edit"):+ ui_defaults.update({+ "guidance_scale": 7.5,+ "flow_shift": 13,+ "video_prompt_type": "MV",+ "sliding_window_size": 129,+ })elif get_model_type(filename) in ("hunyuan_avatar"):ui_defaults.update({"guidance_scale": 7.5,@@ -2570,19 +2594,96 @@ def convert_image(image):image = image.convert('RGB')return cast(Image, ImageOps.exif_transpose(image))-def get_resampled_video(video_in, start_frame, max_frames, target_fps):+def get_resampled_video(video_in, start_frame, max_frames, target_fps, bridge='torch'):from wan.utils.utils import resampleimport decord- decord.bridge.set_bridge('torch')+ decord.bridge.set_bridge(bridge)reader = decord.VideoReader(video_in)- fps = reader.get_avg_fps()+ fps = round(reader.get_avg_fps())frame_nos = resample(fps, len(reader), max_target_frames_count= max_frames, target_fps=target_fps, start_target_frame= start_frame)frames_list = reader.get_batch(frame_nos)+ # print(f"frame nos: {frame_nos}")return frames_list+def preprocess_video_with_mask(input_video_path, input_mask_path, height, width, max_frames, start_frame=0, fit_canvas = False, target_fps = 16, block_size= 16, expand_scale = 2, pose_enhance = True, to_bbox = False):+ if not input_video_path or not input_mask_path:+ return None, None++ from preprocessing.dwpose.pose import PoseBodyFaceVideoAnnotator+ cfg_dict = {+ "DETECTION_MODEL": "ckpts/pose/yolox_l.onnx",+ "POSE_MODEL": "ckpts/pose/dw-ll_ucoco_384.onnx",+ "RESIZE_SIZE": 1024+ }+ dwpose = PoseBodyFaceVideoAnnotator(cfg_dict)++ video = get_resampled_video(input_video_path, start_frame, max_frames, target_fps)+ mask_video = get_resampled_video(input_mask_path, start_frame, max_frames, target_fps)++ if len(video) == 0 or len(mask_video) == 0:+ return None, None++ frame_height, frame_width, _ = video[0].shape++ if fit_canvas :+ scale1 = min(height / frame_height, width / frame_width)+ scale2 = min(height / frame_width, width / frame_height)+ scale = max(scale1, scale2)+ else:+ scale = ((height * width ) / (frame_height * frame_width))**(1/2)++ height = (int(frame_height * scale) // block_size) * block_size+ width = (int(frame_width * scale) // block_size) * block_size++ num_frames = min(len(video), len(mask_video))++ masked_frames = []+ masks = []+ for frame_idx in range(num_frames):+ frame = Image.fromarray(video[frame_idx].cpu().numpy()) #.asnumpy()+ mask = Image.fromarray(mask_video[frame_idx].cpu().numpy()) #.asnumpy()+ frame = frame.resize((width, height), resample=Image.Resampling.LANCZOS)+ mask = mask.resize((width, height), resample=Image.Resampling.LANCZOS)+ frame = np.array(frame)+ mask = np.array(mask)++ if len(mask.shape) == 3 and mask.shape[2] == 3:+ mask = cv2.cvtColor(mask, cv2.COLOR_BGR2GRAY)++ if expand_scale != 0:+ kernel_size = abs(expand_scale)+ kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))+ op_expand = cv2.dilate if expand_scale > 0 else cv2.erode+ mask = op_expand(mask, kernel, iterations=3)++ _, mask = cv2.threshold(mask, 127.5, 255, cv2.THRESH_BINARY)+ if to_bbox and np.sum(mask == 255) > 0:+ x0, y0, x1, y1 = mask_to_xyxy_box(mask)+ mask = mask * 0+ mask[y0:y1, x0:x1] = 255++ inverse_mask = mask == 0+ if pose_enhance:+ pose_img = dwpose.forward([frame])[0]+ masked_frame = np.where(inverse_mask[..., None], frame, pose_img)+ else:+ masked_frame = frame * (inverse_mask[..., None].astype(frame.dtype))++ mask = torch.from_numpy(mask) # to be commented if save one video enabled+ masked_frame = torch.from_numpy(masked_frame) # to be commented if save one video debug enabled+ masks.append(mask)+ masked_frames.append(masked_frame)+++ # from preprocessing.dwpose.pose import save_one_video+ # save_one_video("masked_frames.mp4", masked_frames, fps=target_fps, quality=8, macro_block_size=None)+ # save_one_video("masks.mp4", masks, fps=target_fps, quality=8, macro_block_size=None)++ return torch.stack(masked_frames), torch.stack(masks)+def preprocess_video(process_type, height, width, video_in, max_frames, start_frame=0, fit_canvas = False, target_fps = 16, block_size = 16):frames_list = get_resampled_video(video_in, start_frame, max_frames, target_fps)@@ -2600,9 +2701,6 @@ def preprocess_video(process_type, height, width, video_in, max_frames, start_frnew_height = (int(frame_height * scale) // block_size) * block_sizenew_width = (int(frame_width * scale) // block_size) * block_size- # if fit_canvas :- # new_height = height- # new_width = widthprocessed_frames_list = []for frame in frames_list:@@ -2857,12 +2955,14 @@ def generate_video(hunyuan_t2v = "hunyuan_video_720" in model_filenamehunyuan_i2v = "hunyuan_video_i2v" in model_filenamehunyuan_custom = "hunyuan_video_custom" in model_filename+ hunyuan_custom_audio = hunyuan_custom and "audio" in model_filename+ hunyuan_custom_edit = hunyuan_custom and "edit" in model_filenamehunyuan_avatar = "hunyuan_video_avatar" in model_filenamefantasy = "fantasy" in model_filename- if diffusion_forcing or hunyuan_t2v or hunyuan_i2v or hunyuan_custom:- fps = 24- elif hunyuan_avatar:+ if hunyuan_avatar or hunyuan_custom_audio:fps = 25+ elif diffusion_forcing or hunyuan_t2v or hunyuan_i2v or hunyuan_custom:+ fps = 24elif fantasy:fps = 23elif ltxv:@@ -2913,7 +3013,7 @@ def generate_video(audio_proj_split = Noneaudio_scale = Noneaudio_context_lens = None- if (fantasy or hunyuan_avatar) and audio_guide != None:+ if (fantasy or hunyuan_avatar or hunyuan_custom_audio) and audio_guide != None:from fantasytalking.infer import parse_audioimport librosaduration = librosa.get_duration(path=audio_guide)@@ -2922,6 +3022,12 @@ def generate_video(audio_proj_split, audio_context_lens = parse_audio(audio_guide, num_frames= current_video_length, fps= fps, device= processing_device )audio_scale = 1.0+ if hunyuan_custom_edit and video_guide != None:+ import cv2+ cap = cv2.VideoCapture(video_guide)+ length = int(cap.get(cv2.CAP_PROP_FRAME_COUNT))+ current_video_length = min(current_video_length, length)+import randomif seed == None or seed <0:seed = random.randint(0, 999999999)@@ -2938,13 +3044,10 @@ def generate_video(repeat_no = 0extra_generation = 0initial_total_windows = 0- if diffusion_forcing or vace or ltxv:- reuse_frames = min(sliding_window_size - 4, sliding_window_overlap)- else:- reuse_frames = 0if (diffusion_forcing or ltxv) and source_video != None:current_video_length += sliding_window_overlap- sliding_window = (vace or diffusion_forcing or ltxv) and current_video_length > sliding_window_size+ sliding_window = (vace or diffusion_forcing or ltxv or hunyuan_custom_edit) and current_video_length > sliding_window_size+ reuse_frames = min(sliding_window_size - 4, sliding_window_overlap) if sliding_window else 0discard_last_frames = sliding_window_discard_last_framesdefault_max_frames_to_generate = current_video_length@@ -3094,6 +3197,10 @@ def generate_video(pre_src_video = [pre_video_guide],fit_into_canvas = fit_canvas)+ elif hunyuan_custom_edit:+ progress_args = [0, get_latest_status(state,"Extracting Open Pose Information and Expanding Mask")]+ send_cmd("progress", progress_args)+ src_video, src_mask = preprocess_video_with_mask(video_guide, video_mask, height=height, width = width, max_frames= current_video_length if window_no == 1 else current_video_length - reuse_frames, start_frame = guide_start_frame, fit_canvas = fit_canvas, target_fps = fps, pose_enhance = "P" in video_prompt_type)if window_no == 1:conditioning_latents_size = ( (prefix_video_frames_count-1) // latent_size) + 1 if prefix_video_frames_count > 0 else 0else:@@ -3124,7 +3231,7 @@ def generate_video(input_frames = src_video,input_ref_images= src_ref_images,input_masks = src_mask,- input_video= pre_video_guide if diffusion_forcing or ltxv else source_video,+ input_video= pre_video_guide if diffusion_forcing or ltxv or hunyuan_custom_edit else source_video,target_camera= target_camera,frame_num=(current_video_length // latent_size)* latent_size + 1,height = height,@@ -3693,12 +3800,12 @@ def process_tasks(state):def get_generation_status(prompt_no, prompts_max, repeat_no, repeat_max, window_no, total_windows):if prompts_max == 1:- if repeat_max == 1:+ if repeat_max <= 1:status = ""else:status = f"Sample {repeat_no}/{repeat_max}"else:- if repeat_max == 1:+ if repeat_max <= 1:status = f"Prompt {prompt_no}/{prompts_max}"else:status = f"Prompt {prompt_no}/{prompts_max}, Sample {repeat_no}/{repeat_max}"@@ -4111,7 +4218,7 @@ def prepare_inputs_dict(target, inputs ):inputs.pop(k)- if not "Vace" in model_filename and not "diffusion_forcing" in model_filename and not "ltxv" in model_filename:+ if not "Vace" in model_filename and not "diffusion_forcing" in model_filename and not "ltxv" in model_filename and not "hunyuan_custom_edit" in model_filename:unsaved_params = [ "sliding_window_size", "sliding_window_overlap", "sliding_window_overlap_noise", "sliding_window_discard_last_frames"]for k in unsaved_params:inputs.pop(k)@@ -4556,8 +4663,10 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonhunyuan_t2v = "hunyuan_video_720" in model_filenamehunyuan_i2v = "hunyuan_video_i2v" in model_filenamehunyuan_video_custom = "hunyuan_video_custom" in model_filename+ hunyuan_video_custom_audio = hunyuan_video_custom and "audio" in model_filename+ hunyuan_video_custom_edit = hunyuan_video_custom and "edit" in model_filenamehunyuan_video_avatar = "hunyuan_video_avatar" in model_filename- sliding_window_enabled = vace or diffusion_forcing or ltxv+ sliding_window_enabled = vace or diffusion_forcing or ltxv or hunyuan_video_custom_editnew_line_text = "each new line of prompt will be used for a window" if sliding_window_enabled else "each new line of prompt will generate a new video"with gr.Column(visible= test_class_i2v(model_filename) or diffusion_forcing or ltxv or recammaster) as image_prompt_column:@@ -4630,7 +4739,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonmodel_mode = gr.Dropdown(value=None, visible=False)keep_frames_video_source = gr.Text(visible=False)- with gr.Column(visible= vace or phantom or hunyuan_video_custom or hunyuan_video_avatar) as video_prompt_column:+ with gr.Column(visible= vace or phantom or hunyuan_video_custom or hunyuan_video_avatar or hunyuan_video_custom_edit ) as video_prompt_column:video_prompt_type_value= ui_defaults.get("video_prompt_type","")video_prompt_type = gr.Text(value= video_prompt_type_value, visible= False)with gr.Row():@@ -4648,6 +4757,15 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonvalue=filter_letters(video_prompt_type_value, "ODPCMV"),label="Video to Video", scale = 3, visible= True)+ elif hunyuan_video_custom_edit:+ video_prompt_type_video_guide = gr.Dropdown(+ choices=[+ ("Inpaint Control Video in area defined by Mask", "MV"),+ ("Inpaint and Transfer Human Motion from the Control Video in area defined by Mask", "PMV"),+ ],+ value=filter_letters(video_prompt_type_value, "ODPCMV"),+ label="Video to Video", scale = 3, visible= True+ )else:video_prompt_type_video_guide = gr.Dropdown(visible= False)@@ -4686,7 +4804,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonvideo_mask = gr.Video(label= "Video Mask (for Inpainting or Outpaing, white pixels = Mask)", visible= "M" in video_prompt_type_value, value= ui_defaults.get("video_mask", None))- audio_guide = gr.Audio(value= ui_defaults.get("audio_guide", None), type="filepath", label="Voice to follow", show_download_button= True, visible= fantasy or hunyuan_video_avatar )+ audio_guide = gr.Audio(value= ui_defaults.get("audio_guide", None), type="filepath", label="Voice to follow", show_download_button= True, visible= fantasy or hunyuan_video_avatar or hunyuan_video_custom_audio )advanced_prompt = advanced_uiprompt_vars=[]@@ -4775,9 +4893,9 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonvideo_length = gr.Slider(17, 737, value=ui_defaults.get("video_length", 81), step=4, label="Number of frames (16 = 1s)", interactive= True)elif fantasy:video_length = gr.Slider(5, 233, value=ui_defaults.get("video_length", 81), step=4, label="Number of frames (23 = 1s)", interactive= True)- elif hunyuan_video_avatar:+ elif hunyuan_video_avatar or hunyuan_video_custom_audio:video_length = gr.Slider(5, 401, value=ui_defaults.get("video_length", 81), step=4, label="Number of frames (25 = 1s)", interactive= True)- elif hunyuan_t2v or hunyuan_i2v:+ elif hunyuan_t2v or hunyuan_i2v or hunyuan_video_custom:video_length = gr.Slider(5, 337, value=ui_defaults.get("video_length", 97), step=4, label="Number of frames (24 = 1s)", interactive= True)else:video_length = gr.Slider(5, 193, value=ui_defaults.get("video_length", 81), step=4, label="Number of frames (16 = 1s)", interactive= True)@@ -4915,17 +5033,22 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Nonif diffusion_forcing:sliding_window_size = gr.Slider(37, 137, value=ui_defaults.get("sliding_window_size", 97), step=20, label="Sliding Window Size (recommended to keep it at 97)")sliding_window_overlap = gr.Slider(17, 97, value=ui_defaults.get("sliding_window_overlap",17), step=20, label="Windows Frames Overlap (needed to maintain continuity between windows, a higher value will require more windows)")- sliding_window_overlap_noise = gr.Slider(0, 100, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect")+ sliding_window_overlap_noise = gr.Slider(0, 100, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect", visible = True)sliding_window_discard_last_frames = gr.Slider(0, 20, value=ui_defaults.get("sliding_window_discard_last_frames", 0), step=4, visible = False)elif ltxv:sliding_window_size = gr.Slider(41, 257, value=ui_defaults.get("sliding_window_size", 129), step=8, label="Sliding Window Size")sliding_window_overlap = gr.Slider(9, 97, value=ui_defaults.get("sliding_window_overlap",9), step=8, label="Windows Frames Overlap (needed to maintain continuity between windows, a higher value will require more windows)")- sliding_window_overlap_noise = gr.Slider(0, 100, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect")+ sliding_window_overlap_noise = gr.Slider(0, 100, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect", visible = False)sliding_window_discard_last_frames = gr.Slider(0, 20, value=ui_defaults.get("sliding_window_discard_last_frames", 0), step=4, visible = False)+ elif hunyuan_video_custom_edit:+ sliding_window_size = gr.Slider(5, 257, value=ui_defaults.get("sliding_window_size", 129), step=4, label="Sliding Window Size")+ sliding_window_overlap = gr.Slider(1, 97, value=ui_defaults.get("sliding_window_overlap",5), step=4, label="Windows Frames Overlap (needed to maintain continuity between windows, a higher value will require more windows)")+ sliding_window_overlap_noise = gr.Slider(0, 150, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect", visible = False)+ sliding_window_discard_last_frames = gr.Slider(0, 20, value=ui_defaults.get("sliding_window_discard_last_frames", 0), step=4, label="Discard Last Frames of a Window (that may have bad quality)", visible = True)else:sliding_window_size = gr.Slider(5, 137, value=ui_defaults.get("sliding_window_size", 81), step=4, label="Sliding Window Size")sliding_window_overlap = gr.Slider(1, 97, value=ui_defaults.get("sliding_window_overlap",5), step=4, label="Windows Frames Overlap (needed to maintain continuity between windows, a higher value will require more windows)")- sliding_window_overlap_noise = gr.Slider(0, 150, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect")+ sliding_window_overlap_noise = gr.Slider(0, 150, value=ui_defaults.get("sliding_window_overlap_noise",20), step=1, label="Noise to be added to overlapped frames to reduce blur effect" , visible = True)sliding_window_discard_last_frames = gr.Slider(0, 20, value=ui_defaults.get("sliding_window_discard_last_frames", 8), step=4, label="Discard Last Frames of a Window (that may have bad quality)", visible = True)