Python
support for alpha
DeepBeepMeep9ae8b584b3bc
diff --git a/defaults/alpha.json b/defaults/alpha.jsonnew file mode 100644index 0000000..641fbde--- /dev/null+++ b/defaults/alpha.json@@ -0,0 +1,19 @@+{+ "model":+ {+ "name": "Wan2.1 Alpha 14B",+ "architecture" : "alpha",+ "description": "This model successfully generates various scenes with accurate and clearly rendered transparency. Notably, it can synthesize diverse semi-transparent objects, glowing effects, and fine-grained details such as hair. For each video generated you will find a Zip file with the same name that will contain the corresponding RGBA images.",+ "URLs": "t2v",+ "preload_URLs": [+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan_alpha_2.1_vae_rgb_channel.safetensors",+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan_alpha_2.1_vae_alpha_channel.safetensors"+ ],+ "loras": [+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan_alpha_2.1_dora.safetensors"+ ],+ "loras_multipliers": [ 1 ]+ },+ "prompt": "A large orange octopus is seen resting. The background of the video is transparent."++}\ No newline at end of filediff --git a/defaults/alpha_sf.json b/defaults/alpha_sf.jsonnew file mode 100644index 0000000..4d273b9--- /dev/null+++ b/defaults/alpha_sf.json@@ -0,0 +1,17 @@+{+ "model":+ {+ "name": "Wan2.1 Alpha Lightning 14B",+ "architecture" : "alpha",+ "description": "This model is accelerated by the Lightning / SelfForcing process. It successfully generates various scenes with accurate and clearly rendered transparency. Notably, it can synthesize diverse semi-transparent objects, glowing effects, and fine-grained details such as hair. For each video generated you will find a Zip file with the same name that will contain the corresponding RGBA images.",+ "URLs": "t2v_sf",+ "preload_URLs": "alpha",+ "loras": "alpha",+ "loras_multipliers": [ 1 ],+ "profiles_dir" : [""]+ },+ "prompt": "A large orange octopus is seen resting. The background of the video is transparent.",+ "num_inference_steps": 4,+ "guidance_scale": 1,+ "flow_shift": 3+}\ No newline at end of filediff --git a/defaults/t2v_sf.json b/defaults/t2v_sf.jsonindex 2131413..d061ca3 100644--- a/defaults/t2v_sf.json+++ b/defaults/t2v_sf.json@@ -1,8 +1,8 @@{"model": {- "name": "Wan2.1 Text2video Self-Forcing 14B",+ "name": "Wan2.1 Text2video Lightning 14B","architecture": "t2v",- "description": "This model is an advanced text-to-video generation model. This approach allows the model to generate videos with significantly fewer inference steps (4 or 8 steps) and without classifier-free guidance, substantially reducing video generation time while maintaining high quality outputs.",+ "description": "This model is an advanced text-to-video generation model. The Lightning / SelfForcing process allows the model to generate videos with significantly fewer inference steps (4 or 8 steps) and without classifier-free guidance, substantially reducing video generation time while maintaining high quality outputs.","URLs": ["https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_bf16.safetensors","https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_quanto_bf16_int8.safetensors",@@ -11,28 +11,7 @@"author": "https://huggingface.co/lightx2v/Wan2.1-T2V-14B-StepDistill-CfgDistill","auto_quantize": true},- "negative_prompt": "",- "prompt": "",- "resolution": "832x480",- "video_length": 81,- "seed": -1,"num_inference_steps": 4,"guidance_scale": 1,- "flow_shift": 3,- "embedded_guidance_scale": 6,- "repeat_generation": 1,- "multi_images_gen_type": 0,- "tea_cache_setting": 0,- "tea_cache_start_step_perc": 0,- "loras_multipliers": "",- "temporal_upsampling": "",- "spatial_upsampling": "",- "RIFLEx_setting": 0,- "slg_switch": 0,- "slg_start_perc": 10,- "slg_end_perc": 90,- "cfg_star_switch": 0,- "cfg_zero_step": -1,- "prompt_enhancer": "",- "activated_loras": []+ "flow_shift": 3}\ No newline at end of filediff --git a/models/wan/alpha/utils.py b/models/wan/alpha/utils.pynew file mode 100644index 0000000..066fe4d--- /dev/null+++ b/models/wan/alpha/utils.py@@ -0,0 +1,80 @@+import torchvision+from PIL import Image, ImageDraw+import imageio+import cv2+import torch+import numpy as np+import zipfile++def render_video(tensor_fgr,+ tensor_pha,+ nrow=8,+ normalize=True,+ value_range=(-1, 1)):++ tensor_fgr = tensor_fgr.clamp(min(value_range), max(value_range))+ tensor_fgr = torch.stack([+ torchvision.utils.make_grid(+ u, nrow=nrow, normalize=normalize, value_range=value_range)+ for u in tensor_fgr.unbind(2)+ ],+ dim=1).permute(1, 2, 3, 0)+ tensor_fgr = (tensor_fgr * 255).type(torch.uint8).cpu()++ tensor_pha = tensor_pha.clamp(min(value_range), max(value_range))+ tensor_pha = torch.stack([+ torchvision.utils.make_grid(+ u, nrow=nrow, normalize=normalize, value_range=value_range)+ for u in tensor_pha.unbind(2)+ ],+ dim=1).permute(1, 2, 3, 0)+ tensor_pha = (tensor_pha * 255).type(torch.uint8).cpu()++ frames = []+ frames_fgr = []+ frames_pha = []+ for frame_fgr, frame_pha in zip(tensor_fgr.numpy(), tensor_pha.numpy()):+ frame_pha = (0.0 + frame_pha[:,:,0:1] + frame_pha[:,:,1:2] + frame_pha[:,:,2:3]) / 3.+ frame = np.concatenate([frame_fgr[:,:,::-1], frame_pha.astype(np.uint8)], axis=2)+ frames.append(frame)+ frames_fgr.append(frame_fgr)+ frames_pha.append(frame_pha)++ def create_checkerboard(size=30, pattern_size=(830, 480), color1=(140, 140, 140), color2=(113, 113, 113)):+ img = Image.new('RGB', (pattern_size[0], pattern_size[1]), color1)+ draw = ImageDraw.Draw(img)+ for i in range(0, pattern_size[0], size):+ for j in range(0, pattern_size[1], size):+ if (i + j) // size % 2 == 0:+ draw.rectangle([i, j, i+size, j+size], fill=color2)+ return img++ def blender_background(frame_rgba, checkerboard):+ alpha_channel = frame_rgba[:, :, 3:] / 255.+ checkerboard = np.array(checkerboard)+ checkerboard = cv2.resize(checkerboard, (frame_rgba.shape[1], frame_rgba.shape[0]))++ frame_rgb = frame_rgba[:, :, :3] * alpha_channel + checkerboard * (1-alpha_channel)+ return frame_rgb.astype(np.uint8)[:,:,::-1]++ checkerboard = create_checkerboard()+ video_checkerboard = [torch.from_numpy(blender_background(f, checkerboard).copy()).float().div_(127.5).sub_(1) for f in frames]+ video_checkerboard = torch.stack(video_checkerboard ).permute(3, 0, 1, 2)+ return video_checkerboard, frames++def from_BRGA_numpy_to_RGBA_torch(video):+ video = [torch.from_numpy(f.copy()).float().div_(127.5).sub_(1) for f in video]+ video = torch.stack(video).permute(3, 0, 1, 2)+ video[[0, 2], ...] = video[[2, 0], ...]+ return video++def write_zip_file(zip_path, frames):+ with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:+ for idx, img in enumerate(frames):+ success, buffer = cv2.imencode(".png", img)+ if not success:+ print(f"Failed to encode image {idx}, skipping...")+ continue++ filename = f"img_{idx:03d}.png"+ zipf.writestr(filename, buffer.tobytes())\ No newline at end of filediff --git a/models/wan/any2video.py b/models/wan/any2video.pyindex d2f6780..92c73e9 100644--- a/models/wan/any2video.py+++ b/models/wan/any2video.py@@ -121,18 +121,27 @@ class WanAny2V:ignore_unused_weights = model_def.get("ignore_unused_weights", False)- if base_model_type in ["ti2v_2_2", "lucy_edit"]:+ vae_checkpoint2 = None+ if model_def.get("wan_5B_class", False):self.vae_stride = (4, 16, 16)vae_checkpoint = "Wan2.2_VAE.safetensors"vae = Wan2_2_VAEelse:- self.vae_stride = config.vae_stride- vae_checkpoint = "Wan2.1_VAE.safetensors"vae = WanVAE+ self.vae_stride = config.vae_stride+ if model_def.get("alpha_class", False):+ vae_checkpoint ="wan_alpha_2.1_vae_rgb_channel.safetensors"+ vae_checkpoint2 ="wan_alpha_2.1_vae_alpha_channel.safetensors"+ else:+ vae_checkpoint = "Wan2.1_VAE.safetensors"self.patch_size = config.patch_sizeself.vae = vae( vae_pth=fl.locate_file(vae_checkpoint), dtype= VAE_dtype, device="cpu")self.vae.device = self.device+ self.vae2 = None+ if vae_checkpoint2 is not None:+ self.vae2 = vae( vae_pth=fl.locate_file(vae_checkpoint2), dtype= VAE_dtype, device="cpu")+ self.vae2.device = self.device# config_filename= "configs/t2v_1.3B.json"# import json@@ -306,7 +315,7 @@ class WanAny2V:msk = msk.transpose(1,2)[0]return msk- def encode_reference_images(self, ref_images, ref_prompt="image of a face", any_guidance= False, tile_size = None):+ def encode_reference_images(self, ref_images, ref_prompt="image of a face", any_guidance= False, tile_size = None, enable_loras = True):ref_images = [convert_image_to_tensor(img).unsqueeze(1).to(device=self.device, dtype=self.dtype) for img in ref_images]shape = ref_images[0].shapefreqs = get_rotary_pos_embed( (len(ref_images) , shape[-2] // 8, shape[-1] // 8 ))@@ -320,6 +329,9 @@ class WanAny2V:context = torch.cat([context, context.new_zeros(self.model.text_len -context.size(0), context.size(1)) ]).unsqueeze(0)clear_caches()get_cache("lynx_ref_buffer").update({ 0: {}, 1: {} })+ if not enable_loras:+ _loras_active_adapters = self.model._loras_active_adapters+ self.model._loras_active_adapters = []ref_buffer = self.model(pipeline =self,x = [vae_feat, vae_feat_uncond] if any_guidance else [vae_feat],@@ -328,6 +340,9 @@ class WanAny2V:t=torch.stack([torch.tensor(0, dtype=torch.float)]).to(self.device),lynx_feature_extractor = True,)+ if not enable_loras:+ self.model._loras_active_adapters = _loras_active_adapters+clear_caches()return ref_buffer[0], (ref_buffer[1] if any_guidance else None)@@ -480,29 +495,30 @@ class WanAny2V:if NAG_scale > 1: context = torch.cat([context, context_null], dim=0)# if NAG_scale > 1: context = torch.cat([context, context_NAG], dim=0)if self._interrupt: return None-- vace = model_type in ["vace_1.3B","vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]+ model_def = self.model_def+ vace = model_def.get("vace_class", False)phantom = model_type in ["phantom_1.3B", "phantom_14B"]fantasy = model_type in ["fantasy"]- multitalk = model_type in ["multitalk", "infinitetalk", "vace_multitalk_14B", "i2v_2_2_multitalk"]+ multitalk = model_def.get("multitalk_class", False)infinitetalk = model_type in ["infinitetalk"]- standin = model_type in ["standin", "vace_standin_14B"]- lynx = model_type in ["lynx_lite", "lynx", "vace_lynx_lite_14B", "vace_lynx_14B"]+ standin = model_def.get("standin_class", False)+ lynx = model_def.get("lynx_class", False)recam = model_type in ["recam_1.3B"]- ti2v = model_type in ["ti2v_2_2", "lucy_edit"]+ ti2v = model_def.get("wan_5B_class", False)+ alpha_class = model_def.get("alpha_class", False)lucy_edit= model_type in ["lucy_edit"]animate= model_type in ["animate"]start_step_no = 0ref_images_count = 0trim_frames = 0- extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = None+ extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = latent_slice = Noneno_noise_latents_injection = infinitetalktimestep_injection = Falselat_frames = int((frame_num - 1) // self.vae_stride[0]) + 1extended_input_dim = 0ref_images_before = False# image2video- if model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "flf2v_720p"]:+ if model_def.get("i2v_class", False) and not animate:any_end_frame = Falseif infinitetalk:new_shot = "0" in video_prompt_type@@ -730,12 +746,11 @@ class WanAny2V:image_ref = original_input_ref_images[-1]from preprocessing.face_preprocessor import FaceProcessorface_processor = FaceProcessor()- lynx_ref = face_processor.process(image_ref, resize_to = 256 )- lynx_ref_buffer, lynx_ref_buffer_uncond = self.encode_reference_images([lynx_ref], tile_size=VAE_tile_size, any_guidance= any_guidance_at_all)+ lynx_ref = face_processor.process(image_ref, resize_to = 256)+ lynx_ref_buffer, lynx_ref_buffer_uncond = self.encode_reference_images([lynx_ref], tile_size=VAE_tile_size, any_guidance= any_guidance_at_all, enable_loras = False)lynx_ref = Nonegc.collect()torch.cuda.empty_cache()- vace_lynx = model_type in ["vace_lynx_14B"]kwargs["lynx_ip_scale"] = control_scale_altkwargs["lynx_ref_scale"] = control_scale_alt@@ -744,13 +759,13 @@ class WanAny2V:from preprocessing.face_preprocessor import FaceProcessorstandin_ref_pos = 1 if "K" in video_prompt_type else 0if len(original_input_ref_images) < standin_ref_pos + 1:- if "I" in video_prompt_type and model_type in ["vace_standin_14B"]:+ if "I" in video_prompt_type and vace:print("Warning: Missing Standin ref image, make sure 'Inject only People / Objets' is selected or if there is 'Landscape and then People or Objects' there are at least two ref images.")else:standin_ref_pos = -1image_ref = original_input_ref_images[standin_ref_pos]face_processor = FaceProcessor()- standin_ref = face_processor.process(image_ref, remove_bg = model_type in ["vace_standin_14B"])+ standin_ref = face_processor.process(image_ref, remove_bg = vace)face_processor = Nonegc.collect()torch.cuda.empty_cache()@@ -764,7 +779,7 @@ class WanAny2V:# vace context encodeinput_frames = [input_frames.to(self.device)] +([] if input_frames2 is None else [input_frames2.to(self.device)])input_masks = [input_masks.to(self.device)] + ([] if input_masks2 is None else [input_masks2.to(self.device)])- if model_type in ["vace_lynx_14B"] and input_ref_images is not None:+ if lynx and input_ref_images is not None:input_ref_images,input_ref_masks = input_ref_images[:-1], input_ref_masks[:-1]input_ref_images = None if input_ref_images is None else [ u.to(self.device) for u in input_ref_images]input_ref_masks = None if input_ref_masks is None else [ None if u is None else u.to(self.device) for u in input_ref_masks]@@ -1103,11 +1118,16 @@ class WanAny2V:self.model.release_chipmunk() # need to add it at every exit when in prodvideos = self.vae.decode(x0, VAE_tile_size)+ any_vae2= self.vae2 is not None+ if any_vae2:+ videos2 = self.vae2.decode(x0, VAE_tile_size)if image_outputs:videos = torch.cat([video[:,:1] for video in videos], dim=1) if len(videos) > 1 else videos[0][:,:1]+ if any_vae2: videos2 = torch.cat([video[:,:1] for video in videos2], dim=1) if len(videos2) > 1 else videos2[0][:,:1]else:videos = videos[0] # return only first video+ if any_vae2: videos2 = videos2[0] # return only first videoif color_correction_strength > 0 and (prefix_frames_count > 0 and window_no > 1 or prefix_frames_count > 1 and window_no == 1):if vace and False:# videos = match_and_blend_colors_with_mask(videos.unsqueeze(0), input_frames[0].unsqueeze(0), input_masks[0][:1].unsqueeze(0), color_correction_strength,copy_mode= "progressive_blend").squeeze(0)@@ -1115,9 +1135,16 @@ class WanAny2V:# videos = match_and_blend_colors_with_mask(videos.unsqueeze(0), videos.unsqueeze(0), input_masks[0][:1].unsqueeze(0), color_correction_strength,copy_mode= "reference").squeeze(0)elif color_reference_frame is not None:videos = match_and_blend_colors(videos.unsqueeze(0), color_reference_frame.unsqueeze(0), color_correction_strength).squeeze(0)-- if return_latent_slice != None:- return { "x" : videos, "latent_slice" : latent_slice }++ BGRA_frames = None+ if alpha_class:+ from .alpha.utils import render_video, from_BRGA_numpy_to_RGBA_torch+ videos, BGRA_frames = render_video(videos[None], videos2[None])+ if image_outputs:+ videos = from_BRGA_numpy_to_RGBA_torch(BGRA_frames)+ BGRA_frames = None+ if return_latent_slice != None or BGRA_frames != None:+ return { "x" : videos, "latent_slice" : latent_slice, "BGRA_frames" : BGRA_frames }return videosdef adapt_vace_model(self, model):diff --git a/models/wan/configs/alpha.json b/models/wan/configs/alpha.jsonnew file mode 100644index 0000000..c554e73--- /dev/null+++ b/models/wan/configs/alpha.json@@ -0,0 +1,14 @@+{+ "_class_name": "WanModel",+ "_diffusers_version": "0.30.0",+ "dim": 5120,+ "eps": 1e-06,+ "ffn_dim": 13824,+ "freq_dim": 256,+ "in_dim": 16,+ "model_type": "t2v",+ "num_heads": 40,+ "num_layers": 40,+ "out_dim": 16,+ "text_len": 512+}diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.pyindex f70e376..a81f131 100644--- a/models/wan/wan_handler.py+++ b/models/wan/wan_handler.py@@ -3,9 +3,15 @@ import numpy as npimport gradio as grfrom shared.utils import files_locator as fl+def test_vace(base_model_type):+ return base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]+def test_class_i2v(base_model_type):return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate" ]+def test_class_t2v(base_model_type):+ return base_model_type in ["t2v", "t2v_2_2", "alpha"]+def text_oneframe_overlap(base_model_type):return test_class_i2v(base_model_type) and not (test_multitalk(base_model_type) or base_model_type in ["animate"]) or test_wan_5B(base_model_type)@@ -19,7 +25,10 @@ def test_standin(base_model_type):return base_model_type in ["standin", "vace_standin_14B"]def test_lynx(base_model_type):- return base_model_type in ["lynx_lite", "vace_lynx_lite_14B", "lynx", "vace_lynx_14B"]+ return base_model_type in ["lynx_lite", "vace_lynx_lite_14B", "lynx", "vace_lynx_14B", "alpha_lynx"]++def test_alpha(base_model_type):+ return base_model_type in ["alpha", "alpha_lynx"]def test_wan_5B(base_model_type):return base_model_type in ["ti2v_2_2", "lucy_edit"]@@ -29,7 +38,7 @@ class family_handler():def query_supported_types():return ["multitalk", "infinitetalk", "fantasy", "vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B","t2v_1.3B", "standin", "lynx_lite", "lynx", "t2v", "t2v_2_2", "vace_1.3B", "phantom_1.3B", "phantom_14B",- "recam_1.3B", "animate",+ "recam_1.3B", "animate", "alpha", "alpha_lynx","i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp"]@@ -38,8 +47,9 @@ class family_handler():models_eqv_map = {"flf2v_720p" : "i2v",- "t2v_1.3B" : "t2v",- "t2v_2_2" : "t2v",+ "t2v_1.3B" : "t2v",+ "t2v_2_2" : "t2v",+ "t2v_2_2" : "alpha","vace_standin_14B" : "vace_14B","vace_lynx_14B" : "vace_14B","vace_14B_2_2": "vace_14B",@@ -47,7 +57,7 @@ class family_handler():models_comp_map = {"vace_14B" : [ "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_14B", "vace_14B_2_2"],- "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_14B", "vace_14B_2_2", "t2v_1.3B", "phantom_1.3B","phantom_14B", "standin", "lynx_lite", "lynx"],+ "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_14B", "vace_14B_2_2", "t2v_1.3B", "phantom_1.3B","phantom_14B", "standin", "lynx_lite", "lynx", "alpha"],"i2v" : [ "fantasy", "multitalk", "flf2v_720p" ],"i2v_2_2" : ["i2v_2_2_multitalk"],"fantasy": ["multitalk"],@@ -119,13 +129,14 @@ class family_handler():extra_model_def = {}if "URLs2" in model_def:extra_model_def["no_steps_skipping"] = True- i2v = test_class_i2v(base_model_type)- extra_model_def["i2v_class"] = i2v- extra_model_def["multitalk_class"] = test_multitalk(base_model_type)- extra_model_def["standin_class"] = test_standin(base_model_type)- extra_model_def["lynx_class"] = test_lynx(base_model_type)- vace_class = base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]- extra_model_def["vace_class"] = vace_class+ extra_model_def["i2v_class"] = i2v = test_class_i2v(base_model_type)+ extra_model_def["t2v_class"] = t2v = test_class_t2v(base_model_type)+ extra_model_def["multitalk_class"] = multitalk = test_multitalk(base_model_type)+ extra_model_def["standin_class"] = standin = test_standin(base_model_type)+ extra_model_def["lynx_class"] = lynx = test_lynx(base_model_type)+ extra_model_def["alpha_class"] = alpha = test_alpha(base_model_type)+ extra_model_def["wan_5B_class"] = wan_5B = test_wan_5B(base_model_type)+ extra_model_def["vace_class"] = vace_class = test_vace(base_model_type)if base_model_type in ["vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]:extra_model_def["parent_model_type"] = "vace_14B"@@ -140,6 +151,8 @@ class family_handler():group = "wan2_2"elif test_class_1_3B(base_model_type):profiles_dir = "wan_1.3B"+ elif base_model_type in ["alpha"]:+ profiles_dir = "wan_alpha"else:profiles_dir = "wan"@@ -148,11 +161,11 @@ class family_handler():if base_model_type in ["animate"]:fps = 30- elif test_multitalk(base_model_type):+ elif multitalk:fps = 25elif base_model_type in ["fantasy"]:fps = 23- elif test_wan_5B(base_model_type):+ elif wan_5B:fps = 24else:fps = 16@@ -165,7 +178,7 @@ class family_handler():extra_model_def.update({"frames_minimum" : frames_minimum,"frames_steps" : frames_steps,- "sliding_window" : base_model_type in ["multitalk", "infinitetalk", "t2v", "t2v_2_2", "fantasy", "animate"] or test_class_i2v(base_model_type) or test_wan_5B(base_model_type) or vace_class, #"ti2v_2_2",+ "sliding_window" : base_model_type in ["multitalk", "infinitetalk", "t2v", "t2v_2_2", "fantasy", "animate", "lynx"] or test_class_i2v(base_model_type) or test_wan_5B(base_model_type) or vace_class, #"ti2v_2_2","multiple_submodels" : multiple_submodels,"guidance_max_phases" : 3,"skip_layer_guidance" : True,@@ -184,21 +197,22 @@ class family_handler():})- if base_model_type in ["t2v", "t2v_2_2"]:- extra_model_def["guide_custom_choices"] = {- "choices":[("Use Text Prompt Only", ""),- ("Video to Video guided by Text Prompt", "GUV"),- ("Video to Video guided by Text Prompt and Restricted to the Area of the Video Mask", "GVA")],- "default": "",- "show_label" : False,- "letters_filter": "GUVA",- "label": "Video to Video"- }+ if t2v:+ if not alpha:+ extra_model_def["guide_custom_choices"] = {+ "choices":[("Use Text Prompt Only", ""),+ ("Video to Video guided by Text Prompt", "GUV"),+ ("Video to Video guided by Text Prompt and Restricted to the Area of the Video Mask", "GVA")],+ "default": "",+ "show_label" : False,+ "letters_filter": "GUVA",+ "label": "Video to Video"+ }- extra_model_def["mask_preprocessing"] = {- "selection":[ "", "A"],- "visible": False- }+ extra_model_def["mask_preprocessing"] = {+ "selection":[ "", "A"],+ "visible": False+ }extra_model_def["v2i_switch_supported"] = True@@ -309,7 +323,7 @@ class family_handler():extra_model_def["forced_guide_mask_inputs"] = Trueextra_model_def["return_image_refs_tensor"] = Trueextra_model_def["v2i_switch_supported"] = True- if base_model_type in ["vace_lynx_14B"]:+ if lynx:extra_model_def["set_video_prompt_type"]="Q"extra_model_def["control_net_weight_alt_name"] = "Lynx"extra_model_def["image_ref_choices"]["choices"] = [("None", ""),@@ -321,7 +335,7 @@ class family_handler():- if base_model_type in ["standin"]:+ if (not vace_class) and standin:extra_model_def["v2i_switch_supported"] = Trueextra_model_def["image_ref_choices"] = {"choices": [@@ -333,7 +347,7 @@ class family_handler():}extra_model_def["one_image_ref_needed"] = True- if base_model_type in ["lynx_lite", "lynx"]:+ if (not vace_class) and lynx:extra_model_def["fit_into_canvas_image_refs"] = 0extra_model_def["guide_custom_choices"] = {"choices":[("Use Reference Image which is a Person Face", ""),@@ -396,7 +410,7 @@ class family_handler():"visible" : False,}- if vace_class or base_model_type in ["animate", "t2v", "t2v_2_2"] :+ if vace_class or base_model_type in ["animate", "t2v", "t2v_2_2", "lynx"] :image_prompt_types_allowed = "TVL"elif base_model_type in ["infinitetalk"]:image_prompt_types_allowed = "TSVL"@@ -404,7 +418,7 @@ class family_handler():image_prompt_types_allowed = "TSVL"elif base_model_type in ["lucy_edit"]:image_prompt_types_allowed = "TVL"- elif test_multitalk(base_model_type) or base_model_type in ["fantasy"]:+ elif multitalk or base_model_type in ["fantasy"]:image_prompt_types_allowed = "SVL"elif i2v:image_prompt_types_allowed = "SEVL"@@ -480,6 +494,8 @@ class family_handler():)pipe = {"transformer": wan_model.model, "text_encoder" : wan_model.text_encoder.model, "vae": wan_model.vae.model }+ if wan_model.vae2 is not None:+ pipe["vae2"] = wan_model.vae2.modelif hasattr(wan_model,"model2") and wan_model.model2 is not None:pipe["transformer2"] = wan_model.model2if hasattr(wan_model, "clip"):@@ -600,7 +616,7 @@ class family_handler():"remove_background_images_ref" : 1 ,})- elif base_model_type in ["lynx_lite", "lynx"]:+ elif (base_model_type in ["lynx_lite", "lynx", "alpha_lynx"]):ui_defaults.update({"guidance_scale": 5.0,"flow_shift": 7, # 11 for 720pdiff --git a/profiles/wan_alpha/Text2Video Lightx2v Cfg Step Distill Rank64 - 4 Steps.json b/profiles/wan_alpha/Text2Video Lightx2v Cfg Step Distill Rank64 - 4 Steps.jsonnew file mode 100644index 0000000..1718e3e--- /dev/null+++ b/profiles/wan_alpha/Text2Video Lightx2v Cfg Step Distill Rank64 - 4 Steps.json@@ -0,0 +1,11 @@+{++ "activated_loras": [+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/loras_accelerators/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors"+ ],+ "loras_multipliers": "1",+ "num_inference_steps": 4,+ "guidance_phases": 1,+ "guidance_scale": 1,+ "flow_shift": 2+}\ No newline at end of filediff --git a/shared/utils/audio_video.py b/shared/utils/audio_video.pyindex 224cf34..3f59cc7 100644--- a/shared/utils/audio_video.py+++ b/shared/utils/audio_video.py@@ -301,7 +301,11 @@ def save_image(tensor,quality='jpeg_95', # 'jpeg_95', 'jpeg_85', 'jpeg_70', 'jpeg_50', 'webp_95', 'webp_85', 'webp_70', 'webp_50', 'png', 'webp_lossless'retry=5):"""Save tensor as image with configurable format and quality."""-++ RGBA = tensor.shape[0] == 4+ if RGBA:+ quality = "png"+# Get format and quality settingsformat_info = _get_format_info(quality)@@ -310,16 +314,18 @@ def save_image(tensor,# Save imageerror = None+for _ in range(retry):try:tensor = tensor.clamp(min(value_range), max(value_range))- if format_info['use_pil']:+ if format_info['use_pil'] or RGBA:# Use PIL for WebP and advanced optionsgrid = torchvision.utils.make_grid(tensor, nrow=nrow, normalize=normalize, value_range=value_range)# Convert to PIL Imagegrid = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()- img = Image.fromarray(grid)+ mode = 'RGBA' if RGBA else 'RGB'+ img = Image.fromarray(grid, mode=mode)img.save(save_file, **format_info['params'])else:# Use torchvision for JPEG and PNGdiff --git a/wgp.py b/wgp.pyindex 74af14e..4df3ed0 100644--- a/wgp.py+++ b/wgp.py@@ -239,27 +239,31 @@ def process_prompt_and_add_tasks(state, model_choice):if frames_count > max_source_video_frames:gr.Info(f"Post processing is not supported on videos longer than {max_source_video_frames} frames. Output Video will be truncated")# return- for k in ["image_start", "image_end", "image_refs", "video_guide", "audio_guide", "audio_guide2", "audio_source" , "video_mask", "image_mask"]:+ for prop in ["state", "model_type", "mode"]:+ edit_overrides[prop] = inputs[prop]+ for k,v in inputs.items():inputs[k] = Noneinputs.update(edit_overrides)del gen["edit_video_source"], gen["edit_overrides"]inputs["video_source"]= edit_video_sourceprompt = []- spatial_upsampling = inputs.get("spatial_upsampling","")- if len(spatial_upsampling) >0: prompt += ["Spatial Upsampling"]- temporal_upsampling = inputs.get("temporal_upsampling","")- if len(temporal_upsampling) >0: prompt += ["Temporal Upsampling"]- if has_image_file_extension(edit_video_source) and len(temporal_upsampling) > 0:- gr.Info("Temporal Upsampling can not be used with an Image")- return ret()- film_grain_intensity = inputs.get("film_grain_intensity",0)- film_grain_saturation = inputs.get("film_grain_saturation",0.5)- # if film_grain_intensity >0: prompt += [f"Film Grain: intensity={film_grain_intensity}, saturation={film_grain_saturation}"]- if film_grain_intensity >0: prompt += ["Film Grain"]- MMAudio_setting = inputs.get("MMAudio_setting",0)- repeat_generation= inputs.get("repeat_generation",1)- if mode =="edit_remux":+ repeat_generation = 1+ if mode == "edit_postprocessing":+ spatial_upsampling = inputs.get("spatial_upsampling","")+ if len(spatial_upsampling) >0: prompt += ["Spatial Upsampling"]+ temporal_upsampling = inputs.get("temporal_upsampling","")+ if len(temporal_upsampling) >0: prompt += ["Temporal Upsampling"]+ if has_image_file_extension(edit_video_source) and len(temporal_upsampling) > 0:+ gr.Info("Temporal Upsampling can not be used with an Image")+ return ret()+ film_grain_intensity = inputs.get("film_grain_intensity",0)+ film_grain_saturation = inputs.get("film_grain_saturation",0.5)+ # if film_grain_intensity >0: prompt += [f"Film Grain: intensity={film_grain_intensity}, saturation={film_grain_saturation}"]+ if film_grain_intensity >0: prompt += ["Film Grain"]+ elif mode =="edit_remux":+ MMAudio_setting = inputs.get("MMAudio_setting",0)+ repeat_generation= inputs.get("repeat_generation",1)audio_source = inputs["audio_source"]if MMAudio_setting== 1:prompt += ["MMAudio"]@@ -270,9 +274,9 @@ def process_prompt_and_add_tasks(state, model_choice):gr.Info("You must provide a custom Audio")return ret()prompt += ["Custom Audio"]- repeat_generation == 1-- seed = inputs.get("seed",None)+ repeat_generation = 1+ seed = inputs.get("seed",None)+ inputs["repeat_generation"] = repeat_generationif len(prompt) == 0:if mode=="edit_remux":gr.Info("You must choose at least one Remux Method")@@ -564,7 +568,7 @@ def process_prompt_and_add_tasks(state, model_choice):if test_any_sliding_window(model_type) and image_mode == 0:if video_length > sliding_window_size:- if model_type in ["t2v", "t2v_2_2"] and not "G" in video_prompt_type :+ if test_class_t2v(model_type) and not "G" in video_prompt_type :gr.Info(f"You have requested to Generate Sliding Windows with a Text to Video model. Unless you use the Video to Video feature this is useless as a t2v model doesn't see past frames and it will generate the same video in each new window.")return ret()full_video_length = video_length if video_source is None else video_length + sliding_window_overlap -1@@ -728,10 +732,10 @@ def add_video_task(**inputs):queue.append({"id": current_task_id,"params": inputs.copy(),- "repeats": inputs["repeat_generation"],- "length": inputs["video_length"], # !!!- "steps": inputs["num_inference_steps"],- "prompt": inputs["prompt"],+ "repeats": inputs.get("repeat_generation",1),+ "length": inputs.get("video_length",1),+ "steps": inputs.get("num_inference_steps",1),+ "prompt": inputs.get("prompt", ""),"start_image_labels": start_image_labels,"end_image_labels": end_image_labels,"start_image_data": start_image_data,@@ -1962,6 +1966,10 @@ def test_vace_module(model_type):model_def = get_model_def(model_type)return model_def.get("vace_class", False)+def test_class_t2v(model_type):+ model_def = get_model_def(model_type)+ return model_def.get("t2v_class", False)+def test_any_sliding_window(model_type):model_def = get_model_def(model_type)return model_def.get("sliding_window", False)@@ -4208,7 +4216,7 @@ def edit_video(abort = False-+ MMAudio_setting = MMAudio_setting or 0configs, _ = get_settings_from_file(state, video_source, False, False, False)if configs == None: configs = { "type" : get_model_record("Post Processing") }@@ -4234,28 +4242,31 @@ def edit_video(frames_count = min(frames_count, max_source_video_frames)sample = None- if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 or film_grain_intensity > 0:- send_cmd("progress", [0, get_latest_status(state,"Upsampling" if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 else "Adding Film Grain" )])- sample = get_resampled_video(video_source, 0, max_source_video_frames, fps)- sample = sample.float().div_(127.5).sub_(1.).permute(-1,0,1,2)- frames_count = sample.shape[1]-- output_fps = round(fps)- if len(temporal_upsampling) > 0:- sample, previous_last_frame, output_fps = perform_temporal_upsampling(sample, None, temporal_upsampling, fps)- configs["temporal_upsampling"] = temporal_upsampling- frames_count = sample.shape[1]--- if len(spatial_upsampling) > 0:- sample = perform_spatial_upsampling(sample, spatial_upsampling )- configs["spatial_upsampling"] = spatial_upsampling-- if film_grain_intensity > 0:- from postprocessing.film_grain import add_film_grain- sample = add_film_grain(sample, film_grain_intensity, film_grain_saturation)- configs["film_grain_intensity"] = film_grain_intensity- configs["film_grain_saturation"] = film_grain_saturation+ if mode == "edit_postprocessing":+ if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 or film_grain_intensity > 0:+ send_cmd("progress", [0, get_latest_status(state,"Upsampling" if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 else "Adding Film Grain" )])+ sample = get_resampled_video(video_source, 0, max_source_video_frames, fps)+ sample = sample.float().div_(127.5).sub_(1.).permute(-1,0,1,2)+ frames_count = sample.shape[1]++ output_fps = round(fps)+ if len(temporal_upsampling) > 0:+ sample, previous_last_frame, output_fps = perform_temporal_upsampling(sample, None, temporal_upsampling, fps)+ configs["temporal_upsampling"] = temporal_upsampling+ frames_count = sample.shape[1]+++ if len(spatial_upsampling) > 0:+ sample = perform_spatial_upsampling(sample, spatial_upsampling )+ configs["spatial_upsampling"] = spatial_upsampling++ if film_grain_intensity > 0:+ from postprocessing.film_grain import add_film_grain+ sample = add_film_grain(sample, film_grain_intensity, film_grain_saturation)+ configs["film_grain_intensity"] = film_grain_intensity+ configs["film_grain_saturation"] = film_grain_saturation+ else:+ output_fps = round(fps)any_mmaudio = MMAudio_setting != 0 and server_config.get("mmaudio_enabled", 0) != 0 and frames_count >=output_fpsif any_mmaudio: download_mmaudio()@@ -5379,10 +5390,11 @@ def generate_video(skip_steps_cache.previous_residual = Noneskip_steps_cache.previous_modulated_input = Noneprint(f"Skipped Steps:{skip_steps_cache.skipped_steps}/{skip_steps_cache.num_steps}" )-+ BGRA_frames = Noneif samples != None:if isinstance(samples, dict):overlapped_latents = samples.get("latent_slice", None)+ BGRA_frames = samples.get("BGRA_frames", None)samples= samples["x"]samples = samples.to("cpu")clear_gen_cache()@@ -5463,7 +5475,10 @@ def generate_video(file_name = f"{time_flag}_seed{seed}_{sanitize_file_name(truncate_for_filesystem(save_prompt,100)).strip()}.{extension}"video_path = os.path.join(save_path, file_name)any_mmaudio = MMAudio_setting != 0 and server_config.get("mmaudio_enabled", 0) != 0 and sample.shape[1] >=fps-+ if BGRA_frames is not None:+ from models.wan.alpha.utils import write_zip_file+ write_zip_file(os.path.splitext(video_path)[0] + ".zip", BGRA_frames)+ BGRA_frames = Noneif is_image:image_path = os.path.join(image_save_path, file_name)sample = sample.transpose(1,0) #c f h w -> f c h w@@ -6311,7 +6326,7 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = Noneinputs["base_model_type"] = base_model_typediffusion_forcing = base_model_type in ["sky_df_1.3B", "sky_df_14B"]vace = test_vace_module(base_model_type)- t2v= base_model_type in ["t2v", "t2v_2_2"]+ t2v= test_class_t2v(base_model_type)ltxv = base_model_type in ["ltxv_13B"]recammaster = base_model_type in ["recam_1.3B"]phantom = base_model_type in ["phantom_1.3B", "phantom_14B"]@@ -7621,7 +7636,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Noncancel_lset_btn = gr.Button("Don't do it !", size="sm", min_width= 1 , visible=False)#confirm_save_lset_btn, confirm_delete_lset_btn, save_lset_btn, delete_lset_btn, cancel_lset_btntrigger_refresh_input_type = gr.Text(interactive= False, visible= False)- t2v = base_model_type in ["t2v", "t2v_2_2"]+ t2v = test_class_t2v(base_model_type)t2v_1_3B = base_model_type in ["t2v_1.3B"]flf2v = base_model_type == "flf2v_720p"base_model_family = get_model_family(base_model_type)