Skip to content
Python

support for alpha

DeepBeepMeep9ae8b584b3bc
diff --git a/defaults/alpha.json b/defaults/alpha.json
new file mode 100644
index 0000000..641fbde
--- /dev/null
+++ b/defaults/alpha.json
@@ -0,0 +1,19 @@
+{
+ "model":
+ {
+ "name": "Wan2.1 Alpha 14B",
+ "architecture" : "alpha",
+ "description": "This model successfully generates various scenes with accurate and clearly rendered transparency. Notably, it can synthesize diverse semi-transparent objects, glowing effects, and fine-grained details such as hair. For each video generated you will find a Zip file with the same name that will contain the corresponding RGBA images.",
+ "URLs": "t2v",
+ "preload_URLs": [
+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan_alpha_2.1_vae_rgb_channel.safetensors",
+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan_alpha_2.1_vae_alpha_channel.safetensors"
+ ],
+ "loras": [
+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan_alpha_2.1_dora.safetensors"
+ ],
+ "loras_multipliers": [ 1 ]
+ },
+ "prompt": "A large orange octopus is seen resting. The background of the video is transparent."
+
+}
\ No newline at end of file
diff --git a/defaults/alpha_sf.json b/defaults/alpha_sf.json
new file mode 100644
index 0000000..4d273b9
--- /dev/null
+++ b/defaults/alpha_sf.json
@@ -0,0 +1,17 @@
+{
+ "model":
+ {
+ "name": "Wan2.1 Alpha Lightning 14B",
+ "architecture" : "alpha",
+ "description": "This model is accelerated by the Lightning / SelfForcing process. It successfully generates various scenes with accurate and clearly rendered transparency. Notably, it can synthesize diverse semi-transparent objects, glowing effects, and fine-grained details such as hair. For each video generated you will find a Zip file with the same name that will contain the corresponding RGBA images.",
+ "URLs": "t2v_sf",
+ "preload_URLs": "alpha",
+ "loras": "alpha",
+ "loras_multipliers": [ 1 ],
+ "profiles_dir" : [""]
+ },
+ "prompt": "A large orange octopus is seen resting. The background of the video is transparent.",
+ "num_inference_steps": 4,
+ "guidance_scale": 1,
+ "flow_shift": 3
+}
\ No newline at end of file
diff --git a/defaults/t2v_sf.json b/defaults/t2v_sf.json
index 2131413..d061ca3 100644
--- a/defaults/t2v_sf.json
+++ b/defaults/t2v_sf.json
@@ -1,8 +1,8 @@
{
"model": {
- "name": "Wan2.1 Text2video Self-Forcing 14B",
+ "name": "Wan2.1 Text2video Lightning 14B",
"architecture": "t2v",
- "description": "This model is an advanced text-to-video generation model. This approach allows the model to generate videos with significantly fewer inference steps (4 or 8 steps) and without classifier-free guidance, substantially reducing video generation time while maintaining high quality outputs.",
+ "description": "This model is an advanced text-to-video generation model. The Lightning / SelfForcing process allows the model to generate videos with significantly fewer inference steps (4 or 8 steps) and without classifier-free guidance, substantially reducing video generation time while maintaining high quality outputs.",
"URLs": [
"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_bf16.safetensors",
"https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/wan2.1_StepDistill-CfgDistill_14B_quanto_bf16_int8.safetensors",
@@ -11,28 +11,7 @@
"author": "https://huggingface.co/lightx2v/Wan2.1-T2V-14B-StepDistill-CfgDistill",
"auto_quantize": true
},
- "negative_prompt": "",
- "prompt": "",
- "resolution": "832x480",
- "video_length": 81,
- "seed": -1,
"num_inference_steps": 4,
"guidance_scale": 1,
- "flow_shift": 3,
- "embedded_guidance_scale": 6,
- "repeat_generation": 1,
- "multi_images_gen_type": 0,
- "tea_cache_setting": 0,
- "tea_cache_start_step_perc": 0,
- "loras_multipliers": "",
- "temporal_upsampling": "",
- "spatial_upsampling": "",
- "RIFLEx_setting": 0,
- "slg_switch": 0,
- "slg_start_perc": 10,
- "slg_end_perc": 90,
- "cfg_star_switch": 0,
- "cfg_zero_step": -1,
- "prompt_enhancer": "",
- "activated_loras": []
+ "flow_shift": 3
}
\ No newline at end of file
diff --git a/models/wan/alpha/utils.py b/models/wan/alpha/utils.py
new file mode 100644
index 0000000..066fe4d
--- /dev/null
+++ b/models/wan/alpha/utils.py
@@ -0,0 +1,80 @@
+import torchvision
+from PIL import Image, ImageDraw
+import imageio
+import cv2
+import torch
+import numpy as np
+import zipfile
+
+def render_video(tensor_fgr,
+ tensor_pha,
+ nrow=8,
+ normalize=True,
+ value_range=(-1, 1)):
+
+ tensor_fgr = tensor_fgr.clamp(min(value_range), max(value_range))
+ tensor_fgr = torch.stack([
+ torchvision.utils.make_grid(
+ u, nrow=nrow, normalize=normalize, value_range=value_range)
+ for u in tensor_fgr.unbind(2)
+ ],
+ dim=1).permute(1, 2, 3, 0)
+ tensor_fgr = (tensor_fgr * 255).type(torch.uint8).cpu()
+
+ tensor_pha = tensor_pha.clamp(min(value_range), max(value_range))
+ tensor_pha = torch.stack([
+ torchvision.utils.make_grid(
+ u, nrow=nrow, normalize=normalize, value_range=value_range)
+ for u in tensor_pha.unbind(2)
+ ],
+ dim=1).permute(1, 2, 3, 0)
+ tensor_pha = (tensor_pha * 255).type(torch.uint8).cpu()
+
+ frames = []
+ frames_fgr = []
+ frames_pha = []
+ for frame_fgr, frame_pha in zip(tensor_fgr.numpy(), tensor_pha.numpy()):
+ frame_pha = (0.0 + frame_pha[:,:,0:1] + frame_pha[:,:,1:2] + frame_pha[:,:,2:3]) / 3.
+ frame = np.concatenate([frame_fgr[:,:,::-1], frame_pha.astype(np.uint8)], axis=2)
+ frames.append(frame)
+ frames_fgr.append(frame_fgr)
+ frames_pha.append(frame_pha)
+
+ def create_checkerboard(size=30, pattern_size=(830, 480), color1=(140, 140, 140), color2=(113, 113, 113)):
+ img = Image.new('RGB', (pattern_size[0], pattern_size[1]), color1)
+ draw = ImageDraw.Draw(img)
+ for i in range(0, pattern_size[0], size):
+ for j in range(0, pattern_size[1], size):
+ if (i + j) // size % 2 == 0:
+ draw.rectangle([i, j, i+size, j+size], fill=color2)
+ return img
+
+ def blender_background(frame_rgba, checkerboard):
+ alpha_channel = frame_rgba[:, :, 3:] / 255.
+ checkerboard = np.array(checkerboard)
+ checkerboard = cv2.resize(checkerboard, (frame_rgba.shape[1], frame_rgba.shape[0]))
+
+ frame_rgb = frame_rgba[:, :, :3] * alpha_channel + checkerboard * (1-alpha_channel)
+ return frame_rgb.astype(np.uint8)[:,:,::-1]
+
+ checkerboard = create_checkerboard()
+ video_checkerboard = [torch.from_numpy(blender_background(f, checkerboard).copy()).float().div_(127.5).sub_(1) for f in frames]
+ video_checkerboard = torch.stack(video_checkerboard ).permute(3, 0, 1, 2)
+ return video_checkerboard, frames
+
+def from_BRGA_numpy_to_RGBA_torch(video):
+ video = [torch.from_numpy(f.copy()).float().div_(127.5).sub_(1) for f in video]
+ video = torch.stack(video).permute(3, 0, 1, 2)
+ video[[0, 2], ...] = video[[2, 0], ...]
+ return video
+
+def write_zip_file(zip_path, frames):
+ with zipfile.ZipFile(zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
+ for idx, img in enumerate(frames):
+ success, buffer = cv2.imencode(".png", img)
+ if not success:
+ print(f"Failed to encode image {idx}, skipping...")
+ continue
+
+ filename = f"img_{idx:03d}.png"
+ zipf.writestr(filename, buffer.tobytes())
\ No newline at end of file
diff --git a/models/wan/any2video.py b/models/wan/any2video.py
index d2f6780..92c73e9 100644
--- a/models/wan/any2video.py
+++ b/models/wan/any2video.py
@@ -121,18 +121,27 @@ class WanAny2V:
ignore_unused_weights = model_def.get("ignore_unused_weights", False)
- if base_model_type in ["ti2v_2_2", "lucy_edit"]:
+ vae_checkpoint2 = None
+ if model_def.get("wan_5B_class", False):
self.vae_stride = (4, 16, 16)
vae_checkpoint = "Wan2.2_VAE.safetensors"
vae = Wan2_2_VAE
else:
- self.vae_stride = config.vae_stride
- vae_checkpoint = "Wan2.1_VAE.safetensors"
vae = WanVAE
+ self.vae_stride = config.vae_stride
+ if model_def.get("alpha_class", False):
+ vae_checkpoint ="wan_alpha_2.1_vae_rgb_channel.safetensors"
+ vae_checkpoint2 ="wan_alpha_2.1_vae_alpha_channel.safetensors"
+ else:
+ vae_checkpoint = "Wan2.1_VAE.safetensors"
self.patch_size = config.patch_size
self.vae = vae( vae_pth=fl.locate_file(vae_checkpoint), dtype= VAE_dtype, device="cpu")
self.vae.device = self.device
+ self.vae2 = None
+ if vae_checkpoint2 is not None:
+ self.vae2 = vae( vae_pth=fl.locate_file(vae_checkpoint2), dtype= VAE_dtype, device="cpu")
+ self.vae2.device = self.device
# config_filename= "configs/t2v_1.3B.json"
# import json
@@ -306,7 +315,7 @@ class WanAny2V:
msk = msk.transpose(1,2)[0]
return msk
- def encode_reference_images(self, ref_images, ref_prompt="image of a face", any_guidance= False, tile_size = None):
+ def encode_reference_images(self, ref_images, ref_prompt="image of a face", any_guidance= False, tile_size = None, enable_loras = True):
ref_images = [convert_image_to_tensor(img).unsqueeze(1).to(device=self.device, dtype=self.dtype) for img in ref_images]
shape = ref_images[0].shape
freqs = get_rotary_pos_embed( (len(ref_images) , shape[-2] // 8, shape[-1] // 8 ))
@@ -320,6 +329,9 @@ class WanAny2V:
context = torch.cat([context, context.new_zeros(self.model.text_len -context.size(0), context.size(1)) ]).unsqueeze(0)
clear_caches()
get_cache("lynx_ref_buffer").update({ 0: {}, 1: {} })
+ if not enable_loras:
+ _loras_active_adapters = self.model._loras_active_adapters
+ self.model._loras_active_adapters = []
ref_buffer = self.model(
pipeline =self,
x = [vae_feat, vae_feat_uncond] if any_guidance else [vae_feat],
@@ -328,6 +340,9 @@ class WanAny2V:
t=torch.stack([torch.tensor(0, dtype=torch.float)]).to(self.device),
lynx_feature_extractor = True,
)
+ if not enable_loras:
+ self.model._loras_active_adapters = _loras_active_adapters
+
clear_caches()
return ref_buffer[0], (ref_buffer[1] if any_guidance else None)
@@ -480,29 +495,30 @@ class WanAny2V:
if NAG_scale > 1: context = torch.cat([context, context_null], dim=0)
# if NAG_scale > 1: context = torch.cat([context, context_NAG], dim=0)
if self._interrupt: return None
-
- vace = model_type in ["vace_1.3B","vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]
+ model_def = self.model_def
+ vace = model_def.get("vace_class", False)
phantom = model_type in ["phantom_1.3B", "phantom_14B"]
fantasy = model_type in ["fantasy"]
- multitalk = model_type in ["multitalk", "infinitetalk", "vace_multitalk_14B", "i2v_2_2_multitalk"]
+ multitalk = model_def.get("multitalk_class", False)
infinitetalk = model_type in ["infinitetalk"]
- standin = model_type in ["standin", "vace_standin_14B"]
- lynx = model_type in ["lynx_lite", "lynx", "vace_lynx_lite_14B", "vace_lynx_14B"]
+ standin = model_def.get("standin_class", False)
+ lynx = model_def.get("lynx_class", False)
recam = model_type in ["recam_1.3B"]
- ti2v = model_type in ["ti2v_2_2", "lucy_edit"]
+ ti2v = model_def.get("wan_5B_class", False)
+ alpha_class = model_def.get("alpha_class", False)
lucy_edit= model_type in ["lucy_edit"]
animate= model_type in ["animate"]
start_step_no = 0
ref_images_count = 0
trim_frames = 0
- extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = None
+ extended_overlapped_latents = clip_image_start = clip_image_end = image_mask_latents = latent_slice = None
no_noise_latents_injection = infinitetalk
timestep_injection = False
lat_frames = int((frame_num - 1) // self.vae_stride[0]) + 1
extended_input_dim = 0
ref_images_before = False
# image2video
- if model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "flf2v_720p"]:
+ if model_def.get("i2v_class", False) and not animate:
any_end_frame = False
if infinitetalk:
new_shot = "0" in video_prompt_type
@@ -730,12 +746,11 @@ class WanAny2V:
image_ref = original_input_ref_images[-1]
from preprocessing.face_preprocessor import FaceProcessor
face_processor = FaceProcessor()
- lynx_ref = face_processor.process(image_ref, resize_to = 256 )
- lynx_ref_buffer, lynx_ref_buffer_uncond = self.encode_reference_images([lynx_ref], tile_size=VAE_tile_size, any_guidance= any_guidance_at_all)
+ lynx_ref = face_processor.process(image_ref, resize_to = 256)
+ lynx_ref_buffer, lynx_ref_buffer_uncond = self.encode_reference_images([lynx_ref], tile_size=VAE_tile_size, any_guidance= any_guidance_at_all, enable_loras = False)
lynx_ref = None
gc.collect()
torch.cuda.empty_cache()
- vace_lynx = model_type in ["vace_lynx_14B"]
kwargs["lynx_ip_scale"] = control_scale_alt
kwargs["lynx_ref_scale"] = control_scale_alt
@@ -744,13 +759,13 @@ class WanAny2V:
from preprocessing.face_preprocessor import FaceProcessor
standin_ref_pos = 1 if "K" in video_prompt_type else 0
if len(original_input_ref_images) < standin_ref_pos + 1:
- if "I" in video_prompt_type and model_type in ["vace_standin_14B"]:
+ if "I" in video_prompt_type and vace:
print("Warning: Missing Standin ref image, make sure 'Inject only People / Objets' is selected or if there is 'Landscape and then People or Objects' there are at least two ref images.")
else:
standin_ref_pos = -1
image_ref = original_input_ref_images[standin_ref_pos]
face_processor = FaceProcessor()
- standin_ref = face_processor.process(image_ref, remove_bg = model_type in ["vace_standin_14B"])
+ standin_ref = face_processor.process(image_ref, remove_bg = vace)
face_processor = None
gc.collect()
torch.cuda.empty_cache()
@@ -764,7 +779,7 @@ class WanAny2V:
# vace context encode
input_frames = [input_frames.to(self.device)] +([] if input_frames2 is None else [input_frames2.to(self.device)])
input_masks = [input_masks.to(self.device)] + ([] if input_masks2 is None else [input_masks2.to(self.device)])
- if model_type in ["vace_lynx_14B"] and input_ref_images is not None:
+ if lynx and input_ref_images is not None:
input_ref_images,input_ref_masks = input_ref_images[:-1], input_ref_masks[:-1]
input_ref_images = None if input_ref_images is None else [ u.to(self.device) for u in input_ref_images]
input_ref_masks = None if input_ref_masks is None else [ None if u is None else u.to(self.device) for u in input_ref_masks]
@@ -1103,11 +1118,16 @@ class WanAny2V:
self.model.release_chipmunk() # need to add it at every exit when in prod
videos = self.vae.decode(x0, VAE_tile_size)
+ any_vae2= self.vae2 is not None
+ if any_vae2:
+ videos2 = self.vae2.decode(x0, VAE_tile_size)
if image_outputs:
videos = torch.cat([video[:,:1] for video in videos], dim=1) if len(videos) > 1 else videos[0][:,:1]
+ if any_vae2: videos2 = torch.cat([video[:,:1] for video in videos2], dim=1) if len(videos2) > 1 else videos2[0][:,:1]
else:
videos = videos[0] # return only first video
+ if any_vae2: videos2 = videos2[0] # return only first video
if color_correction_strength > 0 and (prefix_frames_count > 0 and window_no > 1 or prefix_frames_count > 1 and window_no == 1):
if vace and False:
# videos = match_and_blend_colors_with_mask(videos.unsqueeze(0), input_frames[0].unsqueeze(0), input_masks[0][:1].unsqueeze(0), color_correction_strength,copy_mode= "progressive_blend").squeeze(0)
@@ -1115,9 +1135,16 @@ class WanAny2V:
# videos = match_and_blend_colors_with_mask(videos.unsqueeze(0), videos.unsqueeze(0), input_masks[0][:1].unsqueeze(0), color_correction_strength,copy_mode= "reference").squeeze(0)
elif color_reference_frame is not None:
videos = match_and_blend_colors(videos.unsqueeze(0), color_reference_frame.unsqueeze(0), color_correction_strength).squeeze(0)
-
- if return_latent_slice != None:
- return { "x" : videos, "latent_slice" : latent_slice }
+
+ BGRA_frames = None
+ if alpha_class:
+ from .alpha.utils import render_video, from_BRGA_numpy_to_RGBA_torch
+ videos, BGRA_frames = render_video(videos[None], videos2[None])
+ if image_outputs:
+ videos = from_BRGA_numpy_to_RGBA_torch(BGRA_frames)
+ BGRA_frames = None
+ if return_latent_slice != None or BGRA_frames != None:
+ return { "x" : videos, "latent_slice" : latent_slice, "BGRA_frames" : BGRA_frames }
return videos
def adapt_vace_model(self, model):
diff --git a/models/wan/configs/alpha.json b/models/wan/configs/alpha.json
new file mode 100644
index 0000000..c554e73
--- /dev/null
+++ b/models/wan/configs/alpha.json
@@ -0,0 +1,14 @@
+{
+ "_class_name": "WanModel",
+ "_diffusers_version": "0.30.0",
+ "dim": 5120,
+ "eps": 1e-06,
+ "ffn_dim": 13824,
+ "freq_dim": 256,
+ "in_dim": 16,
+ "model_type": "t2v",
+ "num_heads": 40,
+ "num_layers": 40,
+ "out_dim": 16,
+ "text_len": 512
+}
diff --git a/models/wan/wan_handler.py b/models/wan/wan_handler.py
index f70e376..a81f131 100644
--- a/models/wan/wan_handler.py
+++ b/models/wan/wan_handler.py
@@ -3,9 +3,15 @@ import numpy as np
import gradio as gr
from shared.utils import files_locator as fl
+def test_vace(base_model_type):
+ return base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]
+
def test_class_i2v(base_model_type):
return base_model_type in ["i2v", "i2v_2_2", "fun_inp_1.3B", "fun_inp", "flf2v_720p", "fantasy", "multitalk", "infinitetalk", "i2v_2_2_multitalk", "animate" ]
+def test_class_t2v(base_model_type):
+ return base_model_type in ["t2v", "t2v_2_2", "alpha"]
+
def text_oneframe_overlap(base_model_type):
return test_class_i2v(base_model_type) and not (test_multitalk(base_model_type) or base_model_type in ["animate"]) or test_wan_5B(base_model_type)
@@ -19,7 +25,10 @@ def test_standin(base_model_type):
return base_model_type in ["standin", "vace_standin_14B"]
def test_lynx(base_model_type):
- return base_model_type in ["lynx_lite", "vace_lynx_lite_14B", "lynx", "vace_lynx_14B"]
+ return base_model_type in ["lynx_lite", "vace_lynx_lite_14B", "lynx", "vace_lynx_14B", "alpha_lynx"]
+
+def test_alpha(base_model_type):
+ return base_model_type in ["alpha", "alpha_lynx"]
def test_wan_5B(base_model_type):
return base_model_type in ["ti2v_2_2", "lucy_edit"]
@@ -29,7 +38,7 @@ class family_handler():
def query_supported_types():
return ["multitalk", "infinitetalk", "fantasy", "vace_14B", "vace_14B_2_2", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B",
"t2v_1.3B", "standin", "lynx_lite", "lynx", "t2v", "t2v_2_2", "vace_1.3B", "phantom_1.3B", "phantom_14B",
- "recam_1.3B", "animate",
+ "recam_1.3B", "animate", "alpha", "alpha_lynx",
"i2v", "i2v_2_2", "i2v_2_2_multitalk", "ti2v_2_2", "lucy_edit", "flf2v_720p", "fun_inp_1.3B", "fun_inp"]
@@ -38,8 +47,9 @@ class family_handler():
models_eqv_map = {
"flf2v_720p" : "i2v",
- "t2v_1.3B" : "t2v",
- "t2v_2_2" : "t2v",
+ "t2v_1.3B" : "t2v",
+ "t2v_2_2" : "t2v",
+ "t2v_2_2" : "alpha",
"vace_standin_14B" : "vace_14B",
"vace_lynx_14B" : "vace_14B",
"vace_14B_2_2": "vace_14B",
@@ -47,7 +57,7 @@ class family_handler():
models_comp_map = {
"vace_14B" : [ "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_14B", "vace_14B_2_2"],
- "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_14B", "vace_14B_2_2", "t2v_1.3B", "phantom_1.3B","phantom_14B", "standin", "lynx_lite", "lynx"],
+ "t2v" : [ "vace_14B", "vace_1.3B" "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_lite_14B", "vace_lynx_14B", "vace_14B_2_2", "t2v_1.3B", "phantom_1.3B","phantom_14B", "standin", "lynx_lite", "lynx", "alpha"],
"i2v" : [ "fantasy", "multitalk", "flf2v_720p" ],
"i2v_2_2" : ["i2v_2_2_multitalk"],
"fantasy": ["multitalk"],
@@ -119,13 +129,14 @@ class family_handler():
extra_model_def = {}
if "URLs2" in model_def:
extra_model_def["no_steps_skipping"] = True
- i2v = test_class_i2v(base_model_type)
- extra_model_def["i2v_class"] = i2v
- extra_model_def["multitalk_class"] = test_multitalk(base_model_type)
- extra_model_def["standin_class"] = test_standin(base_model_type)
- extra_model_def["lynx_class"] = test_lynx(base_model_type)
- vace_class = base_model_type in ["vace_14B", "vace_14B_2_2", "vace_1.3B", "vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]
- extra_model_def["vace_class"] = vace_class
+ extra_model_def["i2v_class"] = i2v = test_class_i2v(base_model_type)
+ extra_model_def["t2v_class"] = t2v = test_class_t2v(base_model_type)
+ extra_model_def["multitalk_class"] = multitalk = test_multitalk(base_model_type)
+ extra_model_def["standin_class"] = standin = test_standin(base_model_type)
+ extra_model_def["lynx_class"] = lynx = test_lynx(base_model_type)
+ extra_model_def["alpha_class"] = alpha = test_alpha(base_model_type)
+ extra_model_def["wan_5B_class"] = wan_5B = test_wan_5B(base_model_type)
+ extra_model_def["vace_class"] = vace_class = test_vace(base_model_type)
if base_model_type in ["vace_multitalk_14B", "vace_standin_14B", "vace_lynx_14B"]:
extra_model_def["parent_model_type"] = "vace_14B"
@@ -140,6 +151,8 @@ class family_handler():
group = "wan2_2"
elif test_class_1_3B(base_model_type):
profiles_dir = "wan_1.3B"
+ elif base_model_type in ["alpha"]:
+ profiles_dir = "wan_alpha"
else:
profiles_dir = "wan"
@@ -148,11 +161,11 @@ class family_handler():
if base_model_type in ["animate"]:
fps = 30
- elif test_multitalk(base_model_type):
+ elif multitalk:
fps = 25
elif base_model_type in ["fantasy"]:
fps = 23
- elif test_wan_5B(base_model_type):
+ elif wan_5B:
fps = 24
else:
fps = 16
@@ -165,7 +178,7 @@ class family_handler():
extra_model_def.update({
"frames_minimum" : frames_minimum,
"frames_steps" : frames_steps,
- "sliding_window" : base_model_type in ["multitalk", "infinitetalk", "t2v", "t2v_2_2", "fantasy", "animate"] or test_class_i2v(base_model_type) or test_wan_5B(base_model_type) or vace_class, #"ti2v_2_2",
+ "sliding_window" : base_model_type in ["multitalk", "infinitetalk", "t2v", "t2v_2_2", "fantasy", "animate", "lynx"] or test_class_i2v(base_model_type) or test_wan_5B(base_model_type) or vace_class, #"ti2v_2_2",
"multiple_submodels" : multiple_submodels,
"guidance_max_phases" : 3,
"skip_layer_guidance" : True,
@@ -184,21 +197,22 @@ class family_handler():
})
- if base_model_type in ["t2v", "t2v_2_2"]:
- extra_model_def["guide_custom_choices"] = {
- "choices":[("Use Text Prompt Only", ""),
- ("Video to Video guided by Text Prompt", "GUV"),
- ("Video to Video guided by Text Prompt and Restricted to the Area of the Video Mask", "GVA")],
- "default": "",
- "show_label" : False,
- "letters_filter": "GUVA",
- "label": "Video to Video"
- }
+ if t2v:
+ if not alpha:
+ extra_model_def["guide_custom_choices"] = {
+ "choices":[("Use Text Prompt Only", ""),
+ ("Video to Video guided by Text Prompt", "GUV"),
+ ("Video to Video guided by Text Prompt and Restricted to the Area of the Video Mask", "GVA")],
+ "default": "",
+ "show_label" : False,
+ "letters_filter": "GUVA",
+ "label": "Video to Video"
+ }
- extra_model_def["mask_preprocessing"] = {
- "selection":[ "", "A"],
- "visible": False
- }
+ extra_model_def["mask_preprocessing"] = {
+ "selection":[ "", "A"],
+ "visible": False
+ }
extra_model_def["v2i_switch_supported"] = True
@@ -309,7 +323,7 @@ class family_handler():
extra_model_def["forced_guide_mask_inputs"] = True
extra_model_def["return_image_refs_tensor"] = True
extra_model_def["v2i_switch_supported"] = True
- if base_model_type in ["vace_lynx_14B"]:
+ if lynx:
extra_model_def["set_video_prompt_type"]="Q"
extra_model_def["control_net_weight_alt_name"] = "Lynx"
extra_model_def["image_ref_choices"]["choices"] = [("None", ""),
@@ -321,7 +335,7 @@ class family_handler():
- if base_model_type in ["standin"]:
+ if (not vace_class) and standin:
extra_model_def["v2i_switch_supported"] = True
extra_model_def["image_ref_choices"] = {
"choices": [
@@ -333,7 +347,7 @@ class family_handler():
}
extra_model_def["one_image_ref_needed"] = True
- if base_model_type in ["lynx_lite", "lynx"]:
+ if (not vace_class) and lynx:
extra_model_def["fit_into_canvas_image_refs"] = 0
extra_model_def["guide_custom_choices"] = {
"choices":[("Use Reference Image which is a Person Face", ""),
@@ -396,7 +410,7 @@ class family_handler():
"visible" : False,
}
- if vace_class or base_model_type in ["animate", "t2v", "t2v_2_2"] :
+ if vace_class or base_model_type in ["animate", "t2v", "t2v_2_2", "lynx"] :
image_prompt_types_allowed = "TVL"
elif base_model_type in ["infinitetalk"]:
image_prompt_types_allowed = "TSVL"
@@ -404,7 +418,7 @@ class family_handler():
image_prompt_types_allowed = "TSVL"
elif base_model_type in ["lucy_edit"]:
image_prompt_types_allowed = "TVL"
- elif test_multitalk(base_model_type) or base_model_type in ["fantasy"]:
+ elif multitalk or base_model_type in ["fantasy"]:
image_prompt_types_allowed = "SVL"
elif i2v:
image_prompt_types_allowed = "SEVL"
@@ -480,6 +494,8 @@ class family_handler():
)
pipe = {"transformer": wan_model.model, "text_encoder" : wan_model.text_encoder.model, "vae": wan_model.vae.model }
+ if wan_model.vae2 is not None:
+ pipe["vae2"] = wan_model.vae2.model
if hasattr(wan_model,"model2") and wan_model.model2 is not None:
pipe["transformer2"] = wan_model.model2
if hasattr(wan_model, "clip"):
@@ -600,7 +616,7 @@ class family_handler():
"remove_background_images_ref" : 1 ,
})
- elif base_model_type in ["lynx_lite", "lynx"]:
+ elif (base_model_type in ["lynx_lite", "lynx", "alpha_lynx"]):
ui_defaults.update({
"guidance_scale": 5.0,
"flow_shift": 7, # 11 for 720p
diff --git a/profiles/wan_alpha/Text2Video Lightx2v Cfg Step Distill Rank64 - 4 Steps.json b/profiles/wan_alpha/Text2Video Lightx2v Cfg Step Distill Rank64 - 4 Steps.json
new file mode 100644
index 0000000..1718e3e
--- /dev/null
+++ b/profiles/wan_alpha/Text2Video Lightx2v Cfg Step Distill Rank64 - 4 Steps.json
@@ -0,0 +1,11 @@
+{
+
+ "activated_loras": [
+ "https://huggingface.co/DeepBeepMeep/Wan2.1/resolve/main/loras_accelerators/lightx2v_T2V_14B_cfg_step_distill_v2_lora_rank64_bf16.safetensors"
+ ],
+ "loras_multipliers": "1",
+ "num_inference_steps": 4,
+ "guidance_phases": 1,
+ "guidance_scale": 1,
+ "flow_shift": 2
+}
\ No newline at end of file
diff --git a/shared/utils/audio_video.py b/shared/utils/audio_video.py
index 224cf34..3f59cc7 100644
--- a/shared/utils/audio_video.py
+++ b/shared/utils/audio_video.py
@@ -301,7 +301,11 @@ def save_image(tensor,
quality='jpeg_95', # 'jpeg_95', 'jpeg_85', 'jpeg_70', 'jpeg_50', 'webp_95', 'webp_85', 'webp_70', 'webp_50', 'png', 'webp_lossless'
retry=5):
"""Save tensor as image with configurable format and quality."""
-
+
+ RGBA = tensor.shape[0] == 4
+ if RGBA:
+ quality = "png"
+
# Get format and quality settings
format_info = _get_format_info(quality)
@@ -310,16 +314,18 @@ def save_image(tensor,
# Save image
error = None
+
for _ in range(retry):
try:
tensor = tensor.clamp(min(value_range), max(value_range))
- if format_info['use_pil']:
+ if format_info['use_pil'] or RGBA:
# Use PIL for WebP and advanced options
grid = torchvision.utils.make_grid(tensor, nrow=nrow, normalize=normalize, value_range=value_range)
# Convert to PIL Image
grid = grid.mul(255).add_(0.5).clamp_(0, 255).permute(1, 2, 0).to('cpu', torch.uint8).numpy()
- img = Image.fromarray(grid)
+ mode = 'RGBA' if RGBA else 'RGB'
+ img = Image.fromarray(grid, mode=mode)
img.save(save_file, **format_info['params'])
else:
# Use torchvision for JPEG and PNG
diff --git a/wgp.py b/wgp.py
index 74af14e..4df3ed0 100644
--- a/wgp.py
+++ b/wgp.py
@@ -239,27 +239,31 @@ def process_prompt_and_add_tasks(state, model_choice):
if frames_count > max_source_video_frames:
gr.Info(f"Post processing is not supported on videos longer than {max_source_video_frames} frames. Output Video will be truncated")
# return
- for k in ["image_start", "image_end", "image_refs", "video_guide", "audio_guide", "audio_guide2", "audio_source" , "video_mask", "image_mask"]:
+ for prop in ["state", "model_type", "mode"]:
+ edit_overrides[prop] = inputs[prop]
+ for k,v in inputs.items():
inputs[k] = None
inputs.update(edit_overrides)
del gen["edit_video_source"], gen["edit_overrides"]
inputs["video_source"]= edit_video_source
prompt = []
- spatial_upsampling = inputs.get("spatial_upsampling","")
- if len(spatial_upsampling) >0: prompt += ["Spatial Upsampling"]
- temporal_upsampling = inputs.get("temporal_upsampling","")
- if len(temporal_upsampling) >0: prompt += ["Temporal Upsampling"]
- if has_image_file_extension(edit_video_source) and len(temporal_upsampling) > 0:
- gr.Info("Temporal Upsampling can not be used with an Image")
- return ret()
- film_grain_intensity = inputs.get("film_grain_intensity",0)
- film_grain_saturation = inputs.get("film_grain_saturation",0.5)
- # if film_grain_intensity >0: prompt += [f"Film Grain: intensity={film_grain_intensity}, saturation={film_grain_saturation}"]
- if film_grain_intensity >0: prompt += ["Film Grain"]
- MMAudio_setting = inputs.get("MMAudio_setting",0)
- repeat_generation= inputs.get("repeat_generation",1)
- if mode =="edit_remux":
+ repeat_generation = 1
+ if mode == "edit_postprocessing":
+ spatial_upsampling = inputs.get("spatial_upsampling","")
+ if len(spatial_upsampling) >0: prompt += ["Spatial Upsampling"]
+ temporal_upsampling = inputs.get("temporal_upsampling","")
+ if len(temporal_upsampling) >0: prompt += ["Temporal Upsampling"]
+ if has_image_file_extension(edit_video_source) and len(temporal_upsampling) > 0:
+ gr.Info("Temporal Upsampling can not be used with an Image")
+ return ret()
+ film_grain_intensity = inputs.get("film_grain_intensity",0)
+ film_grain_saturation = inputs.get("film_grain_saturation",0.5)
+ # if film_grain_intensity >0: prompt += [f"Film Grain: intensity={film_grain_intensity}, saturation={film_grain_saturation}"]
+ if film_grain_intensity >0: prompt += ["Film Grain"]
+ elif mode =="edit_remux":
+ MMAudio_setting = inputs.get("MMAudio_setting",0)
+ repeat_generation= inputs.get("repeat_generation",1)
audio_source = inputs["audio_source"]
if MMAudio_setting== 1:
prompt += ["MMAudio"]
@@ -270,9 +274,9 @@ def process_prompt_and_add_tasks(state, model_choice):
gr.Info("You must provide a custom Audio")
return ret()
prompt += ["Custom Audio"]
- repeat_generation == 1
-
- seed = inputs.get("seed",None)
+ repeat_generation = 1
+ seed = inputs.get("seed",None)
+ inputs["repeat_generation"] = repeat_generation
if len(prompt) == 0:
if mode=="edit_remux":
gr.Info("You must choose at least one Remux Method")
@@ -564,7 +568,7 @@ def process_prompt_and_add_tasks(state, model_choice):
if test_any_sliding_window(model_type) and image_mode == 0:
if video_length > sliding_window_size:
- if model_type in ["t2v", "t2v_2_2"] and not "G" in video_prompt_type :
+ if test_class_t2v(model_type) and not "G" in video_prompt_type :
gr.Info(f"You have requested to Generate Sliding Windows with a Text to Video model. Unless you use the Video to Video feature this is useless as a t2v model doesn't see past frames and it will generate the same video in each new window.")
return ret()
full_video_length = video_length if video_source is None else video_length + sliding_window_overlap -1
@@ -728,10 +732,10 @@ def add_video_task(**inputs):
queue.append({
"id": current_task_id,
"params": inputs.copy(),
- "repeats": inputs["repeat_generation"],
- "length": inputs["video_length"], # !!!
- "steps": inputs["num_inference_steps"],
- "prompt": inputs["prompt"],
+ "repeats": inputs.get("repeat_generation",1),
+ "length": inputs.get("video_length",1),
+ "steps": inputs.get("num_inference_steps",1),
+ "prompt": inputs.get("prompt", ""),
"start_image_labels": start_image_labels,
"end_image_labels": end_image_labels,
"start_image_data": start_image_data,
@@ -1962,6 +1966,10 @@ def test_vace_module(model_type):
model_def = get_model_def(model_type)
return model_def.get("vace_class", False)
+def test_class_t2v(model_type):
+ model_def = get_model_def(model_type)
+ return model_def.get("t2v_class", False)
+
def test_any_sliding_window(model_type):
model_def = get_model_def(model_type)
return model_def.get("sliding_window", False)
@@ -4208,7 +4216,7 @@ def edit_video(
abort = False
-
+ MMAudio_setting = MMAudio_setting or 0
configs, _ = get_settings_from_file(state, video_source, False, False, False)
if configs == None: configs = { "type" : get_model_record("Post Processing") }
@@ -4234,28 +4242,31 @@ def edit_video(
frames_count = min(frames_count, max_source_video_frames)
sample = None
- if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 or film_grain_intensity > 0:
- send_cmd("progress", [0, get_latest_status(state,"Upsampling" if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 else "Adding Film Grain" )])
- sample = get_resampled_video(video_source, 0, max_source_video_frames, fps)
- sample = sample.float().div_(127.5).sub_(1.).permute(-1,0,1,2)
- frames_count = sample.shape[1]
-
- output_fps = round(fps)
- if len(temporal_upsampling) > 0:
- sample, previous_last_frame, output_fps = perform_temporal_upsampling(sample, None, temporal_upsampling, fps)
- configs["temporal_upsampling"] = temporal_upsampling
- frames_count = sample.shape[1]
-
-
- if len(spatial_upsampling) > 0:
- sample = perform_spatial_upsampling(sample, spatial_upsampling )
- configs["spatial_upsampling"] = spatial_upsampling
-
- if film_grain_intensity > 0:
- from postprocessing.film_grain import add_film_grain
- sample = add_film_grain(sample, film_grain_intensity, film_grain_saturation)
- configs["film_grain_intensity"] = film_grain_intensity
- configs["film_grain_saturation"] = film_grain_saturation
+ if mode == "edit_postprocessing":
+ if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 or film_grain_intensity > 0:
+ send_cmd("progress", [0, get_latest_status(state,"Upsampling" if len(temporal_upsampling) > 0 or len(spatial_upsampling) > 0 else "Adding Film Grain" )])
+ sample = get_resampled_video(video_source, 0, max_source_video_frames, fps)
+ sample = sample.float().div_(127.5).sub_(1.).permute(-1,0,1,2)
+ frames_count = sample.shape[1]
+
+ output_fps = round(fps)
+ if len(temporal_upsampling) > 0:
+ sample, previous_last_frame, output_fps = perform_temporal_upsampling(sample, None, temporal_upsampling, fps)
+ configs["temporal_upsampling"] = temporal_upsampling
+ frames_count = sample.shape[1]
+
+
+ if len(spatial_upsampling) > 0:
+ sample = perform_spatial_upsampling(sample, spatial_upsampling )
+ configs["spatial_upsampling"] = spatial_upsampling
+
+ if film_grain_intensity > 0:
+ from postprocessing.film_grain import add_film_grain
+ sample = add_film_grain(sample, film_grain_intensity, film_grain_saturation)
+ configs["film_grain_intensity"] = film_grain_intensity
+ configs["film_grain_saturation"] = film_grain_saturation
+ else:
+ output_fps = round(fps)
any_mmaudio = MMAudio_setting != 0 and server_config.get("mmaudio_enabled", 0) != 0 and frames_count >=output_fps
if any_mmaudio: download_mmaudio()
@@ -5379,10 +5390,11 @@ def generate_video(
skip_steps_cache.previous_residual = None
skip_steps_cache.previous_modulated_input = None
print(f"Skipped Steps:{skip_steps_cache.skipped_steps}/{skip_steps_cache.num_steps}" )
-
+ BGRA_frames = None
if samples != None:
if isinstance(samples, dict):
overlapped_latents = samples.get("latent_slice", None)
+ BGRA_frames = samples.get("BGRA_frames", None)
samples= samples["x"]
samples = samples.to("cpu")
clear_gen_cache()
@@ -5463,7 +5475,10 @@ def generate_video(
file_name = f"{time_flag}_seed{seed}_{sanitize_file_name(truncate_for_filesystem(save_prompt,100)).strip()}.{extension}"
video_path = os.path.join(save_path, file_name)
any_mmaudio = MMAudio_setting != 0 and server_config.get("mmaudio_enabled", 0) != 0 and sample.shape[1] >=fps
-
+ if BGRA_frames is not None:
+ from models.wan.alpha.utils import write_zip_file
+ write_zip_file(os.path.splitext(video_path)[0] + ".zip", BGRA_frames)
+ BGRA_frames = None
if is_image:
image_path = os.path.join(image_save_path, file_name)
sample = sample.transpose(1,0) #c f h w -> f c h w
@@ -6311,7 +6326,7 @@ def prepare_inputs_dict(target, inputs, model_type = None, model_filename = None
inputs["base_model_type"] = base_model_type
diffusion_forcing = base_model_type in ["sky_df_1.3B", "sky_df_14B"]
vace = test_vace_module(base_model_type)
- t2v= base_model_type in ["t2v", "t2v_2_2"]
+ t2v= test_class_t2v(base_model_type)
ltxv = base_model_type in ["ltxv_13B"]
recammaster = base_model_type in ["recam_1.3B"]
phantom = base_model_type in ["phantom_1.3B", "phantom_14B"]
@@ -7621,7 +7636,7 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
cancel_lset_btn = gr.Button("Don't do it !", size="sm", min_width= 1 , visible=False)
#confirm_save_lset_btn, confirm_delete_lset_btn, save_lset_btn, delete_lset_btn, cancel_lset_btn
trigger_refresh_input_type = gr.Text(interactive= False, visible= False)
- t2v = base_model_type in ["t2v", "t2v_2_2"]
+ t2v = test_class_t2v(base_model_type)
t2v_1_3B = base_model_type in ["t2v_1.3B"]
flf2v = base_model_type == "flf2v_720p"
base_model_family = get_model_family(base_model_type)