Skip to content
Python

added support for Qwen Image Layered

DeepBeepMeep461bc9030d19
diff --git a/README.md b/README.md
index e4d25c8..b9fac8b 100644
--- a/README.md
+++ b/README.md
@@ -36,7 +36,7 @@ WanGP supports the Wan (and derived models), Hunyuan Video and LTV Video models
-----
## 🔥 Latest Updates :
-### December 19 2025: WanGP v9.9, Early Christmas
+### December 19 2025: WanGP v9.91, Early Christmas
- **SCAIL Preview**: enjoy this *Wan Animate*, *Steady Dancer* contender that can support multiple people. Thanks to its 3D positioning, it can take into account which parts of the body are hidden and which are not.
@@ -44,6 +44,10 @@ WanGP version has the following perks: 3D pose Preprocessing entirely rewritten
- **pi-Flux 2**: you don't use Flux 2 because you find it too slow ? You won't be able to use this excuse anymore: pi-Flux 2 is *4 steps distills* of the best image generator. It supports both image edition and text to image generation.
+- **Zandinksy v5** : for the video models collectors among you, you can try the Zandinsky model families, the 2B model quality is especially impressive given its small size
+
+- **Qwen Image Layered**: a new Qwen Image variant that lets you extract RGBA layers of your images so that each layer can be edited separately
+
- **loras accelerator**: *loras accelerator* for *Wan 2.2 t2v* and *Wan 2.1 i2v* have been added (activable using the *Profile settings* as usual)
### December 14 2025: WanGP v9.86, Simple Pleasures...
diff --git a/models/qwen/autoencoder_kl_qwenimage.py b/models/qwen/autoencoder_kl_qwenimage.py
index 026a2f4..80e35a5 100644
--- a/models/qwen/autoencoder_kl_qwenimage.py
+++ b/models/qwen/autoencoder_kl_qwenimage.py
@@ -395,6 +395,7 @@ class QwenImageEncoder3d(nn.Module):
attn_scales=[],
temperal_downsample=[True, True, False],
dropout=0.0,
+ input_channels: int = 3,
non_linearity: str = "silu",
):
super().__init__()
@@ -411,7 +412,7 @@ class QwenImageEncoder3d(nn.Module):
scale = 1.0
# init block
- self.conv_in = QwenImageCausalConv3d(3, dims[0], 3, padding=1)
+ self.conv_in = QwenImageCausalConv3d(input_channels, dims[0], 3, padding=1)
# downsample blocks
self.down_blocks = nn.ModuleList([])
@@ -572,6 +573,7 @@ class QwenImageDecoder3d(nn.Module):
temperal_upsample=[False, True, True],
dropout=0.0,
upsampler_factor = 1,
+ input_channels: int = 3,
non_linearity: str = "silu",
):
super().__init__()
@@ -623,7 +625,8 @@ class QwenImageDecoder3d(nn.Module):
# output blocks
self.norm_out = QwenImageRMS_norm(out_dim, images=False)
- self.conv_out = QwenImageCausalConv3d(out_dim, 3*int(upsampler_factor*upsampler_factor), 3, padding=1)
+ output_channels = input_channels * int(upsampler_factor * upsampler_factor)
+ self.conv_out = QwenImageCausalConv3d(out_dim, output_channels, 3, padding=1)
self.gradient_checkpointing = False
@@ -718,6 +721,7 @@ class AutoencoderKLQwenImage(ModelMixin, ConfigMixin, FromOriginalModelMixin):
attn_scales: List[float] = [],
temperal_downsample: List[bool] = [False, True, True],
dropout: float = 0.0,
+ input_channels: int = 3,
upsampler_factor = 1,
latents_mean: List[float] = [-0.7571, -0.7089, -0.9113, 0.1075, -0.1745, 0.9653, -0.1517, 1.5508, 0.4134, -0.0715, 0.5517, -0.3632, -0.1922, -0.9497, 0.2503, -0.2921],
latents_std: List[float] = [2.8184, 1.4541, 2.3275, 2.6558, 1.2196, 1.7708, 2.6052, 2.0743, 3.2687, 2.1526, 2.8652, 1.5579, 1.6382, 1.1253, 2.8251, 1.9160],
@@ -730,13 +734,28 @@ class AutoencoderKLQwenImage(ModelMixin, ConfigMixin, FromOriginalModelMixin):
self.temperal_upsample = temperal_downsample[::-1]
self.upsampler_factor = upsampler_factor
self.encoder = QwenImageEncoder3d(
- base_dim, z_dim * 2, dim_mult, num_res_blocks, attn_scales, self.temperal_downsample, dropout
+ base_dim,
+ z_dim * 2,
+ dim_mult,
+ num_res_blocks,
+ attn_scales,
+ self.temperal_downsample,
+ dropout,
+ input_channels,
)
self.quant_conv = QwenImageCausalConv3d(z_dim * 2, z_dim * 2, 1)
self.post_quant_conv = QwenImageCausalConv3d(z_dim, z_dim, 1)
self.decoder = QwenImageDecoder3d(
- base_dim, z_dim, dim_mult, num_res_blocks, attn_scales, self.temperal_upsample, dropout, upsampler_factor
+ base_dim,
+ z_dim,
+ dim_mult,
+ num_res_blocks,
+ attn_scales,
+ self.temperal_upsample,
+ dropout,
+ upsampler_factor,
+ input_channels,
)
self.spatial_compression_ratio = 2 ** len(self.temperal_downsample)
diff --git a/models/qwen/pipeline_qwenimage.py b/models/qwen/pipeline_qwenimage.py
index a283b68..f2ef8b3 100644
--- a/models/qwen/pipeline_qwenimage.py
+++ b/models/qwen/pipeline_qwenimage.py
@@ -472,6 +472,7 @@ class QwenImagePipeline(): #DiffusionPipeline
dtype,
device,
generator,
+ num_layers: int = 1,
latents=None,
tile_size=0,
):
@@ -517,10 +518,21 @@ class QwenImagePipeline(): #DiffusionPipeline
f" size of {batch_size}. Make sure the batch size matches the length of the generators."
)
if latents is None:
- latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
- latents = self._pack_latents(latents)
+ if num_layers > 1:
+ layer_shape = (batch_size * num_layers, num_channels_latents, 1, height, width)
+ latents = randn_tensor(layer_shape, generator=generator, device=device, dtype=dtype)
+ latents = self._pack_latents(latents)
+ seq_len = latents.shape[1]
+ dim = latents.shape[2]
+ latents = latents.view(batch_size, num_layers, seq_len, dim)
+ latents = latents.reshape(batch_size, num_layers * seq_len, dim)
+ else:
+ latents = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
+ latents = self._pack_latents(latents)
else:
latents = latents.to(device=device, dtype=dtype)
+ if num_layers > 1 and latents.shape[1] % num_layers != 0:
+ raise ValueError("Provided latents length is not divisible by the number of layers.")
return latents, image_latents
@@ -551,6 +563,8 @@ class QwenImagePipeline(): #DiffusionPipeline
num_inference_steps: int = 50,
sigmas: Optional[List[float]] = None,
num_images_per_prompt: int = 1,
+ layers: int = 1,
+ cfg_normalize: bool = True,
generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
latents: Optional[torch.Tensor] = None,
prompt_embeds: Optional[torch.Tensor] = None,
@@ -683,10 +697,15 @@ class QwenImagePipeline(): #DiffusionPipeline
batch_size = prompt_embeds.shape[0]
device = "cuda"
+ num_layers = max(1, int(layers) if layers is not None else 1)
+ effective_batch_size = batch_size * num_images_per_prompt
+ vae_input_channels = getattr(self.vae.config, "input_channels", 3)
+
condition_images = []
vae_image_sizes = []
vae_images = []
image_mask_latents = None
+ image_mask_rebuilt = None
ref_size = 1024
ref_text_encoder_size = 384 if qwen_edit_plus else 1024
if image is not None:
@@ -694,7 +713,7 @@ class QwenImagePipeline(): #DiffusionPipeline
if height * width < ref_size * ref_size: ref_size = round(math.sqrt(height * width))
for ref_no, img in enumerate(image):
image_width, image_height = img.size
- any_mask = ref_no == 0 and image_mask is not None
+ any_mask = num_layers == 1 and ref_no == 0 and image_mask is not None
if (not qwen_edit_plus) and (image_height * image_width > ref_size * ref_size) and not any_mask:
vae_height, vae_width =calculate_new_dimensions(ref_size, ref_size, image_height, image_width, False, block_size=multiple_of)
else:
@@ -703,17 +722,28 @@ class QwenImagePipeline(): #DiffusionPipeline
vae_height = vae_height // multiple_of * multiple_of
vae_image_sizes.append((vae_width, vae_height))
condition_height, condition_width =calculate_new_dimensions(ref_text_encoder_size, ref_text_encoder_size, image_height, image_width, False, block_size=multiple_of)
- condition_images.append(img.resize((condition_width, condition_height), resample=Image.Resampling.LANCZOS) )
- if img.size != (vae_width, vae_height):
- img = img.resize((vae_width, vae_height), resample=Image.Resampling.LANCZOS)
+ condition_img = img
+ if getattr(condition_img, "mode", None) != "RGB":
+ condition_img = condition_img.convert("RGB")
+ condition_images.append(condition_img.resize((condition_width, condition_height), resample=Image.Resampling.LANCZOS) )
+
+ vae_img = img
+ if vae_input_channels == 4:
+ if getattr(vae_img, "mode", None) != "RGBA":
+ vae_img = vae_img.convert("RGBA")
+ else:
+ if getattr(vae_img, "mode", None) != "RGB":
+ vae_img = vae_img.convert("RGB")
+ if vae_img.size != (vae_width, vae_height):
+ vae_img = vae_img.resize((vae_width, vae_height), resample=Image.Resampling.LANCZOS)
if any_mask :
if lora_inpaint:
image_mask_rebuilt = torch.where(convert_image_to_tensor(image_mask)>-0.5, 1., 0. )[0:1]
- img = convert_image_to_tensor(img)
- green = torch.tensor([-1.0, 1.0, -1.0]).to(img)
- green_image = green[:, None, None] .expand_as(img)
- img = torch.where(image_mask_rebuilt > 0, green_image, img)
- img = convert_tensor_to_image(img)
+ vae_tensor = convert_image_to_tensor(vae_img)
+ green = torch.tensor([-1.0, 1.0, -1.0]).to(vae_tensor)
+ green_image = green[:, None, None] .expand_as(vae_tensor)
+ vae_tensor = torch.where(image_mask_rebuilt > 0, green_image, vae_tensor)
+ vae_img = convert_tensor_to_image(vae_tensor)
else:
image_mask_latents = convert_image_to_tensor(image_mask.resize((vae_width // 8, vae_height // 8), resample=Image.Resampling.LANCZOS))
image_mask_latents = torch.where(image_mask_latents>-0.5, 1., 0. )[0:1]
@@ -722,7 +752,7 @@ class QwenImagePipeline(): #DiffusionPipeline
image_mask_latents = image_mask_latents.to(device).unsqueeze(0).unsqueeze(0).repeat(1,16,1,1,1)
image_mask_latents = self._pack_latents(image_mask_latents)
# img.save("nnn.png")
- vae_images.append( convert_image_to_tensor(img).unsqueeze(0).unsqueeze(2) )
+ vae_images.append( convert_image_to_tensor(vae_img).unsqueeze(0).unsqueeze(2) )
has_neg_prompt = negative_prompt is not None or (
@@ -748,6 +778,15 @@ class QwenImagePipeline(): #DiffusionPipeline
num_images_per_prompt=num_images_per_prompt,
max_sequence_length=max_sequence_length,
)
+ additional_t_cond = None
+ if getattr(self.transformer, "use_additional_t_cond", False):
+ add_value = 1 if image is not None and len(condition_images) > 0 else 0
+ additional_t_cond = torch.full(
+ (effective_batch_size,),
+ add_value,
+ device=device,
+ dtype=torch.long,
+ )
dtype = torch.bfloat16
prompt_embeds = prompt_embeds.to(dtype)
if do_true_cfg:
@@ -757,31 +796,32 @@ class QwenImagePipeline(): #DiffusionPipeline
num_channels_latents = self.transformer.in_channels // 4
latents, image_latents = self.prepare_latents(
vae_images,
- batch_size * num_images_per_prompt,
+ effective_batch_size,
num_channels_latents,
height,
width,
prompt_embeds.dtype,
device,
generator,
- latents,
+ num_layers=num_layers,
+ latents=latents,
tile_size=VAE_tile_size
)
if image is not None:
- img_shapes = [
- [
- (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2),
- # (1, image_height // self.vae_scale_factor // 2, image_width // self.vae_scale_factor // 2),
- *[
- (1, vae_height // self.vae_scale_factor // 2, vae_width // self.vae_scale_factor // 2)
- for vae_width, vae_height in vae_image_sizes
- ],
-
- ]
- ] * batch_size
+ output_shape = (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2)
+ output_shapes = [output_shape] * num_layers if num_layers > 1 else [output_shape]
+ condition_shapes = [
+ (1, vae_height // self.vae_scale_factor // 2, vae_width // self.vae_scale_factor // 2)
+ for vae_width, vae_height in vae_image_sizes
+ ]
+ img_shapes = [output_shapes + condition_shapes] * effective_batch_size
else:
- img_shapes = [(1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2)] * batch_size
+ output_shape = (1, height // self.vae_scale_factor // 2, width // self.vae_scale_factor // 2)
+ if num_layers > 1:
+ img_shapes = [[output_shape] * num_layers] * effective_batch_size
+ else:
+ img_shapes = [output_shape] * effective_batch_size
# 5. Prepare timesteps
sigmas = np.linspace(1.0, 1 / num_inference_steps, num_inference_steps) if sigmas is None else sigmas
@@ -867,6 +907,7 @@ class QwenImagePipeline(): #DiffusionPipeline
img_shapes=img_shapes,
txt_seq_lens_list=[txt_seq_lens, negative_txt_seq_lens],
attention_kwargs=self.attention_kwargs,
+ additional_t_cond=additional_t_cond,
**kwargs
)
if noise_pred == None: return None, None
@@ -882,6 +923,7 @@ class QwenImagePipeline(): #DiffusionPipeline
img_shapes=img_shapes,
txt_seq_lens_list=[txt_seq_lens],
attention_kwargs=self.attention_kwargs,
+ additional_t_cond=additional_t_cond,
**kwargs
)[0]
if noise_pred == None: return None, None
@@ -896,6 +938,7 @@ class QwenImagePipeline(): #DiffusionPipeline
img_shapes=img_shapes,
txt_seq_lens_list=[negative_txt_seq_lens],
attention_kwargs=self.attention_kwargs,
+ additional_t_cond=additional_t_cond,
**kwargs
)[0]
if neg_noise_pred == None: return None, None
@@ -905,10 +948,12 @@ class QwenImagePipeline(): #DiffusionPipeline
if do_true_cfg:
comb_pred = neg_noise_pred + guidance * (noise_pred - neg_noise_pred)
if comb_pred == None: return None
-
- cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
- noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
- noise_pred = comb_pred * (cond_norm / noise_norm)
+ if cfg_normalize:
+ cond_norm = torch.norm(noise_pred, dim=-1, keepdim=True)
+ noise_norm = torch.norm(comb_pred, dim=-1, keepdim=True)
+ noise_pred = comb_pred * (cond_norm / noise_norm)
+ else:
+ noise_pred = comb_pred
return noise_pred
@@ -942,7 +987,11 @@ class QwenImagePipeline(): #DiffusionPipeline
latents = latents.to(latents_dtype)
if callback is not None:
- preview = self._unpack_latents(latents, height, width, self.vae_scale_factor)
+ preview_latents = latents
+ if num_layers > 1:
+ seq_len = latents.shape[1] // num_layers
+ preview_latents = latents[:, :seq_len, :]
+ preview = self._unpack_latents(preview_latents, height, width, self.vae_scale_factor)
preview = preview.transpose(0,2).squeeze(0)
callback(i, preview, False)
@@ -951,23 +1000,35 @@ class QwenImagePipeline(): #DiffusionPipeline
if output_type == "latent":
output_image = latents
else:
- latents = self._unpack_latents(latents, height, width, self.vae_scale_factor)
- latents = latents.to(self.vae.dtype)
+ latents_to_decode = latents
+ if num_layers > 1:
+ seq_len = latents.shape[1] // num_layers
+ dim = latents.shape[2]
+ latents_to_decode = latents.view(latents.shape[0], num_layers, seq_len, dim)
+ latents_to_decode = latents_to_decode.reshape(latents.shape[0] * num_layers, seq_len, dim)
+ latents_to_decode = self._unpack_latents(latents_to_decode, height, width, self.vae_scale_factor)
+ latents_to_decode = latents_to_decode.to(self.vae.dtype)
if self.use_Wan_VAE:
- output_image = torch.cat([image.transpose(0,1) for image in self.vae.decode(latents, tile_size= VAE_tile_size)])
+ output_image = torch.cat([image.transpose(0,1) for image in self.vae.decode(latents_to_decode, tile_size= VAE_tile_size)])
else:
latents_mean = (
torch.tensor(self.vae.config.latents_mean)
.view(1, vae_z_dim, 1, 1, 1)
- .to(latents.device, latents.dtype)
+ .to(latents_to_decode.device, latents_to_decode.dtype)
)
latents_std = 1.0 / torch.tensor(self.vae.config.latents_std).view(1, vae_z_dim, 1, 1, 1).to(
- latents.device, latents.dtype
+ latents_to_decode.device, latents_to_decode.dtype
)
- latents = latents / latents_std + latents_mean
- output_image = self.vae.decode(latents, return_dict=False)[0][:, :, 0]
-
- if image_mask is not None and not lora_inpaint and self.vae.upsampling_set is None and masking_strength == 1: #not (lora_inpaint and outpainting_dims is not None):
+ latents_to_decode = latents_to_decode / latents_std + latents_mean
+ output_image = self.vae.decode(latents_to_decode, return_dict=False)[0][:, :, 0]
+
+ if (
+ num_layers == 1
+ and image_mask_rebuilt is not None
+ and not lora_inpaint
+ and self.vae.upsampling_set is None
+ and masking_strength == 1
+ ):
output_image = vae_images[0].squeeze(2) * (1 - image_mask_rebuilt) + output_image.to(vae_images[0] ) * image_mask_rebuilt
diff --git a/models/qwen/qwen_handler.py b/models/qwen/qwen_handler.py
index 7167d84..c5a18ff 100644
--- a/models/qwen/qwen_handler.py
+++ b/models/qwen/qwen_handler.py
@@ -25,6 +25,17 @@ class family_handler():
extra_model_def["vae_upsampler"] = [1,2]
+ if base_model_type in ["qwen_image_layered_20B"]:
+ extra_model_def["batch_size_label"] = "Number of Layers"
+ extra_model_def["set_video_prompt_type"] = "V"
+ extra_model_def["guide_preprocessing"] = {
+ "selection": ["V"],
+ "labels": {"V": "Control Image"},
+ "visible": False,
+ }
+ extra_model_def["vae_upsampler"] = [1]
+ extra_model_def["sample_solvers"] = [("Default", "default")]
+
if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B"]:
extra_model_def["inpaint_support"] = True
extra_model_def["image_ref_inpaint"]= base_model_type in ["qwen_image_edit_plus_20B"]
@@ -64,7 +75,7 @@ class family_handler():
@staticmethod
def query_supported_types():
- return ["qwen_image_20B", "qwen_image_edit_20B", "qwen_image_edit_plus_20B"]
+ return ["qwen_image_20B", "qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_layered_20B"]
@staticmethod
def query_family_maps():
@@ -94,10 +105,13 @@ class family_handler():
@staticmethod
def query_model_files(computeList, base_model_type, model_filename, text_encoder_quantization):
text_encoder_filename = get_qwen_text_encoder_filename(text_encoder_quantization)
+ vae_files = ["qwen_vae.safetensors", "qwen_vae_config.json"]
+ if base_model_type in ["qwen_image_layered_20B"]:
+ vae_files.append("qwen_image_layered_vae_bf16.safetensors")
download_def = [{
"repoId" : "DeepBeepMeep/Qwen_image",
"sourceFolderList" : ["", "Qwen2.5-VL-7B-Instruct"],
- "fileList" : [ ["qwen_vae.safetensors", "qwen_vae_config.json"], ["merges.txt", "tokenizer_config.json", "config.json", "vocab.json", "video_preprocessor_config.json", "preprocessor_config.json", "chat_template.json"] + computeList(text_encoder_filename) ]
+ "fileList" : [ vae_files, ["merges.txt", "tokenizer_config.json", "config.json", "vocab.json", "video_preprocessor_config.json", "preprocessor_config.json", "chat_template.json"] + computeList(text_encoder_filename) ]
}]
download_def += [{
@@ -159,6 +173,10 @@ class family_handler():
"denoising_strength" : 1.,
"model_mode" : 0,
})
+ elif base_model_type in ["qwen_image_layered_20B"]:
+ ui_defaults.update({
+ "video_prompt_type": "V",
+ })
@staticmethod
def validate_generative_settings(base_model_type, model_def, inputs):
@@ -173,6 +191,9 @@ class family_handler():
gr.Info("Denoising Strength will be ignored while using Lora Inpainting")
if outpainting_dims is not None and model_mode == 0 :
return "Outpainting is not supported with Masked Denoising"
+ if base_model_type in ["qwen_image_layered_20B"]:
+ if inputs.get("image_guide") is None:
+ return "Qwen Image Layered requires a Control Image."
@staticmethod
def get_rgb_factors(base_model_type ):
diff --git a/models/qwen/qwen_main.py b/models/qwen/qwen_main.py
index 192a3ab..45747d4 100644
--- a/models/qwen/qwen_main.py
+++ b/models/qwen/qwen_main.py
@@ -53,17 +53,20 @@ class model_factory():
transformer_filename = model_filename[0]
processor = None
tokenizer = None
- if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B"]:
+ if base_model_type in ["qwen_image_edit_20B", "qwen_image_edit_plus_20B", "qwen_image_layered_20B"]:
processor = Qwen2VLProcessor.from_pretrained(fl.locate_folder("Qwen2.5-VL-7B-Instruct"))
tokenizer = AutoTokenizer.from_pretrained(fl.locate_folder("Qwen2.5-VL-7B-Instruct"))
self.base_model_type = base_model_type
- base_config_file = "models/qwen/configs/qwen_image_20B.json"
+ if base_model_type == "qwen_image_layered_20B":
+ base_config_file = "models/qwen/configs/qwen_image_layered_20B.json"
+ else:
+ base_config_file = "models/qwen/configs/qwen_image_20B.json"
with open(base_config_file, 'r', encoding='utf-8') as f:
transformer_config = json.load(f)
- transformer_config.pop("_diffusers_version")
- transformer_config.pop("_class_name")
- transformer_config.pop("pooled_projection_dim")
+ transformer_config.pop("_diffusers_version", None)
+ transformer_config.pop("_class_name", None)
+ transformer_config.pop("pooled_projection_dim", None)
from accelerate import init_empty_weights
with init_empty_weights():
@@ -89,7 +92,11 @@ class model_factory():
# text_encoder.to(torch.float16)
# offload.save_model(text_encoder, "text_encoder_quanto_fp16.safetensors", do_quantize= True)
use_Wan_VAE = False
- VAE_upsampler_factor = 2 if VAE_upsampling is not None else 1
+ if base_model_type == "qwen_image_layered_20B":
+ VAE_upsampling = None
+ VAE_upsampler_factor = 1
+ else:
+ VAE_upsampler_factor = 2 if VAE_upsampling is not None else 1
if use_Wan_VAE:
vae_checkpoint = "Wan2.1_VAE_upscale2x_imageonly_real_v1.safetensors" if VAE_upsampler_factor == 2 else "Wan2.1_VAE.safetensors"
@@ -97,13 +104,26 @@ class model_factory():
vae = WanVAE( vae_pth=fl.locate_file(vae_checkpoint), dtype= VAE_dtype, upsampler_factor = VAE_upsampler_factor, device="cpu")
vae.device = "cuda" #self.device # need to set to cuda so that vae buffers are properly moved (although the rest will stay in the CPU)
else:
- if VAE_upsampler_factor ==2 :
+ if base_model_type == "qwen_image_layered_20B":
+ convert_state_dict = None
+ vae_checkpoint = "qwen_image_layered_vae_bf16.safetensors"
+ vae_config_file = "models/qwen/configs/qwen_image_layered_vae_config.json"
+ vae_override = model_def.get("vae_URL", None) or model_def.get("vae_URLs", None)
+ if isinstance(vae_override, list):
+ vae_override = vae_override[0] if len(vae_override) > 0 else None
+ if isinstance(vae_override, dict):
+ vae_override = vae_override.get("URLs", None)
+ if vae_override:
+ vae_checkpoint = vae_override
+ elif VAE_upsampler_factor ==2 :
from .convert_diffusers_qwen_vae import convert_state_dict
vae_checkpoint = "Wan2.1_VAE_upscale2x_imageonly_real_v1.safetensors"
+ vae_config_file = "qwen_vae_config.json"
else:
convert_state_dict = None
vae_checkpoint = "qwen_vae.safetensors"
- vae = offload.fast_load_transformers_model( fl.locate_file(vae_checkpoint), writable_tensors= True , modelClass=AutoencoderKLQwenImage, defaultConfigPath= fl.locate_file("qwen_vae_config.json"), configKwargs={"upsampler_factor": VAE_upsampler_factor}, preprocess_sd=convert_state_dict)
+ vae_config_file = "qwen_vae_config.json"
+ vae = offload.fast_load_transformers_model( fl.locate_file(vae_checkpoint), writable_tensors= True , modelClass=AutoencoderKLQwenImage, defaultConfigPath= fl.locate_file(vae_config_file), configKwargs={"upsampler_factor": VAE_upsampler_factor}, preprocess_sd=convert_state_dict)
vae.upsampling_set = VAE_upsampling
self.pipeline = QwenImagePipeline(vae, text_encoder, tokenizer, transformer, processor)
self.pipeline.use_Wan_VAE = use_Wan_VAE
@@ -196,12 +216,16 @@ class model_factory():
tile_size = VAE_tile_size[1]
# tile_size = 256
qwen_edit_plus = self.base_model_type in ["qwen_image_edit_plus_20B"]
+ qwen_layered = self.base_model_type in ["qwen_image_layered_20B"]
if hasattr(self.vae, "enable_slicing"):
self.vae.enable_slicing()
# width, height = aspect_ratios["16:9"]
if n_prompt is None or len(n_prompt) == 0:
- n_prompt= "text, watermark, copyright, blurry, low resolution"
+ if qwen_layered:
+ n_prompt = " "
+ else:
+ n_prompt= "text, watermark, copyright, blurry, low resolution"
image_mask = None if input_masks is None else convert_tensor_to_image(input_masks, mask_levels= True)
if input_frames is not None:
@@ -209,16 +233,21 @@ class model_factory():
if input_ref_images is not None:
# image stiching method
- stiched = input_ref_images[0]
- if "K" in video_prompt_type :
- w, h = input_ref_images[0].size
- height, width = calculate_new_dimensions(height, width, h, w, fit_into_canvas)
+ if qwen_layered:
+ input_ref_images = [input_ref_images[0]]
+ else:
+ stiched = input_ref_images[0]
+ if "K" in video_prompt_type :
+ w, h = input_ref_images[0].size
+ height, width = calculate_new_dimensions(height, width, h, w, fit_into_canvas)
- if not qwen_edit_plus:
- for new_img in input_ref_images[1:]:
- stiched = stitch_images(stiched, new_img)
- input_ref_images = [stiched]
+ if not qwen_edit_plus:
+ for new_img in input_ref_images[1:]:
+ stiched = stitch_images(stiched, new_img)
+ input_ref_images = [stiched]
+ num_images_per_prompt = 1 if qwen_layered else batch_size
+ layers = batch_size if qwen_layered else 1
image = self.pipeline(
prompt=input_prompt,
negative_prompt=n_prompt,
@@ -227,7 +256,9 @@ class model_factory():
width=width,
height=height,
num_inference_steps=sampling_steps,
- num_images_per_prompt = batch_size,
+ num_images_per_prompt = num_images_per_prompt,
+ layers = layers,
+ cfg_normalize = True,
true_cfg_scale=guide_scale,
callback = callback,
pipeline=self,
diff --git a/models/qwen/transformer_qwenimage.py b/models/qwen/transformer_qwenimage.py
index a5fa38b..4723728 100644
--- a/models/qwen/transformer_qwenimage.py
+++ b/models/qwen/transformer_qwenimage.py
@@ -123,17 +123,25 @@ def apply_rotary_emb_qwen_inplace(x_list: list, freqs_cis: torch.Tensor) -> torc
class QwenTimestepProjEmbeddings(nn.Module):
- def __init__(self, embedding_dim):
+ def __init__(self, embedding_dim, use_additional_t_cond=False):
super().__init__()
self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0, scale=1000)
self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim)
+ self.use_additional_t_cond = use_additional_t_cond
+ if use_additional_t_cond:
+ self.addition_t_embedding = nn.Embedding(2, embedding_dim)
- def forward(self, timestep, hidden_states):
+ def forward(self, timestep, hidden_states, addition_t_cond=None):
timesteps_proj = self.time_proj(timestep)
timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_states.dtype)) # (N, D)
conditioning = timesteps_emb
+ if self.use_additional_t_cond:
+ if addition_t_cond is None:
+ raise ValueError("When additional_t_cond is True, addition_t_cond must be provided.")
+ addition_t_emb = self.addition_t_embedding(addition_t_cond)
+ conditioning = conditioning + addition_t_emb.to(dtype=hidden_states.dtype)
return conditioning
@@ -242,6 +250,127 @@ class QwenEmbedRope(nn.Module):
return freqs.clone().contiguous()
+class QwenEmbedLayer3DRope(nn.Module):
+ def __init__(self, theta: int, axes_dim: List[int], scale_rope=False):
+ super().__init__()
+ self.theta = theta
+ self.axes_dim = axes_dim
+ pos_index = torch.arange(4096)
+ neg_index = torch.arange(4096).flip(0) * -1 - 1
+ self.pos_freqs = torch.cat(
+ [
+ self.rope_params(pos_index, self.axes_dim[0], self.theta),
+ self.rope_params(pos_index, self.axes_dim[1], self.theta),
+ self.rope_params(pos_index, self.axes_dim[2], self.theta),
+ ],
+ dim=1,
+ )
+ self.neg_freqs = torch.cat(
+ [
+ self.rope_params(neg_index, self.axes_dim[0], self.theta),
+ self.rope_params(neg_index, self.axes_dim[1], self.theta),
+ self.rope_params(neg_index, self.axes_dim[2], self.theta),
+ ],
+ dim=1,
+ )
+
+ self.scale_rope = scale_rope
+
+ def rope_params(self, index, dim, theta=10000):
+ """
+ Args:
+ index: [0, 1, 2, 3] 1D Tensor representing the position index of the token
+ Returns:
+ Tensor of shape [len(index), dim//2, 2] with cos/sin stacked in last dim
+ """
+ assert dim % 2 == 0
+ freqs = torch.outer(index, 1.0 / torch.pow(theta, torch.arange(0, dim, 2).to(torch.float32).div(dim)))
+ return torch.stack([torch.cos(freqs), torch.sin(freqs)], dim=-1)
+
+ def forward(self, video_fhw, txt_seq_lens, device):
+ """
+ Args: video_fhw: [frame, height, width] a list of 3 integers representing the shape of the video Args:
+ txt_length: [bs] a list of 1 integers representing the length of the text
+ """
+ if self.pos_freqs.device != device:
+ self.pos_freqs = self.pos_freqs.to(device)
+ self.neg_freqs = self.neg_freqs.to(device)
+
+ if isinstance(video_fhw, list):
+ video_fhw = video_fhw[0]
+ if not isinstance(video_fhw, list):
+ video_fhw = [video_fhw]
+
+ vid_freqs = []
+ max_vid_index = 0
+ layer_num = len(video_fhw) - 1
+ for idx, fhw in enumerate(video_fhw):
+ frame, height, width = fhw
+ if idx != layer_num:
+ video_freq = self._compute_video_freqs(frame, height, width, idx)
+ else:
+ # For the condition image, we set the layer index to -1.
+ video_freq = self._compute_condition_freqs(frame, height, width)
+ video_freq = video_freq.to(device)
+ vid_freqs.append(video_freq)
+
+ if self.scale_rope:
+ max_vid_index = max(height // 2, width // 2, max_vid_index)
+ else:
+ max_vid_index = max(height, width, max_vid_index)
+
+ max_vid_index = max(max_vid_index, layer_num)
+ max_len = max(txt_seq_lens)
+ txt_freqs = self.pos_freqs[max_vid_index : max_vid_index + max_len, ...]
+ vid_freqs = torch.cat(vid_freqs, dim=0)
+
+ return vid_freqs, txt_freqs
+
+ @functools.lru_cache(maxsize=None)
+ def _compute_video_freqs(self, frame, height, width, idx=0):
+ seq_lens = frame * height * width
+ freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1)
+ freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1)
+
+ freqs_frame = freqs_pos[0][idx : idx + frame] # [frame, d0//2, 2]
+ freqs_frame = freqs_frame.view(frame, 1, 1, -1, 2).expand(frame, height, width, -1, 2)
+
+ if self.scale_rope:
+ freqs_height = torch.cat([freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], dim=0)
+ freqs_height = freqs_height.view(1, height, 1, -1, 2).expand(frame, height, width, -1, 2)
+ freqs_width = torch.cat([freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], dim=0)
+ freqs_width = freqs_width.view(1, 1, width, -1, 2).expand(frame, height, width, -1, 2)
+ else:
+ freqs_height = freqs_pos[1][:height].view(1, height, 1, -1, 2).expand(frame, height, width, -1, 2)
+ freqs_width = freqs_pos[2][:width].view(1, 1, width, -1, 2).expand(frame, height, width, -1, 2)
+
+ freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-2) # [F, H, W, total//2, 2]
+ freqs = freqs.reshape(seq_lens, -1, 2)
+ return freqs.clone().contiguous()
+
+ @functools.lru_cache(maxsize=None)
+ def _compute_condition_freqs(self, frame, height, width):
+ seq_lens = frame * height * width
+ freqs_pos = self.pos_freqs.split([x // 2 for x in self.axes_dim], dim=1)
+ freqs_neg = self.neg_freqs.split([x // 2 for x in self.axes_dim], dim=1)
+
+ freqs_frame = freqs_neg[0][-1:] # [1, d0//2, 2]
+ freqs_frame = freqs_frame.view(frame, 1, 1, -1, 2).expand(frame, height, width, -1, 2)
+
+ if self.scale_rope:
+ freqs_height = torch.cat([freqs_neg[1][-(height - height // 2) :], freqs_pos[1][: height // 2]], dim=0)
+ freqs_height = freqs_height.view(1, height, 1, -1, 2).expand(frame, height, width, -1, 2)
+ freqs_width = torch.cat([freqs_neg[2][-(width - width // 2) :], freqs_pos[2][: width // 2]], dim=0)
+ freqs_width = freqs_width.view(1, 1, width, -1, 2).expand(frame, height, width, -1, 2)
+ else:
+ freqs_height = freqs_pos[1][:height].view(1, height, 1, -1, 2).expand(frame, height, width, -1, 2)
+ freqs_width = freqs_pos[2][:width].view(1, 1, width, -1, 2).expand(frame, height, width, -1, 2)
+
+ freqs = torch.cat([freqs_frame, freqs_height, freqs_width], dim=-2) # [F, H, W, total//2, 2]
+ freqs = freqs.reshape(seq_lens, -1, 2)
+ return freqs.clone().contiguous()
+
+
class QwenDoubleStreamAttnProcessor2_0:
"""
Attention processor for Qwen double-stream architecture, matching DoubleStreamLayerMegatron logic. This processor
@@ -350,7 +479,13 @@ class QwenDoubleStreamAttnProcessor2_0:
class QwenImageTransformerBlock(nn.Module):
def __init__(
- self, dim: int, num_attention_heads: int, attention_head_dim: int, qk_norm: str = "rms_norm", eps: float = 1e-6
+ self,
+ dim: int,
+ num_attention_heads: int,
+ attention_head_dim: int,
+ qk_norm: str = "rms_norm",
+ eps: float = 1e-6,
+ zero_cond_t: bool = False,
):
super().__init__()
@@ -392,6 +527,7 @@ class QwenImageTransformerBlock(nn.Module):
# FFN expansion factor for chunking (diffusers FeedForward default is 4x)
self.ffn_mult = 4
+ self.zero_cond_t = zero_cond_t
def _apply_ffn_chunked(self, ffn_in: torch.Tensor, mlp: nn.Module) -> None:
_, seq_len, dim = ffn_in.shape
@@ -401,6 +537,34 @@ class QwenImageTransformerBlock(nn.Module):
ffn_chunk[...] = mlp(ffn_chunk)
# ffn_in is already modified in-place via ffn_in_flat view
+ def _modulate(self, x, mod_params, index=None):
+ """Apply modulation to input tensor"""
+ shift, scale, gate = mod_params.chunk(3, dim=-1)
+
+ if index is not None:
+ actual_batch = shift.size(0) // 2
+ shift_0, shift_1 = shift[:actual_batch], shift[actual_batch:]
+ scale_0, scale_1 = scale[:actual_batch], scale[actual_batch:]
+ gate_0, gate_1 = gate[:actual_batch], gate[actual_batch:]
+
+ index_expanded = index.unsqueeze(-1)
+ shift_0_exp = shift_0.unsqueeze(1)
+ shift_1_exp = shift_1.unsqueeze(1)
+ scale_0_exp = scale_0.unsqueeze(1)
+ scale_1_exp = scale_1.unsqueeze(1)
+ gate_0_exp = gate_0.unsqueeze(1)
+ gate_1_exp = gate_1.unsqueeze(1)
+
+ shift_result = torch.where(index_expanded == 0, shift_0_exp, shift_1_exp)
+ scale_result = torch.where(index_expanded == 0, scale_0_exp, scale_1_exp)
+ gate_result = torch.where(index_expanded == 0, gate_0_exp, gate_1_exp)
+ else:
+ shift_result = shift.unsqueeze(1)
+ scale_result = scale.unsqueeze(1)
+ gate_result = gate.unsqueeze(1)
+
+ return x * (1 + scale_result) + shift_result, gate_result
+
def _modulate_inplace(self, x, shift, scale, gate):
scale.add_(1.0)
x.mul_(scale.unsqueeze(1))
@@ -415,46 +579,85 @@ class QwenImageTransformerBlock(nn.Module):
temb: torch.Tensor,
image_rotary_emb: Optional[Tuple[torch.Tensor, torch.Tensor]] = None,
joint_attention_kwargs: Optional[Dict[str, Any]] = None,
+ modulate_index: Optional[torch.Tensor] = None,
) -> Tuple[torch.Tensor, torch.Tensor]:
- # Get modulation parameters for both streams and split into shift, scale, gate
- img_shift1, img_scale1, img_gate1, img_shift2, img_scale2, img_gate2 = self.img_mod(temb).chunk(6, dim=-1)
- txt_shift1, txt_scale1, txt_gate1, txt_shift2, txt_scale2, txt_gate2 = self.txt_mod(temb).chunk(6, dim=-1)
-
- # Process image stream - norm1 + modulation (in-place)
- img_normed = self.img_norm1(hidden_states)
- img_gate1 = self._modulate_inplace(img_normed, img_shift1, img_scale1, img_gate1)
-
- # Process text stream - norm1 + modulation (in-place)
- txt_normed = self.txt_norm1(encoder_hidden_states)
- txt_gate1 = self._modulate_inplace(txt_normed, txt_shift1, txt_scale1, txt_gate1)
-
-
- hidden_states_list = [img_normed, txt_normed]
- del img_normed, txt_normed
- joint_attention_kwargs = joint_attention_kwargs or {}
- img_attn_output, txt_attn_output = self.attn.processor(self.attn,
- hidden_states_list=hidden_states_list,
- encoder_hidden_states_mask=encoder_hidden_states_mask,
- image_rotary_emb=image_rotary_emb,
- **joint_attention_kwargs,
- )
+ img_mod_params = self.img_mod(temb)
+ if self.zero_cond_t:
+ temb_txt = temb.chunk(2, dim=0)[0]
+ else:
+ temb_txt = temb
+ txt_mod_params = self.txt_mod(temb_txt)
+
+ if modulate_index is None:
+ # Get modulation parameters for both streams and split into shift, scale, gate
+ img_shift1, img_scale1, img_gate1, img_shift2, img_scale2, img_gate2 = img_mod_params.chunk(6, dim=-1)
+ txt_shift1, txt_scale1, txt_gate1, txt_shift2, txt_scale2, txt_gate2 = txt_mod_params.chunk(6, dim=-1)
+
+ # Process image stream - norm1 + modulation (in-place)
+ img_normed = self.img_norm1(hidden_states)
+ img_gate1 = self._modulate_inplace(img_normed, img_shift1, img_scale1, img_gate1)
+
+ # Process text stream - norm1 + modulation (in-place)
+ txt_normed = self.txt_norm1(encoder_hidden_states)
+ txt_gate1 = self._modulate_inplace(txt_normed, txt_shift1, txt_scale1, txt_gate1)
+
+ hidden_states_list = [img_normed, txt_normed]
+ del img_normed, txt_normed
+ joint_attention_kwargs = joint_attention_kwargs or {}
+ img_attn_output, txt_attn_output = self.attn.processor(
+ self.attn,
+ hidden_states_list=hidden_states_list,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ image_rotary_emb=image_rotary_emb,
+ **joint_attention_kwargs,
+ )
- # Apply attention gates and add residual in-place
- hidden_states.addcmul_(img_attn_output, img_gate1)
- encoder_hidden_states.addcmul_(txt_attn_output, txt_gate1)
+ # Apply attention gates and add residual in-place
+ hidden_states.addcmul_(img_attn_output, img_gate1)
+ encoder_hidden_states.addcmul_(txt_attn_output, txt_gate1)
- img_normed2 = self.img_norm2(hidden_states)
- img_gate2 = self._modulate_inplace(img_normed2, img_shift2, img_scale2, img_gate2)
- self._apply_ffn_chunked(img_normed2, self.img_mlp)
- # img_normed2 = self.img_mlp(img_normed2)
+ img_normed2 = self.img_norm2(hidden_states)
+ img_gate2 = self._modulate_inplace(img_normed2, img_shift2, img_scale2, img_gate2)
+ self._apply_ffn_chunked(img_normed2, self.img_mlp)
+ hidden_states.addcmul_(img_normed2, img_gate2)
- hidden_states.addcmul_(img_normed2, img_gate2)
+ txt_normed2 = self.txt_norm2(encoder_hidden_states)
+ txt_gate2 = self._modulate_inplace(txt_normed2, txt_shift2, txt_scale2, txt_gate2)
+ txt_normed2 = self.txt_mlp(txt_normed2)
+ encoder_hidden_states.addcmul_(txt_normed2, txt_gate2)
+ else:
+ img_mod1, img_mod2 = img_mod_params.chunk(2, dim=-1)
+ txt_mod1, txt_mod2 = txt_mod_params.chunk(2, dim=-1)
+
+ img_normed = self.img_norm1(hidden_states)
+ img_modulated, img_gate1 = self._modulate(img_normed, img_mod1, modulate_index)
+
+ txt_normed = self.txt_norm1(encoder_hidden_states)
+ txt_modulated, txt_gate1 = self._modulate(txt_normed, txt_mod1)
+
+ hidden_states_list = [img_modulated, txt_modulated]
+ del img_modulated, txt_modulated
+ joint_attention_kwargs = joint_attention_kwargs or {}
+ img_attn_output, txt_attn_output = self.attn.processor(
+ self.attn,
+ hidden_states_list=hidden_states_list,
+ encoder_hidden_states_mask=encoder_hidden_states_mask,
+ image_rotary_emb=image_rotary_emb,
+ **joint_attention_kwargs,
+ )
- txt_normed2 = self.txt_norm2(encoder_hidden_states)
- txt_gate2 = self._modulate_inplace(txt_normed2, txt_shift2, txt_scale2, txt_gate2)
- txt_normed2 = self.txt_mlp(txt_normed2)
+ hidden_states.addcmul_(img_attn_output, img_gate1)
+ encoder_hidden_states.addcmul_(txt_attn_output, txt_gate1)
- encoder_hidden_states.addcmul_(txt_normed2, txt_gate2)
+ img_normed2 = self.img_norm2(hidden_states)
+ img_modulated2, img_gate2 = self._modulate(img_normed2, img_mod2, modulate_index)
+ self._apply_ffn_chunked(img_modulated2, self.img_mlp)
+ hidden_states.addcmul_(img_modulated2, img_gate2)
+
+ txt_normed2 = self.txt_norm2(encoder_hidden_states)
+ txt_modulated2, txt_gate2 = self._modulate(txt_normed2, txt_mod2)
+ txt_modulated2 = self.txt_mlp(txt_modulated2)
+ encoder_hidden_states.addcmul_(txt_modulated2, txt_gate2)
# Clip to prevent overflow for fp16
if encoder_hidden_states.dtype == torch.float16:
@@ -552,15 +755,23 @@ class QwenImageTransformer2DModel(nn.Module):
joint_attention_dim: int = 3584,
guidance_embeds: bool = False, # TODO: this should probably be removed
axes_dims_rope: Tuple[int, int, int] = (16, 56, 56),
+ zero_cond_t: bool = False,
+ use_additional_t_cond: bool = False,
+ use_layer3d_rope: bool = False,
):
super().__init__()
self.out_channels = out_channels or in_channels
self.inner_dim = num_attention_heads * attention_head_dim
self.in_channels = in_channels
self.guidance_embeds = guidance_embeds
- self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
+ if use_layer3d_rope:
+ self.pos_embed = QwenEmbedLayer3DRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
+ else:
+ self.pos_embed = QwenEmbedRope(theta=10000, axes_dim=list(axes_dims_rope), scale_rope=True)
- self.time_text_embed = QwenTimestepProjEmbeddings(embedding_dim=self.inner_dim)
+ self.time_text_embed = QwenTimestepProjEmbeddings(
+ embedding_dim=self.inner_dim, use_additional_t_cond=use_additional_t_cond
+ )
self.txt_norm = RMSNorm(joint_attention_dim, eps=1e-6)
@@ -573,6 +784,7 @@ class QwenImageTransformer2DModel(nn.Module):
dim=self.inner_dim,
num_attention_heads=num_attention_heads,
attention_head_dim=attention_head_dim,
+ zero_cond_t=zero_cond_t,
)
for _ in range(num_layers)
]
@@ -582,6 +794,8 @@ class QwenImageTransformer2DModel(nn.Module):
self.proj_out = nn.Linear(self.inner_dim, patch_size * patch_size * self.out_channels, bias=True)
self.gradient_checkpointing = False
+ self.zero_cond_t = zero_cond_t
+ self.use_additional_t_cond = use_additional_t_cond
def forward(
self,
@@ -595,11 +809,26 @@ class QwenImageTransformer2DModel(nn.Module):
attention_kwargs: Optional[Dict[str, Any]] = None,
callback= None,
pipeline =None,
+ additional_t_cond: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, Transformer2DModelOutput]:
hidden_states = self.img_in(hidden_states)
timestep = timestep.to(hidden_states.dtype)
+ modulate_index = None
+ if self.zero_cond_t:
+ timestep = torch.cat([timestep, timestep * 0], dim=0)
+ if additional_t_cond is not None:
+ additional_t_cond = torch.cat([additional_t_cond, additional_t_cond * 0], dim=0)
+ if img_shapes is None:
+ raise ValueError("img_shapes must be provided when zero_cond_t is enabled.")
+ base_shapes = img_shapes
+ if isinstance(img_shapes, list) and len(img_shapes) > 0 and isinstance(img_shapes[0], list):
+ base_shapes = img_shapes[0]
+ seq_counts = [f * h * w for f, h, w in base_shapes]
+ base_index = [0] * seq_counts[0] + [1] * sum(seq_counts[1:])
+ modulate_index = torch.tensor(base_index, device=timestep.device, dtype=torch.int).unsqueeze(0)
+ modulate_index = modulate_index.repeat(hidden_states.shape[0], 1)
hidden_states_list = [hidden_states if i == 0 else hidden_states.clone() for i, _ in enumerate(encoder_hidden_states_list)]
new_encoder_hidden_states_list = []
@@ -614,9 +843,9 @@ class QwenImageTransformer2DModel(nn.Module):
guidance = guidance.to(hidden_states.dtype) * 1000
temb = (
- self.time_text_embed(timestep, hidden_states)
+ self.time_text_embed(timestep, hidden_states, additional_t_cond)
if guidance is None
- else self.time_text_embed(timestep, guidance, hidden_states)
+ else self.time_text_embed(timestep, guidance, additional_t_cond)
)
image_rotary_emb_list = [ self.pos_embed(img_shapes, txt_seq_lens, device=hidden_states.device) for txt_seq_lens in txt_seq_lens_list]
@@ -636,6 +865,7 @@ class QwenImageTransformer2DModel(nn.Module):
temb=temb,
image_rotary_emb=image_rotary_emb,
joint_attention_kwargs=attention_kwargs,
+ modulate_index=modulate_index,
)
# Use only the image part (hidden_states) from the dual-stream blocks
diff --git a/wgp.py b/wgp.py
index dd1ae77..c0acb56 100644
--- a/wgp.py
+++ b/wgp.py
@@ -6070,7 +6070,8 @@ def generate_video(
video_path = [video_path] if not isinstance(video_path, list) else video_path
for no, path in enumerate(video_path):
if metadata_choice == "json":
- with open(path.replace(f'.{extension}', '.json'), 'w') as f:
+ json_path = os.path.splitext(path)[0] + ".json"
+ with open(json_path, 'w') as f:
json.dump(configs, f, indent=4)
elif metadata_choice == "metadata":
if audio_only:
@@ -8985,7 +8986,8 @@ def generate_video_tab(update_form = False, state_dict = None, ui_defaults = Non
scale = 5
)
with gr.Row(visible= not audio_only) as number_frames_row:
- batch_size = gr.Slider(1, 16, value=ui_get("batch_size"), step=1, label="Number of Images to Generate", visible = image_outputs, show_reset_button= False)
+ batch_label = model_def.get("batch_size_label", "Number of Images to Generate")
+ batch_size = gr.Slider(1, 16, value=ui_get("batch_size"), step=1, label=batch_label, visible = image_outputs, show_reset_button= False)
if image_outputs:
video_length = gr.Slider(1, 9999, value=ui_get("video_length"), step=1, label="Number of frames", visible = False, show_reset_button= False)
else: