Official repository for LTX-Video
Python diffusion-modelsditimage-to-videoimage-to-video-generationtext-to-videotext-to-video-generation
VAE: Support timestep-conditioned decoder
Yoav HaCohen925516ed6fd0
diff --git a/inference.py b/inference.pyindex 1f399f2..704ba6a 100644--- a/inference.py+++ b/inference.py@@ -238,6 +238,20 @@ def main():help="Sets the precision for the transformer and tokenizer. Default is bfloat16. If 'mixed_precision' is enabled, it moves to mixed-precision.",)+ # VAE noise augmentation+ parser.add_argument(+ "--decode_timestep",+ type=float,+ default=0.05,+ help="Timestep for decoding noise",+ )+ parser.add_argument(+ "--decode_noise_scale",+ type=float,+ default=0.025,+ help="Noise level for decoding noise",+ )+# Promptsparser.add_argument("--prompt",@@ -392,6 +406,8 @@ def main():else ConditioningMethod.UNCONDITIONAL),image_cond_noise_scale=args.image_cond_noise_scale,+ decode_timestep=args.decode_timestep,+ decode_noise_scale=args.decode_noise_scale,mixed_precision=(args.precision == "mixed_precision"),offload_to_cpu=offload_to_cpu,).imagesdiff --git a/ltx_video/models/autoencoders/causal_video_autoencoder.py b/ltx_video/models/autoencoders/causal_video_autoencoder.pyindex 0efafa2..c059d4a 100644--- a/ltx_video/models/autoencoders/causal_video_autoencoder.py+++ b/ltx_video/models/autoencoders/causal_video_autoencoder.py@@ -641,7 +641,7 @@ class Decoder(nn.Module):self,sample: torch.FloatTensor,target_shape,- timesteps: Optional[torch.Tensor] = None,+ timestep: Optional[torch.Tensor] = None,) -> torch.FloatTensor:r"""The forward method of the `Decoder` class."""assert target_shape is not None, "target_shape must be provided"@@ -661,14 +661,14 @@ class Decoder(nn.Module):if self.timestep_conditioning:assert (- timesteps is not None- ), "should pass timesteps with timestep_conditioning=True"- scaled_timesteps = timesteps * self.timestep_scale_multiplier+ timestep is not None+ ), "should pass timestep with timestep_conditioning=True"+ scaled_timestep = timestep * self.timestep_scale_multiplierfor up_block in self.up_blocks:if self.timestep_conditioning and isinstance(up_block, UNetMidBlock3D):sample = checkpoint_fn(up_block)(- sample, causal=self.causal, timesteps=scaled_timesteps+ sample, causal=self.causal, timestep=scaled_timestep)else:sample = checkpoint_fn(up_block)(sample, causal=self.causal)@@ -676,25 +676,25 @@ class Decoder(nn.Module):sample = self.conv_norm_out(sample)if self.timestep_conditioning:- embedded_timesteps = self.last_time_embedder(- timestep=scaled_timesteps.flatten(),+ embedded_timestep = self.last_time_embedder(+ timestep=scaled_timestep.flatten(),resolution=None,aspect_ratio=None,batch_size=sample.shape[0],hidden_dtype=sample.dtype,)- embedded_timesteps = embedded_timesteps.view(- batch_size, embedded_timesteps.shape[-1], 1, 1, 1+ embedded_timestep = embedded_timestep.view(+ batch_size, embedded_timestep.shape[-1], 1, 1, 1)ada_values = self.last_scale_shift_table[None, ..., None, None, None- ] + embedded_timesteps.reshape(+ ] + embedded_timestep.reshape(batch_size,2,-1,- embedded_timesteps.shape[-3],- embedded_timesteps.shape[-2],- embedded_timesteps.shape[-1],+ embedded_timestep.shape[-3],+ embedded_timestep.shape[-2],+ embedded_timestep.shape[-1],)shift, scale = ada_values.unbind(dim=1)sample = sample * (1 + scale) + shift@@ -801,16 +801,16 @@ class UNetMidBlock3D(nn.Module):self,hidden_states: torch.FloatTensor,causal: bool = True,- timesteps: Optional[torch.Tensor] = None,+ timestep: Optional[torch.Tensor] = None,) -> torch.FloatTensor:timestep_embed = Noneif self.timestep_conditioning:assert (- timesteps is not None- ), "should pass timesteps with timestep_conditioning=True"+ timestep is not None+ ), "should pass timestep with timestep_conditioning=True"batch_size = hidden_states.shape[0]timestep_embed = self.time_embedder(- timestep=timesteps.flatten(),+ timestep=timestep.flatten(),resolution=None,aspect_ratio=None,batch_size=batch_size,@@ -823,7 +823,7 @@ class UNetMidBlock3D(nn.Module):if self.attention_blocks:for resnet, attention in zip(self.res_blocks, self.attention_blocks):hidden_states = resnet(- hidden_states, causal=causal, timesteps=timestep_embed+ hidden_states, causal=causal, timestep=timestep_embed)# Reshape the hidden states to be (batch_size, frames * height * width, channel)@@ -870,7 +870,7 @@ class UNetMidBlock3D(nn.Module):else:for resnet in self.res_blocks:hidden_states = resnet(- hidden_states, causal=causal, timesteps=timestep_embed+ hidden_states, causal=causal, timestep=timestep_embed)return hidden_states@@ -1055,7 +1055,7 @@ class ResnetBlock3D(nn.Module):self,input_tensor: torch.FloatTensor,causal: bool = True,- timesteps: Optional[torch.Tensor] = None,+ timestep: Optional[torch.Tensor] = None,) -> torch.FloatTensor:hidden_states = input_tensorbatch_size = hidden_states.shape[0]@@ -1063,17 +1063,17 @@ class ResnetBlock3D(nn.Module):hidden_states = self.norm1(hidden_states)if self.timestep_conditioning:assert (- timesteps is not None- ), "should pass timesteps with timestep_conditioning=True"+ timestep is not None+ ), "should pass timestep with timestep_conditioning=True"ada_values = self.scale_shift_table[None, ..., None, None, None- ] + timesteps.reshape(+ ] + timestep.reshape(batch_size,4,-1,- timesteps.shape[-3],- timesteps.shape[-2],- timesteps.shape[-1],+ timestep.shape[-3],+ timestep.shape[-2],+ timestep.shape[-1],)shift1, scale1, shift2, scale2 = ada_values.unbind(dim=1)@@ -1228,9 +1228,9 @@ def demo_video_autoencoder_forward_backward():print(f"input shape={input_videos.shape}")print(f"latent shape={latent.shape}")- timesteps = torch.ones(input_videos.shape[0]) * 0.1+ timestep = torch.ones(input_videos.shape[0]) * 0.1reconstructed_videos = video_autoencoder.decode(- latent, target_shape=input_videos.shape, timesteps=timesteps+ latent, target_shape=input_videos.shape, timestep=timestep).sampleprint(f"reconstructed shape={reconstructed_videos.shape}")@@ -1239,7 +1239,7 @@ def demo_video_autoencoder_forward_backward():input_image = input_videos[:, :, :1, :, :]image_latent = video_autoencoder.encode(input_image).latent_dist.mode()_ = video_autoencoder.decode(- image_latent, target_shape=image_latent.shape, timesteps=timesteps+ image_latent, target_shape=image_latent.shape, timestep=timestep).sample# first_frame_latent = latent[:, :, :1, :, :]diff --git a/ltx_video/models/autoencoders/vae.py b/ltx_video/models/autoencoders/vae.pyindex a67ac7a..0bcab52 100644--- a/ltx_video/models/autoencoders/vae.py+++ b/ltx_video/models/autoencoders/vae.py@@ -257,11 +257,11 @@ class AutoencoderKLWrapper(ModelMixin, ConfigMixin):self,z: torch.FloatTensor,target_shape=None,- timesteps: Optional[torch.Tensor] = None,+ timestep: Optional[torch.Tensor] = None,) -> Union[DecoderOutput, torch.FloatTensor]:z = self.post_quant_conv(z)- if "timesteps" in self.decoder_params:- dec = self.decoder(z, target_shape=target_shape, timesteps=timesteps)+ if "timestep" in self.decoder_params:+ dec = self.decoder(z, target_shape=target_shape, timestep=timestep)else:dec = self.decoder(z, target_shape=target_shape)return dec@@ -271,7 +271,7 @@ class AutoencoderKLWrapper(ModelMixin, ConfigMixin):z: torch.FloatTensor,return_dict: bool = True,target_shape=None,- timesteps: Optional[torch.Tensor] = None,+ timestep: Optional[torch.Tensor] = None,) -> Union[DecoderOutput, torch.FloatTensor]:assert target_shape is not None, "target_shape must be provided for decoding"if self.use_z_tiling and z.shape[2] > self.z_sample_size > 1:@@ -304,7 +304,7 @@ class AutoencoderKLWrapper(ModelMixin, ConfigMixin):decoded = (self._hw_tiled_decode(z, target_shape)if self.use_hw_tiling- else self._decode(z, target_shape=target_shape, timesteps=timesteps)+ else self._decode(z, target_shape=target_shape, timestep=timestep))if not return_dict:diff --git a/ltx_video/models/autoencoders/vae_encode.py b/ltx_video/models/autoencoders/vae_encode.pyindex 3b8b15b..f584ec0 100644--- a/ltx_video/models/autoencoders/vae_encode.py+++ b/ltx_video/models/autoencoders/vae_encode.py@@ -96,6 +96,7 @@ def vae_decode(is_video: bool = True,split_size: int = 1,vae_per_channel_normalize=False,+ timestep=None,) -> Tensor:is_video_shaped = latents.dim() == 5batch_size = latents.shape[0]@@ -111,12 +112,16 @@ def vae_decode()encode_bs = len(latents) // split_sizeimage_batch = [- _run_decoder(latent_batch, vae, is_video, vae_per_channel_normalize)+ _run_decoder(+ latent_batch, vae, is_video, vae_per_channel_normalize, timestep+ )for latent_batch in latents.split(encode_bs)]images = torch.cat(image_batch, dim=0)else:- images = _run_decoder(latents, vae, is_video, vae_per_channel_normalize)+ images = _run_decoder(+ latents, vae, is_video, vae_per_channel_normalize, timestep+ )if is_video_shaped and not isinstance(vae, (VideoAutoencoder, CausalVideoAutoencoder)@@ -126,12 +131,19 @@ def vae_decode(def _run_decoder(- latents: Tensor, vae: AutoencoderKL, is_video: bool, vae_per_channel_normalize=False+ latents: Tensor,+ vae: AutoencoderKL,+ is_video: bool,+ vae_per_channel_normalize=False,+ timestep=None,) -> Tensor:if isinstance(vae, (VideoAutoencoder, CausalVideoAutoencoder)):*_, fl, hl, wl = latents.shapetemporal_scale, spatial_scale, _ = get_vae_size_scale_factor(vae)latents = latents.to(vae.dtype)+ vae_decode_kwargs = {}+ if timestep is not None:+ vae_decode_kwargs["timestep"] = timestepimage = vae.decode(un_normalize_latents(latents, vae, vae_per_channel_normalize),return_dict=False,@@ -142,6 +154,7 @@ def _run_decoder(hl * spatial_scale,wl * spatial_scale,),+ **vae_decode_kwargs,)[0]else:image = vae.decode(diff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.pyindex fa0a87b..19f191d 100644--- a/ltx_video/pipelines/pipeline_ltx_video.py+++ b/ltx_video/pipelines/pipeline_ltx_video.py@@ -792,6 +792,8 @@ class LTXVideoPipeline(DiffusionPipeline):callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,clean_caption: bool = True,media_items: Optional[torch.FloatTensor] = None,+ decode_timestep: Union[List[float], float] = 0.0,+ decode_noise_scale: Optional[List[float]] = None,mixed_precision: bool = False,offload_to_cpu: bool = False,**kwargs,@@ -1171,11 +1173,30 @@ class LTXVideoPipeline(DiffusionPipeline):// math.prod(self.patchifier.patch_size),)if output_type != "latent":+ if self.vae.decoder.timestep_conditioning:+ noise = torch.randn_like(latents)+ if not isinstance(decode_timestep, list):+ decode_timestep = [decode_timestep] * latents.shape[0]+ if decode_noise_scale is None:+ decode_noise_scale = decode_timestep+ elif not isinstance(decode_noise_scale, list):+ decode_noise_scale = [decode_noise_scale] * latents.shape[0]++ decode_timestep = torch.tensor(decode_timestep).to(latents.device)+ decode_noise_scale = torch.tensor(decode_noise_scale).to(+ latents.device+ )[:, None, None, None, None]+ latents = (+ latents * (1 - decode_noise_scale) + noise * decode_noise_scale+ )+ else:+ decode_timestep = Noneimage = vae_decode(latents,self.vae,is_video,vae_per_channel_normalize=kwargs["vae_per_channel_normalize"],+ timestep=decode_timestep,)image = self.image_processor.postprocess(image, output_type=output_type)