Skip to content

Official repository for LTX-Video

VAE: Support timestep-conditioned decoder

Yoav HaCohen925516ed6fd0
diff --git a/inference.py b/inference.py
index 1f399f2..704ba6a 100644
--- a/inference.py
+++ b/inference.py
@@ -238,6 +238,20 @@ def main():
help="Sets the precision for the transformer and tokenizer. Default is bfloat16. If 'mixed_precision' is enabled, it moves to mixed-precision.",
)
+ # VAE noise augmentation
+ parser.add_argument(
+ "--decode_timestep",
+ type=float,
+ default=0.05,
+ help="Timestep for decoding noise",
+ )
+ parser.add_argument(
+ "--decode_noise_scale",
+ type=float,
+ default=0.025,
+ help="Noise level for decoding noise",
+ )
+
# Prompts
parser.add_argument(
"--prompt",
@@ -392,6 +406,8 @@ def main():
else ConditioningMethod.UNCONDITIONAL
),
image_cond_noise_scale=args.image_cond_noise_scale,
+ decode_timestep=args.decode_timestep,
+ decode_noise_scale=args.decode_noise_scale,
mixed_precision=(args.precision == "mixed_precision"),
offload_to_cpu=offload_to_cpu,
).images
diff --git a/ltx_video/models/autoencoders/causal_video_autoencoder.py b/ltx_video/models/autoencoders/causal_video_autoencoder.py
index 0efafa2..c059d4a 100644
--- a/ltx_video/models/autoencoders/causal_video_autoencoder.py
+++ b/ltx_video/models/autoencoders/causal_video_autoencoder.py
@@ -641,7 +641,7 @@ class Decoder(nn.Module):
self,
sample: torch.FloatTensor,
target_shape,
- timesteps: Optional[torch.Tensor] = None,
+ timestep: Optional[torch.Tensor] = None,
) -> torch.FloatTensor:
r"""The forward method of the `Decoder` class."""
assert target_shape is not None, "target_shape must be provided"
@@ -661,14 +661,14 @@ class Decoder(nn.Module):
if self.timestep_conditioning:
assert (
- timesteps is not None
- ), "should pass timesteps with timestep_conditioning=True"
- scaled_timesteps = timesteps * self.timestep_scale_multiplier
+ timestep is not None
+ ), "should pass timestep with timestep_conditioning=True"
+ scaled_timestep = timestep * self.timestep_scale_multiplier
for up_block in self.up_blocks:
if self.timestep_conditioning and isinstance(up_block, UNetMidBlock3D):
sample = checkpoint_fn(up_block)(
- sample, causal=self.causal, timesteps=scaled_timesteps
+ sample, causal=self.causal, timestep=scaled_timestep
)
else:
sample = checkpoint_fn(up_block)(sample, causal=self.causal)
@@ -676,25 +676,25 @@ class Decoder(nn.Module):
sample = self.conv_norm_out(sample)
if self.timestep_conditioning:
- embedded_timesteps = self.last_time_embedder(
- timestep=scaled_timesteps.flatten(),
+ embedded_timestep = self.last_time_embedder(
+ timestep=scaled_timestep.flatten(),
resolution=None,
aspect_ratio=None,
batch_size=sample.shape[0],
hidden_dtype=sample.dtype,
)
- embedded_timesteps = embedded_timesteps.view(
- batch_size, embedded_timesteps.shape[-1], 1, 1, 1
+ embedded_timestep = embedded_timestep.view(
+ batch_size, embedded_timestep.shape[-1], 1, 1, 1
)
ada_values = self.last_scale_shift_table[
None, ..., None, None, None
- ] + embedded_timesteps.reshape(
+ ] + embedded_timestep.reshape(
batch_size,
2,
-1,
- embedded_timesteps.shape[-3],
- embedded_timesteps.shape[-2],
- embedded_timesteps.shape[-1],
+ embedded_timestep.shape[-3],
+ embedded_timestep.shape[-2],
+ embedded_timestep.shape[-1],
)
shift, scale = ada_values.unbind(dim=1)
sample = sample * (1 + scale) + shift
@@ -801,16 +801,16 @@ class UNetMidBlock3D(nn.Module):
self,
hidden_states: torch.FloatTensor,
causal: bool = True,
- timesteps: Optional[torch.Tensor] = None,
+ timestep: Optional[torch.Tensor] = None,
) -> torch.FloatTensor:
timestep_embed = None
if self.timestep_conditioning:
assert (
- timesteps is not None
- ), "should pass timesteps with timestep_conditioning=True"
+ timestep is not None
+ ), "should pass timestep with timestep_conditioning=True"
batch_size = hidden_states.shape[0]
timestep_embed = self.time_embedder(
- timestep=timesteps.flatten(),
+ timestep=timestep.flatten(),
resolution=None,
aspect_ratio=None,
batch_size=batch_size,
@@ -823,7 +823,7 @@ class UNetMidBlock3D(nn.Module):
if self.attention_blocks:
for resnet, attention in zip(self.res_blocks, self.attention_blocks):
hidden_states = resnet(
- hidden_states, causal=causal, timesteps=timestep_embed
+ hidden_states, causal=causal, timestep=timestep_embed
)
# Reshape the hidden states to be (batch_size, frames * height * width, channel)
@@ -870,7 +870,7 @@ class UNetMidBlock3D(nn.Module):
else:
for resnet in self.res_blocks:
hidden_states = resnet(
- hidden_states, causal=causal, timesteps=timestep_embed
+ hidden_states, causal=causal, timestep=timestep_embed
)
return hidden_states
@@ -1055,7 +1055,7 @@ class ResnetBlock3D(nn.Module):
self,
input_tensor: torch.FloatTensor,
causal: bool = True,
- timesteps: Optional[torch.Tensor] = None,
+ timestep: Optional[torch.Tensor] = None,
) -> torch.FloatTensor:
hidden_states = input_tensor
batch_size = hidden_states.shape[0]
@@ -1063,17 +1063,17 @@ class ResnetBlock3D(nn.Module):
hidden_states = self.norm1(hidden_states)
if self.timestep_conditioning:
assert (
- timesteps is not None
- ), "should pass timesteps with timestep_conditioning=True"
+ timestep is not None
+ ), "should pass timestep with timestep_conditioning=True"
ada_values = self.scale_shift_table[
None, ..., None, None, None
- ] + timesteps.reshape(
+ ] + timestep.reshape(
batch_size,
4,
-1,
- timesteps.shape[-3],
- timesteps.shape[-2],
- timesteps.shape[-1],
+ timestep.shape[-3],
+ timestep.shape[-2],
+ timestep.shape[-1],
)
shift1, scale1, shift2, scale2 = ada_values.unbind(dim=1)
@@ -1228,9 +1228,9 @@ def demo_video_autoencoder_forward_backward():
print(f"input shape={input_videos.shape}")
print(f"latent shape={latent.shape}")
- timesteps = torch.ones(input_videos.shape[0]) * 0.1
+ timestep = torch.ones(input_videos.shape[0]) * 0.1
reconstructed_videos = video_autoencoder.decode(
- latent, target_shape=input_videos.shape, timesteps=timesteps
+ latent, target_shape=input_videos.shape, timestep=timestep
).sample
print(f"reconstructed shape={reconstructed_videos.shape}")
@@ -1239,7 +1239,7 @@ def demo_video_autoencoder_forward_backward():
input_image = input_videos[:, :, :1, :, :]
image_latent = video_autoencoder.encode(input_image).latent_dist.mode()
_ = video_autoencoder.decode(
- image_latent, target_shape=image_latent.shape, timesteps=timesteps
+ image_latent, target_shape=image_latent.shape, timestep=timestep
).sample
# first_frame_latent = latent[:, :, :1, :, :]
diff --git a/ltx_video/models/autoencoders/vae.py b/ltx_video/models/autoencoders/vae.py
index a67ac7a..0bcab52 100644
--- a/ltx_video/models/autoencoders/vae.py
+++ b/ltx_video/models/autoencoders/vae.py
@@ -257,11 +257,11 @@ class AutoencoderKLWrapper(ModelMixin, ConfigMixin):
self,
z: torch.FloatTensor,
target_shape=None,
- timesteps: Optional[torch.Tensor] = None,
+ timestep: Optional[torch.Tensor] = None,
) -> Union[DecoderOutput, torch.FloatTensor]:
z = self.post_quant_conv(z)
- if "timesteps" in self.decoder_params:
- dec = self.decoder(z, target_shape=target_shape, timesteps=timesteps)
+ if "timestep" in self.decoder_params:
+ dec = self.decoder(z, target_shape=target_shape, timestep=timestep)
else:
dec = self.decoder(z, target_shape=target_shape)
return dec
@@ -271,7 +271,7 @@ class AutoencoderKLWrapper(ModelMixin, ConfigMixin):
z: torch.FloatTensor,
return_dict: bool = True,
target_shape=None,
- timesteps: Optional[torch.Tensor] = None,
+ timestep: Optional[torch.Tensor] = None,
) -> Union[DecoderOutput, torch.FloatTensor]:
assert target_shape is not None, "target_shape must be provided for decoding"
if self.use_z_tiling and z.shape[2] > self.z_sample_size > 1:
@@ -304,7 +304,7 @@ class AutoencoderKLWrapper(ModelMixin, ConfigMixin):
decoded = (
self._hw_tiled_decode(z, target_shape)
if self.use_hw_tiling
- else self._decode(z, target_shape=target_shape, timesteps=timesteps)
+ else self._decode(z, target_shape=target_shape, timestep=timestep)
)
if not return_dict:
diff --git a/ltx_video/models/autoencoders/vae_encode.py b/ltx_video/models/autoencoders/vae_encode.py
index 3b8b15b..f584ec0 100644
--- a/ltx_video/models/autoencoders/vae_encode.py
+++ b/ltx_video/models/autoencoders/vae_encode.py
@@ -96,6 +96,7 @@ def vae_decode(
is_video: bool = True,
split_size: int = 1,
vae_per_channel_normalize=False,
+ timestep=None,
) -> Tensor:
is_video_shaped = latents.dim() == 5
batch_size = latents.shape[0]
@@ -111,12 +112,16 @@ def vae_decode(
)
encode_bs = len(latents) // split_size
image_batch = [
- _run_decoder(latent_batch, vae, is_video, vae_per_channel_normalize)
+ _run_decoder(
+ latent_batch, vae, is_video, vae_per_channel_normalize, timestep
+ )
for latent_batch in latents.split(encode_bs)
]
images = torch.cat(image_batch, dim=0)
else:
- images = _run_decoder(latents, vae, is_video, vae_per_channel_normalize)
+ images = _run_decoder(
+ latents, vae, is_video, vae_per_channel_normalize, timestep
+ )
if is_video_shaped and not isinstance(
vae, (VideoAutoencoder, CausalVideoAutoencoder)
@@ -126,12 +131,19 @@ def vae_decode(
def _run_decoder(
- latents: Tensor, vae: AutoencoderKL, is_video: bool, vae_per_channel_normalize=False
+ latents: Tensor,
+ vae: AutoencoderKL,
+ is_video: bool,
+ vae_per_channel_normalize=False,
+ timestep=None,
) -> Tensor:
if isinstance(vae, (VideoAutoencoder, CausalVideoAutoencoder)):
*_, fl, hl, wl = latents.shape
temporal_scale, spatial_scale, _ = get_vae_size_scale_factor(vae)
latents = latents.to(vae.dtype)
+ vae_decode_kwargs = {}
+ if timestep is not None:
+ vae_decode_kwargs["timestep"] = timestep
image = vae.decode(
un_normalize_latents(latents, vae, vae_per_channel_normalize),
return_dict=False,
@@ -142,6 +154,7 @@ def _run_decoder(
hl * spatial_scale,
wl * spatial_scale,
),
+ **vae_decode_kwargs,
)[0]
else:
image = vae.decode(
diff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.py
index fa0a87b..19f191d 100644
--- a/ltx_video/pipelines/pipeline_ltx_video.py
+++ b/ltx_video/pipelines/pipeline_ltx_video.py
@@ -792,6 +792,8 @@ class LTXVideoPipeline(DiffusionPipeline):
callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
clean_caption: bool = True,
media_items: Optional[torch.FloatTensor] = None,
+ decode_timestep: Union[List[float], float] = 0.0,
+ decode_noise_scale: Optional[List[float]] = None,
mixed_precision: bool = False,
offload_to_cpu: bool = False,
**kwargs,
@@ -1171,11 +1173,30 @@ class LTXVideoPipeline(DiffusionPipeline):
// math.prod(self.patchifier.patch_size),
)
if output_type != "latent":
+ if self.vae.decoder.timestep_conditioning:
+ noise = torch.randn_like(latents)
+ if not isinstance(decode_timestep, list):
+ decode_timestep = [decode_timestep] * latents.shape[0]
+ if decode_noise_scale is None:
+ decode_noise_scale = decode_timestep
+ elif not isinstance(decode_noise_scale, list):
+ decode_noise_scale = [decode_noise_scale] * latents.shape[0]
+
+ decode_timestep = torch.tensor(decode_timestep).to(latents.device)
+ decode_noise_scale = torch.tensor(decode_noise_scale).to(
+ latents.device
+ )[:, None, None, None, None]
+ latents = (
+ latents * (1 - decode_noise_scale) + noise * decode_noise_scale
+ )
+ else:
+ decode_timestep = None
image = vae_decode(
latents,
self.vae,
is_video,
vae_per_channel_normalize=kwargs["vae_per_channel_normalize"],
+ timestep=decode_timestep,
)
image = self.image_processor.postprocess(image, output_type=output_type)