Skip to content

Official repository for LTX-Video

Pipeline: Support keyframe and sequence conditioning

Yoav HaCohen6189c45aa416
diff --git a/README.md b/README.md
index 6071b7c..8d88f84 100644
--- a/README.md
+++ b/README.md
@@ -88,7 +88,20 @@ python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --height HEIGHT --width
#### For image-to-video generation:
```bash
-python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --input_image_path IMAGE_PATH --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED
+python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --conditioning_media_paths IMAGE_PATH --conditioning_start_frames 0 --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED
+```
+
+#### Extending a video:
+
+```bash
+python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --conditioning_media_paths VIDEO_PATH --conditioning_start_frames START_FRAME --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED
+```
+
+
+#### For video generation with multiple conditions:
+
+```bash
+python inference.py --ckpt_path 'PATH' --prompt "PROMPT" --conditioning_media_paths IMAGE_PATH1 VIDEO_PATH2 --conditioning_start_frames 0 SECOND_COND_START_FRAME --height HEIGHT --width WIDTH --num_frames NUM_FRAMES --seed SEED
```
## ComfyUI Integration
diff --git a/__init__.py b/__init__.py
new file mode 100644
index 0000000..e69de29
diff --git a/inference.py b/inference.py
index a595bfe..ad1f60b 100644
--- a/inference.py
+++ b/inference.py
@@ -3,12 +3,12 @@ import os
import random
from datetime import datetime
from pathlib import Path
-from diffusers.utils import logging
-from typing import Optional, Union
+from typing import List, Optional, Union
import imageio
import numpy as np
import torch
+from diffusers.utils import logging
from PIL import Image
from transformers import T5EncoderModel, T5Tokenizer
@@ -17,9 +17,8 @@ from ltx_video.models.autoencoders.causal_video_autoencoder import (
)
from ltx_video.models.transformers.symmetric_patchifier import SymmetricPatchifier
from ltx_video.models.transformers.transformer3d import Transformer3DModel
-from ltx_video.pipelines.pipeline_ltx_video import LTXVideoPipeline
+from ltx_video.pipelines.pipeline_ltx_video import ConditioningItem, LTXVideoPipeline
from ltx_video.schedulers.rf import RectifiedFlowScheduler
-from ltx_video.utils.conditioning_method import ConditioningMethod
from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
MAX_HEIGHT = 720
@@ -174,14 +173,6 @@ def main():
required=True,
help="Path to a safetensors file that contains all model parts.",
)
- parser.add_argument(
- "--input_video_path",
- type=str,
- help="Path to the input video file (first frame used)",
- )
- parser.add_argument(
- "--input_image_path", type=str, help="Path to the input image file"
- )
parser.add_argument(
"--output_path",
type=str,
@@ -310,7 +301,25 @@ def main():
help="Local path or model identifier for both the tokenizer and text encoder. Defaults to pretrained model on Hugging Face.",
)
- # Add to the argument parser section
+ # Conditioning arguments
+ parser.add_argument(
+ "--conditioning_media_paths",
+ type=str,
+ nargs="*",
+ help="List of paths to conditioning media (images or videos). Each path will be used as a conditioning item.",
+ )
+ parser.add_argument(
+ "--conditioning_strengths",
+ type=float,
+ nargs="*",
+ help="List of conditioning strengths (between 0 and 1) for each conditioning item. Must match the number of conditioning items.",
+ )
+ parser.add_argument(
+ "--conditioning_start_frames",
+ type=int,
+ nargs="*",
+ help="List of frame indices where each conditioning item should be applied. Must match the number of conditioning items.",
+ )
parser.add_argument(
"--sampler",
type=str,
@@ -335,7 +344,7 @@ def create_ltx_video_pipeline(
assert os.path.exists(
ckpt_path
), f"Ckpt path provided (--ckpt_path) {ckpt_path} does not exist"
- vae = CausalVideoAutoencoder.from_pretrained(ckpt_path, torch_dtype=torch.bfloat16)
+ vae = CausalVideoAutoencoder.from_pretrained(ckpt_path)
transformer = Transformer3DModel.from_pretrained(ckpt_path)
# Use constructor if sampler is specified, otherwise use from_pretrained
@@ -401,23 +410,47 @@ def infer(
negative_prompt: str,
offload_to_cpu: bool,
text_encoder_model_name_or_path: str,
- input_image_path: Optional[str] = None,
+ conditioning_media_paths: Optional[List[str]] = None,
+ conditioning_strengths: Optional[List[float]] = None,
+ conditioning_start_frames: Optional[List[int]] = None,
sampler: Optional[str] = None,
device: Optional[str] = None,
**kwargs,
):
- device = device or get_device()
-
- if device == "mps":
- if torch.__version__ in ["2.4", "2.5"]:
- logger.warning(
- "PyTorch versions 2.4 and 2.5 have a known bug with MPS. Please update to PyTorch 2.6. More info: https://github.com/pytorch/pytorch/issues/141471"
+ if kwargs.get("input_image_path", None):
+ logger.warning(
+ "Please use conditioning_media_paths instead of input_image_path."
+ )
+ assert not conditioning_media_paths and not conditioning_start_frames
+ conditioning_media_paths = [kwargs["input_image_path"]]
+ conditioning_start_frames = [0]
+
+ # Validate conditioning arguments
+ if conditioning_media_paths:
+ # Use default strengths of 1.0
+ if not conditioning_strengths:
+ conditioning_strengths = [1.0] * len(conditioning_media_paths)
+ if not conditioning_start_frames:
+ raise ValueError(
+ "If `conditioning_media_paths` is provided, "
+ "`conditioning_start_frames` must also be provided"
+ )
+ if len(conditioning_media_paths) != len(conditioning_strengths) or len(
+ conditioning_media_paths
+ ) != len(conditioning_start_frames):
+ raise ValueError(
+ "`conditioning_media_paths`, `conditioning_strengths`, "
+ "and `conditioning_start_frames` must have the same length"
+ )
+ if any(s < 0 or s > 1 for s in conditioning_strengths):
+ raise ValueError("All conditioning strengths must be between 0 and 1")
+ if any(f < 0 or f >= num_frames for f in conditioning_start_frames):
+ raise ValueError(
+ f"All conditioning start frames must be between 0 and {num_frames-1}"
)
-
- generator = torch.Generator(device=device).manual_seed(seed)
seed_everething(seed)
- if offload_to_cpu and not device == "cpu":
+ if offload_to_cpu and not torch.cuda.is_available():
logger.warning(
"offload_to_cpu is set to True, but offloading will not occur since the model is already running on CPU."
)
@@ -451,23 +484,27 @@ def infer(
device=kwargs.get("device", get_device()),
)
- conditioning_item = (
+ conditioning_items = (
prepare_conditioning(
- path=input_image_path,
+ conditioning_media_paths=conditioning_media_paths,
+ conditioning_strengths=conditioning_strengths,
+ conditioning_start_frames=conditioning_start_frames,
height=height,
width=width,
+ num_frames=num_frames,
padding=padding,
+ pipeline=pipeline,
)
- if input_image_path
+ if conditioning_media_paths
else None
)
# Set spatiotemporal guidance
skip_block_list = [int(x.strip()) for x in stg_skip_layers.split(",")]
- if stg_mode.lower() == "stg_as" or stg_mode.lower() == "attention_skip":
- skip_layer_strategy = SkipLayerStrategy.AttentionSkip
- elif stg_mode.lower() == "stg_av" or stg_mode.lower() == "attention_values":
+ if stg_mode.lower() == "stg_av" or stg_mode.lower() == "attention_values":
skip_layer_strategy = SkipLayerStrategy.AttentionValues
+ elif stg_mode.lower() == "stg_as" or stg_mode.lower() == "attention_skip":
+ skip_layer_strategy = SkipLayerStrategy.AttentionSkip
elif stg_mode.lower() == "stg_r" or stg_mode.lower() == "residual":
skip_layer_strategy = SkipLayerStrategy.Residual
elif stg_mode.lower() == "stg_t" or stg_mode.lower() == "transformer_block":
@@ -475,17 +512,17 @@ def infer(
else:
raise ValueError(f"Invalid spatiotemporal guidance mode: {stg_mode}")
- skip_block_list = [int(x.strip()) for x in stg_skip_layers.split(",")]
-
# Prepare input for the pipeline
sample = {
"prompt": prompt,
"prompt_attention_mask": None,
"negative_prompt": negative_prompt,
"negative_prompt_attention_mask": None,
- "media_items": conditioning_item,
}
+ device = device or get_device()
+ generator = torch.Generator(device=device).manual_seed(seed)
+
images = pipeline(
num_inference_steps=num_inference_steps,
num_images_per_prompt=num_images_per_prompt,
@@ -503,13 +540,9 @@ def infer(
num_frames=num_frames_padded,
frame_rate=frame_rate,
**sample,
+ conditioning_items=conditioning_items,
is_video=True,
vae_per_channel_normalize=True,
- conditioning_method=(
- ConditioningMethod.FIRST_FRAME
- if conditioning_item is not None
- else ConditioningMethod.UNCONDITIONAL
- ),
image_cond_noise_scale=image_cond_noise_scale,
decode_timestep=decode_timestep,
decode_noise_scale=decode_noise_scale,
@@ -565,27 +598,76 @@ def infer(
def prepare_conditioning(
- path: str,
+ conditioning_media_paths: List[str],
+ conditioning_strengths: List[float],
+ conditioning_start_frames: List[int],
height: int,
width: int,
+ num_frames: int,
padding: tuple[int, int, int, int],
-) -> torch.Tensor:
+ pipeline: LTXVideoPipeline,
+) -> Optional[List[ConditioningItem]]:
"""Prepare conditioning items based on input media paths and their parameters.
Args:
- path: Path to conditioning image
- strength: Strength for the conditioning media
+ conditioning_media_paths: List of paths to conditioning media (images or videos)
+ conditioning_strengths: List of conditioning strengths for each media item
+ conditioning_start_frames: List of frame indices where each item should be applied
height: Height of the output frames
width: Width of the output frames
+ num_frames: Number of frames in the output video
padding: Padding to apply to the frames
+ pipeline: LTXVideoPipeline object used for condition video trimming
Returns:
A list of ConditioningItem objects.
"""
+ conditioning_items = []
+ for path, strength, start_frame in zip(
+ conditioning_media_paths, conditioning_strengths, conditioning_start_frames
+ ):
+ # Check if the path points to an image or video
+ is_video = any(
+ path.lower().endswith(ext) for ext in [".mp4", ".avi", ".mov", ".mkv"]
+ )
+
+ if is_video:
+ reader = imageio.get_reader(path)
+ orig_num_input_frames = reader.count_frames()
+ num_input_frames = pipeline.trim_conditioning_sequence(
+ start_frame, orig_num_input_frames, num_frames
+ )
+ if num_input_frames < orig_num_input_frames:
+ logger.warning(
+ f"Trimming conditioning video {path} from {orig_num_input_frames} to {num_input_frames} frames."
+ )
+
+ # Read and preprocess the relevant frames from the video file.
+ frames = []
+ for i in range(num_input_frames):
+ frame = Image.fromarray(reader.get_data(i))
+ frame_tensor = load_image_to_tensor_with_resize_and_crop(
+ frame, height, width
+ )
+ frame_tensor = torch.nn.functional.pad(frame_tensor, padding)
+ frames.append(frame_tensor)
+ reader.close()
+
+ # Stack frames along the temporal dimension
+ video_tensor = torch.cat(frames, dim=2)
+ conditioning_items.append(
+ ConditioningItem(video_tensor, start_frame, strength)
+ )
+ else: # Input image
+ frame_tensor = load_image_to_tensor_with_resize_and_crop(
+ path, height, width
+ )
+ frame_tensor = torch.nn.functional.pad(frame_tensor, padding)
+ conditioning_items.append(
+ ConditioningItem(frame_tensor, start_frame, strength)
+ )
- frame_tensor = load_image_to_tensor_with_resize_and_crop(path, height, width)
- frame_tensor = torch.nn.functional.pad(frame_tensor, padding)
- frame_tensor
+ return conditioning_items
if __name__ == "__main__":
diff --git a/ltx_video/models/autoencoders/vae_encode.py b/ltx_video/models/autoencoders/vae_encode.py
index f584ec0..bfc97f6 100644
--- a/ltx_video/models/autoencoders/vae_encode.py
+++ b/ltx_video/models/autoencoders/vae_encode.py
@@ -1,3 +1,4 @@
+from typing import Tuple
import torch
from diffusers import AutoencoderKL
from einops import rearrange
@@ -186,6 +187,44 @@ def get_vae_size_scale_factor(vae: AutoencoderKL) -> float:
return (temporal, spatial, spatial)
+def latent_to_pixel_coords(
+ latent_coords: Tensor, vae: AutoencoderKL, causal_fix: bool = False
+) -> Tensor:
+ """
+ Converts latent coordinates to pixel coordinates by scaling them according to the VAE's
+ configuration.
+
+ Args:
+ latent_coords (Tensor): A tensor of shape [batch_size, 3, num_latents]
+ containing the latent corner coordinates of each token.
+ vae (AutoencoderKL): The VAE model
+ causal_fix (bool): Whether to take into account the different temporal scale
+ of the first frame. Default = False for backwards compatibility.
+ Returns:
+ Tensor: A tensor of pixel coordinates corresponding to the input latent coordinates.
+ """
+
+ scale_factors = get_vae_size_scale_factor(vae)
+ causal_fix = isinstance(vae, CausalVideoAutoencoder) and causal_fix
+ pixel_coords = latent_to_pixel_coords_from_factors(
+ latent_coords, scale_factors, causal_fix
+ )
+ return pixel_coords
+
+
+def latent_to_pixel_coords_from_factors(
+ latent_coords: Tensor, scale_factors: Tuple, causal_fix: bool = False
+) -> Tensor:
+ pixel_coords = (
+ latent_coords
+ * torch.tensor(scale_factors, device=latent_coords.device)[None, :, None]
+ )
+ if causal_fix:
+ # Fix temporal scale for first frame to 1 due to causality
+ pixel_coords[:, 0] = (pixel_coords[:, 0] + 1 - scale_factors[0]).clamp(min=0)
+ return pixel_coords
+
+
def normalize_latents(
latents: Tensor, vae: AutoencoderKL, vae_per_channel_normalize: bool = False
) -> Tensor:
diff --git a/ltx_video/models/transformers/symmetric_patchifier.py b/ltx_video/models/transformers/symmetric_patchifier.py
index ba1bd6c..2eca320 100644
--- a/ltx_video/models/transformers/symmetric_patchifier.py
+++ b/ltx_video/models/transformers/symmetric_patchifier.py
@@ -6,8 +6,6 @@ from diffusers.configuration_utils import ConfigMixin
from einops import rearrange
from torch import Tensor
-from ltx_video.utils.torch_utils import append_dims
-
class Patchifier(ConfigMixin, ABC):
def __init__(self, patch_size: int):
@@ -15,10 +13,8 @@ class Patchifier(ConfigMixin, ABC):
self._patch_size = (1, patch_size, patch_size)
@abstractmethod
- def patchify(
- self, latents: Tensor, frame_rates: Tensor, scale_grid: bool
- ) -> Tuple[Tensor, Tensor]:
- pass
+ def patchify(self, latents: Tensor) -> Tuple[Tensor, Tensor]:
+ raise NotImplementedError("Patchify method not implemented")
@abstractmethod
def unpatchify(
@@ -26,7 +22,6 @@ class Patchifier(ConfigMixin, ABC):
latents: Tensor,
output_height: int,
output_width: int,
- output_num_frames: int,
out_channels: int,
) -> Tuple[Tensor, Tensor]:
pass
@@ -35,36 +30,31 @@ class Patchifier(ConfigMixin, ABC):
def patch_size(self):
return self._patch_size
- def get_grid(
- self, orig_num_frames, orig_height, orig_width, batch_size, scale_grid, device
+ def get_latent_coords(
+ self, latent_num_frames, latent_height, latent_width, batch_size, device
):
- f = orig_num_frames // self._patch_size[0]
- h = orig_height // self._patch_size[1]
- w = orig_width // self._patch_size[2]
- grid_h = torch.arange(h, dtype=torch.float32, device=device)
- grid_w = torch.arange(w, dtype=torch.float32, device=device)
- grid_f = torch.arange(f, dtype=torch.float32, device=device)
- grid = torch.meshgrid(grid_f, grid_h, grid_w)
- grid = torch.stack(grid, dim=0)
- grid = grid.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)
-
- if scale_grid is not None:
- for i in range(3):
- if isinstance(scale_grid[i], Tensor):
- scale = append_dims(scale_grid[i], grid.ndim - 1)
- else:
- scale = scale_grid[i]
- grid[:, i, ...] = grid[:, i, ...] * scale * self._patch_size[i]
-
- grid = rearrange(grid, "b c f h w -> b c (f h w)", b=batch_size)
- return grid
+ """
+ Return a tensor of shape [batch_size, 3, num_patches] containing the
+ top-left corner latent coordinates of each latent patch.
+ The tensor is repeated for each batch element.
+ """
+ latent_sample_coords = torch.meshgrid(
+ torch.arange(0, latent_num_frames, self._patch_size[0], device=device),
+ torch.arange(0, latent_height, self._patch_size[1], device=device),
+ torch.arange(0, latent_width, self._patch_size[2], device=device),
+ )
+ latent_sample_coords = torch.stack(latent_sample_coords, dim=0)
+ latent_coords = latent_sample_coords.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)
+ latent_coords = rearrange(
+ latent_coords, "b c f h w -> b c (f h w)", b=batch_size
+ )
+ return latent_coords
class SymmetricPatchifier(Patchifier):
- def patchify(
- self,
- latents: Tensor,
- ) -> Tuple[Tensor, Tensor]:
+ def patchify(self, latents: Tensor) -> Tuple[Tensor, Tensor]:
+ b, _, f, h, w = latents.shape
+ latent_coords = self.get_latent_coords(f, h, w, b, latents.device)
latents = rearrange(
latents,
"b c (f p1) (h p2) (w p3) -> b (f h w) (c p1 p2 p3)",
@@ -72,22 +62,20 @@ class SymmetricPatchifier(Patchifier):
p2=self._patch_size[1],
p3=self._patch_size[2],
)
- return latents
+ return latents, latent_coords
def unpatchify(
self,
latents: Tensor,
output_height: int,
output_width: int,
- output_num_frames: int,
out_channels: int,
) -> Tuple[Tensor, Tensor]:
output_height = output_height // self._patch_size[1]
output_width = output_width // self._patch_size[2]
latents = rearrange(
latents,
- "b (f h w) (c p q) -> b c f (h p) (w q) ",
- f=output_num_frames,
+ "b (f h w) (c p q) -> b c f (h p) (w q)",
h=output_height,
w=output_width,
p=self._patch_size[1],
diff --git a/ltx_video/models/transformers/transformer3d.py b/ltx_video/models/transformers/transformer3d.py
index c883e2e..15b388d 100644
--- a/ltx_video/models/transformers/transformer3d.py
+++ b/ltx_video/models/transformers/transformer3d.py
@@ -79,6 +79,7 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
positional_embedding_theta: Optional[float] = None,
positional_embedding_max_pos: Optional[List[int]] = None,
timestep_scale_multiplier: Optional[float] = None,
+ causal_temporal_positioning: bool = False, # For backward compatibility, will be deprecated
):
super().__init__()
self.use_tpu_flash_attention = (
diff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.py
index 9d09aa3..bcfafda 100644
--- a/ltx_video/pipelines/pipeline_ltx_video.py
+++ b/ltx_video/pipelines/pipeline_ltx_video.py
@@ -2,12 +2,12 @@
import inspect
import math
import re
+from contextlib import nullcontext
+from dataclasses import dataclass
from typing import Callable, Dict, List, Optional, Tuple, Union
-
import torch
import torch.nn.functional as F
-from contextlib import nullcontext
from diffusers.image_processor import VaeImageProcessor
from diffusers.models import AutoencoderKL
from diffusers.pipelines.pipeline_utils import DiffusionPipeline, ImagePipelineOutput
@@ -17,18 +17,18 @@ from diffusers.utils.torch_utils import randn_tensor
from einops import rearrange
from transformers import T5EncoderModel, T5Tokenizer
-from ltx_video.models.transformers.transformer3d import Transformer3DModel
-from ltx_video.models.transformers.symmetric_patchifier import Patchifier
+from ltx_video.models.autoencoders.causal_video_autoencoder import (
+ CausalVideoAutoencoder,
+)
from ltx_video.models.autoencoders.vae_encode import (
get_vae_size_scale_factor,
+ latent_to_pixel_coords,
vae_decode,
vae_encode,
)
-from ltx_video.models.autoencoders.causal_video_autoencoder import (
- CausalVideoAutoencoder,
-)
+from ltx_video.models.transformers.symmetric_patchifier import Patchifier
+from ltx_video.models.transformers.transformer3d import Transformer3DModel
from ltx_video.schedulers.rf import TimestepShifter
-from ltx_video.utils.conditioning_method import ConditioningMethod
from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
@@ -154,6 +154,21 @@ def retrieve_timesteps(
return timesteps, num_inference_steps
+@dataclass
+class ConditioningItem:
+ """
+ Defines a single frame-conditioning item - a single frame or a sequence of frames.
+ Attributes:
+ media_item (torch.Tensor), shape=(b, 3, f, h, w): The media item to condition on.
+ media_frame_number (int): The start-frame number of the media item in the generated video.
+ conditioning_strength (float): The strength of the conditioning (1.0 = full conditioning).
+ """
+
+ media_item: torch.Tensor
+ media_frame_number: int
+ conditioning_strength: float
+
+
class LTXVideoPipeline(DiffusionPipeline):
r"""
Pipeline for text-to-image generation using LTX-Video.
@@ -502,61 +517,52 @@ class LTXVideoPipeline(DiffusionPipeline):
return [process(t) for t in text]
- def image_cond_noise_update(
- self,
- t,
- init_latents,
- latents,
- noise_scale,
- conditiong_mask,
+ @staticmethod
+ def add_noise_to_image_conditioning_latents(
+ t: float,
+ init_latents: torch.Tensor,
+ latents: torch.Tensor,
+ noise_scale: float,
+ conditioning_mask: torch.Tensor,
generator,
+ eps=1e-6,
):
+ """
+ Add timestep-dependent noise to the hard-conditioning latents.
+ This helps with motion continuity, especially when conditioned on a single frame.
+ """
noise = randn_tensor(
latents.shape,
generator=generator,
device=latents.device,
dtype=latents.dtype,
)
- latents = (init_latents + noise_scale * noise * (t**2)) * conditiong_mask[
- ..., None
- ] + latents * (1 - conditiong_mask[..., None])
+ # Add noise only to hard-conditioning latents (conditioning_mask = 1.0)
+ need_to_noise = (conditioning_mask > 1.0 - eps).unsqueeze(-1)
+ noised_latents = init_latents + noise_scale * noise * (t**2)
+ latents = torch.where(need_to_noise, noised_latents, latents)
return latents
# Copied from diffusers.pipelines.stable_diffusion.pipeline_stable_diffusion.StableDiffusionPipeline.prepare_latents
def prepare_latents(
self,
- batch_size,
- num_latent_channels,
- num_patches,
+ latent_shape,
dtype,
device,
generator,
- latents=None,
- latents_mask=None,
):
- shape = (
- batch_size,
- num_patches // math.prod(self.patchifier.patch_size),
- num_latent_channels,
- )
-
- if isinstance(generator, list) and len(generator) != batch_size:
+ if isinstance(generator, list) and len(generator) != latent_shape[0]:
raise ValueError(
f"You have passed a list of generators of length {len(generator)}, but requested an effective batch"
- f" size of {batch_size}. Make sure the batch size matches the length of the generators."
+ f" size of {latent_shape[0]}. Make sure the batch size matches the length of the generators."
)
- if latents is None:
- latents = randn_tensor(
- shape, generator=generator, device=device, dtype=dtype
- )
- elif latents_mask is not None:
- noise = randn_tensor(shape, generator=generator, device=device, dtype=dtype)
- latents = latents * latents_mask[..., None] + noise * (
- 1 - latents_mask[..., None]
- )
- else:
- latents = latents.to(device)
+ # For backward compatibility, generate in the "patchified" shape and rearrange
+ b, c, f, h, w = latent_shape
+ latents = randn_tensor(
+ (b, f * h * w, c), generator=generator, device=device, dtype=dtype
+ )
+ latents = rearrange(latents, "b (f h w) c -> b c f h w", f=f, h=h, w=w)
# scale the initial noise by the standard deviation required by the scheduler
latents = latents * self.scheduler.init_noise_sigma
@@ -631,7 +637,7 @@ class LTXVideoPipeline(DiffusionPipeline):
output_type: Optional[str] = "pil",
return_dict: bool = True,
callback_on_step_end: Optional[Callable[[int, int, Dict], None]] = None,
- media_items: Optional[torch.FloatTensor] = None,
+ conditioning_items: Optional[List[ConditioningItem]] = None,
decode_timestep: Union[List[float], float] = 0.0,
decode_noise_scale: Optional[List[float]] = None,
mixed_precision: bool = False,
@@ -724,6 +730,13 @@ class LTXVideoPipeline(DiffusionPipeline):
negative_prompt_attention_mask,
)
+ if kwargs.get("media_items", None) is not None:
+ # Backwards compatibility mode for first-frame conditioning
+ assert (
+ conditioning_items is None
+ ), "Cannot pass both `conditioning_items` and `media_items`."
+ conditioning_items = [ConditioningItem(kwargs["media_items"], 0, 1.0)]
+
# 2. Default height and width to transformer
if prompt is not None and isinstance(prompt, str):
batch_size = 1
@@ -799,17 +812,8 @@ class LTXVideoPipeline(DiffusionPipeline):
# 3b. Encode and prepare conditioning data
self.video_scale_factor = self.video_scale_factor if is_video else 1
- conditioning_method = kwargs.get("conditioning_method", None)
vae_per_channel_normalize = kwargs.get("vae_per_channel_normalize", False)
image_cond_noise_scale = kwargs.get("image_cond_noise_scale", 0.0)
- init_latents, conditioning_mask = self.prepare_conditioning(
- media_items,
- num_frames,
- height,
- width,
- conditioning_method,
- vae_per_channel_normalize,
- )
# 4. Prepare latents.
latent_height = height // self.vae_scale_factor
@@ -817,26 +821,43 @@ class LTXVideoPipeline(DiffusionPipeline):
latent_num_frames = num_frames // self.video_scale_factor
if isinstance(self.vae, CausalVideoAutoencoder) and is_video:
latent_num_frames += 1
- latent_frame_rate = frame_rate / self.video_scale_factor
- num_latent_patches = latent_height * latent_width * latent_num_frames
+ latent_shape = (
+ batch_size * num_images_per_prompt,
+ self.transformer.config.in_channels,
+ latent_num_frames,
+ latent_height,
+ latent_width,
+ )
+
+ # Prepare the initial random latents tensor, shape = (b, c, f, h, w)
latents = self.prepare_latents(
- batch_size=batch_size * num_images_per_prompt,
- num_latent_channels=self.transformer.config.in_channels,
- num_patches=num_latent_patches,
+ latent_shape=latent_shape,
dtype=prompt_embeds_batch.dtype,
device=device,
generator=generator,
- latents=init_latents,
- latents_mask=conditioning_mask,
)
- orig_conditiong_mask = conditioning_mask
+
+ # Update the latents with the conditioning items and patchify them into (b, n, c)
+ latents, pixel_coords, conditioning_mask, num_cond_latents = (
+ self.prepare_conditioning(
+ conditioning_items=conditioning_items,
+ init_latents=latents,
+ num_frames=num_frames,
+ height=height,
+ width=width,
+ vae_per_channel_normalize=vae_per_channel_normalize,
+ generator=generator,
+ )
+ )
+ init_latents = latents.clone() # Used for image_cond_noise_update
+
+ pixel_coords = torch.cat([pixel_coords] * num_conds)
+ orig_conditioning_mask = conditioning_mask
if conditioning_mask is not None and is_video:
assert num_images_per_prompt == 1
- conditioning_mask = (
- torch.cat([conditioning_mask] * num_conds)
- if num_conds > 1
- else conditioning_mask
- )
+ conditioning_mask = torch.cat([conditioning_mask] * num_conds)
+ fractional_coords = pixel_coords.to(torch.float32)
+ fractional_coords[:, 0] = fractional_coords[:, 0] * (1.0 / frame_rate)
# 5. Prepare timesteps
retrieve_timesteps_kwargs = {}
@@ -860,13 +881,13 @@ class LTXVideoPipeline(DiffusionPipeline):
with self.progress_bar(total=num_inference_steps) as progress_bar:
for i, t in enumerate(timesteps):
- if conditioning_method == ConditioningMethod.FIRST_FRAME:
- latents = self.image_cond_noise_update(
+ if conditioning_mask is not None and image_cond_noise_scale > 0.0:
+ latents = self.add_noise_to_image_conditioning_latents(
t,
init_latents,
latents,
image_cond_noise_scale,
- orig_conditiong_mask,
+ orig_conditioning_mask,
generator,
)
@@ -877,13 +898,6 @@ class LTXVideoPipeline(DiffusionPipeline):
latent_model_input, t
)
- latent_frame_rates = (
- torch.ones(
- latent_model_input.shape[0], 1, device=latent_model_input.device
- )
- * latent_frame_rate
- )
-
current_timestep = t
if not torch.is_tensor(current_timestep):
# TODO: this requires sync between CPU and GPU. So try to pass timesteps as tensors if you can
@@ -906,26 +920,14 @@ class LTXVideoPipeline(DiffusionPipeline):
current_timestep = current_timestep.expand(
latent_model_input.shape[0]
).unsqueeze(-1)
- scale_grid = (
- (
- 1 / latent_frame_rates,
- self.vae_scale_factor,
- self.vae_scale_factor,
- )
- if self.transformer.use_rope
- else None
- )
- indices_grid = self.patchifier.get_grid(
- orig_num_frames=latent_num_frames,
- orig_height=latent_height,
- orig_width=latent_width,
- batch_size=latent_model_input.shape[0],
- scale_grid=scale_grid,
- device=latents.device,
- )
if conditioning_mask is not None:
- current_timestep = current_timestep * (1 - conditioning_mask)
+ # Conditioning latents have an initial timestep and noising level of (1.0 - conditioning_mask)
+ # and will start to be denoised when the current timestep is lower than their conditioning timestep.
+ current_timestep = torch.min(
+ current_timestep, 1.0 - conditioning_mask
+ )
+
# Choose the appropriate context manager based on `mixed_precision`
if mixed_precision:
if "xla" in device.type:
@@ -941,7 +943,7 @@ class LTXVideoPipeline(DiffusionPipeline):
with context_manager:
noise_pred = self.transformer(
latent_model_input.to(self.transformer.dtype),
- indices_grid,
+ indices_grid=fractional_coords,
encoder_hidden_states=prompt_embeds_batch.to(
self.transformer.dtype
),
@@ -954,20 +956,32 @@ class LTXVideoPipeline(DiffusionPipeline):
# perform guidance
if do_spatio_temporal_guidance:
- noise_pred_text_perturb = noise_pred[-1:]
+ noise_pred_text, noise_pred_text_perturb = noise_pred.chunk(
+ num_conds
+ )[-2:]
if do_classifier_free_guidance:
- noise_pred_uncond, noise_pred_text = noise_pred[:2].chunk(2)
+ noise_pred_uncond, noise_pred_text = noise_pred.chunk(num_conds)[:2]
noise_pred = noise_pred_uncond + guidance_scale * (
noise_pred_text - noise_pred_uncond
)
+ elif do_spatio_temporal_guidance:
+ noise_pred = noise_pred_text
if do_spatio_temporal_guidance:
noise_pred = noise_pred + stg_scale * (
noise_pred_text - noise_pred_text_perturb
)
if do_rescaling:
- factor = noise_pred_text.std() / noise_pred.std()
+ noise_pred_text_std = noise_pred_text.view(batch_size, -1).std(
+ dim=1, keepdim=True
+ )
+ noise_pred_std = noise_pred.view(batch_size, -1).std(
+ dim=1, keepdim=True
+ )
+
+ factor = noise_pred_text_std / noise_pred_std
factor = rescaling_scale * factor + (1 - rescaling_scale)
- noise_pred = noise_pred * factor
+
+ noise_pred = noise_pred * factor.view(batch_size, 1, 1)
current_timestep = current_timestep[:1]
# learned sigma
@@ -978,13 +992,14 @@ class LTXVideoPipeline(DiffusionPipeline):
noise_pred = noise_pred.chunk(2, dim=1)[0]
# compute previous image: x_t -> x_t-1
- latents = self.scheduler.step(
- noise_pred,
- t if current_timestep is None else current_timestep,
+ latents = self.denoising_step(
latents,
- **extra_step_kwargs,
- return_dict=False,
- )[0]
+ noise_pred,
+ current_timestep,
+ orig_conditioning_mask,
+ t,
+ extra_step_kwargs,
+ )
# call the callback, if provided
if i == len(timesteps) - 1 or (
@@ -1000,11 +1015,13 @@ class LTXVideoPipeline(DiffusionPipeline):
if self._execution_device == "cuda":
torch.cuda.empty_cache()
+ # Remove the added conditioning latents
+ latents = latents[:, num_cond_latents:]
+
latents = self.patchifier.unpatchify(
latents=latents,
output_height=latent_height,
output_width=latent_width,
- output_num_frames=latent_num_frames,
out_channels=self.transformer.in_channels
// math.prod(self.patchifier.patch_size),
)
@@ -1047,64 +1064,298 @@ class LTXVideoPipeline(DiffusionPipeline):
return ImagePipelineOutput(images=image)
+ def denoising_step(
+ self,
+ latents: torch.Tensor,
+ noise_pred: torch.Tensor,
+ current_timestep: torch.Tensor,
+ conditioning_mask: torch.Tensor,
+ t: float,
+ extra_step_kwargs,
+ t_eps=1e-6,
+ ):
+ """
+ Perform the denoising step for the required tokens, based on the current timestep and
+ conditioning mask:
+ Conditioning latents have an initial timestep and noising level of (1.0 - conditioning_mask)
+ and will start to be denoised when the current timestep is equal or lower than their
+ conditioning timestep.
+ (hard-conditioning latents with conditioning_mask = 1.0 are never denoised)
+ """
+ # Denoise the latents using the scheduler
+ denoised_latents = self.scheduler.step(
+ noise_pred,
+ t if current_timestep is None else current_timestep,
+ latents,
+ **extra_step_kwargs,
+ return_dict=False,
+ )[0]
+
+ if conditioning_mask is None:
+ return denoised_latents
+
+ tokens_to_denoise_mask = (t - t_eps < (1.0 - conditioning_mask)).unsqueeze(-1)
+ return torch.where(tokens_to_denoise_mask, denoised_latents, latents)
+
def prepare_conditioning(
self,
- media_items: torch.Tensor,
+ conditioning_items: Optional[List[ConditioningItem]],
+ init_latents: torch.Tensor,
num_frames: int,
height: int,
width: int,
- method: ConditioningMethod = ConditioningMethod.UNCONDITIONAL,
vae_per_channel_normalize: bool = False,
- ) -> Tuple[torch.Tensor, torch.Tensor]:
+ generator=None,
+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
"""
- Prepare the conditioning data for the video generation. If an input media item is provided, encode it
- and set the conditioning_mask to indicate which tokens to condition on. Input media item should have
- the same height and width as the generated video.
+ Prepare conditioning tokens based on the provided conditioning items.
+
+ This method encodes provided conditioning items (video frames or single frames) into latents
+ and integrates them with the initial latent tensor. It also calculates corresponding pixel
+ coordinates, a mask indicating the influence of conditioning latents, and the total number of
+ conditioning latents.
Args:
- media_items (torch.Tensor): media items to condition on (images or videos)
- num_frames (int): number of frames to generate
- height (int): height of the generated video
- width (int): width of the generated video
- method (ConditioningMethod, optional): conditioning method to use. Defaults to ConditioningMethod.UNCONDITIONAL.
- vae_per_channel_normalize (bool, optional): whether to normalize the input to the VAE per channel. Defaults to False.
+ conditioning_items (Optional[List[ConditioningItem]]): A list of ConditioningItem objects.
+ init_latents (torch.Tensor): The initial latent tensor of shape (b, c, f_l, h_l, w_l), where
+ `f_l` is the number of latent frames, and `h_l` and `w_l` are latent spatial dimensions.
+ num_frames, height, width: The dimensions of the generated video.
+ vae_per_channel_normalize (bool, optional): Whether to normalize channels during VAE encoding.
+ Defaults to `False`.
+ generator: The random generator
Returns:
- Tuple[torch.Tensor, torch.Tensor]: the conditioning latents and the conditioning mask
+ Tuple[torch.Tensor, torch.Tensor, torch.Tensor, int]:
+ - `init_latents` (torch.Tensor): The updated latent tensor including conditioning latents,
+ patchified into (b, n, c) shape.
+ - `init_pixel_coords` (torch.Tensor): The pixel coordinates corresponding to the updated
+ latent tensor.
+ - `conditioning_mask` (torch.Tensor): A mask indicating the conditioning-strength of each
+ latent token.
+ - `num_cond_latents` (int): The total number of latent tokens added from conditioning items.
+
+ Raises:
+ AssertionError: If input shapes, dimensions, or conditions for applying conditioning are invalid.
"""
- if media_items is None or method == ConditioningMethod.UNCONDITIONAL:
- return None, None
+ assert isinstance(self.vae, CausalVideoAutoencoder)
+
+ if conditioning_items:
+ batch_size, _, num_latent_frames = init_latents.shape[:3]
+ # Initialize the conditioning mask
+ conditioning_latent_frames_mask = torch.zeros(
+ (batch_size, num_latent_frames),
+ dtype=torch.float32,
+ device=init_latents.device,
+ )
- assert media_items.ndim == 5
- assert height == media_items.shape[-2] and width == media_items.shape[-1]
+ extra_conditioning_latents = []
+ extra_conditioning_pixel_coords = []
+ extra_conditioning_mask = []
+ extra_conditioning_num_latents = 0 # Number of extra conditioning latents added (should be removed before decoding)
+
+ # Process each conditioning item
+ for conditioning_item in conditioning_items:
+ media_item = conditioning_item.media_item
+ media_frame_number = conditioning_item.media_frame_number
+ strength = conditioning_item.conditioning_strength
+ assert media_item.ndim == 5 # (b, c, f, h, w)
+ b, c, n_frames, h, w = media_item.shape
+ assert height == h and width == w
+ assert n_frames % 8 == 1
+ assert (
+ media_frame_number >= 0
+ and media_frame_number + n_frames <= num_frames
+ )
- # Encode the input video and repeat to the required number of frame-tokens
- init_latents = vae_encode(
- media_items.to(dtype=self.vae.dtype, device=self.vae.device),
+ # Encode the provided conditioning media item
+ latents = vae_encode(
+ media_item.to(dtype=self.vae.dtype, device=self.vae.device),
+ self.vae,
+ vae_per_channel_normalize=vae_per_channel_normalize,
+ ).to(dtype=self.transformer.dtype)
+
+ # Handle the different conditioning cases
+ if media_frame_number == 0:
+ # First frame or sequence - just update the initial noise latents and the mask
+ f_l = latents.shape[2]
+ init_latents[:, :, :f_l] = torch.lerp(
+ init_latents[:, :, :f_l], latents, strength
+ )
+ conditioning_latent_frames_mask[:, :f_l] = strength
+ else:
+ # Non-first frame or sequence
+ if n_frames > 1:
+ # Handle non-first sequence.
+ # Encoded latents are either fully consumed, or the prefix is handled separately below.
+ init_latents, conditioning_latent_frames_mask, latents = (
+ self._handle_non_first_conditioning_sequence(
+ init_latents,
+ conditioning_latent_frames_mask,
+ latents,
+ media_frame_number,
+ strength,
+ )
+ )
+
+ if latents is not None: # Single frame or sequence-prefix latents
+ noise = randn_tensor(
+ latents.shape,
+ generator=generator,
+ device=latents.device,
+ dtype=latents.dtype,
+ )
+
+ latents = torch.lerp(noise, latents, strength)
+
+ # Patchify the extra conditioning latents and calculate their pixel coordinates
+ latents, latent_coords = self.patchifier.patchify(
+ latents=latents
+ )
+ pixel_coords = latent_to_pixel_coords(
+ latent_coords,
+ self.vae,
+ causal_fix=self.transformer.config.causal_temporal_positioning,
+ )
+
+ # Update the frame numbers to match the target frame number
+ pixel_coords[:, 0] += media_frame_number
+ extra_conditioning_num_latents += latents.shape[1]
+
+ conditioning_mask = torch.full(
+ latents.shape[:2],
+ strength,
+ dtype=torch.float32,
+ device=conditioning_latent_frames_mask.device,
+ )
+
+ extra_conditioning_latents.append(latents)
+ extra_conditioning_pixel_coords.append(pixel_coords)
+ extra_conditioning_mask.append(conditioning_mask)
+
+ # Patchify the updated latents and calculate their pixel coordinates
+ init_latents, init_latent_coords = self.patchifier.patchify(
+ latents=init_latents
+ )
+ init_pixel_coords = latent_to_pixel_coords(
+ init_latent_coords,
self.vae,
- vae_per_channel_normalize=vae_per_channel_normalize,
- ).float()
+ causal_fix=self.transformer.config.causal_temporal_positioning,
+ )
+
+ if not conditioning_items:
+ return init_latents, init_pixel_coords, None, 0
- init_len, target_len = (
- init_latents.shape[2],
- num_frames // self.video_scale_factor,
+ # Create a per-token mask based on the updated conditioning_latent_frames_mask
+ init_conditioning_mask = conditioning_latent_frames_mask.gather(
+ 1, init_latent_coords[:, 0]
)
- if isinstance(self.vae, CausalVideoAutoencoder):
- target_len += 1
- init_latents = init_latents[:, :, :target_len]
- if target_len > init_len:
- repeat_factor = (target_len + init_len - 1) // init_len # Ceiling division
- init_latents = init_latents.repeat(1, 1, repeat_factor, 1, 1)[
- :, :, :target_len
- ]
-
- # Prepare the conditioning mask (1.0 = condition on this token)
- b, n, f, h, w = init_latents.shape
- conditioning_mask = torch.zeros([b, 1, f, h, w], device=init_latents.device)
- if method == ConditioningMethod.FIRST_FRAME:
- conditioning_mask[:, :, 0] = 1.0
-
- # Patchify the init latents and the mask
- conditioning_mask = self.patchifier.patchify(conditioning_mask).squeeze(-1)
- init_latents = self.patchifier.patchify(latents=init_latents)
- return init_latents, conditioning_mask
+
+ if extra_conditioning_latents:
+ # Stack the extra conditioning latents, pixel coordinates and mask
+ init_latents = torch.cat([*extra_conditioning_latents, init_latents], dim=1)
+ init_pixel_coords = torch.cat(
+ [*extra_conditioning_pixel_coords, init_pixel_coords], dim=2
+ )
+ init_conditioning_mask = torch.cat(
+ [*extra_conditioning_mask, init_conditioning_mask], dim=1
+ )
+
+ return (
+ init_latents,
+ init_pixel_coords,
+ init_conditioning_mask,
+ extra_conditioning_num_latents,
+ )
+
+ @staticmethod
+ def _handle_non_first_conditioning_sequence(
+ init_latents: torch.Tensor,
+ conditioning_latent_frames_mask: torch.Tensor,
+ latents: torch.Tensor,
+ media_frame_number: int,
+ strength: float,
+ num_prefix_latent_frames: int = 2,
+ prefix_latents_mode: str = "concat",
+ prefix_soft_conditioning_strength: float = 0.15,
+ ):
+ """
+ Special handling for a conditioning sequence that does not start on the first frame.
+ The special handling is required to allow a short encoded video to be used as middle
+ (or last) sequence in a longer video.
+ Args:
+ init_latents (torch.Tensor): The initial noise latents to be updated.
+ conditioning_latent_frames_mask (torch.Tensor): A mask indicating the conditioning-strength of each
+ latent token.
+ latents (torch.Tensor): The encoded conditioning item.
+ media_frame_number (int): The target frame number of the first frame in the conditioning sequence.
+ strength (float): The conditioning strength for the conditioning latents.
+ num_prefix_latent_frames (int, optional): The length of the sequence prefix, to be handled
+ separately. Defaults to 2.
+ prefix_latents_mode (str, optional): Special treatment for prefix (boundary) latents.
+ - "drop": Drop the prefix latents.
+ - "soft": Use the prefix latents, but with soft-conditioning
+ - "concat": Add the prefix latents as extra tokens (like single frames)
+ prefix_soft_conditioning_strength (float, optional): The strength of the soft-conditioning for
+ the prefix latents, relevant if `prefix_latents_mode` is "soft". Defaults to 0.1.
+
+ """
+ f_l = latents.shape[2]
+ f_l_p = num_prefix_latent_frames
+ assert f_l >= f_l_p
+ assert media_frame_number % 8 == 0
+ if f_l > f_l_p:
+ # Insert the conditioning latents **excluding the prefix** into the sequence
+ f_l_start = media_frame_number // 8 + f_l_p
+ f_l_end = f_l_start + f_l - f_l_p
+ init_latents[:, :, f_l_start:f_l_end] = torch.lerp(
+ init_latents[:, :, f_l_start:f_l_end],
+ latents[:, :, f_l_p:],
+ strength,
+ )
+ # Mark these latent frames as conditioning latents
+ conditioning_latent_frames_mask[:, f_l_start:f_l_end] = strength
+
+ # Handle the prefix-latents
+ if prefix_latents_mode == "soft":
+ if f_l_p > 1:
+ # Drop the first (single-frame) latent and soft-condition the remaining prefix
+ f_l_start = media_frame_number // 8 + 1
+ f_l_end = f_l_start + f_l_p - 1
+ strength = min(prefix_soft_conditioning_strength, strength)
+ init_latents[:, :, f_l_start:f_l_end] = torch.lerp(
+ init_latents[:, :, f_l_start:f_l_end],
+ latents[:, :, 1:f_l_p],
+ strength,
+ )
+ # Mark these latent frames as conditioning latents
+ conditioning_latent_frames_mask[:, f_l_start:f_l_end] = strength
+ latents = None # No more latents to handle
+ elif prefix_latents_mode == "drop":
+ # Drop the prefix latents
+ latents = None
+ elif prefix_latents_mode == "concat":
+ # Pass-on the prefix latents to be handled as extra conditioning frames
+ latents = latents[:, :, :f_l_p]
+ else:
+ raise ValueError(f"Invalid prefix_latents_mode: {prefix_latents_mode}")
+ return init_latents, conditioning_latent_frames_mask, latents
+
+ def trim_conditioning_sequence(
+ self, start_frame: int, sequence_num_frames: int, target_num_frames: int
+ ):
+ """
+ Trim a conditioning sequence to the allowed number of frames.
+
+ Args:
+ start_frame (int): The target frame number of the first frame in the sequence.
+ sequence_num_frames (int): The number of frames in the sequence.
+ target_num_frames (int): The target number of frames in the generated video.
+
+ Returns:
+ int: updated sequence length
+ """
+ scale_factor = self.video_scale_factor
+ num_frames = min(sequence_num_frames, target_num_frames - start_frame)
+ # Trim down to a multiple of temporal_scale_factor frames plus 1
+ num_frames = (num_frames - 1) // scale_factor * scale_factor + 1
+ return num_frames
diff --git a/ltx_video/schedulers/rf.py b/ltx_video/schedulers/rf.py
index c27d065..ed2f6c5 100644
--- a/ltx_video/schedulers/rf.py
+++ b/ltx_video/schedulers/rf.py
@@ -25,6 +25,8 @@ from ltx_video.utils.diffusers_config_mapping import (
def linear_quadratic_schedule(num_steps, threshold_noise=0.025, linear_steps=None):
if linear_steps is None:
linear_steps = num_steps // 2
+ if num_steps < 2:
+ return torch.tensor([1.0])
linear_sigma_schedule = [
i * threshold_noise / linear_steps for i in range(linear_steps)
]
@@ -189,9 +191,6 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
self.base_resolution = base_resolution
self.target_shift_terminal = target_shift_terminal
self.timesteps = self.sigmas = self.get_initial_timesteps(num_train_timesteps)
- self.delta_timesteps = self.timesteps - torch.cat(
- [self.timesteps[1:], torch.zeros_like(self.timesteps[-1:])]
- )
def get_initial_timesteps(self, num_timesteps: int) -> Tensor:
if self.sampler == "Uniform":
@@ -227,9 +226,6 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
num_inference_steps = min(self.config.num_train_timesteps, num_inference_steps)
self.timesteps = self.get_initial_timesteps(num_inference_steps).to(device)
self.timesteps = self.shift_timesteps(samples, self.timesteps)
- self.delta_timesteps = self.timesteps - torch.cat(
- [self.timesteps[1:], torch.zeros_like(self.timesteps[-1:])]
- )
self.num_inference_steps = num_inference_steps
self.sigmas = self.timesteps
@@ -280,36 +276,23 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
model_output: torch.FloatTensor,
timestep: torch.FloatTensor,
sample: torch.FloatTensor,
- eta: float = 0.0,
- use_clipped_model_output: bool = False,
- generator=None,
- variance_noise: Optional[torch.FloatTensor] = None,
return_dict: bool = True,
+ **kwargs,
) -> Union[RectifiedFlowSchedulerOutput, Tuple]:
- # pylint: disable=unused-argument
"""
Predict the sample from the previous timestep by reversing the SDE. This function propagates the diffusion
process from the learned model outputs (most often the predicted noise).
+ z_{t_1} = z_t - \Delta_t * v
+ The method finds the next timestep that is lower than the input timestep(s) and denoises the latents
+ to that level. The input timestep(s) are not required to be one of the predefined timesteps.
Args:
model_output (`torch.FloatTensor`):
- The direct output from learned diffusion model.
+ The direct output from learned diffusion model - the velocity,
timestep (`float`):
- The current discrete timestep in the diffusion chain.
+ The current discrete timestep in the diffusion chain (global or per-token).
sample (`torch.FloatTensor`):
- A current instance of a sample created by the diffusion process.
- eta (`float`):
- The weight of noise for added noise in diffusion step.
- use_clipped_model_output (`bool`, defaults to `False`):
- If `True`, computes "corrected" `model_output` from the clipped predicted original sample. Necessary
- because predicted original sample is clipped to [-1, 1] when `self.config.clip_sample` is `True`. If no
- clipping has happened, "corrected" `model_output` would coincide with the one provided as input and
- `use_clipped_model_output` has no effect.
- generator (`torch.Generator`, *optional*):
- A random number generator.
- variance_noise (`torch.FloatTensor`):
- Alternative to generating noise with `generator` by directly providing the noise for the variance
- itself. Useful for methods such as [`CycleDiffusion`].
+ A current latent tokens to be de-noised.
return_dict (`bool`, *optional*, defaults to `True`):
Whether or not to return a [`~schedulers.scheduling_ddim.DDIMSchedulerOutput`] or `tuple`.
@@ -322,21 +305,28 @@ class RectifiedFlowScheduler(SchedulerMixin, ConfigMixin, TimestepShifter):
raise ValueError(
"Number of inference steps is 'None', you need to run 'set_timesteps' after creating the scheduler"
)
+ t_eps = 1e-6 # Small epsilon to avoid numerical issues in timestep values
+ timesteps_padded = torch.cat(
+ [self.timesteps, torch.zeros(1, device=self.timesteps.device)]
+ )
+
+ # Find the next lower timestep(s) and compute the dt from the current timestep(s)
if timestep.ndim == 0:
- # Global timestep
- current_index = (self.timesteps - timestep).abs().argmin()
- dt = self.delta_timesteps.gather(0, current_index.unsqueeze(0))
+ # Global timestep case
+ lower_mask = timesteps_padded < timestep - t_eps
+ lower_timestep = timesteps_padded[lower_mask][0] # Closest lower timestep
+ dt = timestep - lower_timestep
+
else:
- # Timestep per token
+ # Per-token case
assert timestep.ndim == 2
- current_index = (
- (self.timesteps[:, None, None] - timestep[None]).abs().argmin(dim=0)
- )
- dt = self.delta_timesteps[current_index]
- # Special treatment for zero timestep tokens - set dt to 0 so prev_sample = sample
- dt = torch.where(timestep == 0.0, torch.zeros_like(dt), dt)[..., None]
+ lower_mask = timesteps_padded[:, None, None] < timestep[None] - t_eps
+ lower_timestep = lower_mask * timesteps_padded[:, None, None]
+ lower_timestep, _ = lower_timestep.max(dim=0)
+ dt = (timestep - lower_timestep)[..., None]
+ # Compute previous sample
prev_sample = sample - dt * model_output
if not return_dict:
diff --git a/ltx_video/utils/conditioning_method.py b/ltx_video/utils/conditioning_method.py
deleted file mode 100644
index 20befcb..0000000
--- a/ltx_video/utils/conditioning_method.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from enum import Enum
-
-
-class ConditioningMethod(Enum):
- UNCONDITIONAL = "unconditional"
- FIRST_FRAME = "first_frame"
diff --git a/tests/conftest.py b/tests/conftest.py
index 693d5d3..41d1921 100644
--- a/tests/conftest.py
+++ b/tests/conftest.py
@@ -12,10 +12,12 @@ def test_paths(request, pytestconfig):
"text_encoder_model_name_or_path"
)
input_image_path = pytestconfig.getoption("input_image_path")
+ input_video_path = pytestconfig.getoption("input_video_path")
config = {
"ckpt_path": ckpt_path,
"input_image_path": input_image_path,
+ "input_video_path": input_video_path,
"output_path": output_path,
"text_encoder_model_name_or_path": text_encoder_model_name_or_path,
}
@@ -49,6 +51,12 @@ def pytest_addoption(parser):
parser.addoption(
"--input_image_path",
action="store",
- default="tests/utils/car.png",
+ default="tests/utils/woman.jpeg",
help="Path to input image file.",
)
+ parser.addoption(
+ "--input_video_path",
+ action="store",
+ default="tests/utils/woman.mp4",
+ help="Path to input video file.",
+ )
diff --git a/tests/test_inference.py b/tests/test_inference.py
index 09b41b0..7a76695 100644
--- a/tests/test_inference.py
+++ b/tests/test_inference.py
@@ -11,15 +11,39 @@ def pytest_make_parametrize_id(config, val, argname):
@pytest.mark.parametrize(
- "do_txt_to_image", [True, False], ids=lambda x: f"do_txt_to_image={x}"
+ "conditioning_test_mode",
+ ["unconditional", "first-frame", "first-sequence", "sequence-and-frame"],
+ ids=lambda x: f"conditioning_test_mode={x}",
)
-def test_infer_runs_on_real_path(test_paths, do_txt_to_image):
- # Update the input_image_path in test_paths if there is an override
- if do_txt_to_image:
- test_paths["input_image_path"] = None
+def test_infer_runs_on_real_path(test_paths, conditioning_test_mode):
+ conditioning_params = {}
+ if conditioning_test_mode == "unconditional":
+ pass
+ elif conditioning_test_mode == "first-frame":
+ conditioning_params["conditioning_media_paths"] = [
+ test_paths["input_image_path"]
+ ]
+ conditioning_params["conditioning_start_frames"] = [0]
+ elif conditioning_test_mode == "first-sequence":
+ conditioning_params["conditioning_media_paths"] = [
+ test_paths["input_video_path"]
+ ]
+ conditioning_params["conditioning_start_frames"] = [0]
+ elif conditioning_test_mode == "sequence-and-frame":
+ conditioning_params["conditioning_media_paths"] = [
+ test_paths["input_video_path"],
+ test_paths["input_image_path"],
+ ]
+ conditioning_params["conditioning_start_frames"] = [16, 32]
+ else:
+ raise ValueError(f"Unknown conditioning mode: {conditioning_test_mode}")
+ test_paths = {
+ k: v
+ for k, v in test_paths.items()
+ if k not in ["input_image_path", "input_video_path"]
+ }
params = {
- "input_video_path": None,
"seed": 42,
"num_inference_steps": 1,
"num_images_per_prompt": 1,
@@ -29,19 +53,18 @@ def test_infer_runs_on_real_path(test_paths, do_txt_to_image):
"stg_mode": "attention_values",
"stg_skip_layers": "1,2,3",
"image_cond_noise_scale": 0.15,
- "height": 480,
- "width": 704,
+ "height": 288,
+ "width": 512,
"num_frames": 33,
"frame_rate": 25,
"precision": "bfloat16",
"decode_timestep": 0.05,
"decode_noise_scale": 0.025,
- "prompt": "A vintage yellow car drives along a wet mountain road, its rear wheels kicking up a light spray as it moves. The camera follows close behind, capturing the curvature of the road as it winds through rocky cliffs and lush green hills. The sunlight pierces through scattered clouds, reflecting off the car's rain-speckled surface, creating a dynamic, cinematic moment. "
- "The scene conveys a sense of freedom and exploration as the car disappears into the distance.",
+ "prompt": "A young woman with wavy, shoulder-length light brown hair stands outdoors on a foggy day. She wears a cozy pink turtleneck sweater, with a serene expression and piercing blue eyes. A wooden fence and a misty, grassy field fade into the background, evoking a calm and introspective mood.",
"negative_prompt": "worst quality, inconsistent motion, blurry, jittery, distorted",
"offload_to_cpu": False,
}
- infer(**{**test_paths, **params})
+ infer(**{**conditioning_params, **test_paths, **params})
def get_device():
@@ -71,7 +94,6 @@ def test_pipeline_on_batch(test_paths):
)
params = {
- "input_video_path": None,
"seed": 42,
"num_inference_steps": 2,
"num_images_per_prompt": 1,
@@ -82,8 +104,8 @@ def test_pipeline_on_batch(test_paths):
"skip_layer_strategy": SkipLayerStrategy.AttentionValues,
"skip_block_list": [1, 2],
"image_cond_noise_scale": 0.15,
- "height": 480,
- "width": 704,
+ "height": 288,
+ "width": 512,
"num_frames": 1,
"frame_rate": 25,
"decode_timestep": 0.05,
@@ -113,7 +135,6 @@ def test_pipeline_on_batch(test_paths):
images = pipeline(
prompt=prompts,
- device=device,
generator=generators,
**sample,
**params,
diff --git a/tests/test_scheduler.py b/tests/test_scheduler.py
new file mode 100644
index 0000000..1392438
--- /dev/null
+++ b/tests/test_scheduler.py
@@ -0,0 +1,96 @@
+import pytest
+import torch
+from ltx_video.schedulers.rf import RectifiedFlowScheduler
+
+
+def init_latents_and_scheduler(sampler):
+ batch_size, n_tokens, n_channels = 2, 4096, 128
+ num_steps = 20
+ scheduler = RectifiedFlowScheduler(
+ sampler=("Uniform" if sampler.lower() == "uniform" else "LinearQuadratic")
+ )
+ latents = torch.randn(size=(batch_size, n_tokens, n_channels))
+ scheduler.set_timesteps(num_inference_steps=num_steps, samples=latents)
+ return scheduler, latents
+
+
[email protected]("sampler", ["LinearQuadratic", "Uniform"])
+def test_scheduler_default_behavior(sampler):
+ """
+ Test the case of a single timestep from the list of timesteps.
+ """
+ scheduler, latents = init_latents_and_scheduler(sampler)
+
+ for i, t in enumerate(scheduler.timesteps):
+ noise_pred = torch.randn_like(latents)
+ denoised_latents = scheduler.step(
+ noise_pred,
+ t,
+ latents,
+ return_dict=False,
+ )[0]
+
+ # Verify the denoising
+ next_t = scheduler.timesteps[i + 1] if i < len(scheduler.timesteps) - 1 else 0.0
+ dt = t - next_t
+ expected_denoised_latents = latents - dt * noise_pred
+ assert torch.allclose(denoised_latents, expected_denoised_latents, atol=1e-06)
+
+
[email protected]("sampler", ["LinearQuadratic", "Uniform"])
+def test_scheduler_per_token(sampler):
+ """
+ Test the case of a timestep per token (from the list of timesteps).
+ Some tokens are set with timestep of 0.
+ """
+ scheduler, latents = init_latents_and_scheduler(sampler)
+ batch_size, n_tokens = latents.shape[:2]
+ for i, t in enumerate(scheduler.timesteps):
+ timesteps = torch.full((batch_size, n_tokens), t)
+ timesteps[:, 0] = 0.0
+ noise_pred = torch.randn_like(latents)
+ denoised_latents = scheduler.step(
+ noise_pred,
+ timesteps,
+ latents,
+ return_dict=False,
+ )[0]
+
+ # Verify the denoising
+ next_t = scheduler.timesteps[i + 1] if i < len(scheduler.timesteps) - 1 else 0.0
+ next_timesteps = torch.full((batch_size, n_tokens), next_t)
+ dt = timesteps - next_timesteps
+ expected_denoised_latents = latents - dt.unsqueeze(-1) * noise_pred
+ assert torch.allclose(
+ denoised_latents[:, 1:], expected_denoised_latents[:, 1:], atol=1e-06
+ )
+ assert torch.allclose(denoised_latents[:, 0], latents[:, 0], atol=1e-06)
+
+
[email protected]("sampler", ["LinearQuadratic", "Uniform"])
+def test_scheduler_t_not_in_list(sampler):
+ """
+ Test the case of a timestep per token NOT from the list of timesteps.
+ """
+ scheduler, latents = init_latents_and_scheduler(sampler)
+ batch_size, n_tokens = latents.shape[:2]
+ for i in range(len(scheduler.timesteps)):
+ if i < len(scheduler.timesteps) - 1:
+ t = (scheduler.timesteps[i] + scheduler.timesteps[i + 1]) / 2
+ else:
+ t = scheduler.timesteps[i] / 2
+ timesteps = torch.full((batch_size, n_tokens), t)
+ noise_pred = torch.randn_like(latents)
+ denoised_latents = scheduler.step(
+ noise_pred,
+ timesteps,
+ latents,
+ return_dict=False,
+ )[0]
+
+ # Verify the denoising
+ next_t = scheduler.timesteps[i + 1] if i < len(scheduler.timesteps) - 1 else 0.0
+ next_timesteps = torch.full((batch_size, n_tokens), next_t)
+ dt = timesteps - next_timesteps
+ expected_denoised_latents = latents - dt.unsqueeze(-1) * noise_pred
+ assert torch.allclose(denoised_latents, expected_denoised_latents, atol=1e-06)
diff --git a/tests/utils/.gitattributes b/tests/utils/.gitattributes
new file mode 100644
index 0000000..f8ff2b5
--- /dev/null
+++ b/tests/utils/.gitattributes
@@ -0,0 +1 @@
+*.mp4 filter=lfs diff=lfs merge=lfs -text
diff --git a/tests/utils/car.png b/tests/utils/car.png
deleted file mode 100644
index 5fb5b40..0000000
--- a/tests/utils/car.png
+++ /dev/null
@@ -1,3 +0,0 @@
-version https://git-lfs.github.com/spec/v1
-oid sha256:ef275020c1d26e926dfbb19c2ed5d7006650040e5cfc941250d920f90fcf33af
-size 798449
diff --git a/tests/utils/woman.jpeg b/tests/utils/woman.jpeg
new file mode 100644
index 0000000..7131d47
--- /dev/null
+++ b/tests/utils/woman.jpeg
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:acf0a6de9f7fb5551702fa8065ab423811e2011c36338f49636f84b033715df8
+size 33249
diff --git a/tests/utils/woman.mp4 b/tests/utils/woman.mp4
new file mode 100644
index 0000000..892de73
--- /dev/null
+++ b/tests/utils/woman.mp4
@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:833c2a0afe03ad60cb0c6361433eb11acf2ed58890f6b411cd306c01069e199f
+size 73228