Official repository for LTX-Video
Python diffusion-modelsditimage-to-videoimage-to-video-generationtext-to-videotext-to-video-generation
Inference: Support guidance with STG / PAG
Yoav HaCohen5a8aeb334b68
diff --git a/inference.py b/inference.pyindex 1a9f8b9..1f399f2 100644--- a/inference.py+++ b/inference.py@@ -20,7 +20,7 @@ from ltx_video.models.transformers.transformer3d import Transformer3DModelfrom ltx_video.pipelines.pipeline_ltx_video import LTXVideoPipelinefrom ltx_video.schedulers.rf import RectifiedFlowSchedulerfrom ltx_video.utils.conditioning_method import ConditioningMethod-+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategyMAX_HEIGHT = 720MAX_WIDTH = 1280@@ -179,6 +179,30 @@ def main():default=3,help="Guidance scale for the pipeline",)+ parser.add_argument(+ "--stg_scale",+ type=float,+ default=1,+ help="Spatiotemporal guidance scale for the pipeline. 0 to disable STG.",+ )+ parser.add_argument(+ "--stg_rescale",+ type=float,+ default=0.7,+ help="Spatiotemporal guidance rescaling scale for the pipeline. 1 to disable rescale.",+ )+ parser.add_argument(+ "--stg_mode",+ type=str,+ default="stg_a",+ help="Spatiotemporal guidance mode for the pipeline. Can be either stg_a or stg_r.",+ )+ parser.add_argument(+ "--stg_skip_layers",+ type=str,+ default="19",+ help="Attention layers to skip for spatiotemporal guidance. Comma separated list of integers.",+ )parser.add_argument("--image_cond_noise_scale",type=float,@@ -308,6 +332,14 @@ def main():transformer = transformer.to(torch.bfloat16)text_encoder = text_encoder.to(torch.bfloat16)+ # Set spatiotemporal guidance+ skip_block_list = [int(x.strip()) for x in args.stg_skip_layers.split(",")]+ skip_layer_strategy = (+ SkipLayerStrategy.Attention+ if args.stg_mode.lower() == "stg_a"+ else SkipLayerStrategy.Residual+ )+# Use submodels for the pipelinesubmodel_dict = {"transformer": transformer,@@ -339,6 +371,11 @@ def main():num_inference_steps=args.num_inference_steps,num_images_per_prompt=args.num_images_per_prompt,guidance_scale=args.guidance_scale,+ skip_layer_strategy=skip_layer_strategy,+ skip_block_list=skip_block_list,+ stg_scale=args.stg_scale,+ do_rescaling=args.stg_rescale != 1,+ rescaling_scale=args.stg_rescale,generator=generator,output_type="pt",callback_on_step_end=None,diff --git a/ltx_video/models/transformers/attention.py b/ltx_video/models/transformers/attention.pyindex e31e8b9..96ed251 100644--- a/ltx_video/models/transformers/attention.py+++ b/ltx_video/models/transformers/attention.py@@ -20,6 +20,8 @@ from diffusers.utils.torch_utils import maybe_allow_in_graphfrom einops import rearrangefrom torch import nn+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy+try:from torch_xla.experimental.custom_kernel import flash_attentionexcept ImportError:@@ -204,6 +206,8 @@ class BasicTransformerBlock(nn.Module):cross_attention_kwargs: Dict[str, Any] = None,class_labels: Optional[torch.LongTensor] = None,added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,+ skip_layer_mask: Optional[torch.Tensor] = None,+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,) -> torch.FloatTensor:if cross_attention_kwargs is not None:if cross_attention_kwargs.get("scale", None) is not None:@@ -253,6 +257,8 @@ class BasicTransformerBlock(nn.Module):encoder_hidden_states if self.only_cross_attention else None),attention_mask=attention_mask,+ skip_layer_mask=skip_layer_mask,+ skip_layer_strategy=skip_layer_strategy,**cross_attention_kwargs,)if gate_msa is not None:@@ -647,6 +653,8 @@ class Attention(nn.Module):freqs_cis: Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] = None,encoder_hidden_states: Optional[torch.FloatTensor] = None,attention_mask: Optional[torch.FloatTensor] = None,+ skip_layer_mask: Optional[torch.Tensor] = None,+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,**cross_attention_kwargs,) -> torch.Tensor:r"""@@ -659,6 +667,10 @@ class Attention(nn.Module):The hidden states of the encoder.attention_mask (`torch.Tensor`, *optional*):The attention mask to use. If `None`, no mask is applied.+ skip_layer_mask (`torch.Tensor`, *optional*):+ The skip layer mask to use. If `None`, no mask is applied.+ skip_layer_strategy (`SkipLayerStrategy`, *optional*, defaults to `None`):+ Controls which layers to skip for spatiotemporal guidance.**cross_attention_kwargs:Additional keyword arguments to pass along to the cross attention.@@ -690,6 +702,8 @@ class Attention(nn.Module):freqs_cis=freqs_cis,encoder_hidden_states=encoder_hidden_states,attention_mask=attention_mask,+ skip_layer_mask=skip_layer_mask,+ skip_layer_strategy=skip_layer_strategy,**cross_attention_kwargs,)@@ -924,6 +938,8 @@ class AttnProcessor2_0:encoder_hidden_states: Optional[torch.FloatTensor] = None,attention_mask: Optional[torch.FloatTensor] = None,temb: Optional[torch.FloatTensor] = None,+ skip_layer_mask: Optional[torch.FloatTensor] = None,+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,*args,**kwargs,) -> torch.FloatTensor:@@ -949,6 +965,9 @@ class AttnProcessor2_0:else encoder_hidden_states.shape)+ if skip_layer_mask is not None:+ skip_layer_mask = skip_layer_mask.reshape(batch_size, 1, 1)+if (attention_mask is not None) and (not attn.use_tpu_flash_attention):attention_mask = attn.prepare_attention_mask(attention_mask, sequence_length, batch_size@@ -1015,7 +1034,7 @@ class AttnProcessor2_0:), f"ERROR: KEY SHAPE must be divisible by 128 (TPU limitation) [{key.shape[2]}]"# run the TPU kernel implemented in jax with pallas- hidden_states = flash_attention(+ hidden_states_a = flash_attention(q=query,k=key,v=value,@@ -1024,7 +1043,7 @@ class AttnProcessor2_0:sm_scale=attn.scale,)else:- hidden_states = F.scaled_dot_product_attention(+ hidden_states_a = F.scaled_dot_product_attention(query,key,value,@@ -1033,10 +1052,20 @@ class AttnProcessor2_0:is_causal=False,)- hidden_states = hidden_states.transpose(1, 2).reshape(+ hidden_states_a = hidden_states_a.transpose(1, 2).reshape(batch_size, -1, attn.heads * head_dim)- hidden_states = hidden_states.to(query.dtype)+ hidden_states_a = hidden_states_a.to(query.dtype)++ if (+ skip_layer_mask is not None+ and skip_layer_strategy == SkipLayerStrategy.Attention+ ):+ hidden_states = hidden_states_a * skip_layer_mask + hidden_states * (+ 1.0 - skip_layer_mask+ )+ else:+ hidden_states = hidden_states_a# linear projhidden_states = attn.to_out[0](hidden_states)@@ -1047,9 +1076,20 @@ class AttnProcessor2_0:hidden_states = hidden_states.transpose(-1, -2).reshape(batch_size, channel, height, width)+ if (+ skip_layer_mask is not None+ and skip_layer_strategy == SkipLayerStrategy.Residual+ ):+ skip_layer_mask = skip_layer_mask.reshape(batch_size, 1, 1, 1)if attn.residual_connection:- hidden_states = hidden_states + residual+ if (+ skip_layer_mask is not None+ and skip_layer_strategy == SkipLayerStrategy.Residual+ ):+ hidden_states = hidden_states + residual * skip_layer_mask+ else:+ hidden_states = hidden_states + residualhidden_states = hidden_states / attn.rescale_output_factordiff --git a/ltx_video/models/transformers/transformer3d.py b/ltx_video/models/transformers/transformer3d.pyindex b53d119..afc98e7 100644--- a/ltx_video/models/transformers/transformer3d.py+++ b/ltx_video/models/transformers/transformer3d.py@@ -20,6 +20,14 @@ from safetensors import safe_openfrom ltx_video.models.transformers.attention import BasicTransformerBlockfrom ltx_video.models.transformers.embeddings import get_3d_sincos_pos_embed+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy++from ltx_video.utils.diffusers_config_mapping import (+ diffusers_and_ours_config_mapping,+ make_hashable_key,+ TRANSFORMER_KEYS_RENAME_DICT,+)+logger = logging.get_logger(__name__)@@ -172,6 +180,21 @@ class Transformer3DModel(ModelMixin, ConfigMixin):for block in self.transformer_blocks:block.set_use_tpu_flash_attention()+ def create_skip_layer_mask(+ self,+ skip_block_list: List[int],+ batch_size: int,+ num_conds: int,+ ptb_index: int,+ ):+ num_layers = len(self.transformer_blocks)+ mask = torch.ones(+ (num_layers, batch_size * num_conds), device=self.device, dtype=self.dtype+ )+ for block_idx in skip_block_list:+ mask[block_idx, ptb_index::num_conds] = 0+ return mask+def initialize(self, embedding_std: float, mode: Literal["ltx_video", "legacy"]):def _basic_init(module):if isinstance(module, nn.Linear):@@ -390,6 +413,8 @@ class Transformer3DModel(ModelMixin, ConfigMixin):cross_attention_kwargs: Dict[str, Any] = None,attention_mask: Optional[torch.Tensor] = None,encoder_attention_mask: Optional[torch.Tensor] = None,+ skip_layer_mask: Optional[torch.Tensor] = None,+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,return_dict: bool = True,):"""@@ -423,6 +448,11 @@ class Transformer3DModel(ModelMixin, ConfigMixin):If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the formatabove. This bias will be added to the cross-attention scores.+ skip_layer_mask ( `torch.Tensor`, *optional*):+ A mask of shape `(num_layers, batch)` that indicates which layers to skip. `0` at position+ `layer, batch_idx` indicates that the layer should be skipped for the corresponding batch index.+ skip_layer_strategy ( `SkipLayerStrategy`, *optional*, defaults to `None`):+ Controls which layers are skipped when calculating a perturbed latent for spatiotemporal guidance.return_dict (`bool`, *optional*, defaults to `True`):Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plaintuple.@@ -488,6 +518,11 @@ class Transformer3DModel(ModelMixin, ConfigMixin):batch_size, -1, embedded_timestep.shape[-1])+ if skip_layer_mask is None:+ skip_layer_mask = torch.ones(+ len(self.transformer_blocks), batch_size, device=hidden_states.device+ )+# 2. Blocksif self.caption_projection is not None:batch_size = hidden_states.shape[0]@@ -496,7 +531,7 @@ class Transformer3DModel(ModelMixin, ConfigMixin):batch_size, -1, hidden_states.shape[-1])- for block in self.transformer_blocks:+ for block_idx, block in enumerate(self.transformer_blocks):if self.training and self.gradient_checkpointing:def create_custom_forward(module, return_dict=None):@@ -521,6 +556,8 @@ class Transformer3DModel(ModelMixin, ConfigMixin):timestep,cross_attention_kwargs,class_labels,+ skip_layer_mask[block_idx],+ skip_layer_strategy,**ckpt_kwargs,)else:@@ -533,6 +570,8 @@ class Transformer3DModel(ModelMixin, ConfigMixin):timestep=timestep,cross_attention_kwargs=cross_attention_kwargs,class_labels=class_labels,+ skip_layer_mask=skip_layer_mask[block_idx],+ skip_layer_strategy=skip_layer_strategy,)# 3. Outputdiff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.pyindex c5b2c71..fa0a87b 100644--- a/ltx_video/pipelines/pipeline_ltx_video.py+++ b/ltx_video/pipelines/pipeline_ltx_video.py@@ -37,6 +37,7 @@ from ltx_video.models.autoencoders.causal_video_autoencoder import ()from ltx_video.schedulers.rf import TimestepShifterfrom ltx_video.utils.conditioning_method import ConditioningMethod+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategylogger = logging.get_logger(__name__) # pylint: disable=invalid-name@@ -773,6 +774,11 @@ class LTXVideoPipeline(DiffusionPipeline):num_inference_steps: int = 20,timesteps: List[int] = None,guidance_scale: float = 4.5,+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,+ skip_block_list: List[int] = None,+ stg_scale: float = 1.0,+ do_rescaling: bool = True,+ rescaling_scale: float = 0.7,num_images_per_prompt: Optional[int] = 1,eta: float = 0.0,generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,@@ -894,6 +900,19 @@ class LTXVideoPipeline(DiffusionPipeline):# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`# corresponds to doing no classifier free guidance.do_classifier_free_guidance = guidance_scale > 1.0+ do_spatio_temporal_guidance = stg_scale > 0.0++ num_conds = 1+ if do_classifier_free_guidance:+ num_conds += 1+ if do_spatio_temporal_guidance:+ num_conds += 1++ skip_layer_mask = None+ if do_spatio_temporal_guidance:+ skip_layer_mask = self.transformer.create_skip_layer_mask(+ skip_block_list, batch_size, num_conds, 2+ )# 3. Encode input promptself.text_encoder = self.text_encoder.to(self._execution_device)@@ -921,11 +940,24 @@ class LTXVideoPipeline(DiffusionPipeline):self.transformer = self.transformer.to(self._execution_device)+ prompt_embeds_batch = prompt_embeds+ prompt_attention_mask_batch = prompt_attention_maskif do_classifier_free_guidance:- prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)- prompt_attention_mask = torch.cat(+ prompt_embeds_batch = torch.cat(+ [negative_prompt_embeds, prompt_embeds], dim=0+ )+ prompt_attention_mask_batch = torch.cat([negative_prompt_attention_mask, prompt_attention_mask], dim=0)+ if do_spatio_temporal_guidance:+ prompt_embeds_batch = torch.cat([prompt_embeds_batch, prompt_embeds], dim=0)+ prompt_attention_mask_batch = torch.cat(+ [+ prompt_attention_mask_batch,+ prompt_attention_mask,+ ],+ dim=0,+ )# 3b. Encode and prepare conditioning dataself.video_scale_factor = self.video_scale_factor if is_video else 1@@ -953,7 +985,7 @@ class LTXVideoPipeline(DiffusionPipeline):batch_size=batch_size * num_images_per_prompt,num_latent_channels=self.transformer.config.in_channels,num_patches=num_latent_patches,- dtype=prompt_embeds.dtype,+ dtype=prompt_embeds_batch.dtype,device=device,generator=generator,latents=init_latents,@@ -963,8 +995,8 @@ class LTXVideoPipeline(DiffusionPipeline):if conditioning_mask is not None and is_video:assert num_images_per_prompt == 1conditioning_mask = (- torch.cat([conditioning_mask] * 2)- if do_classifier_free_guidance+ torch.cat([conditioning_mask] * num_conds)+ if num_conds > 1else conditioning_mask)@@ -999,8 +1031,9 @@ class LTXVideoPipeline(DiffusionPipeline):orig_conditiong_mask,generator,)+latent_model_input = (- torch.cat([latents] * 2) if do_classifier_free_guidance else latents+ torch.cat([latents] * num_conds) if num_conds > 1 else latents)latent_model_input = self.scheduler.scale_model_input(latent_model_input, t@@ -1071,20 +1104,34 @@ class LTXVideoPipeline(DiffusionPipeline):noise_pred = self.transformer(latent_model_input.to(self.transformer.dtype),indices_grid,- encoder_hidden_states=prompt_embeds.to(self.transformer.dtype),- encoder_attention_mask=prompt_attention_mask,+ encoder_hidden_states=prompt_embeds_batch.to(+ self.transformer.dtype+ ),+ encoder_attention_mask=prompt_attention_mask_batch,timestep=current_timestep,+ skip_layer_mask=skip_layer_mask,+ skip_layer_strategy=skip_layer_strategy,return_dict=False,)[0]# perform guidance+ if do_spatio_temporal_guidance:+ noise_pred_text_perturb = noise_pred[-1:]if do_classifier_free_guidance:- noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)+ noise_pred_uncond, noise_pred_text = noise_pred[:2].chunk(2)noise_pred = noise_pred_uncond + guidance_scale * (noise_pred_text - noise_pred_uncond)- current_timestep, _ = current_timestep.chunk(2)+ if do_spatio_temporal_guidance:+ noise_pred = noise_pred + stg_scale * (+ noise_pred_text - noise_pred_text_perturb+ )+ if do_rescaling:+ factor = noise_pred_text.std() / noise_pred.std()+ factor = rescaling_scale * factor + (1 - rescaling_scale)+ noise_pred = noise_pred * factor+ current_timestep = current_timestep[:1]# learned sigmaif (self.transformer.config.out_channels // 2diff --git a/ltx_video/utils/skip_layer_strategy.py b/ltx_video/utils/skip_layer_strategy.pynew file mode 100644index 0000000..fd4c133--- /dev/null+++ b/ltx_video/utils/skip_layer_strategy.py@@ -0,0 +1,6 @@+from enum import Enum, auto+++class SkipLayerStrategy(Enum):+ Attention = auto()+ Residual = auto()