Skip to content

Official repository for LTX-Video

Inference: Support guidance with STG / PAG

Yoav HaCohen5a8aeb334b68
diff --git a/inference.py b/inference.py
index 1a9f8b9..1f399f2 100644
--- a/inference.py
+++ b/inference.py
@@ -20,7 +20,7 @@ from ltx_video.models.transformers.transformer3d import Transformer3DModel
from ltx_video.pipelines.pipeline_ltx_video import LTXVideoPipeline
from ltx_video.schedulers.rf import RectifiedFlowScheduler
from ltx_video.utils.conditioning_method import ConditioningMethod
-
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
MAX_HEIGHT = 720
MAX_WIDTH = 1280
@@ -179,6 +179,30 @@ def main():
default=3,
help="Guidance scale for the pipeline",
)
+ parser.add_argument(
+ "--stg_scale",
+ type=float,
+ default=1,
+ help="Spatiotemporal guidance scale for the pipeline. 0 to disable STG.",
+ )
+ parser.add_argument(
+ "--stg_rescale",
+ type=float,
+ default=0.7,
+ help="Spatiotemporal guidance rescaling scale for the pipeline. 1 to disable rescale.",
+ )
+ parser.add_argument(
+ "--stg_mode",
+ type=str,
+ default="stg_a",
+ help="Spatiotemporal guidance mode for the pipeline. Can be either stg_a or stg_r.",
+ )
+ parser.add_argument(
+ "--stg_skip_layers",
+ type=str,
+ default="19",
+ help="Attention layers to skip for spatiotemporal guidance. Comma separated list of integers.",
+ )
parser.add_argument(
"--image_cond_noise_scale",
type=float,
@@ -308,6 +332,14 @@ def main():
transformer = transformer.to(torch.bfloat16)
text_encoder = text_encoder.to(torch.bfloat16)
+ # Set spatiotemporal guidance
+ skip_block_list = [int(x.strip()) for x in args.stg_skip_layers.split(",")]
+ skip_layer_strategy = (
+ SkipLayerStrategy.Attention
+ if args.stg_mode.lower() == "stg_a"
+ else SkipLayerStrategy.Residual
+ )
+
# Use submodels for the pipeline
submodel_dict = {
"transformer": transformer,
@@ -339,6 +371,11 @@ def main():
num_inference_steps=args.num_inference_steps,
num_images_per_prompt=args.num_images_per_prompt,
guidance_scale=args.guidance_scale,
+ skip_layer_strategy=skip_layer_strategy,
+ skip_block_list=skip_block_list,
+ stg_scale=args.stg_scale,
+ do_rescaling=args.stg_rescale != 1,
+ rescaling_scale=args.stg_rescale,
generator=generator,
output_type="pt",
callback_on_step_end=None,
diff --git a/ltx_video/models/transformers/attention.py b/ltx_video/models/transformers/attention.py
index e31e8b9..96ed251 100644
--- a/ltx_video/models/transformers/attention.py
+++ b/ltx_video/models/transformers/attention.py
@@ -20,6 +20,8 @@ from diffusers.utils.torch_utils import maybe_allow_in_graph
from einops import rearrange
from torch import nn
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
+
try:
from torch_xla.experimental.custom_kernel import flash_attention
except ImportError:
@@ -204,6 +206,8 @@ class BasicTransformerBlock(nn.Module):
cross_attention_kwargs: Dict[str, Any] = None,
class_labels: Optional[torch.LongTensor] = None,
added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,
+ skip_layer_mask: Optional[torch.Tensor] = None,
+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,
) -> torch.FloatTensor:
if cross_attention_kwargs is not None:
if cross_attention_kwargs.get("scale", None) is not None:
@@ -253,6 +257,8 @@ class BasicTransformerBlock(nn.Module):
encoder_hidden_states if self.only_cross_attention else None
),
attention_mask=attention_mask,
+ skip_layer_mask=skip_layer_mask,
+ skip_layer_strategy=skip_layer_strategy,
**cross_attention_kwargs,
)
if gate_msa is not None:
@@ -647,6 +653,8 @@ class Attention(nn.Module):
freqs_cis: Optional[Tuple[torch.FloatTensor, torch.FloatTensor]] = None,
encoder_hidden_states: Optional[torch.FloatTensor] = None,
attention_mask: Optional[torch.FloatTensor] = None,
+ skip_layer_mask: Optional[torch.Tensor] = None,
+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,
**cross_attention_kwargs,
) -> torch.Tensor:
r"""
@@ -659,6 +667,10 @@ class Attention(nn.Module):
The hidden states of the encoder.
attention_mask (`torch.Tensor`, *optional*):
The attention mask to use. If `None`, no mask is applied.
+ skip_layer_mask (`torch.Tensor`, *optional*):
+ The skip layer mask to use. If `None`, no mask is applied.
+ skip_layer_strategy (`SkipLayerStrategy`, *optional*, defaults to `None`):
+ Controls which layers to skip for spatiotemporal guidance.
**cross_attention_kwargs:
Additional keyword arguments to pass along to the cross attention.
@@ -690,6 +702,8 @@ class Attention(nn.Module):
freqs_cis=freqs_cis,
encoder_hidden_states=encoder_hidden_states,
attention_mask=attention_mask,
+ skip_layer_mask=skip_layer_mask,
+ skip_layer_strategy=skip_layer_strategy,
**cross_attention_kwargs,
)
@@ -924,6 +938,8 @@ class AttnProcessor2_0:
encoder_hidden_states: Optional[torch.FloatTensor] = None,
attention_mask: Optional[torch.FloatTensor] = None,
temb: Optional[torch.FloatTensor] = None,
+ skip_layer_mask: Optional[torch.FloatTensor] = None,
+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,
*args,
**kwargs,
) -> torch.FloatTensor:
@@ -949,6 +965,9 @@ class AttnProcessor2_0:
else encoder_hidden_states.shape
)
+ if skip_layer_mask is not None:
+ skip_layer_mask = skip_layer_mask.reshape(batch_size, 1, 1)
+
if (attention_mask is not None) and (not attn.use_tpu_flash_attention):
attention_mask = attn.prepare_attention_mask(
attention_mask, sequence_length, batch_size
@@ -1015,7 +1034,7 @@ class AttnProcessor2_0:
), f"ERROR: KEY SHAPE must be divisible by 128 (TPU limitation) [{key.shape[2]}]"
# run the TPU kernel implemented in jax with pallas
- hidden_states = flash_attention(
+ hidden_states_a = flash_attention(
q=query,
k=key,
v=value,
@@ -1024,7 +1043,7 @@ class AttnProcessor2_0:
sm_scale=attn.scale,
)
else:
- hidden_states = F.scaled_dot_product_attention(
+ hidden_states_a = F.scaled_dot_product_attention(
query,
key,
value,
@@ -1033,10 +1052,20 @@ class AttnProcessor2_0:
is_causal=False,
)
- hidden_states = hidden_states.transpose(1, 2).reshape(
+ hidden_states_a = hidden_states_a.transpose(1, 2).reshape(
batch_size, -1, attn.heads * head_dim
)
- hidden_states = hidden_states.to(query.dtype)
+ hidden_states_a = hidden_states_a.to(query.dtype)
+
+ if (
+ skip_layer_mask is not None
+ and skip_layer_strategy == SkipLayerStrategy.Attention
+ ):
+ hidden_states = hidden_states_a * skip_layer_mask + hidden_states * (
+ 1.0 - skip_layer_mask
+ )
+ else:
+ hidden_states = hidden_states_a
# linear proj
hidden_states = attn.to_out[0](hidden_states)
@@ -1047,9 +1076,20 @@ class AttnProcessor2_0:
hidden_states = hidden_states.transpose(-1, -2).reshape(
batch_size, channel, height, width
)
+ if (
+ skip_layer_mask is not None
+ and skip_layer_strategy == SkipLayerStrategy.Residual
+ ):
+ skip_layer_mask = skip_layer_mask.reshape(batch_size, 1, 1, 1)
if attn.residual_connection:
- hidden_states = hidden_states + residual
+ if (
+ skip_layer_mask is not None
+ and skip_layer_strategy == SkipLayerStrategy.Residual
+ ):
+ hidden_states = hidden_states + residual * skip_layer_mask
+ else:
+ hidden_states = hidden_states + residual
hidden_states = hidden_states / attn.rescale_output_factor
diff --git a/ltx_video/models/transformers/transformer3d.py b/ltx_video/models/transformers/transformer3d.py
index b53d119..afc98e7 100644
--- a/ltx_video/models/transformers/transformer3d.py
+++ b/ltx_video/models/transformers/transformer3d.py
@@ -20,6 +20,14 @@ from safetensors import safe_open
from ltx_video.models.transformers.attention import BasicTransformerBlock
from ltx_video.models.transformers.embeddings import get_3d_sincos_pos_embed
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
+
+from ltx_video.utils.diffusers_config_mapping import (
+ diffusers_and_ours_config_mapping,
+ make_hashable_key,
+ TRANSFORMER_KEYS_RENAME_DICT,
+)
+
logger = logging.get_logger(__name__)
@@ -172,6 +180,21 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
for block in self.transformer_blocks:
block.set_use_tpu_flash_attention()
+ def create_skip_layer_mask(
+ self,
+ skip_block_list: List[int],
+ batch_size: int,
+ num_conds: int,
+ ptb_index: int,
+ ):
+ num_layers = len(self.transformer_blocks)
+ mask = torch.ones(
+ (num_layers, batch_size * num_conds), device=self.device, dtype=self.dtype
+ )
+ for block_idx in skip_block_list:
+ mask[block_idx, ptb_index::num_conds] = 0
+ return mask
+
def initialize(self, embedding_std: float, mode: Literal["ltx_video", "legacy"]):
def _basic_init(module):
if isinstance(module, nn.Linear):
@@ -390,6 +413,8 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
cross_attention_kwargs: Dict[str, Any] = None,
attention_mask: Optional[torch.Tensor] = None,
encoder_attention_mask: Optional[torch.Tensor] = None,
+ skip_layer_mask: Optional[torch.Tensor] = None,
+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,
return_dict: bool = True,
):
"""
@@ -423,6 +448,11 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
If `ndim == 2`: will be interpreted as a mask, then converted into a bias consistent with the format
above. This bias will be added to the cross-attention scores.
+ skip_layer_mask ( `torch.Tensor`, *optional*):
+ A mask of shape `(num_layers, batch)` that indicates which layers to skip. `0` at position
+ `layer, batch_idx` indicates that the layer should be skipped for the corresponding batch index.
+ skip_layer_strategy ( `SkipLayerStrategy`, *optional*, defaults to `None`):
+ Controls which layers are skipped when calculating a perturbed latent for spatiotemporal guidance.
return_dict (`bool`, *optional*, defaults to `True`):
Whether or not to return a [`~models.unets.unet_2d_condition.UNet2DConditionOutput`] instead of a plain
tuple.
@@ -488,6 +518,11 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
batch_size, -1, embedded_timestep.shape[-1]
)
+ if skip_layer_mask is None:
+ skip_layer_mask = torch.ones(
+ len(self.transformer_blocks), batch_size, device=hidden_states.device
+ )
+
# 2. Blocks
if self.caption_projection is not None:
batch_size = hidden_states.shape[0]
@@ -496,7 +531,7 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
batch_size, -1, hidden_states.shape[-1]
)
- for block in self.transformer_blocks:
+ for block_idx, block in enumerate(self.transformer_blocks):
if self.training and self.gradient_checkpointing:
def create_custom_forward(module, return_dict=None):
@@ -521,6 +556,8 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
timestep,
cross_attention_kwargs,
class_labels,
+ skip_layer_mask[block_idx],
+ skip_layer_strategy,
**ckpt_kwargs,
)
else:
@@ -533,6 +570,8 @@ class Transformer3DModel(ModelMixin, ConfigMixin):
timestep=timestep,
cross_attention_kwargs=cross_attention_kwargs,
class_labels=class_labels,
+ skip_layer_mask=skip_layer_mask[block_idx],
+ skip_layer_strategy=skip_layer_strategy,
)
# 3. Output
diff --git a/ltx_video/pipelines/pipeline_ltx_video.py b/ltx_video/pipelines/pipeline_ltx_video.py
index c5b2c71..fa0a87b 100644
--- a/ltx_video/pipelines/pipeline_ltx_video.py
+++ b/ltx_video/pipelines/pipeline_ltx_video.py
@@ -37,6 +37,7 @@ from ltx_video.models.autoencoders.causal_video_autoencoder import (
)
from ltx_video.schedulers.rf import TimestepShifter
from ltx_video.utils.conditioning_method import ConditioningMethod
+from ltx_video.utils.skip_layer_strategy import SkipLayerStrategy
logger = logging.get_logger(__name__) # pylint: disable=invalid-name
@@ -773,6 +774,11 @@ class LTXVideoPipeline(DiffusionPipeline):
num_inference_steps: int = 20,
timesteps: List[int] = None,
guidance_scale: float = 4.5,
+ skip_layer_strategy: Optional[SkipLayerStrategy] = None,
+ skip_block_list: List[int] = None,
+ stg_scale: float = 1.0,
+ do_rescaling: bool = True,
+ rescaling_scale: float = 0.7,
num_images_per_prompt: Optional[int] = 1,
eta: float = 0.0,
generator: Optional[Union[torch.Generator, List[torch.Generator]]] = None,
@@ -894,6 +900,19 @@ class LTXVideoPipeline(DiffusionPipeline):
# of the Imagen paper: https://arxiv.org/pdf/2205.11487.pdf . `guidance_scale = 1`
# corresponds to doing no classifier free guidance.
do_classifier_free_guidance = guidance_scale > 1.0
+ do_spatio_temporal_guidance = stg_scale > 0.0
+
+ num_conds = 1
+ if do_classifier_free_guidance:
+ num_conds += 1
+ if do_spatio_temporal_guidance:
+ num_conds += 1
+
+ skip_layer_mask = None
+ if do_spatio_temporal_guidance:
+ skip_layer_mask = self.transformer.create_skip_layer_mask(
+ skip_block_list, batch_size, num_conds, 2
+ )
# 3. Encode input prompt
self.text_encoder = self.text_encoder.to(self._execution_device)
@@ -921,11 +940,24 @@ class LTXVideoPipeline(DiffusionPipeline):
self.transformer = self.transformer.to(self._execution_device)
+ prompt_embeds_batch = prompt_embeds
+ prompt_attention_mask_batch = prompt_attention_mask
if do_classifier_free_guidance:
- prompt_embeds = torch.cat([negative_prompt_embeds, prompt_embeds], dim=0)
- prompt_attention_mask = torch.cat(
+ prompt_embeds_batch = torch.cat(
+ [negative_prompt_embeds, prompt_embeds], dim=0
+ )
+ prompt_attention_mask_batch = torch.cat(
[negative_prompt_attention_mask, prompt_attention_mask], dim=0
)
+ if do_spatio_temporal_guidance:
+ prompt_embeds_batch = torch.cat([prompt_embeds_batch, prompt_embeds], dim=0)
+ prompt_attention_mask_batch = torch.cat(
+ [
+ prompt_attention_mask_batch,
+ prompt_attention_mask,
+ ],
+ dim=0,
+ )
# 3b. Encode and prepare conditioning data
self.video_scale_factor = self.video_scale_factor if is_video else 1
@@ -953,7 +985,7 @@ class LTXVideoPipeline(DiffusionPipeline):
batch_size=batch_size * num_images_per_prompt,
num_latent_channels=self.transformer.config.in_channels,
num_patches=num_latent_patches,
- dtype=prompt_embeds.dtype,
+ dtype=prompt_embeds_batch.dtype,
device=device,
generator=generator,
latents=init_latents,
@@ -963,8 +995,8 @@ class LTXVideoPipeline(DiffusionPipeline):
if conditioning_mask is not None and is_video:
assert num_images_per_prompt == 1
conditioning_mask = (
- torch.cat([conditioning_mask] * 2)
- if do_classifier_free_guidance
+ torch.cat([conditioning_mask] * num_conds)
+ if num_conds > 1
else conditioning_mask
)
@@ -999,8 +1031,9 @@ class LTXVideoPipeline(DiffusionPipeline):
orig_conditiong_mask,
generator,
)
+
latent_model_input = (
- torch.cat([latents] * 2) if do_classifier_free_guidance else latents
+ torch.cat([latents] * num_conds) if num_conds > 1 else latents
)
latent_model_input = self.scheduler.scale_model_input(
latent_model_input, t
@@ -1071,20 +1104,34 @@ class LTXVideoPipeline(DiffusionPipeline):
noise_pred = self.transformer(
latent_model_input.to(self.transformer.dtype),
indices_grid,
- encoder_hidden_states=prompt_embeds.to(self.transformer.dtype),
- encoder_attention_mask=prompt_attention_mask,
+ encoder_hidden_states=prompt_embeds_batch.to(
+ self.transformer.dtype
+ ),
+ encoder_attention_mask=prompt_attention_mask_batch,
timestep=current_timestep,
+ skip_layer_mask=skip_layer_mask,
+ skip_layer_strategy=skip_layer_strategy,
return_dict=False,
)[0]
# perform guidance
+ if do_spatio_temporal_guidance:
+ noise_pred_text_perturb = noise_pred[-1:]
if do_classifier_free_guidance:
- noise_pred_uncond, noise_pred_text = noise_pred.chunk(2)
+ noise_pred_uncond, noise_pred_text = noise_pred[:2].chunk(2)
noise_pred = noise_pred_uncond + guidance_scale * (
noise_pred_text - noise_pred_uncond
)
- current_timestep, _ = current_timestep.chunk(2)
+ if do_spatio_temporal_guidance:
+ noise_pred = noise_pred + stg_scale * (
+ noise_pred_text - noise_pred_text_perturb
+ )
+ if do_rescaling:
+ factor = noise_pred_text.std() / noise_pred.std()
+ factor = rescaling_scale * factor + (1 - rescaling_scale)
+ noise_pred = noise_pred * factor
+ current_timestep = current_timestep[:1]
# learned sigma
if (
self.transformer.config.out_channels // 2
diff --git a/ltx_video/utils/skip_layer_strategy.py b/ltx_video/utils/skip_layer_strategy.py
new file mode 100644
index 0000000..fd4c133
--- /dev/null
+++ b/ltx_video/utils/skip_layer_strategy.py
@@ -0,0 +1,6 @@
+from enum import Enum, auto
+
+
+class SkipLayerStrategy(Enum):
+ Attention = auto()
+ Residual = auto()