The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Reduce LTX2 VRAM use by more efficient timestep embed handling (#11829)
Jukka Seppänenfd5c0755af18
diff --git a/comfy/ldm/lightricks/av_model.py b/comfy/ldm/lightricks/av_model.pyindex 759535501..c12ace241 100644--- a/comfy/ldm/lightricks/av_model.py+++ b/comfy/ldm/lightricks/av_model.py@@ -11,6 +11,69 @@ from comfy.ldm.lightricks.model import (from comfy.ldm.lightricks.symmetric_patchifier import AudioPatchifierimport comfy.ldm.common_dit+class CompressedTimestep:+ """Store video timestep embeddings in compressed form using per-frame indexing."""+ __slots__ = ('data', 'batch_size', 'num_frames', 'patches_per_frame', 'feature_dim')++ def __init__(self, tensor: torch.Tensor, patches_per_frame: int):+ """+ tensor: [batch_size, num_tokens, feature_dim] tensor where num_tokens = num_frames * patches_per_frame+ patches_per_frame: Number of spatial patches per frame (height * width in latent space)+ """+ self.batch_size, num_tokens, self.feature_dim = tensor.shape++ # Check if compression is valid (num_tokens must be divisible by patches_per_frame)+ if num_tokens % patches_per_frame == 0 and num_tokens >= patches_per_frame:+ self.patches_per_frame = patches_per_frame+ self.num_frames = num_tokens // patches_per_frame++ # Reshape to [batch, frames, patches_per_frame, feature_dim] and store one value per frame+ # All patches in a frame are identical, so we only keep the first one+ reshaped = tensor.view(self.batch_size, self.num_frames, patches_per_frame, self.feature_dim)+ self.data = reshaped[:, :, 0, :].contiguous() # [batch, frames, feature_dim]+ else:+ # Not divisible or too small - store directly without compression+ self.patches_per_frame = 1+ self.num_frames = num_tokens+ self.data = tensor++ def expand(self):+ """Expand back to original tensor."""+ if self.patches_per_frame == 1:+ return self.data++ # [batch, frames, feature_dim] -> [batch, frames, patches_per_frame, feature_dim] -> [batch, tokens, feature_dim]+ expanded = self.data.unsqueeze(2).expand(self.batch_size, self.num_frames, self.patches_per_frame, self.feature_dim)+ return expanded.reshape(self.batch_size, -1, self.feature_dim)++ def expand_for_computation(self, scale_shift_table: torch.Tensor, batch_size: int, indices: slice = slice(None, None)):+ """Compute ada values on compressed per-frame data, then expand spatially."""+ num_ada_params = scale_shift_table.shape[0]++ # No compression - compute directly+ if self.patches_per_frame == 1:+ num_tokens = self.data.shape[1]+ dim_per_param = self.feature_dim // num_ada_params+ reshaped = self.data.reshape(batch_size, num_tokens, num_ada_params, dim_per_param)[:, :, indices, :]+ table_values = scale_shift_table[indices].unsqueeze(0).unsqueeze(0).to(device=self.data.device, dtype=self.data.dtype)+ ada_values = (table_values + reshaped).unbind(dim=2)+ return ada_values++ # Compressed: compute on per-frame data then expand spatially+ # Reshape: [batch, frames, feature_dim] -> [batch, frames, num_ada_params, dim_per_param]+ frame_reshaped = self.data.reshape(batch_size, self.num_frames, num_ada_params, -1)[:, :, indices, :]+ table_values = scale_shift_table[indices].unsqueeze(0).unsqueeze(0).to(+ device=self.data.device, dtype=self.data.dtype+ )+ frame_ada = (table_values + frame_reshaped).unbind(dim=2)++ # Expand each ada parameter spatially: [batch, frames, dim] -> [batch, frames, patches, dim] -> [batch, tokens, dim]+ return tuple(+ frame_val.unsqueeze(2).expand(batch_size, self.num_frames, self.patches_per_frame, -1)+ .reshape(batch_size, -1, frame_val.shape[-1])+ for frame_val in frame_ada+ )+class BasicAVTransformerBlock(nn.Module):def __init__(self,@@ -119,6 +182,9 @@ class BasicAVTransformerBlock(nn.Module):def get_ada_values(self, scale_shift_table: torch.Tensor, batch_size: int, timestep: torch.Tensor, indices: slice = slice(None, None)):+ if isinstance(timestep, CompressedTimestep):+ return timestep.expand_for_computation(scale_shift_table, batch_size, indices)+num_ada_params = scale_shift_table.shape[0]ada_values = (@@ -146,10 +212,7 @@ class BasicAVTransformerBlock(nn.Module):gate_timestep,)- scale_shift_chunks = [t.squeeze(2) for t in scale_shift_ada_values]- gate_ada_values = [t.squeeze(2) for t in gate_ada_values]-- return (*scale_shift_chunks, *gate_ada_values)+ return (*scale_shift_ada_values, *gate_ada_values)def forward(self,@@ -543,72 +606,80 @@ class LTXAVModel(LTXVModel):if grid_mask is not None:timestep = timestep[:, grid_mask]- timestep = timestep * self.timestep_scale_multiplier+ timestep_scaled = timestep * self.timestep_scale_multiplier+v_timestep, v_embedded_timestep = self.adaln_single(- timestep.flatten(),+ timestep_scaled.flatten(),{"resolution": None, "aspect_ratio": None},batch_size=batch_size,hidden_dtype=hidden_dtype,)- # Second dimension is 1 or number of tokens (if timestep_per_token)- v_timestep = v_timestep.view(batch_size, -1, v_timestep.shape[-1])- v_embedded_timestep = v_embedded_timestep.view(- batch_size, -1, v_embedded_timestep.shape[-1]- )+ # Calculate patches_per_frame from orig_shape: [batch, channels, frames, height, width]+ # Video tokens are arranged as (frames * height * width), so patches_per_frame = height * width+ orig_shape = kwargs.get("orig_shape")+ v_patches_per_frame = None+ if orig_shape is not None and len(orig_shape) == 5:+ # orig_shape[3] = height, orig_shape[4] = width (in latent space)+ v_patches_per_frame = orig_shape[3] * orig_shape[4]++ # Reshape to [batch_size, num_tokens, dim] and compress for storage+ v_timestep = CompressedTimestep(v_timestep.view(batch_size, -1, v_timestep.shape[-1]), v_patches_per_frame)+ v_embedded_timestep = CompressedTimestep(v_embedded_timestep.view(batch_size, -1, v_embedded_timestep.shape[-1]), v_patches_per_frame)# Prepare audio timestepa_timestep = kwargs.get("a_timestep")if a_timestep is not None:- a_timestep = a_timestep * self.timestep_scale_multiplier+ a_timestep_scaled = a_timestep * self.timestep_scale_multiplier+ a_timestep_flat = a_timestep_scaled.flatten()+ timestep_flat = timestep_scaled.flatten()av_ca_factor = self.av_ca_timestep_scale_multiplier / self.timestep_scale_multiplier+ # Cross-attention timesteps - compress these tooav_ca_audio_scale_shift_timestep, _ = self.av_ca_audio_scale_shift_adaln_single(- a_timestep.flatten(),+ a_timestep_flat,{"resolution": None, "aspect_ratio": None},batch_size=batch_size,hidden_dtype=hidden_dtype,)av_ca_video_scale_shift_timestep, _ = self.av_ca_video_scale_shift_adaln_single(- timestep.flatten(),+ timestep_flat,{"resolution": None, "aspect_ratio": None},batch_size=batch_size,hidden_dtype=hidden_dtype,)av_ca_a2v_gate_noise_timestep, _ = self.av_ca_a2v_gate_adaln_single(- timestep.flatten() * av_ca_factor,+ timestep_flat * av_ca_factor,{"resolution": None, "aspect_ratio": None},batch_size=batch_size,hidden_dtype=hidden_dtype,)av_ca_v2a_gate_noise_timestep, _ = self.av_ca_v2a_gate_adaln_single(- a_timestep.flatten() * av_ca_factor,+ a_timestep_flat * av_ca_factor,{"resolution": None, "aspect_ratio": None},batch_size=batch_size,hidden_dtype=hidden_dtype,)+ # Compress cross-attention timesteps (only video side, audio is too small to benefit)+ cross_av_timestep_ss = [+ av_ca_audio_scale_shift_timestep.view(batch_size, -1, av_ca_audio_scale_shift_timestep.shape[-1]),+ CompressedTimestep(av_ca_video_scale_shift_timestep.view(batch_size, -1, av_ca_video_scale_shift_timestep.shape[-1]), v_patches_per_frame), # video - compressed+ CompressedTimestep(av_ca_a2v_gate_noise_timestep.view(batch_size, -1, av_ca_a2v_gate_noise_timestep.shape[-1]), v_patches_per_frame), # video - compressed+ av_ca_v2a_gate_noise_timestep.view(batch_size, -1, av_ca_v2a_gate_noise_timestep.shape[-1]),+ ]+a_timestep, a_embedded_timestep = self.audio_adaln_single(- a_timestep.flatten(),+ a_timestep_flat,{"resolution": None, "aspect_ratio": None},batch_size=batch_size,hidden_dtype=hidden_dtype,)+ # Audio timestepsa_timestep = a_timestep.view(batch_size, -1, a_timestep.shape[-1])- a_embedded_timestep = a_embedded_timestep.view(- batch_size, -1, a_embedded_timestep.shape[-1]- )- cross_av_timestep_ss = [- av_ca_audio_scale_shift_timestep,- av_ca_video_scale_shift_timestep,- av_ca_a2v_gate_noise_timestep,- av_ca_v2a_gate_noise_timestep,- ]- cross_av_timestep_ss = list(- [t.view(batch_size, -1, t.shape[-1]) for t in cross_av_timestep_ss]- )+ a_embedded_timestep = a_embedded_timestep.view(batch_size, -1, a_embedded_timestep.shape[-1])else:- a_timestep = timestep+ a_timestep = timestep_scaleda_embedded_timestep = kwargs.get("embedded_timestep")cross_av_timestep_ss = []@@ -767,6 +838,11 @@ class LTXAVModel(LTXVModel):ax = x[1]v_embedded_timestep = embedded_timestep[0]a_embedded_timestep = embedded_timestep[1]++ # Expand compressed video timestep if needed+ if isinstance(v_embedded_timestep, CompressedTimestep):+ v_embedded_timestep = v_embedded_timestep.expand()+vx = super()._process_output(vx, v_embedded_timestep, keyframe_idxs, **kwargs)# Process audio output