The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support the LTXV 2 model. (#11632)
comfyanonymousf2b002372b71
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex f1ca0151e..9bbe30b53 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -407,6 +407,9 @@ class LTXV(LatentFormat):self.latent_rgb_factors_bias = [-0.0571, -0.1657, -0.2512]+class LTXAV(LTXV):+ pass+class HunyuanVideo(LatentFormat):latent_channels = 16latent_dimensions = 3diff --git a/comfy/ldm/lightricks/av_model.py b/comfy/ldm/lightricks/av_model.pynew file mode 100644index 000000000..759535501--- /dev/null+++ b/comfy/ldm/lightricks/av_model.py@@ -0,0 +1,837 @@+from typing import Tuple+import torch+import torch.nn as nn+from comfy.ldm.lightricks.model import (+ CrossAttention,+ FeedForward,+ AdaLayerNormSingle,+ PixArtAlphaTextProjection,+ LTXVModel,+)+from comfy.ldm.lightricks.symmetric_patchifier import AudioPatchifier+import comfy.ldm.common_dit++class BasicAVTransformerBlock(nn.Module):+ def __init__(+ self,+ v_dim,+ a_dim,+ v_heads,+ a_heads,+ vd_head,+ ad_head,+ v_context_dim=None,+ a_context_dim=None,+ attn_precision=None,+ dtype=None,+ device=None,+ operations=None,+ ):+ super().__init__()++ self.attn_precision = attn_precision++ self.attn1 = CrossAttention(+ query_dim=v_dim,+ heads=v_heads,+ dim_head=vd_head,+ context_dim=None,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )+ self.audio_attn1 = CrossAttention(+ query_dim=a_dim,+ heads=a_heads,+ dim_head=ad_head,+ context_dim=None,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ self.attn2 = CrossAttention(+ query_dim=v_dim,+ context_dim=v_context_dim,+ heads=v_heads,+ dim_head=vd_head,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )+ self.audio_attn2 = CrossAttention(+ query_dim=a_dim,+ context_dim=a_context_dim,+ heads=a_heads,+ dim_head=ad_head,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ # Q: Video, K,V: Audio+ self.audio_to_video_attn = CrossAttention(+ query_dim=v_dim,+ context_dim=a_dim,+ heads=a_heads,+ dim_head=ad_head,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ # Q: Audio, K,V: Video+ self.video_to_audio_attn = CrossAttention(+ query_dim=a_dim,+ context_dim=v_dim,+ heads=a_heads,+ dim_head=ad_head,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ self.ff = FeedForward(+ v_dim, dim_out=v_dim, glu=True, dtype=dtype, device=device, operations=operations+ )+ self.audio_ff = FeedForward(+ a_dim, dim_out=a_dim, glu=True, dtype=dtype, device=device, operations=operations+ )++ self.scale_shift_table = nn.Parameter(torch.empty(6, v_dim, device=device, dtype=dtype))+ self.audio_scale_shift_table = nn.Parameter(+ torch.empty(6, a_dim, device=device, dtype=dtype)+ )++ self.scale_shift_table_a2v_ca_audio = nn.Parameter(+ torch.empty(5, a_dim, device=device, dtype=dtype)+ )+ self.scale_shift_table_a2v_ca_video = nn.Parameter(+ torch.empty(5, v_dim, device=device, dtype=dtype)+ )++ def get_ada_values(+ self, scale_shift_table: torch.Tensor, batch_size: int, timestep: torch.Tensor, indices: slice = slice(None, None)+ ):+ num_ada_params = scale_shift_table.shape[0]++ ada_values = (+ scale_shift_table[indices].unsqueeze(0).unsqueeze(0).to(device=timestep.device, dtype=timestep.dtype)+ + timestep.reshape(batch_size, timestep.shape[1], num_ada_params, -1)[:, :, indices, :]+ ).unbind(dim=2)+ return ada_values++ def get_av_ca_ada_values(+ self,+ scale_shift_table: torch.Tensor,+ batch_size: int,+ scale_shift_timestep: torch.Tensor,+ gate_timestep: torch.Tensor,+ num_scale_shift_values: int = 4,+ ):+ scale_shift_ada_values = self.get_ada_values(+ scale_shift_table[:num_scale_shift_values, :],+ batch_size,+ scale_shift_timestep,+ )+ gate_ada_values = self.get_ada_values(+ scale_shift_table[num_scale_shift_values:, :],+ batch_size,+ gate_timestep,+ )++ scale_shift_chunks = [t.squeeze(2) for t in scale_shift_ada_values]+ gate_ada_values = [t.squeeze(2) for t in gate_ada_values]++ return (*scale_shift_chunks, *gate_ada_values)++ def forward(+ self,+ x: Tuple[torch.Tensor, torch.Tensor],+ v_context=None,+ a_context=None,+ attention_mask=None,+ v_timestep=None,+ a_timestep=None,+ v_pe=None,+ a_pe=None,+ v_cross_pe=None,+ a_cross_pe=None,+ v_cross_scale_shift_timestep=None,+ a_cross_scale_shift_timestep=None,+ v_cross_gate_timestep=None,+ a_cross_gate_timestep=None,+ transformer_options=None,+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ run_vx = transformer_options.get("run_vx", True)+ run_ax = transformer_options.get("run_ax", True)++ vx, ax = x+ run_ax = run_ax and ax.numel() > 0+ run_a2v = run_vx and transformer_options.get("a2v_cross_attn", True) and ax.numel() > 0+ run_v2a = run_ax and transformer_options.get("v2a_cross_attn", True)++ if run_vx:+ vshift_msa, vscale_msa, vgate_msa = (+ self.get_ada_values(self.scale_shift_table, vx.shape[0], v_timestep, slice(0, 3))+ )++ norm_vx = comfy.ldm.common_dit.rms_norm(vx) * (1 + vscale_msa) + vshift_msa+ vx += self.attn1(norm_vx, pe=v_pe, transformer_options=transformer_options) * vgate_msa+ vx += self.attn2(+ comfy.ldm.common_dit.rms_norm(vx),+ context=v_context,+ mask=attention_mask,+ transformer_options=transformer_options,+ )++ del vshift_msa, vscale_msa, vgate_msa++ if run_ax:+ ashift_msa, ascale_msa, agate_msa = (+ self.get_ada_values(self.audio_scale_shift_table, ax.shape[0], a_timestep, slice(0, 3))+ )++ norm_ax = comfy.ldm.common_dit.rms_norm(ax) * (1 + ascale_msa) + ashift_msa+ ax += (+ self.audio_attn1(norm_ax, pe=a_pe, transformer_options=transformer_options)+ * agate_msa+ )+ ax += self.audio_attn2(+ comfy.ldm.common_dit.rms_norm(ax),+ context=a_context,+ mask=attention_mask,+ transformer_options=transformer_options,+ )++ del ashift_msa, ascale_msa, agate_msa++ # Audio - Video cross attention.+ if run_a2v or run_v2a:+ # norm3+ vx_norm3 = comfy.ldm.common_dit.rms_norm(vx)+ ax_norm3 = comfy.ldm.common_dit.rms_norm(ax)++ (+ scale_ca_audio_hidden_states_a2v,+ shift_ca_audio_hidden_states_a2v,+ scale_ca_audio_hidden_states_v2a,+ shift_ca_audio_hidden_states_v2a,+ gate_out_v2a,+ ) = self.get_av_ca_ada_values(+ self.scale_shift_table_a2v_ca_audio,+ ax.shape[0],+ a_cross_scale_shift_timestep,+ a_cross_gate_timestep,+ )++ (+ scale_ca_video_hidden_states_a2v,+ shift_ca_video_hidden_states_a2v,+ scale_ca_video_hidden_states_v2a,+ shift_ca_video_hidden_states_v2a,+ gate_out_a2v,+ ) = self.get_av_ca_ada_values(+ self.scale_shift_table_a2v_ca_video,+ vx.shape[0],+ v_cross_scale_shift_timestep,+ v_cross_gate_timestep,+ )++ if run_a2v:+ vx_scaled = (+ vx_norm3 * (1 + scale_ca_video_hidden_states_a2v)+ + shift_ca_video_hidden_states_a2v+ )+ ax_scaled = (+ ax_norm3 * (1 + scale_ca_audio_hidden_states_a2v)+ + shift_ca_audio_hidden_states_a2v+ )+ vx += (+ self.audio_to_video_attn(+ vx_scaled,+ context=ax_scaled,+ pe=v_cross_pe,+ k_pe=a_cross_pe,+ transformer_options=transformer_options,+ )+ * gate_out_a2v+ )++ del gate_out_a2v+ del scale_ca_video_hidden_states_a2v,\+ shift_ca_video_hidden_states_a2v,\+ scale_ca_audio_hidden_states_a2v,\+ shift_ca_audio_hidden_states_a2v,\++ if run_v2a:+ ax_scaled = (+ ax_norm3 * (1 + scale_ca_audio_hidden_states_v2a)+ + shift_ca_audio_hidden_states_v2a+ )+ vx_scaled = (+ vx_norm3 * (1 + scale_ca_video_hidden_states_v2a)+ + shift_ca_video_hidden_states_v2a+ )+ ax += (+ self.video_to_audio_attn(+ ax_scaled,+ context=vx_scaled,+ pe=a_cross_pe,+ k_pe=v_cross_pe,+ transformer_options=transformer_options,+ )+ * gate_out_v2a+ )++ del gate_out_v2a+ del scale_ca_video_hidden_states_v2a,\+ shift_ca_video_hidden_states_v2a,\+ scale_ca_audio_hidden_states_v2a,\+ shift_ca_audio_hidden_states_v2a++ if run_vx:+ vshift_mlp, vscale_mlp, vgate_mlp = (+ self.get_ada_values(self.scale_shift_table, vx.shape[0], v_timestep, slice(3, None))+ )++ vx_scaled = comfy.ldm.common_dit.rms_norm(vx) * (1 + vscale_mlp) + vshift_mlp+ vx += self.ff(vx_scaled) * vgate_mlp+ del vshift_mlp, vscale_mlp, vgate_mlp++ if run_ax:+ ashift_mlp, ascale_mlp, agate_mlp = (+ self.get_ada_values(self.audio_scale_shift_table, ax.shape[0], a_timestep, slice(3, None))+ )++ ax_scaled = comfy.ldm.common_dit.rms_norm(ax) * (1 + ascale_mlp) + ashift_mlp+ ax += self.audio_ff(ax_scaled) * agate_mlp++ del ashift_mlp, ascale_mlp, agate_mlp+++ return vx, ax+++class LTXAVModel(LTXVModel):+ """LTXAV model for audio-video generation."""++ def __init__(+ self,+ in_channels=128,+ audio_in_channels=128,+ cross_attention_dim=4096,+ audio_cross_attention_dim=2048,+ attention_head_dim=128,+ audio_attention_head_dim=64,+ num_attention_heads=32,+ audio_num_attention_heads=32,+ caption_channels=3840,+ num_layers=48,+ positional_embedding_theta=10000.0,+ positional_embedding_max_pos=[20, 2048, 2048],+ audio_positional_embedding_max_pos=[20],+ causal_temporal_positioning=False,+ vae_scale_factors=(8, 32, 32),+ use_middle_indices_grid=False,+ timestep_scale_multiplier=1000.0,+ av_ca_timestep_scale_multiplier=1.0,+ dtype=None,+ device=None,+ operations=None,+ **kwargs,+ ):+ # Store audio-specific parameters+ self.audio_in_channels = audio_in_channels+ self.audio_cross_attention_dim = audio_cross_attention_dim+ self.audio_attention_head_dim = audio_attention_head_dim+ self.audio_num_attention_heads = audio_num_attention_heads+ self.audio_positional_embedding_max_pos = audio_positional_embedding_max_pos++ # Calculate audio dimensions+ self.audio_inner_dim = audio_num_attention_heads * audio_attention_head_dim+ self.audio_out_channels = audio_in_channels++ # Audio-specific constants+ self.num_audio_channels = 8+ self.audio_frequency_bins = 16++ self.av_ca_timestep_scale_multiplier = av_ca_timestep_scale_multiplier++ super().__init__(+ in_channels=in_channels,+ cross_attention_dim=cross_attention_dim,+ attention_head_dim=attention_head_dim,+ num_attention_heads=num_attention_heads,+ caption_channels=caption_channels,+ num_layers=num_layers,+ positional_embedding_theta=positional_embedding_theta,+ positional_embedding_max_pos=positional_embedding_max_pos,+ causal_temporal_positioning=causal_temporal_positioning,+ vae_scale_factors=vae_scale_factors,+ use_middle_indices_grid=use_middle_indices_grid,+ timestep_scale_multiplier=timestep_scale_multiplier,+ dtype=dtype,+ device=device,+ operations=operations,+ **kwargs,+ )++ def _init_model_components(self, device, dtype, **kwargs):+ """Initialize LTXAV-specific components."""+ # Audio-specific projections+ self.audio_patchify_proj = self.operations.Linear(+ self.audio_in_channels, self.audio_inner_dim, bias=True, dtype=dtype, device=device+ )++ # Audio-specific AdaLN+ self.audio_adaln_single = AdaLayerNormSingle(+ self.audio_inner_dim,+ use_additional_conditions=False,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )++ num_scale_shift_values = 4+ self.av_ca_video_scale_shift_adaln_single = AdaLayerNormSingle(+ self.inner_dim,+ use_additional_conditions=False,+ embedding_coefficient=num_scale_shift_values,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )+ self.av_ca_a2v_gate_adaln_single = AdaLayerNormSingle(+ self.inner_dim,+ use_additional_conditions=False,+ embedding_coefficient=1,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )+ self.av_ca_audio_scale_shift_adaln_single = AdaLayerNormSingle(+ self.audio_inner_dim,+ use_additional_conditions=False,+ embedding_coefficient=num_scale_shift_values,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )+ self.av_ca_v2a_gate_adaln_single = AdaLayerNormSingle(+ self.audio_inner_dim,+ use_additional_conditions=False,+ embedding_coefficient=1,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )++ # Audio caption projection+ self.audio_caption_projection = PixArtAlphaTextProjection(+ in_features=self.caption_channels,+ hidden_size=self.audio_inner_dim,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )++ def _init_transformer_blocks(self, device, dtype, **kwargs):+ """Initialize transformer blocks for LTXAV."""+ self.transformer_blocks = nn.ModuleList(+ [+ BasicAVTransformerBlock(+ v_dim=self.inner_dim,+ a_dim=self.audio_inner_dim,+ v_heads=self.num_attention_heads,+ a_heads=self.audio_num_attention_heads,+ vd_head=self.attention_head_dim,+ ad_head=self.audio_attention_head_dim,+ v_context_dim=self.cross_attention_dim,+ a_context_dim=self.audio_cross_attention_dim,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )+ for _ in range(self.num_layers)+ ]+ )++ def _init_output_components(self, device, dtype):+ """Initialize output components for LTXAV."""+ # Video output components+ super()._init_output_components(device, dtype)+ # Audio output components+ self.audio_scale_shift_table = nn.Parameter(+ torch.empty(2, self.audio_inner_dim, dtype=dtype, device=device)+ )+ self.audio_norm_out = self.operations.LayerNorm(+ self.audio_inner_dim, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device+ )+ self.audio_proj_out = self.operations.Linear(+ self.audio_inner_dim, self.audio_out_channels, dtype=dtype, device=device+ )+ self.a_patchifier = AudioPatchifier(1, start_end=True)++ def separate_audio_and_video_latents(self, x, audio_length):+ """Separate audio and video latents from combined input."""+ # vx = x[:, : self.in_channels]+ # ax = x[:, self.in_channels :]+ #+ # ax = ax.reshape(ax.shape[0], -1)+ # ax = ax[:, : audio_length * self.num_audio_channels * self.audio_frequency_bins]+ #+ # ax = ax.reshape(+ # ax.shape[0], self.num_audio_channels, audio_length, self.audio_frequency_bins+ # )++ vx = x[0]+ ax = x[1] if len(x) > 1 else torch.zeros(+ (vx.shape[0], self.num_audio_channels, 0, self.audio_frequency_bins),+ device=vx.device, dtype=vx.dtype+ )+ return vx, ax++ def recombine_audio_and_video_latents(self, vx, ax, target_shape=None):+ if ax.numel() == 0:+ return vx+ else:+ return [vx, ax]+ """Recombine audio and video latents for output."""+ # if ax.device != vx.device or ax.dtype != vx.dtype:+ # logging.warning("Audio and video latents are on different devices or dtypes.")+ # ax = ax.to(device=vx.device, dtype=vx.dtype)+ # logging.warning(f"Audio audio latent moved to device: {ax.device}, dtype: {ax.dtype}")+ #+ # ax = ax.reshape(ax.shape[0], -1)+ # # pad to f x h x w of the video latents+ # divisor = vx.shape[-1] * vx.shape[-2] * vx.shape[-3]+ # if target_shape is None:+ # repetitions = math.ceil(ax.shape[-1] / divisor)+ # else:+ # repetitions = target_shape[1] - vx.shape[1]+ # padded_len = repetitions * divisor+ # ax = F.pad(ax, (0, padded_len - ax.shape[-1]))+ # ax = ax.reshape(ax.shape[0], -1, vx.shape[-3], vx.shape[-2], vx.shape[-1])+ # return torch.cat([vx, ax], dim=1)++ def _process_input(self, x, keyframe_idxs, denoise_mask, **kwargs):+ """Process input for LTXAV - separate audio and video, then patchify."""+ audio_length = kwargs.get("audio_length", 0)+ # Separate audio and video latents+ vx, ax = self.separate_audio_and_video_latents(x, audio_length)+ [vx, v_pixel_coords, additional_args] = super()._process_input(+ vx, keyframe_idxs, denoise_mask, **kwargs+ )++ ax, a_latent_coords = self.a_patchifier.patchify(ax)+ ax = self.audio_patchify_proj(ax)++ # additional_args.update({"av_orig_shape": list(x.shape)})+ return [vx, ax], [v_pixel_coords, a_latent_coords], additional_args++ def _prepare_timestep(self, timestep, batch_size, hidden_dtype, **kwargs):+ """Prepare timestep embeddings."""+ # TODO: some code reuse is needed here.+ grid_mask = kwargs.get("grid_mask", None)+ if grid_mask is not None:+ timestep = timestep[:, grid_mask]++ timestep = timestep * self.timestep_scale_multiplier+ v_timestep, v_embedded_timestep = self.adaln_single(+ timestep.flatten(),+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,+ )++ # Second dimension is 1 or number of tokens (if timestep_per_token)+ v_timestep = v_timestep.view(batch_size, -1, v_timestep.shape[-1])+ v_embedded_timestep = v_embedded_timestep.view(+ batch_size, -1, v_embedded_timestep.shape[-1]+ )++ # Prepare audio timestep+ a_timestep = kwargs.get("a_timestep")+ if a_timestep is not None:+ a_timestep = a_timestep * self.timestep_scale_multiplier+ av_ca_factor = self.av_ca_timestep_scale_multiplier / self.timestep_scale_multiplier++ av_ca_audio_scale_shift_timestep, _ = self.av_ca_audio_scale_shift_adaln_single(+ a_timestep.flatten(),+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,+ )+ av_ca_video_scale_shift_timestep, _ = self.av_ca_video_scale_shift_adaln_single(+ timestep.flatten(),+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,+ )+ av_ca_a2v_gate_noise_timestep, _ = self.av_ca_a2v_gate_adaln_single(+ timestep.flatten() * av_ca_factor,+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,+ )+ av_ca_v2a_gate_noise_timestep, _ = self.av_ca_v2a_gate_adaln_single(+ a_timestep.flatten() * av_ca_factor,+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,+ )++ a_timestep, a_embedded_timestep = self.audio_adaln_single(+ a_timestep.flatten(),+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,+ )+ a_timestep = a_timestep.view(batch_size, -1, a_timestep.shape[-1])+ a_embedded_timestep = a_embedded_timestep.view(+ batch_size, -1, a_embedded_timestep.shape[-1]+ )+ cross_av_timestep_ss = [+ av_ca_audio_scale_shift_timestep,+ av_ca_video_scale_shift_timestep,+ av_ca_a2v_gate_noise_timestep,+ av_ca_v2a_gate_noise_timestep,+ ]+ cross_av_timestep_ss = list(+ [t.view(batch_size, -1, t.shape[-1]) for t in cross_av_timestep_ss]+ )+ else:+ a_timestep = timestep+ a_embedded_timestep = kwargs.get("embedded_timestep")+ cross_av_timestep_ss = []++ return [v_timestep, a_timestep, cross_av_timestep_ss], [+ v_embedded_timestep,+ a_embedded_timestep,+ ]++ def _prepare_context(self, context, batch_size, x, attention_mask=None):+ vx = x[0]+ ax = x[1]+ v_context, a_context = torch.split(+ context, int(context.shape[-1] / 2), len(context.shape) - 1+ )++ v_context, attention_mask = super()._prepare_context(+ v_context, batch_size, vx, attention_mask+ )+ if self.audio_caption_projection is not None:+ a_context = self.audio_caption_projection(a_context)+ a_context = a_context.view(batch_size, -1, ax.shape[-1])++ return [v_context, a_context], attention_mask++ def _prepare_positional_embeddings(self, pixel_coords, frame_rate, x_dtype):+ v_pixel_coords = pixel_coords[0]+ v_pe = super()._prepare_positional_embeddings(v_pixel_coords, frame_rate, x_dtype)++ a_latent_coords = pixel_coords[1]+ a_pe = self._precompute_freqs_cis(+ a_latent_coords,+ dim=self.audio_inner_dim,+ out_dtype=x_dtype,+ max_pos=self.audio_positional_embedding_max_pos,+ use_middle_indices_grid=self.use_middle_indices_grid,+ num_attention_heads=self.audio_num_attention_heads,+ )++ # calculate positional embeddings for the middle of the token duration, to use in av cross attention layers.+ max_pos = max(+ self.positional_embedding_max_pos[0], self.audio_positional_embedding_max_pos[0]+ )+ v_pixel_coords = v_pixel_coords.to(torch.float32)+ v_pixel_coords[:, 0] = v_pixel_coords[:, 0] * (1.0 / frame_rate)+ av_cross_video_freq_cis = self._precompute_freqs_cis(+ v_pixel_coords[:, 0:1, :],+ dim=self.audio_cross_attention_dim,+ out_dtype=x_dtype,+ max_pos=[max_pos],+ use_middle_indices_grid=True,+ num_attention_heads=self.audio_num_attention_heads,+ )+ av_cross_audio_freq_cis = self._precompute_freqs_cis(+ a_latent_coords[:, 0:1, :],+ dim=self.audio_cross_attention_dim,+ out_dtype=x_dtype,+ max_pos=[max_pos],+ use_middle_indices_grid=True,+ num_attention_heads=self.audio_num_attention_heads,+ )++ return [(v_pe, av_cross_video_freq_cis), (a_pe, av_cross_audio_freq_cis)]++ def _process_transformer_blocks(+ self, x, context, attention_mask, timestep, pe, transformer_options={}, **kwargs+ ):+ vx = x[0]+ ax = x[1]+ v_context = context[0]+ a_context = context[1]+ v_timestep = timestep[0]+ a_timestep = timestep[1]+ v_pe, av_cross_video_freq_cis = pe[0]+ a_pe, av_cross_audio_freq_cis = pe[1]++ (+ av_ca_audio_scale_shift_timestep,+ av_ca_video_scale_shift_timestep,+ av_ca_a2v_gate_noise_timestep,+ av_ca_v2a_gate_noise_timestep,+ ) = timestep[2]++ """Process transformer blocks for LTXAV."""+ patches_replace = transformer_options.get("patches_replace", {})+ blocks_replace = patches_replace.get("dit", {})++ # Process transformer blocks+ for i, block in enumerate(self.transformer_blocks):+ if ("double_block", i) in blocks_replace:++ def block_wrap(args):+ out = {}+ out["img"] = block(+ args["img"],+ v_context=args["v_context"],+ a_context=args["a_context"],+ attention_mask=args["attention_mask"],+ v_timestep=args["v_timestep"],+ a_timestep=args["a_timestep"],+ v_pe=args["v_pe"],+ a_pe=args["a_pe"],+ v_cross_pe=args["v_cross_pe"],+ a_cross_pe=args["a_cross_pe"],+ v_cross_scale_shift_timestep=args["v_cross_scale_shift_timestep"],+ a_cross_scale_shift_timestep=args["a_cross_scale_shift_timestep"],+ v_cross_gate_timestep=args["v_cross_gate_timestep"],+ a_cross_gate_timestep=args["a_cross_gate_timestep"],+ transformer_options=args["transformer_options"],+ )+ return out++ out = blocks_replace[("double_block", i)](+ {+ "img": (vx, ax),+ "v_context": v_context,+ "a_context": a_context,+ "attention_mask": attention_mask,+ "v_timestep": v_timestep,+ "a_timestep": a_timestep,+ "v_pe": v_pe,+ "a_pe": a_pe,+ "v_cross_pe": av_cross_video_freq_cis,+ "a_cross_pe": av_cross_audio_freq_cis,+ "v_cross_scale_shift_timestep": av_ca_video_scale_shift_timestep,+ "a_cross_scale_shift_timestep": av_ca_audio_scale_shift_timestep,+ "v_cross_gate_timestep": av_ca_a2v_gate_noise_timestep,+ "a_cross_gate_timestep": av_ca_v2a_gate_noise_timestep,+ "transformer_options": transformer_options,+ },+ {"original_block": block_wrap},+ )+ vx, ax = out["img"]+ else:+ vx, ax = block(+ (vx, ax),+ v_context=v_context,+ a_context=a_context,+ attention_mask=attention_mask,+ v_timestep=v_timestep,+ a_timestep=a_timestep,+ v_pe=v_pe,+ a_pe=a_pe,+ v_cross_pe=av_cross_video_freq_cis,+ a_cross_pe=av_cross_audio_freq_cis,+ v_cross_scale_shift_timestep=av_ca_video_scale_shift_timestep,+ a_cross_scale_shift_timestep=av_ca_audio_scale_shift_timestep,+ v_cross_gate_timestep=av_ca_a2v_gate_noise_timestep,+ a_cross_gate_timestep=av_ca_v2a_gate_noise_timestep,+ transformer_options=transformer_options,+ )++ return [vx, ax]++ def _process_output(self, x, embedded_timestep, keyframe_idxs, **kwargs):+ vx = x[0]+ ax = x[1]+ v_embedded_timestep = embedded_timestep[0]+ a_embedded_timestep = embedded_timestep[1]+ vx = super()._process_output(vx, v_embedded_timestep, keyframe_idxs, **kwargs)++ # Process audio output+ a_scale_shift_values = (+ self.audio_scale_shift_table[None, None].to(device=a_embedded_timestep.device, dtype=a_embedded_timestep.dtype)+ + a_embedded_timestep[:, :, None]+ )+ a_shift, a_scale = a_scale_shift_values[:, :, 0], a_scale_shift_values[:, :, 1]++ ax = self.audio_norm_out(ax)+ ax = ax * (1 + a_scale) + a_shift+ ax = self.audio_proj_out(ax)++ # Unpatchify audio+ ax = self.a_patchifier.unpatchify(+ ax, channels=self.num_audio_channels, freq=self.audio_frequency_bins+ )++ # Recombine audio and video+ original_shape = kwargs.get("av_orig_shape")+ return self.recombine_audio_and_video_latents(vx, ax, original_shape)++ def forward(+ self,+ x,+ timestep,+ context,+ attention_mask=None,+ frame_rate=25,+ transformer_options={},+ keyframe_idxs=None,+ **kwargs,+ ):+ """+ Forward pass for LTXAV model.++ Args:+ x: Combined audio-video input tensor+ timestep: Tuple of (video_timestep, audio_timestep) or single timestep+ context: Context tensor (e.g., text embeddings)+ attention_mask: Attention mask tensor+ frame_rate: Frame rate for temporal processing+ transformer_options: Additional options for transformer blocks+ keyframe_idxs: Keyframe indices for temporal processing+ **kwargs: Additional keyword arguments including audio_length++ Returns:+ Combined audio-video output tensor+ """+ # Handle timestep format+ if isinstance(timestep, (tuple, list)) and len(timestep) == 2:+ v_timestep, a_timestep = timestep+ kwargs["a_timestep"] = a_timestep+ timestep = v_timestep+ else:+ kwargs["a_timestep"] = timestep++ # Call parent forward method+ return super().forward(+ x,+ timestep,+ context,+ attention_mask,+ frame_rate,+ transformer_options,+ keyframe_idxs,+ **kwargs,+ )diff --git a/comfy/ldm/lightricks/embeddings_connector.py b/comfy/ldm/lightricks/embeddings_connector.pynew file mode 100644index 000000000..f7a43f3c3--- /dev/null+++ b/comfy/ldm/lightricks/embeddings_connector.py@@ -0,0 +1,305 @@+import math+from typing import Optional++import comfy.ldm.common_dit+import torch+from comfy.ldm.lightricks.model import (+ CrossAttention,+ FeedForward,+ generate_freq_grid_np,+ interleaved_freqs_cis,+ split_freqs_cis,+)+from torch import nn+++class BasicTransformerBlock1D(nn.Module):+ r"""+ A basic Transformer block.++ Parameters:++ dim (`int`): The number of channels in the input and output.+ num_attention_heads (`int`): The number of heads to use for multi-head attention.+ attention_head_dim (`int`): The number of channels in each head.+ dropout (`float`, *optional*, defaults to 0.0): The dropout probability to use.+ activation_fn (`str`, *optional*, defaults to `"geglu"`): Activation function to be used in feed-forward.+ attention_bias (:+ obj: `bool`, *optional*, defaults to `False`): Configure if the attentions should contain a bias parameter.+ upcast_attention (`bool`, *optional*):+ Whether to upcast the attention computation to float32. This is useful for mixed precision training.+ norm_elementwise_affine (`bool`, *optional*, defaults to `True`):+ Whether to use learnable elementwise affine parameters for normalization.+ standardization_norm (`str`, *optional*, defaults to `"layer_norm"`): The type of pre-normalization to use. Can be `"layer_norm"` or `"rms_norm"`.+ norm_eps (`float`, *optional*, defaults to 1e-5): Epsilon value for normalization layers.+ qk_norm (`str`, *optional*, defaults to None):+ Set to 'layer_norm' or `rms_norm` to perform query and key normalization.+ final_dropout (`bool` *optional*, defaults to False):+ Whether to apply a final dropout after the last feed-forward layer.+ ff_inner_dim (`int`, *optional*): Dimension of the inner feed-forward layer. If not provided, defaults to `dim * 4`.+ ff_bias (`bool`, *optional*, defaults to `True`): Whether to use bias in the feed-forward layer.+ attention_out_bias (`bool`, *optional*, defaults to `True`): Whether to use bias in the attention output layer.+ use_rope (`bool`, *optional*, defaults to `False`): Whether to use Rotary Position Embeddings (RoPE).+ ffn_dim_mult (`int`, *optional*, defaults to 4): Multiplier for the inner dimension of the feed-forward layer.+ """++ def __init__(+ self,+ dim,+ n_heads,+ d_head,+ context_dim=None,+ attn_precision=None,+ dtype=None,+ device=None,+ operations=None,+ ):+ super().__init__()++ # Define 3 blocks. Each block has its own normalization layer.+ # 1. Self-Attn+ self.attn1 = CrossAttention(+ query_dim=dim,+ heads=n_heads,+ dim_head=d_head,+ context_dim=None,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ # 3. Feed-forward+ self.ff = FeedForward(+ dim,+ dim_out=dim,+ glu=True,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ def forward(self, hidden_states, attention_mask=None, pe=None) -> torch.FloatTensor:++ # Notice that normalization is always applied before the real computation in the following blocks.++ # 1. Normalization Before Self-Attention+ norm_hidden_states = comfy.ldm.common_dit.rms_norm(hidden_states)++ norm_hidden_states = norm_hidden_states.squeeze(1)++ # 2. Self-Attention+ attn_output = self.attn1(norm_hidden_states, mask=attention_mask, pe=pe)++ hidden_states = attn_output + hidden_states+ if hidden_states.ndim == 4:+ hidden_states = hidden_states.squeeze(1)++ # 3. Normalization before Feed-Forward+ norm_hidden_states = comfy.ldm.common_dit.rms_norm(hidden_states)++ # 4. Feed-forward+ ff_output = self.ff(norm_hidden_states)++ hidden_states = ff_output + hidden_states+ if hidden_states.ndim == 4:+ hidden_states = hidden_states.squeeze(1)++ return hidden_states+++class Embeddings1DConnector(nn.Module):+ _supports_gradient_checkpointing = True++ def __init__(+ self,+ in_channels=128,+ cross_attention_dim=2048,+ attention_head_dim=128,+ num_attention_heads=30,+ num_layers=2,+ positional_embedding_theta=10000.0,+ positional_embedding_max_pos=[4096],+ causal_temporal_positioning=False,+ num_learnable_registers: Optional[int] = 128,+ dtype=None,+ device=None,+ operations=None,+ split_rope=False,+ double_precision_rope=False,+ **kwargs,+ ):+ super().__init__()+ self.dtype = dtype+ self.out_channels = in_channels+ self.num_attention_heads = num_attention_heads+ self.inner_dim = num_attention_heads * attention_head_dim+ self.causal_temporal_positioning = causal_temporal_positioning+ self.positional_embedding_theta = positional_embedding_theta+ self.positional_embedding_max_pos = positional_embedding_max_pos+ self.split_rope = split_rope+ self.double_precision_rope = double_precision_rope+ self.transformer_1d_blocks = nn.ModuleList(+ [+ BasicTransformerBlock1D(+ self.inner_dim,+ num_attention_heads,+ attention_head_dim,+ context_dim=cross_attention_dim,+ dtype=dtype,+ device=device,+ operations=operations,+ )+ for _ in range(num_layers)+ ]+ )++ inner_dim = num_attention_heads * attention_head_dim+ self.num_learnable_registers = num_learnable_registers+ if self.num_learnable_registers:+ self.learnable_registers = nn.Parameter(+ torch.rand(+ self.num_learnable_registers, inner_dim, dtype=dtype, device=device+ )+ * 2.0+ - 1.0+ )++ def get_fractional_positions(self, indices_grid):+ fractional_positions = torch.stack(+ [+ indices_grid[:, i] / self.positional_embedding_max_pos[i]+ for i in range(1)+ ],+ dim=-1,+ )+ return fractional_positions++ def precompute_freqs(self, indices_grid, spacing):+ source_dtype = indices_grid.dtype+ dtype = (+ torch.float32+ if source_dtype in (torch.bfloat16, torch.float16)+ else source_dtype+ )++ fractional_positions = self.get_fractional_positions(indices_grid)+ indices = (+ generate_freq_grid_np(+ self.positional_embedding_theta,+ indices_grid.shape[1],+ self.inner_dim,+ )+ if self.double_precision_rope+ else self.generate_freq_grid(spacing, dtype, fractional_positions.device)+ ).to(device=fractional_positions.device)++ if spacing == "exp_2":+ freqs = (+ (indices * fractional_positions.unsqueeze(-1))+ .transpose(-1, -2)+ .flatten(2)+ )+ else:+ freqs = (+ (indices * (fractional_positions.unsqueeze(-1) * 2 - 1))+ .transpose(-1, -2)+ .flatten(2)+ )+ return freqs++ def generate_freq_grid(self, spacing, dtype, device):+ dim = self.inner_dim+ theta = self.positional_embedding_theta+ n_pos_dims = 1+ n_elem = 2 * n_pos_dims # 2 for cos and sin e.g. x 3 = 6+ start = 1+ end = theta++ if spacing == "exp":+ indices = theta ** (torch.arange(0, dim, n_elem, device="cpu", dtype=torch.float32) / (dim - n_elem))+ indices = indices.to(dtype=dtype, device=device)+ elif spacing == "exp_2":+ indices = 1.0 / theta ** (torch.arange(0, dim, n_elem, device=device) / dim)+ indices = indices.to(dtype=dtype)+ elif spacing == "linear":+ indices = torch.linspace(+ start, end, dim // n_elem, device=device, dtype=dtype+ )+ elif spacing == "sqrt":+ indices = torch.linspace(+ start**2, end**2, dim // n_elem, device=device, dtype=dtype+ ).sqrt()++ indices = indices * math.pi / 2++ return indices++ def precompute_freqs_cis(self, indices_grid, spacing="exp"):+ dim = self.inner_dim+ n_elem = 2 # 2 because of cos and sin+ freqs = self.precompute_freqs(indices_grid, spacing)+ if self.split_rope:+ expected_freqs = dim // 2+ current_freqs = freqs.shape[-1]+ pad_size = expected_freqs - current_freqs+ cos_freq, sin_freq = split_freqs_cis(+ freqs, pad_size, self.num_attention_heads+ )+ else:+ cos_freq, sin_freq = interleaved_freqs_cis(freqs, dim % n_elem)+ return cos_freq.to(self.dtype), sin_freq.to(self.dtype), self.split_rope++ def forward(+ self,+ hidden_states: torch.Tensor,+ attention_mask: Optional[torch.Tensor] = None,+ ):+ """+ The [`Transformer2DModel`] forward method.++ Args:+ hidden_states (`torch.LongTensor` of shape `(batch size, num latent pixels)` if discrete, `torch.FloatTensor` of shape `(batch size, channel, height, width)` if continuous):+ Input `hidden_states`.+ indices_grid (`torch.LongTensor` of shape `(batch size, 3, num latent pixels)`):+ attention_mask ( `torch.Tensor`, *optional*):+ An attention mask of shape `(batch, key_tokens)` is applied to `encoder_hidden_states`. If `1` the mask+ is kept, otherwise if `0` it is discarded. Mask will be converted into a bias, which adds large+ negative values to the attention scores corresponding to "discard" tokens.+ Returns:+ If `return_dict` is True, an [`~models.transformer_2d.Transformer2DModelOutput`] is returned, otherwise a+ `tuple` where the first element is the sample tensor.+ """+ # 1. Input++ if self.num_learnable_registers:+ num_registers_duplications = math.ceil(+ max(1024, hidden_states.shape[1]) / self.num_learnable_registers+ )+ learnable_registers = torch.tile(+ self.learnable_registers, (num_registers_duplications, 1)+ )++ hidden_states = torch.cat((hidden_states, learnable_registers[hidden_states.shape[1]:].unsqueeze(0).repeat(hidden_states.shape[0], 1, 1)), dim=1)++ if attention_mask is not None:+ attention_mask = torch.zeros([1, 1, 1, hidden_states.shape[1]], dtype=attention_mask.dtype, device=attention_mask.device)++ indices_grid = torch.arange(+ hidden_states.shape[1], dtype=torch.float32, device=hidden_states.device+ )+ indices_grid = indices_grid[None, None, :]+ freqs_cis = self.precompute_freqs_cis(indices_grid)++ # 2. Blocks+ for block_idx, block in enumerate(self.transformer_1d_blocks):+ hidden_states = block(+ hidden_states, attention_mask=attention_mask, pe=freqs_cis+ )++ # 3. Output+ # if self.output_scale is not None:+ # hidden_states = hidden_states / self.output_scale++ hidden_states = comfy.ldm.common_dit.rms_norm(hidden_states)++ return hidden_states, attention_maskdiff --git a/comfy/ldm/lightricks/latent_upsampler.py b/comfy/ldm/lightricks/latent_upsampler.pynew file mode 100644index 000000000..78ed7653f--- /dev/null+++ b/comfy/ldm/lightricks/latent_upsampler.py@@ -0,0 +1,292 @@+from typing import Optional, Tuple+import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange+++def _rational_for_scale(scale: float) -> Tuple[int, int]:+ mapping = {0.75: (3, 4), 1.5: (3, 2), 2.0: (2, 1), 4.0: (4, 1)}+ if float(scale) not in mapping:+ raise ValueError(+ f"Unsupported spatial_scale {scale}. Choose from {list(mapping.keys())}"+ )+ return mapping[float(scale)]+++class PixelShuffleND(nn.Module):+ def __init__(self, dims, upscale_factors=(2, 2, 2)):+ super().__init__()+ assert dims in [1, 2, 3], "dims must be 1, 2, or 3"+ self.dims = dims+ self.upscale_factors = upscale_factors++ def forward(self, x):+ if self.dims == 3:+ return rearrange(+ x,+ "b (c p1 p2 p3) d h w -> b c (d p1) (h p2) (w p3)",+ p1=self.upscale_factors[0],+ p2=self.upscale_factors[1],+ p3=self.upscale_factors[2],+ )+ elif self.dims == 2:+ return rearrange(+ x,+ "b (c p1 p2) h w -> b c (h p1) (w p2)",+ p1=self.upscale_factors[0],+ p2=self.upscale_factors[1],+ )+ elif self.dims == 1:+ return rearrange(+ x,+ "b (c p1) f h w -> b c (f p1) h w",+ p1=self.upscale_factors[0],+ )+++class BlurDownsample(nn.Module):+ """+ Anti-aliased spatial downsampling by integer stride using a fixed separable binomial kernel.+ Applies only on H,W. Works for dims=2 or dims=3 (per-frame).+ """++ def __init__(self, dims: int, stride: int):+ super().__init__()+ assert dims in (2, 3)+ assert stride >= 1 and isinstance(stride, int)+ self.dims = dims+ self.stride = stride++ # 5x5 separable binomial kernel [1,4,6,4,1] (outer product), normalized+ k = torch.tensor([1.0, 4.0, 6.0, 4.0, 1.0])+ k2d = k[:, None] @ k[None, :]+ k2d = (k2d / k2d.sum()).float() # shape (5,5)+ self.register_buffer("kernel", k2d[None, None, :, :]) # (1,1,5,5)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ if self.stride == 1:+ return x++ def _apply_2d(x2d: torch.Tensor) -> torch.Tensor:+ # x2d: (B, C, H, W)+ B, C, H, W = x2d.shape+ weight = self.kernel.expand(C, 1, 5, 5) # depthwise+ x2d = F.conv2d(+ x2d, weight=weight, bias=None, stride=self.stride, padding=2, groups=C+ )+ return x2d++ if self.dims == 2:+ return _apply_2d(x)+ else:+ # dims == 3: apply per-frame on H,W+ b, c, f, h, w = x.shape+ x = rearrange(x, "b c f h w -> (b f) c h w")+ x = _apply_2d(x)+ h2, w2 = x.shape[-2:]+ x = rearrange(x, "(b f) c h w -> b c f h w", b=b, f=f, h=h2, w=w2)+ return x+++class SpatialRationalResampler(nn.Module):+ """+ Fully-learned rational spatial scaling: up by 'num' via PixelShuffle, then anti-aliased+ downsample by 'den' using fixed blur + stride. Operates on H,W only.++ For dims==3, work per-frame for spatial scaling (temporal axis untouched).+ """++ def __init__(self, mid_channels: int, scale: float):+ super().__init__()+ self.scale = float(scale)+ self.num, self.den = _rational_for_scale(self.scale)+ self.conv = nn.Conv2d(+ mid_channels, (self.num**2) * mid_channels, kernel_size=3, padding=1+ )+ self.pixel_shuffle = PixelShuffleND(2, upscale_factors=(self.num, self.num))+ self.blur_down = BlurDownsample(dims=2, stride=self.den)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ b, c, f, h, w = x.shape+ x = rearrange(x, "b c f h w -> (b f) c h w")+ x = self.conv(x)+ x = self.pixel_shuffle(x)+ x = self.blur_down(x)+ x = rearrange(x, "(b f) c h w -> b c f h w", b=b, f=f)+ return x+++class ResBlock(nn.Module):+ def __init__(+ self, channels: int, mid_channels: Optional[int] = None, dims: int = 3+ ):+ super().__init__()+ if mid_channels is None:+ mid_channels = channels++ Conv = nn.Conv2d if dims == 2 else nn.Conv3d++ self.conv1 = Conv(channels, mid_channels, kernel_size=3, padding=1)+ self.norm1 = nn.GroupNorm(32, mid_channels)+ self.conv2 = Conv(mid_channels, channels, kernel_size=3, padding=1)+ self.norm2 = nn.GroupNorm(32, channels)+ self.activation = nn.SiLU()++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ residual = x+ x = self.conv1(x)+ x = self.norm1(x)+ x = self.activation(x)+ x = self.conv2(x)+ x = self.norm2(x)+ x = self.activation(x + residual)+ return x+++class LatentUpsampler(nn.Module):+ """+ Model to spatially upsample VAE latents.++ Args:+ in_channels (`int`): Number of channels in the input latent+ mid_channels (`int`): Number of channels in the middle layers+ num_blocks_per_stage (`int`): Number of ResBlocks to use in each stage (pre/post upsampling)+ dims (`int`): Number of dimensions for convolutions (2 or 3)+ spatial_upsample (`bool`): Whether to spatially upsample the latent+ temporal_upsample (`bool`): Whether to temporally upsample the latent+ """++ def __init__(+ self,+ in_channels: int = 128,+ mid_channels: int = 512,+ num_blocks_per_stage: int = 4,+ dims: int = 3,+ spatial_upsample: bool = True,+ temporal_upsample: bool = False,+ spatial_scale: float = 2.0,+ rational_resampler: bool = False,+ ):+ super().__init__()++ self.in_channels = in_channels+ self.mid_channels = mid_channels+ self.num_blocks_per_stage = num_blocks_per_stage+ self.dims = dims+ self.spatial_upsample = spatial_upsample+ self.temporal_upsample = temporal_upsample+ self.spatial_scale = float(spatial_scale)+ self.rational_resampler = rational_resampler++ Conv = nn.Conv2d if dims == 2 else nn.Conv3d++ self.initial_conv = Conv(in_channels, mid_channels, kernel_size=3, padding=1)+ self.initial_norm = nn.GroupNorm(32, mid_channels)+ self.initial_activation = nn.SiLU()++ self.res_blocks = nn.ModuleList(+ [ResBlock(mid_channels, dims=dims) for _ in range(num_blocks_per_stage)]+ )++ if spatial_upsample and temporal_upsample:+ self.upsampler = nn.Sequential(+ nn.Conv3d(mid_channels, 8 * mid_channels, kernel_size=3, padding=1),+ PixelShuffleND(3),+ )+ elif spatial_upsample:+ if rational_resampler:+ self.upsampler = SpatialRationalResampler(+ mid_channels=mid_channels, scale=self.spatial_scale+ )+ else:+ self.upsampler = nn.Sequential(+ nn.Conv2d(mid_channels, 4 * mid_channels, kernel_size=3, padding=1),+ PixelShuffleND(2),+ )+ elif temporal_upsample:+ self.upsampler = nn.Sequential(+ nn.Conv3d(mid_channels, 2 * mid_channels, kernel_size=3, padding=1),+ PixelShuffleND(1),+ )+ else:+ raise ValueError(+ "Either spatial_upsample or temporal_upsample must be True"+ )++ self.post_upsample_res_blocks = nn.ModuleList(+ [ResBlock(mid_channels, dims=dims) for _ in range(num_blocks_per_stage)]+ )++ self.final_conv = Conv(mid_channels, in_channels, kernel_size=3, padding=1)++ def forward(self, latent: torch.Tensor) -> torch.Tensor:+ b, c, f, h, w = latent.shape++ if self.dims == 2:+ x = rearrange(latent, "b c f h w -> (b f) c h w")+ x = self.initial_conv(x)+ x = self.initial_norm(x)+ x = self.initial_activation(x)++ for block in self.res_blocks:+ x = block(x)++ x = self.upsampler(x)++ for block in self.post_upsample_res_blocks:+ x = block(x)++ x = self.final_conv(x)+ x = rearrange(x, "(b f) c h w -> b c f h w", b=b, f=f)+ else:+ x = self.initial_conv(latent)+ x = self.initial_norm(x)+ x = self.initial_activation(x)++ for block in self.res_blocks:+ x = block(x)++ if self.temporal_upsample:+ x = self.upsampler(x)+ x = x[:, :, 1:, :, :]+ else:+ if isinstance(self.upsampler, SpatialRationalResampler):+ x = self.upsampler(x)+ else:+ x = rearrange(x, "b c f h w -> (b f) c h w")+ x = self.upsampler(x)+ x = rearrange(x, "(b f) c h w -> b c f h w", b=b, f=f)++ for block in self.post_upsample_res_blocks:+ x = block(x)++ x = self.final_conv(x)++ return x++ @classmethod+ def from_config(cls, config):+ return cls(+ in_channels=config.get("in_channels", 4),+ mid_channels=config.get("mid_channels", 128),+ num_blocks_per_stage=config.get("num_blocks_per_stage", 4),+ dims=config.get("dims", 2),+ spatial_upsample=config.get("spatial_upsample", True),+ temporal_upsample=config.get("temporal_upsample", False),+ spatial_scale=config.get("spatial_scale", 2.0),+ rational_resampler=config.get("rational_resampler", False),+ )++ def config(self):+ return {+ "_class_name": "LatentUpsampler",+ "in_channels": self.in_channels,+ "mid_channels": self.mid_channels,+ "num_blocks_per_stage": self.num_blocks_per_stage,+ "dims": self.dims,+ "spatial_upsample": self.spatial_upsample,+ "temporal_upsample": self.temporal_upsample,+ "spatial_scale": self.spatial_scale,+ "rational_resampler": self.rational_resampler,+ }diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.pyindex 593f7940f..d61e19d6e 100644--- a/comfy/ldm/lightricks/model.py+++ b/comfy/ldm/lightricks/model.py@@ -1,13 +1,47 @@+from abc import ABC, abstractmethod+from enum import Enum+import functools+import math+from typing import Dict, Optional, Tuple++from einops import rearrange+import numpy as npimport torchfrom torch import nnimport comfy.patcher_extensionimport comfy.ldm.modules.attentionimport comfy.ldm.common_dit-import math-from typing import Dict, Optional, Tuplefrom .symmetric_patchifier import SymmetricPatchifier, latent_to_pixel_coords-from comfy.ldm.flux.math import apply_rope1++def _log_base(x, base):+ return np.log(x) / np.log(base)++class LTXRopeType(str, Enum):+ INTERLEAVED = "interleaved"+ SPLIT = "split"++ KEY = "rope_type"++ @classmethod+ def from_dict(cls, kwargs, default=None):+ if default is None:+ default = cls.INTERLEAVED+ return cls(kwargs.get(cls.KEY, default))+++class LTXFrequenciesPrecision(str, Enum):+ FLOAT32 = "float32"+ FLOAT64 = "float64"++ KEY = "frequencies_precision"++ @classmethod+ def from_dict(cls, kwargs, default=None):+ if default is None:+ default = cls.FLOAT32+ return cls(kwargs.get(cls.KEY, default))+def get_timestep_embedding(timesteps: torch.Tensor,@@ -39,9 +73,7 @@ def get_timestep_embedding(assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"half_dim = embedding_dim // 2- exponent = -math.log(max_period) * torch.arange(- start=0, end=half_dim, dtype=torch.float32, device=timesteps.device- )+ exponent = -math.log(max_period) * torch.arange(start=0, end=half_dim, dtype=torch.float32, device=timesteps.device)exponent = exponent / (half_dim - downscale_freq_shift)emb = torch.exp(exponent)@@ -73,7 +105,9 @@ class TimestepEmbedding(nn.Module):post_act_fn: Optional[str] = None,cond_proj_dim=None,sample_proj_bias=True,- dtype=None, device=None, operations=None,+ dtype=None,+ device=None,+ operations=None,):super().__init__()@@ -90,7 +124,9 @@ class TimestepEmbedding(nn.Module):time_embed_dim_out = out_dimelse:time_embed_dim_out = time_embed_dim- self.linear_2 = operations.Linear(time_embed_dim, time_embed_dim_out, sample_proj_bias, dtype=dtype, device=device)+ self.linear_2 = operations.Linear(+ time_embed_dim, time_embed_dim_out, sample_proj_bias, dtype=dtype, device=device+ )if post_act_fn is None:self.post_act = None@@ -139,12 +175,22 @@ class PixArtAlphaCombinedTimestepSizeEmbeddings(nn.Module):https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L164C9-L168C29"""- def __init__(self, embedding_dim, size_emb_dim, use_additional_conditions: bool = False, dtype=None, device=None, operations=None):+ def __init__(+ self,+ embedding_dim,+ size_emb_dim,+ use_additional_conditions: bool = False,+ dtype=None,+ device=None,+ operations=None,+ ):super().__init__()self.outdim = size_emb_dimself.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0)- self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim, dtype=dtype, device=device, operations=operations)+ self.timestep_embedder = TimestepEmbedding(+ in_channels=256, time_embed_dim=embedding_dim, dtype=dtype, device=device, operations=operations+ )def forward(self, timestep, resolution, aspect_ratio, batch_size, hidden_dtype):timesteps_proj = self.time_proj(timestep)@@ -163,15 +209,22 @@ class AdaLayerNormSingle(nn.Module):use_additional_conditions (`bool`): To use additional conditions for normalization or not."""- def __init__(self, embedding_dim: int, use_additional_conditions: bool = False, dtype=None, device=None, operations=None):+ def __init__(+ self, embedding_dim: int, embedding_coefficient: int = 6, use_additional_conditions: bool = False, dtype=None, device=None, operations=None+ ):super().__init__()self.emb = PixArtAlphaCombinedTimestepSizeEmbeddings(- embedding_dim, size_emb_dim=embedding_dim // 3, use_additional_conditions=use_additional_conditions, dtype=dtype, device=device, operations=operations+ embedding_dim,+ size_emb_dim=embedding_dim // 3,+ use_additional_conditions=use_additional_conditions,+ dtype=dtype,+ device=device,+ operations=operations,)self.silu = nn.SiLU()- self.linear = operations.Linear(embedding_dim, 6 * embedding_dim, bias=True, dtype=dtype, device=device)+ self.linear = operations.Linear(embedding_dim, embedding_coefficient * embedding_dim, bias=True, dtype=dtype, device=device)def forward(self,@@ -185,6 +238,7 @@ class AdaLayerNormSingle(nn.Module):embedded_timestep = self.emb(timestep, **added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_dtype)return self.linear(self.silu(embedded_timestep)), embedded_timestep+class PixArtAlphaTextProjection(nn.Module):"""Projects caption embeddings. Also handles dropout for classifier-free guidance.@@ -192,18 +246,24 @@ class PixArtAlphaTextProjection(nn.Module):Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py"""- def __init__(self, in_features, hidden_size, out_features=None, act_fn="gelu_tanh", dtype=None, device=None, operations=None):+ def __init__(+ self, in_features, hidden_size, out_features=None, act_fn="gelu_tanh", dtype=None, device=None, operations=None+ ):super().__init__()if out_features is None:out_features = hidden_size- self.linear_1 = operations.Linear(in_features=in_features, out_features=hidden_size, bias=True, dtype=dtype, device=device)+ self.linear_1 = operations.Linear(+ in_features=in_features, out_features=hidden_size, bias=True, dtype=dtype, device=device+ )if act_fn == "gelu_tanh":self.act_1 = nn.GELU(approximate="tanh")elif act_fn == "silu":self.act_1 = nn.SiLU()else:raise ValueError(f"Unknown activation function: {act_fn}")- self.linear_2 = operations.Linear(in_features=hidden_size, out_features=out_features, bias=True, dtype=dtype, device=device)+ self.linear_2 = operations.Linear(+ in_features=hidden_size, out_features=out_features, bias=True, dtype=dtype, device=device+ )def forward(self, caption):hidden_states = self.linear_1(caption)@@ -222,23 +282,68 @@ class GELU_approx(nn.Module):class FeedForward(nn.Module):- def __init__(self, dim, dim_out, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=None):+ def __init__(self, dim, dim_out, mult=4, glu=False, dropout=0.0, dtype=None, device=None, operations=None):super().__init__()inner_dim = int(dim * mult)project_in = GELU_approx(dim, inner_dim, dtype=dtype, device=device, operations=operations)self.net = nn.Sequential(- project_in,- nn.Dropout(dropout),- operations.Linear(inner_dim, dim_out, dtype=dtype, device=device)+ project_in, nn.Dropout(dropout), operations.Linear(inner_dim, dim_out, dtype=dtype, device=device))def forward(self, x):return self.net(x)+def apply_rotary_emb(input_tensor, freqs_cis):+ cos_freqs, sin_freqs = freqs_cis[0], freqs_cis[1]+ split_pe = freqs_cis[2] if len(freqs_cis) > 2 else False+ return (+ apply_split_rotary_emb(input_tensor, cos_freqs, sin_freqs)+ if split_pe else+ apply_interleaved_rotary_emb(input_tensor, cos_freqs, sin_freqs)+ )++def apply_interleaved_rotary_emb(input_tensor, cos_freqs, sin_freqs): # TODO: remove duplicate funcs and pick the best/fastest one+ t_dup = rearrange(input_tensor, "... (d r) -> ... d r", r=2)+ t1, t2 = t_dup.unbind(dim=-1)+ t_dup = torch.stack((-t2, t1), dim=-1)+ input_tensor_rot = rearrange(t_dup, "... d r -> ... (d r)")++ out = input_tensor * cos_freqs + input_tensor_rot * sin_freqs++ return out++def apply_split_rotary_emb(input_tensor, cos, sin):+ needs_reshape = False+ if input_tensor.ndim != 4 and cos.ndim == 4:+ B, H, T, _ = cos.shape+ input_tensor = input_tensor.reshape(B, T, H, -1).swapaxes(1, 2)+ needs_reshape = True+ split_input = rearrange(input_tensor, "... (d r) -> ... d r", d=2)+ first_half_input = split_input[..., :1, :]+ second_half_input = split_input[..., 1:, :]+ output = split_input * cos.unsqueeze(-2)+ first_half_output = output[..., :1, :]+ second_half_output = output[..., 1:, :]+ first_half_output.addcmul_(-sin.unsqueeze(-2), second_half_input)+ second_half_output.addcmul_(sin.unsqueeze(-2), first_half_input)+ output = rearrange(output, "... d r -> ... (d r)")+ return output.swapaxes(1, 2).reshape(B, T, -1) if needs_reshape else output+class CrossAttention(nn.Module):- def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., attn_precision=None, dtype=None, device=None, operations=None):+ def __init__(+ self,+ query_dim,+ context_dim=None,+ heads=8,+ dim_head=64,+ dropout=0.0,+ attn_precision=None,+ dtype=None,+ device=None,+ operations=None,+ ):super().__init__()inner_dim = dim_head * headscontext_dim = query_dim if context_dim is None else context_dim@@ -254,9 +359,11 @@ class CrossAttention(nn.Module):self.to_k = operations.Linear(context_dim, inner_dim, bias=True, dtype=dtype, device=device)self.to_v = operations.Linear(context_dim, inner_dim, bias=True, dtype=dtype, device=device)- self.to_out = nn.Sequential(operations.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout))+ self.to_out = nn.Sequential(+ operations.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout)+ )- def forward(self, x, context=None, mask=None, pe=None, transformer_options={}):+ def forward(self, x, context=None, mask=None, pe=None, k_pe=None, transformer_options={}):q = self.to_q(x)context = x if context is None else contextk = self.to_k(context)@@ -266,8 +373,8 @@ class CrossAttention(nn.Module):k = self.k_norm(k)if pe is not None:- q = apply_rope1(q.unsqueeze(1), pe).squeeze(1)- k = apply_rope1(k.unsqueeze(1), pe).squeeze(1)+ q = apply_rotary_emb(q, pe)+ k = apply_rotary_emb(k, pe if k_pe is None else k_pe)if mask is None:out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision, transformer_options=transformer_options)@@ -277,14 +384,34 @@ class CrossAttention(nn.Module):class BasicTransformerBlock(nn.Module):- def __init__(self, dim, n_heads, d_head, context_dim=None, attn_precision=None, dtype=None, device=None, operations=None):+ def __init__(+ self, dim, n_heads, d_head, context_dim=None, attn_precision=None, dtype=None, device=None, operations=None+ ):super().__init__()self.attn_precision = attn_precision- self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, context_dim=None, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations)+ self.attn1 = CrossAttention(+ query_dim=dim,+ heads=n_heads,+ dim_head=d_head,+ context_dim=None,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )self.ff = FeedForward(dim, dim_out=dim, glu=True, dtype=dtype, device=device, operations=operations)- self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, heads=n_heads, dim_head=d_head, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations)+ self.attn2 = CrossAttention(+ query_dim=dim,+ context_dim=context_dim,+ heads=n_heads,+ dim_head=d_head,+ attn_precision=self.attn_precision,+ dtype=dtype,+ device=device,+ operations=operations,+ )self.scale_shift_table = nn.Parameter(torch.empty(6, dim, device=device, dtype=dtype))@@ -306,116 +433,446 @@ class BasicTransformerBlock(nn.Module):return xdef get_fractional_positions(indices_grid, max_pos):+ n_pos_dims = indices_grid.shape[1]+ assert n_pos_dims == len(max_pos), f'Number of position dimensions ({n_pos_dims}) must match max_pos length ({len(max_pos)})'fractional_positions = torch.stack(- [- indices_grid[:, i] / max_pos[i]- for i in range(3)- ],- dim=-1,+ [indices_grid[:, i] / max_pos[i] for i in range(n_pos_dims)],+ axis=-1,)return fractional_positions-def precompute_freqs_cis(indices_grid, dim, out_dtype, theta=10000.0, max_pos=[20, 2048, 2048]):- dtype = torch.float32- device = indices_grid.device[email protected]_cache(maxsize=5)+def generate_freq_grid_np(positional_embedding_theta, positional_embedding_max_pos_count, inner_dim, _ = None):+ theta = positional_embedding_theta+ start = 1+ end = theta++ n_elem = 2 * positional_embedding_max_pos_count+ pow_indices = np.power(+ theta,+ np.linspace(+ _log_base(start, theta),+ _log_base(end, theta),+ inner_dim // n_elem,+ dtype=np.float64,+ ),+ )+ return torch.tensor(pow_indices * math.pi / 2, dtype=torch.float32)++def generate_freq_grid_pytorch(positional_embedding_theta, positional_embedding_max_pos_count, inner_dim, device):+ theta = positional_embedding_theta+ start = 1+ end = theta+ n_elem = 2 * positional_embedding_max_pos_count++ indices = theta ** (+ torch.linspace(+ math.log(start, theta),+ math.log(end, theta),+ inner_dim // n_elem,+ device=device,+ dtype=torch.float32,+ )+ )+ indices = indices.to(dtype=torch.float32)++ indices = indices * math.pi / 2++ return indices++def generate_freqs(indices, indices_grid, max_pos, use_middle_indices_grid):+ if use_middle_indices_grid:+ assert(len(indices_grid.shape) == 4 and indices_grid.shape[-1] ==2)+ indices_grid_start, indices_grid_end = indices_grid[..., 0], indices_grid[..., 1]+ indices_grid = (indices_grid_start + indices_grid_end) / 2.0+ elif len(indices_grid.shape) == 4:+ indices_grid = indices_grid[..., 0]# Get fractional positions and compute frequency indicesfractional_positions = get_fractional_positions(indices_grid, max_pos)- indices = theta ** torch.linspace(0, 1, dim // 6, device=device, dtype=dtype) * math.pi / 2+ indices = indices.to(device=fractional_positions.device)++ freqs = (+ (indices * (fractional_positions.unsqueeze(-1) * 2 - 1))+ .transpose(-1, -2)+ .flatten(2)+ )+ return freqs- # Compute frequencies and apply cos/sin- freqs = (indices * (fractional_positions.unsqueeze(-1) * 2 - 1)).transpose(-1, -2).flatten(2)- cos_vals = freqs.cos().repeat_interleave(2, dim=-1)- sin_vals = freqs.sin().repeat_interleave(2, dim=-1)+def interleaved_freqs_cis(freqs, pad_size):+ cos_freq = freqs.cos().repeat_interleave(2, dim=-1)+ sin_freq = freqs.sin().repeat_interleave(2, dim=-1)+ if pad_size != 0:+ cos_padding = torch.ones_like(cos_freq[:, :, : pad_size])+ sin_padding = torch.zeros_like(cos_freq[:, :, : pad_size])+ cos_freq = torch.cat([cos_padding, cos_freq], dim=-1)+ sin_freq = torch.cat([sin_padding, sin_freq], dim=-1)+ return cos_freq, sin_freq- # Pad if dim is not divisible by 6- if dim % 6 != 0:- padding_size = dim % 6- cos_vals = torch.cat([torch.ones_like(cos_vals[:, :, :padding_size]), cos_vals], dim=-1)- sin_vals = torch.cat([torch.zeros_like(sin_vals[:, :, :padding_size]), sin_vals], dim=-1)+def split_freqs_cis(freqs, pad_size, num_attention_heads):+ cos_freq = freqs.cos()+ sin_freq = freqs.sin()- # Reshape and extract one value per pair (since repeat_interleave duplicates each value)- cos_vals = cos_vals.reshape(*cos_vals.shape[:2], -1, 2)[..., 0].to(out_dtype) # [B, N, dim//2]- sin_vals = sin_vals.reshape(*sin_vals.shape[:2], -1, 2)[..., 0].to(out_dtype) # [B, N, dim//2]+ if pad_size != 0:+ cos_padding = torch.ones_like(cos_freq[:, :, :pad_size])+ sin_padding = torch.zeros_like(sin_freq[:, :, :pad_size])- # Build rotation matrix [[cos, -sin], [sin, cos]] and add heads dimension- freqs_cis = torch.stack([- torch.stack([cos_vals, -sin_vals], dim=-1),- torch.stack([sin_vals, cos_vals], dim=-1)- ], dim=-2).unsqueeze(1) # [B, 1, N, dim//2, 2, 2]+ cos_freq = torch.concatenate([cos_padding, cos_freq], axis=-1)+ sin_freq = torch.concatenate([sin_padding, sin_freq], axis=-1)- return freqs_cis+ # Reshape freqs to be compatible with multi-head attention+ B , T, half_HD = cos_freq.shape+ cos_freq = cos_freq.reshape(B, T, num_attention_heads, half_HD // num_attention_heads)+ sin_freq = sin_freq.reshape(B, T, num_attention_heads, half_HD // num_attention_heads)-class LTXVModel(torch.nn.Module):- def __init__(self,- in_channels=128,- cross_attention_dim=2048,- attention_head_dim=64,- num_attention_heads=32,+ cos_freq = torch.swapaxes(cos_freq, 1, 2) # (B,H,T,D//2)+ sin_freq = torch.swapaxes(sin_freq, 1, 2) # (B,H,T,D//2)+ return cos_freq, sin_freq- caption_channels=4096,- num_layers=28,+class LTXBaseModel(torch.nn.Module, ABC):+ """+ Abstract base class for LTX models (Lightricks Transformer models).+ This class defines the common interface and shared functionality for all LTX models,+ including LTXV (video) and LTXAV (audio-video) variants.+ """- positional_embedding_theta=10000.0,- positional_embedding_max_pos=[20, 2048, 2048],- causal_temporal_positioning=False,- vae_scale_factors=(8, 32, 32),- dtype=None, device=None, operations=None, **kwargs):+ def __init__(+ self,+ in_channels: int,+ cross_attention_dim: int,+ attention_head_dim: int,+ num_attention_heads: int,+ caption_channels: int,+ num_layers: int,+ positional_embedding_theta: float = 10000.0,+ positional_embedding_max_pos: list = [20, 2048, 2048],+ causal_temporal_positioning: bool = False,+ vae_scale_factors: tuple = (8, 32, 32),+ use_middle_indices_grid=False,+ timestep_scale_multiplier = 1000.0,+ dtype=None,+ device=None,+ operations=None,+ **kwargs,+ ):super().__init__()self.generator = Noneself.vae_scale_factors = vae_scale_factors+ self.use_middle_indices_grid = use_middle_indices_gridself.dtype = dtype- self.out_channels = in_channels- self.inner_dim = num_attention_heads * attention_head_dim+ self.in_channels = in_channels+ self.cross_attention_dim = cross_attention_dim+ self.attention_head_dim = attention_head_dim+ self.num_attention_heads = num_attention_heads+ self.caption_channels = caption_channels+ self.num_layers = num_layers+ self.positional_embedding_theta = positional_embedding_theta+ self.positional_embedding_max_pos = positional_embedding_max_pos+ self.split_positional_embedding = LTXRopeType.from_dict(kwargs)+ self.freq_grid_generator = (+ generate_freq_grid_np if LTXFrequenciesPrecision.from_dict(kwargs) == LTXFrequenciesPrecision.FLOAT64+ else generate_freq_grid_pytorch+ )self.causal_temporal_positioning = causal_temporal_positioning+ self.operations = operations+ self.timestep_scale_multiplier = timestep_scale_multiplier- self.patchify_proj = operations.Linear(in_channels, self.inner_dim, bias=True, dtype=dtype, device=device)+ # Common dimensions+ self.inner_dim = num_attention_heads * attention_head_dim+ self.out_channels = in_channels- self.adaln_single = AdaLayerNormSingle(- self.inner_dim, use_additional_conditions=False, dtype=dtype, device=device, operations=operations+ # Initialize common components+ self._init_common_components(device, dtype)++ # Initialize model-specific components+ self._init_model_components(device, dtype, **kwargs)++ # Initialize transformer blocks+ self._init_transformer_blocks(device, dtype, **kwargs)++ # Initialize output components+ self._init_output_components(device, dtype)++ def _init_common_components(self, device, dtype):+ """Initialize components common to all LTX models+ - patchify_proj: Linear projection for patchifying input+ - adaln_single: AdaLN layer for timestep embedding+ - caption_projection: Linear projection for caption embedding+ """+ self.patchify_proj = self.operations.Linear(+ self.in_channels, self.inner_dim, bias=True, dtype=dtype, device=device)- # self.adaln_single.linear = operations.Linear(self.inner_dim, 4 * self.inner_dim, bias=True, dtype=dtype, device=device)+ self.adaln_single = AdaLayerNormSingle(+ self.inner_dim, use_additional_conditions=False, dtype=dtype, device=device, operations=self.operations+ )self.caption_projection = PixArtAlphaTextProjection(- in_features=caption_channels, hidden_size=self.inner_dim, dtype=dtype, device=device, operations=operations+ in_features=self.caption_channels,+ hidden_size=self.inner_dim,+ dtype=dtype,+ device=device,+ operations=self.operations,)- self.transformer_blocks = nn.ModuleList(- [- BasicTransformerBlock(- self.inner_dim,- num_attention_heads,- attention_head_dim,- context_dim=cross_attention_dim,- # attn_precision=attn_precision,- dtype=dtype, device=device, operations=operations- )- for d in range(num_layers)- ]+ @abstractmethod+ def _init_model_components(self, device, dtype, **kwargs):+ """Initialize model-specific components. Must be implemented by subclasses."""+ pass++ @abstractmethod+ def _init_transformer_blocks(self, device, dtype, **kwargs):+ """Initialize transformer blocks. Must be implemented by subclasses."""+ pass++ @abstractmethod+ def _init_output_components(self, device, dtype):+ """Initialize output components. Must be implemented by subclasses."""+ pass++ @abstractmethod+ def _process_input(self, x, keyframe_idxs, denoise_mask, **kwargs):+ """Process input data. Must be implemented by subclasses."""+ pass++ @abstractmethod+ def _process_transformer_blocks(self, x, context, attention_mask, timestep, pe, **kwargs):+ """Process transformer blocks. Must be implemented by subclasses."""+ pass++ @abstractmethod+ def _process_output(self, x, embedded_timestep, keyframe_idxs, **kwargs):+ """Process output data. Must be implemented by subclasses."""+ pass++ def _prepare_timestep(self, timestep, batch_size, hidden_dtype, **kwargs):+ """Prepare timestep embeddings."""+ grid_mask = kwargs.get("grid_mask", None)+ if grid_mask is not None:+ timestep = timestep[:, grid_mask]++ timestep = timestep * self.timestep_scale_multiplier+ timestep, embedded_timestep = self.adaln_single(+ timestep.flatten(),+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=hidden_dtype,)- self.scale_shift_table = nn.Parameter(torch.empty(2, self.inner_dim, dtype=dtype, device=device))- self.norm_out = operations.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)- self.proj_out = operations.Linear(self.inner_dim, self.out_channels, dtype=dtype, device=device)+ # Second dimension is 1 or number of tokens (if timestep_per_token)+ timestep = timestep.view(batch_size, -1, timestep.shape[-1])+ embedded_timestep = embedded_timestep.view(batch_size, -1, embedded_timestep.shape[-1])++ return timestep, embedded_timestep++ def _prepare_context(self, context, batch_size, x, attention_mask=None):+ """Prepare context for transformer blocks."""+ if self.caption_projection is not None:+ context = self.caption_projection(context)+ context = context.view(batch_size, -1, x.shape[-1])++ return context, attention_mask++ def _precompute_freqs_cis(+ self,+ indices_grid,+ dim,+ out_dtype,+ theta=10000.0,+ max_pos=[20, 2048, 2048],+ use_middle_indices_grid=False,+ num_attention_heads=32,+ ):+ split_mode = self.split_positional_embedding == LTXRopeType.SPLIT+ indices = self.freq_grid_generator(theta, indices_grid.shape[1], dim, indices_grid.device)+ freqs = generate_freqs(indices, indices_grid, max_pos, use_middle_indices_grid)++ if split_mode:+ expected_freqs = dim // 2+ current_freqs = freqs.shape[-1]+ pad_size = expected_freqs - current_freqs+ cos_freq, sin_freq = split_freqs_cis(freqs, pad_size, num_attention_heads)+ else:+ # 2 because of cos and sin by 3 for (t, x, y), 1 for temporal only+ n_elem = 2 * indices_grid.shape[1]+ cos_freq, sin_freq = interleaved_freqs_cis(freqs, dim % n_elem)+ return cos_freq.to(out_dtype), sin_freq.to(out_dtype), split_mode++ def _prepare_positional_embeddings(self, pixel_coords, frame_rate, x_dtype):+ """Prepare positional embeddings."""+ fractional_coords = pixel_coords.to(torch.float32)+ fractional_coords[:, 0] = fractional_coords[:, 0] * (1.0 / frame_rate)+ pe = self._precompute_freqs_cis(+ fractional_coords,+ dim=self.inner_dim,+ out_dtype=x_dtype,+ max_pos=self.positional_embedding_max_pos,+ use_middle_indices_grid=self.use_middle_indices_grid,+ num_attention_heads=self.num_attention_heads,+ )+ return pe- self.patchifier = SymmetricPatchifier(1)+ def _prepare_attention_mask(self, attention_mask, x_dtype):+ """Prepare attention mask."""+ if attention_mask is not None and not torch.is_floating_point(attention_mask):+ attention_mask = (attention_mask - 1).to(x_dtype).reshape(+ (attention_mask.shape[0], 1, -1, attention_mask.shape[-1])+ ) * torch.finfo(x_dtype).max+ return attention_mask- def forward(self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, **kwargs):+ def forward(+ self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, denoise_mask=None, **kwargs+ ):+ """+ Forward pass for LTX models.++ Args:+ x: Input tensor+ timestep: Timestep tensor+ context: Context tensor (e.g., text embeddings)+ attention_mask: Attention mask tensor+ frame_rate: Frame rate for temporal processing+ transformer_options: Additional options for transformer blocks+ keyframe_idxs: Keyframe indices for temporal processing+ **kwargs: Additional keyword arguments++ Returns:+ Processed output tensor+ """return comfy.patcher_extension.WrapperExecutor.new_class_executor(self._forward,self,- comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)- ).execute(x, timestep, context, attention_mask, frame_rate, transformer_options, keyframe_idxs, **kwargs)+ comfy.patcher_extension.get_all_wrappers(+ comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options+ ),+ ).execute(x, timestep, context, attention_mask, frame_rate, transformer_options, keyframe_idxs, denoise_mask=denoise_mask, **kwargs)++ def _forward(+ self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, denoise_mask=None, **kwargs+ ):+ """+ Internal forward pass for LTX models.++ Args:+ x: Input tensor+ timestep: Timestep tensor+ context: Context tensor (e.g., text embeddings)+ attention_mask: Attention mask tensor+ frame_rate: Frame rate for temporal processing+ transformer_options: Additional options for transformer blocks+ keyframe_idxs: Keyframe indices for temporal processing+ **kwargs: Additional keyword arguments++ Returns:+ Processed output tensor+ """+ if isinstance(x, list):+ input_dtype = x[0].dtype+ batch_size = x[0].shape[0]+ else:+ input_dtype = x.dtype+ batch_size = x.shape[0]+ # Process input+ merged_args = {**transformer_options, **kwargs}+ x, pixel_coords, additional_args = self._process_input(x, keyframe_idxs, denoise_mask, **merged_args)+ merged_args.update(additional_args)++ # Prepare timestep and context+ timestep, embedded_timestep = self._prepare_timestep(timestep, batch_size, input_dtype, **merged_args)+ context, attention_mask = self._prepare_context(context, batch_size, x, attention_mask)++ # Prepare attention mask and positional embeddings+ attention_mask = self._prepare_attention_mask(attention_mask, input_dtype)+ pe = self._prepare_positional_embeddings(pixel_coords, frame_rate, input_dtype)++ # Process transformer blocks+ x = self._process_transformer_blocks(+ x, context, attention_mask, timestep, pe, transformer_options=transformer_options, **merged_args+ )++ # Process output+ x = self._process_output(x, embedded_timestep, keyframe_idxs, **merged_args)+ return x- def _forward(self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, **kwargs):- patches_replace = transformer_options.get("patches_replace", {})- orig_shape = list(x.shape)+class LTXVModel(LTXBaseModel):+ """LTXV model for video generation."""+ def __init__(+ self,+ in_channels=128,+ cross_attention_dim=2048,+ attention_head_dim=64,+ num_attention_heads=32,+ caption_channels=4096,+ num_layers=28,+ positional_embedding_theta=10000.0,+ positional_embedding_max_pos=[20, 2048, 2048],+ causal_temporal_positioning=False,+ vae_scale_factors=(8, 32, 32),+ use_middle_indices_grid=False,+ timestep_scale_multiplier = 1000.0,+ dtype=None,+ device=None,+ operations=None,+ **kwargs,+ ):+ super().__init__(+ in_channels=in_channels,+ cross_attention_dim=cross_attention_dim,+ attention_head_dim=attention_head_dim,+ num_attention_heads=num_attention_heads,+ caption_channels=caption_channels,+ num_layers=num_layers,+ positional_embedding_theta=positional_embedding_theta,+ positional_embedding_max_pos=positional_embedding_max_pos,+ causal_temporal_positioning=causal_temporal_positioning,+ vae_scale_factors=vae_scale_factors,+ use_middle_indices_grid=use_middle_indices_grid,+ timestep_scale_multiplier=timestep_scale_multiplier,+ dtype=dtype,+ device=device,+ operations=operations,+ **kwargs,+ )++ def _init_model_components(self, device, dtype, **kwargs):+ """Initialize LTXV-specific components."""+ # No additional components needed for LTXV beyond base class+ pass++ def _init_transformer_blocks(self, device, dtype, **kwargs):+ """Initialize transformer blocks for LTXV."""+ self.transformer_blocks = nn.ModuleList(+ [+ BasicTransformerBlock(+ self.inner_dim,+ self.num_attention_heads,+ self.attention_head_dim,+ context_dim=self.cross_attention_dim,+ dtype=dtype,+ device=device,+ operations=self.operations,+ )+ for _ in range(self.num_layers)+ ]+ )++ def _init_output_components(self, device, dtype):+ """Initialize output components for LTXV."""+ self.scale_shift_table = nn.Parameter(torch.empty(2, self.inner_dim, dtype=dtype, device=device))+ self.norm_out = self.operations.LayerNorm(+ self.inner_dim, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device+ )+ self.proj_out = self.operations.Linear(self.inner_dim, self.out_channels, dtype=dtype, device=device)+ self.patchifier = SymmetricPatchifier(1, start_end=True)++ def _process_input(self, x, keyframe_idxs, denoise_mask, **kwargs):+ """Process input for LTXV."""+ additional_args = {"orig_shape": list(x.shape)}x, latent_coords = self.patchifier.patchify(x)pixel_coords = latent_to_pixel_coords(latent_coords=latent_coords,@@ -423,44 +880,30 @@ class LTXVModel(torch.nn.Module):causal_fix=self.causal_temporal_positioning,)+ grid_mask = Noneif keyframe_idxs is not None:- pixel_coords[:, :, -keyframe_idxs.shape[2]:] = keyframe_idxs+ additional_args.update({ "orig_patchified_shape": list(x.shape)})+ denoise_mask = self.patchifier.patchify(denoise_mask)[0]+ grid_mask = ~torch.any(denoise_mask < 0, dim=-1)[0]+ additional_args.update({"grid_mask": grid_mask})+ x = x[:, grid_mask, :]+ pixel_coords = pixel_coords[:, :, grid_mask, ...]- fractional_coords = pixel_coords.to(torch.float32)- fractional_coords[:, 0] = fractional_coords[:, 0] * (1.0 / frame_rate)+ kf_grid_mask = grid_mask[-keyframe_idxs.shape[2]:]+ keyframe_idxs = keyframe_idxs[..., kf_grid_mask, :]+ pixel_coords[:, :, -keyframe_idxs.shape[2]:, :] = keyframe_idxsx = self.patchify_proj(x)- timestep = timestep * 1000.0-- if attention_mask is not None and not torch.is_floating_point(attention_mask):- attention_mask = (attention_mask - 1).to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])) * torch.finfo(x.dtype).max-- pe = precompute_freqs_cis(fractional_coords, dim=self.inner_dim, out_dtype=x.dtype)-- batch_size = x.shape[0]- timestep, embedded_timestep = self.adaln_single(- timestep.flatten(),- {"resolution": None, "aspect_ratio": None},- batch_size=batch_size,- hidden_dtype=x.dtype,- )- # Second dimension is 1 or number of tokens (if timestep_per_token)- timestep = timestep.view(batch_size, -1, timestep.shape[-1])- embedded_timestep = embedded_timestep.view(- batch_size, -1, embedded_timestep.shape[-1]- )-- # 2. Blocks- if self.caption_projection is not None:- batch_size = x.shape[0]- context = self.caption_projection(context)- context = context.view(- batch_size, -1, x.shape[-1]- )+ return x, pixel_coords, additional_args+ def _process_transformer_blocks(self, x, context, attention_mask, timestep, pe, transformer_options={}, **kwargs):+ """Process transformer blocks for LTXV."""+ patches_replace = transformer_options.get("patches_replace", {})blocks_replace = patches_replace.get("dit", {})+for i, block in enumerate(self.transformer_blocks):if ("double_block", i) in blocks_replace:+def block_wrap(args):out = {}out["img"] = block(args["img"], context=args["txt"], attention_mask=args["attention_mask"], timestep=args["vec"], pe=args["pe"], transformer_options=args["transformer_options"])@@ -478,16 +921,28 @@ class LTXVModel(torch.nn.Module):transformer_options=transformer_options,)- # 3. Output+ return x++ def _process_output(self, x, embedded_timestep, keyframe_idxs, **kwargs):+ """Process output for LTXV."""+ # Apply scale-shift modulationscale_shift_values = (self.scale_shift_table[None, None].to(device=x.device, dtype=x.dtype) + embedded_timestep[:, :, None])shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1]+x = self.norm_out(x)- # Modulation- x = torch.addcmul(x, x, scale).add_(shift)+ x = x * (1 + scale) + shiftx = self.proj_out(x)+ if keyframe_idxs is not None:+ grid_mask = kwargs["grid_mask"]+ orig_patchified_shape = kwargs["orig_patchified_shape"]+ full_x = torch.zeros(orig_patchified_shape, dtype=x.dtype, device=x.device)+ full_x[:, grid_mask, :] = x+ x = full_x+ # Unpatchify to restore original dimensions+ orig_shape = kwargs["orig_shape"]x = self.patchifier.unpatchify(latents=x,output_height=orig_shape[3],diff --git a/comfy/ldm/lightricks/symmetric_patchifier.py b/comfy/ldm/lightricks/symmetric_patchifier.pyindex 4b9972b9f..8f9a41186 100644--- a/comfy/ldm/lightricks/symmetric_patchifier.py+++ b/comfy/ldm/lightricks/symmetric_patchifier.py@@ -21,20 +21,23 @@ def latent_to_pixel_coords(Returns:Tensor: A tensor of pixel coordinates corresponding to the input latent coordinates."""+ shape = [1] * latent_coords.ndim+ shape[1] = -1pixel_coords = (latent_coords- * torch.tensor(scale_factors, device=latent_coords.device)[None, :, None]+ * torch.tensor(scale_factors, device=latent_coords.device).view(*shape))if causal_fix:# Fix temporal scale for first frame to 1 due to causality- pixel_coords[:, 0] = (pixel_coords[:, 0] + 1 - scale_factors[0]).clamp(min=0)+ pixel_coords[:, 0, ...] = (pixel_coords[:, 0, ...] + 1 - scale_factors[0]).clamp(min=0)return pixel_coordsclass Patchifier(ABC):- def __init__(self, patch_size: int):+ def __init__(self, patch_size: int, start_end: bool=False):super().__init__()self._patch_size = (1, patch_size, patch_size)+ self.start_end = start_end@abstractmethoddef patchify(@@ -71,11 +74,23 @@ class Patchifier(ABC):torch.arange(0, latent_width, self._patch_size[2], device=device),indexing="ij",)- latent_sample_coords = torch.stack(latent_sample_coords, dim=0)- latent_coords = latent_sample_coords.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)- latent_coords = rearrange(- latent_coords, "b c f h w -> b c (f h w)", b=batch_size+ latent_sample_coords_start = torch.stack(latent_sample_coords, dim=0)+ delta = torch.tensor(self._patch_size, device=latent_sample_coords_start.device, dtype=latent_sample_coords_start.dtype)[:, None, None, None]+ latent_sample_coords_end = latent_sample_coords_start + delta++ latent_sample_coords_start = latent_sample_coords_start.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)+ latent_sample_coords_start = rearrange(+ latent_sample_coords_start, "b c f h w -> b c (f h w)", b=batch_size)+ if self.start_end:+ latent_sample_coords_end = latent_sample_coords_end.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)+ latent_sample_coords_end = rearrange(+ latent_sample_coords_end, "b c f h w -> b c (f h w)", b=batch_size+ )++ latent_coords = torch.stack((latent_sample_coords_start, latent_sample_coords_end), dim=-1)+ else:+ latent_coords = latent_sample_coords_startreturn latent_coords@@ -115,3 +130,61 @@ class SymmetricPatchifier(Patchifier):q=self._patch_size[2],)return latents+++class AudioPatchifier(Patchifier):+ def __init__(self, patch_size: int,+ sample_rate=16000,+ hop_length=160,+ audio_latent_downsample_factor=4,+ is_causal=True,+ start_end=False,+ shift = 0+ ):+ super().__init__(patch_size, start_end=start_end)+ self.hop_length = hop_length+ self.sample_rate = sample_rate+ self.audio_latent_downsample_factor = audio_latent_downsample_factor+ self.is_causal = is_causal+ self.shift = shift++ def copy_with_shift(self, shift):+ return AudioPatchifier(+ self.patch_size, self.sample_rate, self.hop_length, self.audio_latent_downsample_factor,+ self.is_causal, self.start_end, shift+ )++ def _get_audio_latent_time_in_sec(self, start_latent, end_latent: int, dtype: torch.dtype, device=torch.device):+ audio_latent_frame = torch.arange(start_latent, end_latent, dtype=dtype, device=device)+ audio_mel_frame = audio_latent_frame * self.audio_latent_downsample_factor+ if self.is_causal:+ audio_mel_frame = (audio_mel_frame + 1 - self.audio_latent_downsample_factor).clip(min=0)+ return audio_mel_frame * self.hop_length / self.sample_rate+++ def patchify(self, audio_latents: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor]:+ # audio_latents: (batch, channels, time, freq)+ b, _, t, _ = audio_latents.shape+ audio_latents = rearrange(+ audio_latents,+ "b c t f -> b t (c f)",+ )++ audio_latents_start_timings = self._get_audio_latent_time_in_sec(self.shift, t + self.shift, torch.float32, audio_latents.device)+ audio_latents_start_timings = audio_latents_start_timings.unsqueeze(0).expand(b, -1).unsqueeze(1)++ if self.start_end:+ audio_latents_end_timings = self._get_audio_latent_time_in_sec(self.shift + 1, t + self.shift + 1, torch.float32, audio_latents.device)+ audio_latents_end_timings = audio_latents_end_timings.unsqueeze(0).expand(b, -1).unsqueeze(1)++ audio_latents_timings = torch.stack([audio_latents_start_timings, audio_latents_end_timings], dim=-1)+ else:+ audio_latents_timings = audio_latents_start_timings+ return audio_latents, audio_latents_timings++ def unpatchify(self, audio_latents: torch.Tensor, channels: int, freq: int) -> torch.Tensor:+ # audio_latents: (batch, time, freq * channels)+ audio_latents = rearrange(+ audio_latents, "b t (c f) -> b c t f", c=channels, f=freq+ )+ return audio_latentsdiff --git a/comfy/ldm/lightricks/vae/audio_vae.py b/comfy/ldm/lightricks/vae/audio_vae.pynew file mode 100644index 000000000..a9111d3bd--- /dev/null+++ b/comfy/ldm/lightricks/vae/audio_vae.py@@ -0,0 +1,286 @@+import json+from dataclasses import dataclass+import math+import torch+import torchaudio++import comfy.model_management+import comfy.model_patcher+import comfy.utils as utils+from comfy.ldm.mmaudio.vae.distributions import DiagonalGaussianDistribution+from comfy.ldm.lightricks.symmetric_patchifier import AudioPatchifier+from comfy.ldm.lightricks.vae.causal_audio_autoencoder import (+ CausalityAxis,+ CausalAudioAutoencoder,+)+from comfy.ldm.lightricks.vocoders.vocoder import Vocoder++LATENT_DOWNSAMPLE_FACTOR = 4+++@dataclass(frozen=True)+class AudioVAEComponentConfig:+ """Container for model component configuration extracted from metadata."""++ autoencoder: dict+ vocoder: dict++ @classmethod+ def from_metadata(cls, metadata: dict) -> "AudioVAEComponentConfig":+ assert metadata is not None and "config" in metadata, "Metadata is required for audio VAE"++ raw_config = metadata["config"]+ if isinstance(raw_config, str):+ parsed_config = json.loads(raw_config)+ else:+ parsed_config = raw_config++ audio_config = parsed_config.get("audio_vae")+ vocoder_config = parsed_config.get("vocoder")++ assert audio_config is not None, "Audio VAE config is required for audio VAE"+ assert vocoder_config is not None, "Vocoder config is required for audio VAE"++ return cls(autoencoder=audio_config, vocoder=vocoder_config)+++class ModelDeviceManager:+ """Manages device placement and GPU residency for the composed model."""++ def __init__(self, module: torch.nn.Module):+ load_device = comfy.model_management.get_torch_device()+ offload_device = comfy.model_management.vae_offload_device()+ self.patcher = comfy.model_patcher.ModelPatcher(module, load_device, offload_device)++ def ensure_model_loaded(self) -> None:+ comfy.model_management.free_memory(+ self.patcher.model_size(),+ self.patcher.load_device,+ )+ comfy.model_management.load_model_gpu(self.patcher)++ def move_to_load_device(self, tensor: torch.Tensor) -> torch.Tensor:+ return tensor.to(self.patcher.load_device)++ @property+ def load_device(self):+ return self.patcher.load_device+++class AudioLatentNormalizer:+ """Applies per-channel statistics in patch space and restores original layout."""++ def __init__(self, patchfier: AudioPatchifier, statistics_processor: torch.nn.Module):+ self.patchifier = patchfier+ self.statistics = statistics_processor++ def normalize(self, latents: torch.Tensor) -> torch.Tensor:+ channels = latents.shape[1]+ freq = latents.shape[3]+ patched, _ = self.patchifier.patchify(latents)+ normalized = self.statistics.normalize(patched)+ return self.patchifier.unpatchify(normalized, channels=channels, freq=freq)++ def denormalize(self, latents: torch.Tensor) -> torch.Tensor:+ channels = latents.shape[1]+ freq = latents.shape[3]+ patched, _ = self.patchifier.patchify(latents)+ denormalized = self.statistics.un_normalize(patched)+ return self.patchifier.unpatchify(denormalized, channels=channels, freq=freq)+++class AudioPreprocessor:+ """Prepares raw waveforms for the autoencoder by matching training conditions."""++ def __init__(self, target_sample_rate: int, mel_bins: int, mel_hop_length: int, n_fft: int):+ self.target_sample_rate = target_sample_rate+ self.mel_bins = mel_bins+ self.mel_hop_length = mel_hop_length+ self.n_fft = n_fft++ def resample(self, waveform: torch.Tensor, source_rate: int) -> torch.Tensor:+ if source_rate == self.target_sample_rate:+ return waveform+ return torchaudio.functional.resample(waveform, source_rate, self.target_sample_rate)++ @staticmethod+ def normalize_amplitude(+ waveform: torch.Tensor, max_amplitude: float = 0.5, eps: float = 1e-5+ ) -> torch.Tensor:+ waveform = waveform - waveform.mean(dim=2, keepdim=True)+ peak = torch.max(torch.abs(waveform)) + eps+ scale = peak.clamp(max=max_amplitude) / peak+ return waveform * scale++ def waveform_to_mel(+ self, waveform: torch.Tensor, waveform_sample_rate: int, device+ ) -> torch.Tensor:+ waveform = self.resample(waveform, waveform_sample_rate)+ waveform = self.normalize_amplitude(waveform)++ mel_transform = torchaudio.transforms.MelSpectrogram(+ sample_rate=self.target_sample_rate,+ n_fft=self.n_fft,+ win_length=self.n_fft,+ hop_length=self.mel_hop_length,+ f_min=0.0,+ f_max=self.target_sample_rate / 2.0,+ n_mels=self.mel_bins,+ window_fn=torch.hann_window,+ center=True,+ pad_mode="reflect",+ power=1.0,+ mel_scale="slaney",+ norm="slaney",+ ).to(device)++ mel = mel_transform(waveform)+ mel = torch.log(torch.clamp(mel, min=1e-5))+ return mel.permute(0, 1, 3, 2).contiguous()+++class AudioVAE(torch.nn.Module):+ """High-level Audio VAE wrapper exposing encode and decode entry points."""++ def __init__(self, state_dict: dict, metadata: dict):+ super().__init__()++ component_config = AudioVAEComponentConfig.from_metadata(metadata)++ vae_sd = utils.state_dict_prefix_replace(state_dict, {"audio_vae.": ""}, filter_keys=True)+ vocoder_sd = utils.state_dict_prefix_replace(state_dict, {"vocoder.": ""}, filter_keys=True)++ self.autoencoder = CausalAudioAutoencoder(config=component_config.autoencoder)+ self.vocoder = Vocoder(config=component_config.vocoder)++ self.autoencoder.load_state_dict(vae_sd, strict=False)+ self.vocoder.load_state_dict(vocoder_sd, strict=False)++ autoencoder_config = self.autoencoder.get_config()+ self.normalizer = AudioLatentNormalizer(+ AudioPatchifier(+ patch_size=1,+ audio_latent_downsample_factor=LATENT_DOWNSAMPLE_FACTOR,+ sample_rate=autoencoder_config["sampling_rate"],+ hop_length=autoencoder_config["mel_hop_length"],+ is_causal=autoencoder_config["is_causal"],+ ),+ self.autoencoder.per_channel_statistics,+ )++ self.preprocessor = AudioPreprocessor(+ target_sample_rate=autoencoder_config["sampling_rate"],+ mel_bins=autoencoder_config["mel_bins"],+ mel_hop_length=autoencoder_config["mel_hop_length"],+ n_fft=autoencoder_config["n_fft"],+ )++ self.device_manager = ModelDeviceManager(self)++ def encode(self, audio: dict) -> torch.Tensor:+ """Encode a waveform dictionary into normalized latent tensors."""++ waveform = audio["waveform"]+ waveform_sample_rate = audio["sample_rate"]+ input_device = waveform.device+ # Ensure that Audio VAE is loaded on the correct device.+ self.device_manager.ensure_model_loaded()++ waveform = self.device_manager.move_to_load_device(waveform)+ expected_channels = self.autoencoder.encoder.in_channels+ if waveform.shape[1] != expected_channels:+ raise ValueError(+ f"Input audio must have {expected_channels} channels, got {waveform.shape[1]}"+ )++ mel_spec = self.preprocessor.waveform_to_mel(+ waveform, waveform_sample_rate, device=self.device_manager.load_device+ )++ latents = self.autoencoder.encode(mel_spec)+ posterior = DiagonalGaussianDistribution(latents)+ latent_mode = posterior.mode()++ normalized = self.normalizer.normalize(latent_mode)+ return normalized.to(input_device)++ def decode(self, latents: torch.Tensor) -> torch.Tensor:+ """Decode normalized latent tensors into an audio waveform."""+ original_shape = latents.shape++ # Ensure that Audio VAE is loaded on the correct device.+ self.device_manager.ensure_model_loaded()++ latents = self.device_manager.move_to_load_device(latents)+ latents = self.normalizer.denormalize(latents)++ target_shape = self.target_shape_from_latents(original_shape)+ mel_spec = self.autoencoder.decode(latents, target_shape=target_shape)++ waveform = self.run_vocoder(mel_spec)+ return self.device_manager.move_to_load_device(waveform)++ def target_shape_from_latents(self, latents_shape):+ batch, _, time, _ = latents_shape+ target_length = time * LATENT_DOWNSAMPLE_FACTOR+ if self.autoencoder.causality_axis != CausalityAxis.NONE:+ target_length -= LATENT_DOWNSAMPLE_FACTOR - 1+ return (+ batch,+ self.autoencoder.decoder.out_ch,+ target_length,+ self.autoencoder.mel_bins,+ )++ def num_of_latents_from_frames(self, frames_number: int, frame_rate: int) -> int:+ return math.ceil((float(frames_number) / frame_rate) * self.latents_per_second)++ def run_vocoder(self, mel_spec: torch.Tensor) -> torch.Tensor:+ audio_channels = self.autoencoder.decoder.out_ch+ vocoder_input = mel_spec.transpose(2, 3)++ if audio_channels == 1:+ vocoder_input = vocoder_input.squeeze(1)+ elif audio_channels != 2:+ raise ValueError(f"Unsupported audio_channels: {audio_channels}")++ return self.vocoder(vocoder_input)++ @property+ def sample_rate(self) -> int:+ return int(self.autoencoder.sampling_rate)++ @property+ def mel_hop_length(self) -> int:+ return int(self.autoencoder.mel_hop_length)++ @property+ def mel_bins(self) -> int:+ return int(self.autoencoder.mel_bins)++ @property+ def latent_channels(self) -> int:+ return int(self.autoencoder.decoder.z_channels)++ @property+ def latent_frequency_bins(self) -> int:+ return int(self.mel_bins // LATENT_DOWNSAMPLE_FACTOR)++ @property+ def latents_per_second(self) -> float:+ return self.sample_rate / self.mel_hop_length / LATENT_DOWNSAMPLE_FACTOR++ @property+ def output_sample_rate(self) -> int:+ output_rate = getattr(self.vocoder, "output_sample_rate", None)+ if output_rate is not None:+ return int(output_rate)+ upsample_factor = getattr(self.vocoder, "upsample_factor", None)+ if upsample_factor is None:+ raise AttributeError(+ "Vocoder is missing upsample_factor; cannot infer output sample rate"+ )+ return int(self.sample_rate * upsample_factor / self.mel_hop_length)++ def memory_required(self, input_shape):+ return self.device_manager.patcher.model_size()diff --git a/comfy/ldm/lightricks/vae/causal_audio_autoencoder.py b/comfy/ldm/lightricks/vae/causal_audio_autoencoder.pynew file mode 100644index 000000000..f12b9bb53--- /dev/null+++ b/comfy/ldm/lightricks/vae/causal_audio_autoencoder.py@@ -0,0 +1,909 @@+from __future__ import annotations+import torch+from torch import nn+from torch.nn import functional as F+from typing import Optional+from enum import Enum+from .pixel_norm import PixelNorm+import comfy.ops+import logging++ops = comfy.ops.disable_weight_init+++class StringConvertibleEnum(Enum):+ """+ Base enum class that provides string-to-enum conversion functionality.++ This mixin adds a str_to_enum() class method that handles conversion from+ strings, None, or existing enum instances with case-insensitive matching.+ """++ @classmethod+ def str_to_enum(cls, value):+ """+ Convert a string, enum instance, or None to the appropriate enum member.++ Args:+ value: Can be an enum instance of this class, a string, or None++ Returns:+ Enum member of this class++ Raises:+ ValueError: If the value cannot be converted to a valid enum member+ """+ # Already an enum instance of this class+ if isinstance(value, cls):+ return value++ # None maps to NONE member if it exists+ if value is None:+ if hasattr(cls, "NONE"):+ return cls.NONE+ raise ValueError(f"{cls.__name__} does not have a NONE member to map None to")++ # String conversion (case-insensitive)+ if isinstance(value, str):+ value_lower = value.lower()++ # Try to match against enum values+ for member in cls:+ # Handle members with None values+ if member.value is None:+ if value_lower == "none":+ return member+ # Handle members with string values+ elif isinstance(member.value, str) and member.value.lower() == value_lower:+ return member++ # Build helpful error message with valid values+ valid_values = []+ for member in cls:+ if member.value is None:+ valid_values.append("none")+ elif isinstance(member.value, str):+ valid_values.append(member.value)++ raise ValueError(f"Invalid {cls.__name__} string: '{value}'. " f"Valid values are: {valid_values}")++ raise ValueError(+ f"Cannot convert type {type(value).__name__} to {cls.__name__} enum. "+ f"Expected string, None, or {cls.__name__} instance."+ )+++class AttentionType(StringConvertibleEnum):+ """Enum for specifying the attention mechanism type."""++ VANILLA = "vanilla"+ LINEAR = "linear"+ NONE = "none"+++class CausalityAxis(StringConvertibleEnum):+ """Enum for specifying the causality axis in causal convolutions."""++ NONE = None+ WIDTH = "width"+ HEIGHT = "height"+ WIDTH_COMPATIBILITY = "width-compatibility"+++def Normalize(in_channels, *, num_groups=32, normtype="group"):+ if normtype == "group":+ return torch.nn.GroupNorm(num_groups=num_groups, num_channels=in_channels, eps=1e-6, affine=True)+ elif normtype == "pixel":+ return PixelNorm(dim=1, eps=1e-6)+ else:+ raise ValueError(f"Invalid normalization type: {normtype}")+++class CausalConv2d(nn.Module):+ """+ A causal 2D convolution.++ This layer ensures that the output at time `t` only depends on inputs+ at time `t` and earlier. It achieves this by applying asymmetric padding+ to the time dimension (width) before the convolution.+ """++ def __init__(+ self,+ in_channels,+ out_channels,+ kernel_size,+ stride=1,+ dilation=1,+ groups=1,+ bias=True,+ causality_axis: CausalityAxis = CausalityAxis.HEIGHT,+ ):+ super().__init__()++ self.causality_axis = causality_axis++ # Ensure kernel_size and dilation are tuples+ kernel_size = nn.modules.utils._pair(kernel_size)+ dilation = nn.modules.utils._pair(dilation)++ # Calculate padding dimensions+ pad_h = (kernel_size[0] - 1) * dilation[0]+ pad_w = (kernel_size[1] - 1) * dilation[1]++ # The padding tuple for F.pad is (pad_left, pad_right, pad_top, pad_bottom)+ match self.causality_axis:+ case CausalityAxis.NONE:+ self.padding = (pad_w // 2, pad_w - pad_w // 2, pad_h // 2, pad_h - pad_h // 2)+ case CausalityAxis.WIDTH | CausalityAxis.WIDTH_COMPATIBILITY:+ self.padding = (pad_w, 0, pad_h // 2, pad_h - pad_h // 2)+ case CausalityAxis.HEIGHT:+ self.padding = (pad_w // 2, pad_w - pad_w // 2, pad_h, 0)+ case _:+ raise ValueError(f"Invalid causality_axis: {causality_axis}")++ # The internal convolution layer uses no padding, as we handle it manually+ self.conv = ops.Conv2d(+ in_channels,+ out_channels,+ kernel_size,+ stride=stride,+ padding=0,+ dilation=dilation,+ groups=groups,+ bias=bias,+ )++ def forward(self, x):+ # Apply causal padding before convolution+ x = F.pad(x, self.padding)+ return self.conv(x)+++def make_conv2d(+ in_channels,+ out_channels,+ kernel_size,+ stride=1,+ padding=None,+ dilation=1,+ groups=1,+ bias=True,+ causality_axis: Optional[CausalityAxis] = None,+):+ """+ Create a 2D convolution layer that can be either causal or non-causal.++ Args:+ in_channels: Number of input channels+ out_channels: Number of output channels+ kernel_size: Size of the convolution kernel+ stride: Convolution stride+ padding: Padding (if None, will be calculated based on causal flag)+ dilation: Dilation rate+ groups: Number of groups for grouped convolution+ bias: Whether to use bias+ causality_axis: Dimension along which to apply causality.++ Returns:+ Either a regular Conv2d or CausalConv2d layer+ """+ if causality_axis is not None:+ # For causal convolution, padding is handled internally by CausalConv2d+ return CausalConv2d(in_channels, out_channels, kernel_size, stride, dilation, groups, bias, causality_axis)+ else:+ # For non-causal convolution, use symmetric padding if not specified+ if padding is None:+ if isinstance(kernel_size, int):+ padding = kernel_size // 2+ else:+ padding = tuple(k // 2 for k in kernel_size)+ return ops.Conv2d(+ in_channels,+ out_channels,+ kernel_size,+ stride,+ padding,+ dilation,+ groups,+ bias,+ )+++class Upsample(nn.Module):+ def __init__(self, in_channels, with_conv, causality_axis: CausalityAxis = CausalityAxis.HEIGHT):+ super().__init__()+ self.with_conv = with_conv+ self.causality_axis = causality_axis+ if self.with_conv:+ self.conv = make_conv2d(in_channels, in_channels, kernel_size=3, stride=1, causality_axis=causality_axis)++ def forward(self, x):+ x = torch.nn.functional.interpolate(x, scale_factor=2.0, mode="nearest")+ if self.with_conv:+ x = self.conv(x)+ # Drop FIRST element in the causal axis to undo encoder's padding, while keeping the length 1 + 2 * n.+ # For example, if the input is [0, 1, 2], after interpolation, the output is [0, 0, 1, 1, 2, 2].+ # The causal convolution will pad the first element as [-, -, 0, 0, 1, 1, 2, 2],+ # So the output elements rely on the following windows:+ # 0: [-,-,0]+ # 1: [-,0,0]+ # 2: [0,0,1]+ # 3: [0,1,1]+ # 4: [1,1,2]+ # 5: [1,2,2]+ # Notice that the first and second elements in the output rely only on the first element in the input,+ # while all other elements rely on two elements in the input.+ # So we can drop the first element to undo the padding (rather than the last element).+ # This is a no-op for non-causal convolutions.+ match self.causality_axis:+ case CausalityAxis.NONE:+ pass # x remains unchanged+ case CausalityAxis.HEIGHT:+ x = x[:, :, 1:, :]+ case CausalityAxis.WIDTH:+ x = x[:, :, :, 1:]+ case CausalityAxis.WIDTH_COMPATIBILITY:+ pass # x remains unchanged+ case _:+ raise ValueError(f"Invalid causality_axis: {self.causality_axis}")++ return x+++class Downsample(nn.Module):+ """+ A downsampling layer that can use either a strided convolution+ or average pooling. Supports standard and causal padding for the+ convolutional mode.+ """++ def __init__(self, in_channels, with_conv, causality_axis: CausalityAxis = CausalityAxis.WIDTH):+ super().__init__()+ self.with_conv = with_conv+ self.causality_axis = causality_axis++ if self.causality_axis != CausalityAxis.NONE and not self.with_conv:+ raise ValueError("causality is only supported when `with_conv=True`.")++ if self.with_conv:+ # Do time downsampling here+ # no asymmetric padding in torch conv, must do it ourselves+ self.conv = ops.Conv2d(in_channels, in_channels, kernel_size=3, stride=2, padding=0)++ def forward(self, x):+ if self.with_conv:+ # (pad_left, pad_right, pad_top, pad_bottom)+ match self.causality_axis:+ case CausalityAxis.NONE:+ pad = (0, 1, 0, 1)+ case CausalityAxis.WIDTH:+ pad = (2, 0, 0, 1)+ case CausalityAxis.HEIGHT:+ pad = (0, 1, 2, 0)+ case CausalityAxis.WIDTH_COMPATIBILITY:+ pad = (1, 0, 0, 1)+ case _:+ raise ValueError(f"Invalid causality_axis: {self.causality_axis}")++ x = torch.nn.functional.pad(x, pad, mode="constant", value=0)+ x = self.conv(x)+ else:+ # This branch is only taken if with_conv=False, which implies causality_axis is NONE.+ x = torch.nn.functional.avg_pool2d(x, kernel_size=2, stride=2)++ return x+++class ResnetBlock(nn.Module):+ def __init__(+ self,+ *,+ in_channels,+ out_channels=None,+ conv_shortcut=False,+ dropout,+ temb_channels=512,+ norm_type="group",+ causality_axis: CausalityAxis = CausalityAxis.HEIGHT,+ ):+ super().__init__()+ self.causality_axis = causality_axis++ if self.causality_axis != CausalityAxis.NONE and norm_type == "group":+ raise ValueError("Causal ResnetBlock with GroupNorm is not supported.")+ self.in_channels = in_channels+ out_channels = in_channels if out_channels is None else out_channels+ self.out_channels = out_channels+ self.use_conv_shortcut = conv_shortcut++ self.norm1 = Normalize(in_channels, normtype=norm_type)+ self.non_linearity = nn.SiLU()+ self.conv1 = make_conv2d(in_channels, out_channels, kernel_size=3, stride=1, causality_axis=causality_axis)+ if temb_channels > 0:+ self.temb_proj = ops.Linear(temb_channels, out_channels)+ self.norm2 = Normalize(out_channels, normtype=norm_type)+ self.dropout = torch.nn.Dropout(dropout)+ self.conv2 = make_conv2d(out_channels, out_channels, kernel_size=3, stride=1, causality_axis=causality_axis)+ if self.in_channels != self.out_channels:+ if self.use_conv_shortcut:+ self.conv_shortcut = make_conv2d(+ in_channels, out_channels, kernel_size=3, stride=1, causality_axis=causality_axis+ )+ else:+ self.nin_shortcut = make_conv2d(+ in_channels, out_channels, kernel_size=1, stride=1, causality_axis=causality_axis+ )++ def forward(self, x, temb):+ h = x+ h = self.norm1(h)+ h = self.non_linearity(h)+ h = self.conv1(h)++ if temb is not None:+ h = h + self.temb_proj(self.non_linearity(temb))[:, :, None, None]++ h = self.norm2(h)+ h = self.non_linearity(h)+ h = self.dropout(h)+ h = self.conv2(h)++ if self.in_channels != self.out_channels:+ if self.use_conv_shortcut:+ x = self.conv_shortcut(x)+ else:+ x = self.nin_shortcut(x)++ return x + h+++class AttnBlock(nn.Module):+ def __init__(self, in_channels, norm_type="group"):+ super().__init__()+ self.in_channels = in_channels++ self.norm = Normalize(in_channels, normtype=norm_type)+ self.q = ops.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)+ self.k = ops.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)+ self.v = ops.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)+ self.proj_out = ops.Conv2d(in_channels, in_channels, kernel_size=1, stride=1, padding=0)++ def forward(self, x):+ h_ = x+ h_ = self.norm(h_)+ q = self.q(h_)+ k = self.k(h_)+ v = self.v(h_)++ # compute attention+ b, c, h, w = q.shape+ q = q.reshape(b, c, h * w).contiguous()+ q = q.permute(0, 2, 1).contiguous() # b,hw,c+ k = k.reshape(b, c, h * w).contiguous() # b,c,hw+ w_ = torch.bmm(q, k).contiguous() # b,hw,hw w[b,i,j]=sum_c q[b,i,c]k[b,c,j]+ w_ = w_ * (int(c) ** (-0.5))+ w_ = torch.nn.functional.softmax(w_, dim=2)++ # attend to values+ v = v.reshape(b, c, h * w).contiguous()+ w_ = w_.permute(0, 2, 1).contiguous() # b,hw,hw (first hw of k, second of q)+ h_ = torch.bmm(v, w_).contiguous() # b, c,hw (hw of q) h_[b,c,j] = sum_i v[b,c,i] w_[b,i,j]+ h_ = h_.reshape(b, c, h, w).contiguous()++ h_ = self.proj_out(h_)++ return x + h_+++def make_attn(in_channels, attn_type="vanilla", norm_type="group"):+ # Convert string to enum if needed+ attn_type = AttentionType.str_to_enum(attn_type)++ if attn_type != AttentionType.NONE:+ logging.info(f"making attention of type '{attn_type.value}' with {in_channels} in_channels")+ else:+ logging.info(f"making identity attention with {in_channels} in_channels")++ match attn_type:+ case AttentionType.VANILLA:+ return AttnBlock(in_channels, norm_type=norm_type)+ case AttentionType.NONE:+ return nn.Identity(in_channels)+ case AttentionType.LINEAR:+ raise NotImplementedError(f"Attention type {attn_type.value} is not supported yet.")+ case _:+ raise ValueError(f"Unknown attention type: {attn_type}")+++class Encoder(nn.Module):+ def __init__(+ self,+ *,+ ch,+ out_ch,+ ch_mult=(1, 2, 4, 8),+ num_res_blocks,+ attn_resolutions,+ dropout=0.0,+ resamp_with_conv=True,+ in_channels,+ resolution,+ z_channels,+ double_z=True,+ attn_type="vanilla",+ mid_block_add_attention=True,+ norm_type="group",+ causality_axis=CausalityAxis.WIDTH.value,+ **ignore_kwargs,+ ):+ super().__init__()+ self.ch = ch+ self.temb_ch = 0+ self.num_resolutions = len(ch_mult)+ self.num_res_blocks = num_res_blocks+ self.resolution = resolution+ self.in_channels = in_channels+ self.z_channels = z_channels+ self.double_z = double_z+ self.norm_type = norm_type+ # Convert string to enum if needed (for config loading)+ causality_axis = CausalityAxis.str_to_enum(causality_axis)+ self.attn_type = AttentionType.str_to_enum(attn_type)++ # downsampling+ self.conv_in = make_conv2d(+ in_channels,+ self.ch,+ kernel_size=3,+ stride=1,+ causality_axis=causality_axis,+ )++ self.non_linearity = nn.SiLU()++ curr_res = resolution+ in_ch_mult = (1,) + tuple(ch_mult)+ self.in_ch_mult = in_ch_mult+ self.down = nn.ModuleList()++ for i_level in range(self.num_resolutions):+ block = nn.ModuleList()+ attn = nn.ModuleList()+ block_in = ch * in_ch_mult[i_level]+ block_out = ch * ch_mult[i_level]++ for _ in range(self.num_res_blocks):+ block.append(+ ResnetBlock(+ in_channels=block_in,+ out_channels=block_out,+ temb_channels=self.temb_ch,+ dropout=dropout,+ norm_type=self.norm_type,+ causality_axis=causality_axis,+ )+ )+ block_in = block_out+ if curr_res in attn_resolutions:+ attn.append(make_attn(block_in, attn_type=self.attn_type, norm_type=self.norm_type))++ down = nn.Module()+ down.block = block+ down.attn = attn+ if i_level != self.num_resolutions - 1:+ down.downsample = Downsample(block_in, resamp_with_conv, causality_axis=causality_axis)+ curr_res = curr_res // 2+ self.down.append(down)++ # middle+ self.mid = nn.Module()+ self.mid.block_1 = ResnetBlock(+ in_channels=block_in,+ out_channels=block_in,+ temb_channels=self.temb_ch,+ dropout=dropout,+ norm_type=self.norm_type,+ causality_axis=causality_axis,+ )+ if mid_block_add_attention:+ self.mid.attn_1 = make_attn(block_in, attn_type=self.attn_type, norm_type=self.norm_type)+ else:+ self.mid.attn_1 = nn.Identity()+ self.mid.block_2 = ResnetBlock(+ in_channels=block_in,+ out_channels=block_in,+ temb_channels=self.temb_ch,+ dropout=dropout,+ norm_type=self.norm_type,+ causality_axis=causality_axis,+ )++ # end+ self.norm_out = Normalize(block_in, normtype=self.norm_type)+ self.conv_out = make_conv2d(+ block_in,+ 2 * z_channels if double_z else z_channels,+ kernel_size=3,+ stride=1,+ causality_axis=causality_axis,+ )++ def forward(self, x):+ """+ Forward pass through the encoder.++ Args:+ x: Input tensor of shape [batch, channels, time, n_mels]++ Returns:+ Encoded latent representation+ """+ feature_maps = [self.conv_in(x)]++ # Process each resolution level (from high to low resolution)+ for resolution_level in range(self.num_resolutions):+ # Apply residual blocks at current resolution level+ for block_idx in range(self.num_res_blocks):+ # Apply ResNet block with optional timestep embedding+ current_features = self.down[resolution_level].block[block_idx](feature_maps[-1], temb=None)++ # Apply attention if configured for this resolution level+ if len(self.down[resolution_level].attn) > 0:+ current_features = self.down[resolution_level].attn[block_idx](current_features)++ # Store processed features+ feature_maps.append(current_features)++ # Downsample spatial dimensions (except at the final resolution level)+ if resolution_level != self.num_resolutions - 1:+ downsampled_features = self.down[resolution_level].downsample(feature_maps[-1])+ feature_maps.append(downsampled_features)++ # === MIDDLE PROCESSING PHASE ===+ # Take the lowest resolution features for middle processing+ bottleneck_features = feature_maps[-1]++ # Apply first middle ResNet block+ bottleneck_features = self.mid.block_1(bottleneck_features, temb=None)++ # Apply middle attention block+ bottleneck_features = self.mid.attn_1(bottleneck_features)++ # Apply second middle ResNet block+ bottleneck_features = self.mid.block_2(bottleneck_features, temb=None)++ # === OUTPUT PHASE ===+ # Normalize the bottleneck features+ output_features = self.norm_out(bottleneck_features)++ # Apply non-linearity (SiLU activation)+ output_features = self.non_linearity(output_features)++ # Final convolution to produce latent representation+ # [batch, channels, time, n_mels] -> [batch, 2 * z_channels if double_z else z_channels, time, n_mels]+ return self.conv_out(output_features)+++class Decoder(nn.Module):+ def __init__(+ self,+ *,+ ch,+ out_ch,+ ch_mult=(1, 2, 4, 8),+ num_res_blocks,+ attn_resolutions,+ dropout=0.0,+ resamp_with_conv=True,+ in_channels,+ resolution,+ z_channels,+ give_pre_end=False,+ tanh_out=False,+ attn_type="vanilla",+ mid_block_add_attention=True,+ norm_type="group",+ causality_axis=CausalityAxis.WIDTH.value,+ **ignorekwargs,+ ):+ super().__init__()+ self.ch = ch+ self.temb_ch = 0+ self.num_resolutions = len(ch_mult)+ self.num_res_blocks = num_res_blocks+ self.resolution = resolution+ self.in_channels = in_channels+ self.out_ch = out_ch+ self.give_pre_end = give_pre_end+ self.tanh_out = tanh_out+ self.norm_type = norm_type+ self.z_channels = z_channels+ # Convert string to enum if needed (for config loading)+ causality_axis = CausalityAxis.str_to_enum(causality_axis)+ self.attn_type = AttentionType.str_to_enum(attn_type)++ # compute block_in and curr_res at lowest res+ block_in = ch * ch_mult[self.num_resolutions - 1]+ curr_res = resolution // 2 ** (self.num_resolutions - 1)+ self.z_shape = (1, z_channels, curr_res, curr_res)++ # z to block_in+ self.conv_in = make_conv2d(z_channels, block_in, kernel_size=3, stride=1, causality_axis=causality_axis)++ self.non_linearity = nn.SiLU()++ # middle+ self.mid = nn.Module()+ self.mid.block_1 = ResnetBlock(+ in_channels=block_in,+ out_channels=block_in,+ temb_channels=self.temb_ch,+ dropout=dropout,+ norm_type=self.norm_type,+ causality_axis=causality_axis,+ )+ if mid_block_add_attention:+ self.mid.attn_1 = make_attn(block_in, attn_type=self.attn_type, norm_type=self.norm_type)+ else:+ self.mid.attn_1 = nn.Identity()+ self.mid.block_2 = ResnetBlock(+ in_channels=block_in,+ out_channels=block_in,+ temb_channels=self.temb_ch,+ dropout=dropout,+ norm_type=self.norm_type,+ causality_axis=causality_axis,+ )++ # upsampling+ self.up = nn.ModuleList()+ for i_level in reversed(range(self.num_resolutions)):+ block = nn.ModuleList()+ attn = nn.ModuleList()+ block_out = ch * ch_mult[i_level]+ for _ in range(self.num_res_blocks + 1):+ block.append(+ ResnetBlock(+ in_channels=block_in,+ out_channels=block_out,+ temb_channels=self.temb_ch,+ dropout=dropout,+ norm_type=self.norm_type,+ causality_axis=causality_axis,+ )+ )+ block_in = block_out+ if curr_res in attn_resolutions:+ attn.append(make_attn(block_in, attn_type=self.attn_type, norm_type=self.norm_type))+ up = nn.Module()+ up.block = block+ up.attn = attn+ if i_level != 0:+ up.upsample = Upsample(block_in, resamp_with_conv, causality_axis=causality_axis)+ curr_res = curr_res * 2+ self.up.insert(0, up) # prepend to get consistent order++ # end+ self.norm_out = Normalize(block_in, normtype=self.norm_type)+ self.conv_out = make_conv2d(block_in, out_ch, kernel_size=3, stride=1, causality_axis=causality_axis)++ def _adjust_output_shape(self, decoded_output, target_shape):+ """+ Adjust output shape to match target dimensions for variable-length audio.++ This function handles the common case where decoded audio spectrograms need to be+ resized to match a specific target shape.++ Args:+ decoded_output: Tensor of shape (batch, channels, time, frequency)+ target_shape: Target shape tuple (batch, channels, time, frequency)++ Returns:+ Tensor adjusted to match target_shape exactly+ """+ # Current output shape: (batch, channels, time, frequency)+ _, _, current_time, current_freq = decoded_output.shape+ _, target_channels, target_time, target_freq = target_shape++ # Step 1: Crop first to avoid exceeding target dimensions+ decoded_output = decoded_output[+ :, :target_channels, : min(current_time, target_time), : min(current_freq, target_freq)+ ]++ # Step 2: Calculate padding needed for time and frequency dimensions+ time_padding_needed = target_time - decoded_output.shape[2]+ freq_padding_needed = target_freq - decoded_output.shape[3]++ # Step 3: Apply padding if needed+ if time_padding_needed > 0 or freq_padding_needed > 0:+ # PyTorch padding format: (pad_left, pad_right, pad_top, pad_bottom)+ # For audio: pad_left/right = frequency, pad_top/bottom = time+ padding = (+ 0,+ max(freq_padding_needed, 0), # frequency padding (left, right)+ 0,+ max(time_padding_needed, 0), # time padding (top, bottom)+ )+ decoded_output = F.pad(decoded_output, padding)++ # Step 4: Final safety crop to ensure exact target shape+ decoded_output = decoded_output[:, :target_channels, :target_time, :target_freq]++ return decoded_output++ def get_config(self):+ return {+ "ch": self.ch,+ "out_ch": self.out_ch,+ "ch_mult": self.ch_mult,+ "num_res_blocks": self.num_res_blocks,+ "in_channels": self.in_channels,+ "resolution": self.resolution,+ "z_channels": self.z_channels,+ }++ def forward(self, latent_features, target_shape=None):+ """+ Decode latent features back to audio spectrograms.++ Args:+ latent_features: Encoded latent representation of shape (batch, channels, height, width)+ target_shape: Optional target output shape (batch, channels, time, frequency)+ If provided, output will be cropped/padded to match this shape++ Returns:+ Reconstructed audio spectrogram of shape (batch, channels, time, frequency)+ """+ assert target_shape is not None, "Target shape is required for CausalAudioAutoencoder Decoder"++ # Transform latent features to decoder's internal feature dimension+ hidden_features = self.conv_in(latent_features)++ # Middle processing+ hidden_features = self.mid.block_1(hidden_features, temb=None)+ hidden_features = self.mid.attn_1(hidden_features)+ hidden_features = self.mid.block_2(hidden_features, temb=None)++ # Upsampling+ # Progressively increase spatial resolution from lowest to highest+ for resolution_level in reversed(range(self.num_resolutions)):+ # Apply residual blocks at current resolution level+ for block_index in range(self.num_res_blocks + 1):+ hidden_features = self.up[resolution_level].block[block_index](hidden_features, temb=None)++ if len(self.up[resolution_level].attn) > 0:+ hidden_features = self.up[resolution_level].attn[block_index](hidden_features)++ if resolution_level != 0:+ hidden_features = self.up[resolution_level].upsample(hidden_features)++ # Output+ if self.give_pre_end:+ # Return intermediate features before final processing (for debugging/analysis)+ decoded_output = hidden_features+ else:+ # Standard output path: normalize, activate, and convert to output channels+ # Final normalization layer+ hidden_features = self.norm_out(hidden_features)++ # Apply SiLU (Swish) activation function+ hidden_features = self.non_linearity(hidden_features)++ # Final convolution to map to output channels (typically 2 for stereo audio)+ decoded_output = self.conv_out(hidden_features)++ # Optional tanh activation to bound output values to [-1, 1] range+ if self.tanh_out:+ decoded_output = torch.tanh(decoded_output)++ # Adjust shape for audio data+ if target_shape is not None:+ decoded_output = self._adjust_output_shape(decoded_output, target_shape)++ return decoded_output+++class processor(nn.Module):+ def __init__(self):+ super().__init__()+ self.register_buffer("std-of-means", torch.empty(128))+ self.register_buffer("mean-of-means", torch.empty(128))++ def un_normalize(self, x):+ return (x * self.get_buffer("std-of-means").to(x)) + self.get_buffer("mean-of-means").to(x)++ def normalize(self, x):+ return (x - self.get_buffer("mean-of-means").to(x)) / self.get_buffer("std-of-means").to(x)+++class CausalAudioAutoencoder(nn.Module):+ def __init__(self, config=None):+ super().__init__()++ if config is None:+ config = self._guess_config()++ # Extract encoder and decoder configs from the new format+ model_config = config.get("model", {}).get("params", {})+ variables_config = config.get("variables", {})++ self.sampling_rate = variables_config.get(+ "sampling_rate",+ model_config.get("sampling_rate", config.get("sampling_rate", 16000)),+ )+ encoder_config = model_config.get("encoder", model_config.get("ddconfig", {}))+ decoder_config = model_config.get("decoder", encoder_config)++ # Load mel spectrogram parameters+ self.mel_bins = encoder_config.get("mel_bins", 64)+ self.mel_hop_length = model_config.get("preprocessing", {}).get("stft", {}).get("hop_length", 160)+ self.n_fft = model_config.get("preprocessing", {}).get("stft", {}).get("filter_length", 1024)++ # Store causality configuration at VAE level (not just in encoder internals)+ causality_axis_value = encoder_config.get("causality_axis", CausalityAxis.WIDTH.value)+ self.causality_axis = CausalityAxis.str_to_enum(causality_axis_value)+ self.is_causal = self.causality_axis == CausalityAxis.HEIGHT++ self.encoder = Encoder(**encoder_config)+ self.decoder = Decoder(**decoder_config)++ self.per_channel_statistics = processor()++ def _guess_config(self):+ encoder_config = {+ # Required parameters - based on ltx-video-av-1679000 model metadata+ "ch": 128,+ "out_ch": 8,+ "ch_mult": [1, 2, 4], # Based on metadata: [1, 2, 4] not [1, 2, 4, 8]+ "num_res_blocks": 2,+ "attn_resolutions": [], # Based on metadata: empty list, no attention+ "dropout": 0.0,+ "resamp_with_conv": True,+ "in_channels": 2, # stereo+ "resolution": 256,+ "z_channels": 8,+ "double_z": True,+ "attn_type": "vanilla",+ "mid_block_add_attention": False, # Based on metadata: false+ "norm_type": "pixel",+ "causality_axis": "height", # Based on metadata+ "mel_bins": 64, # Based on metadata: mel_bins = 64+ }++ decoder_config = {+ # Inherits encoder config, can override specific params+ **encoder_config,+ "out_ch": 2, # Stereo audio output (2 channels)+ "give_pre_end": False,+ "tanh_out": False,+ }++ config = {+ "_class_name": "CausalAudioAutoencoder",+ "sampling_rate": 16000,+ "model": {+ "params": {+ "encoder": encoder_config,+ "decoder": decoder_config,+ }+ },+ }++ return config++ def get_config(self):+ return {+ "sampling_rate": self.sampling_rate,+ "mel_bins": self.mel_bins,+ "mel_hop_length": self.mel_hop_length,+ "n_fft": self.n_fft,+ "causality_axis": self.causality_axis.value,+ "is_causal": self.is_causal,+ }++ def encode(self, x):+ return self.encoder(x)++ def decode(self, x, target_shape=None):+ return self.decoder(x, target_shape=target_shape)diff --git a/comfy/ldm/lightricks/vocoders/vocoder.py b/comfy/ldm/lightricks/vocoders/vocoder.pynew file mode 100644index 000000000..b1f15f2c5--- /dev/null+++ b/comfy/ldm/lightricks/vocoders/vocoder.py@@ -0,0 +1,213 @@+import torch+import torch.nn.functional as F+import torch.nn as nn+import comfy.ops+import numpy as np++ops = comfy.ops.disable_weight_init++LRELU_SLOPE = 0.1++def get_padding(kernel_size, dilation=1):+ return int((kernel_size * dilation - dilation) / 2)+++class ResBlock1(torch.nn.Module):+ def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):+ super(ResBlock1, self).__init__()+ self.convs1 = nn.ModuleList(+ [+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[0],+ padding=get_padding(kernel_size, dilation[0]),+ ),+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[1],+ padding=get_padding(kernel_size, dilation[1]),+ ),+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[2],+ padding=get_padding(kernel_size, dilation[2]),+ ),+ ]+ )++ self.convs2 = nn.ModuleList(+ [+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=1,+ padding=get_padding(kernel_size, 1),+ ),+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=1,+ padding=get_padding(kernel_size, 1),+ ),+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=1,+ padding=get_padding(kernel_size, 1),+ ),+ ]+ )++ def forward(self, x):+ for c1, c2 in zip(self.convs1, self.convs2):+ xt = F.leaky_relu(x, LRELU_SLOPE)+ xt = c1(xt)+ xt = F.leaky_relu(xt, LRELU_SLOPE)+ xt = c2(xt)+ x = xt + x+ return x+++class ResBlock2(torch.nn.Module):+ def __init__(self, channels, kernel_size=3, dilation=(1, 3)):+ super(ResBlock2, self).__init__()+ self.convs = nn.ModuleList(+ [+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[0],+ padding=get_padding(kernel_size, dilation[0]),+ ),+ ops.Conv1d(+ channels,+ channels,+ kernel_size,+ 1,+ dilation=dilation[1],+ padding=get_padding(kernel_size, dilation[1]),+ ),+ ]+ )++ def forward(self, x):+ for c in self.convs:+ xt = F.leaky_relu(x, LRELU_SLOPE)+ xt = c(xt)+ x = xt + x+ return x+++class Vocoder(torch.nn.Module):+ """+ Vocoder model for synthesizing audio from spectrograms, based on: https://github.com/jik876/hifi-gan.++ """++ def __init__(self, config=None):+ super(Vocoder, self).__init__()++ if config is None:+ config = self.get_default_config()++ resblock_kernel_sizes = config.get("resblock_kernel_sizes", [3, 7, 11])+ upsample_rates = config.get("upsample_rates", [6, 5, 2, 2, 2])+ upsample_kernel_sizes = config.get("upsample_kernel_sizes", [16, 15, 8, 4, 4])+ resblock_dilation_sizes = config.get("resblock_dilation_sizes", [[1, 3, 5], [1, 3, 5], [1, 3, 5]])+ upsample_initial_channel = config.get("upsample_initial_channel", 1024)+ stereo = config.get("stereo", True)+ resblock = config.get("resblock", "1")++ self.output_sample_rate = config.get("output_sample_rate")+ self.num_kernels = len(resblock_kernel_sizes)+ self.num_upsamples = len(upsample_rates)+ in_channels = 128 if stereo else 64+ self.conv_pre = ops.Conv1d(in_channels, upsample_initial_channel, 7, 1, padding=3)+ resblock_class = ResBlock1 if resblock == "1" else ResBlock2++ self.ups = nn.ModuleList()+ for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):+ self.ups.append(+ ops.ConvTranspose1d(+ upsample_initial_channel // (2**i),+ upsample_initial_channel // (2 ** (i + 1)),+ k,+ u,+ padding=(k - u) // 2,+ )+ )++ self.resblocks = nn.ModuleList()+ for i in range(len(self.ups)):+ ch = upsample_initial_channel // (2 ** (i + 1))+ for _, (k, d) in enumerate(zip(resblock_kernel_sizes, resblock_dilation_sizes)):+ self.resblocks.append(resblock_class(ch, k, d))++ out_channels = 2 if stereo else 1+ self.conv_post = ops.Conv1d(ch, out_channels, 7, 1, padding=3)++ self.upsample_factor = np.prod([self.ups[i].stride[0] for i in range(len(self.ups))])++ def get_default_config(self):+ """Generate default configuration for the vocoder."""++ config = {+ "resblock_kernel_sizes": [3, 7, 11],+ "upsample_rates": [6, 5, 2, 2, 2],+ "upsample_kernel_sizes": [16, 15, 8, 4, 4],+ "resblock_dilation_sizes": [[1, 3, 5], [1, 3, 5], [1, 3, 5]],+ "upsample_initial_channel": 1024,+ "stereo": True,+ "resblock": "1",+ }++ return config++ def forward(self, x):+ """+ Forward pass of the vocoder.++ Args:+ x: Input spectrogram tensor. Can be:+ - 3D: (batch_size, channels, time_steps) for mono+ - 4D: (batch_size, 2, channels, time_steps) for stereo++ Returns:+ Audio tensor of shape (batch_size, out_channels, audio_length)+ """+ if x.dim() == 4: # stereo+ assert x.shape[1] == 2, "Input must have 2 channels for stereo"+ x = torch.cat((x[:, 0, :, :], x[:, 1, :, :]), dim=1)+ x = self.conv_pre(x)+ for i in range(self.num_upsamples):+ x = F.leaky_relu(x, LRELU_SLOPE)+ x = self.ups[i](x)+ xs = None+ for j in range(self.num_kernels):+ if xs is None:+ xs = self.resblocks[i * self.num_kernels + j](x)+ else:+ xs += self.resblocks[i * self.num_kernels + j](x)+ x = xs / self.num_kernels+ x = F.leaky_relu(x)+ x = self.conv_post(x)+ x = torch.tanh(x)++ return xdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex c4f3c0639..49efd700b 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -20,6 +20,7 @@ import comfy.ldm.hunyuan3dv2_1import comfy.ldm.hunyuan3dv2_1.hunyuanditimport torchimport logging+import comfy.ldm.lightricks.av_modelfrom comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestepfrom comfy.ldm.cascade.stage_c import StageCfrom comfy.ldm.cascade.stage_b import StageB@@ -946,7 +947,7 @@ class GenmoMochi(BaseModel):class LTXV(BaseModel):def __init__(self, model_config, model_type=ModelType.FLUX, device=None):- super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lightricks.model.LTXVModel) #TODO+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lightricks.model.LTXVModel)def extra_conds(self, **kwargs):out = super().extra_conds(**kwargs)@@ -977,6 +978,60 @@ class LTXV(BaseModel):def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):return latent_image+class LTXAV(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lightricks.av_model.LTXAVModel) #TODO++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ attention_mask = kwargs.get("attention_mask", None)+ if attention_mask is not None:+ out['attention_mask'] = comfy.conds.CONDRegular(attention_mask)+ cross_attn = kwargs.get("cross_attn", None)+ if cross_attn is not None:+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)++ out['frame_rate'] = comfy.conds.CONDConstant(kwargs.get("frame_rate", 25))++ denoise_mask = kwargs.get("concat_mask", kwargs.get("denoise_mask", None))++ audio_denoise_mask = None+ if denoise_mask is not None and "latent_shapes" in kwargs:+ denoise_mask = utils.unpack_latents(denoise_mask, kwargs["latent_shapes"])+ if len(denoise_mask) > 1:+ audio_denoise_mask = denoise_mask[1]+ denoise_mask = denoise_mask[0]++ if denoise_mask is not None:+ out["denoise_mask"] = comfy.conds.CONDRegular(denoise_mask)++ if audio_denoise_mask is not None:+ out["audio_denoise_mask"] = comfy.conds.CONDRegular(audio_denoise_mask)++ keyframe_idxs = kwargs.get("keyframe_idxs", None)+ if keyframe_idxs is not None:+ out['keyframe_idxs'] = comfy.conds.CONDRegular(keyframe_idxs)++ latent_shapes = kwargs.get("latent_shapes", None)+ if latent_shapes is not None:+ out['latent_shapes'] = comfy.conds.CONDConstant(latent_shapes)++ return out++ def process_timestep(self, timestep, x, denoise_mask=None, audio_denoise_mask=None, **kwargs):+ v_timestep = timestep+ a_timestep = timestep++ if denoise_mask is not None:+ v_timestep = self.diffusion_model.patchifier.patchify(((denoise_mask) * timestep.view([timestep.shape[0]] + [1] * (denoise_mask.ndim - 1)))[:, :1])[0]+ if audio_denoise_mask is not None:+ a_timestep = self.diffusion_model.a_patchifier.patchify(((audio_denoise_mask) * timestep.view([timestep.shape[0]] + [1] * (audio_denoise_mask.ndim - 1)))[:, :1, :, :1])[0]++ return v_timestep, a_timestep++ def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):+ return latent_image+class HunyuanVideo(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 539e296ed..0853b3aec 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -305,7 +305,7 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxvdit_config = {}- dit_config["image_model"] = "ltxv"+ dit_config["image_model"] = "ltxav" if f'{key_prefix}audio_adaln_single.linear.weight' in state_dict_keys else "ltxv"dit_config["num_layers"] = count_blocks(state_dict_keys, '{}transformer_blocks.'.format(key_prefix) + '{}.')shape = state_dict['{}transformer_blocks.0.attn2.to_k.weight'.format(key_prefix)].shapedit_config["attention_head_dim"] = shape[0] // 32diff --git a/comfy/sd.py b/comfy/sd.pyindex 7de7dd9c6..32157e18b 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1041,7 +1041,8 @@ class TEModel(Enum):MISTRAL3_24B_PRUNED_FLUX2 = 15QWEN3_4B = 16QWEN3_2B = 17- JINA_CLIP_2 = 18+ GEMMA_3_12B = 18+ JINA_CLIP_2 = 19def detect_te_model(sd):@@ -1067,6 +1068,8 @@ def detect_te_model(sd):return TEModel.BYT5_SMALL_GLYPHreturn TEModel.T5_BASEif 'model.layers.0.post_feedforward_layernorm.weight' in sd:+ if 'model.layers.47.self_attn.q_norm.weight' in sd:+ return TEModel.GEMMA_3_12Bif 'model.layers.0.self_attn.q_norm.weight' in sd:return TEModel.GEMMA_3_4Breturn TEModel.GEMMA_2_2B@@ -1271,6 +1274,10 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipelif clip_type == CLIPType.KANDINSKY5_IMAGE:clip_target.clip = comfy.text_encoders.kandinsky5.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.kandinsky5.Kandinsky5TokenizerImage+ elif clip_type == CLIPType.LTXV:+ clip_target.clip = comfy.text_encoders.lt.ltxav_te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.lt.LTXAVGemmaTokenizer+ tokenizer_data["spiece_model"] = clip_data[0].get("spiece_model", None)elif clip_type == CLIPType.NEWBIE:clip_target.clip = comfy.text_encoders.newbie.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.newbie.NewBieTokenizerdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 1888f35ba..ee9a79001 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -836,6 +836,21 @@ class LTXV(supported_models_base.BASE):t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.lt.LTXVT5Tokenizer, comfy.text_encoders.lt.ltxv_te(**t5_detect))+class LTXAV(LTXV):+ unet_config = {+ "image_model": "ltxav",+ }++ latent_format = latent_formats.LTXAV++ def __init__(self, unet_config):+ super().__init__(unet_config)+ self.memory_usage_factor = 0.055 # TODO++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.LTXAV(self, device=device)+ return out+class HunyuanVideo(supported_models_base.BASE):unet_config = {"image_model": "hunyuan_video",@@ -1536,6 +1551,6 @@ class Kandinsky5Image(Kandinsky5):return supported_models_base.ClipTarget(comfy.text_encoders.kandinsky5.Kandinsky5TokenizerImage, comfy.text_encoders.kandinsky5.te(**hunyuan_detect))-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5]+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5]models += [SVD_img2vid]diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex faa4e1de8..76731576b 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -7,6 +7,7 @@ import mathfrom comfy.ldm.modules.attention import optimized_attention_for_deviceimport comfy.model_managementimport comfy.ldm.common_dit+import comfy.clip_modelfrom . import qwen_vl@@ -188,6 +189,31 @@ class Gemma3_4B_Config:rope_scale = [8.0, 1.0]final_norm: bool = True+@dataclass+class Gemma3_12B_Config:+ vocab_size: int = 262208+ hidden_size: int = 3840+ intermediate_size: int = 15360+ num_hidden_layers: int = 48+ num_attention_heads: int = 16+ num_key_value_heads: int = 8+ max_position_embeddings: int = 131072+ rms_norm_eps: float = 1e-6+ rope_theta = [1000000.0, 10000.0]+ transformer_type: str = "gemma3"+ head_dim = 256+ rms_norm_add = True+ mlp_activation = "gelu_pytorch_tanh"+ qkv_bias = False+ rope_dims = None+ q_norm = "gemma3"+ k_norm = "gemma3"+ sliding_attention = [1024, 1024, 1024, 1024, 1024, False]+ rope_scale = [8.0, 1.0]+ final_norm: bool = True+ vision_config = {"num_channels": 3, "hidden_act": "gelu_pytorch_tanh", "hidden_size": 1152, "image_size": 896, "intermediate_size": 4304, "model_type": "siglip_vision_model", "num_attention_heads": 16, "num_hidden_layers": 27, "patch_size": 14}+ mm_tokens_per_image = 256+class RMSNorm(nn.Module):def __init__(self, dim: int, eps: float = 1e-5, add=False, device=None, dtype=None):super().__init__()@@ -520,6 +546,41 @@ class Llama2_(nn.Module):return x, intermediate++class Gemma3MultiModalProjector(torch.nn.Module):+ def __init__(self, config, dtype, device, operations):+ super().__init__()++ self.mm_input_projection_weight = nn.Parameter(+ torch.empty(config.vision_config["hidden_size"], config.hidden_size, device=device, dtype=dtype)+ )++ self.mm_soft_emb_norm = RMSNorm(config.vision_config["hidden_size"], eps=config.rms_norm_eps, add=config.rms_norm_add, device=device, dtype=dtype)++ self.patches_per_image = int(config.vision_config["image_size"] // config.vision_config["patch_size"])+ self.tokens_per_side = int(config.mm_tokens_per_image**0.5)+ self.kernel_size = self.patches_per_image // self.tokens_per_side+ self.avg_pool = nn.AvgPool2d(kernel_size=self.kernel_size, stride=self.kernel_size)++ def forward(self, vision_outputs: torch.Tensor):+ batch_size, _, seq_length = vision_outputs.shape++ reshaped_vision_outputs = vision_outputs.transpose(1, 2)+ reshaped_vision_outputs = reshaped_vision_outputs.reshape(+ batch_size, seq_length, self.patches_per_image, self.patches_per_image+ )+ reshaped_vision_outputs = reshaped_vision_outputs.contiguous()++ pooled_vision_outputs = self.avg_pool(reshaped_vision_outputs)+ pooled_vision_outputs = pooled_vision_outputs.flatten(2)+ pooled_vision_outputs = pooled_vision_outputs.transpose(1, 2)++ normed_vision_outputs = self.mm_soft_emb_norm(pooled_vision_outputs)++ projected_vision_outputs = torch.matmul(normed_vision_outputs, comfy.model_management.cast_to_device(self.mm_input_projection_weight, device=normed_vision_outputs.device, dtype=normed_vision_outputs.dtype))+ return projected_vision_outputs.type_as(vision_outputs)++class BaseLlama:def get_input_embeddings(self):return self.model.embed_tokens@@ -636,3 +697,21 @@ class Gemma3_4B(BaseLlama, torch.nn.Module):self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)self.dtype = dtype++class Gemma3_12B(BaseLlama, torch.nn.Module):+ def __init__(self, config_dict, dtype, device, operations):+ super().__init__()+ config = Gemma3_12B_Config(**config_dict)+ self.num_layers = config.num_hidden_layers++ self.model = Llama2_(config, device=device, dtype=dtype, ops=operations)+ self.multi_modal_projector = Gemma3MultiModalProjector(config, dtype, device, operations)+ self.vision_model = comfy.clip_model.CLIPVision(config.vision_config, dtype, device, operations)+ self.dtype = dtype+ self.image_size = config.vision_config["image_size"]++ def preprocess_embed(self, embed, device):+ if embed["type"] == "image":+ image = comfy.clip_model.clip_preprocess(embed["data"], size=self.image_size, mean=[0.5, 0.5, 0.5], std=[0.5, 0.5, 0.5], crop=True)+ return self.multi_modal_projector(self.vision_model(image.to(device, dtype=torch.float32))[0]), None+ return None, Nonediff --git a/comfy/text_encoders/lt.py b/comfy/text_encoders/lt.pyindex 48ea67e67..2c2d453e8 100644--- a/comfy/text_encoders/lt.py+++ b/comfy/text_encoders/lt.py@@ -1,7 +1,11 @@from comfy import sd1_clipimport osfrom transformers import T5TokenizerFast+from .spiece_tokenizer import SPieceTokenizerimport comfy.text_encoders.genmo+from comfy.ldm.lightricks.embeddings_connector import Embeddings1DConnector+import torch+import comfy.utilsclass T5XXLTokenizer(sd1_clip.SDTokenizer):def __init__(self, embedding_directory=None, tokenizer_data={}):@@ -16,3 +20,110 @@ class LTXVT5Tokenizer(sd1_clip.SD1Tokenizer):def ltxv_te(*args, **kwargs):return comfy.text_encoders.genmo.mochi_te(*args, **kwargs)+++class Gemma3_12BTokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer = tokenizer_data.get("spiece_model", None)+ super().__init__(tokenizer, pad_with_end=False, embedding_size=3840, embedding_key='gemma3_12b', tokenizer_class=SPieceTokenizer, has_end_token=False, pad_to_max_length=False, max_length=99999999, min_length=1, tokenizer_args={"add_bos": True, "add_eos": False}, tokenizer_data=tokenizer_data)++ def state_dict(self):+ return {"spiece_model": self.tokenizer.serialize_model()}++class LTXAVGemmaTokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="gemma3_12b", tokenizer=Gemma3_12BTokenizer)++class Gemma3_12BModel(sd1_clip.SDClipModel):+ def __init__(self, device="cpu", layer="all", layer_idx=None, dtype=None, attention_mask=True, model_options={}):+ llama_scaled_fp8 = model_options.get("gemma_scaled_fp8", None)+ if llama_scaled_fp8 is not None:+ model_options = model_options.copy()+ model_options["scaled_fp8"] = llama_scaled_fp8++ super().__init__(device=device, layer=layer, layer_idx=layer_idx, textmodel_json_config={}, dtype=dtype, special_tokens={"start": 2, "pad": 0}, layer_norm_hidden_state=False, model_class=comfy.text_encoders.llama.Gemma3_12B, enable_attention_masks=attention_mask, return_attention_masks=attention_mask, model_options=model_options)++ def tokenize_with_weights(self, text, return_word_ids=False, llama_template="{}", image_embeds=None, **kwargs):+ text = llama_template.format(text)+ text_tokens = super().tokenize_with_weights(text, return_word_ids)+ embed_count = 0+ for k in text_tokens:+ tt = text_tokens[k]+ for r in tt:+ for i in range(len(r)):+ if r[i][0] == 262144:+ if image_embeds is not None and embed_count < image_embeds.shape[0]:+ r[i] = ({"type": "embedding", "data": image_embeds[embed_count], "original_type": "image"},) + r[i][1:]+ embed_count += 1+ return text_tokens++class LTXAVTEModel(torch.nn.Module):+ def __init__(self, dtype_llama=None, device="cpu", dtype=None, model_options={}):+ super().__init__()+ self.dtypes = set()+ self.dtypes.add(dtype)++ self.gemma3_12b = Gemma3_12BModel(device=device, dtype=dtype_llama, model_options=model_options, layer="all", layer_idx=None)+ self.dtypes.add(dtype_llama)++ operations = self.gemma3_12b.operations # TODO+ self.text_embedding_projection = operations.Linear(3840 * 49, 3840, bias=False, dtype=dtype, device=device)++ self.audio_embeddings_connector = Embeddings1DConnector(+ split_rope=True,+ double_precision_rope=True,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ self.video_embeddings_connector = Embeddings1DConnector(+ split_rope=True,+ double_precision_rope=True,+ dtype=dtype,+ device=device,+ operations=operations,+ )++ def set_clip_options(self, options):+ self.gemma3_12b.set_clip_options(options)++ def reset_clip_options(self):+ self.gemma3_12b.reset_clip_options()++ def encode_token_weights(self, token_weight_pairs):+ token_weight_pairs = token_weight_pairs["gemma3_12b"]++ out, pooled, extra = self.gemma3_12b.encode_token_weights(token_weight_pairs)+ out_device = out.device+ out = out.movedim(1, -1).to(self.text_embedding_projection.weight.device)+ out = 8.0 * (out - out.mean(dim=(1, 2), keepdim=True)) / (out.amax(dim=(1, 2), keepdim=True) - out.amin(dim=(1, 2), keepdim=True) + 1e-6)+ out = out.reshape((out.shape[0], out.shape[1], -1))+ out = self.text_embedding_projection(out)+ out_vid = self.video_embeddings_connector(out)[0]+ out_audio = self.audio_embeddings_connector(out)[0]+ out = torch.concat((out_vid, out_audio), dim=-1)++ return out.to(out_device), pooled++ def load_sd(self, sd):+ if "model.layers.47.self_attn.q_norm.weight" in sd:+ return self.gemma3_12b.load_sd(sd)+ else:+ sdo = comfy.utils.state_dict_prefix_replace(sd, {"text_embedding_projection.aggregate_embed.weight": "text_embedding_projection.weight", "model.diffusion_model.video_embeddings_connector.": "video_embeddings_connector.", "model.diffusion_model.audio_embeddings_connector.": "audio_embeddings_connector."}, filter_keys=True)+ if len(sdo) == 0:+ sdo = sd++ return self.load_state_dict(sdo, strict=False)+++def ltxav_te(dtype_llama=None, llama_scaled_fp8=None):+ class LTXAVTEModel_(LTXAVTEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if llama_scaled_fp8 is not None and "llama_scaled_fp8" not in model_options:+ model_options = model_options.copy()+ model_options["llama_scaled_fp8"] = llama_scaled_fp8+ if dtype_llama is not None:+ dtype = dtype_llama+ super().__init__(dtype_llama=dtype_llama, device=device, dtype=dtype, model_options=model_options)+ return LTXAVTEModel_diff --git a/comfy/utils.py b/comfy/utils.pyindex e4162d7ac..ffa98c9b1 100644--- a/comfy/utils.py+++ b/comfy/utils.py@@ -1198,7 +1198,7 @@ def unpack_latents(combined_latent, latent_shapes):combined_latent = combined_latent[:, :, cut:]output_tensors.append(tens.reshape([tens.shape[0]] + list(shape)[1:]))else:- output_tensors = combined_latent+ output_tensors = [combined_latent]return output_tensorsdef detect_layer_quantization(state_dict, prefix):diff --git a/comfy_extras/nodes_audio.py b/comfy_extras/nodes_audio.pyindex c7916443c..94ad5e8a8 100644--- a/comfy_extras/nodes_audio.py+++ b/comfy_extras/nodes_audio.py@@ -112,7 +112,7 @@ class VAEDecodeAudio(IO.ComfyNode):std = torch.std(audio, dim=[1,2], keepdim=True) * 5.0std[std < 1.0] = 1.0audio /= std- return IO.NodeOutput({"waveform": audio, "sample_rate": 44100})+ return IO.NodeOutput({"waveform": audio, "sample_rate": 44100 if "sample_rate" not in samples else samples["sample_rate"]})decode = execute # TODO: removediff --git a/comfy_extras/nodes_hunyuan.py b/comfy_extras/nodes_hunyuan.pyindex 32be182f1..ceff657d3 100644--- a/comfy_extras/nodes_hunyuan.py+++ b/comfy_extras/nodes_hunyuan.py@@ -5,7 +5,9 @@ import comfy.model_managementfrom typing_extensions import overridefrom comfy_api.latest import ComfyExtension, iofrom comfy.ldm.hunyuan_video.upsampler import HunyuanVideo15SRModel+from comfy.ldm.lightricks.latent_upsampler import LatentUpsamplerimport folder_paths+import jsonclass CLIPTextEncodeHunyuanDiT(io.ComfyNode):@classmethod@@ -186,7 +188,7 @@ class LatentUpscaleModelLoader(io.ComfyNode):@classmethoddef execute(cls, model_name) -> io.NodeOutput:model_path = folder_paths.get_full_path_or_raise("latent_upscale_models", model_name)- sd = comfy.utils.load_torch_file(model_path, safe_load=True)+ sd, metadata = comfy.utils.load_torch_file(model_path, safe_load=True, return_metadata=True)if "blocks.0.block.0.conv.weight" in sd:config = {@@ -197,6 +199,8 @@ class LatentUpscaleModelLoader(io.ComfyNode):"global_residual": False,}model_type = "720p"+ model = HunyuanVideo15SRModel(model_type, config)+ model.load_sd(sd)elif "up.0.block.0.conv1.conv.weight" in sd:sd = {key.replace("nin_shortcut", "nin_shortcut.conv", 1): value for key, value in sd.items()}config = {@@ -205,9 +209,12 @@ class LatentUpscaleModelLoader(io.ComfyNode):"block_out_channels": tuple(sd[f"up.{i}.block.0.conv1.conv.weight"].shape[0] for i in range(len([k for k in sd.keys() if k.startswith("up.") and k.endswith(".block.0.conv1.conv.weight")]))),}model_type = "1080p"-- model = HunyuanVideo15SRModel(model_type, config)- model.load_sd(sd)+ model = HunyuanVideo15SRModel(model_type, config)+ model.load_sd(sd)+ elif "post_upsample_res_blocks.0.conv2.bias" in sd:+ config = json.loads(metadata["config"])+ model = LatentUpsampler.from_config(config).to(dtype=comfy.model_management.vae_dtype(allowed_dtypes=[torch.bfloat16, torch.float32]))+ model.load_state_dict(sd)return io.NodeOutput(model)diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.pyindex 50da5f4eb..b91a22309 100644--- a/comfy_extras/nodes_lt.py+++ b/comfy_extras/nodes_lt.py@@ -81,6 +81,59 @@ class LTXVImgToVideo(io.ComfyNode):generate = execute # TODO: remove+class LTXVImgToVideoInplace(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="LTXVImgToVideoInplace",+ category="conditioning/video_models",+ inputs=[+ io.Vae.Input("vae"),+ io.Image.Input("image"),+ io.Latent.Input("latent"),+ io.Float.Input("strength", default=1.0, min=0.0, max=1.0),+ io.Boolean.Input("bypass", default=False, tooltip="Bypass the conditioning.")+ ],+ outputs=[+ io.Latent.Output(display_name="latent"),+ ],+ )++ @classmethod+ def execute(cls, vae, image, latent, strength, bypass=False) -> io.NodeOutput:+ if bypass:+ return (latent,)++ samples = latent["samples"]+ _, height_scale_factor, width_scale_factor = (+ vae.downscale_index_formula+ )++ batch, _, latent_frames, latent_height, latent_width = samples.shape+ width = latent_width * width_scale_factor+ height = latent_height * height_scale_factor++ if image.shape[1] != height or image.shape[2] != width:+ pixels = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)+ else:+ pixels = image+ encode_pixels = pixels[:, :, :, :3]+ t = vae.encode(encode_pixels)++ samples[:, :, :t.shape[2]] = t++ conditioning_latent_frames_mask = torch.ones(+ (batch, 1, latent_frames, 1, 1),+ dtype=torch.float32,+ device=samples.device,+ )+ conditioning_latent_frames_mask[:, :, :t.shape[2]] = 1.0 - strength++ return io.NodeOutput({"samples": samples, "noise_mask": conditioning_latent_frames_mask})++ generate = execute # TODO: remove++def conditioning_get_any_value(conditioning, key, default=None):for t in conditioning:if key in t[1]:@@ -106,12 +159,12 @@ def get_keyframe_idxs(cond):keyframe_idxs = conditioning_get_any_value(cond, "keyframe_idxs", None)if keyframe_idxs is None:return None, 0- num_keyframes = torch.unique(keyframe_idxs[:, 0]).shape[0]+ # keyframe_idxs contains start/end positions (last dimension), checking for unqiue values only for start+ num_keyframes = torch.unique(keyframe_idxs[:, 0, :, 0]).shape[0]return keyframe_idxs, num_keyframesclass LTXVAddGuide(io.ComfyNode):- NUM_PREFIX_FRAMES = 2- PATCHIFIER = SymmetricPatchifier(1)+ PATCHIFIER = SymmetricPatchifier(1, start_end=True)@classmethoddef define_schema(cls):@@ -182,26 +235,35 @@ class LTXVAddGuide(io.ComfyNode):return node_helpers.conditioning_set_values(cond, {"keyframe_idxs": keyframe_idxs})@classmethod- def append_keyframe(cls, positive, negative, frame_idx, latent_image, noise_mask, guiding_latent, strength, scale_factors):- _, latent_idx = cls.get_latent_index(- cond=positive,- latent_length=latent_image.shape[2],- guide_length=guiding_latent.shape[2],- frame_idx=frame_idx,- scale_factors=scale_factors,- )- noise_mask[:, :, latent_idx:latent_idx + guiding_latent.shape[2]] = 1.0+ def append_keyframe(cls, positive, negative, frame_idx, latent_image, noise_mask, guiding_latent, strength, scale_factors, guide_mask=None, in_channels=128):+ if latent_image.shape[1] != in_channels or guiding_latent.shape[1] != in_channels:+ raise ValueError("Adding guide to a combined AV latent is not supported.")positive = cls.add_keyframe_index(positive, frame_idx, guiding_latent, scale_factors)negative = cls.add_keyframe_index(negative, frame_idx, guiding_latent, scale_factors)- mask = torch.full(- (noise_mask.shape[0], 1, guiding_latent.shape[2], noise_mask.shape[3], noise_mask.shape[4]),- 1.0 - strength,- dtype=noise_mask.dtype,- device=noise_mask.device,- )+ if guide_mask is not None:+ target_h = max(noise_mask.shape[3], guide_mask.shape[3])+ target_w = max(noise_mask.shape[4], guide_mask.shape[4])+ if noise_mask.shape[3] == 1 or noise_mask.shape[4] == 1:+ noise_mask = noise_mask.expand(-1, -1, -1, target_h, target_w)++ if guide_mask.shape[3] == 1 or guide_mask.shape[4] == 1:+ guide_mask = guide_mask.expand(-1, -1, -1, target_h, target_w)+ mask = guide_mask - strength+ else:+ mask = torch.full(+ (noise_mask.shape[0], 1, guiding_latent.shape[2], noise_mask.shape[3], noise_mask.shape[4]),+ 1.0 - strength,+ dtype=noise_mask.dtype,+ device=noise_mask.device,+ )+ # This solves audio video combined latent case where latent_image has audio latent concatenated+ # in channel dimension with video latent. The solution is to pad guiding latent accordingly.+ if latent_image.shape[1] > guiding_latent.shape[1]:+ pad_len = latent_image.shape[1] - guiding_latent.shape[1]+ guiding_latent = torch.nn.functional.pad(guiding_latent, pad=(0, 0, 0, 0, 0, 0, 0, pad_len), value=0)latent_image = torch.cat([latent_image, guiding_latent], dim=2)noise_mask = torch.cat([noise_mask, mask], dim=2)return positive, negative, latent_image, noise_mask@@ -238,31 +300,15 @@ class LTXVAddGuide(io.ComfyNode):frame_idx, latent_idx = cls.get_latent_index(positive, latent_length, len(image), frame_idx, scale_factors)assert latent_idx + t.shape[2] <= latent_length, "Conditioning frames exceed the length of the latent sequence."- num_prefix_frames = min(cls.NUM_PREFIX_FRAMES, t.shape[2])-positive, negative, latent_image, noise_mask = cls.append_keyframe(positive,negative,frame_idx,- latent_image,- noise_mask,- t[:, :, :num_prefix_frames],- strength,- scale_factors,- )-- latent_idx += num_prefix_frames-- t = t[:, :, num_prefix_frames:]- if t.shape[2] == 0:- return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})-- latent_image, noise_mask = cls.replace_latent_frames(latent_image,noise_mask,t,- latent_idx,strength,+ scale_factors,)return io.NodeOutput(positive, negative, {"samples": latent_image, "noise_mask": noise_mask})@@ -507,18 +553,90 @@ class LTXVPreprocess(io.ComfyNode):preprocess = execute # TODO: remove++import comfy.nested_tensor+class LTXVConcatAVLatent(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="LTXVConcatAVLatent",+ category="latent/video/ltxv",+ inputs=[+ io.Latent.Input("video_latent"),+ io.Latent.Input("audio_latent"),+ ],+ outputs=[+ io.Latent.Output(display_name="latent"),+ ],+ )++ @classmethod+ def execute(cls, video_latent, audio_latent) -> io.NodeOutput:+ output = {}+ output.update(video_latent)+ output.update(audio_latent)+ video_noise_mask = video_latent.get("noise_mask", None)+ audio_noise_mask = audio_latent.get("noise_mask", None)++ if video_noise_mask is not None or audio_noise_mask is not None:+ if video_noise_mask is None:+ video_noise_mask = torch.ones_like(video_latent["samples"])+ if audio_noise_mask is None:+ audio_noise_mask = torch.ones_like(audio_latent["samples"])+ output["noise_mask"] = comfy.nested_tensor.NestedTensor((video_noise_mask, audio_noise_mask))++ output["samples"] = comfy.nested_tensor.NestedTensor((video_latent["samples"], audio_latent["samples"]))++ return io.NodeOutput(output)+++class LTXVSeparateAVLatent(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="LTXVSeparateAVLatent",+ category="latent/video/ltxv",+ description="LTXV Separate AV Latent",+ inputs=[+ io.Latent.Input("av_latent"),+ ],+ outputs=[+ io.Latent.Output(display_name="video_latent"),+ io.Latent.Output(display_name="audio_latent"),+ ],+ )++ @classmethod+ def execute(cls, av_latent) -> io.NodeOutput:+ latents = av_latent["samples"].unbind()+ video_latent = av_latent.copy()+ video_latent["samples"] = latents[0]+ audio_latent = av_latent.copy()+ audio_latent["samples"] = latents[1]+ if "noise_mask" in av_latent:+ masks = av_latent["noise_mask"]+ if masks is not None:+ masks = masks.unbind()+ video_latent["noise_mask"] = masks[0]+ audio_latent["noise_mask"] = masks[1]+ return io.NodeOutput(video_latent, audio_latent)++class LtxvExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[io.ComfyNode]]:return [EmptyLTXVLatentVideo,LTXVImgToVideo,+ LTXVImgToVideoInplace,ModelSamplingLTXV,LTXVConditioning,LTXVScheduler,LTXVAddGuide,LTXVPreprocess,LTXVCropGuides,+ LTXVConcatAVLatent,+ LTXVSeparateAVLatent,]diff --git a/comfy_extras/nodes_lt_audio.py b/comfy_extras/nodes_lt_audio.pynew file mode 100644index 000000000..b0b7000ef--- /dev/null+++ b/comfy_extras/nodes_lt_audio.py@@ -0,0 +1,183 @@+import folder_paths+import comfy.utils+import comfy.model_management+import torch++from comfy.ldm.lightricks.vae.audio_vae import AudioVAE+from comfy_api.latest import ComfyExtension, io+++class LTXVAudioVAELoader(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="LTXVAudioVAELoader",+ display_name="LTXV Audio VAE Loader",+ category="audio",+ inputs=[+ io.Combo.Input(+ "ckpt_name",+ options=folder_paths.get_filename_list("checkpoints"),+ tooltip="Audio VAE checkpoint to load.",+ )+ ],+ outputs=[io.Vae.Output(display_name="Audio VAE")],+ )++ @classmethod+ def execute(cls, ckpt_name: str) -> io.NodeOutput:+ ckpt_path = folder_paths.get_full_path_or_raise("checkpoints", ckpt_name)+ sd, metadata = comfy.utils.load_torch_file(ckpt_path, return_metadata=True)+ return io.NodeOutput(AudioVAE(sd, metadata))+++class LTXVAudioVAEEncode(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="LTXVAudioVAEEncode",+ display_name="LTXV Audio VAE Encode",+ category="audio",+ inputs=[+ io.Audio.Input("audio", tooltip="The audio to be encoded."),+ io.Vae.Input(+ id="audio_vae",+ display_name="Audio VAE",+ tooltip="The Audio VAE model to use for encoding.",+ ),+ ],+ outputs=[io.Latent.Output(display_name="Audio Latent")],+ )++ @classmethod+ def execute(cls, audio, audio_vae: AudioVAE) -> io.NodeOutput:+ audio_latents = audio_vae.encode(audio)+ return io.NodeOutput(+ {+ "samples": audio_latents,+ "sample_rate": int(audio_vae.sample_rate),+ "type": "audio",+ }+ )+++class LTXVAudioVAEDecode(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="LTXVAudioVAEDecode",+ display_name="LTXV Audio VAE Decode",+ category="audio",+ inputs=[+ io.Latent.Input("samples", tooltip="The latent to be decoded."),+ io.Vae.Input(+ id="audio_vae",+ display_name="Audio VAE",+ tooltip="The Audio VAE model used for decoding the latent.",+ ),+ ],+ outputs=[io.Audio.Output(display_name="Audio")],+ )++ @classmethod+ def execute(cls, samples, audio_vae: AudioVAE) -> io.NodeOutput:+ audio_latent = samples["samples"]+ if audio_latent.is_nested:+ audio_latent = audio_latent.unbind()[-1]+ audio = audio_vae.decode(audio_latent).to(audio_latent.device)+ output_audio_sample_rate = audio_vae.output_sample_rate+ return io.NodeOutput(+ {+ "waveform": audio,+ "sample_rate": int(output_audio_sample_rate),+ }+ )+++class LTXVEmptyLatentAudio(io.ComfyNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ return io.Schema(+ node_id="LTXVEmptyLatentAudio",+ display_name="LTXV Empty Latent Audio",+ category="latent/audio",+ inputs=[+ io.Int.Input(+ "frames_number",+ default=97,+ min=1,+ max=1000,+ step=1,+ display_mode=io.NumberDisplay.number,+ tooltip="Number of frames.",+ ),+ io.Int.Input(+ "frame_rate",+ default=25,+ min=1,+ max=1000,+ step=1,+ display_mode=io.NumberDisplay.number,+ tooltip="Number of frames per second.",+ ),+ io.Int.Input(+ "batch_size",+ default=1,+ min=1,+ max=4096,+ display_mode=io.NumberDisplay.number,+ tooltip="The number of latent audio samples in the batch.",+ ),+ io.Vae.Input(+ id="audio_vae",+ display_name="Audio VAE",+ tooltip="The Audio VAE model to get configuration from.",+ ),+ ],+ outputs=[io.Latent.Output(display_name="Latent")],+ )++ @classmethod+ def execute(+ cls,+ frames_number: int,+ frame_rate: int,+ batch_size: int,+ audio_vae: AudioVAE,+ ) -> io.NodeOutput:+ """Generate empty audio latents matching the reference pipeline structure."""++ assert audio_vae is not None, "Audio VAE model is required"++ z_channels = audio_vae.latent_channels+ audio_freq = audio_vae.latent_frequency_bins+ sampling_rate = int(audio_vae.sample_rate)++ num_audio_latents = audio_vae.num_of_latents_from_frames(frames_number, frame_rate)++ audio_latents = torch.zeros(+ (batch_size, z_channels, num_audio_latents, audio_freq),+ device=comfy.model_management.intermediate_device(),+ )++ return io.NodeOutput(+ {+ "samples": audio_latents,+ "sample_rate": sampling_rate,+ "type": "audio",+ }+ )+++class LTXVAudioExtension(ComfyExtension):+ async def get_node_list(self) -> list[type[io.ComfyNode]]:+ return [+ LTXVAudioVAELoader,+ LTXVAudioVAEEncode,+ LTXVAudioVAEDecode,+ LTXVEmptyLatentAudio,+ ]+++async def comfy_entrypoint() -> ComfyExtension:+ return LTXVAudioExtension()diff --git a/comfy_extras/nodes_lt_upsampler.py b/comfy_extras/nodes_lt_upsampler.pynew file mode 100644index 000000000..f99ba13fb--- /dev/null+++ b/comfy_extras/nodes_lt_upsampler.py@@ -0,0 +1,75 @@+from comfy import model_management+import math++class LTXVLatentUpsampler:+ """+ Upsamples a video latent by a factor of 2.+ """++ @classmethod+ def INPUT_TYPES(s):+ return {+ "required": {+ "samples": ("LATENT",),+ "upscale_model": ("LATENT_UPSCALE_MODEL",),+ "vae": ("VAE",),+ }+ }++ RETURN_TYPES = ("LATENT",)+ FUNCTION = "upsample_latent"+ CATEGORY = "latent/video"+ EXPERIMENTAL = True++ def upsample_latent(+ self,+ samples: dict,+ upscale_model,+ vae,+ ) -> tuple:+ """+ Upsample the input latent using the provided model.++ Args:+ samples (dict): Input latent samples+ upscale_model (LatentUpsampler): Loaded upscale model+ vae: VAE model for normalization+ auto_tiling (bool): Whether to automatically tile the input for processing++ Returns:+ tuple: Tuple containing the upsampled latent+ """+ device = model_management.get_torch_device()+ memory_required = model_management.module_size(upscale_model)++ model_dtype = next(upscale_model.parameters()).dtype+ latents = samples["samples"]+ input_dtype = latents.dtype++ memory_required += math.prod(latents.shape) * 3000.0 # TODO: more accurate+ model_management.free_memory(memory_required, device)++ try:+ upscale_model.to(device) # TODO: use the comfy model management system.++ latents = latents.to(dtype=model_dtype, device=device)++ """Upsample latents without tiling."""+ latents = vae.first_stage_model.per_channel_statistics.un_normalize(latents)+ upsampled_latents = upscale_model(latents)+ finally:+ upscale_model.cpu()++ upsampled_latents = vae.first_stage_model.per_channel_statistics.normalize(+ upsampled_latents+ )+ upsampled_latents = upsampled_latents.to(dtype=input_dtype, device=model_management.intermediate_device())+ return_dict = samples.copy()+ return_dict["samples"] = upsampled_latents+ return_dict.pop("noise_mask", None)+ return (return_dict,)+++NODE_CLASS_MAPPINGS = {+ "LTXVLatentUpsampler": LTXVLatentUpsampler,+}diff --git a/nodes.py b/nodes.pyindex 662907ae6..56b74ebe3 100644--- a/nodes.py+++ b/nodes.py@@ -295,7 +295,11 @@ class VAEDecode:DESCRIPTION = "Decodes latent images back into pixel space images."def decode(self, vae, samples):- images = vae.decode(samples["samples"])+ latent = samples["samples"]+ if latent.is_nested:+ latent = latent.unbind()[0]++ images = vae.decode(latent)if len(images.shape) == 5: #Combine batchesimages = images.reshape(-1, images.shape[-3], images.shape[-2], images.shape[-1])return (images, )@@ -970,7 +974,7 @@ class DualCLIPLoader:def INPUT_TYPES(s):return {"required": { "clip_name1": (folder_paths.get_filename_list("text_encoders"), ),"clip_name2": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["sdxl", "sd3", "flux", "hunyuan_video", "hidream", "hunyuan_image", "hunyuan_video_15", "kandinsky5", "kandinsky5_image", "newbie"], ),+ "type": (["sdxl", "sd3", "flux", "hunyuan_video", "hidream", "hunyuan_image", "hunyuan_video_15", "kandinsky5", "kandinsky5_image", "ltxv", "newbie"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -2331,6 +2335,8 @@ async def init_builtin_extra_nodes():"nodes_mochi.py","nodes_slg.py","nodes_mahiro.py",+ "nodes_lt_upsampler.py",+ "nodes_lt_audio.py","nodes_lt.py","nodes_hooks.py","nodes_load_3d.py",diff --git a/pyproject.toml b/pyproject.tomlindex 60378de1e..a7d159be9 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -3,7 +3,7 @@ name = "ComfyUI"version = "0.7.0"readme = "README.md"license = { file = "LICENSE" }-requires-python = ">=3.9"+requires-python = ">=3.10"[project.urls]homepage = "https://www.comfy.org/"