The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support Lightricks LTX-Video model.
comfyanonymous5e16f1d24beb
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex f9fd16d8c..199a3641d 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -216,3 +216,7 @@ class Mochi(LatentFormat):latents_mean = self.latents_mean.to(latent.device, latent.dtype)latents_std = self.latents_std.to(latent.device, latent.dtype)return latent * latents_std / self.scale_factor + latents_mean++class LTXV(LatentFormat):+ latent_channels = 128+diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.pynew file mode 100644index 000000000..2792384d5--- /dev/null+++ b/comfy/ldm/lightricks/model.py@@ -0,0 +1,502 @@+import torch+from torch import nn+import comfy.ldm.modules.attention+from comfy.ldm.genmo.joint_model.layers import RMSNorm+import comfy.ldm.common_dit+from einops import rearrange+import math+from typing import Dict, Optional, Tuple++from .symmetric_patchifier import SymmetricPatchifier+++def get_timestep_embedding(+ timesteps: torch.Tensor,+ embedding_dim: int,+ flip_sin_to_cos: bool = False,+ downscale_freq_shift: float = 1,+ scale: float = 1,+ max_period: int = 10000,+):+ """+ This matches the implementation in Denoising Diffusion Probabilistic Models: Create sinusoidal timestep embeddings.++ Args+ timesteps (torch.Tensor):+ a 1-D Tensor of N indices, one per batch element. These may be fractional.+ embedding_dim (int):+ the dimension of the output.+ flip_sin_to_cos (bool):+ Whether the embedding order should be `cos, sin` (if True) or `sin, cos` (if False)+ downscale_freq_shift (float):+ Controls the delta between frequencies between dimensions+ scale (float):+ Scaling factor applied to the embeddings.+ max_period (int):+ Controls the maximum frequency of the embeddings+ Returns+ torch.Tensor: an [N x dim] Tensor of positional embeddings.+ """+ assert len(timesteps.shape) == 1, "Timesteps should be a 1d-array"++ half_dim = embedding_dim // 2+ exponent = -math.log(max_period) * torch.arange(+ start=0, end=half_dim, dtype=torch.float32, device=timesteps.device+ )+ exponent = exponent / (half_dim - downscale_freq_shift)++ emb = torch.exp(exponent)+ emb = timesteps[:, None].float() * emb[None, :]++ # scale embeddings+ emb = scale * emb++ # concat sine and cosine embeddings+ emb = torch.cat([torch.sin(emb), torch.cos(emb)], dim=-1)++ # flip sine and cosine embeddings+ if flip_sin_to_cos:+ emb = torch.cat([emb[:, half_dim:], emb[:, :half_dim]], dim=-1)++ # zero pad+ if embedding_dim % 2 == 1:+ emb = torch.nn.functional.pad(emb, (0, 1, 0, 0))+ return emb+++class TimestepEmbedding(nn.Module):+ def __init__(+ self,+ in_channels: int,+ time_embed_dim: int,+ act_fn: str = "silu",+ out_dim: int = None,+ post_act_fn: Optional[str] = None,+ cond_proj_dim=None,+ sample_proj_bias=True,+ dtype=None, device=None, operations=None,+ ):+ super().__init__()++ self.linear_1 = operations.Linear(in_channels, time_embed_dim, sample_proj_bias, dtype=dtype, device=device)++ if cond_proj_dim is not None:+ self.cond_proj = operations.Linear(cond_proj_dim, in_channels, bias=False, dtype=dtype, device=device)+ else:+ self.cond_proj = None++ self.act = nn.SiLU()++ if out_dim is not None:+ time_embed_dim_out = out_dim+ else:+ time_embed_dim_out = time_embed_dim+ self.linear_2 = operations.Linear(time_embed_dim, time_embed_dim_out, sample_proj_bias, dtype=dtype, device=device)++ if post_act_fn is None:+ self.post_act = None+ # else:+ # self.post_act = get_activation(post_act_fn)++ def forward(self, sample, condition=None):+ if condition is not None:+ sample = sample + self.cond_proj(condition)+ sample = self.linear_1(sample)++ if self.act is not None:+ sample = self.act(sample)++ sample = self.linear_2(sample)++ if self.post_act is not None:+ sample = self.post_act(sample)+ return sample+++class Timesteps(nn.Module):+ def __init__(self, num_channels: int, flip_sin_to_cos: bool, downscale_freq_shift: float, scale: int = 1):+ super().__init__()+ self.num_channels = num_channels+ self.flip_sin_to_cos = flip_sin_to_cos+ self.downscale_freq_shift = downscale_freq_shift+ self.scale = scale++ def forward(self, timesteps):+ t_emb = get_timestep_embedding(+ timesteps,+ self.num_channels,+ flip_sin_to_cos=self.flip_sin_to_cos,+ downscale_freq_shift=self.downscale_freq_shift,+ scale=self.scale,+ )+ return t_emb+++class PixArtAlphaCombinedTimestepSizeEmbeddings(nn.Module):+ """+ For PixArt-Alpha.++ Reference:+ https://github.com/PixArt-alpha/PixArt-alpha/blob/0f55e922376d8b797edd44d25d0e7464b260dcab/diffusion/model/nets/PixArtMS.py#L164C9-L168C29+ """++ def __init__(self, embedding_dim, size_emb_dim, use_additional_conditions: bool = False, dtype=None, device=None, operations=None):+ super().__init__()++ self.outdim = size_emb_dim+ self.time_proj = Timesteps(num_channels=256, flip_sin_to_cos=True, downscale_freq_shift=0)+ self.timestep_embedder = TimestepEmbedding(in_channels=256, time_embed_dim=embedding_dim, dtype=dtype, device=device, operations=operations)++ def forward(self, timestep, resolution, aspect_ratio, batch_size, hidden_dtype):+ timesteps_proj = self.time_proj(timestep)+ timesteps_emb = self.timestep_embedder(timesteps_proj.to(dtype=hidden_dtype)) # (N, D)+ return timesteps_emb+++class AdaLayerNormSingle(nn.Module):+ r"""+ Norm layer adaptive layer norm single (adaLN-single).++ As proposed in PixArt-Alpha (see: https://arxiv.org/abs/2310.00426; Section 2.3).++ Parameters:+ embedding_dim (`int`): The size of each embedding vector.+ use_additional_conditions (`bool`): To use additional conditions for normalization or not.+ """++ def __init__(self, embedding_dim: int, use_additional_conditions: bool = False, dtype=None, device=None, operations=None):+ super().__init__()++ self.emb = PixArtAlphaCombinedTimestepSizeEmbeddings(+ embedding_dim, size_emb_dim=embedding_dim // 3, use_additional_conditions=use_additional_conditions, dtype=dtype, device=device, operations=operations+ )++ self.silu = nn.SiLU()+ self.linear = operations.Linear(embedding_dim, 6 * embedding_dim, bias=True, dtype=dtype, device=device)++ def forward(+ self,+ timestep: torch.Tensor,+ added_cond_kwargs: Optional[Dict[str, torch.Tensor]] = None,+ batch_size: Optional[int] = None,+ hidden_dtype: Optional[torch.dtype] = None,+ ) -> Tuple[torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor, torch.Tensor]:+ # No modulation happening here.+ added_cond_kwargs = added_cond_kwargs or {"resolution": None, "aspect_ratio": None}+ embedded_timestep = self.emb(timestep, **added_cond_kwargs, batch_size=batch_size, hidden_dtype=hidden_dtype)+ return self.linear(self.silu(embedded_timestep)), embedded_timestep++class PixArtAlphaTextProjection(nn.Module):+ """+ Projects caption embeddings. Also handles dropout for classifier-free guidance.++ Adapted from https://github.com/PixArt-alpha/PixArt-alpha/blob/master/diffusion/model/nets/PixArt_blocks.py+ """++ def __init__(self, in_features, hidden_size, out_features=None, act_fn="gelu_tanh", dtype=None, device=None, operations=None):+ super().__init__()+ if out_features is None:+ out_features = hidden_size+ self.linear_1 = operations.Linear(in_features=in_features, out_features=hidden_size, bias=True, dtype=dtype, device=device)+ if act_fn == "gelu_tanh":+ self.act_1 = nn.GELU(approximate="tanh")+ elif act_fn == "silu":+ self.act_1 = nn.SiLU()+ else:+ raise ValueError(f"Unknown activation function: {act_fn}")+ self.linear_2 = operations.Linear(in_features=hidden_size, out_features=out_features, bias=True, dtype=dtype, device=device)++ def forward(self, caption):+ hidden_states = self.linear_1(caption)+ hidden_states = self.act_1(hidden_states)+ hidden_states = self.linear_2(hidden_states)+ return hidden_states+++class GELU_approx(nn.Module):+ def __init__(self, dim_in, dim_out, dtype=None, device=None, operations=None):+ super().__init__()+ self.proj = operations.Linear(dim_in, dim_out, dtype=dtype, device=device)++ def forward(self, x):+ return torch.nn.functional.gelu(self.proj(x), approximate="tanh")+++class FeedForward(nn.Module):+ def __init__(self, dim, dim_out, mult=4, glu=False, dropout=0., dtype=None, device=None, operations=None):+ super().__init__()+ inner_dim = int(dim * mult)+ project_in = GELU_approx(dim, inner_dim, dtype=dtype, device=device, operations=operations)++ self.net = nn.Sequential(+ project_in,+ nn.Dropout(dropout),+ operations.Linear(inner_dim, dim_out, dtype=dtype, device=device)+ )++ def forward(self, x):+ return self.net(x)+++def apply_rotary_emb(input_tensor, freqs_cis): #TODO: remove duplicate funcs and pick the best/fastest one+ cos_freqs = freqs_cis[0]+ sin_freqs = freqs_cis[1]++ t_dup = rearrange(input_tensor, "... (d r) -> ... d r", r=2)+ t1, t2 = t_dup.unbind(dim=-1)+ t_dup = torch.stack((-t2, t1), dim=-1)+ input_tensor_rot = rearrange(t_dup, "... d r -> ... (d r)")++ out = input_tensor * cos_freqs + input_tensor_rot * sin_freqs++ return out+++class CrossAttention(nn.Module):+ def __init__(self, query_dim, context_dim=None, heads=8, dim_head=64, dropout=0., attn_precision=None, dtype=None, device=None, operations=None):+ super().__init__()+ inner_dim = dim_head * heads+ context_dim = query_dim if context_dim is None else context_dim+ self.attn_precision = attn_precision++ self.heads = heads+ self.dim_head = dim_head++ self.q_norm = RMSNorm(inner_dim, dtype=dtype, device=device)+ self.k_norm = RMSNorm(inner_dim, dtype=dtype, device=device)++ self.to_q = operations.Linear(query_dim, inner_dim, bias=True, dtype=dtype, device=device)+ self.to_k = operations.Linear(context_dim, inner_dim, bias=True, dtype=dtype, device=device)+ self.to_v = operations.Linear(context_dim, inner_dim, bias=True, dtype=dtype, device=device)++ self.to_out = nn.Sequential(operations.Linear(inner_dim, query_dim, dtype=dtype, device=device), nn.Dropout(dropout))++ def forward(self, x, context=None, mask=None, pe=None):+ q = self.to_q(x)+ context = x if context is None else context+ k = self.to_k(context)+ v = self.to_v(context)++ q = self.q_norm(q)+ k = self.k_norm(k)++ if pe is not None:+ q = apply_rotary_emb(q, pe)+ k = apply_rotary_emb(k, pe)++ if mask is None:+ out = comfy.ldm.modules.attention.optimized_attention(q, k, v, self.heads, attn_precision=self.attn_precision)+ else:+ out = comfy.ldm.modules.attention.optimized_attention_masked(q, k, v, self.heads, mask, attn_precision=self.attn_precision)+ return self.to_out(out)+++class BasicTransformerBlock(nn.Module):+ def __init__(self, dim, n_heads, d_head, context_dim=None, attn_precision=None, dtype=None, device=None, operations=None):+ super().__init__()++ self.attn_precision = attn_precision+ self.attn1 = CrossAttention(query_dim=dim, heads=n_heads, dim_head=d_head, context_dim=None, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations)+ self.ff = FeedForward(dim, dim_out=dim, glu=True, dtype=dtype, device=device, operations=operations)++ self.attn2 = CrossAttention(query_dim=dim, context_dim=context_dim, heads=n_heads, dim_head=d_head, attn_precision=self.attn_precision, dtype=dtype, device=device, operations=operations)++ self.scale_shift_table = nn.Parameter(torch.empty(6, dim, device=device, dtype=dtype))++ def forward(self, x, context=None, attention_mask=None, timestep=None, pe=None):+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = (self.scale_shift_table[None, None] + timestep.reshape(x.shape[0], timestep.shape[1], self.scale_shift_table.shape[0], -1)).unbind(dim=2)++ x += self.attn1(comfy.ldm.common_dit.rms_norm(x) * (1 + scale_msa) + shift_msa, pe=pe) * gate_msa++ x += self.attn2(x, context=context, mask=attention_mask)++ y = comfy.ldm.common_dit.rms_norm(x) * (1 + scale_mlp) + shift_mlp+ x += self.ff(y) * gate_mlp++ return x++def get_fractional_positions(indices_grid, max_pos):+ fractional_positions = torch.stack(+ [+ indices_grid[:, i] / max_pos[i]+ for i in range(3)+ ],+ dim=-1,+ )+ return fractional_positions+++def precompute_freqs_cis(indices_grid, dim, out_dtype, theta=10000.0, max_pos=[20, 2048, 2048]):+ dtype = torch.float32 #self.dtype++ fractional_positions = get_fractional_positions(indices_grid, max_pos)++ start = 1+ end = theta+ device = fractional_positions.device++ indices = theta ** (+ torch.linspace(+ math.log(start, theta),+ math.log(end, theta),+ dim // 6,+ device=device,+ dtype=dtype,+ )+ )+ indices = indices.to(dtype=dtype)++ indices = indices * math.pi / 2++ freqs = (+ (indices * (fractional_positions.unsqueeze(-1) * 2 - 1))+ .transpose(-1, -2)+ .flatten(2)+ )++ cos_freq = freqs.cos().repeat_interleave(2, dim=-1)+ sin_freq = freqs.sin().repeat_interleave(2, dim=-1)+ if dim % 6 != 0:+ cos_padding = torch.ones_like(cos_freq[:, :, : dim % 6])+ sin_padding = torch.zeros_like(cos_freq[:, :, : dim % 6])+ cos_freq = torch.cat([cos_padding, cos_freq], dim=-1)+ sin_freq = torch.cat([sin_padding, sin_freq], dim=-1)+ return cos_freq.to(out_dtype), sin_freq.to(out_dtype)+++class LTXVModel(torch.nn.Module):+ def __init__(self,+ in_channels=128,+ cross_attention_dim=2048,+ attention_head_dim=64,+ num_attention_heads=32,++ caption_channels=4096,+ num_layers=28,+++ positional_embedding_theta=10000.0,+ positional_embedding_max_pos=[20, 2048, 2048],+ dtype=None, device=None, operations=None, **kwargs):+ super().__init__()+ self.dtype = dtype+ self.out_channels = in_channels+ self.inner_dim = num_attention_heads * attention_head_dim++ self.patchify_proj = operations.Linear(in_channels, self.inner_dim, bias=True, dtype=dtype, device=device)++ self.adaln_single = AdaLayerNormSingle(+ self.inner_dim, use_additional_conditions=False, dtype=dtype, device=device, operations=operations+ )++ # self.adaln_single.linear = operations.Linear(self.inner_dim, 4 * self.inner_dim, bias=True, dtype=dtype, device=device)++ self.caption_projection = PixArtAlphaTextProjection(+ in_features=caption_channels, hidden_size=self.inner_dim, dtype=dtype, device=device, operations=operations+ )++ self.transformer_blocks = nn.ModuleList(+ [+ BasicTransformerBlock(+ self.inner_dim,+ num_attention_heads,+ attention_head_dim,+ context_dim=cross_attention_dim,+ # attn_precision=attn_precision,+ dtype=dtype, device=device, operations=operations+ )+ for d in range(num_layers)+ ]+ )++ self.scale_shift_table = nn.Parameter(torch.empty(2, self.inner_dim, dtype=dtype, device=device))+ self.norm_out = operations.LayerNorm(self.inner_dim, elementwise_affine=False, eps=1e-6, dtype=dtype, device=device)+ self.proj_out = operations.Linear(self.inner_dim, self.out_channels, dtype=dtype, device=device)++ self.patchifier = SymmetricPatchifier(1)++ def forward(self, x, timestep, context, attention_mask, frame_rate=25, guiding_latent=None, **kwargs):+ indices_grid = self.patchifier.get_grid(+ orig_num_frames=x.shape[2],+ orig_height=x.shape[3],+ orig_width=x.shape[4],+ batch_size=x.shape[0],+ scale_grid=((1 / frame_rate) * 8, 32, 32), #TODO: controlable frame rate+ device=x.device,+ )++ if guiding_latent is not None:+ ts = torch.ones([x.shape[0], 1, x.shape[2], x.shape[3], x.shape[4]], device=x.device, dtype=x.dtype)+ input_ts = timestep.view([timestep.shape[0]] + [1] * (x.ndim - 1))+ ts *= input_ts+ ts[:, :, 0] = 0.0+ timestep = self.patchifier.patchify(ts)+ input_x = x.clone()+ x[:, :, 0] = guiding_latent[:, :, 0]++ orig_shape = list(x.shape)++ x = self.patchifier.patchify(x)++ x = self.patchify_proj(x)+ timestep = timestep * 1000.0++ attention_mask = 1.0 - attention_mask.to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1]))+ attention_mask = attention_mask.masked_fill(attention_mask.to(torch.bool), float("-inf")) # not sure about this+ # attention_mask = (context != 0).any(dim=2).to(dtype=x.dtype)++ pe = precompute_freqs_cis(indices_grid, dim=self.inner_dim, out_dtype=x.dtype)++ batch_size = x.shape[0]+ timestep, embedded_timestep = self.adaln_single(+ timestep.flatten(),+ {"resolution": None, "aspect_ratio": None},+ batch_size=batch_size,+ hidden_dtype=x.dtype,+ )+ # Second dimension is 1 or number of tokens (if timestep_per_token)+ timestep = timestep.view(batch_size, -1, timestep.shape[-1])+ embedded_timestep = embedded_timestep.view(+ batch_size, -1, embedded_timestep.shape[-1]+ )++ # 2. Blocks+ if self.caption_projection is not None:+ batch_size = x.shape[0]+ context = self.caption_projection(context)+ context = context.view(+ batch_size, -1, x.shape[-1]+ )++ for block in self.transformer_blocks:+ x = block(+ x,+ context=context,+ attention_mask=attention_mask,+ timestep=timestep,+ pe=pe+ )++ # 3. Output+ scale_shift_values = (+ self.scale_shift_table[None, None] + embedded_timestep[:, :, None]+ )+ shift, scale = scale_shift_values[:, :, 0], scale_shift_values[:, :, 1]+ x = self.norm_out(x)+ # Modulation+ x = x * (1 + scale) + shift+ x = self.proj_out(x)++ x = self.patchifier.unpatchify(+ latents=x,+ output_height=orig_shape[3],+ output_width=orig_shape[4],+ output_num_frames=orig_shape[2],+ out_channels=orig_shape[1] // math.prod(self.patchifier.patch_size),+ )++ if guiding_latent is not None:+ x[:, :, 0] = (input_x[:, :, 0] - guiding_latent[:, :, 0]) / input_ts[:, :, 0]++ # print("res", x)+ return xdiff --git a/comfy/ldm/lightricks/symmetric_patchifier.py b/comfy/ldm/lightricks/symmetric_patchifier.pynew file mode 100644index 000000000..51ce50589--- /dev/null+++ b/comfy/ldm/lightricks/symmetric_patchifier.py@@ -0,0 +1,105 @@+from abc import ABC, abstractmethod+from typing import Tuple++import torch+from einops import rearrange+from torch import Tensor+++def append_dims(x: torch.Tensor, target_dims: int) -> torch.Tensor:+ """Appends dimensions to the end of a tensor until it has target_dims dimensions."""+ dims_to_append = target_dims - x.ndim+ if dims_to_append < 0:+ raise ValueError(+ f"input has {x.ndim} dims but target_dims is {target_dims}, which is less"+ )+ elif dims_to_append == 0:+ return x+ return x[(...,) + (None,) * dims_to_append]+++class Patchifier(ABC):+ def __init__(self, patch_size: int):+ super().__init__()+ self._patch_size = (1, patch_size, patch_size)++ @abstractmethod+ def patchify(+ self, latents: Tensor, frame_rates: Tensor, scale_grid: bool+ ) -> Tuple[Tensor, Tensor]:+ pass++ @abstractmethod+ def unpatchify(+ self,+ latents: Tensor,+ output_height: int,+ output_width: int,+ output_num_frames: int,+ out_channels: int,+ ) -> Tuple[Tensor, Tensor]:+ pass++ @property+ def patch_size(self):+ return self._patch_size++ def get_grid(+ self, orig_num_frames, orig_height, orig_width, batch_size, scale_grid, device+ ):+ f = orig_num_frames // self._patch_size[0]+ h = orig_height // self._patch_size[1]+ w = orig_width // self._patch_size[2]+ grid_h = torch.arange(h, dtype=torch.float32, device=device)+ grid_w = torch.arange(w, dtype=torch.float32, device=device)+ grid_f = torch.arange(f, dtype=torch.float32, device=device)+ grid = torch.meshgrid(grid_f, grid_h, grid_w)+ grid = torch.stack(grid, dim=0)+ grid = grid.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)++ if scale_grid is not None:+ for i in range(3):+ if isinstance(scale_grid[i], Tensor):+ scale = append_dims(scale_grid[i], grid.ndim - 1)+ else:+ scale = scale_grid[i]+ grid[:, i, ...] = grid[:, i, ...] * scale * self._patch_size[i]++ grid = rearrange(grid, "b c f h w -> b c (f h w)", b=batch_size)+ return grid+++class SymmetricPatchifier(Patchifier):+ def patchify(+ self,+ latents: Tensor,+ ) -> Tuple[Tensor, Tensor]:+ latents = rearrange(+ latents,+ "b c (f p1) (h p2) (w p3) -> b (f h w) (c p1 p2 p3)",+ p1=self._patch_size[0],+ p2=self._patch_size[1],+ p3=self._patch_size[2],+ )+ return latents++ def unpatchify(+ self,+ latents: Tensor,+ output_height: int,+ output_width: int,+ output_num_frames: int,+ out_channels: int,+ ) -> Tuple[Tensor, Tensor]:+ output_height = output_height // self._patch_size[1]+ output_width = output_width // self._patch_size[2]+ latents = rearrange(+ latents,+ "b (f h w) (c p q) -> b c f (h p) (w q) ",+ f=output_num_frames,+ h=output_height,+ w=output_width,+ p=self._patch_size[1],+ q=self._patch_size[2],+ )+ return latentsdiff --git a/comfy/ldm/lightricks/vae/causal_conv3d.py b/comfy/ldm/lightricks/vae/causal_conv3d.pynew file mode 100644index 000000000..146dea19b--- /dev/null+++ b/comfy/ldm/lightricks/vae/causal_conv3d.py@@ -0,0 +1,62 @@+from typing import Tuple, Union++import torch+import torch.nn as nn+++class CausalConv3d(nn.Module):+ def __init__(+ self,+ in_channels,+ out_channels,+ kernel_size: int = 3,+ stride: Union[int, Tuple[int]] = 1,+ dilation: int = 1,+ groups: int = 1,+ **kwargs,+ ):+ super().__init__()++ self.in_channels = in_channels+ self.out_channels = out_channels++ kernel_size = (kernel_size, kernel_size, kernel_size)+ self.time_kernel_size = kernel_size[0]++ dilation = (dilation, 1, 1)++ height_pad = kernel_size[1] // 2+ width_pad = kernel_size[2] // 2+ padding = (0, height_pad, width_pad)++ self.conv = nn.Conv3d(+ in_channels,+ out_channels,+ kernel_size,+ stride=stride,+ dilation=dilation,+ padding=padding,+ padding_mode="zeros",+ groups=groups,+ )++ def forward(self, x, causal: bool = True):+ if causal:+ first_frame_pad = x[:, :, :1, :, :].repeat(+ (1, 1, self.time_kernel_size - 1, 1, 1)+ )+ x = torch.concatenate((first_frame_pad, x), dim=2)+ else:+ first_frame_pad = x[:, :, :1, :, :].repeat(+ (1, 1, (self.time_kernel_size - 1) // 2, 1, 1)+ )+ last_frame_pad = x[:, :, -1:, :, :].repeat(+ (1, 1, (self.time_kernel_size - 1) // 2, 1, 1)+ )+ x = torch.concatenate((first_frame_pad, x, last_frame_pad), dim=2)+ x = self.conv(x)+ return x++ @property+ def weight(self):+ return self.conv.weightdiff --git a/comfy/ldm/lightricks/vae/causal_video_autoencoder.py b/comfy/ldm/lightricks/vae/causal_video_autoencoder.pynew file mode 100644index 000000000..4138fdf3c--- /dev/null+++ b/comfy/ldm/lightricks/vae/causal_video_autoencoder.py@@ -0,0 +1,698 @@+import torch+from torch import nn+from functools import partial+import math+from einops import rearrange+from typing import Any, Mapping, Optional, Tuple, Union, List+from .conv_nd_factory import make_conv_nd, make_linear_nd+from .pixel_norm import PixelNorm+++class Encoder(nn.Module):+ r"""+ The `Encoder` layer of a variational autoencoder that encodes its input into a latent representation.++ Args:+ dims (`int` or `Tuple[int, int]`, *optional*, defaults to 3):+ The number of dimensions to use in convolutions.+ in_channels (`int`, *optional*, defaults to 3):+ The number of input channels.+ out_channels (`int`, *optional*, defaults to 3):+ The number of output channels.+ blocks (`List[Tuple[str, int]]`, *optional*, defaults to `[("res_x", 1)]`):+ The blocks to use. Each block is a tuple of the block name and the number of layers.+ base_channels (`int`, *optional*, defaults to 128):+ The number of output channels for the first convolutional layer.+ norm_num_groups (`int`, *optional*, defaults to 32):+ The number of groups for normalization.+ patch_size (`int`, *optional*, defaults to 1):+ The patch size to use. Should be a power of 2.+ norm_layer (`str`, *optional*, defaults to `group_norm`):+ The normalization layer to use. Can be either `group_norm` or `pixel_norm`.+ latent_log_var (`str`, *optional*, defaults to `per_channel`):+ The number of channels for the log variance. Can be either `per_channel`, `uniform`, or `none`.+ """++ def __init__(+ self,+ dims: Union[int, Tuple[int, int]] = 3,+ in_channels: int = 3,+ out_channels: int = 3,+ blocks=[("res_x", 1)],+ base_channels: int = 128,+ norm_num_groups: int = 32,+ patch_size: Union[int, Tuple[int]] = 1,+ norm_layer: str = "group_norm", # group_norm, pixel_norm+ latent_log_var: str = "per_channel",+ ):+ super().__init__()+ self.patch_size = patch_size+ self.norm_layer = norm_layer+ self.latent_channels = out_channels+ self.latent_log_var = latent_log_var+ self.blocks_desc = blocks++ in_channels = in_channels * patch_size**2+ output_channel = base_channels++ self.conv_in = make_conv_nd(+ dims=dims,+ in_channels=in_channels,+ out_channels=output_channel,+ kernel_size=3,+ stride=1,+ padding=1,+ causal=True,+ )++ self.down_blocks = nn.ModuleList([])++ for block_name, block_params in blocks:+ input_channel = output_channel+ if isinstance(block_params, int):+ block_params = {"num_layers": block_params}++ if block_name == "res_x":+ block = UNetMidBlock3D(+ dims=dims,+ in_channels=input_channel,+ num_layers=block_params["num_layers"],+ resnet_eps=1e-6,+ resnet_groups=norm_num_groups,+ norm_layer=norm_layer,+ )+ elif block_name == "res_x_y":+ output_channel = block_params.get("multiplier", 2) * output_channel+ block = ResnetBlock3D(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ eps=1e-6,+ groups=norm_num_groups,+ norm_layer=norm_layer,+ )+ elif block_name == "compress_time":+ block = make_conv_nd(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ kernel_size=3,+ stride=(2, 1, 1),+ causal=True,+ )+ elif block_name == "compress_space":+ block = make_conv_nd(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ kernel_size=3,+ stride=(1, 2, 2),+ causal=True,+ )+ elif block_name == "compress_all":+ block = make_conv_nd(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ kernel_size=3,+ stride=(2, 2, 2),+ causal=True,+ )+ elif block_name == "compress_all_x_y":+ output_channel = block_params.get("multiplier", 2) * output_channel+ block = make_conv_nd(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ kernel_size=3,+ stride=(2, 2, 2),+ causal=True,+ )+ else:+ raise ValueError(f"unknown block: {block_name}")++ self.down_blocks.append(block)++ # out+ if norm_layer == "group_norm":+ self.conv_norm_out = nn.GroupNorm(+ num_channels=output_channel, num_groups=norm_num_groups, eps=1e-6+ )+ elif norm_layer == "pixel_norm":+ self.conv_norm_out = PixelNorm()+ elif norm_layer == "layer_norm":+ self.conv_norm_out = LayerNorm(output_channel, eps=1e-6)++ self.conv_act = nn.SiLU()++ conv_out_channels = out_channels+ if latent_log_var == "per_channel":+ conv_out_channels *= 2+ elif latent_log_var == "uniform":+ conv_out_channels += 1+ elif latent_log_var != "none":+ raise ValueError(f"Invalid latent_log_var: {latent_log_var}")+ self.conv_out = make_conv_nd(+ dims, output_channel, conv_out_channels, 3, padding=1, causal=True+ )++ self.gradient_checkpointing = False++ def forward(self, sample: torch.FloatTensor) -> torch.FloatTensor:+ r"""The forward method of the `Encoder` class."""++ sample = patchify(sample, patch_size_hw=self.patch_size, patch_size_t=1)+ sample = self.conv_in(sample)++ checkpoint_fn = (+ partial(torch.utils.checkpoint.checkpoint, use_reentrant=False)+ if self.gradient_checkpointing and self.training+ else lambda x: x+ )++ for down_block in self.down_blocks:+ sample = checkpoint_fn(down_block)(sample)++ sample = self.conv_norm_out(sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample)++ if self.latent_log_var == "uniform":+ last_channel = sample[:, -1:, ...]+ num_dims = sample.dim()++ if num_dims == 4:+ # For shape (B, C, H, W)+ repeated_last_channel = last_channel.repeat(+ 1, sample.shape[1] - 2, 1, 1+ )+ sample = torch.cat([sample, repeated_last_channel], dim=1)+ elif num_dims == 5:+ # For shape (B, C, F, H, W)+ repeated_last_channel = last_channel.repeat(+ 1, sample.shape[1] - 2, 1, 1, 1+ )+ sample = torch.cat([sample, repeated_last_channel], dim=1)+ else:+ raise ValueError(f"Invalid input shape: {sample.shape}")++ return sample+++class Decoder(nn.Module):+ r"""+ The `Decoder` layer of a variational autoencoder that decodes its latent representation into an output sample.++ Args:+ dims (`int` or `Tuple[int, int]`, *optional*, defaults to 3):+ The number of dimensions to use in convolutions.+ in_channels (`int`, *optional*, defaults to 3):+ The number of input channels.+ out_channels (`int`, *optional*, defaults to 3):+ The number of output channels.+ blocks (`List[Tuple[str, int]]`, *optional*, defaults to `[("res_x", 1)]`):+ The blocks to use. Each block is a tuple of the block name and the number of layers.+ base_channels (`int`, *optional*, defaults to 128):+ The number of output channels for the first convolutional layer.+ norm_num_groups (`int`, *optional*, defaults to 32):+ The number of groups for normalization.+ patch_size (`int`, *optional*, defaults to 1):+ The patch size to use. Should be a power of 2.+ norm_layer (`str`, *optional*, defaults to `group_norm`):+ The normalization layer to use. Can be either `group_norm` or `pixel_norm`.+ causal (`bool`, *optional*, defaults to `True`):+ Whether to use causal convolutions or not.+ """++ def __init__(+ self,+ dims,+ in_channels: int = 3,+ out_channels: int = 3,+ blocks=[("res_x", 1)],+ base_channels: int = 128,+ layers_per_block: int = 2,+ norm_num_groups: int = 32,+ patch_size: int = 1,+ norm_layer: str = "group_norm",+ causal: bool = True,+ ):+ super().__init__()+ self.patch_size = patch_size+ self.layers_per_block = layers_per_block+ out_channels = out_channels * patch_size**2+ self.causal = causal+ self.blocks_desc = blocks++ # Compute output channel to be product of all channel-multiplier blocks+ output_channel = base_channels+ for block_name, block_params in list(reversed(blocks)):+ block_params = block_params if isinstance(block_params, dict) else {}+ if block_name == "res_x_y":+ output_channel = output_channel * block_params.get("multiplier", 2)++ self.conv_in = make_conv_nd(+ dims,+ in_channels,+ output_channel,+ kernel_size=3,+ stride=1,+ padding=1,+ causal=True,+ )++ self.up_blocks = nn.ModuleList([])++ for block_name, block_params in list(reversed(blocks)):+ input_channel = output_channel+ if isinstance(block_params, int):+ block_params = {"num_layers": block_params}++ if block_name == "res_x":+ block = UNetMidBlock3D(+ dims=dims,+ in_channels=input_channel,+ num_layers=block_params["num_layers"],+ resnet_eps=1e-6,+ resnet_groups=norm_num_groups,+ norm_layer=norm_layer,+ )+ elif block_name == "res_x_y":+ output_channel = output_channel // block_params.get("multiplier", 2)+ block = ResnetBlock3D(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ eps=1e-6,+ groups=norm_num_groups,+ norm_layer=norm_layer,+ )+ elif block_name == "compress_time":+ block = DepthToSpaceUpsample(+ dims=dims, in_channels=input_channel, stride=(2, 1, 1)+ )+ elif block_name == "compress_space":+ block = DepthToSpaceUpsample(+ dims=dims, in_channels=input_channel, stride=(1, 2, 2)+ )+ elif block_name == "compress_all":+ block = DepthToSpaceUpsample(+ dims=dims,+ in_channels=input_channel,+ stride=(2, 2, 2),+ residual=block_params.get("residual", False),+ )+ else:+ raise ValueError(f"unknown layer: {block_name}")++ self.up_blocks.append(block)++ if norm_layer == "group_norm":+ self.conv_norm_out = nn.GroupNorm(+ num_channels=output_channel, num_groups=norm_num_groups, eps=1e-6+ )+ elif norm_layer == "pixel_norm":+ self.conv_norm_out = PixelNorm()+ elif norm_layer == "layer_norm":+ self.conv_norm_out = LayerNorm(output_channel, eps=1e-6)++ self.conv_act = nn.SiLU()+ self.conv_out = make_conv_nd(+ dims, output_channel, out_channels, 3, padding=1, causal=True+ )++ self.gradient_checkpointing = False++ # def forward(self, sample: torch.FloatTensor, target_shape) -> torch.FloatTensor:+ def forward(self, sample: torch.FloatTensor) -> torch.FloatTensor:+ r"""The forward method of the `Decoder` class."""+ # assert target_shape is not None, "target_shape must be provided"++ sample = self.conv_in(sample, causal=self.causal)++ upscale_dtype = next(iter(self.up_blocks.parameters())).dtype++ checkpoint_fn = (+ partial(torch.utils.checkpoint.checkpoint, use_reentrant=False)+ if self.gradient_checkpointing and self.training+ else lambda x: x+ )++ sample = sample.to(upscale_dtype)++ for up_block in self.up_blocks:+ sample = checkpoint_fn(up_block)(sample, causal=self.causal)++ sample = self.conv_norm_out(sample)+ sample = self.conv_act(sample)+ sample = self.conv_out(sample, causal=self.causal)++ sample = unpatchify(sample, patch_size_hw=self.patch_size, patch_size_t=1)++ return sample+++class UNetMidBlock3D(nn.Module):+ """+ A 3D UNet mid-block [`UNetMidBlock3D`] with multiple residual blocks.++ Args:+ in_channels (`int`): The number of input channels.+ dropout (`float`, *optional*, defaults to 0.0): The dropout rate.+ num_layers (`int`, *optional*, defaults to 1): The number of residual blocks.+ resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks.+ resnet_groups (`int`, *optional*, defaults to 32):+ The number of groups to use in the group normalization layers of the resnet blocks.++ Returns:+ `torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size,+ in_channels, height, width)`.++ """++ def __init__(+ self,+ dims: Union[int, Tuple[int, int]],+ in_channels: int,+ dropout: float = 0.0,+ num_layers: int = 1,+ resnet_eps: float = 1e-6,+ resnet_groups: int = 32,+ norm_layer: str = "group_norm",+ ):+ super().__init__()+ resnet_groups = (+ resnet_groups if resnet_groups is not None else min(in_channels // 4, 32)+ )++ self.res_blocks = nn.ModuleList(+ [+ ResnetBlock3D(+ dims=dims,+ in_channels=in_channels,+ out_channels=in_channels,+ eps=resnet_eps,+ groups=resnet_groups,+ dropout=dropout,+ norm_layer=norm_layer,+ )+ for _ in range(num_layers)+ ]+ )++ def forward(+ self, hidden_states: torch.FloatTensor, causal: bool = True+ ) -> torch.FloatTensor:+ for resnet in self.res_blocks:+ hidden_states = resnet(hidden_states, causal=causal)++ return hidden_states+++class DepthToSpaceUpsample(nn.Module):+ def __init__(self, dims, in_channels, stride, residual=False):+ super().__init__()+ self.stride = stride+ self.out_channels = math.prod(stride) * in_channels+ self.conv = make_conv_nd(+ dims=dims,+ in_channels=in_channels,+ out_channels=self.out_channels,+ kernel_size=3,+ stride=1,+ causal=True,+ )+ self.residual = residual++ def forward(self, x, causal: bool = True):+ if self.residual:+ # Reshape and duplicate the input to match the output shape+ x_in = rearrange(+ x,+ "b (c p1 p2 p3) d h w -> b c (d p1) (h p2) (w p3)",+ p1=self.stride[0],+ p2=self.stride[1],+ p3=self.stride[2],+ )+ x_in = x_in.repeat(1, math.prod(self.stride), 1, 1, 1)+ if self.stride[0] == 2:+ x_in = x_in[:, :, 1:, :, :]+ x = self.conv(x, causal=causal)+ x = rearrange(+ x,+ "b (c p1 p2 p3) d h w -> b c (d p1) (h p2) (w p3)",+ p1=self.stride[0],+ p2=self.stride[1],+ p3=self.stride[2],+ )+ if self.stride[0] == 2:+ x = x[:, :, 1:, :, :]+ if self.residual:+ x = x + x_in+ return x+++class LayerNorm(nn.Module):+ def __init__(self, dim, eps, elementwise_affine=True) -> None:+ super().__init__()+ self.norm = nn.LayerNorm(dim, eps=eps, elementwise_affine=elementwise_affine)++ def forward(self, x):+ x = rearrange(x, "b c d h w -> b d h w c")+ x = self.norm(x)+ x = rearrange(x, "b d h w c -> b c d h w")+ return x+++class ResnetBlock3D(nn.Module):+ r"""+ A Resnet block.++ Parameters:+ in_channels (`int`): The number of channels in the input.+ out_channels (`int`, *optional*, default to be `None`):+ The number of output channels for the first conv layer. If None, same as `in_channels`.+ dropout (`float`, *optional*, defaults to `0.0`): The dropout probability to use.+ groups (`int`, *optional*, default to `32`): The number of groups to use for the first normalization layer.+ eps (`float`, *optional*, defaults to `1e-6`): The epsilon to use for the normalization.+ """++ def __init__(+ self,+ dims: Union[int, Tuple[int, int]],+ in_channels: int,+ out_channels: Optional[int] = None,+ dropout: float = 0.0,+ groups: int = 32,+ eps: float = 1e-6,+ norm_layer: str = "group_norm",+ ):+ super().__init__()+ self.in_channels = in_channels+ out_channels = in_channels if out_channels is None else out_channels+ self.out_channels = out_channels++ if norm_layer == "group_norm":+ self.norm1 = nn.GroupNorm(+ num_groups=groups, num_channels=in_channels, eps=eps, affine=True+ )+ elif norm_layer == "pixel_norm":+ self.norm1 = PixelNorm()+ elif norm_layer == "layer_norm":+ self.norm1 = LayerNorm(in_channels, eps=eps, elementwise_affine=True)++ self.non_linearity = nn.SiLU()++ self.conv1 = make_conv_nd(+ dims,+ in_channels,+ out_channels,+ kernel_size=3,+ stride=1,+ padding=1,+ causal=True,+ )++ if norm_layer == "group_norm":+ self.norm2 = nn.GroupNorm(+ num_groups=groups, num_channels=out_channels, eps=eps, affine=True+ )+ elif norm_layer == "pixel_norm":+ self.norm2 = PixelNorm()+ elif norm_layer == "layer_norm":+ self.norm2 = LayerNorm(out_channels, eps=eps, elementwise_affine=True)++ self.dropout = torch.nn.Dropout(dropout)++ self.conv2 = make_conv_nd(+ dims,+ out_channels,+ out_channels,+ kernel_size=3,+ stride=1,+ padding=1,+ causal=True,+ )++ self.conv_shortcut = (+ make_linear_nd(+ dims=dims, in_channels=in_channels, out_channels=out_channels+ )+ if in_channels != out_channels+ else nn.Identity()+ )++ self.norm3 = (+ LayerNorm(in_channels, eps=eps, elementwise_affine=True)+ if in_channels != out_channels+ else nn.Identity()+ )++ def forward(+ self,+ input_tensor: torch.FloatTensor,+ causal: bool = True,+ ) -> torch.FloatTensor:+ hidden_states = input_tensor++ hidden_states = self.norm1(hidden_states)++ hidden_states = self.non_linearity(hidden_states)++ hidden_states = self.conv1(hidden_states, causal=causal)++ hidden_states = self.norm2(hidden_states)++ hidden_states = self.non_linearity(hidden_states)++ hidden_states = self.dropout(hidden_states)++ hidden_states = self.conv2(hidden_states, causal=causal)++ input_tensor = self.norm3(input_tensor)++ input_tensor = self.conv_shortcut(input_tensor)++ output_tensor = input_tensor + hidden_states++ return output_tensor+++def patchify(x, patch_size_hw, patch_size_t=1):+ if patch_size_hw == 1 and patch_size_t == 1:+ return x+ if x.dim() == 4:+ x = rearrange(+ x, "b c (h q) (w r) -> b (c r q) h w", q=patch_size_hw, r=patch_size_hw+ )+ elif x.dim() == 5:+ x = rearrange(+ x,+ "b c (f p) (h q) (w r) -> b (c p r q) f h w",+ p=patch_size_t,+ q=patch_size_hw,+ r=patch_size_hw,+ )+ else:+ raise ValueError(f"Invalid input shape: {x.shape}")++ return x+++def unpatchify(x, patch_size_hw, patch_size_t=1):+ if patch_size_hw == 1 and patch_size_t == 1:+ return x++ if x.dim() == 4:+ x = rearrange(+ x, "b (c r q) h w -> b c (h q) (w r)", q=patch_size_hw, r=patch_size_hw+ )+ elif x.dim() == 5:+ x = rearrange(+ x,+ "b (c p r q) f h w -> b c (f p) (h q) (w r)",+ p=patch_size_t,+ q=patch_size_hw,+ r=patch_size_hw,+ )++ return x++class processor(nn.Module):+ def __init__(self):+ super().__init__()+ self.register_buffer("std-of-means", torch.empty(128))+ self.register_buffer("mean-of-means", torch.empty(128))+ self.register_buffer("mean-of-stds", torch.empty(128))+ self.register_buffer("mean-of-stds_over_std-of-means", torch.empty(128))+ self.register_buffer("channel", torch.empty(128))++ def un_normalize(self, x):+ return (x * self.get_buffer("std-of-means").view(1, -1, 1, 1, 1)) + self.get_buffer("mean-of-means").view(1, -1, 1, 1, 1)++ def normalize(self, x):+ return (x - self.get_buffer("mean-of-means").view(1, -1, 1, 1, 1)) / self.get_buffer("std-of-means").view(1, -1, 1, 1, 1)++class VideoVAE(nn.Module):+ def __init__(self):+ super().__init__()+ config = {+ "_class_name": "CausalVideoAutoencoder",+ "dims": 3,+ "in_channels": 3,+ "out_channels": 3,+ "latent_channels": 128,+ "blocks": [+ ["res_x", 4],+ ["compress_all", 1],+ ["res_x_y", 1],+ ["res_x", 3],+ ["compress_all", 1],+ ["res_x_y", 1],+ ["res_x", 3],+ ["compress_all", 1],+ ["res_x", 3],+ ["res_x", 4],+ ],+ "scaling_factor": 1.0,+ "norm_layer": "pixel_norm",+ "patch_size": 4,+ "latent_log_var": "uniform",+ "use_quant_conv": False,+ "causal_decoder": False,+ }++ double_z = config.get("double_z", True)+ latent_log_var = config.get(+ "latent_log_var", "per_channel" if double_z else "none"+ )++ self.encoder = Encoder(+ dims=config["dims"],+ in_channels=config.get("in_channels", 3),+ out_channels=config["latent_channels"],+ blocks=config.get("encoder_blocks", config.get("blocks")),+ patch_size=config.get("patch_size", 1),+ latent_log_var=latent_log_var,+ norm_layer=config.get("norm_layer", "group_norm"),+ )++ self.decoder = Decoder(+ dims=config["dims"],+ in_channels=config["latent_channels"],+ out_channels=config.get("out_channels", 3),+ blocks=config.get("decoder_blocks", config.get("blocks")),+ patch_size=config.get("patch_size", 1),+ norm_layer=config.get("norm_layer", "group_norm"),+ causal=config.get("causal_decoder", False),+ )++ self.per_channel_statistics = processor()++ def encode(self, x):+ means, logvar = torch.chunk(self.encoder(x), 2, dim=1)+ return self.per_channel_statistics.normalize(means)++ def decode(self, x):+ return self.decoder(self.per_channel_statistics.un_normalize(x))+diff --git a/comfy/ldm/lightricks/vae/conv_nd_factory.py b/comfy/ldm/lightricks/vae/conv_nd_factory.pynew file mode 100644index 000000000..389f81659--- /dev/null+++ b/comfy/ldm/lightricks/vae/conv_nd_factory.py@@ -0,0 +1,82 @@+from typing import Tuple, Union++import torch++from .dual_conv3d import DualConv3d+from .causal_conv3d import CausalConv3d+++def make_conv_nd(+ dims: Union[int, Tuple[int, int]],+ in_channels: int,+ out_channels: int,+ kernel_size: int,+ stride=1,+ padding=0,+ dilation=1,+ groups=1,+ bias=True,+ causal=False,+):+ if dims == 2:+ return torch.nn.Conv2d(+ in_channels=in_channels,+ out_channels=out_channels,+ kernel_size=kernel_size,+ stride=stride,+ padding=padding,+ dilation=dilation,+ groups=groups,+ bias=bias,+ )+ elif dims == 3:+ if causal:+ return CausalConv3d(+ in_channels=in_channels,+ out_channels=out_channels,+ kernel_size=kernel_size,+ stride=stride,+ padding=padding,+ dilation=dilation,+ groups=groups,+ bias=bias,+ )+ return torch.nn.Conv3d(+ in_channels=in_channels,+ out_channels=out_channels,+ kernel_size=kernel_size,+ stride=stride,+ padding=padding,+ dilation=dilation,+ groups=groups,+ bias=bias,+ )+ elif dims == (2, 1):+ return DualConv3d(+ in_channels=in_channels,+ out_channels=out_channels,+ kernel_size=kernel_size,+ stride=stride,+ padding=padding,+ bias=bias,+ )+ else:+ raise ValueError(f"unsupported dimensions: {dims}")+++def make_linear_nd(+ dims: int,+ in_channels: int,+ out_channels: int,+ bias=True,+):+ if dims == 2:+ return torch.nn.Conv2d(+ in_channels=in_channels, out_channels=out_channels, kernel_size=1, bias=bias+ )+ elif dims == 3 or dims == (2, 1):+ return torch.nn.Conv3d(+ in_channels=in_channels, out_channels=out_channels, kernel_size=1, bias=bias+ )+ else:+ raise ValueError(f"unsupported dimensions: {dims}")diff --git a/comfy/ldm/lightricks/vae/dual_conv3d.py b/comfy/ldm/lightricks/vae/dual_conv3d.pynew file mode 100644index 000000000..6bd54c0a6--- /dev/null+++ b/comfy/ldm/lightricks/vae/dual_conv3d.py@@ -0,0 +1,195 @@+import math+from typing import Tuple, Union++import torch+import torch.nn as nn+import torch.nn.functional as F+from einops import rearrange+++class DualConv3d(nn.Module):+ def __init__(+ self,+ in_channels,+ out_channels,+ kernel_size,+ stride: Union[int, Tuple[int, int, int]] = 1,+ padding: Union[int, Tuple[int, int, int]] = 0,+ dilation: Union[int, Tuple[int, int, int]] = 1,+ groups=1,+ bias=True,+ ):+ super(DualConv3d, self).__init__()++ self.in_channels = in_channels+ self.out_channels = out_channels+ # Ensure kernel_size, stride, padding, and dilation are tuples of length 3+ if isinstance(kernel_size, int):+ kernel_size = (kernel_size, kernel_size, kernel_size)+ if kernel_size == (1, 1, 1):+ raise ValueError(+ "kernel_size must be greater than 1. Use make_linear_nd instead."+ )+ if isinstance(stride, int):+ stride = (stride, stride, stride)+ if isinstance(padding, int):+ padding = (padding, padding, padding)+ if isinstance(dilation, int):+ dilation = (dilation, dilation, dilation)++ # Set parameters for convolutions+ self.groups = groups+ self.bias = bias++ # Define the size of the channels after the first convolution+ intermediate_channels = (+ out_channels if in_channels < out_channels else in_channels+ )++ # Define parameters for the first convolution+ self.weight1 = nn.Parameter(+ torch.Tensor(+ intermediate_channels,+ in_channels // groups,+ 1,+ kernel_size[1],+ kernel_size[2],+ )+ )+ self.stride1 = (1, stride[1], stride[2])+ self.padding1 = (0, padding[1], padding[2])+ self.dilation1 = (1, dilation[1], dilation[2])+ if bias:+ self.bias1 = nn.Parameter(torch.Tensor(intermediate_channels))+ else:+ self.register_parameter("bias1", None)++ # Define parameters for the second convolution+ self.weight2 = nn.Parameter(+ torch.Tensor(+ out_channels, intermediate_channels // groups, kernel_size[0], 1, 1+ )+ )+ self.stride2 = (stride[0], 1, 1)+ self.padding2 = (padding[0], 0, 0)+ self.dilation2 = (dilation[0], 1, 1)+ if bias:+ self.bias2 = nn.Parameter(torch.Tensor(out_channels))+ else:+ self.register_parameter("bias2", None)++ # Initialize weights and biases+ self.reset_parameters()++ def reset_parameters(self):+ nn.init.kaiming_uniform_(self.weight1, a=math.sqrt(5))+ nn.init.kaiming_uniform_(self.weight2, a=math.sqrt(5))+ if self.bias:+ fan_in1, _ = nn.init._calculate_fan_in_and_fan_out(self.weight1)+ bound1 = 1 / math.sqrt(fan_in1)+ nn.init.uniform_(self.bias1, -bound1, bound1)+ fan_in2, _ = nn.init._calculate_fan_in_and_fan_out(self.weight2)+ bound2 = 1 / math.sqrt(fan_in2)+ nn.init.uniform_(self.bias2, -bound2, bound2)++ def forward(self, x, use_conv3d=False, skip_time_conv=False):+ if use_conv3d:+ return self.forward_with_3d(x=x, skip_time_conv=skip_time_conv)+ else:+ return self.forward_with_2d(x=x, skip_time_conv=skip_time_conv)++ def forward_with_3d(self, x, skip_time_conv):+ # First convolution+ x = F.conv3d(+ x,+ self.weight1,+ self.bias1,+ self.stride1,+ self.padding1,+ self.dilation1,+ self.groups,+ )++ if skip_time_conv:+ return x++ # Second convolution+ x = F.conv3d(+ x,+ self.weight2,+ self.bias2,+ self.stride2,+ self.padding2,+ self.dilation2,+ self.groups,+ )++ return x++ def forward_with_2d(self, x, skip_time_conv):+ b, c, d, h, w = x.shape++ # First 2D convolution+ x = rearrange(x, "b c d h w -> (b d) c h w")+ # Squeeze the depth dimension out of weight1 since it's 1+ weight1 = self.weight1.squeeze(2)+ # Select stride, padding, and dilation for the 2D convolution+ stride1 = (self.stride1[1], self.stride1[2])+ padding1 = (self.padding1[1], self.padding1[2])+ dilation1 = (self.dilation1[1], self.dilation1[2])+ x = F.conv2d(x, weight1, self.bias1, stride1, padding1, dilation1, self.groups)++ _, _, h, w = x.shape++ if skip_time_conv:+ x = rearrange(x, "(b d) c h w -> b c d h w", b=b)+ return x++ # Second convolution which is essentially treated as a 1D convolution across the 'd' dimension+ x = rearrange(x, "(b d) c h w -> (b h w) c d", b=b)++ # Reshape weight2 to match the expected dimensions for conv1d+ weight2 = self.weight2.squeeze(-1).squeeze(-1)+ # Use only the relevant dimension for stride, padding, and dilation for the 1D convolution+ stride2 = self.stride2[0]+ padding2 = self.padding2[0]+ dilation2 = self.dilation2[0]+ x = F.conv1d(x, weight2, self.bias2, stride2, padding2, dilation2, self.groups)+ x = rearrange(x, "(b h w) c d -> b c d h w", b=b, h=h, w=w)++ return x++ @property+ def weight(self):+ return self.weight2+++def test_dual_conv3d_consistency():+ # Initialize parameters+ in_channels = 3+ out_channels = 5+ kernel_size = (3, 3, 3)+ stride = (2, 2, 2)+ padding = (1, 1, 1)++ # Create an instance of the DualConv3d class+ dual_conv3d = DualConv3d(+ in_channels=in_channels,+ out_channels=out_channels,+ kernel_size=kernel_size,+ stride=stride,+ padding=padding,+ bias=True,+ )++ # Example input tensor+ test_input = torch.randn(1, 3, 10, 10, 10)++ # Perform forward passes with both 3D and 2D settings+ output_conv3d = dual_conv3d(test_input, use_conv3d=True)+ output_2d = dual_conv3d(test_input, use_conv3d=False)++ # Assert that the outputs from both methods are sufficiently close+ assert torch.allclose(+ output_conv3d, output_2d, atol=1e-6+ ), "Outputs are not consistent between 3D and 2D convolutions."diff --git a/comfy/ldm/lightricks/vae/pixel_norm.py b/comfy/ldm/lightricks/vae/pixel_norm.pynew file mode 100644index 000000000..9bc3ea60e--- /dev/null+++ b/comfy/ldm/lightricks/vae/pixel_norm.py@@ -0,0 +1,12 @@+import torch+from torch import nn+++class PixelNorm(nn.Module):+ def __init__(self, dim=1, eps=1e-8):+ super(PixelNorm, self).__init__()+ self.dim = dim+ self.eps = eps++ def forward(self, x):+ return x / torch.sqrt(torch.mean(x**2, dim=self.dim, keepdim=True) + self.eps)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 6f1aa5708..e7bfc8d76 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -30,6 +30,7 @@ import comfy.ldm.hydit.modelsimport comfy.ldm.audio.ditimport comfy.ldm.audio.embeddersimport comfy.ldm.flux.model+import comfy.ldm.lightricks.modelimport comfy.model_managementimport comfy.conds@@ -141,6 +142,7 @@ class BaseModel(torch.nn.Module):extra = extra.to(dtype)extra_conds[o] = extra+ print(t)model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()return self.model_sampling.calculate_denoised(sigma, model_output, x)@@ -767,3 +769,23 @@ class GenmoMochi(BaseModel):if cross_attn is not None:out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)return out++class LTXV(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLUX, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.lightricks.model.LTXVModel) #TODO++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ attention_mask = kwargs.get("attention_mask", None)+ if attention_mask is not None:+ out['attention_mask'] = comfy.conds.CONDRegular(attention_mask)+ cross_attn = kwargs.get("cross_attn", None)+ if cross_attn is not None:+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)++ guiding_latent = kwargs.get("guiding_latent", None)+ if guiding_latent is not None:+ out['guiding_latent'] = comfy.conds.CONDRegular(guiding_latent)++ out['frame_rate'] = comfy.conds.CONDConstant(kwargs.get("frame_rate", 25))+ return outdiff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 008e4b199..c5c5dbb2a 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -183,6 +183,10 @@ def detect_unet_config(state_dict, key_prefix):dit_config["rope_theta"] = 10000.0return dit_config+ if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxv+ dit_config = {}+ dit_config["image_model"] = "ltxv"+ return dit_configif '{}input_blocks.0.0.weight'.format(key_prefix) not in state_dict_keys:return Nonediff --git a/comfy/sd.py b/comfy/sd.pyindex 3dea043fe..b07b5fe37 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -8,6 +8,7 @@ from .ldm.cascade.stage_a import StageAfrom .ldm.cascade.stage_c_coder import StageC_coderfrom .ldm.audio.autoencoder import AudioOobleckVAEimport comfy.ldm.genmo.vae.model+import comfy.ldm.lightricks.vae.causal_video_autoencoderimport yamlimport comfy.utils@@ -27,6 +28,7 @@ import comfy.text_encoders.hyditimport comfy.text_encoders.fluximport comfy.text_encoders.long_cliplimport comfy.text_encoders.genmo+import comfy.text_encoders.ltimport comfy.model_patcherimport comfy.lora@@ -261,6 +263,14 @@ class VAE:self.memory_used_encode = lambda shape, dtype: (1.5 * max(shape[2], 7) * shape[3] * shape[4] * (6 * 8 * 8)) * model_management.dtype_size(dtype)self.upscale_ratio = (lambda a: max(0, a * 6 - 5), 8, 8)self.working_dtypes = [torch.float16, torch.float32]+ elif "decoder.up_blocks.0.res_blocks.0.conv1.conv.weight" in sd: #lightricks ltxv+ self.first_stage_model = comfy.ldm.lightricks.vae.causal_video_autoencoder.VideoVAE()+ self.latent_channels = 128+ self.latent_dim = 3+ self.memory_used_decode = lambda shape, dtype: (900 * shape[2] * shape[3] * shape[4] * (8 * 8 * 8)) * model_management.dtype_size(dtype)+ self.memory_used_encode = lambda shape, dtype: (70 * max(shape[2], 7) * shape[3] * shape[4]) * model_management.dtype_size(dtype)+ self.upscale_ratio = 8+ self.working_dtypes = [torch.bfloat16, torch.float32]else:logging.warning("WARNING: No VAE weights detected, VAE not initalized.")self.first_stage_model = None@@ -452,6 +462,7 @@ class CLIPType(Enum):HUNYUAN_DIT = 5FLUX = 6MOCHI = 7+ LTXV = 8def load_clip(ckpt_paths, embedding_directory=None, clip_type=CLIPType.STABLE_DIFFUSION, model_options={}):clip_data = []@@ -530,6 +541,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipif clip_type == CLIPType.SD3:clip_target.clip = comfy.text_encoders.sd3_clip.sd3_clip(clip_l=False, clip_g=False, t5=True, **t5xxl_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.sd3_clip.SD3Tokenizer+ elif clip_type == CLIPType.LTXV:+ clip_target.clip = comfy.text_encoders.lt.ltxv_te(**t5xxl_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.lt.LTXVT5Tokenizerelse: #CLIPType.MOCHIclip_target.clip = comfy.text_encoders.genmo.mochi_te(**t5xxl_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.genmo.MochiT5Tokenizerdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 75ddaee57..caa1a1846 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -11,6 +11,7 @@ import comfy.text_encoders.aura_t5import comfy.text_encoders.hyditimport comfy.text_encoders.fluximport comfy.text_encoders.genmo+import comfy.text_encoders.ltfrom . import supported_models_basefrom . import latent_formats@@ -702,7 +703,34 @@ class GenmoMochi(supported_models_base.BASE):t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))return supported_models_base.ClipTarget(comfy.text_encoders.genmo.MochiT5Tokenizer, comfy.text_encoders.genmo.mochi_te(**t5_detect))+class LTXV(supported_models_base.BASE):+ unet_config = {+ "image_model": "ltxv",+ }++ sampling_settings = {+ "shift": 2.37,+ }++ unet_extra_config = {}+ latent_format = latent_formats.LTXV++ memory_usage_factor = 2.7++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.LTXV(self, device=device)+ return out++ def clip_target(self, state_dict={}):+ pref = self.text_encoder_key_prefix[0]+ t5_detect = comfy.text_encoders.sd3_clip.t5_xxl_detect(state_dict, "{}t5xxl.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.lt.LTXVT5Tokenizer, comfy.text_encoders.lt.ltxv_te(**t5_detect))-models = [Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, HunyuanDiT, HunyuanDiT1, Flux, FluxSchnell, GenmoMochi]+models = [Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, HunyuanDiT, HunyuanDiT1, Flux, FluxSchnell, GenmoMochi, LTXV]models += [SVD_img2vid]diff --git a/comfy/text_encoders/lt.py b/comfy/text_encoders/lt.pynew file mode 100644index 000000000..5c2ce583f--- /dev/null+++ b/comfy/text_encoders/lt.py@@ -0,0 +1,18 @@+from comfy import sd1_clip+import os+from transformers import T5TokenizerFast+import comfy.text_encoders.genmo++class T5XXLTokenizer(sd1_clip.SDTokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "t5_tokenizer")+ super().__init__(tokenizer_path, embedding_directory=embedding_directory, pad_with_end=False, embedding_size=4096, embedding_key='t5xxl', tokenizer_class=T5TokenizerFast, has_start_token=False, pad_to_max_length=False, max_length=99999999, min_length=128) #pad to 128?+++class LTXVT5Tokenizer(sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, clip_name="t5xxl", tokenizer=T5XXLTokenizer)+++def ltxv_te(*args, **kwargs):+ return comfy.text_encoders.genmo.mochi_te(*args, **kwargs)diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.pynew file mode 100644index 000000000..17177b662--- /dev/null+++ b/comfy_extras/nodes_lt.py@@ -0,0 +1,183 @@+import nodes+import node_helpers+import torch+import comfy.model_management+import comfy.model_sampling+import math++class EmptyLTXVLatentVideo:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "width": ("INT", {"default": 768, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 32}),+ "height": ("INT", {"default": 512, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 32}),+ "length": ("INT", {"default": 97, "min": 9, "max": nodes.MAX_RESOLUTION, "step": 8}),+ "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})}}+ RETURN_TYPES = ("LATENT",)+ FUNCTION = "generate"++ CATEGORY = "latent/video/ltxv"++ def generate(self, width, height, length, batch_size=1):+ latent = torch.zeros([batch_size, 128, ((length - 1) // 8) + 1, height // 32, width // 32], device=comfy.model_management.intermediate_device())+ return ({"samples": latent}, )+++class LTXVImgToVideo:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {"positive": ("CONDITIONING", ),+ "negative": ("CONDITIONING", ),+ "vae": ("VAE",),+ "image": ("IMAGE",),+ "width": ("INT", {"default": 768, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 32}),+ "height": ("INT", {"default": 512, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 32}),+ "length": ("INT", {"default": 97, "min": 9, "max": nodes.MAX_RESOLUTION, "step": 8}),+ "batch_size": ("INT", {"default": 1, "min": 1, "max": 4096})}}++ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")+ RETURN_NAMES = ("positive", "negative", "latent")++ CATEGORY = "conditioning/video_models"+ FUNCTION = "generate"++ def generate(self, positive, negative, image, vae, width, height, length, batch_size):+ pixels = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)+ encode_pixels = pixels[:, :, :, :3]+ t = vae.encode(encode_pixels)+ positive = node_helpers.conditioning_set_values(positive, {"guiding_latent": t})+ negative = node_helpers.conditioning_set_values(negative, {"guiding_latent": t})++ latent = torch.zeros([batch_size, 128, ((length - 1) // 8) + 1, height // 32, width // 32], device=comfy.model_management.intermediate_device())+ latent[:, :, :t.shape[2]] = t+ return (positive, negative, {"samples": latent}, )+++class LTXVConditioning:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {"positive": ("CONDITIONING", ),+ "negative": ("CONDITIONING", ),+ "frame_rate": ("FLOAT", {"default": 25.0, "min": 0.0, "max": 1000.0, "step": 0.01}),+ }}+ RETURN_TYPES = ("CONDITIONING", "CONDITIONING")+ RETURN_NAMES = ("positive", "negative")+ FUNCTION = "append"++ CATEGORY = "conditioning/video_models"++ def append(self, positive, negative, frame_rate):+ positive = node_helpers.conditioning_set_values(positive, {"frame_rate": frame_rate})+ negative = node_helpers.conditioning_set_values(negative, {"frame_rate": frame_rate})+ return (positive, negative)+++class ModelSamplingLTXV:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": { "model": ("MODEL",),+ "max_shift": ("FLOAT", {"default": 2.05, "min": 0.0, "max": 100.0, "step":0.01}),+ "base_shift": ("FLOAT", {"default": 0.95, "min": 0.0, "max": 100.0, "step":0.01}),+ },+ "optional": {"latent": ("LATENT",), }+ }++ RETURN_TYPES = ("MODEL",)+ FUNCTION = "patch"++ CATEGORY = "advanced/model"++ def patch(self, model, max_shift, base_shift, latent=None):+ m = model.clone()++ if latent is None:+ tokens = 4096+ else:+ tokens = math.prod(latent["samples"].shape[2:])++ x1 = 1024+ x2 = 4096+ mm = (max_shift - base_shift) / (x2 - x1)+ b = base_shift - mm * x1+ shift = (tokens) * mm + b++ sampling_base = comfy.model_sampling.ModelSamplingFlux+ sampling_type = comfy.model_sampling.CONST++ class ModelSamplingAdvanced(sampling_base, sampling_type):+ pass++ model_sampling = ModelSamplingAdvanced(model.model.model_config)+ model_sampling.set_parameters(shift=shift)+ m.add_object_patch("model_sampling", model_sampling)+ return (m, )+++class LTXVScheduler:+ @classmethod+ def INPUT_TYPES(s):+ return {"required":+ {"steps": ("INT", {"default": 20, "min": 1, "max": 10000}),+ "max_shift": ("FLOAT", {"default": 2.05, "min": 0.0, "max": 100.0, "step":0.01}),+ "base_shift": ("FLOAT", {"default": 0.95, "min": 0.0, "max": 100.0, "step":0.01}),+ "stretch": ("BOOLEAN", {+ "default": True,+ "tooltip": "Stretch the sigmas to be in the range [terminal, 1]."+ }),+ "terminal": (+ "FLOAT",+ {+ "default": 0.1, "min": 0.0, "max": 0.99, "step": 0.01,+ "tooltip": "The terminal value of the sigmas after stretching."+ },+ ),+ },+ "optional": {"latent": ("LATENT",), }+ }++ RETURN_TYPES = ("SIGMAS",)+ CATEGORY = "sampling/custom_sampling/schedulers"++ FUNCTION = "get_sigmas"++ def get_sigmas(self, steps, max_shift, base_shift, stretch, terminal, latent=None):+ if latent is None:+ tokens = 4096+ else:+ tokens = math.prod(latent["samples"].shape[2:])++ sigmas = torch.linspace(1.0, 0.0, steps + 1)++ x1 = 1024+ x2 = 4096+ mm = (max_shift - base_shift) / (x2 - x1)+ b = base_shift - mm * x1+ sigma_shift = (tokens) * mm + b+ print(sigma_shift)++ power = 1+ sigmas = torch.where(+ sigmas != 0,+ math.exp(sigma_shift) / (math.exp(sigma_shift) + (1 / sigmas - 1) ** power),+ 0,+ )++ # Stretch sigmas so that its final value matches the given terminal value.+ if stretch:+ non_zero_mask = sigmas != 0+ non_zero_sigmas = sigmas[non_zero_mask]+ one_minus_z = 1.0 - non_zero_sigmas+ scale_factor = one_minus_z[-1] / (1.0 - terminal)+ stretched = 1.0 - (one_minus_z / scale_factor)+ sigmas[non_zero_mask] = stretched++ print(sigmas)+ return (sigmas,)+++NODE_CLASS_MAPPINGS = {+ "EmptyLTXVLatentVideo": EmptyLTXVLatentVideo,+ "LTXVImgToVideo": LTXVImgToVideo,+ "ModelSamplingLTXV": ModelSamplingLTXV,+ "LTXVConditioning": LTXVConditioning,+ "LTXVScheduler": LTXVScheduler,+}diff --git a/nodes.py b/nodes.pyindex 7f1515867..01af6c68d 100644--- a/nodes.py+++ b/nodes.py@@ -897,7 +897,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv"], ),}}RETURN_TYPES = ("CLIP",)FUNCTION = "load_clip"@@ -915,6 +915,8 @@ class CLIPLoader:clip_type = comfy.sd.CLIPType.STABLE_AUDIOelif type == "mochi":clip_type = comfy.sd.CLIPType.MOCHI+ elif type == "ltxv":+ clip_type = comfy.sd.CLIPType.LTXVelse:clip_type = comfy.sd.CLIPType.STABLE_DIFFUSION@@ -2136,6 +2138,7 @@ def init_builtin_extra_nodes():"nodes_torch_compile.py","nodes_mochi.py","nodes_slg.py",+ "nodes_lt.py",]import_failed = []