The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support LTXV 0.9.5.
comfyanonymous93fedd92fe0e
diff --git a/comfy/ldm/lightricks/model.py b/comfy/ldm/lightricks/model.pyindex 2a02acd65..6e8e06181 100644--- a/comfy/ldm/lightricks/model.py+++ b/comfy/ldm/lightricks/model.py@@ -7,7 +7,7 @@ from einops import rearrangeimport mathfrom typing import Dict, Optional, Tuple-from .symmetric_patchifier import SymmetricPatchifier+from .symmetric_patchifier import SymmetricPatchifier, latent_to_pixel_coordsdef get_timestep_embedding(@@ -377,12 +377,16 @@ class LTXVModel(torch.nn.Module):positional_embedding_theta=10000.0,positional_embedding_max_pos=[20, 2048, 2048],+ causal_temporal_positioning=False,+ vae_scale_factors=(8, 32, 32),dtype=None, device=None, operations=None, **kwargs):super().__init__()self.generator = None+ self.vae_scale_factors = vae_scale_factorsself.dtype = dtypeself.out_channels = in_channelsself.inner_dim = num_attention_heads * attention_head_dim+ self.causal_temporal_positioning = causal_temporal_positioningself.patchify_proj = operations.Linear(in_channels, self.inner_dim, bias=True, dtype=dtype, device=device)@@ -416,42 +420,23 @@ class LTXVModel(torch.nn.Module):self.patchifier = SymmetricPatchifier(1)- def forward(self, x, timestep, context, attention_mask, frame_rate=25, guiding_latent=None, guiding_latent_noise_scale=0, transformer_options={}, **kwargs):+ def forward(self, x, timestep, context, attention_mask, frame_rate=25, transformer_options={}, keyframe_idxs=None, **kwargs):patches_replace = transformer_options.get("patches_replace", {})- indices_grid = self.patchifier.get_grid(- orig_num_frames=x.shape[2],- orig_height=x.shape[3],- orig_width=x.shape[4],- batch_size=x.shape[0],- scale_grid=((1 / frame_rate) * 8, 32, 32),- device=x.device,- )-- if guiding_latent is not None:- ts = torch.ones([x.shape[0], 1, x.shape[2], x.shape[3], x.shape[4]], device=x.device, dtype=x.dtype)- input_ts = timestep.view([timestep.shape[0]] + [1] * (x.ndim - 1))- ts *= input_ts- ts[:, :, 0] = guiding_latent_noise_scale * (input_ts[:, :, 0] ** 2)- timestep = self.patchifier.patchify(ts)- input_x = x.clone()- x[:, :, 0] = guiding_latent[:, :, 0]- if guiding_latent_noise_scale > 0:- if self.generator is None:- self.generator = torch.Generator(device=x.device).manual_seed(42)- elif self.generator.device != x.device:- self.generator = torch.Generator(device=x.device).set_state(self.generator.get_state())-- noise_shape = [guiding_latent.shape[0], guiding_latent.shape[1], 1, guiding_latent.shape[3], guiding_latent.shape[4]]- scale = guiding_latent_noise_scale * (input_ts ** 2)- guiding_noise = scale * torch.randn(size=noise_shape, device=x.device, generator=self.generator)-- x[:, :, 0] = guiding_noise[:, :, 0] + x[:, :, 0] * (1.0 - scale[:, :, 0])+ orig_shape = list(x.shape)+ x, latent_coords = self.patchifier.patchify(x)+ pixel_coords = latent_to_pixel_coords(+ latent_coords=latent_coords,+ scale_factors=self.vae_scale_factors,+ causal_fix=self.causal_temporal_positioning,+ )- orig_shape = list(x.shape)+ if keyframe_idxs is not None:+ pixel_coords[:, :, -keyframe_idxs.shape[2]:] = keyframe_idxs- x = self.patchifier.patchify(x)+ fractional_coords = pixel_coords.to(torch.float32)+ fractional_coords[:, 0] = fractional_coords[:, 0] * (1.0 / frame_rate)x = self.patchify_proj(x)timestep = timestep * 1000.0@@ -459,7 +444,7 @@ class LTXVModel(torch.nn.Module):if attention_mask is not None and not torch.is_floating_point(attention_mask):attention_mask = (attention_mask - 1).to(x.dtype).reshape((attention_mask.shape[0], 1, -1, attention_mask.shape[-1])) * torch.finfo(x.dtype).max- pe = precompute_freqs_cis(indices_grid, dim=self.inner_dim, out_dtype=x.dtype)+ pe = precompute_freqs_cis(fractional_coords, dim=self.inner_dim, out_dtype=x.dtype)batch_size = x.shape[0]timestep, embedded_timestep = self.adaln_single(@@ -519,8 +504,4 @@ class LTXVModel(torch.nn.Module):out_channels=orig_shape[1] // math.prod(self.patchifier.patch_size),)- if guiding_latent is not None:- x[:, :, 0] = (input_x[:, :, 0] - guiding_latent[:, :, 0]) / input_ts[:, :, 0]-- # print("res", x)return xdiff --git a/comfy/ldm/lightricks/symmetric_patchifier.py b/comfy/ldm/lightricks/symmetric_patchifier.pyindex c58dfb20b..4b9972b9f 100644--- a/comfy/ldm/lightricks/symmetric_patchifier.py+++ b/comfy/ldm/lightricks/symmetric_patchifier.py@@ -6,16 +6,29 @@ from einops import rearrangefrom torch import Tensor-def append_dims(x: torch.Tensor, target_dims: int) -> torch.Tensor:- """Appends dimensions to the end of a tensor until it has target_dims dimensions."""- dims_to_append = target_dims - x.ndim- if dims_to_append < 0:- raise ValueError(- f"input has {x.ndim} dims but target_dims is {target_dims}, which is less"- )- elif dims_to_append == 0:- return x- return x[(...,) + (None,) * dims_to_append]+def latent_to_pixel_coords(+ latent_coords: Tensor, scale_factors: Tuple[int, int, int], causal_fix: bool = False+) -> Tensor:+ """+ Converts latent coordinates to pixel coordinates by scaling them according to the VAE's+ configuration.+ Args:+ latent_coords (Tensor): A tensor of shape [batch_size, 3, num_latents]+ containing the latent corner coordinates of each token.+ scale_factors (Tuple[int, int, int]): The scale factors of the VAE's latent space.+ causal_fix (bool): Whether to take into account the different temporal scale+ of the first frame. Default = False for backwards compatibility.+ Returns:+ Tensor: A tensor of pixel coordinates corresponding to the input latent coordinates.+ """+ pixel_coords = (+ latent_coords+ * torch.tensor(scale_factors, device=latent_coords.device)[None, :, None]+ )+ if causal_fix:+ # Fix temporal scale for first frame to 1 due to causality+ pixel_coords[:, 0] = (pixel_coords[:, 0] + 1 - scale_factors[0]).clamp(min=0)+ return pixel_coordsclass Patchifier(ABC):@@ -44,29 +57,26 @@ class Patchifier(ABC):def patch_size(self):return self._patch_size- def get_grid(- self, orig_num_frames, orig_height, orig_width, batch_size, scale_grid, device+ def get_latent_coords(+ self, latent_num_frames, latent_height, latent_width, batch_size, device):- f = orig_num_frames // self._patch_size[0]- h = orig_height // self._patch_size[1]- w = orig_width // self._patch_size[2]- grid_h = torch.arange(h, dtype=torch.float32, device=device)- grid_w = torch.arange(w, dtype=torch.float32, device=device)- grid_f = torch.arange(f, dtype=torch.float32, device=device)- grid = torch.meshgrid(grid_f, grid_h, grid_w, indexing='ij')- grid = torch.stack(grid, dim=0)- grid = grid.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)-- if scale_grid is not None:- for i in range(3):- if isinstance(scale_grid[i], Tensor):- scale = append_dims(scale_grid[i], grid.ndim - 1)- else:- scale = scale_grid[i]- grid[:, i, ...] = grid[:, i, ...] * scale * self._patch_size[i]-- grid = rearrange(grid, "b c f h w -> b c (f h w)", b=batch_size)- return grid+ """+ Return a tensor of shape [batch_size, 3, num_patches] containing the+ top-left corner latent coordinates of each latent patch.+ The tensor is repeated for each batch element.+ """+ latent_sample_coords = torch.meshgrid(+ torch.arange(0, latent_num_frames, self._patch_size[0], device=device),+ torch.arange(0, latent_height, self._patch_size[1], device=device),+ torch.arange(0, latent_width, self._patch_size[2], device=device),+ indexing="ij",+ )+ latent_sample_coords = torch.stack(latent_sample_coords, dim=0)+ latent_coords = latent_sample_coords.unsqueeze(0).repeat(batch_size, 1, 1, 1, 1)+ latent_coords = rearrange(+ latent_coords, "b c f h w -> b c (f h w)", b=batch_size+ )+ return latent_coordsclass SymmetricPatchifier(Patchifier):@@ -74,6 +84,8 @@ class SymmetricPatchifier(Patchifier):self,latents: Tensor,) -> Tuple[Tensor, Tensor]:+ b, _, f, h, w = latents.shape+ latent_coords = self.get_latent_coords(f, h, w, b, latents.device)latents = rearrange(latents,"b c (f p1) (h p2) (w p3) -> b (f h w) (c p1 p2 p3)",@@ -81,7 +93,7 @@ class SymmetricPatchifier(Patchifier):p2=self._patch_size[1],p3=self._patch_size[2],)- return latents+ return latents, latent_coordsdef unpatchify(self,diff --git a/comfy/ldm/lightricks/vae/causal_conv3d.py b/comfy/ldm/lightricks/vae/causal_conv3d.pyindex c572e7e86..70d612e86 100644--- a/comfy/ldm/lightricks/vae/causal_conv3d.py+++ b/comfy/ldm/lightricks/vae/causal_conv3d.py@@ -15,6 +15,7 @@ class CausalConv3d(nn.Module):stride: Union[int, Tuple[int]] = 1,dilation: int = 1,groups: int = 1,+ spatial_padding_mode: str = "zeros",**kwargs,):super().__init__()@@ -38,7 +39,7 @@ class CausalConv3d(nn.Module):stride=stride,dilation=dilation,padding=padding,- padding_mode="zeros",+ padding_mode=spatial_padding_mode,groups=groups,)diff --git a/comfy/ldm/lightricks/vae/causal_video_autoencoder.py b/comfy/ldm/lightricks/vae/causal_video_autoencoder.pyindex e0344deec..043ca0496 100644--- a/comfy/ldm/lightricks/vae/causal_video_autoencoder.py+++ b/comfy/ldm/lightricks/vae/causal_video_autoencoder.py@@ -1,13 +1,15 @@+from __future__ import annotationsimport torchfrom torch import nnfrom functools import partialimport mathfrom einops import rearrange-from typing import Optional, Tuple, Union+from typing import List, Optional, Tuple, Unionfrom .conv_nd_factory import make_conv_nd, make_linear_ndfrom .pixel_norm import PixelNormfrom ..model import PixArtAlphaCombinedTimestepSizeEmbeddingsimport comfy.ops+ops = comfy.ops.disable_weight_initclass Encoder(nn.Module):@@ -32,7 +34,7 @@ class Encoder(nn.Module):norm_layer (`str`, *optional*, defaults to `group_norm`):The normalization layer to use. Can be either `group_norm` or `pixel_norm`.latent_log_var (`str`, *optional*, defaults to `per_channel`):- The number of channels for the log variance. Can be either `per_channel`, `uniform`, or `none`.+ The number of channels for the log variance. Can be either `per_channel`, `uniform`, `constant` or `none`."""def __init__(@@ -40,12 +42,13 @@ class Encoder(nn.Module):dims: Union[int, Tuple[int, int]] = 3,in_channels: int = 3,out_channels: int = 3,- blocks=[("res_x", 1)],+ blocks: List[Tuple[str, int | dict]] = [("res_x", 1)],base_channels: int = 128,norm_num_groups: int = 32,patch_size: Union[int, Tuple[int]] = 1,norm_layer: str = "group_norm", # group_norm, pixel_normlatent_log_var: str = "per_channel",+ spatial_padding_mode: str = "zeros",):super().__init__()self.patch_size = patch_size@@ -65,6 +68,7 @@ class Encoder(nn.Module):stride=1,padding=1,causal=True,+ spatial_padding_mode=spatial_padding_mode,)self.down_blocks = nn.ModuleList([])@@ -82,6 +86,7 @@ class Encoder(nn.Module):resnet_eps=1e-6,resnet_groups=norm_num_groups,norm_layer=norm_layer,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "res_x_y":output_channel = block_params.get("multiplier", 2) * output_channel@@ -92,6 +97,7 @@ class Encoder(nn.Module):eps=1e-6,groups=norm_num_groups,norm_layer=norm_layer,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_time":block = make_conv_nd(@@ -101,6 +107,7 @@ class Encoder(nn.Module):kernel_size=3,stride=(2, 1, 1),causal=True,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_space":block = make_conv_nd(@@ -110,6 +117,7 @@ class Encoder(nn.Module):kernel_size=3,stride=(1, 2, 2),causal=True,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_all":block = make_conv_nd(@@ -119,6 +127,7 @@ class Encoder(nn.Module):kernel_size=3,stride=(2, 2, 2),causal=True,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_all_x_y":output_channel = block_params.get("multiplier", 2) * output_channel@@ -129,6 +138,34 @@ class Encoder(nn.Module):kernel_size=3,stride=(2, 2, 2),causal=True,+ spatial_padding_mode=spatial_padding_mode,+ )+ elif block_name == "compress_all_res":+ output_channel = block_params.get("multiplier", 2) * output_channel+ block = SpaceToDepthDownsample(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ stride=(2, 2, 2),+ spatial_padding_mode=spatial_padding_mode,+ )+ elif block_name == "compress_space_res":+ output_channel = block_params.get("multiplier", 2) * output_channel+ block = SpaceToDepthDownsample(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ stride=(1, 2, 2),+ spatial_padding_mode=spatial_padding_mode,+ )+ elif block_name == "compress_time_res":+ output_channel = block_params.get("multiplier", 2) * output_channel+ block = SpaceToDepthDownsample(+ dims=dims,+ in_channels=input_channel,+ out_channels=output_channel,+ stride=(2, 1, 1),+ spatial_padding_mode=spatial_padding_mode,)else:raise ValueError(f"unknown block: {block_name}")@@ -152,10 +189,18 @@ class Encoder(nn.Module):conv_out_channels *= 2elif latent_log_var == "uniform":conv_out_channels += 1+ elif latent_log_var == "constant":+ conv_out_channels += 1elif latent_log_var != "none":raise ValueError(f"Invalid latent_log_var: {latent_log_var}")self.conv_out = make_conv_nd(- dims, output_channel, conv_out_channels, 3, padding=1, causal=True+ dims,+ output_channel,+ conv_out_channels,+ 3,+ padding=1,+ causal=True,+ spatial_padding_mode=spatial_padding_mode,)self.gradient_checkpointing = False@@ -197,6 +242,15 @@ class Encoder(nn.Module):sample = torch.cat([sample, repeated_last_channel], dim=1)else:raise ValueError(f"Invalid input shape: {sample.shape}")+ elif self.latent_log_var == "constant":+ sample = sample[:, :-1, ...]+ approx_ln_0 = (+ -30+ ) # this is the minimal clamp value in DiagonalGaussianDistribution objects+ sample = torch.cat(+ [sample, torch.ones_like(sample, device=sample.device) * approx_ln_0],+ dim=1,+ )return sample@@ -231,7 +285,7 @@ class Decoder(nn.Module):dims,in_channels: int = 3,out_channels: int = 3,- blocks=[("res_x", 1)],+ blocks: List[Tuple[str, int | dict]] = [("res_x", 1)],base_channels: int = 128,layers_per_block: int = 2,norm_num_groups: int = 32,@@ -239,6 +293,7 @@ class Decoder(nn.Module):norm_layer: str = "group_norm",causal: bool = True,timestep_conditioning: bool = False,+ spatial_padding_mode: str = "zeros",):super().__init__()self.patch_size = patch_size@@ -264,6 +319,7 @@ class Decoder(nn.Module):stride=1,padding=1,causal=True,+ spatial_padding_mode=spatial_padding_mode,)self.up_blocks = nn.ModuleList([])@@ -283,6 +339,7 @@ class Decoder(nn.Module):norm_layer=norm_layer,inject_noise=block_params.get("inject_noise", False),timestep_conditioning=timestep_conditioning,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "attn_res_x":block = UNetMidBlock3D(@@ -294,6 +351,7 @@ class Decoder(nn.Module):inject_noise=block_params.get("inject_noise", False),timestep_conditioning=timestep_conditioning,attention_head_dim=block_params["attention_head_dim"],+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "res_x_y":output_channel = output_channel // block_params.get("multiplier", 2)@@ -306,14 +364,21 @@ class Decoder(nn.Module):norm_layer=norm_layer,inject_noise=block_params.get("inject_noise", False),timestep_conditioning=False,+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_time":block = DepthToSpaceUpsample(- dims=dims, in_channels=input_channel, stride=(2, 1, 1)+ dims=dims,+ in_channels=input_channel,+ stride=(2, 1, 1),+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_space":block = DepthToSpaceUpsample(- dims=dims, in_channels=input_channel, stride=(1, 2, 2)+ dims=dims,+ in_channels=input_channel,+ stride=(1, 2, 2),+ spatial_padding_mode=spatial_padding_mode,)elif block_name == "compress_all":output_channel = output_channel // block_params.get("multiplier", 1)@@ -323,6 +388,7 @@ class Decoder(nn.Module):stride=(2, 2, 2),residual=block_params.get("residual", False),out_channels_reduction_factor=block_params.get("multiplier", 1),+ spatial_padding_mode=spatial_padding_mode,)else:raise ValueError(f"unknown layer: {block_name}")@@ -340,7 +406,13 @@ class Decoder(nn.Module):self.conv_act = nn.SiLU()self.conv_out = make_conv_nd(- dims, output_channel, out_channels, 3, padding=1, causal=True+ dims,+ output_channel,+ out_channels,+ 3,+ padding=1,+ causal=True,+ spatial_padding_mode=spatial_padding_mode,)self.gradient_checkpointing = False@@ -433,6 +505,12 @@ class UNetMidBlock3D(nn.Module):resnet_eps (`float`, *optional*, 1e-6 ): The epsilon value for the resnet blocks.resnet_groups (`int`, *optional*, defaults to 32):The number of groups to use in the group normalization layers of the resnet blocks.+ norm_layer (`str`, *optional*, defaults to `group_norm`):+ The normalization layer to use. Can be either `group_norm` or `pixel_norm`.+ inject_noise (`bool`, *optional*, defaults to `False`):+ Whether to inject noise into the hidden states.+ timestep_conditioning (`bool`, *optional*, defaults to `False`):+ Whether to condition the hidden states on the timestep.Returns:`torch.FloatTensor`: The output of the last residual block, which is a tensor of shape `(batch_size,@@ -451,6 +529,7 @@ class UNetMidBlock3D(nn.Module):norm_layer: str = "group_norm",inject_noise: bool = False,timestep_conditioning: bool = False,+ spatial_padding_mode: str = "zeros",):super().__init__()resnet_groups = (@@ -476,13 +555,17 @@ class UNetMidBlock3D(nn.Module):norm_layer=norm_layer,inject_noise=inject_noise,timestep_conditioning=timestep_conditioning,+ spatial_padding_mode=spatial_padding_mode,)for _ in range(num_layers)])def forward(- self, hidden_states: torch.FloatTensor, causal: bool = True, timestep: Optional[torch.Tensor] = None+ self,+ hidden_states: torch.FloatTensor,+ causal: bool = True,+ timestep: Optional[torch.Tensor] = None,) -> torch.FloatTensor:timestep_embed = Noneif self.timestep_conditioning:@@ -507,9 +590,62 @@ class UNetMidBlock3D(nn.Module):return hidden_states+class SpaceToDepthDownsample(nn.Module):+ def __init__(self, dims, in_channels, out_channels, stride, spatial_padding_mode):+ super().__init__()+ self.stride = stride+ self.group_size = in_channels * math.prod(stride) // out_channels+ self.conv = make_conv_nd(+ dims=dims,+ in_channels=in_channels,+ out_channels=out_channels // math.prod(stride),+ kernel_size=3,+ stride=1,+ causal=True,+ spatial_padding_mode=spatial_padding_mode,+ )++ def forward(self, x, causal: bool = True):+ if self.stride[0] == 2:+ x = torch.cat(+ [x[:, :, :1, :, :], x], dim=2+ ) # duplicate first frames for padding++ # skip connection+ x_in = rearrange(+ x,+ "b c (d p1) (h p2) (w p3) -> b (c p1 p2 p3) d h w",+ p1=self.stride[0],+ p2=self.stride[1],+ p3=self.stride[2],+ )+ x_in = rearrange(x_in, "b (c g) d h w -> b c g d h w", g=self.group_size)+ x_in = x_in.mean(dim=2)++ # conv+ x = self.conv(x, causal=causal)+ x = rearrange(+ x,+ "b c (d p1) (h p2) (w p3) -> b (c p1 p2 p3) d h w",+ p1=self.stride[0],+ p2=self.stride[1],+ p3=self.stride[2],+ )++ x = x + x_in++ return x++class DepthToSpaceUpsample(nn.Module):def __init__(- self, dims, in_channels, stride, residual=False, out_channels_reduction_factor=1+ self,+ dims,+ in_channels,+ stride,+ residual=False,+ out_channels_reduction_factor=1,+ spatial_padding_mode="zeros",):super().__init__()self.stride = stride@@ -523,6 +659,7 @@ class DepthToSpaceUpsample(nn.Module):kernel_size=3,stride=1,causal=True,+ spatial_padding_mode=spatial_padding_mode,)self.residual = residualself.out_channels_reduction_factor = out_channels_reduction_factor@@ -591,6 +728,7 @@ class ResnetBlock3D(nn.Module):norm_layer: str = "group_norm",inject_noise: bool = False,timestep_conditioning: bool = False,+ spatial_padding_mode: str = "zeros",):super().__init__()self.in_channels = in_channels@@ -617,6 +755,7 @@ class ResnetBlock3D(nn.Module):stride=1,padding=1,causal=True,+ spatial_padding_mode=spatial_padding_mode,)if inject_noise:@@ -641,6 +780,7 @@ class ResnetBlock3D(nn.Module):stride=1,padding=1,causal=True,+ spatial_padding_mode=spatial_padding_mode,)if inject_noise:@@ -801,9 +941,44 @@ class processor(nn.Module):return (x - self.get_buffer("mean-of-means").view(1, -1, 1, 1, 1).to(x)) / self.get_buffer("std-of-means").view(1, -1, 1, 1, 1).to(x)class VideoVAE(nn.Module):- def __init__(self, version=0):+ def __init__(self, version=0, config=None):super().__init__()+ if config is None:+ config = self.guess_config(version)++ self.timestep_conditioning = config.get("timestep_conditioning", False)+ double_z = config.get("double_z", True)+ latent_log_var = config.get(+ "latent_log_var", "per_channel" if double_z else "none"+ )++ self.encoder = Encoder(+ dims=config["dims"],+ in_channels=config.get("in_channels", 3),+ out_channels=config["latent_channels"],+ blocks=config.get("encoder_blocks", config.get("encoder_blocks", config.get("blocks"))),+ patch_size=config.get("patch_size", 1),+ latent_log_var=latent_log_var,+ norm_layer=config.get("norm_layer", "group_norm"),+ spatial_padding_mode=config.get("spatial_padding_mode", "zeros"),+ )++ self.decoder = Decoder(+ dims=config["dims"],+ in_channels=config["latent_channels"],+ out_channels=config.get("out_channels", 3),+ blocks=config.get("decoder_blocks", config.get("decoder_blocks", config.get("blocks"))),+ patch_size=config.get("patch_size", 1),+ norm_layer=config.get("norm_layer", "group_norm"),+ causal=config.get("causal_decoder", False),+ timestep_conditioning=self.timestep_conditioning,+ spatial_padding_mode=config.get("spatial_padding_mode", "zeros"),+ )++ self.per_channel_statistics = processor()++ def guess_config(self, version):if version == 0:config = {"_class_name": "CausalVideoAutoencoder",@@ -830,7 +1005,7 @@ class VideoVAE(nn.Module):"use_quant_conv": False,"causal_decoder": False,}- else:+ elif version == 1:config = {"_class_name": "CausalVideoAutoencoder","dims": 3,@@ -866,37 +1041,47 @@ class VideoVAE(nn.Module):"causal_decoder": False,"timestep_conditioning": True,}-- double_z = config.get("double_z", True)- latent_log_var = config.get(- "latent_log_var", "per_channel" if double_z else "none"- )-- self.encoder = Encoder(- dims=config["dims"],- in_channels=config.get("in_channels", 3),- out_channels=config["latent_channels"],- blocks=config.get("encoder_blocks", config.get("encoder_blocks", config.get("blocks"))),- patch_size=config.get("patch_size", 1),- latent_log_var=latent_log_var,- norm_layer=config.get("norm_layer", "group_norm"),- )-- self.decoder = Decoder(- dims=config["dims"],- in_channels=config["latent_channels"],- out_channels=config.get("out_channels", 3),- blocks=config.get("decoder_blocks", config.get("decoder_blocks", config.get("blocks"))),- patch_size=config.get("patch_size", 1),- norm_layer=config.get("norm_layer", "group_norm"),- causal=config.get("causal_decoder", False),- timestep_conditioning=config.get("timestep_conditioning", False),- )-- self.timestep_conditioning = config.get("timestep_conditioning", False)- self.per_channel_statistics = processor()+ else:+ config = {+ "_class_name": "CausalVideoAutoencoder",+ "dims": 3,+ "in_channels": 3,+ "out_channels": 3,+ "latent_channels": 128,+ "encoder_blocks": [+ ["res_x", {"num_layers": 4}],+ ["compress_space_res", {"multiplier": 2}],+ ["res_x", {"num_layers": 6}],+ ["compress_time_res", {"multiplier": 2}],+ ["res_x", {"num_layers": 6}],+ ["compress_all_res", {"multiplier": 2}],+ ["res_x", {"num_layers": 2}],+ ["compress_all_res", {"multiplier": 2}],+ ["res_x", {"num_layers": 2}]+ ],+ "decoder_blocks": [+ ["res_x", {"num_layers": 5, "inject_noise": False}],+ ["compress_all", {"residual": True, "multiplier": 2}],+ ["res_x", {"num_layers": 5, "inject_noise": False}],+ ["compress_all", {"residual": True, "multiplier": 2}],+ ["res_x", {"num_layers": 5, "inject_noise": False}],+ ["compress_all", {"residual": True, "multiplier": 2}],+ ["res_x", {"num_layers": 5, "inject_noise": False}]+ ],+ "scaling_factor": 1.0,+ "norm_layer": "pixel_norm",+ "patch_size": 4,+ "latent_log_var": "uniform",+ "use_quant_conv": False,+ "causal_decoder": False,+ "timestep_conditioning": True+ }+ return configdef encode(self, x):+ frames_count = x.shape[2]+ if ((frames_count - 1) % 8) != 0:+ raise ValueError("Invalid number of frames: Encode input must have 1 + 8 * x frames (e.g., 1, 9, 17, ...). Please check your input.")means, logvar = torch.chunk(self.encoder(x), 2, dim=1)return self.per_channel_statistics.normalize(means)diff --git a/comfy/ldm/lightricks/vae/conv_nd_factory.py b/comfy/ldm/lightricks/vae/conv_nd_factory.pyindex 52df4ee22..b4026b14f 100644--- a/comfy/ldm/lightricks/vae/conv_nd_factory.py+++ b/comfy/ldm/lightricks/vae/conv_nd_factory.py@@ -17,7 +17,11 @@ def make_conv_nd(groups=1,bias=True,causal=False,+ spatial_padding_mode="zeros",+ temporal_padding_mode="zeros",):+ if not (spatial_padding_mode == temporal_padding_mode or causal):+ raise NotImplementedError("spatial and temporal padding modes must be equal")if dims == 2:return ops.Conv2d(in_channels=in_channels,@@ -28,6 +32,7 @@ def make_conv_nd(dilation=dilation,groups=groups,bias=bias,+ padding_mode=spatial_padding_mode,)elif dims == 3:if causal:@@ -40,6 +45,7 @@ def make_conv_nd(dilation=dilation,groups=groups,bias=bias,+ spatial_padding_mode=spatial_padding_mode,)return ops.Conv3d(in_channels=in_channels,@@ -50,6 +56,7 @@ def make_conv_nd(dilation=dilation,groups=groups,bias=bias,+ padding_mode=spatial_padding_mode,)elif dims == (2, 1):return DualConv3d(@@ -59,6 +66,7 @@ def make_conv_nd(stride=stride,padding=padding,bias=bias,+ padding_mode=spatial_padding_mode,)else:raise ValueError(f"unsupported dimensions: {dims}")diff --git a/comfy/ldm/lightricks/vae/dual_conv3d.py b/comfy/ldm/lightricks/vae/dual_conv3d.pyindex 6bd54c0a6..dcf889296 100644--- a/comfy/ldm/lightricks/vae/dual_conv3d.py+++ b/comfy/ldm/lightricks/vae/dual_conv3d.py@@ -18,11 +18,13 @@ class DualConv3d(nn.Module):dilation: Union[int, Tuple[int, int, int]] = 1,groups=1,bias=True,+ padding_mode="zeros",):super(DualConv3d, self).__init__()self.in_channels = in_channelsself.out_channels = out_channels+ self.padding_mode = padding_mode# Ensure kernel_size, stride, padding, and dilation are tuples of length 3if isinstance(kernel_size, int):kernel_size = (kernel_size, kernel_size, kernel_size)@@ -108,6 +110,7 @@ class DualConv3d(nn.Module):self.padding1,self.dilation1,self.groups,+ padding_mode=self.padding_mode,)if skip_time_conv:@@ -122,6 +125,7 @@ class DualConv3d(nn.Module):self.padding2,self.dilation2,self.groups,+ padding_mode=self.padding_mode,)return x@@ -137,7 +141,16 @@ class DualConv3d(nn.Module):stride1 = (self.stride1[1], self.stride1[2])padding1 = (self.padding1[1], self.padding1[2])dilation1 = (self.dilation1[1], self.dilation1[2])- x = F.conv2d(x, weight1, self.bias1, stride1, padding1, dilation1, self.groups)+ x = F.conv2d(+ x,+ weight1,+ self.bias1,+ stride1,+ padding1,+ dilation1,+ self.groups,+ padding_mode=self.padding_mode,+ )_, _, h, w = x.shape@@ -154,7 +167,16 @@ class DualConv3d(nn.Module):stride2 = self.stride2[0]padding2 = self.padding2[0]dilation2 = self.dilation2[0]- x = F.conv1d(x, weight2, self.bias2, stride2, padding2, dilation2, self.groups)+ x = F.conv1d(+ x,+ weight2,+ self.bias2,+ stride2,+ padding2,+ dilation2,+ self.groups,+ padding_mode=self.padding_mode,+ )x = rearrange(x, "(b h w) c d -> b c d h w", b=b, h=h, w=w)return xdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex cddc4663e..07fd2db43 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -161,9 +161,13 @@ class BaseModel(torch.nn.Module):extra = extra.to(dtype)extra_conds[o] = extra+ t = self.process_timestep(t, x=x, **extra_conds)model_output = self.diffusion_model(xc, t, context=context, control=control, transformer_options=transformer_options, **extra_conds).float()return self.model_sampling.calculate_denoised(sigma, model_output, x)+ def process_timestep(self, timestep, **kwargs):+ return timestep+def get_dtype(self):return self.diffusion_model.dtype@@ -855,17 +859,26 @@ class LTXV(BaseModel):if cross_attn is not None:out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)- guiding_latent = kwargs.get("guiding_latent", None)- if guiding_latent is not None:- out['guiding_latent'] = comfy.conds.CONDRegular(guiding_latent)+ out['frame_rate'] = comfy.conds.CONDConstant(kwargs.get("frame_rate", 25))++ denoise_mask = kwargs.get("concat_mask", kwargs.get("denoise_mask", None))+ if denoise_mask is not None:+ out["denoise_mask"] = comfy.conds.CONDRegular(denoise_mask)- guiding_latent_noise_scale = kwargs.get("guiding_latent_noise_scale", None)- if guiding_latent_noise_scale is not None:- out["guiding_latent_noise_scale"] = comfy.conds.CONDConstant(guiding_latent_noise_scale)+ keyframe_idxs = kwargs.get("keyframe_idxs", None)+ if keyframe_idxs is not None:+ out['keyframe_idxs'] = comfy.conds.CONDRegular(keyframe_idxs)- out['frame_rate'] = comfy.conds.CONDConstant(kwargs.get("frame_rate", 25))return out+ def process_timestep(self, timestep, x, denoise_mask=None, **kwargs):+ if denoise_mask is None:+ return timestep+ return self.diffusion_model.patchifier.patchify(((denoise_mask) * timestep.view([timestep.shape[0]] + [1] * (denoise_mask.ndim - 1)))[:, :1])[0]++ def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):+ return latent_image+class HunyuanVideo(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan_video.model.HunyuanVideo)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex f149a4bf7..1aef549f4 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -1,3 +1,4 @@+import jsonimport comfy.supported_modelsimport comfy.supported_models_baseimport comfy.utils@@ -33,7 +34,7 @@ def calculate_transformer_depth(prefix, state_dict_keys, state_dict):return last_transformer_depth, context_dim, use_linear_in_transformer, time_stack, time_stack_crossreturn None-def detect_unet_config(state_dict, key_prefix):+def detect_unet_config(state_dict, key_prefix, metadata=None):state_dict_keys = list(state_dict.keys())if '{}joint_blocks.0.context_block.attn.qkv.weight'.format(key_prefix) in state_dict_keys: #mmdit model@@ -210,6 +211,8 @@ def detect_unet_config(state_dict, key_prefix):if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxvdit_config = {}dit_config["image_model"] = "ltxv"+ if metadata is not None and "config" in metadata:+ dit_config.update(json.loads(metadata["config"]).get("transformer", {}))return dit_configif '{}t_block.1.weight'.format(key_prefix) in state_dict_keys: # PixArt@@ -454,8 +457,8 @@ def model_config_from_unet_config(unet_config, state_dict=None):logging.error("no match {}".format(unet_config))return None-def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=False):- unet_config = detect_unet_config(state_dict, unet_key_prefix)+def model_config_from_unet(state_dict, unet_key_prefix, use_base_if_no_match=False, metadata=None):+ unet_config = detect_unet_config(state_dict, unet_key_prefix, metadata=metadata)if unet_config is None:return Nonemodel_config = model_config_from_unet_config(unet_config, state_dict)diff --git a/comfy/sd.py b/comfy/sd.pyindex b866c66c4..fd98585a1 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1,4 +1,5 @@from __future__ import annotations+import jsonimport torchfrom enum import Enumimport logging@@ -249,7 +250,7 @@ class CLIP:return self.patcher.get_key_patches()class VAE:- def __init__(self, sd=None, device=None, config=None, dtype=None):+ def __init__(self, sd=None, device=None, config=None, dtype=None, metadata=None):if 'decoder.up_blocks.0.resnets.0.norm1.weight' in sd.keys(): #diffusers formatsd = diffusers_convert.convert_vae_state_dict(sd)@@ -357,7 +358,12 @@ class VAE:version = 0elif tensor_conv1.shape[0] == 1024:version = 1- self.first_stage_model = comfy.ldm.lightricks.vae.causal_video_autoencoder.VideoVAE(version=version)+ if "encoder.down_blocks.1.conv.conv.bias" in sd:+ version = 2+ vae_config = None+ if metadata is not None and "config" in metadata:+ vae_config = json.loads(metadata["config"]).get("vae", None)+ self.first_stage_model = comfy.ldm.lightricks.vae.causal_video_autoencoder.VideoVAE(version=version, config=vae_config)self.latent_channels = 128self.latent_dim = 3self.memory_used_decode = lambda shape, dtype: (900 * shape[2] * shape[3] * shape[4] * (8 * 8 * 8)) * model_management.dtype_size(dtype)@@ -873,13 +879,13 @@ def load_checkpoint(config_path=None, ckpt_path=None, output_vae=True, output_clreturn (model, clip, vae)def load_checkpoint_guess_config(ckpt_path, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}, te_model_options={}):- sd = comfy.utils.load_torch_file(ckpt_path)- out = load_state_dict_guess_config(sd, output_vae, output_clip, output_clipvision, embedding_directory, output_model, model_options, te_model_options=te_model_options)+ sd, metadata = comfy.utils.load_torch_file(ckpt_path, return_metadata=True)+ out = load_state_dict_guess_config(sd, output_vae, output_clip, output_clipvision, embedding_directory, output_model, model_options, te_model_options=te_model_options, metadata=metadata)if out is None:raise RuntimeError("ERROR: Could not detect model type of: {}".format(ckpt_path))return out-def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}, te_model_options={}):+def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_clipvision=False, embedding_directory=None, output_model=True, model_options={}, te_model_options={}, metadata=None):clip = Noneclipvision = Nonevae = None@@ -891,7 +897,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_cweight_dtype = comfy.utils.weight_dtype(sd, diffusion_model_prefix)load_device = model_management.get_torch_device()- model_config = model_detection.model_config_from_unet(sd, diffusion_model_prefix)+ model_config = model_detection.model_config_from_unet(sd, diffusion_model_prefix, metadata=metadata)if model_config is None:return None@@ -920,7 +926,7 @@ def load_state_dict_guess_config(sd, output_vae=True, output_clip=True, output_cif output_vae:vae_sd = comfy.utils.state_dict_prefix_replace(sd, {k: "" for k in model_config.vae_key_prefix}, filter_keys=True)vae_sd = model_config.process_vae_state_dict(vae_sd)- vae = VAE(sd=vae_sd)+ vae = VAE(sd=vae_sd, metadata=metadata)if output_clip:clip_target = model_config.clip_target(state_dict=sd)diff --git a/comfy/utils.py b/comfy/utils.pyindex df7057c6a..a826e41bf 100644--- a/comfy/utils.py+++ b/comfy/utils.py@@ -46,12 +46,18 @@ if hasattr(torch.serialization, "add_safe_globals"): # TODO: this was added inelse:logging.info("Warning, you are using an old pytorch version and some ckpt/pt files might be loaded unsafely. Upgrading to 2.4 or above is recommended.")-def load_torch_file(ckpt, safe_load=False, device=None):+def load_torch_file(ckpt, safe_load=False, device=None, return_metadata=False):if device is None:device = torch.device("cpu")+ metadata = Noneif ckpt.lower().endswith(".safetensors") or ckpt.lower().endswith(".sft"):try:- sd = safetensors.torch.load_file(ckpt, device=device.type)+ with safetensors.safe_open(ckpt, framework="pt", device=device.type) as f:+ sd = {}+ for k in f.keys():+ sd[k] = f.get_tensor(k)+ if return_metadata:+ metadata = f.metadata()except Exception as e:if len(e.args) > 0:message = e.args[0]@@ -77,7 +83,7 @@ def load_torch_file(ckpt, safe_load=False, device=None):sd = pl_sdelse:sd = pl_sd- return sd+ return (sd, metadata) if return_metadata else sddef save_torch_file(sd, ckpt, metadata=None):if metadata is not None:diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.pyindex dec912416..8bd548bcd 100644--- a/comfy_extras/nodes_lt.py+++ b/comfy_extras/nodes_lt.py@@ -1,9 +1,14 @@+import ioimport nodesimport node_helpersimport torchimport comfy.model_managementimport comfy.model_sampling+import comfy.utilsimport math+import numpy as np+import av+from comfy.ldm.lightricks.symmetric_patchifier import SymmetricPatchifier, latent_to_pixel_coordsclass EmptyLTXVLatentVideo:@classmethod@@ -33,7 +38,6 @@ class LTXVImgToVideo:"height": ("INT", {"default": 512, "min": 64, "max": nodes.MAX_RESOLUTION, "step": 32}),"length": ("INT", {"default": 97, "min": 9, "max": nodes.MAX_RESOLUTION, "step": 8}),"batch_size": ("INT", {"default": 1, "min": 1, "max": 4096}),- "image_noise_scale": ("FLOAT", {"default": 0.15, "min": 0, "max": 1.0, "step": 0.01, "tooltip": "Amount of noise to apply on conditioning image latent."})}}RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")@@ -42,16 +46,220 @@ class LTXVImgToVideo:CATEGORY = "conditioning/video_models"FUNCTION = "generate"- def generate(self, positive, negative, image, vae, width, height, length, batch_size, image_noise_scale):+ def generate(self, positive, negative, image, vae, width, height, length, batch_size):pixels = comfy.utils.common_upscale(image.movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)encode_pixels = pixels[:, :, :, :3]t = vae.encode(encode_pixels)- positive = node_helpers.conditioning_set_values(positive, {"guiding_latent": t, "guiding_latent_noise_scale": image_noise_scale})- negative = node_helpers.conditioning_set_values(negative, {"guiding_latent": t, "guiding_latent_noise_scale": image_noise_scale})latent = torch.zeros([batch_size, 128, ((length - 1) // 8) + 1, height // 32, width // 32], device=comfy.model_management.intermediate_device())latent[:, :, :t.shape[2]] = t- return (positive, negative, {"samples": latent}, )++ conditioning_latent_frames_mask = torch.ones(+ (batch_size, 1, latent.shape[2], 1, 1),+ dtype=torch.float32,+ device=latent.device,+ )+ conditioning_latent_frames_mask[:, :, :t.shape[2]] = 0++ return (positive, negative, {"samples": latent, "noise_mask": conditioning_latent_frames_mask}, )+++def conditioning_get_any_value(conditioning, key, default=None):+ for t in conditioning:+ if key in t[1]:+ return t[1][key]+ return default+++def get_noise_mask(latent):+ noise_mask = latent.get("noise_mask", None)+ latent_image = latent["samples"]+ if noise_mask is None:+ batch_size, _, latent_length, _, _ = latent_image.shape+ noise_mask = torch.ones(+ (batch_size, 1, latent_length, 1, 1),+ dtype=torch.float32,+ device=latent_image.device,+ )+ else:+ noise_mask = noise_mask.clone()+ return noise_mask++def get_keyframe_idxs(cond):+ keyframe_idxs = conditioning_get_any_value(cond, "keyframe_idxs", None)+ if keyframe_idxs is None:+ return None, 0+ num_keyframes = torch.unique(keyframe_idxs[:, 0]).shape[0]+ return keyframe_idxs, num_keyframes++class LTXVAddGuide:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {"positive": ("CONDITIONING", ),+ "negative": ("CONDITIONING", ),+ "vae": ("VAE",),+ "latent": ("LATENT",),+ "image": ("IMAGE", {"tooltip": "Image or video to condition the latent video on. Must be 8*n + 1 frames." \+ "If the video is not 8*n + 1 frames, it will be cropped to the nearest 8*n + 1 frames."}),+ "frame_idx": ("INT", {"default": 0, "min": -9999, "max": 9999,+ "tooltip": "Frame index to start the conditioning at. Must be divisible by 8. " \+ "If a frame is not divisible by 8, it will be rounded down to the nearest multiple of 8. " \+ "Negative values are counted from the end of the video."}),+ "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 1.0, "step": 0.01}),+ }+ }++ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")+ RETURN_NAMES = ("positive", "negative", "latent")++ CATEGORY = "conditioning/video_models"+ FUNCTION = "generate"++ def __init__(self):+ self._num_prefix_frames = 2+ self._patchifier = SymmetricPatchifier(1)++ def encode(self, vae, latent_width, latent_height, images, scale_factors):+ time_scale_factor, width_scale_factor, height_scale_factor = scale_factors+ images = images[:(images.shape[0] - 1) // time_scale_factor * time_scale_factor + 1]+ pixels = comfy.utils.common_upscale(images.movedim(-1, 1), latent_width * width_scale_factor, latent_height * height_scale_factor, "bilinear", crop="disabled").movedim(1, -1)+ encode_pixels = pixels[:, :, :, :3]+ t = vae.encode(encode_pixels)+ return encode_pixels, t++ def get_latent_index(self, cond, latent_length, frame_idx, scale_factors):+ time_scale_factor, _, _ = scale_factors+ _, num_keyframes = get_keyframe_idxs(cond)+ latent_count = latent_length - num_keyframes+ frame_idx = frame_idx if frame_idx >= 0 else max((latent_count - 1) * 8 + 1 + frame_idx, 0)+ frame_idx = frame_idx // time_scale_factor * time_scale_factor # frame index must be divisible by 8++ latent_idx = (frame_idx + time_scale_factor - 1) // time_scale_factor++ return frame_idx, latent_idx++ def add_keyframe_index(self, cond, frame_idx, guiding_latent, scale_factors):+ keyframe_idxs, _ = get_keyframe_idxs(cond)+ _, latent_coords = self._patchifier.patchify(guiding_latent)+ pixel_coords = latent_to_pixel_coords(latent_coords, scale_factors, True)+ pixel_coords[:, 0] += frame_idx+ if keyframe_idxs is None:+ keyframe_idxs = pixel_coords+ else:+ keyframe_idxs = torch.cat([keyframe_idxs, pixel_coords], dim=2)+ return node_helpers.conditioning_set_values(cond, {"keyframe_idxs": keyframe_idxs})++ def append_keyframe(self, positive, negative, frame_idx, latent_image, noise_mask, guiding_latent, strength, scale_factors):+ positive = self.add_keyframe_index(positive, frame_idx, guiding_latent, scale_factors)+ negative = self.add_keyframe_index(negative, frame_idx, guiding_latent, scale_factors)++ mask = torch.full(+ (noise_mask.shape[0], 1, guiding_latent.shape[2], 1, 1),+ 1.0 - strength,+ dtype=noise_mask.dtype,+ device=noise_mask.device,+ )++ latent_image = torch.cat([latent_image, guiding_latent], dim=2)+ noise_mask = torch.cat([noise_mask, mask], dim=2)+ return positive, negative, latent_image, noise_mask++ def replace_latent_frames(self, latent_image, noise_mask, guiding_latent, latent_idx, strength):+ cond_length = guiding_latent.shape[2]+ assert latent_image.shape[2] >= latent_idx + cond_length, "Conditioning frames exceed the length of the latent sequence."++ mask = torch.full(+ (noise_mask.shape[0], 1, cond_length, 1, 1),+ 1.0 - strength,+ dtype=noise_mask.dtype,+ device=noise_mask.device,+ )++ latent_image = latent_image.clone()+ noise_mask = noise_mask.clone()++ latent_image[:, :, latent_idx : latent_idx + cond_length] = guiding_latent+ noise_mask[:, :, latent_idx : latent_idx + cond_length] = mask++ return latent_image, noise_mask++ def generate(self, positive, negative, vae, latent, image, frame_idx, strength):+ scale_factors = vae.downscale_index_formula+ latent_image = latent["samples"]+ noise_mask = get_noise_mask(latent)++ _, _, latent_length, latent_height, latent_width = latent_image.shape+ image, t = self.encode(vae, latent_width, latent_height, image, scale_factors)++ frame_idx, latent_idx = self.get_latent_index(positive, latent_length, frame_idx, scale_factors)+ assert latent_idx + t.shape[2] <= latent_length, "Conditioning frames exceed the length of the latent sequence."++ if frame_idx == 0:+ latent_image, noise_mask = self.replace_latent_frames(latent_image, noise_mask, t, latent_idx, strength)+ return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask},)+++ num_prefix_frames = min(self._num_prefix_frames, t.shape[2])++ positive, negative, latent_image, noise_mask = self.append_keyframe(+ positive,+ negative,+ frame_idx,+ latent_image,+ noise_mask,+ t[:, :, :num_prefix_frames],+ strength,+ scale_factors,+ )++ latent_idx += num_prefix_frames++ t = t[:, :, num_prefix_frames:]+ if t.shape[2] == 0:+ return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask},)++ latent_image, noise_mask = self.replace_latent_frames(+ latent_image,+ noise_mask,+ t,+ latent_idx,+ strength,+ )++ return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask},)+++class LTXVCropGuides:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {"positive": ("CONDITIONING", ),+ "negative": ("CONDITIONING", ),+ "latent": ("LATENT",),+ }+ }++ RETURN_TYPES = ("CONDITIONING", "CONDITIONING", "LATENT")+ RETURN_NAMES = ("positive", "negative", "latent")++ CATEGORY = "conditioning/video_models"+ FUNCTION = "crop"++ def __init__(self):+ self._patchifier = SymmetricPatchifier(1)++ def crop(self, positive, negative, latent):+ latent_image = latent["samples"].clone()+ noise_mask = get_noise_mask(latent)++ _, num_keyframes = get_keyframe_idxs(positive)++ latent_image = latent_image[:, :, :-num_keyframes]+ noise_mask = noise_mask[:, :, :-num_keyframes]++ positive = node_helpers.conditioning_set_values(positive, {"keyframe_idxs": None})+ negative = node_helpers.conditioning_set_values(negative, {"keyframe_idxs": None})++ return (positive, negative, {"samples": latent_image, "noise_mask": noise_mask},)class LTXVConditioning:@@ -174,6 +382,78 @@ class LTXVScheduler:return (sigmas,)+def encode_single_frame(output_file, image_array: np.ndarray, crf):+ container = av.open(output_file, "w", format="mp4")+ try:+ stream = container.add_stream(+ "h264", rate=1, options={"crf": str(crf), "preset": "veryfast"}+ )+ stream.height = image_array.shape[0]+ stream.width = image_array.shape[1]+ av_frame = av.VideoFrame.from_ndarray(image_array, format="rgb24").reformat(+ format="yuv420p"+ )+ container.mux(stream.encode(av_frame))+ container.mux(stream.encode())+ finally:+ container.close()+++def decode_single_frame(video_file):+ container = av.open(video_file)+ try:+ stream = next(s for s in container.streams if s.type == "video")+ frame = next(container.decode(stream))+ finally:+ container.close()+ return frame.to_ndarray(format="rgb24")+++def preprocess(image: torch.Tensor, crf=29):+ if crf == 0:+ return image++ image_array = (image * 255.0).byte().cpu().numpy()+ with io.BytesIO() as output_file:+ encode_single_frame(output_file, image_array, crf)+ video_bytes = output_file.getvalue()+ with io.BytesIO(video_bytes) as video_file:+ image_array = decode_single_frame(video_file)+ tensor = torch.tensor(image_array, dtype=image.dtype, device=image.device) / 255.0+ return tensor+++class LTXVPreprocess:+ @classmethod+ def INPUT_TYPES(s):+ return {+ "required": {+ "image": ("IMAGE",),+ "img_compression": (+ "INT",+ {+ "default": 35,+ "min": 0,+ "max": 100,+ "tooltip": "Amount of compression to apply on image.",+ },+ ),+ }+ }++ FUNCTION = "preprocess"+ RETURN_TYPES = ("IMAGE",)+ RETURN_NAMES = ("output_image",)+ CATEGORY = "image"++ def preprocess(self, image, img_compression):+ output_image = image+ if img_compression > 0:+ output_image = torch.zeros_like(image)+ for i in range(image.shape[0]):+ output_image[i] = preprocess(image[i], img_compression)+ return (output_image,)+NODE_CLASS_MAPPINGS = {"EmptyLTXVLatentVideo": EmptyLTXVLatentVideo,@@ -181,4 +461,7 @@ NODE_CLASS_MAPPINGS = {"ModelSamplingLTXV": ModelSamplingLTXV,"LTXVConditioning": LTXVConditioning,"LTXVScheduler": LTXVScheduler,+ "LTXVAddGuide": LTXVAddGuide,+ "LTXVPreprocess": LTXVPreprocess,+ "LTXVCropGuides": LTXVCropGuides,}