The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix VAE Compute Dtype crash on the LTX 2.5 diffusion VAE decoder (#17487)
DoronGenzelHass4d988b647853
diff --git a/comfy/ldm/lightricks/vae/na_diffusion_decoder.py b/comfy/ldm/lightricks/vae/na_diffusion_decoder.pyindex ec539c665..cf16051f7 100644--- a/comfy/ldm/lightricks/vae/na_diffusion_decoder.py+++ b/comfy/ldm/lightricks/vae/na_diffusion_decoder.py@@ -23,12 +23,15 @@ import torch.nn.functional as Ffrom einops import rearrangefrom torch import nnimport comfy.model_management+import comfy.opsfrom comfy.ldm.lightricks.model import get_timestep_embeddingfrom .causal_video_autoencoder import Encoder, processorimport comfy_kitchen+ops = comfy.ops.disable_weight_init+# Token chunk for the SwiGLU MLP (bounds the [chunk, hidden] workspace).MLP_TOKEN_CHUNK = 65536@@ -123,8 +126,8 @@ class NeighborhoodAttention3D(nn.Module):self.rope_split = default_rope_dim_split(head_dim)self.rope_base = rope_base- self.qkv = nn.Linear(dim, dim * 3, bias=True)- self.proj = nn.Linear(dim, dim, bias=True)+ self.qkv = ops.Linear(dim, dim * 3, bias=True)+ self.proj = ops.Linear(dim, dim, bias=True)self.q_norm = RMSNorm(head_dim, eps=1e-6)self.k_norm = RMSNorm(head_dim, eps=1e-6)@@ -177,9 +180,9 @@ class SwiGLU(nn.Module):def __init__(self, dim, hidden_dim):super().__init__()- self.w_up = nn.Linear(dim, hidden_dim, bias=False)- self.w_gate = nn.Linear(dim, hidden_dim, bias=False)- self.w_down = nn.Linear(hidden_dim, dim, bias=False)+ self.w_up = ops.Linear(dim, hidden_dim, bias=False)+ self.w_gate = ops.Linear(dim, hidden_dim, bias=False)+ self.w_down = ops.Linear(hidden_dim, dim, bias=False)def forward(self, x, pre=None, add_to=None):"""``pre``/``add_to`` as in ``NeighborhoodAttention3D.forward``."""@@ -224,7 +227,7 @@ class AdaLNZero(nn.Module):def __init__(self, dim, t_emb_dim):super().__init__()- self.proj = nn.Linear(t_emb_dim, self.NUM_CHUNKS * dim, bias=True)+ self.proj = ops.Linear(t_emb_dim, self.NUM_CHUNKS * dim, bias=True)def forward(self, t_emb):h = self.proj(F.silu(t_emb))@@ -236,7 +239,7 @@ class DiffusionNABlock(nn.Module):def __init__(self, dim, kernel_size, context_channels, head_dim=64, mlp_ratio=4.0):super().__init__()- self.context_proj = nn.Linear(context_channels, dim, bias=True)+ self.context_proj = ops.Linear(context_channels, dim, bias=True)self.scale_shift_table = nn.Parameter(torch.zeros(AdaLNZero.NUM_CHUNKS, dim))self.norm1 = RMSNorm(dim, eps=1e-6)self.attn = NeighborhoodAttention3D(dim, kernel_size, head_dim=head_dim)@@ -245,8 +248,9 @@ class DiffusionNABlock(nn.Module):self.mlp = SwiGLU(dim, hidden)def forward(self, x, latent_context, modulation):+ table = comfy.ops.cast_to_input(self.scale_shift_table, modulation[0], copy=False)scale_msa, shift_msa, _, scale_mlp, shift_mlp, _, _ = [- modulation[i] + self.scale_shift_table[i].view(1, 1, 1, 1, -1) for i in range(AdaLNZero.NUM_CHUNKS)+ modulation[i] + table[i].view(1, 1, 1, 1, -1) for i in range(AdaLNZero.NUM_CHUNKS)]chunk = max(1, MLP_TOKEN_CHUNK // max(x.shape[2] * x.shape[3], 1))for t0 in range(0, x.shape[1], chunk):@@ -263,7 +267,7 @@ class LinearPixelShuffleUpsample(nn.Module):self.stride = tuple(stride)proj_out_channels = math.prod(stride) * in_channels // out_channels_reduction_factorself.out_channels = proj_out_channels // math.prod(stride)- self.proj = nn.Linear(in_channels, proj_out_channels, bias=True)+ self.proj = ops.Linear(in_channels, proj_out_channels, bias=True)def forward(self, x, drop_leading_frame=True):batch, t, h, w, _ = x.shape@@ -289,9 +293,9 @@ class TimestepEmbedder(nn.Module):super().__init__()self.freq_dim = freq_dimself.mlp = nn.Sequential(- nn.Linear(freq_dim, t_emb_dim, bias=True),+ ops.Linear(freq_dim, t_emb_dim, bias=True),nn.SiLU(),- nn.Linear(t_emb_dim, t_emb_dim, bias=True),+ ops.Linear(t_emb_dim, t_emb_dim, bias=True),)def forward(self, timestep, dtype):@@ -339,7 +343,7 @@ class NADiffusionDecoder(nn.Module):# frame through stages 1-4, crop the appendix off the context after.self.trailing_pad_latent_frames = (stage_kernels[0][0] // 2) * 2- self.conv_in = nn.Linear(in_channels, stage_channels[0], bias=True)+ self.conv_in = ops.Linear(in_channels, stage_channels[0], bias=True)self.det_stages = nn.ModuleList()self.upsamples = nn.ModuleList()@@ -356,14 +360,14 @@ class NADiffusionDecoder(nn.Module):c5 = stage_channels[-1]self.context_channels = c5noised_pixel_channels = out_channels * (patch_size ** 2)- self.conv_in_x_t = nn.Linear(noised_pixel_channels, c5, bias=True)+ self.conv_in_x_t = ops.Linear(noised_pixel_channels, c5, bias=True)self.shared_adaln = AdaLNZero(c5, t_emb_dim)self.diff_blocks = nn.ModuleList([DiffusionNABlock(c5, stage5_kernel, context_channels=c5, head_dim=head_dim)for _ in range(stage_depths[-1])])self.norm_out = RMSNorm(c5, eps=1e-6)- self.conv_out = nn.Linear(c5, noised_pixel_channels, bias=True)+ self.conv_out = ops.Linear(c5, noised_pixel_channels, bias=True)def forward_pre_diffusion(self, z, drop_leading_frame=True, pad_trailing=True):"""Stages 1-4: latent -> stage-5 context, channels-last.diff --git a/tests-unit/comfy_test/test_ltx_diffusion_vae_dtype.py b/tests-unit/comfy_test/test_ltx_diffusion_vae_dtype.pynew file mode 100644index 000000000..10ef4e617--- /dev/null+++ b/tests-unit/comfy_test/test_ltx_diffusion_vae_dtype.py@@ -0,0 +1,52 @@+import torch++from comfy.cli_args import args as cli_args++if not torch.cuda.is_available():+ cli_args.cpu = True++import comfy.model_patcher+import comfy.ldm.lightricks.vae.na_diffusion_decoder as na_decoder++TINY_DECODER = dict(+ in_channels=8, out_channels=3, patch_size=2, head_dim=32,+ stage_channels=(256, 128, 64, 64, 32), stage_depths=(1, 1, 1, 1, 2),+ stage_kernels=((3, 3, 3),) * 5, stage5_kernel=(3, 3, 3), t_emb_dim=64,+)+++def _tiny_decoder(dtype):+ torch.manual_seed(0)+ model = na_decoder.NADiffusionDecoder(**TINY_DECODER)+ for p in model.parameters():+ torch.nn.init.normal_(p, std=0.05)+ return model.eval().requires_grad_(False).to(dtype)+++def _decode(model, z):+ return model(z, generator=torch.Generator(device=z.device).manual_seed(0))+++def test_ltx_diffusion_decoder_linears_use_comfy_ops():+ model = _tiny_decoder(torch.float32)+ plain = [name for name, m in model.named_modules() if type(m) is torch.nn.Linear]+ assert plain == []+++def test_ltx_diffusion_decoder_vae_compute_dtype_fp32():+ device = torch.device("cpu")+ z = torch.randn(1, 8, 2, 2, 2)+ reference = _decode(_tiny_decoder(torch.bfloat16).float(), z)++ model = _tiny_decoder(torch.bfloat16)+ patcher = comfy.model_patcher.ModelPatcher(model, load_device=device, offload_device=device)+ patcher.set_model_compute_dtype(torch.float32)+ patcher.patch_model(device_to=device)+ try:+ out = _decode(model, z)+ finally:+ patcher.unpatch_model(device_to=device)++ assert model.conv_in.weight.dtype == torch.bfloat16+ assert out.dtype == torch.float32+ torch.testing.assert_close(out, reference)