The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add support for taeh3. (#15695)
comfyanonymous8e869efc8764
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex ed5167075..34c6d700c 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -577,6 +577,7 @@ class MiniMaxH3Video(LatentFormat):spacial_downscale_ratio = 16temporal_downscale_ratio = 4scale_factor = 1.0+ taesd_decoder_name = "taeh3"latent_rgb_factors = [[-0.018555, 0.024344, -0.017536],diff --git a/comfy/sd.py b/comfy/sd.pyindex 4bdaa978c..06679c6fb 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -907,7 +907,14 @@ class VAE:self.upscale_index_formula = (4, 16, 16)self.downscale_ratio = (lambda a: max(0, math.floor((a + 3) / 4)), 16, 16)self.downscale_index_formula = (4, 16, 16)- if self.latent_channels in [48, 128]: # Wan 2.2 and LTX2+ if self.latent_channels == 24 and sd["decoder.22.bias"].shape[0] == 12: # MiniMax H3+ self.first_stage_model = comfy.taesd.taehv.TAEHV(latent_channels=self.latent_channels, latent_format=None)+ self.process_input = self.process_output = lambda image: image+ self.upscale_ratio = (lambda a: max(1, (a - 2) // 5 * 17 + 5), 16, 16)+ self.downscale_ratio = (lambda a: max(1, (a - 1) // 17 * 5 + 2) if a > 1 else 1, 16, 16)+ self.memory_used_encode = lambda shape, dtype: (400 * ((shape[-3] + 16) // 17) * shape[-2] * shape[-1] * model_management.dtype_size(dtype))+ self.memory_used_decode = lambda shape, dtype: ((260 * 16 * 16 + shape[1] * shape[-3]) * shape[-2] * shape[-1] * model_management.dtype_size(dtype))+ elif self.latent_channels in [48, 128]: # Wan 2.2 and LTX2self.first_stage_model = comfy.taesd.taehv.TAEHV(latent_channels=self.latent_channels, latent_format=None) # taehv doesn't need scalingself.process_input = self.process_output = lambda image: imageself.process_output = lambda image: imagediff --git a/comfy/taesd/taehv.py b/comfy/taesd/taehv.pyindex 696013200..ffa9f89d1 100644--- a/comfy/taesd/taehv.py+++ b/comfy/taesd/taehv.py@@ -131,10 +131,11 @@ class TAEHV(nn.Module):self.latent_channels = latent_channelsself.parallel = parallelself.latent_format = latent_format+ self.is_h3 = self.latent_channels == 24self.show_progress_bar = show_progress_barself.process_in = latent_format().process_in if latent_format is not None else (lambda x: x)self.process_out = latent_format().process_out if latent_format is not None else (lambda x: x)- if self.latent_channels in [48, 32]: # Wan 2.2 and HunyuanVideo1.5+ if self.latent_channels in [48, 32, 24]: # Wan 2.2, HunyuanVideo1.5 and MiniMax H3self.patch_size = 2elif self.latent_channels == 128: # LTX2self.patch_size, self.latent_channels, encoder_time_downscale, decoder_time_upscale = 4, 128, (True, True, True), (True, True, True)@@ -176,6 +177,21 @@ class TAEHV(nn.Module):def encode(self, x, **kwargs):x = x.movedim(2, 1) # [B, C, T, H, W] -> [B, T, C, H, W]+ if self.is_h3:+ single_frame = x.shape[1] == 1+ batch = x.shape[0]+ x = torch.cat([x, x[:, -1:].expand(-1, -x.shape[1] % 17, -1, -1, -1)], dim=1)+ x = F.pad(x.reshape(batch, -1, 17, *x.shape[2:]), (0, 0, 0, 0, 0, 0, 3, 0))+ if self.parallel:+ x = apply_model_with_memblocks(self.encoder, x.flatten(0, 1), True, self.show_progress_bar,+ patch_size=self.patch_size)+ x = x.reshape(batch, -1, *x.shape[2:])+ else:+ x = torch.cat([apply_model_with_memblocks(self.encoder, chunk, False, False,+ patch_size=self.patch_size)+ for chunk in tqdm(x.unbind(1), disable=not self.show_progress_bar)], dim=1)+ x = x[:, :1] if single_frame else x[:, :-3]+ return self.process_out(x.movedim(2, 1))if x.shape[1] % self.t_downscale != 0:# pad at end to multiple of t_downscalen_pad = self.t_downscale - x.shape[1] % self.t_downscale@@ -189,7 +205,16 @@ class TAEHV(nn.Module):x = x.unsqueeze(0) if x.ndim == 4 else x # [T, C, H, W] -> [1, T, C, H, W]x = x.movedim(1, 2) if x.shape[1] != self.latent_channels else x # [B, T, C, H, W] or [B, C, T, H, W]x = self.process_in(x).movedim(2, 1) # [B, C, T, H, W] -> [B, T, C, H, W]+ if self.is_h3:+ single_frame = x.shape[1] == 1x = apply_model_with_memblocks(self.decoder, x, self.parallel, self.show_progress_bar,output_device=comfy.model_management.intermediate_device(),patch_size=self.patch_size, decode=True)+ if self.is_h3:+ x.clamp_(0, 1)+ if not single_frame:+ chunk_frames = 5 * self.t_upscale+ x = F.pad(x, (0, 0, 0, 0, 0, 0, 0, -x.shape[1] % chunk_frames))+ x = x.unflatten(1, (-1, chunk_frames))[:, :, self.frames_to_trim:].flatten(1, 2)+ return x[:, :-3 * self.t_upscale].movedim(2, 1)return x[:, self.frames_to_trim:].movedim(2, 1)diff --git a/latent_preview.py b/latent_preview.pyindex 6bf2c1869..d98b70019 100644--- a/latent_preview.py+++ b/latent_preview.py@@ -11,7 +11,7 @@ import loggingdefault_preview_method = args.preview_methodMAX_PREVIEW_RESOLUTION = args.preview_size-VIDEO_TAES = ["taehv", "lighttaew2_2", "lighttaew2_1", "lighttaehy1_5", "taeltx_2"]+VIDEO_TAES = ["taehv", "lighttaew2_2", "lighttaew2_1", "lighttaehy1_5", "taeltx_2", "taeh3"]def preview_to_image(latent_image, do_scale=True):if do_scale:@@ -136,4 +136,3 @@ def set_preview_method(override: str = None):args.preview_method = methodreturnargs.preview_method = default_preview_method-diff --git a/nodes.py b/nodes.pyindex 1a3dd3f48..fa3a77949 100644--- a/nodes.py+++ b/nodes.py@@ -768,7 +768,7 @@ class LoraLoaderModelOnly(LoraLoader):return (self.load_lora(model, None, lora_name, strength_model, 0)[0],)class VAELoader:- video_taes = ["taehv", "lighttaew2_2", "lighttaew2_1", "lighttaehy1_5", "taeltx_2"]+ video_taes = ["taehv", "lighttaew2_2", "lighttaew2_1", "lighttaehy1_5", "taeltx_2", "taeh3"]image_taes = ["taesd", "taesdxl", "taesd3", "taef1", "taef2"]@staticmethod