The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support tiny VAE for Qwen-Image 2.1 (#16552)
Jukka Seppänena73d24ba1771
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex 44141146e..0e7fba912 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -958,6 +958,7 @@ class QwenImage21(LatentFormat):latent_channels = 64latent_dimensions = 2spacial_downscale_ratio = 16+ taesd_decoder_name = "taeqi2_1_decoder"latent_rgb_factors = [[-0.0158, -0.0115, -0.0174], [ 0.0030, 0.0120, 0.0027], [ 0.0637, 0.0470, -0.0127], [ 0.0360, 0.0661, -0.0030],diff --git a/comfy/sd.py b/comfy/sd.pyindex 104caae9a..1023ad9ea 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -556,6 +556,11 @@ class VAE:else:self.latent_channels = sd["taesd_decoder.1.weight"].shape[1]self.first_stage_model = comfy.taesd.taesd.TAESD(latent_channels=self.latent_channels)+ if self.latent_channels == 64: # Qwen Image 2.1 TAESD: 16x, RGBA+ self.downscale_ratio = 16+ self.upscale_ratio = 16+ self.output_channels = 4+ self.pad_channel_value = 1.0elif "vquantizer.codebook.weight" in sd: #VQGan: stage a of stable cascadeself.first_stage_model = StageA()self.downscale_ratio = 4diff --git a/comfy/taesd/taesd.py b/comfy/taesd/taesd.pyindex 05d370209..bc033e4ca 100644--- a/comfy/taesd/taesd.py+++ b/comfy/taesd/taesd.py@@ -58,6 +58,27 @@ class Decoder(nn.Sequential):Block(64, 64), conv(64, 3),)+class EncoderF16(nn.Sequential):+ # 16x: 2x2 pixel unshuffle on the way in and wider low resolution stages (Qwen Image 2.1, RGBA)+ def __init__(self, latent_channels: int = 64, image_channels: int = 4):+ super().__init__(+ nn.PixelUnshuffle(2), conv(image_channels * 4, 64), nn.ReLU(inplace=True), Block(64, 64),+ conv(64, 64, stride=2, bias=False), Block(64, 64), Block(64, 64), Block(64, 64),+ conv(64, 128, stride=2, bias=False), Block(128, 128), Block(128, 128), Block(128, 128),+ conv(128, 256, stride=2, bias=False), Block(256, 256), Block(256, 256), Block(256, 256),+ conv(256, latent_channels),+ )++class DecoderF16(nn.Sequential):+ def __init__(self, latent_channels: int = 64, image_channels: int = 4):+ super().__init__(+ Clamp(), conv(latent_channels, 256), nn.ReLU(),+ Block(256, 256), Block(256, 256), Block(256, 256), nn.Upsample(scale_factor=2), conv(256, 128, bias=False),+ Block(128, 128), Block(128, 128), Block(128, 128), nn.Upsample(scale_factor=2), conv(128, 64, bias=False),+ Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False),+ Block(64, 64), conv(64, image_channels * 4), nn.PixelShuffle(2),+ )+class DecoderFlux2(Decoder):def __init__(self, latent_channels: int = 128, use_gn: bool = True):if latent_channels != 128 or not use_gn:@@ -101,14 +122,19 @@ class TAESD(nn.Module):if latent_channels == 128:encoder_class = EncoderFlux2decoder_class = DecoderFlux2+ elif latent_channels == 64:+ encoder_class = EncoderF16+ decoder_class = DecoderF16else:encoder_class = Encoderdecoder_class = Decoderself.taesd_encoder = encoder_class(latent_channels=latent_channels)self.taesd_decoder = decoder_class(latent_channels=latent_channels)- self.vae_scale = torch.nn.Parameter(torch.tensor(1.0))- self.vae_shift = torch.nn.Parameter(torch.tensor(0.0))+ # the 64 channel latents are normalized per channel+ shape = (latent_channels, 1, 1) if latent_channels == 64 else ()+ self.vae_scale = torch.nn.Parameter(torch.ones(shape))+ self.vae_shift = torch.nn.Parameter(torch.zeros(shape))if encoder_path is not None:self.taesd_encoder.load_state_dict(comfy.utils.load_torch_file(encoder_path, safe_load=True))if decoder_path is not None:diff --git a/latent_preview.py b/latent_preview.pyindex e3864e938..f50594003 100644--- a/latent_preview.py+++ b/latent_preview.py@@ -42,7 +42,8 @@ class TAESDPreviewerImpl(LatentPreviewer):self.taesd = taesddef decode_latent_to_preview(self, x0):- x_sample = self.taesd.decode(x0[:1])[0].movedim(0, 2)+ # RGB only: previews are JPEG+ x_sample = self.taesd.decode(x0[:1])[0, :3].movedim(0, 2)return preview_to_image(x_sample)class TAEHVPreviewerImpl(TAESDPreviewerImpl):diff --git a/nodes.py b/nodes.pyindex 1d3fd6b16..12f152c77 100644--- a/nodes.py+++ b/nodes.py@@ -25,6 +25,7 @@ import comfy.samplersimport comfy.sampleimport comfy.sdimport comfy.utils+import comfy.latent_formatsimport comfy.controlnetfrom comfy.comfy_types import IO, ComfyNodeABC, InputTypeDict, FileLocatorfrom comfy_api.internal import register_versions, ComfyAPIWithVersion@@ -773,7 +774,7 @@ class LoraLoaderModelOnly(LoraLoader):class VAELoader:video_taes = ["taehv", "lighttaew2_2", "lighttaew2_1", "lighttaehy1_5", "taeltx_2", "taeh3"]- image_taes = ["taesd", "taesdxl", "taesd3", "taef1", "taef2"]+ image_taes = ["taesd", "taesdxl", "taesd3", "taef1", "taef2", "taeqi2_1"]@staticmethoddef vae_list(s):@@ -781,7 +782,7 @@ class VAELoader:approx_vaes = folder_paths.get_filename_list("vae_approx")have_img_encoder, have_img_decoder = set(), set()for v in approx_vaes:- parts = v.split("_", 1)+ parts = v.rsplit("_", 1)if len(parts) != 2 or parts[0] not in s.image_taes:for tae in s.video_taes:if v.startswith(tae):@@ -824,6 +825,10 @@ class VAELoader:elif name == "taef1":sd["vae_scale"] = torch.tensor(0.3611)sd["vae_shift"] = torch.tensor(0.1159)+ elif name == "taeqi2_1":+ latent_format = comfy.latent_formats.QwenImage21()+ sd["vae_scale"] = 1.0 / latent_format.latents_std[0]+ sd["vae_shift"] = latent_format.latents_mean[0]return sd@classmethod