Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support tiny VAE for Qwen-Image 2.1 (#16552)

Jukka Seppänena73d24ba1771
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.py
index 44141146e..0e7fba912 100644
--- a/comfy/latent_formats.py
+++ b/comfy/latent_formats.py
@@ -958,6 +958,7 @@ class QwenImage21(LatentFormat):
latent_channels = 64
latent_dimensions = 2
spacial_downscale_ratio = 16
+ taesd_decoder_name = "taeqi2_1_decoder"
latent_rgb_factors = [
[-0.0158, -0.0115, -0.0174], [ 0.0030, 0.0120, 0.0027], [ 0.0637, 0.0470, -0.0127], [ 0.0360, 0.0661, -0.0030],
diff --git a/comfy/sd.py b/comfy/sd.py
index 104caae9a..1023ad9ea 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -556,6 +556,11 @@ class VAE:
else:
self.latent_channels = sd["taesd_decoder.1.weight"].shape[1]
self.first_stage_model = comfy.taesd.taesd.TAESD(latent_channels=self.latent_channels)
+ if self.latent_channels == 64: # Qwen Image 2.1 TAESD: 16x, RGBA
+ self.downscale_ratio = 16
+ self.upscale_ratio = 16
+ self.output_channels = 4
+ self.pad_channel_value = 1.0
elif "vquantizer.codebook.weight" in sd: #VQGan: stage a of stable cascade
self.first_stage_model = StageA()
self.downscale_ratio = 4
diff --git a/comfy/taesd/taesd.py b/comfy/taesd/taesd.py
index 05d370209..bc033e4ca 100644
--- a/comfy/taesd/taesd.py
+++ b/comfy/taesd/taesd.py
@@ -58,6 +58,27 @@ class Decoder(nn.Sequential):
Block(64, 64), conv(64, 3),
)
+class EncoderF16(nn.Sequential):
+ # 16x: 2x2 pixel unshuffle on the way in and wider low resolution stages (Qwen Image 2.1, RGBA)
+ def __init__(self, latent_channels: int = 64, image_channels: int = 4):
+ super().__init__(
+ nn.PixelUnshuffle(2), conv(image_channels * 4, 64), nn.ReLU(inplace=True), Block(64, 64),
+ conv(64, 64, stride=2, bias=False), Block(64, 64), Block(64, 64), Block(64, 64),
+ conv(64, 128, stride=2, bias=False), Block(128, 128), Block(128, 128), Block(128, 128),
+ conv(128, 256, stride=2, bias=False), Block(256, 256), Block(256, 256), Block(256, 256),
+ conv(256, latent_channels),
+ )
+
+class DecoderF16(nn.Sequential):
+ def __init__(self, latent_channels: int = 64, image_channels: int = 4):
+ super().__init__(
+ Clamp(), conv(latent_channels, 256), nn.ReLU(),
+ Block(256, 256), Block(256, 256), Block(256, 256), nn.Upsample(scale_factor=2), conv(256, 128, bias=False),
+ Block(128, 128), Block(128, 128), Block(128, 128), nn.Upsample(scale_factor=2), conv(128, 64, bias=False),
+ Block(64, 64), Block(64, 64), Block(64, 64), nn.Upsample(scale_factor=2), conv(64, 64, bias=False),
+ Block(64, 64), conv(64, image_channels * 4), nn.PixelShuffle(2),
+ )
+
class DecoderFlux2(Decoder):
def __init__(self, latent_channels: int = 128, use_gn: bool = True):
if latent_channels != 128 or not use_gn:
@@ -101,14 +122,19 @@ class TAESD(nn.Module):
if latent_channels == 128:
encoder_class = EncoderFlux2
decoder_class = DecoderFlux2
+ elif latent_channels == 64:
+ encoder_class = EncoderF16
+ decoder_class = DecoderF16
else:
encoder_class = Encoder
decoder_class = Decoder
self.taesd_encoder = encoder_class(latent_channels=latent_channels)
self.taesd_decoder = decoder_class(latent_channels=latent_channels)
- self.vae_scale = torch.nn.Parameter(torch.tensor(1.0))
- self.vae_shift = torch.nn.Parameter(torch.tensor(0.0))
+ # the 64 channel latents are normalized per channel
+ shape = (latent_channels, 1, 1) if latent_channels == 64 else ()
+ self.vae_scale = torch.nn.Parameter(torch.ones(shape))
+ self.vae_shift = torch.nn.Parameter(torch.zeros(shape))
if encoder_path is not None:
self.taesd_encoder.load_state_dict(comfy.utils.load_torch_file(encoder_path, safe_load=True))
if decoder_path is not None:
diff --git a/latent_preview.py b/latent_preview.py
index e3864e938..f50594003 100644
--- a/latent_preview.py
+++ b/latent_preview.py
@@ -42,7 +42,8 @@ class TAESDPreviewerImpl(LatentPreviewer):
self.taesd = taesd
def decode_latent_to_preview(self, x0):
- x_sample = self.taesd.decode(x0[:1])[0].movedim(0, 2)
+ # RGB only: previews are JPEG
+ x_sample = self.taesd.decode(x0[:1])[0, :3].movedim(0, 2)
return preview_to_image(x_sample)
class TAEHVPreviewerImpl(TAESDPreviewerImpl):
diff --git a/nodes.py b/nodes.py
index 1d3fd6b16..12f152c77 100644
--- a/nodes.py
+++ b/nodes.py
@@ -25,6 +25,7 @@ import comfy.samplers
import comfy.sample
import comfy.sd
import comfy.utils
+import comfy.latent_formats
import comfy.controlnet
from comfy.comfy_types import IO, ComfyNodeABC, InputTypeDict, FileLocator
from comfy_api.internal import register_versions, ComfyAPIWithVersion
@@ -773,7 +774,7 @@ class LoraLoaderModelOnly(LoraLoader):
class VAELoader:
video_taes = ["taehv", "lighttaew2_2", "lighttaew2_1", "lighttaehy1_5", "taeltx_2", "taeh3"]
- image_taes = ["taesd", "taesdxl", "taesd3", "taef1", "taef2"]
+ image_taes = ["taesd", "taesdxl", "taesd3", "taef1", "taef2", "taeqi2_1"]
@staticmethod
def vae_list(s):
@@ -781,7 +782,7 @@ class VAELoader:
approx_vaes = folder_paths.get_filename_list("vae_approx")
have_img_encoder, have_img_decoder = set(), set()
for v in approx_vaes:
- parts = v.split("_", 1)
+ parts = v.rsplit("_", 1)
if len(parts) != 2 or parts[0] not in s.image_taes:
for tae in s.video_taes:
if v.startswith(tae):
@@ -824,6 +825,10 @@ class VAELoader:
elif name == "taef1":
sd["vae_scale"] = torch.tensor(0.3611)
sd["vae_shift"] = torch.tensor(0.1159)
+ elif name == "taeqi2_1":
+ latent_format = comfy.latent_formats.QwenImage21()
+ sd["vae_scale"] = 1.0 / latent_format.latents_std[0]
+ sd["vae_shift"] = latent_format.latents_mean[0]
return sd
@classmethod