The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Decode MiniMax H3 single images as the first latent of a clip (#16881)
Jukka Seppänenec1537de684c
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex 84962b6a3..a2ffb6893 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -424,6 +424,7 @@ class MiniMaxH3VideoVAE(nn.Module):tiling=True,operations=ops,num_layers=36,+ still_frame=None,):super().__init__()self.vae_ratio = int(math.prod(space_down))@@ -437,6 +438,11 @@ class MiniMaxH3VideoVAE(nn.Module):self.token_overlap = (-token_drop) % self.tokens_chunk_sizeself.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0)+ # a decoder fine-tuned to decode one latent directly keeps this output frame of it+ if still_frame is not None and not 0 <= still_frame < self.vae_ratio_t:+ raise ValueError(f"still_frame must be in [0, {self.vae_ratio_t}), got {still_frame}")+ self.still_frame = still_frame+# spatial tiling parametersself.tiling = tilingself.tile_size = tile_size@@ -498,10 +504,10 @@ class MiniMaxH3VideoVAE(nn.Module):return self.tiled_encode(x)return self._encode_moments(x)- def _adaptive_decode(self, z):+ def _adaptive_decode(self, z, frames=slice(None)):if self.tiling:- return self.tiled_decode(z)- return self._decode_pixels(z)+ return self.tiled_decode(z, frames)+ return self._decode_pixels(z)[:, :, frames]# spatial tiling@@ -595,7 +601,7 @@ class MiniMaxH3VideoVAE(nn.Module):group = slices[k:k + batch]yield from self._decode_pixels(torch.cat(group)).chunk(len(group))- def tiled_decode(self, z):+ def tiled_decode(self, z, frames=slice(None)):height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratioy_idx, y_len, y_overlap = self.split_tiles(height)x_idx, x_len, x_overlap = self.split_tiles(width)@@ -613,7 +619,7 @@ class MiniMaxH3VideoVAE(nn.Module):out_x = 0# enumerate would retain the previous tile while the next batch decodes.for j in range(len(x_idx)):- tile = next(tiles)+ tile = next(tiles)[:, :, frames]if i > 0:tile = self.blend(strip[..., :, x_idx[j]:x_idx[j] + x_len[j]], tile, y_overlap[i - 1], dim=-2)if j > 0:@@ -799,7 +805,14 @@ class MiniMaxH3VideoVAE(nn.Module):z = z * latents_std + latents_meanif z.shape[2] == 1:- dec = self._finalize_pixels(self._adaptive_decode(z)[:, :, -1:, :, :])+ if self.still_frame is not None:+ dec = self._adaptive_decode(z, slice(self.still_frame, self.still_frame + 1))+ else:+ # A still latent is the first latent of a clip and decodes badly alone: decode it as a+ # full clip of copies and keep only its first real frame.+ first = slice(self.frame_pre_padding, self.frame_pre_padding + 1)+ dec = self._adaptive_decode(z.repeat(1, 1, self.tokens_chunk_size, 1, 1), first)+ dec = self._finalize_pixels(dec)if output_buffer is None:return decoutput_buffer.copy_(dec)diff --git a/comfy/sd.py b/comfy/sd.pyindex 50176f79c..2f106a7bd 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -1032,7 +1032,10 @@ class VAE:if minimax_quant is not None: # int8+convrot quantized decoderminimax_ops = comfy.ops.mixed_precision_ops(minimax_quant, dtype if dtype is not None else torch.float16)minimax_layers = sum(k.startswith("decoder.transformer_blocks.") and k.endswith(".scale1") for k in sd)- self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers)+ still_frame = None+ if metadata is not None and metadata.get("h3_t1_direct") == "true": # decoder fine-tuned for single-latent stills+ still_frame = int(metadata.get("h3_t1_output_slice", 3))+ self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers, still_frame=still_frame)self.latent_channels = 24self.latent_dim = 3# frames 17k+5 <-> latents 5k+2, 16x spatial@@ -1056,7 +1059,8 @@ class VAE:return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03def estimate_decode_memory(frames, height, width, dtype):- fixed = 110_000_000 if frames <= 22 else 270_000_000+ # the stock decoder decodes a still as a full clip of copies, so it needs a clip's activations+ fixed = 150_000_000 if frames == 1 and still_frame is None else 110_000_000 if frames <= 22 else 270_000_000frames = min(frames, chunk_frames + 2)return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03diff --git a/tests-unit/comfy_test/test_minimax_h3_vae.py b/tests-unit/comfy_test/test_minimax_h3_vae.pyindex 515d43476..793a9c59c 100644--- a/tests-unit/comfy_test/test_minimax_h3_vae.py+++ b/tests-unit/comfy_test/test_minimax_h3_vae.py@@ -6,7 +6,7 @@ if not torch.cuda.is_available():args.cpu = Trueimport comfy.quant_ops-from comfy.ldm.minimax.vae import Attention+from comfy.ldm.minimax.vae import Attention, MiniMaxH3VideoVAEclass _OffloadedScale:@@ -73,3 +73,23 @@ def test_attention_moves_offloaded_qk_norm_scale_to_input_device(monkeypatch):assert len(qkv_outputs) == 1torch.testing.assert_close(qkv_outputs[0], expected)assert not torch.allclose(qkv_outputs[0], unnormalized)++++def test_still_decodes_as_first_latent_of_a_full_clip():+ with torch.device("meta"):+ vae = MiniMaxH3VideoVAE()+ vae.latents_mean, vae.latents_std = torch.zeros(24), torch.ones(24)+ vae.pixel_mean, vae.pixel_std = torch.zeros(1, 3, 1, 1, 1), torch.ones(1, 3, 1, 1, 1)+ seen = []++ def decode_pixels(z): # every output frame holds its own index / 100+ seen.append(z.shape[2])+ frames = torch.arange(z.shape[2] * vae.vae_ratio_t, dtype=torch.float32) / 100+ return frames.view(1, 1, -1, 1, 1).expand(z.shape[0], 3, -1, z.shape[-2] * vae.vae_ratio, z.shape[-1] * vae.vae_ratio)++ vae._decode_pixels = decode_pixels+ out = vae.decode(torch.zeros(1, 24, 1, 40, 24)) # 640x384, several tiles+ assert out.shape == (1, 3, 1, 640, 384)+ assert set(seen) == {vae.tokens_chunk_size}+ torch.testing.assert_close(out, torch.full_like(out, vae.frame_pre_padding / 100))