Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Decode MiniMax H3 single images as the first latent of a clip (#16881)

Jukka Seppänenec1537de684c
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.py
index 84962b6a3..a2ffb6893 100644
--- a/comfy/ldm/minimax/vae.py
+++ b/comfy/ldm/minimax/vae.py
@@ -424,6 +424,7 @@ class MiniMaxH3VideoVAE(nn.Module):
tiling=True,
operations=ops,
num_layers=36,
+ still_frame=None,
):
super().__init__()
self.vae_ratio = int(math.prod(space_down))
@@ -437,6 +438,11 @@ class MiniMaxH3VideoVAE(nn.Module):
self.token_overlap = (-token_drop) % self.tokens_chunk_size
self.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0)
+ # a decoder fine-tuned to decode one latent directly keeps this output frame of it
+ if still_frame is not None and not 0 <= still_frame < self.vae_ratio_t:
+ raise ValueError(f"still_frame must be in [0, {self.vae_ratio_t}), got {still_frame}")
+ self.still_frame = still_frame
+
# spatial tiling parameters
self.tiling = tiling
self.tile_size = tile_size
@@ -498,10 +504,10 @@ class MiniMaxH3VideoVAE(nn.Module):
return self.tiled_encode(x)
return self._encode_moments(x)
- def _adaptive_decode(self, z):
+ def _adaptive_decode(self, z, frames=slice(None)):
if self.tiling:
- return self.tiled_decode(z)
- return self._decode_pixels(z)
+ return self.tiled_decode(z, frames)
+ return self._decode_pixels(z)[:, :, frames]
# spatial tiling
@@ -595,7 +601,7 @@ class MiniMaxH3VideoVAE(nn.Module):
group = slices[k:k + batch]
yield from self._decode_pixels(torch.cat(group)).chunk(len(group))
- def tiled_decode(self, z):
+ def tiled_decode(self, z, frames=slice(None)):
height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratio
y_idx, y_len, y_overlap = self.split_tiles(height)
x_idx, x_len, x_overlap = self.split_tiles(width)
@@ -613,7 +619,7 @@ class MiniMaxH3VideoVAE(nn.Module):
out_x = 0
# enumerate would retain the previous tile while the next batch decodes.
for j in range(len(x_idx)):
- tile = next(tiles)
+ tile = next(tiles)[:, :, frames]
if i > 0:
tile = self.blend(strip[..., :, x_idx[j]:x_idx[j] + x_len[j]], tile, y_overlap[i - 1], dim=-2)
if j > 0:
@@ -799,7 +805,14 @@ class MiniMaxH3VideoVAE(nn.Module):
z = z * latents_std + latents_mean
if z.shape[2] == 1:
- dec = self._finalize_pixels(self._adaptive_decode(z)[:, :, -1:, :, :])
+ if self.still_frame is not None:
+ dec = self._adaptive_decode(z, slice(self.still_frame, self.still_frame + 1))
+ else:
+ # A still latent is the first latent of a clip and decodes badly alone: decode it as a
+ # full clip of copies and keep only its first real frame.
+ first = slice(self.frame_pre_padding, self.frame_pre_padding + 1)
+ dec = self._adaptive_decode(z.repeat(1, 1, self.tokens_chunk_size, 1, 1), first)
+ dec = self._finalize_pixels(dec)
if output_buffer is None:
return dec
output_buffer.copy_(dec)
diff --git a/comfy/sd.py b/comfy/sd.py
index 50176f79c..2f106a7bd 100644
--- a/comfy/sd.py
+++ b/comfy/sd.py
@@ -1032,7 +1032,10 @@ class VAE:
if minimax_quant is not None: # int8+convrot quantized decoder
minimax_ops = comfy.ops.mixed_precision_ops(minimax_quant, dtype if dtype is not None else torch.float16)
minimax_layers = sum(k.startswith("decoder.transformer_blocks.") and k.endswith(".scale1") for k in sd)
- self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers)
+ still_frame = None
+ if metadata is not None and metadata.get("h3_t1_direct") == "true": # decoder fine-tuned for single-latent stills
+ still_frame = int(metadata.get("h3_t1_output_slice", 3))
+ self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE(operations=minimax_ops, num_layers=minimax_layers, still_frame=still_frame)
self.latent_channels = 24
self.latent_dim = 3
# frames 17k+5 <-> latents 5k+2, 16x spatial
@@ -1056,7 +1059,8 @@ class VAE:
return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03
def estimate_decode_memory(frames, height, width, dtype):
- fixed = 110_000_000 if frames <= 22 else 270_000_000
+ # the stock decoder decodes a still as a full clip of copies, so it needs a clip's activations
+ fixed = 150_000_000 if frames == 1 and still_frame is None else 110_000_000 if frames <= 22 else 270_000_000
frames = min(frames, chunk_frames + 2)
return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03
diff --git a/tests-unit/comfy_test/test_minimax_h3_vae.py b/tests-unit/comfy_test/test_minimax_h3_vae.py
index 515d43476..793a9c59c 100644
--- a/tests-unit/comfy_test/test_minimax_h3_vae.py
+++ b/tests-unit/comfy_test/test_minimax_h3_vae.py
@@ -6,7 +6,7 @@ if not torch.cuda.is_available():
args.cpu = True
import comfy.quant_ops
-from comfy.ldm.minimax.vae import Attention
+from comfy.ldm.minimax.vae import Attention, MiniMaxH3VideoVAE
class _OffloadedScale:
@@ -73,3 +73,23 @@ def test_attention_moves_offloaded_qk_norm_scale_to_input_device(monkeypatch):
assert len(qkv_outputs) == 1
torch.testing.assert_close(qkv_outputs[0], expected)
assert not torch.allclose(qkv_outputs[0], unnormalized)
+
+
+
+def test_still_decodes_as_first_latent_of_a_full_clip():
+ with torch.device("meta"):
+ vae = MiniMaxH3VideoVAE()
+ vae.latents_mean, vae.latents_std = torch.zeros(24), torch.ones(24)
+ vae.pixel_mean, vae.pixel_std = torch.zeros(1, 3, 1, 1, 1), torch.ones(1, 3, 1, 1, 1)
+ seen = []
+
+ def decode_pixels(z): # every output frame holds its own index / 100
+ seen.append(z.shape[2])
+ frames = torch.arange(z.shape[2] * vae.vae_ratio_t, dtype=torch.float32) / 100
+ return frames.view(1, 1, -1, 1, 1).expand(z.shape[0], 3, -1, z.shape[-2] * vae.vae_ratio, z.shape[-1] * vae.vae_ratio)
+
+ vae._decode_pixels = decode_pixels
+ out = vae.decode(torch.zeros(1, 24, 1, 40, 24)) # 640x384, several tiles
+ assert out.shape == (1, 3, 1, 640, 384)
+ assert set(seen) == {vae.tokens_chunk_size}
+ torch.testing.assert_close(out, torch.full_like(out, vae.frame_pre_padding / 100))