The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Optimize MiniMax-H3 VAE (#15446)
Jukka Seppänen2a68ce33b4c9
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex 65d06f3e9..e1c146ec4 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -6,6 +6,7 @@ import torchimport torch.nn as nnimport torch.nn.functional as F+import comfy.model_managementimport comfy.opsimport comfy.quant_opsimport comfy.rmsnorm@@ -321,6 +322,8 @@ class ViT3DDecoder(nn.Module):# Full VAEclass MiniMaxH3VideoVAE(nn.Module):+ comfy_has_chunked_io = True+def __init__(self,in_channels=3,@@ -389,6 +392,23 @@ class MiniMaxH3VideoVAE(nn.Module):def _decode_pixels(self, z):return self.decoder(self.post_quant_conv(z))+ def _normalize_pixels(self, x):+ return x.add(1.0).mul_(0.5).sub_(self.pixel_mean.to(x)).div_(self.pixel_std.to(x))++ def _finalize_pixels(self, part):+ # raw decoder output -> float32 pixels in [0, 1] (the VAE wrapper's process_output is identity)+ part = part * self.pixel_std.to(device=part.device, dtype=torch.float32)+ return part.add_(self.pixel_mean.to(device=part.device, dtype=torch.float32)).clamp_(0.0, 1.0)++ def decode_output_shape(self, input_shape):+ b, c, t, h, w = input_shape+ if t == 1:+ frames = 1+ else:+ pad_tokens, num_chunks = self._decode_temporal_chunks(t)+ frames = self._decode_temporal_frame_plan(t + pad_tokens, num_chunks, pad_tokens)+ return (b, self.decoder.out_channels, frames, h * self.vae_ratio, w * self.vae_ratio)+def _adaptive_encode(self, x):if self.tiling:return self.tiled_encode(x)@@ -521,18 +541,15 @@ class MiniMaxH3VideoVAE(nn.Module):# temporal chunking- def encode_temporal(self, x):- if x.shape[2] % self.clip_length != 0:- pad_size = (-x.shape[2]) % self.clip_length- pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1)- x = torch.cat([x, pad_frames], dim=2)-- num_chunks = x.shape[2] // self.clip_length-+ def encode_temporal(self, x, device):+ # chunked input io: x may live on the CPU, clips move to the device as they encodez_list = []- for i in range(num_chunks):- clip_x = x[:, :, i * self.clip_length:(i + 1) * self.clip_length, :, :]- z_list.append(self._adaptive_encode(clip_x))+ for i in range(math.ceil(x.shape[2] / self.clip_length)):+ clip_x = x[:, :, i * self.clip_length:(i + 1) * self.clip_length, :, :].to(device)+ if clip_x.shape[2] < self.clip_length:+ pad_frames = clip_x[:, :, -1:].repeat(1, 1, self.clip_length - clip_x.shape[2], 1, 1)+ clip_x = torch.cat([clip_x, pad_frames], dim=2)+ z_list.append(self._adaptive_encode(self._normalize_pixels(clip_x)))z = torch.cat(z_list, dim=2)if self.token_drop > 0:@@ -577,43 +594,42 @@ class MiniMaxH3VideoVAE(nn.Module):total_frames += final_overlap_framesreturn total_frames - self._decode_temporal_pad_frames(z_len, pad_tokens)- def decode_temporal(self, z):- chunk_dec = self.tokens_chunk_size * self.vae_ratio_t- split_count = int(self.token_drop > 0) + 1-- pseudo_total_tokens = z.shape[2] + self.token_drop-- pad_tokens = 0- remainder = pseudo_total_tokens % self.tokens_chunk_size- if remainder != 0:- pad_tokens = self.tokens_chunk_size - remainder- pseudo_total_tokens += pad_tokens+ def _decode_temporal_chunks(self, z_len):+ pseudo_total_tokens = z_len + self.token_drop+ pad_tokens = (-pseudo_total_tokens) % self.tokens_chunk_size+ pseudo_total_tokens += pad_tokensnum_chunks = pseudo_total_tokens // self.tokens_chunk_size - int(self.token_drop > 0)if num_chunks < 1:# too few tokens for one chunk (e.g. T_lat == 2): pad one extra chunkpad_tokens += self.tokens_chunk_sizenum_chunks += 1+ return pad_tokens, num_chunks++ def decode_temporal(self, z, output_buffer=None):+ chunk_dec = self.tokens_chunk_size * self.vae_ratio_t+ split_count = int(self.token_drop > 0) + 1++ if output_buffer is None:+ # finalized chunks stream out of VRAM so the full video never sits on the GPU+ output_buffer = torch.empty(self.decode_output_shape(z.shape), dtype=torch.float32,+ device=comfy.model_management.intermediate_device())+ pad_tokens, num_chunks = self._decode_temporal_chunks(z.shape[2])if pad_tokens > 0:pad_z = z[:, :, -1:, :, :].repeat(1, 1, pad_tokens, 1, 1)z = torch.cat([z, pad_z], dim=2)- output_frames = self._decode_temporal_frame_plan(z.shape[2], num_chunks, pad_tokens)-- dec = None+ dec = output_bufferdec_overlap = Nonewrite_pos = 0def write_part(part):- nonlocal dec, write_pos+ nonlocal write_pospart_frames = part.shape[2]if part_frames <= 0:return- if dec is None:- out_shape = list(part.shape)- out_shape[2] = output_frames- dec = torch.empty(out_shape, dtype=part.dtype, device=part.device)+ part = self._finalize_pixels(part)copy_frames = min(part_frames, max(0, dec.shape[2] - write_pos))if copy_frames > 0:dec[:, :, write_pos:write_pos + copy_frames, :, :].copy_(@@ -653,18 +669,18 @@ class MiniMaxH3VideoVAE(nn.Module):return dec- def encode(self, x):+ def encode(self, x, device=None):# x: [B, 3, T, H, W] in [-1, 1] -> normalized latents [B, 24, T_lat, H/16, W/16]if x.ndim == 4:x = x.unsqueeze(2)-- x = x.add(1.0).mul_(0.5).sub_(self.pixel_mean.to(x)).div_(self.pixel_std.to(x))+ if device is None:+ device = x.deviceif x.shape[2] == 1:- moments = self._adaptive_encode(x)+ moments = self._adaptive_encode(self._normalize_pixels(x.to(device)))moments = moments[:, :, -1:, :, :]else:- moments = self.encode_temporal(x)+ moments = self.encode_temporal(x, device)mean = torch.chunk(moments.float(), 2, dim=1)[0]@@ -679,18 +695,16 @@ class MiniMaxH3VideoVAE(nn.Module):def decode_tiled(self, z, **kwargs):return self.decode(z)- def decode(self, z):- # z: [B, 24, T_lat, H_lat, W_lat] normalized latents -> pixels [B, 3, T, H, W] in [-1, 1]+ def decode(self, z, output_buffer=None):+ # z: [B, 24, T_lat, H_lat, W_lat] normalized latents -> float32 pixels [B, 3, T, H, W] in [0, 1]latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(z)latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(z)z = z * latents_std + latents_meanif z.shape[2] == 1:- dec = self._adaptive_decode(z)- dec = dec[:, :, -1:, :, :]- else:- dec = self.decode_temporal(z)-- dec = dec.float()- dec.mul_(self.pixel_std.to(dec)).add_(self.pixel_mean.to(dec)).clamp_(0.0, 1.0).mul_(2.0).sub_(1.0)- return dec+ dec = self._finalize_pixels(self._adaptive_decode(z)[:, :, -1:, :, :])+ if output_buffer is None:+ return dec+ output_buffer.copy_(dec)+ return output_buffer+ return self.decode_temporal(z, output_buffer)diff --git a/comfy/sd.py b/comfy/sd.pyindex 9ccd561bc..5fed4ca9a 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -955,13 +955,21 @@ class VAE:self.working_dtypes = [torch.float16, torch.float32]# the model tiles internally (256px spatial, 17-frame temporal chunks)self.handles_tiling = True+ # decode finalizes straight to [0, 1] while streaming chunks out+ self.process_output = lambda image: image+ # one decoded temporal chunk (with overlap) is all that ever sits in VRAM+ chunk_frames = (self.first_stage_model.tokens_chunk_size + self.first_stage_model.token_overlap) * self.first_stage_model.vae_ratio_t+def estimate_encode_memory(frames, height, width, dtype):fixed = 110_000_000 if frames == 1 else 1_300_000_000elements_per_pixel = 7 if frames == 1 else 9.5+ # only one clip of the input video is ever resident on the GPU+ frames = min(frames, self.first_stage_model.clip_length)return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03def estimate_decode_memory(frames, height, width, dtype):fixed = 110_000_000 if frames <= 22 else 270_000_000+ frames = min(frames, chunk_frames + 2)return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], shape[3], shape[4], dtype)@@ -1198,6 +1206,7 @@ class VAE:do_tile = Trueif do_tile:+ pixel_samples = Nonecomfy.model_management.soft_empty_cache()dims = samples_in.ndim - 2if dims == 1 or self.extra_1d_channel is not None: