The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
MiniMax-H3 VAE optimizations (#16187)
Jukka Seppänenb2e31e89412a
diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pyindex e1c146ec4..dafd460cb 100644--- a/comfy/ldm/minimax/vae.py+++ b/comfy/ldm/minimax/vae.py@@ -10,7 +10,7 @@ import comfy.model_managementimport comfy.opsimport comfy.quant_opsimport comfy.rmsnorm-from comfy.ldm.modules.attention import optimized_attention+from comfy.ldm.modules.attention import COMFY_KITCHEN_INT8_ATTENTION_IS_AVAILABLE, optimized_attentionops = comfy.ops.disable_weight_init@@ -38,22 +38,85 @@ LATENTS_STD = [# 3D causal CNN encoder+def _kitchen_ndhwc(x):+ # NDHWC when the kitchen pad kernel can feed it and the conv consumes it natively:+ # cuDNN does, MIOpen converts back and loses more than the fused pad saves+ ck = getattr(comfy.quant_ops, "ck", None)+ return (ck is not None and hasattr(ck, "group_norm_silu_pad3d") and torch.version.hip is None+ and x.is_cuda and x.dtype in (torch.float16, torch.bfloat16))+++def _fused_norm_pad(x, norm, spatial_pad, front):+ # per-frame GroupNorm + SiLU + padding in one kitchen pass, NDHWC out+ if not _kitchen_ndhwc(x) or x.shape[1] % 8:+ return None+ ck = comfy.quant_ops.ck+ if norm is None:+ return ck.group_norm_silu_pad3d(x, None, None, 1, 0.0, (*spatial_pad, front), silu=False)+ # cast_bias_weight: works for offloaded (dynamic VRAM) layers+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(norm, x, offloadable=True)+ try:+ return ck.group_norm_silu_pad3d(x, weight, bias, norm.num_groups, norm.eps, (*spatial_pad, front), silu=True)+ finally:+ comfy.ops.uncast_bias_weight(norm, weight, bias, offload_stream)+++def _fp16_accum_conv(conv, x, weight, bias, residual):+ # kitchen fp16-accumulate conv, same opt-in as the fp16 GEMMs+ ck = comfy.quant_ops.ck+ if not hasattr(ck, "fp16_conv3d") or not comfy.ops._fp16_linear_wanted(x):+ return None+ return ck.fp16_conv3d(x, weight, bias, residual, conv.stride)++class CausalConv3d(ops.Conv3d):# Reflect spatial padding, causal (zeros, front-only) temporal padding.def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride)self.causal_padding = (padding,) * 3 if isinstance(padding, int) else tuple(padding)- def forward(self, x):- if sum(self.causal_padding) == 0:- return super().forward(x)-- x = F.pad(x, (self.causal_padding[2], self.causal_padding[2], self.causal_padding[1], self.causal_padding[1], 0, 0), mode="reflect")- if x.shape[2] == 1:+ def forward(self, x, pre_norm=None, spatial_pad=None, residual=None):+ # pre_norm, spatial_pad and residual fold into the kitchen pad pass and conv epilogue+ pad_t, pad_h, pad_w = self.causal_padding+ if spatial_pad is None:+ spatial_pad = (pad_w, pad_w, pad_h, pad_h)+ front = 0 if x.shape[2] == 1 else pad_t * 2++ ndhwc = _kitchen_ndhwc(x)+ if ndhwc and self.weight.is_cuda and not self.weight.is_contiguous(memory_format=torch.channels_last_3d):+ self.weight.data = self.weight.data.contiguous(memory_format=torch.channels_last_3d)++ if pre_norm is not None or front or any(spatial_pad): # the 1x1 shortcut has nothing to fold+ fused = _fused_norm_pad(x, pre_norm, spatial_pad, front)+ if fused is not None:+ x = fused+ else:+ if pre_norm is not None:+ x = F.silu(pre_norm(x), inplace=True)+ if any(spatial_pad):+ x = F.pad(x, (*spatial_pad, 0, 0), mode="reflect")+ if front:+ x = F.pad(x, (0, 0, 0, 0, front, 0), mode="constant")++ if x.shape[2] == 1 and pad_t:# single frame: the causal front padding is all zeros truncate the temporal taps instead of convolving zero frames- return super().forward(x, autopad="causal_zero")- x = F.pad(x, (0, 0, 0, 0, self.causal_padding[0] * 2, 0), mode="constant")- return super().forward(x)+ out = super().forward(x, autopad="causal_zero")+ elif ndhwc:+ # cast_bias_weight handles offloaded layers; channels_last keeps cuDNN's output NDHWC+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self, x, offloadable=True)+ try:+ weight_cl = weight.contiguous(memory_format=torch.channels_last_3d)+ out = _fp16_accum_conv(self, x, weight_cl, bias, residual)+ if out is not None:+ return out+ out = F.conv3d(x, weight_cl, bias, self.stride)+ finally:+ comfy.ops.uncast_bias_weight(self, weight, bias, offload_stream)+ else:+ out = super().forward(x)+ if residual is not None:+ out += residual+ return outclass TemporalIsolatedGroupNorm(ops.GroupNorm):@@ -85,7 +148,7 @@ class Downsample3D(nn.Module):def forward(self, x):if self.space_stride == 2:- x = F.pad(x, (0, 1, 0, 1, 0, 0), mode="reflect")+ return self.conv(x, spatial_pad=(0, 1, 0, 1))return self.conv(x)@@ -104,11 +167,10 @@ class ResnetBlock3D(nn.Module):self.nin_shortcut = CausalConv3d(in_channels, out_channels, kernel_size=1)def forward(self, x):- h = self.conv1(F.silu(self.norm1(x), inplace=True))- h = self.conv2(F.silu(self.norm2(h), inplace=True))+ h = self.conv1(x, pre_norm=self.norm1)if self.in_channels != self.out_channels:x = self.nin_shortcut(x)- return h.add_(x)+ return self.conv2(h, pre_norm=self.norm2, residual=x)class EncoderFCN3D(nn.Module):@@ -160,8 +222,7 @@ class EncoderFCN3D(nn.Module):h = self.down[i_level].block[i_block](h)if hasattr(self.down[i_level], "downsample"):h = self.down[i_level].downsample(h)- h = F.silu(self.norm_out(h))- return self.conv_out(h)+ return self.conv_out(h, pre_norm=self.norm_out)# ViT3D decoder@@ -206,9 +267,11 @@ class FeedForward(nn.Module):self.w1 = operations.Linear(dim, inner_dim * 2, bias=bias)self.w2 = operations.Linear(inner_dim, dim, bias=bias)- def forward(self, x):- gate, x = self.w1(x).chunk(2, dim=-1)- return self.w2(F.silu(gate).mul_(x))+ def forward(self, x, pre_norm, residual, residual_scale):+ # norm, gated silu and residual addcmul fold into the INT8 kernels+ h = comfy.ops.linear_input_act(self.w1, x, "rms_norm", pre_norm.weight, pre_norm.eps)+ return comfy.ops.linear_input_act(+ self.w2, h, "swiglu", residual=residual, residual_scale=residual_scale)class Attention(nn.Module):@@ -219,27 +282,39 @@ class Attention(nn.Module):inner_dim = dim_head * headsself.norm_q = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False)self.norm_k = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False)+ # unit scale for the fused rms+rope kernel; the qk norms are not affine+ self.register_buffer("qk_norm_scale", torch.ones(dim_head), persistent=False)self.to_qkv = operations.Linear(inner_dim, inner_dim * 3, bias=bias)self.to_out = operations.Linear(inner_dim, inner_dim, bias=bias)- def forward(self, x, rotary_pos_emb=None):+ def forward(self, x, rotary_pos_emb, pre_norm, residual, residual_scale):batch_size, seq_len, _ = x.shape- qkv = self.to_qkv(x)+ qkv = comfy.ops.linear_input_act(self.to_qkv, x, "rms_norm", pre_norm.weight, pre_norm.eps)qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)query, key, value = torch.chunk(qkv, 3, dim=-1)- query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps)- key = comfy.rmsnorm.rms_norm(key, self.norm_k.weight, self.norm_k.eps)-if rotary_pos_emb is not None:- rot = rotary_pos_emb.shape[-3] * 2- query[..., :rot], key[..., :rot] = comfy.quant_ops.ck.apply_rope_split_half(- query[..., :rot], key[..., :rot], rotary_pos_emb)-- out = optimized_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2),- self.heads, skip_reshape=True).nan_to_num_(0.0)- return self.to_out(out)+ # fused per-head RMSNorm + partial split-half rope, in place when autograd is off+ rms_rope = (comfy.quant_ops.ck.rms_rope_split_half if torch.is_grad_enabled()+ else comfy.quant_ops.ck.rms_rope_split_half_)+ query, key = rms_rope(+ query, key, rotary_pos_emb, self.qk_norm_scale,+ epsilon=self.norm_q.eps, rot_dim=rotary_pos_emb.shape[-3] * 2)+ else:+ query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps)+ key = comfy.rmsnorm.rms_norm(key, self.norm_k.weight, self.norm_k.eps)++ query, key, value = (t.transpose(1, 2) for t in (query, key, value))+ # an int8 decoder already accepts int8 numerics; fp16 keeps exact flash+ if isinstance(self.to_qkv.weight, comfy.ops.QuantizedTensor) and COMFY_KITCHEN_INT8_ATTENTION_IS_AVAILABLE:+ out = comfy.quant_ops.ck.int8_attention(query, key, value)+ out = out.transpose(1, 2).reshape(batch_size, seq_len, -1)+ else:+ out = optimized_attention(query, key, value, self.heads, skip_reshape=True)+ return comfy.ops.linear_input_act(+ self.to_out, torch.nan_to_num(out), None,+ residual=residual, residual_scale=residual_scale)class TransformerBlock(nn.Module):@@ -253,9 +328,17 @@ class TransformerBlock(nn.Module):self.ff = FeedForward(dim=dim, bias=bias, operations=operations)self.scale2 = nn.Parameter(torch.empty(dim))+ def _residual_scale(self, scale, x):+ # cast_to_input always copies, defeating the kernels' identity-keyed caches+ if scale.dtype == x.dtype and scale.device == x.device:+ return scale+ return comfy.ops.cast_to_input(scale, x)+def forward(self, x, rotary_pos_emb=None):- x = x.addcmul_(self.attn(comfy.rmsnorm.rms_norm(x, self.norm1.weight, self.norm1.eps), rotary_pos_emb), comfy.ops.cast_to_input(self.scale1, x))- return x.addcmul_(self.ff(comfy.rmsnorm.rms_norm(x, self.norm2.weight, self.norm2.eps)), comfy.ops.cast_to_input(self.scale2, x))+ x = self.attn(x, rotary_pos_emb, pre_norm=self.norm1,+ residual=x, residual_scale=self._residual_scale(self.scale1, x))+ return self.ff(x, pre_norm=self.norm2,+ residual=x, residual_scale=self._residual_scale(self.scale2, x))class ViT3DDecoder(nn.Module):@@ -502,6 +585,17 @@ class MiniMaxH3VideoVAE(nn.Module):result_rows.append(torch.cat(result_row, dim=-1))return torch.cat(result_rows, dim=-2)+ def _decode_tile_row(self, z_row, x_idx, x_len):+ # a few tiles per decoder call (~7%); each extra tile holds ~100 MB of activations+ free = comfy.model_management.get_free_memory(z_row.device)+ batch = int(max(1, min(4, free // (128 * 2**20 * z_row.shape[0]))))+ slices = [z_row[..., j_pos // self.vae_ratio:(j_pos + j_len) // self.vae_ratio] for j_pos, j_len in zip(x_idx, x_len)]+ tiles = []+ for k in range(0, len(slices), batch):+ group = slices[k:k + batch]+ tiles.extend(self._decode_pixels(torch.cat(group)).chunk(len(group)))+ return tiles+def tiled_decode(self, z):height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratioy_idx, y_len, y_overlap = self.split_tiles(height)@@ -513,12 +607,11 @@ class MiniMaxH3VideoVAE(nn.Module):out_y = 0for i, (i_pos, i_len) in enumerate(zip(y_idx, y_len)):zi, zl = i_pos // self.vae_ratio, i_len // self.vae_ratio+ tiles = self._decode_tile_row(z[..., zi:zi + zl, :], x_idx, x_len)new_tails = []left_tail = Noneout_x = 0- for j, (j_pos, j_len) in enumerate(zip(x_idx, x_len)):- zj, zw = j_pos // self.vae_ratio, j_len // self.vae_ratio- tile = self._decode_pixels(z[..., zi:zi + zl, zj:zj + zw])+ for j, tile in enumerate(tiles):if i < len(y_idx) - 1:new_tails.append(tile[..., -y_overlap[i]:, :].clone())next_left_tail = tile[..., :, -x_overlap[j]:].clone() if j < len(x_idx) - 1 else Nonediff --git a/comfy/ops.py b/comfy/ops.pyindex e0bc79ad4..3089454aa 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -26,6 +26,7 @@ import comfy.floatimport jsonimport comfy.memory_managementimport comfy.pinned_memory+import comfy.rmsnormimport comfy.utilsimport comfy_aimdo.model_vbar@@ -955,22 +956,61 @@ INPUT_ACT_EAGER = {}-def linear_input_act(linear, x, input_act):+def _eager_input_act(x, input_act, act_weight=None, act_eps=0.0):+ if input_act is None:+ return x+ if input_act == "rms_norm":+ return comfy.rmsnorm.rms_norm(x, act_weight, act_eps)+ return INPUT_ACT_EAGER[input_act](x)+++def _fp16_linear_wanted(x):+ """kitchen's fp16-accumulate GEMM replaces a plain linear when the user opted into+ fp16 accumulation and the activation is fp16 on CUDA; weights come through cast_bias_weight."""+ return (getattr(torch.backends.cuda.matmul, "allow_fp16_accumulation", False)+ and x.dtype == torch.float16 and x.is_cuda)+++def linear_input_act(linear, x, input_act, act_weight=None, act_eps=0.0,+ residual=None, residual_scale=None):"""``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.An INT8 linear quantizes its input anyway, so an elementwise activation canride along inside that kernel instead of writing a full-size intermediate toHBM and reading it straight back. Worth it for an MLP's down-projection,- where the intermediate is several times the hidden size.+ where the intermediate is several times the hidden size, and for a pre-norm+ block's ``linear(rms_norm(x))`` ("rms_norm", which reads the norm's weight+ and eps from act_weight/act_eps).++ With ``residual``/``residual_scale`` the result is the pre-norm block's+ addcmul, ``residual + residual_scale * linear(act(x))``, fused into the+ INT8 GEMM epilogue where supported."""+ def _residual_out(out):+ if residual is None:+ return out+ return torch.addcmul(residual, out, residual_scale)+weight = linear.weight+ full_precision_mm = getattr(linear, "_full_precision_mm", False)if (comfy.model_management.in_trainingor not isinstance(weight, QuantizedTensor)or weight._layout_cls != "TensorWiseINT8Layout"or getattr(weight._params, "transposed", False)- or getattr(linear, "_full_precision_mm", False)):- return linear(INPUT_ACT_EAGER[input_act](x))+ or full_precision_mm):+ if (not comfy.model_management.in_training+ and not isinstance(weight, QuantizedTensor)+ and not full_precision_mm+ and _fp16_linear_wanted(x)):+ weight, bias, offload_stream = cast_bias_weight(linear, x, offloadable=True)+ try:+ return quant_ops.ck.fp16_linear(+ _eager_input_act(x, input_act, act_weight, act_eps),+ weight, bias, residual=residual, residual_scale=residual_scale)+ finally:+ uncast_bias_weight(linear, weight, bias, offload_stream)+ return _residual_out(linear(_eager_input_act(x, input_act, act_weight, act_eps)))# want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is# patched in on the fly; without it the cast hands back a dequantized weight.@@ -980,13 +1020,18 @@ def linear_input_act(linear, x, input_act):if not isinstance(weight, QuantizedTensor):# A LoRA weight_function, or activations whose dtype differs from the# weight's, make the cast hand back a dequantized tensor.- return torch.nn.functional.linear(INPUT_ACT_EAGER[input_act](x), weight, bias)+ return _residual_out(torch.nn.functional.linear(+ _eager_input_act(x, input_act, act_weight, act_eps), weight, bias))qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)return quant_ops.ck.int8_linear(x, qdata, scale, bias, x.dtype,convrot=getattr(weight._params, "convrot", False),convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256),input_act=input_act,+ input_act_weight=act_weight,+ input_act_eps=act_eps,+ residual=residual,+ residual_scale=residual_scale,)finally:uncast_bias_weight(linear, weight, bias, offload_stream)