The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support MiniMax-H3 latent upscaler model (#17412)
Jukka Seppänen35060f015618
diff --git a/comfy/ldm/minimax/latent_upscaler.py b/comfy/ldm/minimax/latent_upscaler.pynew file mode 100644index 000000000..482046cc9--- /dev/null+++ b/comfy/ldm/minimax/latent_upscaler.py@@ -0,0 +1,148 @@+import comfy_kitchen+import torch+import torch.nn as nn+import torch.nn.functional as F++import comfy.ops+from comfy.ldm.minimax.vae import LATENTS_MEAN, LATENTS_STD+++SLAB = 16 # latent frames per slab; the GroupNorm statistics stay clip-wide, so slabbing is exact+++def clip_stats(x, norm):+ n, c, t, h, w = x.shape+ return comfy_kitchen.group_norm_stats3d(x.reshape(n, c, 1, t * h, w), norm.num_groups, norm.eps)+++def slabs(t, halo):+ for t0 in range(0, t, SLAB):+ t1 = min(t0 + SLAB, t)+ yield t0, t1, max(t0 - halo, 0), min(t1 + halo, t)+++def norm_silu(x, norm, stats, scale=None, shift=None):+ """SiLU(GroupNorm(x) * (1 + scale) + shift) of a slab, normalized with the clip's stats; channels-last out."""+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(norm, x, offloadable=True)+ try:+ w, b = weight, bias+ if scale is not None: # the modulation folds into the norm affine+ w, b = weight * (1 + scale), torch.addcmul(shift, bias, 1 + scale)+ n, c, t, h, wd = x.shape+ # kitchen normalizes per frame; one frame spanning the clip gives nn.GroupNorm's statistics+ out = comfy_kitchen.group_norm_silu_pad3d(x.reshape(n, c, 1, t * h, wd), w, b, norm.num_groups, norm.eps, (0, 0, 0, 0, 0), silu=True, stats=stats)+ return out.view(n, c, t, h, wd)+ finally:+ comfy.ops.uncast_bias_weight(norm, weight, bias, offload_stream)+++def conv3d_cl(conv, x):+ # cast weights arrive channels-first; matching the activations keeps cuDNN and the residual adds on fast paths+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(conv, x, offloadable=True)+ try:+ return F.conv3d(x, weight.contiguous(memory_format=torch.channels_last_3d), bias, conv.stride, conv.padding, conv.dilation, conv.groups)+ finally:+ comfy.ops.uncast_bias_weight(conv, weight, bias, offload_stream)+++class ResBlockEmb3D(nn.Module):+ def __init__(self, channels, emb_channels, dtype=None, device=None, operations=None):+ super().__init__()+ # Identity slots keep the checkpoint's module indices (the activations run in norm_silu)+ self.in_layers = nn.Sequential(+ operations.GroupNorm(32, channels, dtype=dtype, device=device), nn.Identity(),+ operations.Conv3d(channels, channels, 3, padding=1, dtype=dtype, device=device))+ self.emb_layers = nn.Sequential(nn.SiLU(), operations.Linear(emb_channels, 2 * channels, dtype=dtype, device=device))+ self.out_norm = operations.GroupNorm(32, channels, dtype=dtype, device=device)+ self.out_layers = nn.Sequential(+ nn.Identity(), nn.Identity(),+ operations.Conv3d(channels, channels, 3, padding=1, dtype=dtype, device=device))++ def forward(self, x, emb):+ scale, shift = self.emb_layers(emb)[0].chunk(2)+ h = torch.empty_like(x)+ stats = clip_stats(x, self.in_layers[0])+ for t0, t1, a, b in slabs(x.shape[2], 1):+ h[:, :, t0:t1] = conv3d_cl(self.in_layers[2], norm_silu(x[:, :, a:b], self.in_layers[0], stats))[:, :, t0 - a:t1 - a]+ stats = clip_stats(h, self.out_norm)+ for t0, t1, a, b in slabs(x.shape[2], 1):+ x[:, :, t0:t1] += conv3d_cl(self.out_layers[2], norm_silu(h[:, :, a:b], self.out_norm, stats, scale, shift))[:, :, t0 - a:t1 - a]+ return x+++class TemporalConv(nn.Module):+ def __init__(self, channels, kernel_size, dtype=None, device=None, operations=None):+ super().__init__()+ self.norm = operations.GroupNorm(32, channels, dtype=dtype, device=device)+ self.dwconv = operations.Conv3d(channels, channels, (kernel_size, 1, 1), padding=(kernel_size // 2, 0, 0), groups=channels, dtype=dtype, device=device)+ self.pwconv = operations.Conv3d(channels, channels, 1, dtype=dtype, device=device)++ def forward(self, x):+ out = torch.empty_like(x)+ stats = clip_stats(x, self.norm)+ # the depthwise temporal conv as shifted multiply-adds: torch's depthwise conv3d has no channels-last kernel+ weight, bias, offload_stream = comfy.ops.cast_bias_weight(self.dwconv, x, offloadable=True)+ try:+ k = weight.shape[2]+ for t0, t1, a, b in slabs(x.shape[2], k // 2):+ h = norm_silu(x[:, :, a:b], self.norm, stats)+ acc = torch.empty_like(h).copy_(bias.view(1, -1, 1, 1, 1))+ for i in range(k):+ s = i - k // 2+ acc[:, :, max(0, -s):b - a - max(0, s)].addcmul_(h[:, :, max(0, s):b - a + min(0, s)], weight[:, 0, i].view(1, -1, 1, 1, 1))+ out[:, :, t0:t1] = self.pwconv(acc[:, :, t0 - a:t1 - a])+ finally:+ comfy.ops.uncast_bias_weight(self.dwconv, weight, bias, offload_stream)+ return x.add_(out)+++class MiniMaxH3LatentUpscaler(nn.Module):+ """3D-conv blocks at the input size, a trilinear resize, then blocks at the output size; every res block is+ conditioned on the scale factor and every other one is followed by a temporal conv."""++ def __init__(self, in_channels=24, channels=512, in_blocks=12, out_blocks=12, temporal_kernel=5, dtype=None, device=None, operations=None):+ super().__init__()+ self.dtype = dtype+ ops = {"dtype": dtype, "device": device, "operations": operations}+ self.conv_in = operations.Conv3d(in_channels, channels, 3, padding=1, dtype=dtype, device=device)+ self.embed = nn.Sequential(operations.Linear(1, 64, dtype=dtype, device=device), nn.SiLU(), operations.Linear(64, 64, dtype=dtype, device=device))+ self.in_blocks = nn.ModuleList()+ self.out_blocks = nn.ModuleList()+ for blocks, count in ((self.in_blocks, in_blocks), (self.out_blocks, out_blocks)):+ for i in range(count):+ blocks.append(ResBlockEmb3D(channels, 64, **ops))+ if i % 2 == 0:+ blocks.append(TemporalConv(channels, temporal_kernel, **ops))+ self.norm_out = operations.GroupNorm(32, channels, dtype=dtype, device=device)+ self.conv_out = operations.Conv3d(channels, in_channels, 3, padding=1, dtype=dtype, device=device)++ def get_dtype(self):+ return self.dtype++ def memory_required(self, shape, size):+ # two full-size activations plus slab temporaries worth ~75 frames (measured), at the larger of the two stages+ n, _, t, h, w = shape+ return n * self.conv_in.out_channels * (2 * t + 96) * max(h * w, size[1] * size[2]) * self.dtype.itemsize++ def run_blocks(self, blocks, x, emb):+ for block in blocks:+ x = block(x, emb) if isinstance(block, ResBlockEmb3D) else block(x)+ return x++ def forward(self, x, size):+ """Normalized H3 video latent [B, 24, T, H, W] -> the same at size (T, H', W')."""+ scale = (size[-1] / x.shape[-1] + size[-2] / x.shape[-2]) / 2+ mean = torch.tensor(LATENTS_MEAN, device=x.device, dtype=x.dtype).view(1, -1, 1, 1, 1)+ std = torch.tensor(LATENTS_STD, device=x.device, dtype=x.dtype).view(1, -1, 1, 1, 1)+ # trained on VAE latents normalized a second time with the same stats+ x = (x - mean) / std+ emb = self.embed(torch.full((1, 1), scale - 1, device=x.device, dtype=x.dtype))+ # activations stay channels-last, the layout norm_silu emits and the convs consume natively+ x = self.run_blocks(self.in_blocks, self.conv_in(x).contiguous(memory_format=torch.channels_last_3d), emb)+ x = F.interpolate(x, size=size, mode="trilinear", align_corners=False).contiguous(memory_format=torch.channels_last_3d)+ x = self.run_blocks(self.out_blocks, x, emb)+ out = torch.empty((x.shape[0], self.conv_out.out_channels) + tuple(x.shape[2:]), device=x.device, dtype=x.dtype)+ stats = clip_stats(x, self.norm_out)+ for t0, t1, a, b in slabs(x.shape[2], 1):+ out[:, :, t0:t1] = conv3d_cl(self.conv_out, norm_silu(x[:, :, a:b], self.norm_out, stats))[:, :, t0 - a:t1 - a]+ return torch.addcmul(mean, out, std)diff --git a/comfy_extras/nodes_hunyuan.py b/comfy_extras/nodes_hunyuan.pyindex 1d7f39aaf..70cf7a6ba 100644--- a/comfy_extras/nodes_hunyuan.py+++ b/comfy_extras/nodes_hunyuan.py@@ -9,6 +9,7 @@ from typing_extensions import overridefrom comfy_api.latest import ComfyExtension, iofrom comfy.ldm.hunyuan_video.upsampler import HunyuanVideo15SRModelfrom comfy.ldm.lightricks.latent_upsampler import LatentUpsampler+from comfy.ldm.minimax.latent_upscaler import MiniMaxH3LatentUpscalerimport folder_pathsimport json@@ -226,6 +227,19 @@ class LatentUpscaleModelLoader(io.ComfyNode):model_patcher = comfy.model_patcher.CoreModelPatcher(model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device(), fast_disk=fast_disk)model.load_state_dict(sd, assign=model_patcher.is_dynamic())model = model_patcher+ elif "in_blocks.0.emb_layers.1.weight" in sd and "conv_in.weight" in sd and "in_blocks.1.dwconv.weight" in sd:+ model = MiniMaxH3LatentUpscaler(+ in_channels=sd["conv_in.weight"].shape[1],+ channels=sd["conv_in.weight"].shape[0],+ in_blocks=len({k.split(".")[1] for k in sd if k.startswith("in_blocks.") and ".in_layers." in k}),+ out_blocks=len({k.split(".")[1] for k in sd if k.startswith("out_blocks.") and ".in_layers." in k}),+ temporal_kernel=sd["in_blocks.1.dwconv.weight"].shape[2],+ dtype=comfy.model_management.vae_dtype(allowed_dtypes=[torch.float16, torch.bfloat16, torch.float32]),+ operations=comfy.ops.disable_weight_init)+ comfy.model_management.archive_model_dtypes(model)+ model_patcher = comfy.model_patcher.CoreModelPatcher(model, load_device=comfy.model_management.get_torch_device(), offload_device=comfy.model_management.unet_offload_device(), fast_disk=fast_disk)+ model.load_state_dict(sd, assign=model_patcher.is_dynamic())+ model = model_patcherreturn io.NodeOutput(model)diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.pyindex 1ad3c6a40..6989453f8 100644--- a/comfy_extras/nodes_minimax_h3.py+++ b/comfy_extras/nodes_minimax_h3.py@@ -621,6 +621,39 @@ class MiniMaxH3FunControlNetApply(io.ComfyNode):return io.NodeOutput(model_patched)+class MiniMaxH3LatentUpscaleWithModel(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="MiniMaxH3LatentUpscaleWithModel",+ display_name="MiniMax H3 Latent Upscale With Model",+ category="model/latent/minimax",+ description="Resizes the video of a MiniMax H3 latent with a latent upscale model; audio passes through unchanged.",+ inputs=[+ io.LatentUpscaleModel.Input("model"),+ io.Latent.Input("samples"),+ io.Int.Input("width", default=2560, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("height", default=1440, min=32, max=nodes.MAX_RESOLUTION, step=32),+ ],+ outputs=[io.Latent.Output()],+ )++ @classmethod+ def execute(cls, model, samples, width, height) -> io.NodeOutput:+ latents = samples["samples"]+ video = latents.tensors[0] if latents.is_nested else latents+ size = (video.shape[2], height // 16, width // 16)+ if size == tuple(video.shape[2:]):+ return io.NodeOutput(samples)+ comfy.model_management.load_models_gpu([model], memory_required=model.model.memory_required(video.shape, size))+ upscaled = model.model(video.to(device=model.load_device, dtype=model.model_dtype()), size)+ upscaled = upscaled.to(device=comfy.model_management.intermediate_device(), dtype=video.dtype)+ out = samples.copy()+ out["samples"] = comfy.nested_tensor.NestedTensor((upscaled, latents.tensors[1])) if latents.is_nested else upscaled+ out.pop("noise_mask", None)+ return io.NodeOutput(out)++class MiniMaxH3Extension(ComfyExtension):async def get_node_list(self):return [@@ -630,6 +663,7 @@ class MiniMaxH3Extension(ComfyExtension):MiniMaxH3ReferenceToVideo,MiniMaxH3SigmaShift,MiniMaxH3FunControlNetApply,+ MiniMaxH3LatentUpscaleWithModel,]