The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat: Support MiniMax-H3 (CORE-375) (#15224)
Jukka Seppänen57500fc5bc92
diff --git a/comfy/latent_formats.py b/comfy/latent_formats.pyindex 63eefd420..c4270022b 100644--- a/comfy/latent_formats.py+++ b/comfy/latent_formats.py@@ -567,6 +567,45 @@ class LTXAV(LTXV):]self.latent_rgb_factors_bias = [-0.347892, -0.363814, -0.370287]+class MiniMaxH3Video(LatentFormat):+ latent_channels = 24+ latent_dimensions = 3+ spacial_downscale_ratio = 16+ temporal_downscale_ratio = 4+ scale_factor = 1.0++ latent_rgb_factors = [+ [-0.018555, 0.024344, -0.017536],+ [ 0.150164, 0.137244, 0.129221],+ [ 0.027367, -0.050369, -0.208606],+ [-0.000793, -0.164622, -0.323161],+ [-0.048556, 0.013970, -0.074286],+ [ 0.011740, 0.014172, -0.006906],+ [ 0.061517, 0.061212, 0.110025],+ [ 0.035321, 0.086879, 0.110059],+ [-0.017426, 0.002997, 0.035356],+ [ 0.531539, 0.548819, 0.624404],+ [-0.024968, -0.040234, -0.034302],+ [-0.032549, -0.029096, -0.017221],+ [ 0.022609, 0.020286, 0.050661],+ [-0.084001, -0.038131, -0.020805],+ [-0.018830, 0.010412, 0.061120],+ [ 0.020777, 0.011196, -0.030994],+ [-0.008390, -0.012201, -0.025687],+ [-0.013281, -0.002924, 0.006331],+ [ 0.000260, 0.001833, -0.011038],+ [ 0.105471, 0.100482, 0.132106],+ [ 0.016529, 0.015213, 0.009999],+ [-0.014015, -0.017438, -0.019134],+ [-0.033787, -0.009984, -0.019725],+ [ 0.004224, 0.017284, 0.027196],+ ]+ latent_rgb_factors_bias = [ 0.057426, -0.022078, -0.071449]++class MiniMaxH3AV(MiniMaxH3Video):+ # max channels across the two streams (video 24, audio 32) so per-stream slices keep both streams whole+ latent_channels = 32+class HunyuanVideo(LatentFormat):latent_channels = 16latent_dimensions = 3diff --git a/comfy/ldm/minimax/audio_vae.py b/comfy/ldm/minimax/audio_vae.pynew file mode 100644index 000000000..033ae6966--- /dev/null+++ b/comfy/ldm/minimax/audio_vae.py@@ -0,0 +1,442 @@+# MiniMax H3 audio VAE: DAC-lineage waveform encoder + BigVGAN decoder.+# Weight-norm parametrizations are folded into plain conv weights, so this+# module uses ordinary ops.Conv1d / ops.ConvTranspose1d and loads the converted+# checkpoint (plain "*.weight" tensors) with strict=True.+#+# Lineage / licenses of the reference implementation:+# DAC encoder: descript-audio-codec (MIT)+# BigVGAN decoder: NVIDIA BigVGAN (MIT), adapted from hifi-gan (MIT)+# Alias-free ops: junjun3518/alias-free-torch (Apache-2.0), julius (MIT)++import math++import torch+import torch.nn as nn+import torch.nn.functional as F++import comfy.ops++ops = comfy.ops.disable_weight_init+++# Snake activations++def snake(x, alpha, beta):+ # x + 1/beta * sin^2(alpha * x)+ t = torch.sin(alpha * x)+ return t.mul_(t).mul_((beta + 1e-9).reciprocal()).add_(x)+++class Snake1d(nn.Module):+ """Snake activation with per-channel alpha (encoder side)."""++ def __init__(self, channels):+ super().__init__()+ self.alpha = nn.Parameter(torch.empty(1, channels, 1))++ def forward(self, x):+ return snake(x, self.alpha, self.alpha)+++class SnakeBeta(nn.Module):+ """SnakeBeta := x + 1/beta * sin^2(alpha * x); alpha/beta stored in log scale."""++ def __init__(self, in_features):+ super().__init__()+ self.alpha = nn.Parameter(torch.empty(in_features))+ self.beta = nn.Parameter(torch.empty(in_features))++ def forward(self, x):+ alpha = torch.exp(self.alpha).view(1, -1, 1)+ beta = torch.exp(self.beta).view(1, -1, 1)+ return snake(x, alpha, beta)+++# Alias-free (anti-aliased) activation: kaiser-windowed sinc resampling++def kaiser_sinc_filter1d(cutoff, half_width, kernel_size):+ # returns filter [1, 1, kernel_size]+ even = kernel_size % 2 == 0+ half_size = kernel_size // 2++ # kaiser window design+ delta_f = 4 * half_width+ A = 2.285 * (half_size - 1) * math.pi * delta_f + 7.95+ if A > 50.0:+ beta = 0.1102 * (A - 8.7)+ elif A >= 21.0:+ beta = 0.5842 * (A - 21) ** 0.4 + 0.07886 * (A - 21.0)+ else:+ beta = 0.0+ window = torch.kaiser_window(kernel_size, beta=beta, periodic=False)++ if even:+ time = torch.arange(-half_size, half_size) + 0.5+ else:+ time = torch.arange(kernel_size) - half_size++ filter_ = 2 * cutoff * window * torch.sinc(2 * cutoff * time)+ # Normalize filter to have sum = 1, otherwise there is a small leakage of+ # the constant component in the input signal.+ filter_ /= filter_.sum()+ return filter_.view(1, 1, kernel_size)+++class UpSample1d(nn.Module):+ def __init__(self, ratio=2, kernel_size=12):+ super().__init__()+ self.ratio = ratio+ self.stride = ratio+ self.pad = kernel_size // ratio - 1+ self.pad_left = self.pad * ratio + (kernel_size - ratio) // 2+ self.pad_right = self.pad * ratio + (kernel_size - ratio + 1) // 2+ self.register_buffer(+ "filter",+ kaiser_sinc_filter1d(cutoff=0.5 / ratio, half_width=0.6 / ratio, kernel_size=kernel_size),+ )++ def forward(self, x):+ _, C, _ = x.shape+ x = F.pad(x, (self.pad, self.pad), mode="replicate")+ x = F.conv_transpose1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C).mul_(self.ratio)+ x = x[..., self.pad_left:-self.pad_right]+ return x+++class LowPassFilter1d(nn.Module):+ def __init__(self, cutoff=0.5, half_width=0.6, stride=1, kernel_size=12):+ super().__init__()+ self.pad_left = kernel_size // 2 - int(kernel_size % 2 == 0)+ self.pad_right = kernel_size // 2+ self.stride = stride+ self.register_buffer("filter", kaiser_sinc_filter1d(cutoff, half_width, kernel_size))++ def forward(self, x):+ _, C, _ = x.shape+ x = F.pad(x, (self.pad_left, self.pad_right), mode="replicate")+ return F.conv1d(x, self.filter.expand(C, -1, -1).to(x.dtype), stride=self.stride, groups=C)+++class DownSample1d(nn.Module):+ def __init__(self, ratio=2, kernel_size=12):+ super().__init__()+ self.ratio = ratio+ self.kernel_size = kernel_size+ self.lowpass = LowPassFilter1d(+ cutoff=0.5 / ratio,+ half_width=0.6 / ratio,+ stride=ratio,+ kernel_size=self.kernel_size,+ )++ def forward(self, x):+ return self.lowpass(x)+++class Activation1d(nn.Module):+ """upsample x2 -> pointwise activation -> downsample x2 (anti-aliased)."""++ def __init__(self, activation, up_ratio=2, down_ratio=2, up_kernel_size=12, down_kernel_size=12):+ super().__init__()+ self.act = activation+ self.upsample = UpSample1d(up_ratio, up_kernel_size)+ self.downsample = DownSample1d(down_ratio, down_kernel_size)++ def forward(self, x):+ x = self.upsample(x)+ x = self.act(x)+ x = self.downsample(x)+ return x+++# DAC encoder++class ResidualUnit(nn.Module):+ def __init__(self, dim=16, dilation=1):+ super().__init__()+ pad = ((7 - 1) * dilation) // 2+ self.block = nn.Sequential(+ Snake1d(dim),+ ops.Conv1d(dim, dim, kernel_size=7, dilation=dilation, padding=pad),+ Snake1d(dim),+ ops.Conv1d(dim, dim, kernel_size=1),+ )++ def forward(self, x):+ y = self.block(x)+ pad = (x.shape[-1] - y.shape[-1]) // 2+ if pad > 0:+ x = x[..., pad:-pad]+ return y.add_(x)+++class EncoderBlock(nn.Module):+ def __init__(self, dim=16, stride=1):+ super().__init__()+ self.block = nn.Sequential(+ ResidualUnit(dim // 2, dilation=1),+ ResidualUnit(dim // 2, dilation=3),+ ResidualUnit(dim // 2, dilation=9),+ Snake1d(dim // 2),+ ops.Conv1d(+ dim // 2,+ dim,+ kernel_size=2 * stride,+ stride=stride,+ padding=math.ceil(stride / 2),+ ),+ )++ def forward(self, x):+ return self.block(x)+++class Encoder(nn.Module):+ def __init__(self, d_model=64, strides=(2, 4, 4, 5, 5), d_latent=2048):+ super().__init__()+ block = [ops.Conv1d(1, d_model, kernel_size=7, padding=3)]+ for stride in strides:+ d_model *= 2+ block += [EncoderBlock(d_model, stride=stride)]+ block += [+ Snake1d(d_model),+ ops.Conv1d(d_model, d_latent, kernel_size=3, padding=1),+ ]+ self.block = nn.Sequential(*block)++ def forward(self, x):+ return self.block(x)+++# Attention projection (encoder posterior head)++class GeGluMlp(nn.Module):+ def __init__(self, in_features, hidden_features):+ super().__init__()+ self.norm = ops.LayerNorm(in_features)+ self.act = nn.GELU(approximate="tanh")+ self.w0 = ops.Linear(in_features, hidden_features)+ self.w1 = ops.Linear(in_features, hidden_features)+ self.w2 = ops.Linear(hidden_features, in_features)++ def forward(self, x):+ x = self.norm(x)+ return self.w2(self.act(self.w0(x)).mul_(self.w1(x)))+++class CausalAttention(nn.Module):+ def __init__(self, in_dim, out_dim, num_heads):+ super().__init__()+ self.head_dim = in_dim // num_heads+ self.num_heads = num_heads+ self.out_dim = out_dim+ self.qkv = ops.Linear(in_dim, in_dim * 3, bias=False)+ self.q_bias = nn.Parameter(torch.empty(in_dim))+ self.v_bias = nn.Parameter(torch.empty(in_dim))+ self.register_buffer("zero_k_bias", torch.empty(in_dim))+ self.proj = ops.Linear(out_dim, out_dim)++ def forward(self, x):+ B, N, C = x.shape+ weight, _, offload_stream = comfy.ops.cast_bias_weight(self.qkv, x, offloadable=True)+ qkv = F.linear(x, weight=weight, bias=torch.cat((self.q_bias, self.zero_k_bias, self.v_bias)))+ comfy.ops.uncast_bias_weight(self.qkv, weight, None, offload_stream)+ q, k, v = qkv.reshape(B, N, 3, self.num_heads, self.head_dim).permute(2, 0, 3, 1, 4).unbind(0)++ # mean over heads then pool down to the latent width (in_dim >> out_dim)+ x = comfy.ops.scaled_dot_product_attention(q, k, v, is_causal=True)+ x = F.adaptive_avg_pool1d(torch.mean(x, dim=1), self.out_dim)+ return self.proj(x)+++class AttnProjection(nn.Module):+ def __init__(self, in_dim, out_dim, num_heads, mlp_ratio=2):+ super().__init__()+ self.norm1 = ops.LayerNorm(in_dim)+ self.attn = CausalAttention(in_dim, out_dim, num_heads)+ self.proj = ops.Linear(in_dim, out_dim)+ self.norm3 = ops.LayerNorm(in_dim)++ self.norm2 = ops.LayerNorm(out_dim)+ hidden_dim = int(out_dim * mlp_ratio)+ self.mlp = GeGluMlp(in_features=out_dim, hidden_features=hidden_dim)++ def forward(self, x):+ # x: [B, T, in_dim]+ x = self.proj(self.norm3(x)).add_(self.attn(self.norm1(x)))+ return x.add_(self.mlp(self.norm2(x)))+++# BigVGAN decoder++def get_padding(kernel_size, dilation=1):+ return int((kernel_size * dilation - dilation) / 2)+++class AMPBlock1(nn.Module):+ def __init__(self, channels, kernel_size=3, dilation=(1, 3, 5)):+ super().__init__()+ self.convs1 = nn.ModuleList(+ [+ ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=d, padding=get_padding(kernel_size, d))+ for d in dilation+ ]+ )+ self.convs2 = nn.ModuleList(+ [+ ops.Conv1d(channels, channels, kernel_size, stride=1, dilation=1, padding=get_padding(kernel_size, 1))+ for _ in range(len(dilation))+ ]+ )+ self.num_layers = len(self.convs1) + len(self.convs2)+ self.activations = nn.ModuleList(+ [Activation1d(activation=SnakeBeta(channels)) for _ in range(self.num_layers)]+ )++ def forward(self, x):+ acts1, acts2 = self.activations[::2], self.activations[1::2]+ for c1, c2, a1, a2 in zip(self.convs1, self.convs2, acts1, acts2):+ xt = a1(x)+ xt = c1(xt)+ xt = a2(xt)+ xt = c2(xt)+ x = xt.add_(x)+ return x+++class BigVGAN(nn.Module):+ """BigVGAN vocoder (MiniMax H3 32 kHz configuration).++ use_bias_at_final=False, use_tanh_at_final=False (output clamped to [-1, 1]).+ """++ def __init__(+ self,+ num_mels=2048,+ upsample_initial_channel=1024,+ upsample_rates=(5, 5, 2, 2, 2, 2, 2),+ upsample_kernel_sizes=(9, 9, 4, 4, 4, 4, 4),+ resblock_kernel_sizes=(3, 7, 11),+ resblock_dilation_sizes=((1, 3, 5), (1, 3, 5), (1, 3, 5)),+ ):+ super().__init__()+ self.num_kernels = len(resblock_kernel_sizes)+ self.num_upsamples = len(upsample_rates)++ self.conv_pre = ops.Conv1d(num_mels, upsample_initial_channel, 7, 1, padding=3)++ self.ups = nn.ModuleList()+ for i, (u, k) in enumerate(zip(upsample_rates, upsample_kernel_sizes)):+ self.ups.append(+ nn.ModuleList(+ [+ ops.ConvTranspose1d(+ upsample_initial_channel // (2 ** i),+ upsample_initial_channel // (2 ** (i + 1)),+ k,+ u,+ padding=(k - u) // 2,+ )+ ]+ )+ )++ self.resblocks = nn.ModuleList()+ for i in range(len(self.ups)):+ ch = upsample_initial_channel // (2 ** (i + 1))+ for k, d in zip(resblock_kernel_sizes, resblock_dilation_sizes):+ self.resblocks.append(AMPBlock1(ch, k, d))++ self.activation_post = Activation1d(activation=SnakeBeta(ch))+ self.conv_post = ops.Conv1d(ch, 1, 7, 1, padding=3, bias=False)++ def forward(self, x):+ x = self.conv_pre(x)++ for i in range(self.num_upsamples):+ for i_up in range(len(self.ups[i])):+ x = self.ups[i][i_up](x)+ xs = None+ for j in range(self.num_kernels):+ if xs is None:+ xs = self.resblocks[i * self.num_kernels + j](x)+ else:+ xs += self.resblocks[i * self.num_kernels + j](x)+ x = xs.div_(self.num_kernels)++ x = self.activation_post(x)+ return self.conv_post(x).clamp_(-1.0, 1.0)+++# Top-level VAE++class MiniMaxH3AudioVAE(nn.Module):+ """MiniMax H3 stereo audio VAE at 32 kHz.++ Latents are [B, 32, 2, T]: 32 channels, 2 stereo channels, T frames at+ 40 latent frames per second (800 audio samples per latent frame). The+ stereo channels are processed independently by the mono encoder/decoder.+ Latents are normalized with the stored per-channel latents_mean/std.+ """++ def __init__(+ self,+ encoder_dim=64,+ encoder_rates=(2, 4, 4, 5, 5),+ latent_dim=2048,+ decoder_dim=1024,+ vae_latent_channels=32,+ ):+ super().__init__()+ self.sample_rate = 32000++ self.hop_length = 1+ for r in encoder_rates:+ self.hop_length *= r+ self.samples_per_latent = self.hop_length # 800+ self.latents_per_second = self.sample_rate // self.hop_length # 40+ self.output_sample_rate = self.sample_rate # read by LTXVAudioVAEDecode++ self.encoder = Encoder(encoder_dim, encoder_rates, latent_dim)++ self.pre_block = AttnProjection(latent_dim, vae_latent_channels, num_heads=8)++ self.mean_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1)+ # logs_proj exists in the checkpoint but is unused at inference+ # (encode returns the posterior mean, no sampling).+ self.logs_proj = ops.Conv1d(vae_latent_channels, vae_latent_channels, 1)++ self.dec_in_proj = ops.Conv1d(vae_latent_channels, latent_dim, 1)+ self.decoder = BigVGAN(num_mels=latent_dim, upsample_initial_channel=decoder_dim)++ self.register_buffer("latents_mean", torch.empty(vae_latent_channels))+ self.register_buffer("latents_std", torch.empty(vae_latent_channels))++ def decode(self, z):+ """Decode normalized latents [B, 32, 2, T] to stereo waveforms [B, 2, L] at 32 kHz."""+ b, c, s, t = z.shape+ z = z.permute(0, 2, 1, 3).reshape(b * s, c, t)+ mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype)+ std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype)+ z = z * std + mean+ x = self.dec_in_proj(z)+ x = self.decoder(x) # [b * s, 1, L], already clamped to [-1, 1]+ return x.reshape(b, s, -1)++ def encode(self, waveform):+ """Encode stereo waveforms [B, 2, L] at 32 kHz (in [-1, 1]) to normalized latents [B, 32, 2, T].++ L is right-padded with zeros to a multiple of 800 samples; the returned+ posterior mean is used directly (no sampling).+ """+ b, s, length = waveform.shape+ right_pad = math.ceil(length / self.hop_length) * self.hop_length - length+ waveform = F.pad(waveform, (0, right_pad))+ x = waveform.reshape(b * s, 1, -1)+ x = self.encoder(x) # [b * s, latent_dim, T]+ x = self.pre_block(x.transpose(1, 2)).transpose(1, 2) # [b * s, 32, T]+ z = self.mean_proj(x)+ mean = self.latents_mean.view(1, -1, 1).to(device=z.device, dtype=z.dtype)+ std = self.latents_std.view(1, -1, 1).to(device=z.device, dtype=z.dtype)+ z = (z - mean) / std+ return z.reshape(b, s, z.shape[1], z.shape[2]).permute(0, 2, 1, 3)diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.pynew file mode 100644index 000000000..494350d40--- /dev/null+++ b/comfy/ldm/minimax/model.py@@ -0,0 +1,646 @@+"""MiniMax H3 audio-video DiT.++Single-stream packed-token transformer denoising video (24ch, patch 1x2x2) and+stereo audio (32ch, 40 Hz) latents jointly, conditioned on Qwen3-VL layer-50 hidden states.+The packed sequence is:+[text | cond rows | audio | video] for t2va/fl2va+[text | reference blocks | audio | video] for ref2va++Timestep domain: the model receives the *video* sigma from the sampler and+derives per-token timesteps t = 1 - sigma internally; the audio stream runs on+its own shifted schedule (sigma_shift video 12.0 / audio 3.0), mapped from the+video sigma in closed form. The audio velocity is returned scaled by the+schedule map's derivative d(sigma_a)/d(sigma_v).+"""++import math++import torch+import torch.nn as nn++import comfy.ldm.common_dit+import comfy.model_management+import comfy.model_prefetch+import comfy.ops+import comfy.patcher_extension+import comfy.quant_ops+from comfy.ldm.modules.attention import optimized_attention++FRAME_PER_TOKEN = (1, 4, 4, 4, 4)+FRAME_RESCALE = 5.0 / 3.0+VISUAL_COND_TIMESTEP = 0.999+AUDIO_COND_TIMESTEP = 1.0+++def time_shift_sigma(sigma, from_shift, to_shift):+ # invert sigma = s*b/(1+(s-1)*b) to the base grid, re-apply the other shift+ base = sigma / (from_shift + sigma * (1.0 - from_shift))+ return to_shift * base / (1.0 + (to_shift - 1.0) * base)+++def time_shift_slope(sigma, from_shift, to_shift):+ """d(sigma_to)/d(sigma_from) at the same base-grid point.++ Scaling a stream's returned velocity by this slope makes the flat ODE that+ any sampler integrates on the from-schedule equal to that stream's true ODE+ on its own schedule.+ """+ base = sigma / (from_shift + sigma * (1.0 - from_shift))+ return (to_shift * (1.0 + (from_shift - 1.0) * base) ** 2) / (from_shift * (1.0 + (to_shift - 1.0) * base) ** 2)+++def patchify_video(latent, patch_size=(1, 2, 2)):+ # [B, C, T, H, W] -> [B*t*h*w, C*pt*ph*pw]+ b, c, t_full, h_full, w_full = latent.shape+ pt, ph, pw = patch_size+ t, h, w = t_full // pt, h_full // ph, w_full // pw+ x = latent.reshape(b, c, t, pt, h, ph, w, pw)+ x = torch.einsum("nctrhpwq->nthwcrpq", x)+ return x.reshape(b * t * h * w, c * pt * ph * pw)+++def unpatchify_video(rows, t, h, w, c=24, patch_size=(1, 2, 2)):+ pt, ph, pw = patch_size+ x = rows.reshape(-1, t, h, w, c, pt, ph, pw)+ x = torch.einsum("nthwcrpq->nctrhpwq", x)+ return x.reshape(-1, c, t * pt, h * ph, w * pw)+++def pack_audio(latent):+ # [B, C=32, ch=2, T] -> [ch*T, 32] channel-major (ch0 t0..T-1, ch1 t0..T-1)+ b, c, ch, t = latent.shape+ return latent[0].permute(1, 2, 0).reshape(ch * t, c)+++def unpack_audio(rows, ch=2):+ t = rows.shape[0] // ch+ return rows.reshape(ch, t, rows.shape[-1]).permute(2, 0, 1).unsqueeze(0)+++def _axis_from_sqrt_area(dim, patch, sqrt_area):+ # linspace((1 - ratio) / 2, (1 + ratio) / 2, dim // patch, endpoint=False) * 32+ ratio = dim / sqrt_area+ n = dim // patch+ return (torch.arange(n, dtype=torch.float64) * (ratio / n) + (1.0 - ratio) / 2.0) * 32.0+++def _frame_grid(h, w):+ # area-normalized (h, w) coordinates of one latent frame's 2x2-patch rows+ area = math.sqrt(h * w)+ hh, ww = torch.meshgrid(_axis_from_sqrt_area(h, 2, area), _axis_from_sqrt_area(w, 2, area), indexing="ij")+ return torch.stack([hh.reshape(-1), ww.reshape(-1)], dim=-1), _axis_from_sqrt_area(w, 2, area)+++def _video_t_spans(n):+ return [FRAME_RESCALE * FRAME_PER_TOKEN[k % 5] for k in range(n)]+++def _video_t_grid(n, origin):+ # origin + exclusive cumsum+ spans = torch.tensor(_video_t_spans(n), dtype=torch.float64)+ return float(origin) + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)])+++def _audio_grid(cursor, t, w_low, w_high):+ # channel-major stereo rows: t advances per latent frame, w pinned to the grid extremes per stereo channel, h stays 0+ g = torch.zeros(t * 2, 3, dtype=torch.float64)+ g[:, 0] = (cursor + torch.arange(t, dtype=torch.float64)).repeat(2)+ g[:t, 2] = w_low+ g[t:, 2] = w_high+ return g+++def _video_grid(vt, frame, cursor):+ g = torch.empty(vt, frame.shape[0], 3, dtype=torch.float64)+ g[:, :, 0] = _video_t_grid(vt, cursor)[:, None]+ g[:, :, 1:] = frame[None]+ return g.reshape(-1, 3)+++class TimeEmbedder(nn.Module):+ def __init__(self, freq_dim, hidden, out, dtype=None, device=None, operations=None):+ super().__init__()+ self.freq_dim = freq_dim+ self.proj_in = operations.Linear(freq_dim, hidden, bias=True, dtype=dtype, device=device)+ self.proj_out = operations.Linear(hidden, out, bias=True, dtype=dtype, device=device)++ def forward(self, t):+ # t: [M] in [0, 1]; fp32 throughout, cos before sin+ half = self.freq_dim // 2+ freqs = torch.exp(-math.log(10000.0) * torch.arange(half, dtype=torch.float32, device=t.device) / half)+ args = t.to(torch.float32)[:, None] * freqs[None]+ emb = torch.cat([torch.cos(args), torch.sin(args)], dim=-1)+ return self.proj_out(nn.functional.silu(self.proj_in(emb)))+++def rope_rotation_table(angles, dtype):+ """[S, rot_dim] pair angles -> [1, S, 1, rot_dim/2, 2, 2] rotation matrices."""+ half = angles.shape[-1] // 2+ ang = angles[:, :half] # duplicated halves: [:, :half] == [:, half:]+ c, s = torch.cos(ang), torch.sin(ang)+ table = torch.stack([c, -s, s, c], dim=-1).reshape(1, angles.shape[0], 1, half, 2, 2)+ return table.to(dtype)+++class Attention(nn.Module):+ def __init__(self, hidden, heads, head_dim, eps, dtype=None, device=None, operations=None):+ super().__init__()+ self.heads = heads+ self.head_dim = head_dim+ inner = heads * head_dim+ self.qkv_proj = operations.Linear(hidden, inner * 3, bias=False, dtype=dtype, device=device)+ self.q_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device)+ self.k_norm = operations.RMSNorm(head_dim, eps=eps, dtype=dtype, device=device)+ self.out_proj = operations.Linear(inner, hidden, bias=False, dtype=dtype, device=device)++ def forward(self, x, rope_freqs=None, transformer_options={}):+ s = x.shape[0]+ q, k, v = self.qkv_proj(x).split(self.heads * self.head_dim, dim=-1)+ v = v.view(s, self.heads, self.head_dim)+ if rope_freqs is not None:+ # fused per-head RMSNorm + partial split-half rope, in place on the qkv buffer+ q = q.view(1, s, self.heads, self.head_dim)+ k = k.view(1, s, self.heads, self.head_dim)+ qw = comfy.model_management.cast_to(self.q_norm.weight, device=x.device)+ kw = comfy.model_management.cast_to(self.k_norm.weight, device=x.device)+ rot = rope_freqs.shape[-3] * 2+ if comfy.model_management.in_training:+ q, k = comfy.quant_ops.ck.rms_rope_split_half(+ q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot)+ else:+ comfy.quant_ops.ck.rms_rope_split_half_(+ q, k, rope_freqs, qw, kw, epsilon=self.q_norm.eps, rot_dim=rot)+ q = q[0]+ k = k[0]+ else:+ q = self.q_norm(q.view(s, self.heads, self.head_dim))+ k = self.k_norm(k.view(s, self.heads, self.head_dim))+ q = q.transpose(0, 1).unsqueeze(0)+ k = k.transpose(0, 1).unsqueeze(0)+ v = v.transpose(0, 1).unsqueeze(0)+ out = optimized_attention(q, k, v, self.heads, mask=None, skip_reshape=True, transformer_options=transformer_options)+ return self.out_proj(out.squeeze(0))+++class MLP(nn.Module):+ def __init__(self, hidden, ffn, dtype=None, device=None, operations=None):+ super().__init__()+ self.fc1 = operations.Linear(hidden, ffn * 2, bias=False, dtype=dtype, device=device)+ self.fc2 = operations.Linear(ffn, hidden, bias=False, dtype=dtype, device=device)++ def forward(self, x):+ return comfy.ops.linear_input_act(self.fc2, self.fc1(x), "swiglu")+++class AdalnProj(nn.Module):+ def __init__(self, t_dim, hidden, expand, modalities, apply_silu=True,+ dtype=None, device=None, operations=None):+ super().__init__()+ self.expand = expand+ self.modalities = modalities+ self.hidden = hidden+ self.apply_silu = apply_silu+ self.linear = operations.Linear(t_dim, expand * hidden * modalities, bias=True, dtype=dtype, device=device)++ def forward(self, t_emb):+ # [M, t_dim] -> expand tensors of [M*modalities, hidden]+ x = self.linear(nn.functional.silu(t_emb) if self.apply_silu else t_emb)+ x = x.view(x.shape[0] * self.modalities, self.expand * self.hidden)+ return x.chunk(self.expand, dim=-1)+++def _mod_scale_shift(h, shift, scale, segments):+ # segments: [(start, stop, mod_row)] covering h contiguously.+ for a, b, row in segments:+ h[a:b].mul_(1.0 + scale[row].to(h.dtype)).add_(shift[row].to(h.dtype))+ return h+++def _mod_gate(x, gate, other, segments):+ # other is the fresh attn/mlp output: accumulate the gated residual into the stream in place, one fused kernel per segment+ for a, b, row in segments:+ x[a:b].addcmul_(other[a:b], gate[row].to(x.dtype))+ return x+++class RefinerBlock(nn.Module):+ def __init__(self, hidden, heads, head_dim, ffn, eps, qk_eps, dtype=None, device=None, operations=None):+ super().__init__()+ self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)+ self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)+ self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations)+ self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations)++ def forward(self, x, transformer_options={}):+ # attn/mlp outputs are fresh: accumulate residuals in place+ x = self.attn(self.norm1(x), transformer_options=transformer_options).add_(x)+ return self.mlp(self.norm2(x)).add_(x)+++class TokenRefiner(nn.Module):+ def __init__(self, num_layers, hidden, heads, head_dim, ffn, eps, qk_eps, final_eps,+ dtype=None, device=None, operations=None):+ super().__init__()+ self.blocks = nn.ModuleList([+ RefinerBlock(hidden, heads, head_dim, ffn, eps, qk_eps, dtype=dtype, device=device, operations=operations)+ for _ in range(num_layers)])+ self.final_norm = operations.RMSNorm(hidden, eps=final_eps, dtype=dtype, device=device)++ def forward(self, x, transformer_options={}):+ for block in self.blocks:+ x = block(x, transformer_options=transformer_options)+ return self.final_norm(x)+++class DiTBlock(nn.Module):+ def __init__(self, hidden, heads, head_dim, ffn, t_dim, eps, qk_eps,+ apply_silu=True, adaln_dtype=None, dtype=None, device=None, operations=None):+ super().__init__()+ self.norm1 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)+ self.norm2 = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)+ self.attn = Attention(hidden, heads, head_dim, qk_eps, dtype=dtype, device=device, operations=operations)+ self.mlp = MLP(hidden, ffn, dtype=dtype, device=device, operations=operations)+ self.adaln_proj = AdalnProj(t_dim, hidden, 6, 3, apply_silu=apply_silu,+ dtype=adaln_dtype if adaln_dtype is not None else dtype,+ device=device, operations=operations)++ def forward(self, x, t_emb, mod_segments, rope_freqs, transformer_options={}):+ shift_msa, scale_msa, gate_msa, shift_mlp, scale_mlp, gate_mlp = self.adaln_proj(t_emb)+ h = _mod_scale_shift(self.norm1(x), shift_msa, scale_msa, mod_segments)+ x = _mod_gate(x, gate_msa, self.attn(h, rope_freqs=rope_freqs, transformer_options=transformer_options), mod_segments)+ h = _mod_scale_shift(self.norm2(x), shift_mlp, scale_mlp, mod_segments)+ return _mod_gate(x, gate_mlp, self.mlp(h), mod_segments)+++class FinalLayer(nn.Module):+ def __init__(self, hidden, t_dim, video_dim, audio_dim, eps, apply_silu=True, adaln_dtype=None,+ dtype=None, device=None, operations=None):+ super().__init__()+ self.norm = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)+ self.adaln_proj = AdalnProj(t_dim, hidden, 2, 1, apply_silu=apply_silu,+ dtype=adaln_dtype if adaln_dtype is not None else dtype,+ device=device, operations=operations)+ # output heads are the checkpoint's fp32 island; norm/adaln are stored at model dtype+ self.video_out = operations.Linear(hidden, video_dim, bias=True, dtype=torch.float32, device=device)+ self.audio_out = operations.Linear(hidden, audio_dim, bias=True, dtype=torch.float32, device=device)++ def forward(self, x, t_emb, video_seg, audio_seg):+ # video_seg / audio_seg: (start, stop, timestep_row) of the target streams+ shift, scale = self.adaln_proj(t_emb)+ va, vb, vrow = video_seg+ aa, ab, arow = audio_seg+ hv = (self.norm(x[va:vb]) * (1.0 + scale[vrow]) + shift[vrow]).to(torch.float32)+ ha = (self.norm(x[aa:ab]) * (1.0 + scale[arow]) + shift[arow]).to(torch.float32)+ return self.video_out(hv), self.audio_out(ha)+++class PackedLayout:+ """Static packed-sequence structure for one shape/conditioning signature."""++ def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None, frame_count=None):+ frame, w_grid = _frame_grid(latent_h, latent_w)+ frame_rows = frame.shape[0]++ segments = [("text", text_len)] # (kind, n_rows)+ g = torch.zeros(text_len, 3, dtype=torch.float64)+ g[:, 0] = torch.arange(text_len, dtype=torch.float64)+ pos = [g] # per segment: [n, 3] float64 (t, h, w)++ img_pos, img_update = [], []+ audio_pos, audio_update = [], []+ cursor = text_len+ row = text_len++ if keyframes:+ # fl2va: keyframe cond rows right after text, sharing the target spatial grid+ for kf in keyframes:+ pixel_index = kf["resolved_frame_index"]+ if pixel_index == 0:+ cond_t = float(text_len)+ elif frame_count is not None and pixel_index == frame_count - 1:+ cond_t = float(text_len) + sum(_video_t_spans(latent_t)) - FRAME_RESCALE+ else:+ raise ValueError("only first/last keyframe anchors are supported")+ g = torch.empty(frame_rows, 3, dtype=torch.float64)+ g[:, 0] = cond_t+ g[:, 1:] = frame+ segments.append(("cond", frame_rows))+ pos.append(g)+ img_pos.append(torch.arange(row, row + frame_rows))+ img_update.append(torch.zeros(frame_rows, dtype=torch.bool))+ row += frame_rows++ target_audio_w = (float(w_grid[0]), float(w_grid[-1]))+ if refs:+ cursor = float(text_len)+ for blk in refs:+ kind = blk["kind"]+ if kind == "image":+ r_frame, _ = _frame_grid(blk["latent_h"], blk["latent_w"])+ n = r_frame.shape[0]+ g = torch.empty(n, 3, dtype=torch.float64)+ g[:, 0] = cursor+ g[:, 1:] = r_frame+ segments.append(("ref_img", n))+ pos.append(g)+ img_pos.append(torch.arange(row, row + n))+ img_update.append(torch.zeros(n, dtype=torch.bool))+ row += n+ cursor += 1.0+ elif kind == "audio":+ rt = blk["ref_audio_t"]+ if rt > 0:+ segments.append(("ref_audio", rt * 2))+ pos.append(_audio_grid(cursor, rt, *target_audio_w))+ audio_pos.append(torch.arange(row, row + rt * 2))+ audio_update.append(torch.zeros(rt * 2, dtype=torch.bool))+ row += rt * 2+ cursor += float(rt)+ elif kind in ("video", "video_audio"):+ # the block's audio rows pack immediately before its video+ # rows, both sharing the cursor origin+ rt = blk["ref_audio_t"]+ vt = blk["latent_t"]+ r_frame, r_w_grid = _frame_grid(blk["latent_h"], blk["latent_w"])+ if rt > 0:+ segments.append(("ref_audio", rt * 2))+ pos.append(_audio_grid(cursor, rt, float(r_w_grid[0]), float(r_w_grid[-1])))+ audio_pos.append(torch.arange(row, row + rt * 2))+ audio_update.append(torch.zeros(rt * 2, dtype=torch.bool))+ row += rt * 2+ n = vt * r_frame.shape[0]+ segments.append(("ref_img", n))+ pos.append(_video_grid(vt, r_frame, cursor))+ img_pos.append(torch.arange(row, row + n))+ img_update.append(torch.zeros(n, dtype=torch.bool))+ row += n+ cursor += max(float(rt), sum(_video_t_spans(vt)))++ # target audio then target video, always the last two segments+ segments.append(("audio", audio_t * 2))+ pos.append(_audio_grid(cursor, audio_t, *target_audio_w))+ audio_pos.append(torch.arange(row, row + audio_t * 2))+ audio_update.append(torch.ones(audio_t * 2, dtype=torch.bool))+ row += audio_t * 2++ n_video = latent_t * frame_rows+ segments.append(("video", n_video))+ pos.append(_video_grid(latent_t, frame, cursor))+ img_pos.append(torch.arange(row, row + n_video))+ img_update.append(torch.ones(n_video, dtype=torch.bool))+ row += n_video++ self.seq_len = row+ self.position_ids = torch.cat(pos) # [S, 3] float64+ self.img_pos = torch.cat(img_pos)+ self.img_update = torch.cat(img_update)+ self.audio_pos = torch.cat(audio_pos)+ self.audio_update = torch.cat(audio_update)+ self.signature = (text_len, latent_t, latent_h, latent_w, audio_t)+ # contiguous segment table (start, stop, kind)+ # kinds: text / cond / ref_img / ref_audio / audio / video+ # the packed sequence is uniform per segment in (modality tag, timestep class),+ # except the text span (tag runs resolved at forward time from the presentation tags)+ seg_abs = []+ off = 0+ for kind, n in segments:+ seg_abs.append((off, off + n, kind))+ off += n+ self.segments = seg_abs+++class MiniMaxH3Model(nn.Module):+ def __init__(self, hidden_size=5376, num_layers=50, token_refiner_num_layers=2,+ num_attention_heads=56, attention_head_dim=128, ffn_hidden_size=14336,+ latents_dim=24, audio_latents_dim=32, patch_size=(1, 2, 2), text_dim=5120,+ timestep_input_dim=256, time_embed_hidden_size=5376, time_embed_dim=2688,+ rope_inv_freq_len=16, norm_eps=1e-5, qk_norm_eps=1e-5, final_norm_eps=1e-5,+ sigma_shift_video=12.0, sigma_shift_audio=3.0,+ adaln_curve_grid=None,+ image_model=None, dtype=None, device=None, operations=None, **kwargs):+ super().__init__()+ self.dtype = dtype+ self.hidden_size = hidden_size+ self.patch_size = tuple(patch_size)+ self.latents_dim = latents_dim+ self.audio_latents_dim = audio_latents_dim+ self.sigma_shift_video = sigma_shift_video+ self.sigma_shift_audio = sigma_shift_audio+ self.use_adaln_curves = adaln_curve_grid is not None+ # curve-form checkpoints replace the time embedder and full-width adaln weights with a small shared basis of the time-embedding curve+ curve = {"apply_silu": not self.use_adaln_curves,+ "adaln_dtype": torch.float32 if self.use_adaln_curves else dtype}+ video_patch_dim = latents_dim * self.patch_size[0] * self.patch_size[1] * self.patch_size[2]++ self.video_patch_proj = operations.Linear(video_patch_dim, hidden_size, bias=True, dtype=torch.float32, device=device)+ self.audio_patch_proj = operations.Linear(audio_latents_dim, hidden_size, bias=True, dtype=torch.float32, device=device)+ self.condition_proj = operations.Linear(text_dim, hidden_size, bias=True, dtype=dtype, device=device)+ if self.use_adaln_curves:+ self.register_buffer("adaln_t_table", torch.empty(adaln_curve_grid, time_embed_dim, dtype=torch.float32))+ else:+ self.time_embedder = TimeEmbedder(timestep_input_dim, time_embed_hidden_size, time_embed_dim,+ dtype=torch.float32, device=device, operations=operations)+ self.rope = nn.Module()+ self.rope.register_buffer("inv_freq", torch.empty(rope_inv_freq_len, dtype=torch.float32))+ self.token_refiner = TokenRefiner(token_refiner_num_layers, hidden_size, num_attention_heads,+ attention_head_dim, ffn_hidden_size, norm_eps, qk_norm_eps,+ final_norm_eps, dtype=dtype, device=device, operations=operations)+ self.blocks = nn.ModuleList([+ DiTBlock(hidden_size, num_attention_heads, attention_head_dim, ffn_hidden_size,+ time_embed_dim, norm_eps, qk_norm_eps, **curve, dtype=dtype, device=device, operations=operations)+ for _ in range(num_layers)])+ self.final_layer = FinalLayer(hidden_size, time_embed_dim, video_patch_dim, audio_latents_dim,+ final_norm_eps, **curve, dtype=dtype, device=device, operations=operations)++ def preprocess_text_embeds(self, text_states):+ """[B, L, text_dim] Qwen states -> [B, L, hidden] refined text embeds."""+ if text_states.shape[-1] == self.hidden_size:+ return text_states+ return self.token_refiner(self.condition_proj(text_states[0])).unsqueeze(0)++ def rope_freqs(self, position_ids, device):+ # [S, 3] float64 -> [S, 96] fp32+ pos = position_ids.to(torch.float32).to(device)+ inv = comfy.model_management.cast_to(self.rope.inv_freq, device=device)+ per_axis = pos.unsqueeze(-1) * inv.view(1, 1, -1) # [S, 3, 16]+ t_f, h_f, w_f = per_axis.unbind(dim=1)+ half = torch.cat((t_f, h_f, w_f), dim=-1) # [S, 48]+ return torch.cat((half, half), dim=-1) # [S, 96]++ def _cond_video_rows(self, payload, device):+ """Concatenated visual condition rows (normalized latents -> patchified), with condition noise augmentation."""+ rows = []+ aug = payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP)+ seed = int(payload.get("seed", 0))+ # every condition intentionally restarts the same RNG stream+ for z in payload.get("cond_video_latents", []):+ r = patchify_video(z.to(torch.float32), self.patch_size)+ if aug < 1.0:+ gen = torch.Generator("cpu").manual_seed(seed)+ noise = torch.randn(r.shape, generator=gen, dtype=torch.float32)+ r = aug * r + (1.0 - aug) * noise.to(r.device)+ rows.append(r.to(device))+ return torch.cat(rows, dim=0) if rows else None++ def _cond_audio_rows(self, payload, device):+ rows = []+ aug = payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP)+ seed = int(payload.get("seed", 0)) + 1+ for z in payload.get("cond_audio_latents", []):+ r = pack_audio(z.to(torch.float32))+ if aug < 1.0:+ gen = torch.Generator("cpu").manual_seed(seed)+ noise = torch.randn(r.shape, generator=gen, dtype=torch.float32)+ r = aug * r + (1.0 - aug) * noise.to(r.device)+ rows.append(r.to(device))+ return torch.cat(rows, dim=0) if rows else None++ def forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs):+ return comfy.patcher_extension.WrapperExecutor.new_class_executor(+ self._forward,+ self,+ comfy.patcher_extension.get_all_wrappers(comfy.patcher_extension.WrappersMP.DIFFUSION_MODEL, transformer_options)+ ).execute(x, timestep, context, transformer_options, minimax_payload=minimax_payload, **kwargs)++ def _forward(self, x, timestep, context, transformer_options={}, minimax_payload=None, **kwargs):+ video_x, audio_x = x[0], x[1]+ orig_t, orig_h, orig_w = video_x.shape[2], video_x.shape[3], video_x.shape[4]+ video_x = comfy.ldm.common_dit.pad_to_patch_size(video_x, self.patch_size)+ if video_x.shape[0] != 1:+ raise ValueError("MiniMax H3 supports batch size 1")+ payload = minimax_payload or {}+ device = video_x.device+ dtype = context.dtype # compute dtype++ latent_t, lat_h, lat_w = video_x.shape[2], video_x.shape[3], video_x.shape[4]+ audio_t = audio_x.shape[-1]+ text_len = context.shape[1]+ # extra_conds prebuilds the layout once per sampling run+ layout = payload.get("layout")+ if layout is None or layout.signature != (text_len, latent_t, lat_h, lat_w, audio_t):+ layout = PackedLayout(text_len, latent_t, lat_h, lat_w, audio_t,+ keyframes=payload.get("keyframes"),+ refs=payload.get("refs"),+ frame_count=payload.get("frame_count"))++ # model_base passes model_sampling.timestep(sigma) = sigma * 1000+ shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video))+ shift_a = float(transformer_options.get("minimax_h3_sigma_shift_audio", self.sigma_shift_audio))+ sigma_v = (timestep.flatten()[0] / 1000.0).float().clamp(min=1e-6)+ t_v = float(1.0 - sigma_v)+ t_a = float(1.0 - time_shift_sigma(sigma_v, shift_v, shift_a))++ # distinct timesteps are known analytically: text/pad follow video, cond rows pin near 1+ vis_aug = float(payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP))+ aud_aug = float(payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP))+ has_vis_cond = any(k in ("cond", "ref_img") for _, _, k in layout.segments)+ has_aud_cond = any(k == "ref_audio" for _, _, k in layout.segments)+ seg_t = {"text": t_v, "video": t_v, "audio": t_a,+ "cond": max(t_v, vis_aug), "ref_img": max(t_v, vis_aug),+ "ref_audio": max(t_a, aud_aug)}+ unique_t = sorted({t_v, t_a} | ({seg_t["cond"]} if has_vis_cond else set())+ | ({seg_t["ref_audio"]} if has_aud_cond else set()))+ t_row = {t: i for i, t in enumerate(unique_t)}+ seg_tag = {"text": 1, "video": 0, "audio": 2, "cond": 0, "ref_img": 0, "ref_audio": 2}++ text_tags = payload.get("text_token_tags")+ mod_segments = []+ for a, b, kind in layout.segments:+ row_base = t_row[seg_t[kind]] * 3+ if kind == "text" and text_tags is not None:+ # the presentation text span mixes tags (vision pads carry the video modality) split into tag runs+ tags = text_tags.view(-1).tolist()+ run_start = 0+ for i in range(1, b - a + 1):+ if i == b - a or tags[i] != tags[run_start]:+ mod_segments.append((a + run_start, a + i, row_base + int(tags[run_start])))+ run_start = i+ else:+ mod_segments.append((a, b, row_base + seg_tag[kind]))++ # embed+ img_update = layout.img_update.to(device)+ audio_update = layout.audio_update.to(device)+ video_rows = patchify_video(video_x.to(torch.float32), self.patch_size)+ audio_rows = pack_audio(audio_x.to(torch.float32))+ cond_video_rows = self._cond_video_rows(payload, device)+ cond_audio_rows = self._cond_audio_rows(payload, device)++ all_video_rows = video_rows+ if cond_video_rows is not None:+ all_video_rows = torch.empty(img_update.shape[0], video_rows.shape[1], dtype=torch.float32, device=device)+ all_video_rows[~img_update] = cond_video_rows+ all_video_rows[img_update] = video_rows+ all_audio_rows = audio_rows+ if cond_audio_rows is not None:+ all_audio_rows = torch.empty(audio_update.shape[0], audio_rows.shape[1], dtype=torch.float32, device=device)+ all_audio_rows[~audio_update] = cond_audio_rows+ all_audio_rows[audio_update] = audio_rows++ video_embed = self.video_patch_proj(all_video_rows).to(dtype)+ audio_embed = self.audio_patch_proj(all_audio_rows).to(dtype)+ text_states = context[0]+ if text_states.shape[-1] != self.hidden_size:+ text_states = self.token_refiner(self.condition_proj(text_states),+ transformer_options=transformer_options)++ # segments are contiguous: assemble by slices, embed rows follow segment order+ h = torch.empty(layout.seq_len, self.hidden_size, dtype=dtype, device=device)+ voff = aoff = 0+ for a, b, kind in layout.segments:+ n = b - a+ if kind == "text":+ h[a:b] = text_states+ elif kind in ("cond", "ref_img", "video"):+ h[a:b] = video_embed[voff:voff + n]+ voff += n+ else: # ref_audio / audio+ h[a:b] = audio_embed[aoff:aoff + n]+ aoff += n++ t_vals = torch.tensor(unique_t, dtype=torch.float32, device=device)+ if self.use_adaln_curves:+ # adaln projections consume interpolated coordinates of the time-embedding curve+ table = comfy.model_management.cast_to(self.adaln_t_table, device=device)+ pos = t_vals.clamp(0.0, 1.0) * (table.shape[0] - 1) # t in [0,1] -> fractional grid index, out-of-range t clamps to the curve ends+ i0 = pos.floor().long().clamp(max=table.shape[0] - 2) # lower grid row, max-clamp keeps t=1.0 on the last interval instead of reading past the table+ t_emb = torch.lerp(table[i0], table[i0 + 1], (pos - i0).unsqueeze(1)) # blend the two rows by the fractional part+ else:+ t_emb = self.time_embedder(t_vals).to(dtype)++ # rotation table computed once per forward, consumed by the kitchen split-half rope+ rope_freqs = rope_rotation_table(self.rope_freqs(layout.position_ids, device), dtype)++ # blocks+ patches_replace = transformer_options.get("patches_replace", {})+ blocks_replace = patches_replace.get("dit", {})+ prefetch_queue = comfy.model_prefetch.make_prefetch_queue(list(self.blocks), device, transformer_options)+ for i, block in enumerate(self.blocks):+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, block)+ if ("double_block", i) in blocks_replace:+ def block_wrap(args):+ return {"img": block(args["img"], args["t_emb"], args["mod_segments"], args["rope_freqs"],+ transformer_options=args["transformer_options"])}+ h = blocks_replace[("double_block", i)](+ {"img": h, "t_emb": t_emb, "mod_segments": mod_segments, "rope_freqs": rope_freqs,+ "transformer_options": transformer_options},+ {"original_block": block_wrap})["img"]+ else:+ h = block(h, t_emb, mod_segments, rope_freqs, transformer_options=transformer_options)+ if prefetch_queue is not None:+ comfy.model_prefetch.prefetch_queue_pop(prefetch_queue, device, None)++ # target streams are single contiguous segments (audio then video, last two)+ video_seg = next((a, b, t_row[seg_t["video"]]) for a, b, k in layout.segments if k == "video")+ audio_seg = next((a, b, t_row[seg_t["audio"]]) for a, b, k in layout.segments if k == "audio")+ v, a = self.final_layer(h, t_emb, video_seg, audio_seg)++ video_out = unpatchify_video(v, latent_t, lat_h // 2, lat_w // 2, self.latents_dim, self.patch_size)+ video_out = video_out[:, :, :orig_t, :orig_h, :orig_w]+ audio_out = unpack_audio(a)++ # The sampler integrates the flat ODE dX/dsigma_v = (X - denoised)/sigma_v.+ # Scaling the audio velocity by d(sigma_a)/d(sigma_v) makes that ODE equal+ # to the audio stream's true ODE on its own shifted schedule.+ slope_a = time_shift_slope(sigma_v, shift_v, shift_a).to(audio_out.dtype)+ return [-video_out.to(video_x.dtype), (-slope_a) * audio_out.to(audio_x.dtype)]diff --git a/comfy/ldm/minimax/vae.py b/comfy/ldm/minimax/vae.pynew file mode 100644index 000000000..aeb3421a2--- /dev/null+++ b/comfy/ldm/minimax/vae.py@@ -0,0 +1,694 @@+# MiniMax H3 video VAE: 3D causal CNN encoder + ViT3D decoder.++import math++import torch+import torch.nn as nn+import torch.nn.functional as F++import comfy.ops+import comfy.quant_ops+import comfy.rmsnorm+from comfy.ldm.modules.attention import optimized_attention++ops = comfy.ops.disable_weight_init++IMAGENET_MEAN = (0.485, 0.456, 0.406)+IMAGENET_STD = (0.229, 0.224, 0.225)++LATENTS_MEAN = [+ 0.858090341091156, -0.9606591463088989, 1.0661640167236328, -0.5090325474739075,+ -0.2727581858634949, -1.3675414323806763, -0.2553254961967468, -0.26907554268836975,+ -0.5376840829849243, -0.0464097298681736, 0.6657370328903198, 0.19690127670764923,+ -0.5460608005523682, -0.4035342037677765, -0.23683024942874908, 0.25928452610969543,+ -0.30133944749832153, 0.211341992020607, -1.1206848621368408, 0.3581933379173279,+ -0.04225143790245056, 0.2604829967021942, 0.22864092886447906, 0.7056031823158264,+]++LATENTS_STD = [+ 1.2223774194717407, 1.2767263650894165, 1.68317747116088865, 1.7549455165863037,+ 1.5636216402053833, 2.194143533706665, 0.96531379222869875, 1.05698859691619875,+ 0.841948926448822, 0.7729952931404114, 1.8955937623977661, 0.946841835975647,+ 0.7996809482574463, 0.44988900423049925, 0.7197399735450745, 0.69362932443618775,+ 2.961095094680786, 2.7694199085235595, 3.0496184825897215, 2.1088054180145265,+ 3.276226282119751, 3.1627357006073, 2.28168129920959475, 2.6127843856811525,+]+++# 3D causal CNN encoder++class CausalConv3d(ops.Conv3d):+ # Reflect spatial padding, causal (zeros, front-only) temporal padding.+ def __init__(self, in_channels, out_channels, kernel_size, stride=1, padding=0):+ super().__init__(in_channels, out_channels, kernel_size=kernel_size, stride=stride)+ self.causal_padding = (padding,) * 3 if isinstance(padding, int) else tuple(padding)++ def forward(self, x):+ if sum(self.causal_padding) == 0:+ return super().forward(x)++ x = F.pad(x, (self.causal_padding[2], self.causal_padding[2], self.causal_padding[1], self.causal_padding[1], 0, 0), mode="reflect")+ if x.shape[2] == 1:+ # single frame: the causal front padding is all zeros truncate the temporal taps instead of convolving zero frames+ return super().forward(x, autopad="causal_zero")+ x = F.pad(x, (0, 0, 0, 0, self.causal_padding[0] * 2, 0), mode="constant")+ return super().forward(x)+++class TemporalIsolatedGroupNorm(ops.GroupNorm):+ # GroupNorm with statistics computed per frame (time merged into batch).+ def forward(self, x):+ if x.dim() == 5:+ b, c, t, h, w = x.shape+ x = x.permute(0, 2, 1, 3, 4).contiguous().view(b * t, c, 1, h, w)+ x = super().forward(x)+ return x.view(b, t, c, h, w).permute(0, 2, 1, 3, 4).contiguous()+ return super().forward(x)+++def group_norm_3d(num_channels):+ return TemporalIsolatedGroupNorm(num_groups=32, num_channels=num_channels, eps=1e-6, affine=True)+++class Downsample3D(nn.Module):+ def __init__(self, in_channels, out_channels, time_stride=1, space_stride=2):+ super().__init__()+ self.space_stride = space_stride+ self.conv = CausalConv3d(+ in_channels,+ out_channels,+ kernel_size=3,+ padding=(1, 0, 0),+ stride=(time_stride, space_stride, space_stride),+ )++ def forward(self, x):+ if self.space_stride == 2:+ x = F.pad(x, (0, 1, 0, 1, 0, 0), mode="reflect")+ return self.conv(x)+++class ResnetBlock3D(nn.Module):+ def __init__(self, in_channels, out_channels=None):+ super().__init__()+ self.in_channels = in_channels+ out_channels = in_channels if out_channels is None else out_channels+ self.out_channels = out_channels++ self.norm1 = group_norm_3d(in_channels)+ self.norm2 = group_norm_3d(out_channels)+ self.conv1 = CausalConv3d(in_channels, out_channels, kernel_size=3, padding=1)+ self.conv2 = CausalConv3d(out_channels, out_channels, kernel_size=3, padding=1)+ if in_channels != out_channels:+ self.nin_shortcut = CausalConv3d(in_channels, out_channels, kernel_size=1)++ def forward(self, x):+ h = self.conv1(F.silu(self.norm1(x), inplace=True))+ h = self.conv2(F.silu(self.norm2(h), inplace=True))+ if self.in_channels != self.out_channels:+ x = self.nin_shortcut(x)+ return h.add_(x)+++class EncoderFCN3D(nn.Module):+ def __init__(self, ch, ch_mult, space_down, time_down, num_res_blocks, in_channels, z_channels, double_z=True):+ super().__init__()+ self.num_levels = len(ch_mult)+ if isinstance(num_res_blocks, int):+ num_res_blocks = [num_res_blocks] * self.num_levels+ self.num_res_blocks = num_res_blocks++ block_mid = [ch * ch_mult[i] for i in range(self.num_levels)]+ block_in = [block_mid[0]] + block_mid[:-1]+ block_out = block_mid++ self.conv_in = CausalConv3d(in_channels, block_in[0], kernel_size=3, padding=1)++ self.down = nn.ModuleList()+ for i_level in range(self.num_levels):+ down = nn.Module()+ down.block = nn.ModuleList()+ for i in range(self.num_res_blocks[i_level]):+ down.block.append(+ ResnetBlock3D(+ in_channels=block_in[i_level] if i == 0 else block_mid[i_level],+ out_channels=block_mid[i_level],+ )+ )+ if space_down[i_level] * time_down[i_level] > 1:+ down.downsample = Downsample3D(+ block_mid[i_level],+ block_out[i_level],+ time_stride=time_down[i_level],+ space_stride=space_down[i_level],+ )+ self.down.append(down)++ self.norm_out = group_norm_3d(block_out[-1])+ self.conv_out = CausalConv3d(+ block_out[-1],+ 2 * z_channels if double_z else z_channels,+ kernel_size=3,+ padding=1,+ )++ def forward(self, x):+ h = self.conv_in(x)+ for i_level in range(self.num_levels):+ for i_block in range(self.num_res_blocks[i_level]):+ h = self.down[i_level].block[i_block](h)+ if hasattr(self.down[i_level], "downsample"):+ h = self.down[i_level].downsample(h)+ h = F.silu(self.norm_out(h))+ return self.conv_out(h)+++# ViT3D decoder++def create_token_ids(patch_dims, device, dtype):+ coords_list = []+ for dim_size in patch_dims:+ coords = torch.arange(0.5, dim_size, dtype=dtype, device=device)+ coords = coords / dim_size+ coords = 2.0 * coords - 1.0+ coords_list.append(coords)+ coords = torch.stack(torch.meshgrid(*coords_list, indexing="ij"), dim=-1)+ return coords.flatten(0, len(patch_dims) - 1).unsqueeze(0)+++class RotaryEmbeddingND(nn.Module):+ def __init__(self, dim, rotary_base=100.0, n_dim=3):+ super().__init__()+ self.n_dim = n_dim+ self.angle_scale = 2.0 * math.pi+ inv_freq = 1 / rotary_base ** torch.arange(0, 1, 2 * n_dim / dim, dtype=torch.float32)+ self.register_buffer("inv_freq", inv_freq, persistent=False)++ def forward(self, img_ids):+ # [B, S, n_dim] -> [B, S, 1, pairs, 2, 2] rotation table for the kitchen split-half rope+ angles = (+ self.angle_scale+ * img_ids[:, :, :, None].float()+ * self.inv_freq.to(img_ids.device)[None, None, None, :]+ )+ angles = angles.flatten(2, 3)+ c, s = torch.cos(angles), torch.sin(angles)+ table = torch.stack([c, -s, s, c], dim=-1).reshape(*angles.shape[:2], 1, angles.shape[-1], 2, 2)+ return table.to(img_ids.dtype)+++class FeedForward(nn.Module):+ # Gated SiLU FFN.+ def __init__(self, dim, mult=4, bias=True):+ super().__init__()+ inner_dim = dim * mult+ self.w1 = ops.Linear(dim, inner_dim * 2, bias=bias)+ self.w2 = ops.Linear(inner_dim, dim, bias=bias)++ def forward(self, x):+ gate, x = self.w1(x).chunk(2, dim=-1)+ return self.w2(F.silu(gate).mul_(x))+++class Attention(nn.Module):+ def __init__(self, heads, dim_head, bias=True, eps=1e-5):+ super().__init__()+ self.dim_head = dim_head+ self.heads = heads+ inner_dim = dim_head * heads+ self.norm_q = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False)+ self.norm_k = ops.RMSNorm(dim_head, eps=eps, elementwise_affine=False)+ self.to_qkv = ops.Linear(inner_dim, inner_dim * 3, bias=bias)+ self.to_out = ops.Linear(inner_dim, inner_dim, bias=bias)++ def forward(self, x, rotary_pos_emb=None):+ batch_size, seq_len, _ = x.shape++ qkv = self.to_qkv(x)+ qkv = qkv.view(batch_size, seq_len, -1, 3 * self.dim_head)+ query, key, value = torch.chunk(qkv, 3, dim=-1)++ query = comfy.rmsnorm.rms_norm(query, self.norm_q.weight, self.norm_q.eps)+ key = comfy.rmsnorm.rms_norm(key, self.norm_k.weight, self.norm_k.eps)++ if rotary_pos_emb is not None:+ rot = rotary_pos_emb.shape[-3] * 2+ query[..., :rot], key[..., :rot] = comfy.quant_ops.ck.apply_rope_split_half(+ query[..., :rot], key[..., :rot], rotary_pos_emb)++ out = optimized_attention(query.transpose(1, 2), key.transpose(1, 2), value.transpose(1, 2),+ self.heads, skip_reshape=True).nan_to_num_(0.0)+ return self.to_out(out)+++class TransformerBlock(nn.Module):+ def __init__(self, heads, dim_head, bias=True, eps=1e-5):+ super().__init__()+ dim = heads * dim_head+ self.norm1 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps)+ self.attn = Attention(heads=heads, dim_head=dim_head, bias=bias, eps=eps)+ self.scale1 = nn.Parameter(torch.empty(dim))+ self.norm2 = ops.RMSNorm(dim, elementwise_affine=True, eps=eps)+ self.ff = FeedForward(dim=dim, bias=bias)+ self.scale2 = nn.Parameter(torch.empty(dim))++ def forward(self, x, rotary_pos_emb=None):+ x = x.addcmul_(self.attn(comfy.rmsnorm.rms_norm(x, self.norm1.weight, self.norm1.eps), rotary_pos_emb), self.scale1)+ return x.addcmul_(self.ff(comfy.rmsnorm.rms_norm(x, self.norm2.weight, self.norm2.eps)), self.scale2)+++class ViT3DDecoder(nn.Module):+ def __init__(self, patch_size=16, patch_size_t=4, in_channels=24, out_channels=3, num_layers=36, heads=32, dim_head=64, rope_theta=100.0,+ rope_dim_ratio=0.75, bias=True, eps=1e-5, num_register_tokens=4):+ super().__init__()+ dim = heads * dim_head+ self.patch_size = patch_size+ self.patch_size_t = patch_size_t+ self.out_channels = out_channels+ self.num_register_tokens = num_register_tokens++ self.pos_embed = RotaryEmbeddingND(int(dim_head * rope_dim_ratio), rope_theta, n_dim=3)+ self.x_embedder = ops.Linear(in_channels, dim)+ self.register_tokens = nn.Parameter(torch.empty(1, num_register_tokens, dim))+ # unused at inference; kept so the checkpoint loads without leftover keys+ self.register_buffer("mask_token", torch.empty(1, 1, dim))++ self.transformer_blocks = nn.ModuleList(+ [TransformerBlock(heads=heads, dim_head=dim_head, bias=bias, eps=eps)+ for _ in range(num_layers)]+ )++ self.norm_out = ops.LayerNorm(dim, elementwise_affine=True, eps=eps)+ self.proj_out = ops.Linear(dim, out_channels * patch_size_t * patch_size * patch_size)++ def forward(self, x):+ B, C, latent_T, latent_H, latent_W = x.shape++ h = self.x_embedder(x.flatten(2).transpose(1, 2)) # [B, T*H*W, C]++ num_patches = h.shape[1]+ num_suffix = 1 + self.num_register_tokens++ h = torch.cat([h, self.register_tokens.expand(B, -1, -1), torch.zeros_like(h[:, 0:1, :])], dim=1)++ img_ids = create_token_ids((latent_T, latent_H, latent_W), x.device, x.dtype).expand(B, -1, -1)+ suffix_ids = torch.zeros((B, num_suffix, 3), device=x.device, dtype=img_ids.dtype)+ img_ids = torch.cat([img_ids, suffix_ids], dim=1)++ rotary_pos_emb = self.pos_embed(img_ids)++ for block in self.transformer_blocks:+ h = block(h, rotary_pos_emb)++ output = self.proj_out(self.norm_out(h))++ output = output[:, :num_patches, :]++ output = output.view(+ B, latent_T, latent_H, latent_W,+ self.out_channels, self.patch_size_t, self.patch_size, self.patch_size,+ )+ output = output.permute(0, 4, 1, 5, 2, 6, 3, 7).contiguous()+ output = output.reshape(+ B, self.out_channels,+ latent_T * self.patch_size_t,+ latent_H * self.patch_size,+ latent_W * self.patch_size,+ )+ return output+++# Full VAE++class MiniMaxH3VideoVAE(nn.Module):+ def __init__(+ self,+ in_channels=3,+ out_ch=3,+ ch=128,+ embed_dim=24,+ z_channels=24,+ ch_mult=(1, 2, 2, 4, 4, 8),+ num_res_blocks=2,+ space_down=(2, 2, 2, 2, 1, 1),+ time_down=(1, 2, 2, 1, 1, 1),+ clip_length=17,+ token_drop=3,+ tile_size=256,+ tile_overlap_min=64,+ tiling=True,+ ):+ super().__init__()+ self.vae_ratio = int(math.prod(space_down))+ self.vae_ratio_t = int(math.prod(time_down))++ # temporal chunking parameters+ self.clip_length = clip_length+ self.token_drop = token_drop+ self.frame_pre_padding = (-clip_length) % self.vae_ratio_t+ self.tokens_chunk_size = math.ceil(clip_length / self.vae_ratio_t)+ self.token_overlap = (-token_drop) % self.tokens_chunk_size+ self.frame_overlap = max(self.token_overlap * self.vae_ratio_t - self.frame_pre_padding, 0)++ # spatial tiling parameters+ self.tiling = tiling+ self.tile_size = tile_size+ self.tile_overlap_min = tile_overlap_min++ self.encoder = EncoderFCN3D(+ ch=ch,+ ch_mult=list(ch_mult),+ space_down=list(space_down),+ time_down=list(time_down),+ num_res_blocks=num_res_blocks,+ in_channels=in_channels,+ z_channels=z_channels,+ double_z=True,+ )+ self.quant_conv = ops.Conv3d(z_channels * 2, 2 * embed_dim, 1)+ self.post_quant_conv = ops.Conv3d(embed_dim, z_channels, 1)+ self.decoder = ViT3DDecoder(+ patch_size=self.vae_ratio,+ patch_size_t=self.vae_ratio_t,+ in_channels=z_channels,+ out_channels=out_ch,+ )++ self.register_buffer("latents_mean", torch.tensor(LATENTS_MEAN))+ self.register_buffer("latents_std", torch.tensor(LATENTS_STD))+ self.register_buffer("pixel_mean", torch.tensor(IMAGENET_MEAN).view(1, 3, 1, 1, 1), persistent=False)+ self.register_buffer("pixel_std", torch.tensor(IMAGENET_STD).view(1, 3, 1, 1, 1), persistent=False)++ # single-shot forward++ def _encode_moments(self, x):+ return self.quant_conv(self.encoder(x))++ def _decode_pixels(self, z):+ return self.decoder(self.post_quant_conv(z))++ def _adaptive_encode(self, x):+ if self.tiling:+ return self.tiled_encode(x)+ return self._encode_moments(x)++ def _adaptive_decode(self, z):+ if self.tiling:+ return self.tiled_decode(z)+ return self._decode_pixels(z)++ # spatial tiling++ def split_tiles(self, input_len):+ tile_size = self.tile_size+ if tile_size >= input_len:+ return [0], [input_len], []++ N = math.ceil(input_len / tile_size)+ while True:+ overlaps = [self.tile_overlap_min] * (N - 1)+ remaining = tile_size * N - sum(overlaps) - input_len+ if remaining < 0:+ N += 1+ else:+ break++ remaining_units = remaining // self.vae_ratio+ for i in range(remaining_units):+ overlaps[i % (N - 1)] += self.vae_ratio++ tile_start_idx = [0]+ for i in range(N - 1):+ tile_start_idx.append(tile_start_idx[-1] + tile_size - overlaps[i])++ return tile_start_idx, [tile_size] * N, overlaps++ def blend(self, a, b, blend_extent, dim):+ blend_extent = min(a.shape[dim], b.shape[dim], blend_extent)++ positions = torch.arange(blend_extent, device=b.device, dtype=b.dtype)+ weight_a = 1 - positions / blend_extent+ weight_b = positions / blend_extent++ shape = [1] * a.ndim+ shape[dim] = blend_extent+ weight_a = weight_a.view(shape)+ weight_b = weight_b.view(shape)++ slice_a = [slice(None)] * a.ndim+ slice_a[dim] = slice(-blend_extent, None)+ slice_b = [slice(None)] * b.ndim+ slice_b[dim] = slice(0, blend_extent)++ blended = a[tuple(slice_a)] * weight_a + b[tuple(slice_b)] * weight_b++ if blend_extent < b.shape[dim]:+ slice_b_rest = [slice(None)] * b.ndim+ slice_b_rest[dim] = slice(blend_extent, None)+ return torch.cat([blended, b[tuple(slice_b_rest)]], dim=dim)+ return blended++ def tiled_encode(self, x):+ height, width = x.shape[-2], x.shape[-1]+ y_idx, y_len, y_overlap = self.split_tiles(height)+ x_idx, x_len, x_overlap = self.split_tiles(width)++ rows = []+ for i_pos, i_len in zip(y_idx, y_len):+ row = []+ for j_pos, j_len in zip(x_idx, x_len):+ tile = x[..., i_pos:i_pos + i_len, j_pos:j_pos + j_len]+ row.append(self._encode_moments(tile))+ rows.append(row)++ latent_y_overlap = [o // self.vae_ratio for o in y_overlap]+ latent_x_overlap = [o // self.vae_ratio for o in x_overlap]++ result_rows = []+ for i, row in enumerate(rows):+ result_row = []+ for j, tile in enumerate(row):+ if i > 0:+ tile = self.blend(rows[i - 1][j], tile, latent_y_overlap[i - 1], dim=-2)+ if j > 0:+ tile = self.blend(row[j - 1], tile, latent_x_overlap[j - 1], dim=-1)+ if i < len(rows) - 1:+ tile = tile[..., :-latent_y_overlap[i], :]+ if j < len(row) - 1:+ tile = tile[..., :, :-latent_x_overlap[j]]+ result_row.append(tile)+ result_rows.append(torch.cat(result_row, dim=-1))+ return torch.cat(result_rows, dim=-2)++ def tiled_decode(self, z):+ height, width = z.shape[-2] * self.vae_ratio, z.shape[-1] * self.vae_ratio+ y_idx, y_len, y_overlap = self.split_tiles(height)+ x_idx, x_len, x_overlap = self.split_tiles(width)++ # Blended tiles are written straight into a pre-allocated canvas.+ canvas = None+ row_tails = []+ out_y = 0+ for i, (i_pos, i_len) in enumerate(zip(y_idx, y_len)):+ zi, zl = i_pos // self.vae_ratio, i_len // self.vae_ratio+ new_tails = []+ left_tail = None+ out_x = 0+ for j, (j_pos, j_len) in enumerate(zip(x_idx, x_len)):+ zj, zw = j_pos // self.vae_ratio, j_len // self.vae_ratio+ tile = self._decode_pixels(z[..., zi:zi + zl, zj:zj + zw])+ if i < len(y_idx) - 1:+ new_tails.append(tile[..., -y_overlap[i]:, :].clone())+ next_left_tail = tile[..., :, -x_overlap[j]:].clone() if j < len(x_idx) - 1 else None+ if i > 0:+ tile = self.blend(row_tails[j], tile, y_overlap[i - 1], dim=-2)+ if j > 0:+ tile = self.blend(left_tail, tile, x_overlap[j - 1], dim=-1)+ left_tail = next_left_tail+ if i < len(y_idx) - 1:+ tile = tile[..., :-y_overlap[i], :]+ if j < len(x_idx) - 1:+ tile = tile[..., :, :-x_overlap[j]]+ if canvas is None:+ canvas = torch.empty(*tile.shape[:-2], height, width, dtype=tile.dtype, device=tile.device)+ canvas[..., out_y:out_y + tile.shape[-2], out_x:out_x + tile.shape[-1]].copy_(tile)+ out_x += tile.shape[-1]+ row_tails = new_tails+ out_y += tile.shape[-2]+ return canvas++ # temporal chunking++ def encode_temporal(self, x):+ if x.shape[2] % self.clip_length != 0:+ pad_size = (-x.shape[2]) % self.clip_length+ pad_frames = x[:, :, -1:].repeat(1, 1, pad_size, 1, 1)+ x = torch.cat([x, pad_frames], dim=2)++ num_chunks = x.shape[2] // self.clip_length++ z_list = []+ for i in range(num_chunks):+ clip_x = x[:, :, i * self.clip_length:(i + 1) * self.clip_length, :, :]+ z_list.append(self._adaptive_encode(clip_x))++ z = torch.cat(z_list, dim=2)+ if self.token_drop > 0:+ z = z[:, :, :-self.token_drop]+ return z++ def _decode_temporal_pad_frames(self, z_len, pad_tokens):+ if pad_tokens <= 0:+ return 0+ intra_tail = self.clip_length % self.vae_ratio_t+ if intra_tail == 0:+ return pad_tokens * self.vae_ratio_t++ z_len_before_pad = z_len - pad_tokens+ return sum(+ (intra_tail if (z_len_before_pad + k) % self.tokens_chunk_size == 0+ else self.vae_ratio_t)+ for k in range(pad_tokens)+ )++ def _decode_temporal_frame_plan(self, z_len, num_chunks, pad_tokens):+ chunk_dec = self.tokens_chunk_size * self.vae_ratio_t+ split_count = int(self.token_drop > 0) + 1+ total_frames = 0+ final_overlap_frames = 0++ for i in range(num_chunks):+ t_start_idx = i * self.tokens_chunk_size+ t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap+ clip_token_len = max(0, min(t_end_idx, z_len) - min(t_start_idx, z_len))+ clip_frame_len = clip_token_len * self.vae_ratio_t++ for j in range(split_count):+ f_start_idx = j * chunk_dec+ f_end_idx = min(f_start_idx + chunk_dec, clip_frame_len)+ chunk_frames = max(0, f_end_idx - f_start_idx - self.frame_pre_padding)+ if j == 0:+ total_frames += chunk_frames+ else:+ final_overlap_frames = chunk_frames++ total_frames += final_overlap_frames+ return total_frames - self._decode_temporal_pad_frames(z_len, pad_tokens)++ def decode_temporal(self, z):+ chunk_dec = self.tokens_chunk_size * self.vae_ratio_t+ split_count = int(self.token_drop > 0) + 1++ pseudo_total_tokens = z.shape[2] + self.token_drop++ pad_tokens = 0+ remainder = pseudo_total_tokens % self.tokens_chunk_size+ if remainder != 0:+ pad_tokens = self.tokens_chunk_size - remainder+ pseudo_total_tokens += pad_tokens++ num_chunks = pseudo_total_tokens // self.tokens_chunk_size - int(self.token_drop > 0)+ if num_chunks < 1:+ # too few tokens for one chunk (e.g. T_lat == 2): pad one extra chunk+ pad_tokens += self.tokens_chunk_size+ num_chunks += 1++ if pad_tokens > 0:+ pad_z = z[:, :, -1:, :, :].repeat(1, 1, pad_tokens, 1, 1)+ z = torch.cat([z, pad_z], dim=2)++ output_frames = self._decode_temporal_frame_plan(z.shape[2], num_chunks, pad_tokens)++ dec = None+ dec_overlap = None+ write_pos = 0++ def write_part(part):+ nonlocal dec, write_pos+ part_frames = part.shape[2]+ if part_frames <= 0:+ return+ if dec is None:+ out_shape = list(part.shape)+ out_shape[2] = output_frames+ dec = torch.empty(out_shape, dtype=part.dtype, device=part.device)+ copy_frames = min(part_frames, max(0, dec.shape[2] - write_pos))+ if copy_frames > 0:+ dec[:, :, write_pos:write_pos + copy_frames, :, :].copy_(+ part[:, :, :copy_frames, :, :]+ )+ write_pos += copy_frames++ for i in range(num_chunks):+ t_start_idx = i * self.tokens_chunk_size+ t_end_idx = t_start_idx + self.tokens_chunk_size + self.token_overlap+ clip_z = z[:, :, t_start_idx:t_end_idx, :, :]++ clip_dec = self._adaptive_decode(clip_z)++ for j in range(split_count):+ f_start_idx = j * chunk_dec+ f_end_idx = min(f_start_idx + chunk_dec, clip_dec.shape[2])+ clip_dec_chunk = clip_dec[:, :, f_start_idx:f_end_idx, :, :]+ clip_dec_chunk = clip_dec_chunk[:, :, self.frame_pre_padding:, :, :]++ if j == 0:+ if dec_overlap is not None:+ clip_dec_chunk = self.blend(+ dec_overlap, clip_dec_chunk, self.frame_overlap, dim=-3+ )+ dec_overlap = None+ write_part(clip_dec_chunk)+ else:+ dec_overlap = clip_dec_chunk.contiguous()++ if i == num_chunks - 1 and dec_overlap is not None:+ write_part(dec_overlap)+ dec_overlap = None++ del clip_dec, clip_z++ return dec+++ def encode(self, x):+ # x: [B, 3, T, H, W] in [-1, 1] -> normalized latents [B, 24, T_lat, H/16, W/16]+ if x.ndim == 4:+ x = x.unsqueeze(2)++ x = x.add(1.0).mul_(0.5).sub_(self.pixel_mean.to(x)).div_(self.pixel_std.to(x))++ if x.shape[2] == 1:+ moments = self._adaptive_encode(x)+ moments = moments[:, :, -1:, :, :]+ else:+ moments = self.encode_temporal(x)++ mean = torch.chunk(moments.float(), 2, dim=1)[0]++ latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(mean)+ latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(mean)+ return (mean - latents_mean) / latents_std++ def encode_tiled(self, x, **kwargs):+ # tiling is always on internally with the reference's semantic tile sizes, ignore tiling fallbacks+ return self.encode(x)++ def decode_tiled(self, z, **kwargs):+ return self.decode(z)++ def decode(self, z):+ # z: [B, 24, T_lat, H_lat, W_lat] normalized latents -> pixels [B, 3, T, H, W] in [-1, 1]+ latents_mean = self.latents_mean.view(1, -1, 1, 1, 1).to(z)+ latents_std = self.latents_std.view(1, -1, 1, 1, 1).to(z)+ z = z * latents_std + latents_mean++ if z.shape[2] == 1:+ dec = self._adaptive_decode(z)+ dec = dec[:, :, -1:, :, :]+ else:+ dec = self.decode_temporal(z)++ dec = dec.float()+ dec.mul_(self.pixel_std.to(dec)).add_(self.pixel_mean.to(dec)).clamp_(0.0, 1.0).mul_(2.0).sub_(1.0)+ return decdiff --git a/comfy/model_base.py b/comfy/model_base.pyindex 7c68c6eba..6631c9eb0 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -21,6 +21,7 @@ import comfy.ldm.hunyuan3dv2_1.hunyuanditimport torchimport loggingimport comfy.ldm.lightricks.av_model+import comfy.ldm.minimax.modelimport comfy.ldm.lightricks.symmetric_patchifierimport comfy.context_windowsfrom comfy.ldm.modules.diffusionmodules.openaimodel import UNetModel, Timestep@@ -2063,6 +2064,57 @@ class Hunyuan3Dv2_1(BaseModel):out['guidance'] = comfy.conds.CONDRegular(torch.FloatTensor([guidance]))return out+class MiniMaxH3(BaseModel):+ def __init__(self, model_config, model_type=ModelType.FLOW, device=None):+ super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.minimax.model.MiniMaxH3Model)++ def extra_conds(self, **kwargs):+ out = super().extra_conds(**kwargs)+ cross_attn = kwargs.get("cross_attn", None)+ if cross_attn is not None:+ # run condition_proj + token refiner once per sampling instead of per step+ cross_attn = self.diffusion_model.preprocess_text_embeds(+ cross_attn.to(device=kwargs["device"], dtype=self.get_dtype_inference()))+ out['c_crossattn'] = comfy.conds.CONDRegular(cross_attn)++ latent_shapes = kwargs.get("latent_shapes", None)+ if latent_shapes is not None:+ out['latent_shapes'] = comfy.conds.CONDConstant(latent_shapes)++ # Everything H3-specific rides in one dict so _apply_model's dtype cast+ # (which would flatten fp32 cond latents and long tags to bf16) skips it.+ payload = {}+ tags = kwargs.get("minimax_token_tags", None)+ if tags is not None:+ payload["text_token_tags"] = tags+ keyframes = kwargs.get("minimax_keyframes", None)+ if keyframes is not None:+ payload["keyframes"] = keyframes+ payload["frame_count"] = kwargs.get("minimax_frame_count", None)+ payload["cond_video_latents"] = [kf["latent"] for kf in keyframes]+ refs = kwargs.get("minimax_refs", None)+ if refs is not None:+ payload["refs"] = refs+ payload["cond_video_latents"] = [r["latent"] for r in refs if "latent" in r]+ payload["cond_audio_latents"] = [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]+ if kwargs.get("minimax_visual_cond_noise_aug", None) is not None:+ payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"]+ if kwargs.get("minimax_audio_cond_noise_aug", None) is not None:+ payload["audio_cond_noise_aug"] = kwargs["minimax_audio_cond_noise_aug"]+ payload["seed"] = kwargs.get("seed", 0)+ if cross_attn is not None and latent_shapes is not None and len(latent_shapes) > 1:+ # packed layout built once per sampling run, h/w rounded up to the DiT's 2x2 patch+ vs = latent_shapes[0]+ payload["layout"] = comfy.ldm.minimax.model.PackedLayout(+ cross_attn.shape[1], vs[2], (vs[3] + 1) // 2 * 2, (vs[4] + 1) // 2 * 2,+ latent_shapes[1][-1], keyframes=payload.get("keyframes"),+ refs=payload.get("refs"), frame_count=payload.get("frame_count"))+ out['minimax_payload'] = comfy.conds.CONDConstant(payload)+ return out++ def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):+ return latent_image+class TripoSplat(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.triposplat.model.LatentSeqMMFlowModel)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 39e973d36..103680fd1 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -359,6 +359,35 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):# PixArt diffusersreturn None+ if '{}video_patch_proj.weight'.format(key_prefix) in state_dict_keys and '{}audio_patch_proj.weight'.format(key_prefix) in state_dict_keys: # MiniMax H3+ dit_config = {}+ dit_config["image_model"] = "minimax_h3"+ dit_config["num_layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.')+ dit_config["token_refiner_num_layers"] = count_blocks(state_dict_keys, '{}token_refiner.blocks.'.format(key_prefix) + '{}.')+ dit_config["hidden_size"] = state_dict['{}video_patch_proj.weight'.format(key_prefix)].shape[0]+ dit_config["latents_dim"] = state_dict['{}final_layer.video_out.weight'.format(key_prefix)].shape[0] // 4 # patch 1x2x2+ dit_config["audio_latents_dim"] = state_dict['{}final_layer.audio_out.weight'.format(key_prefix)].shape[0]+ dit_config["attention_head_dim"] = state_dict['{}blocks.0.attn.q_norm.weight'.format(key_prefix)].shape[0]+ qkv = state_dict['{}blocks.0.attn.qkv_proj.weight'.format(key_prefix)]+ dit_config["num_attention_heads"] = qkv.shape[0] // (3 * dit_config["attention_head_dim"])+ dit_config["ffn_hidden_size"] = state_dict['{}blocks.0.mlp.fc1.weight'.format(key_prefix)].shape[0] // 2+ dit_config["text_dim"] = state_dict['{}condition_proj.weight'.format(key_prefix)].shape[1]+ table_key = '{}adaln_t_table'.format(key_prefix)+ if table_key in state_dict_keys:+ # adaln shipped over a precomputed curve basis: the adaln linears span a small shared basis of the time-embedding curve (no time embedder)+ table = state_dict[table_key].shape # [grid, k]+ dit_config["adaln_curve_grid"] = table[0]+ dit_config["time_embed_dim"] = table[1]+ else:+ te = state_dict['{}time_embedder.proj_in.weight'.format(key_prefix)]+ dit_config["timestep_input_dim"] = te.shape[1]+ dit_config["time_embed_hidden_size"] = te.shape[0]+ dit_config["time_embed_dim"] = state_dict['{}time_embedder.proj_out.weight'.format(key_prefix)].shape[0]+ dit_config["rope_inv_freq_len"] = state_dict['{}rope.inv_freq'.format(key_prefix)].shape[0]+ if metadata is not None and "config" in metadata:+ dit_config.update(json.loads(metadata["config"]).get("transformer", {}))+ return dit_config+if '{}adaln_single.emb.timestep_embedder.linear_1.bias'.format(key_prefix) in state_dict_keys: #Lightricks ltxvdit_config = {}dit_config["image_model"] = "ltxav" if f'{key_prefix}audio_adaln_single.linear.weight' in state_dict_keys else "ltxv"diff --git a/comfy/ops.py b/comfy/ops.pyindex 6c3845eef..077a42351 100644--- a/comfy/ops.py+++ b/comfy/ops.py@@ -943,13 +943,61 @@ if CUBLAS_IS_AVAILABLE:# ==============================================================================# Mixed Precision Operations# ==============================================================================+from . import quant_opsfrom .quant_ops import (QuantizedTensor,QUANT_ALGOS,TensorCoreFP8Layout,+ TensorWiseINT8Layout,get_layout_class,)+def _swiglu_eager(x):+ gate, up = x.chunk(2, dim=-1)+ return torch.nn.functional.silu(gate).mul_(up)+++INPUT_ACT_EAGER = {+ "gelu_tanh": lambda x: torch.nn.functional.gelu(x, approximate="tanh"),+ "swiglu": _swiglu_eager,+}+++def linear_input_act(linear, x, input_act):+ """``linear(act(x))``, with ``act`` folded into an INT8 activation quantizer.++ An INT8 linear quantizes its input anyway, so an elementwise activation can+ ride along inside that kernel instead of writing a full-size intermediate to+ HBM and reading it straight back. Worth it for an MLP's down-projection,+ where the intermediate is several times the hidden size.++ """+ weight = linear.weight+ if (comfy.model_management.in_training+ or not isinstance(weight, QuantizedTensor)+ or weight._layout_cls != "TensorWiseINT8Layout"+ or getattr(weight._params, "transposed", False)):+ return linear(INPUT_ACT_EAGER[input_act](x))++ # want_requant keeps a vbar-streamed layer on the INT8 path when a LoRA is+ # patched in on the fly; without it the cast hands back a dequantized weight.+ weight, bias, offload_stream = cast_bias_weight(+ linear, x, offloadable=True, compute_dtype=x.dtype, want_requant=True)+ try:+ if not isinstance(weight, QuantizedTensor):+ # A LoRA weight_function, or activations whose dtype differs from the+ # weight's, make the cast hand back a dequantized tensor.+ return torch.nn.functional.linear(INPUT_ACT_EAGER[input_act](x), weight, bias)+ qdata, scale = TensorWiseINT8Layout.get_plain_tensors(weight)+ return quant_ops.ck.int8_linear(+ x, qdata, scale, bias, x.dtype,+ convrot=getattr(weight._params, "convrot", False),+ convrot_groupsize=getattr(weight._params, "convrot_groupsize", 256),+ input_act=input_act,+ )+ finally:+ uncast_bias_weight(linear, weight, bias, offload_stream)+class QuantLinearFunc(torch.autograd.Function):"""Custom autograd function for quantized linear: quantized forward, optionally FP8 backward.@@ -1259,7 +1307,7 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef state_dict(self, *args, destination=None, prefix="", **kwargs):sd = destination if destination is not None else {}- return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale",))+ return _quantized_weight_state_dict(self, sd, prefix, extra_quant_params=("input_scale", "pre_quant_scale"))def _forward(self, input, weight, bias):return torch.nn.functional.linear(input, weight, bias)@@ -1298,6 +1346,11 @@ def mixed_precision_ops(quant_config={}, compute_dtype=torch.bfloat16, full_precdef forward(self, input, *args, **kwargs):run_every_op()+ # ModelOpt AWQ-style smoothing+ pre_quant_scale = getattr(self, 'pre_quant_scale', None)+ if pre_quant_scale is not None:+ input = input * comfy.model_management.cast_to_device(pre_quant_scale, input.device, input.dtype)+input_shape = input.shapereshaped_nd = False#If cast needs to apply lora, it should be done in the compute dtypediff --git a/comfy/quant_ops.py b/comfy/quant_ops.pyindex 15f9b1fdb..53586956a 100644--- a/comfy/quant_ops.py+++ b/comfy/quant_ops.py@@ -240,7 +240,7 @@ QUANT_ALGOS = {},"nvfp4": {"storage_t": torch.uint8,- "parameters": {"weight_scale", "weight_scale_2", "input_scale"},+ "parameters": {"weight_scale", "weight_scale_2", "input_scale", "pre_quant_scale"},"comfy_tensor_layout": "TensorCoreNVFP4Layout","group_size": 16,},diff --git a/comfy/sd.py b/comfy/sd.pyindex caf78222d..8d670106e 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -72,6 +72,9 @@ import comfy.text_encoders.ace15import comfy.text_encoders.longcat_imageimport comfy.text_encoders.qwen35import comfy.text_encoders.qwen3vl+import comfy.text_encoders.minimax+import comfy.ldm.minimax.vae+import comfy.ldm.minimax.audio_vaeimport comfy.text_encoders.booguimport comfy.text_encoders.ernieimport comfy.text_encoders.gemma4@@ -936,6 +939,50 @@ class VAE:#Force cast it for --disable-dynamic-vram users until there is a true core fix.if not comfy.memory_management.aimdo_enabled:self.disable_offload = True+ elif "decoder.transformer_blocks.0.scale1" in sd and "encoder.down.5.block.0.conv1.weight" in sd: # MiniMax H3 video VAE+ self.first_stage_model = comfy.ldm.minimax.vae.MiniMaxH3VideoVAE()+ self.latent_channels = 24+ self.latent_dim = 3+ # frames 17k+5 <-> latents 5k+2, 16x spatial+ self.upscale_ratio = (lambda a: max(1, (a - 2) // 5 * 17 + 5), 16, 16)+ self.upscale_index_formula = (4, 16, 16)+ self.downscale_ratio = (lambda a: max(1, (a - 5) // 17 * 5 + 2) if a > 1 else 1, 16, 16)+ self.downscale_index_formula = (4, 16, 16)+ self.working_dtypes = [torch.float16, torch.float32]+ # the model tiles internally (256px spatial, 17-frame temporal chunks)+ self.handles_tiling = True+ def estimate_encode_memory(frames, height, width, dtype):+ fixed = 110_000_000 if frames == 1 else 1_300_000_000+ elements_per_pixel = 7 if frames == 1 else 9.5+ return (elements_per_pixel * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03++ def estimate_decode_memory(frames, height, width, dtype):+ fixed = 110_000_000 if frames <= 22 else 270_000_000+ return (9.5 * frames * height * width + fixed) * model_management.dtype_size(dtype) * 1.03++ self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], shape[3], shape[4], dtype)+ self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(self.upscale_ratio[0](shape[2]), shape[3] * self.upscale_ratio[1], shape[4] * self.upscale_ratio[2], dtype)+ elif "pre_block.attn.zero_k_bias" in sd: # MiniMax H3 audio VAE (DAC encoder + BigVGAN decoder)+ self.first_stage_model = comfy.ldm.minimax.audio_vae.MiniMaxH3AudioVAE()+ self.latent_channels = 32+ self.output_channels = 2+ self.pad_channel_value = "replicate"+ self.audio_sample_rate = 32000+ self.upscale_ratio = 800+ self.downscale_ratio = 800+ self.latent_dim = 2 # [B, 32, stereo 2, T]+ self.process_output = lambda audio: audio+ self.process_input = lambda audio: audio+ self.working_dtypes = [torch.float32]+ # encode gets the waveform shape [B, 2, samples], decode the latent shape [B, 32, 2, T]+ def estimate_encode_memory(samples, dtype):+ return (900 * samples + 105_000_000) * model_management.dtype_size(dtype) * 1.03++ def estimate_decode_memory(samples, dtype):+ return max(42_000_000, 220 * samples + 20_000_000) * model_management.dtype_size(dtype) * 1.03++ self.memory_used_encode = lambda shape, dtype: estimate_encode_memory(shape[2], dtype)+ self.memory_used_decode = lambda shape, dtype: estimate_decode_memory(shape[-1] * self.upscale_ratio, dtype)elif "gs.base_offset_scale" in sd and "octree.out_proj.weight" in sd: # TripoSplat octree gaussian decoderself.first_stage_model = comfy.ldm.triposplat.vae.OctreeGaussianDecoder()self.latent_channels = 16@@ -1393,6 +1440,7 @@ class CLIPType(Enum):KREA2 = 32JOYIMAGE = 33MAGE = 34+ MINIMAX = 35@@ -1449,6 +1497,7 @@ class TEModel(Enum):QWEN3VL_4B = 34QWEN3VL_8B = 35GEMMA_4_12B = 36+ QWEN3VL_32B = 37def detect_te_model(sd):@@ -1515,6 +1564,9 @@ def detect_te_model(sd):return TEModel.QWEN35_2Bif "model.visual.deepstack_merger_list.0.norm.weight" in sd: # DeepStack is unique to Qwen3-VLreturn TEModel.QWEN3VL_4B if sd["model.visual.merger.linear_fc2.weight"].shape[0] == 2560 else TEModel.QWEN3VL_8B+ if "visual.deepstack_merger_list.0.norm.weight" in sd and "model.layers.49.self_attn.q_proj.weight" in sd:+ # MiniMax H3 conditioning encoder: Qwen3-VL-32B, truncated to 50 layers+ return TEModel.QWEN3VL_32Bif "model.layers.0.post_attention_layernorm.weight" in sd:weight = sd['model.layers.0.post_attention_layernorm.weight']if 'model.layers.0.self_attn.q_norm.weight' in sd:@@ -1744,6 +1796,9 @@ def load_text_encoder_state_dicts(state_dicts=[], embedding_directory=None, clipqwen3vl_type = {TEModel.QWEN3VL_4B: "qwen3vl_4b", TEModel.QWEN3VL_8B: "qwen3vl_8b"}[te_model]clip_target.clip = comfy.text_encoders.qwen3vl.te(**llama_detect(clip_data), model_type=qwen3vl_type)clip_target.tokenizer = comfy.text_encoders.qwen3vl.tokenizer(model_type=qwen3vl_type)+ elif te_model == TEModel.QWEN3VL_32B:+ clip_target.clip = comfy.text_encoders.minimax.te(**llama_detect(clip_data))+ clip_target.tokenizer = comfy.text_encoders.minimax.MiniMaxH3Tokenizerelif te_model == TEModel.QWEN3_06B:clip_target.clip = comfy.text_encoders.anima.te(**llama_detect(clip_data))clip_target.tokenizer = comfy.text_encoders.anima.AnimaTokenizerdiff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex ca89850a5..51b58ed1e 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -15,6 +15,7 @@ import comfy.text_encoders.fluximport comfy.text_encoders.genmoimport comfy.text_encoders.ltimport comfy.text_encoders.hunyuan_video+import comfy.text_encoders.minimaximport comfy.text_encoders.cosmosimport comfy.text_encoders.lumina2import comfy.text_encoders.wan@@ -955,6 +956,33 @@ class LTXAV(LTXV):out = model_base.LTXAV(self, device=device)return out+class MiniMaxH3(supported_models_base.BASE):+ unet_config = {+ "image_model": "minimax_h3",+ }++ sampling_settings = {+ "shift": 12.0,+ }++ unet_extra_config = {}+ latent_format = latent_formats.MiniMaxH3AV++ memory_usage_factor = 0.114++ supported_inference_dtypes = [torch.bfloat16, torch.float32]++ vae_key_prefix = ["vae."]+ text_encoder_key_prefix = ["text_encoders."]++ def get_model(self, state_dict, prefix="", device=None):+ return model_base.MiniMaxH3(self, device=device)++ def clip_target(self, state_dict={}, prefix=""):+ pref = self.text_encoder_key_prefix[0]+ detect = comfy.text_encoders.hunyuan_video.llama_detect(state_dict, "{}qwen3vl_32b.transformer.".format(pref))+ return supported_models_base.ClipTarget(comfy.text_encoders.minimax.MiniMaxH3Tokenizer, comfy.text_encoders.minimax.te(**detect))+class HunyuanVideo(supported_models_base.BASE):unet_config = {"image_model": "hunyuan_video",@@ -2407,6 +2435,7 @@ models = [GenmoMochi,LTXV,LTXAV,+ MiniMaxH3,HunyuanVideo15_SR_Distilled,HunyuanVideo15,HunyuanImage21Refiner,diff --git a/comfy/text_encoders/llama.py b/comfy/text_encoders/llama.pyindex 40d04007e..f5c5597ef 100644--- a/comfy/text_encoders/llama.py+++ b/comfy/text_encoders/llama.py@@ -264,6 +264,17 @@ class Qwen3VL_4BConfig(Qwen3VL_8BConfig):intermediate_size: int = 9728lm_head: bool = False # 4B ties word embeddings+@dataclass+class Qwen3VL_32BConfig(Qwen3VL_8BConfig):+ # MiniMax H3 conditioning checkpoint: truncated to the first 50 of 64 layers,+ # consumed as the unnormalized hidden state after layer 50 (no final norm, no lm_head)+ hidden_size: int = 5120+ intermediate_size: int = 25600+ num_hidden_layers: int = 50+ num_attention_heads: int = 64+ lm_head: bool = False+ final_norm: bool = False+@dataclassclass Ovis25_2BConfig:vocab_size: int = 151936diff --git a/comfy/text_encoders/minimax.py b/comfy/text_encoders/minimax.pynew file mode 100644index 000000000..c2dc47f7f--- /dev/null+++ b/comfy/text_encoders/minimax.py@@ -0,0 +1,201 @@+"""MiniMax H3 text/vision conditioning: Qwen3-VL-32B (truncated to 50 layers).++The H3 presentation is NOT chat-templated: token ids are raw prompt/label text+(no special tokens) with explicit vision blocks spliced in:++ t2va: <prompt>+ fl2va: "<Picture 1>: " <vision block> ["<Picture 2>: " <vision block>] <prompt>+ ref2va: per condition in request order (1-based ordinals per type):+ image -> "<Picture i>: " <vision block>+ audio -> "<Audio j>: " (audio never enters Qwen)+ video -> "<Video k>: " then per 2-frame temporal block+ "<T.T seconds>" <vision block(2 frames)>+ then <prompt>++The conditioning is the unnormalized hidden state after LM layer 50 (the+converted checkpoint is truncated there, so this is simply the last-layer+output with no final norm). Vision-pad positions carry adaLN token tag 0+(video modality) in the DiT; text positions carry tag 1 — the tags are+returned alongside the embeddings as "minimax_token_tags".+"""++import math++import torch+import torch.nn.functional as F+import comfy.sd1_clip+from .qwen3vl import Qwen3VL, Qwen3VLSDTokenizer++VISION_START = 151652+VISION_END = 151653+QWEN_IMAGE_MEAN = [0.5, 0.5, 0.5]+QWEN_IMAGE_STD = [0.5, 0.5, 0.5]+++def process_video_block(frames, patch_size=16, temporal_patch_size=2, merge_size=2,+ min_pixels=3136, max_pixels=12845056):+ """[2, H, W, C] frame pair -> (flatten_patches, grid_thw) with grid_t=1.++ Same resize/normalize policy as process_qwen2vl_images, but the two frames+ fill the temporal patch instead of repeating a single frame.+ """+ t, height, width, _ = frames.shape+ imgs = frames.permute(0, 3, 1, 2)+ factor = patch_size * merge_size+ h_bar = round(height / factor) * factor+ w_bar = round(width / factor) * factor+ if h_bar * w_bar > max_pixels:+ beta = math.sqrt((height * width) / max_pixels)+ h_bar = max(factor, math.floor(height / beta / factor) * factor)+ w_bar = max(factor, math.floor(width / beta / factor) * factor)+ elif h_bar * w_bar < min_pixels:+ beta = math.sqrt(min_pixels / (height * width))+ h_bar = math.ceil(height * beta / factor) * factor+ w_bar = math.ceil(width * beta / factor) * factor++ imgs = F.interpolate(imgs, size=(h_bar, w_bar), mode="bilinear", align_corners=False)+ mean = torch.tensor(QWEN_IMAGE_MEAN, device=imgs.device).view(1, 3, 1, 1)+ std = torch.tensor(QWEN_IMAGE_STD, device=imgs.device).view(1, 3, 1, 1)+ imgs = (imgs - mean) / std++ grid_h = h_bar // patch_size+ grid_w = w_bar // patch_size+ patches = imgs.reshape(1, temporal_patch_size, 3, grid_h // merge_size, merge_size,+ patch_size, grid_w // merge_size, merge_size, patch_size)+ patches = patches.permute(0, 3, 6, 4, 7, 2, 1, 5, 8)+ flatten = patches.reshape(grid_h * grid_w, 3 * temporal_patch_size * patch_size * patch_size)+ grid_thw = torch.stack([torch.tensor([1, grid_h, grid_w], device=frames.device, dtype=torch.long)])+ return flatten, grid_thw+++def token_tags_from_embeds_info(seq_len, embeds_info):+ # whole vision block VIDEO(0), including the flanking <|vision_start|>/<|vision_end|> tokens+ # embeds_info spans cover only the expanded embeddings, so widen by one on each side.+ tags = torch.ones(seq_len, dtype=torch.long)+ for e in embeds_info:+ if e.get("type") == "image":+ tags[max(0, e["index"] - 1):e["index"] + e["size"] + 1] = 0+ return tags+++class MiniMaxQwen3VL(Qwen3VL):+ model_type = "qwen3vl_32b"++ def preprocess_embed(self, embed, device):+ if embed["type"] == "image" and embed.get("minimax_video_block", False):+ flatten, grid = process_video_block(embed["data"])+ merged, deepstack = self.visual(flatten.to(device, dtype=torch.float32), grid)+ return merged, {"grid": grid, "deepstack": deepstack}+ return super().preprocess_embed(embed, device)++ def forward(self, input_ids, attention_mask=None, embeds=None, num_tokens=None,+ intermediate_output=None, final_layer_norm_intermediate=True,+ dtype=None, embeds_info=[], **kwargs):+ seq = embeds.shape[1] if embeds is not None else input_ids.shape[1]+ self.last_token_tags = token_tags_from_embeds_info(seq, embeds_info)+ return super().forward(input_ids, attention_mask=attention_mask, embeds=embeds,+ num_tokens=num_tokens, intermediate_output=intermediate_output,+ final_layer_norm_intermediate=final_layer_norm_intermediate,+ dtype=dtype, embeds_info=embeds_info, **kwargs)+++class MiniMaxH3ClipModel(comfy.sd1_clip.SDClipModel):+ def __init__(self, device="cpu", layer="last", layer_idx=None, dtype=None, model_options={}):+ super().__init__(device=device, layer="last", layer_idx=None, textmodel_json_config={},+ dtype=dtype, special_tokens={"pad": 151643}, layer_norm_hidden_state=False,+ model_class=MiniMaxQwen3VL, enable_attention_masks=False,+ return_attention_masks=False, model_options=model_options)++ def encode_token_weights(self, token_weight_pairs):+ out = super().encode_token_weights(token_weight_pairs)+ tags = getattr(self.transformer, "last_token_tags", None)+ if tags is not None:+ extra = out[2] if len(out) > 2 and isinstance(out[2], dict) else {}+ extra["minimax_token_tags"] = tags+ out = (out[0], out[1], extra)+ return out+++class MiniMaxH3TEModel(comfy.sd1_clip.SD1ClipModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ super().__init__(device=device, dtype=dtype, name="qwen3vl_32b",+ clip_model=MiniMaxH3ClipModel, model_options=model_options)+++class MiniMaxH3Tokenizer(comfy.sd1_clip.SD1Tokenizer):+ def __init__(self, embedding_directory=None, tokenizer_data={}):+ tokenizer = lambda *a, **kw: Qwen3VLSDTokenizer(*a, **kw, embedding_size=5120, embedding_key="qwen3vl_32b")+ super().__init__(embedding_directory=embedding_directory, tokenizer_data=tokenizer_data, name="qwen3vl_32b", tokenizer=tokenizer)++ def _text_ids(self, text):+ tok = self.qwen3vl_32b.tokenizer+ return tok(text, add_special_tokens=False)["input_ids"]++ @staticmethod+ def _vision_entry(data, video_block=False):+ emb = {"type": "image", "data": data, "original_type": "image"}+ if video_block:+ emb["minimax_video_block"] = True+ return emb++ def tokenize_with_weights(self, text, return_word_ids=False, images=[],+ minimax_ref_items=None, **kwargs):+ entries = []++ def add_text(s):+ entries.extend((tid, 1.0) for tid in self._text_ids(s))++ def add_vision(data, video_block=False):+ entries.append((VISION_START, 1.0))+ entries.append((self._vision_entry(data, video_block), 1.0))+ entries.append((VISION_END, 1.0))++ if minimax_ref_items:+ counters = {"image": 0, "audio": 0, "video": 0}+ for item in minimax_ref_items:+ kind = item["type"]+ counters[kind] += 1+ if kind == "image":+ add_text("<Picture %d>: " % counters["image"])+ add_vision(item["data"])+ elif kind == "audio":+ add_text("<Audio %d>: " % counters["audio"])+ elif kind == "video":+ frames = item["data"] # [T, H, W, C], sampled at 2 fps+ timestamps = item.get("timestamps")+ if timestamps is None:+ timestamps = [i / 2.0 for i in range(frames.shape[0])]+ if frames.shape[0] % 2 == 1: # repeat-pad to temporal patch of 2+ frames = torch.cat([frames, frames[-1:]], dim=0)+ timestamps = list(timestamps) + [timestamps[-1]]+ add_text("<Video %d>: " % counters["video"])+ for i in range(0, frames.shape[0], 2):+ block_ts = (timestamps[i] + timestamps[i + 1]) / 2.0+ add_text("<%.1f seconds>" % block_ts)+ add_vision(frames[i:i + 2], video_block=True)+ else:+ for i, img in enumerate(images):+ add_text("<Picture %d>: " % (i + 1))+ add_vision(img)++ add_text(text)+ if len(entries) == 0:+ entries.append((151643, 1.0))+ if return_word_ids:+ entries = [t + (0,) for t in entries]+ return {"qwen3vl_32b": [entries]}++ def untokenize(self, token_weight_pair):+ return self.qwen3vl_32b.untokenize(token_weight_pair)+++def te(dtype_llama=None, llama_quantization_metadata=None, **kwargs):+ class MiniMaxH3TEModel_(MiniMaxH3TEModel):+ def __init__(self, device="cpu", dtype=None, model_options={}):+ if dtype_llama is not None:+ dtype = dtype_llama+ if llama_quantization_metadata is not None:+ model_options = model_options.copy()+ model_options["quantization_metadata"] = llama_quantization_metadata+ super().__init__(device=device, dtype=dtype, model_options=model_options)+ return MiniMaxH3TEModel_diff --git a/comfy/text_encoders/qwen3vl.py b/comfy/text_encoders/qwen3vl.pyindex 2dd60d4e6..fd7d0d267 100644--- a/comfy/text_encoders/qwen3vl.py+++ b/comfy/text_encoders/qwen3vl.py@@ -8,17 +8,18 @@ from transformers import Qwen2Tokenizerfrom comfy import sd1_clipimport comfy.text_encoders.qwen_vlfrom .qwen35 import Qwen35VisionModel-from .llama import BaseLlama, BaseQwen3, BaseGenerate, Llama2_, Qwen3VL_4BConfig, Qwen3VL_8BConfig+from .llama import BaseLlama, BaseQwen3, BaseGenerate, Llama2_, Qwen3VL_4BConfig, Qwen3VL_8BConfig, Qwen3VL_32BConfigQWEN3VL_VISION = {"qwen3vl_4b": dict(hidden_size=1024, intermediate_size=4096, depth=24, deepstack_visual_indexes=[5, 11, 17]),"qwen3vl_8b": dict(hidden_size=1152, intermediate_size=4304, depth=27, deepstack_visual_indexes=[8, 16, 24]),+ "qwen3vl_32b": dict(hidden_size=1152, intermediate_size=4304, depth=27, deepstack_visual_indexes=[8, 16, 24]),}QWEN3VL_VISION_COMMON = dict(num_heads=16, patch_size=16, temporal_patch_size=2, in_channels=3,spatial_merge_size=2, num_position_embeddings=2304)-QWEN3VL_CONFIGS = {"qwen3vl_4b": Qwen3VL_4BConfig, "qwen3vl_8b": Qwen3VL_8BConfig}+QWEN3VL_CONFIGS = {"qwen3vl_4b": Qwen3VL_4BConfig, "qwen3vl_8b": Qwen3VL_8BConfig, "qwen3vl_32b": Qwen3VL_32BConfig}class Qwen3VLDeepstackMerger(nn.Module):diff --git a/comfy_extras/nodes_lt.py b/comfy_extras/nodes_lt.pyindex 044d82cc8..b4806bd6a 100644--- a/comfy_extras/nodes_lt.py+++ b/comfy_extras/nodes_lt.py@@ -746,6 +746,8 @@ class LTXVConcatAVLatent(io.ComfyNode):def define_schema(cls):return io.Schema(node_id="LTXVConcatAVLatent",+ display_name="Concat AV Latent",+ description="Merge a video latent and an audio latent into a joint AV latent (any AV model, e.g. LTXV or MiniMax H3).",category="model/latent/ltxv",inputs=[io.Latent.Input("video_latent"),@@ -781,8 +783,9 @@ class LTXVSeparateAVLatent(io.ComfyNode):def define_schema(cls):return io.Schema(node_id="LTXVSeparateAVLatent",+ display_name="Separate AV Latent",category="model/latent/ltxv",- description="LTXV Separate AV Latent",+ description="Split a joint AV latent into its video and audio latents (any AV model, e.g. LTXV or MiniMax H3).",inputs=[io.Latent.Input("av_latent"),],diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.pynew file mode 100644index 000000000..22bc91cd4--- /dev/null+++ b/comfy_extras/nodes_minimax_h3.py@@ -0,0 +1,337 @@+"""MiniMax H3 nodes: AV latent creation and task conditioning (t2va / fl2va / ref2va).++The H3 packed-DiT consumes, via conditioning:+- Qwen3-VL-32B hidden states with per-token modality tags (from the minimax CLIP)+- keyframe / reference condition latents, re-injected every step (never denoised)++Latents are NestedTensor pairs (video [B,24,T,H/16,W/16], audio [B,32,2,T40]);+sampling runs on the flat pack with any stock sampler (the model handles the+audio stream's shifted schedule internally).+"""++import math++import torch+import torchaudio++import nodes+import comfy.model_management+import comfy.model_sampling+import comfy.nested_tensor+import comfy.utils+import node_helpers+from comfy_api.latest import ComfyExtension, io++CANVAS_MULTIPLE = 32+BASE_SHORT_EDGE = 768+MAX_PIXELS = 768 * 1344+REF_IMAGE_SHORT_EDGE = 2048+FPS = 24+AUDIO_LATENT_FPS = 40+++def align_frame_count(n):+ while n % 17 != 5:+ n += 1+ return n+++def video_latent_t(frame_count):+ return 2 if frame_count <= 5 else ((frame_count - 5) // 17) * 5 + 2+++def temporal_shape(length):+ frame_count = align_frame_count(max(5, length))+ duration = frame_count / FPS+ return frame_count, video_latent_t(frame_count), round(duration * AUDIO_LATENT_FPS)+++def adapt_canvas(width, height):+ """768-short-edge canvas with 768*1344 area cap, per-axis round to 32."""+ ratio = width / height+ if ratio >= 1.0:+ nom_w, nom_h = BASE_SHORT_EDGE * ratio, BASE_SHORT_EDGE+ else:+ nom_w, nom_h = BASE_SHORT_EDGE, BASE_SHORT_EDGE / ratio+ if nom_w * nom_h > MAX_PIXELS:+ s = math.sqrt(MAX_PIXELS / (nom_w * nom_h))+ nom_w, nom_h = nom_w * s, nom_h * s+ return (max(CANVAS_MULTIPLE, round(nom_w / CANVAS_MULTIPLE) * CANVAS_MULTIPLE),+ max(CANVAS_MULTIPLE, round(nom_h / CANVAS_MULTIPLE) * CANVAS_MULTIPLE))+++def _resize(image, width, height, crop):+ # image [B, H, W, C] -> [B, height, width, 3]+ samples = image[..., :3].movedim(-1, 1)+ samples = comfy.utils.common_upscale(samples, width, height, "lanczos", crop)+ return samples.movedim(1, -1)+++def _empty_av_latent(width, height, length, batch_size=1):+ frame_count, latent_t, audio_t = temporal_shape(length)+ video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16],+ device=comfy.model_management.intermediate_device())+ audio = torch.zeros([batch_size, 32, 2, audio_t],+ device=comfy.model_management.intermediate_device())+ return {"samples": comfy.nested_tensor.NestedTensor((video, audio))}, frame_count+++class EmptyMiniMaxH3LatentAV(io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="EmptyMiniMaxH3LatentAV",+ display_name="Empty MiniMax H3 AV Latent",+ category="model/latent/minimax",+ description="Joint video+audio latent for MiniMax H3. Duration snaps to the model's 17k+5 frame grid at 24 fps.",+ inputs=[+ io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, snapped up to the model's 17k+5 grid (124 = ~5s; trained range is ~124-362, longer is untested)"),+ ],+ outputs=[io.Latent.Output()],+ )++ @classmethod+ def execute(cls, width, height, length) -> io.NodeOutput:+ latent, _ = _empty_av_latent(width, height, length)+ return io.NodeOutput(latent)+++class MiniMaxH3ImageToVideo(io.ComfyNode):+ """t2va and fl2va: prompt (+ optional first/last keyframes) -> conditioning + AV latent."""++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="MiniMaxH3ImageToVideo",+ display_name="MiniMax H3 Image to Video",+ category="model/conditioning/minimax",+ inputs=[+ io.Clip.Input("clip"),+ io.Vae.Input("vae"),+ io.String.Input("prompt", multiline=True, dynamic_prompts=True),+ io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, snapped up to the model's 17k+5 grid (124 = ~5s; trained range is ~124-362, longer is untested)"),+ io.Image.Input("first_frame", optional=True),+ io.Image.Input("last_frame", optional=True),+ ],+ outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],+ )++ @classmethod+ def execute(cls, clip, vae, prompt, width, height, length,+ first_frame=None, last_frame=None) -> io.NodeOutput:+ latent, frame_count = _empty_av_latent(width, height, length)++ images = []+ keyframes = []+ if first_frame is not None:+ # geometry anchor: plain stretch to canvas+ img = _resize(first_frame[:1], width, height, "disabled")+ images.append(img)+ keyframes.append({"resolved_frame_index": 0, "image": img})+ if last_frame is not None:+ # follower: aspect-preserving cover-crop+ img = _resize(last_frame[:1], width, height, "center")+ images.append(img)+ keyframes.append({"resolved_frame_index": frame_count - 1, "image": img})++ tokens = clip.tokenize(prompt, images=images)+ cond = clip.encode_from_tokens_scheduled(tokens)++ if keyframes:+ for kf in keyframes:+ kf["latent"] = vae.encode(kf.pop("image"))+ cond = node_helpers.conditioning_set_values(cond, {+ "minimax_keyframes": keyframes,+ "minimax_frame_count": frame_count,+ })+ return io.NodeOutput(cond, latent)+++class MiniMaxH3ReferenceToVideo(io.ComfyNode):+ """ref2va: prompt + reference images / videos / audio -> conditioning + AV latent.++ References enter the presentation in fixed order: images, then videos (each+ soundtrack's <Audio j> label right before its <Video k>), then standalone+ audio. Ordinals are 1-based per type, so the prompt refers to them as+ <Picture i> / <Video k> / <Audio j>.+ """++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="MiniMaxH3ReferenceToVideo",+ description="<Picture i> / <Video k> / <Audio j> reference conditioning for MiniMax H3. Use the same tags when prompting.",+ display_name="MiniMax H3 Reference to Video",+ category="model/conditioning/minimax",+ inputs=[+ io.Clip.Input("clip"),+ io.Vae.Input("vae"),+ io.Vae.Input("audio_vae"),+ io.String.Input("prompt", multiline=True, dynamic_prompts=True),+ io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),+ io.Int.Input("length", default=124, min=5, max=3600, step=17, tooltip="Frame count at 24 fps, (124 = ~5s, trained range is ~124-362)"),+ io.Combo.Input("ref_image_size", options=["match", "max"], default="match",+ tooltip="Reference image sizing. 'match' scales each ref (down only, keeping aspect) to the generation's pixel area; 'max' uses the reference pipeline's 2048px short edge for best identity fidelity. Reference tokens ride through every sampling step, so 'max' can be several times slower."),+ io.Autogrow.Input("ref_images", optional=True,+ template=io.Autogrow.TemplatePrefix(+ input=io.Image.Input("ref_image", tooltip="Reference image (downscaled to 2048 short edge if larger, never upscaled)"),+ prefix="ref_image_", min=0, max=9)),+ io.Autogrow.Input("ref_videos", optional=True,+ template=io.Autogrow.TemplatePrefix(+ input=io.Image.Input("ref_video", tooltip="Reference video frames at 24 fps (2-15s)"),+ prefix="ref_video_", min=0, max=3)),+ io.Autogrow.Input("ref_video_audios", optional=True,+ template=io.Autogrow.TemplatePrefix(+ input=io.Audio.Input("ref_video_audio", tooltip="Soundtrack of the same-numbered reference video"),+ prefix="ref_video_audio_", min=0, max=3)),+ io.Autogrow.Input("ref_audios", optional=True,+ template=io.Autogrow.TemplatePrefix(+ input=io.Audio.Input("ref_audio", tooltip="Standalone reference audio"),+ prefix="ref_audio_", min=0, max=3)),+ ],+ outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],+ )++ @staticmethod+ def _encode_ref_audio(audio_vae, audio):+ waveform = audio["waveform"] # [B, C, L]+ sr = audio["sample_rate"]+ vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)+ if sr != vae_sr:+ waveform = torchaudio.functional.resample(waveform, sr, vae_sr)+ z = audio_vae.encode(waveform[:1].movedim(1, -1)) # [1, 32, 2, T]+ return z, z.shape[-1]++ @classmethod+ def execute(cls, clip, vae, audio_vae, prompt, width, height, length, ref_image_size="match",+ ref_images=None, ref_videos=None, ref_video_audios=None, ref_audios=None) -> io.NodeOutput:+ latent, frame_count = _empty_av_latent(width, height, length)++ ref_items = [] # for the tokenizer presentation, in request order+ ref_blocks = [] # for the DiT payload, same order++ for img in (ref_images or {}).values():+ if img is None:+ continue+ h, w = img.shape[1], img.shape[2]+ if ref_image_size == "match":+ # aspect-preserving scale (down only) to the generation's pixel area+ scale = min(1.0, math.sqrt((width * height) / (w * h)))+ else:+ scale = min(1.0, REF_IMAGE_SHORT_EDGE / min(w, h))+ tw = max(CANVAS_MULTIPLE, round(w * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)+ th = max(CANVAS_MULTIPLE, round(h * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)+ resized = _resize(img[:1], tw, th, "disabled")+ z = vae.encode(resized)+ ref_items.append({"type": "image", "data": resized})+ ref_blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, "latent": z})++ ref_video_audios = ref_video_audios or {}+ for name, video_frames in (ref_videos or {}).items():+ if video_frames is None:+ continue+ # index-paired soundtrack: ref_video_audio_N belongs to ref_video_N+ soundtrack = ref_video_audios.get("ref_video_audio_" + name.rsplit("_", 1)[-1])+ vh, vw = video_frames.shape[1], video_frames.shape[2]+ cw, ch = adapt_canvas(vw, vh)+ if vw * vh < cw * ch:+ cw = max(CANVAS_MULTIPLE, round(vw / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)+ ch = max(CANVAS_MULTIPLE, round(vh / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)+ frames = _resize(video_frames, cw, ch, "disabled")+ if frames.shape[0] > frame_count:+ frames = frames[:frame_count]+ n = frames.shape[0]+ if n < 5:+ raise ValueError("MiniMax H3 reference videos need at least 5 frames (~0.2s at 24 fps)")+ while n % 17 != 5:+ n -= 1+ frames = frames[:n]+ z = vae.encode(frames)+ audio_latent, ref_audio_t = (None, 0)+ if soundtrack is not None:+ audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, soundtrack)+ # the soundtrack gets its own <Audio j> label, emitted before <Video k>+ ref_items.append({"type": "audio"})+ # Qwen sees the video at 2 fps with timestamps+ sample_idx = list(range(0, frames.shape[0], FPS // 2))+ qwen_frames = frames[sample_idx]+ ref_items.append({"type": "video", "data": qwen_frames,+ "timestamps": [i / 2.0 for i in range(len(sample_idx))]})+ ref_blocks.append({"kind": "video_audio" if ref_audio_t else "video",+ "latent_t": z.shape[2], "latent_h": ch // 16, "latent_w": cw // 16,+ "ref_audio_t": ref_audio_t, "latent": z, "audio_latent": audio_latent})++ for audio in (ref_audios or {}).values():+ if audio is None:+ continue+ audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, audio)+ ref_items.append({"type": "audio"})+ ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t, "audio_latent": audio_latent})++ tokens = clip.tokenize(prompt, minimax_ref_items=ref_items)+ cond = clip.encode_from_tokens_scheduled(tokens)+ if ref_blocks:+ cond = node_helpers.conditioning_set_values(cond, {"minimax_refs": ref_blocks})+ return io.NodeOutput(cond, latent)+++class MiniMaxH3SigmaShift(io.ComfyNode):+ """Set the video/audio flow shifts coherently.++ The video shift drives the sampler's sigma schedule; both values are also+ handed to the DiT, which inverts the video schedule to the shared base grid+ and derives the audio schedule from it.+ """++ @classmethod+ def define_schema(cls):+ return io.Schema(+ node_id="MiniMaxH3SigmaShift",+ description="Set the video/audio flow shifts.",+ display_name="MiniMax H3 Sigma Shift",+ category="model/patch/minimax",+ inputs=[+ io.Model.Input("model"),+ io.Float.Input("shift_video", default=12.0, min=0.01, max=100.0, step=0.01),+ io.Float.Input("shift_audio", default=3.0, min=0.01, max=100.0, step=0.01),+ ],+ outputs=[io.Model.Output()],+ )++ @classmethod+ def execute(cls, model, shift_video, shift_audio) -> io.NodeOutput:+ m = model.clone()++ class ModelSamplingAdvanced(comfy.model_sampling.ModelSamplingDiscreteFlow, comfy.model_sampling.CONST):+ pass++ original = m.get_model_object("model_sampling")+ model_sampling = ModelSamplingAdvanced(model.model.model_config)+ model_sampling.set_parameters(shift=shift_video)+ if hasattr(original, "noise_scale"):+ model_sampling.set_noise_scale(original.noise_scale)+ m.add_object_patch("model_sampling", model_sampling)++ to = m.model_options["transformer_options"] = m.model_options.get("transformer_options", {}).copy()+ to["minimax_h3_sigma_shift_video"] = shift_video+ to["minimax_h3_sigma_shift_audio"] = shift_audio+ return io.NodeOutput(m)+++class MiniMaxH3Extension(ComfyExtension):+ async def get_node_list(self):+ return [+ EmptyMiniMaxH3LatentAV,+ MiniMaxH3ImageToVideo,+ MiniMaxH3ReferenceToVideo,+ MiniMaxH3SigmaShift+ ]+++async def comfy_entrypoint() -> MiniMaxH3Extension:+ return MiniMaxH3Extension()diff --git a/nodes.py b/nodes.pyindex 243a55bf2..de86e93fe 100644--- a/nodes.py+++ b/nodes.py@@ -992,7 +992,7 @@ class CLIPLoader:@classmethoddef INPUT_TYPES(s):return {"required": { "clip_name": (folder_paths.get_filename_list("text_encoders"), ),- "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage"], ),+ "type": (["stable_diffusion", "stable_cascade", "sd3", "stable_audio", "mochi", "ltxv", "pixart", "cosmos", "lumina2", "wan", "hidream", "chroma", "ace", "omnigen2", "qwen_image", "hunyuan_image", "flux2", "ovis", "longcat_image", "cogvideox", "lens", "pixeldit", "ideogram4", "boogu", "krea2", "joyimage", "mage", "minimax"], ),},"optional": {"device": (["default", "cpu"], {"advanced": True}),@@ -2436,6 +2436,7 @@ async def init_builtin_extra_nodes():"nodes_mahiro.py","nodes_lt_upsampler.py","nodes_lt_audio.py",+ "nodes_minimax_h3.py","nodes_lt.py","nodes_hooks.py","nodes_multigpu.py",