The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support models with PDD baked in (#17157)
Jukka Seppänence42de5f64ac
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.pyindex 440d47ef3..6e3884b71 100644--- a/comfy/ldm/minimax/model.py+++ b/comfy/ldm/minimax/model.py@@ -299,16 +299,18 @@ class DiTBlock(nn.Module):class FinalLayer(nn.Module):- def __init__(self, hidden, t_dim, video_dim, audio_dim, eps, apply_silu=True, adaln_dtype=None,+ def __init__(self, hidden, t_dim, video_dim, audio_dim, eps, apply_silu=True, adaln_dtype=None, pdd_steps=1,dtype=None, device=None, operations=None):super().__init__()+ self.video_dim = video_dimself.norm = operations.RMSNorm(hidden, eps=eps, dtype=dtype, device=device)self.adaln_proj = AdalnProj(t_dim, hidden, 2, 1, apply_silu=apply_silu,dtype=adaln_dtype if adaln_dtype is not None else dtype,device=device, operations=operations)- # output heads are the checkpoint's fp32 island; norm/adaln are stored at model dtype- self.video_out = operations.Linear(hidden, video_dim, bias=True, dtype=torch.float32, device=device)- self.audio_out = operations.Linear(hidden, audio_dim, bias=True, dtype=torch.float32, device=device)+ # output heads are the checkpoint's fp32 island; norm/adaln are stored at model dtype.+ # a PDD checkpoint ships pdd_steps stacked heads per stream (a PDD LoRA grows plain heads to the same layout)+ self.video_out = operations.Linear(hidden, video_dim * pdd_steps, bias=True, dtype=torch.float32, device=device)+ self.audio_out = operations.Linear(hidden, audio_dim * pdd_steps, bias=True, dtype=torch.float32, device=device)def forward(self, x, t_emb, video_seg, audio_seg, sigma, sample_sigmas, shifts):# video_seg / audio_seg: (start, stop, row) of the target streams, where row@@ -319,7 +321,7 @@ class FinalLayer(nn.Module):a, b, row = segreturn (self.norm(x[a:b]) * (1.0 + _mod_row(scale, row, scale.dtype)) + _mod_row(shift, row, shift.dtype)).to(torch.float32)- n = self.video_out.weight.shape[0] // self.video_out.out_features+ n = self.video_out.weight.shape[0] // self.video_dimif n == 1:return self.video_out(mod(video_seg)), self.audio_out(mod(audio_seg))@@ -478,7 +480,7 @@ class MiniMaxH3Model(nn.Module):timestep_input_dim=256, time_embed_hidden_size=5376, time_embed_dim=2688,rope_inv_freq_len=16, norm_eps=1e-5, qk_norm_eps=1e-5, final_norm_eps=1e-5,sigma_shift_video=12.0, sigma_shift_audio=3.0,- adaln_curve_grid=None, gate_compress=False,+ adaln_curve_grid=None, gate_compress=False, pdd_steps=1,image_model=None, dtype=None, device=None, operations=None, **kwargs):super().__init__()self.dtype = dtype@@ -513,7 +515,7 @@ class MiniMaxH3Model(nn.Module):dtype=dtype, device=device, operations=operations)for _ in range(num_layers)])self.final_layer = FinalLayer(hidden_size, time_embed_dim, video_patch_dim, audio_latents_dim,- final_norm_eps, **curve, dtype=dtype, device=device, operations=operations)+ final_norm_eps, **curve, pdd_steps=pdd_steps, dtype=dtype, device=device, operations=operations)def preprocess_text_embeds(self, text_states):"""[B, L, text_dim] Qwen states -> [B, L, hidden] refined text embeds."""diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 78b9e52e3..79c0e143e 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -404,8 +404,12 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["num_layers"] = count_blocks(state_dict_keys, '{}blocks.'.format(key_prefix) + '{}.')dit_config["token_refiner_num_layers"] = count_blocks(state_dict_keys, '{}token_refiner.blocks.'.format(key_prefix) + '{}.')dit_config["hidden_size"] = state_dict['{}video_patch_proj.weight'.format(key_prefix)].shape[0]- dit_config["latents_dim"] = state_dict['{}final_layer.video_out.weight'.format(key_prefix)].shape[0] // 4 # patch 1x2x2- dit_config["audio_latents_dim"] = state_dict['{}final_layer.audio_out.weight'.format(key_prefix)].shape[0]+ # a PDD checkpoint stacks pdd_steps output heads per stream (declared in its config metadata)+ pdd_steps = 1+ if metadata is not None and "config" in metadata:+ pdd_steps = json.loads(metadata["config"]).get("transformer", {}).get("pdd_steps", 1)+ dit_config["latents_dim"] = state_dict['{}final_layer.video_out.weight'.format(key_prefix)].shape[0] // (4 * pdd_steps) # patch 1x2x2+ dit_config["audio_latents_dim"] = state_dict['{}final_layer.audio_out.weight'.format(key_prefix)].shape[0] // pdd_stepsdit_config["attention_head_dim"] = state_dict['{}blocks.0.attn.q_norm.weight'.format(key_prefix)].shape[0]qkv = state_dict['{}blocks.0.attn.qkv_proj.weight'.format(key_prefix)]dit_config["num_attention_heads"] = qkv.shape[0] // (3 * dit_config["attention_head_dim"])