Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support the HuMo 17B model. (#9912)

comfyanonymousdd611a770095
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.py
index b3b7da5d5..9cf3c171d 100644
--- a/comfy/ldm/wan/model.py
+++ b/comfy/ldm/wan/model.py
@@ -1364,7 +1364,7 @@ class AudioCrossAttentionWrapper(nn.Module):
def __init__(self, dim, kv_dim, num_heads, qk_norm=True, eps=1e-6, operation_settings={}):
super().__init__()
- self.audio_cross_attn = WanT2VCrossAttentionGather(dim, num_heads, qk_norm, kv_dim, eps, operation_settings=operation_settings)
+ self.audio_cross_attn = WanT2VCrossAttentionGather(dim, num_heads, qk_norm=qk_norm, kv_dim=kv_dim, eps=eps, operation_settings=operation_settings)
self.norm1_audio = operation_settings.get("operations").LayerNorm(dim, eps, elementwise_affine=True, device=operation_settings.get("device"), dtype=operation_settings.get("dtype"))
def forward(self, x, audio, transformer_options={}):
diff --git a/comfy/model_base.py b/comfy/model_base.py
index cf99035da..70b67b7c1 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -1220,14 +1220,37 @@ class WAN21_HuMo(WAN21):
def extra_conds(self, **kwargs):
out = super().extra_conds(**kwargs)
+ noise = kwargs.get("noise", None)
audio_embed = kwargs.get("audio_embed", None)
if audio_embed is not None:
out['audio_embed'] = comfy.conds.CONDRegular(audio_embed)
- reference_latents = kwargs.get("reference_latents", None)
- if reference_latents is not None:
- out['reference_latent'] = comfy.conds.CONDRegular(self.process_latent_in(reference_latents[-1]))
+ if "c_concat" not in out: # 1.7B model
+ reference_latents = kwargs.get("reference_latents", None)
+ if reference_latents is not None:
+ out['reference_latent'] = comfy.conds.CONDRegular(self.process_latent_in(reference_latents[-1]))
+ else:
+ noise_shape = list(noise.shape)
+ noise_shape[1] += 4
+ concat_latent = torch.zeros(noise_shape, device=noise.device, dtype=noise.dtype)
+ zero_vae_values_first = torch.tensor([0.8660, -0.4326, -0.0017, -0.4884, -0.5283, 0.9207, -0.9896, 0.4433, -0.5543, -0.0113, 0.5753, -0.6000, -0.8346, -0.3497, -0.1926, -0.6938]).view(1, 16, 1, 1, 1)
+ zero_vae_values_second = torch.tensor([1.0869, -1.2370, 0.0206, -0.4357, -0.6411, 2.0307, -1.5972, 1.2659, -0.8595, -0.4654, 0.9638, -1.6330, -1.4310, -0.1098, -0.3856, -1.4583]).view(1, 16, 1, 1, 1)
+ zero_vae_values = torch.tensor([0.8642, -1.8583, 0.1577, 0.1350, -0.3641, 2.5863, -1.9670, 1.6065, -1.0475, -0.8678, 1.1734, -1.8138, -1.5933, -0.7721, -0.3289, -1.3745]).view(1, 16, 1, 1, 1)
+ concat_latent[:, 4:] = zero_vae_values
+ concat_latent[:, 4:, :1] = zero_vae_values_first
+ concat_latent[:, 4:, 1:2] = zero_vae_values_second
+ out['c_concat'] = comfy.conds.CONDNoiseShape(concat_latent)
+ reference_latents = kwargs.get("reference_latents", None)
+ if reference_latents is not None:
+ ref_latent = self.process_latent_in(reference_latents[-1])
+ ref_latent_shape = list(ref_latent.shape)
+ ref_latent_shape[1] += 4 + ref_latent_shape[1]
+ ref_latent_full = torch.zeros(ref_latent_shape, device=ref_latent.device, dtype=ref_latent.dtype)
+ ref_latent_full[:, 20:] = ref_latent
+ ref_latent_full[:, 16:20] = 1.0
+ out['reference_latent'] = comfy.conds.CONDRegular(ref_latent_full)
+
return out
class WAN22_S2V(WAN21):