The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Implement wan2.2 camera model. (#9357)
comfyanonymous1702e6df16b0
diff --git a/comfy/ldm/wan/model.py b/comfy/ldm/wan/model.pyindex 4e2d99566..9d3741be3 100644--- a/comfy/ldm/wan/model.py+++ b/comfy/ldm/wan/model.py@@ -768,7 +768,12 @@ class CameraWanModel(WanModel):operations=None,):- super().__init__(model_type='i2v', patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, image_model=image_model, device=device, dtype=dtype, operations=operations)+ if model_type == 'camera':+ model_type = 'i2v'+ else:+ model_type = 't2v'++ super().__init__(model_type=model_type, patch_size=patch_size, text_len=text_len, in_dim=in_dim, dim=dim, ffn_dim=ffn_dim, freq_dim=freq_dim, text_dim=text_dim, out_dim=out_dim, num_heads=num_heads, num_layers=num_layers, window_size=window_size, qk_norm=qk_norm, cross_attn_norm=cross_attn_norm, eps=eps, flf_pos_embed_token_number=flf_pos_embed_token_number, image_model=image_model, device=device, dtype=dtype, operations=operations)operation_settings = {"operations": operations, "device": device, "dtype": dtype}self.control_adapter = WanCamAdapter(in_dim_control_adapter, dim, kernel_size=patch_size[1:], stride=patch_size[1:], operation_settings=operation_settings)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 8acc51e20..2bec0541e 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -364,7 +364,10 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):dit_config["vace_in_dim"] = state_dict['{}vace_patch_embedding.weight'.format(key_prefix)].shape[1]dit_config["vace_layers"] = count_blocks(state_dict_keys, '{}vace_blocks.'.format(key_prefix) + '{}.')elif '{}control_adapter.conv.weight'.format(key_prefix) in state_dict_keys:- dit_config["model_type"] = "camera"+ if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:+ dit_config["model_type"] = "camera"+ else:+ dit_config["model_type"] = "camera_2.2"else:if '{}img_emb.proj.0.bias'.format(key_prefix) in state_dict_keys:dit_config["model_type"] = "i2v"diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex 156ff9e26..7ed6dfd69 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -1046,6 +1046,18 @@ class WAN21_Camera(WAN21_T2V):def get_model(self, state_dict, prefix="", device=None):out = model_base.WAN21_Camera(self, image_to_video=False, device=device)return out++class WAN22_Camera(WAN21_T2V):+ unet_config = {+ "image_model": "wan2.1",+ "model_type": "camera_2.2",+ "in_dim": 36,+ }++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.WAN21_Camera(self, image_to_video=False, device=device)+ return out+class WAN21_Vace(WAN21_T2V):unet_config = {"image_model": "wan2.1",@@ -1260,6 +1272,6 @@ class QwenImage(supported_models_base.BASE):return supported_models_base.ClipTarget(comfy.text_encoders.qwen_image.QwenImageTokenizer, comfy.text_encoders.qwen_image.te(**hunyuan_detect))-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma, ACEStep, Omnigen2, QwenImage]+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, Hunyuan3Dv2mini, Hunyuan3Dv2, HiDream, Chroma, ACEStep, Omnigen2, QwenImage]models += [SVD_img2vid]diff --git a/comfy_extras/nodes_wan.py b/comfy_extras/nodes_wan.pyindex 694a183f6..83a990688 100644--- a/comfy_extras/nodes_wan.py+++ b/comfy_extras/nodes_wan.py@@ -422,9 +422,12 @@ class WanCameraImageToVideo(io.ComfyNode):start_image = comfy.utils.common_upscale(start_image[:length].movedim(-1, 1), width, height, "bilinear", "center").movedim(1, -1)concat_latent_image = vae.encode(start_image[:, :, :, :3])concat_latent[:,:,:concat_latent_image.shape[2]] = concat_latent_image[:,:,:concat_latent.shape[2]]+ mask = torch.ones((1, 1, latent.shape[2] * 4, latent.shape[-2], latent.shape[-1]))+ mask[:, :, :start_image.shape[0] + 3] = 0.0+ mask = mask.view(1, mask.shape[2] // 4, 4, mask.shape[3], mask.shape[4]).transpose(1, 2)- positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent})- negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent})+ positive = node_helpers.conditioning_set_values(positive, {"concat_latent_image": concat_latent, "concat_mask": mask})+ negative = node_helpers.conditioning_set_values(negative, {"concat_latent_image": concat_latent, "concat_mask": mask})if camera_conditions is not None:positive = node_helpers.conditioning_set_values(positive, {'camera_conditions': camera_conditions})