The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
initial FlowRVS support (#12637)
Jukka Seppänen907e5dcbbffa
diff --git a/comfy/ldm/wan/vae.py b/comfy/ldm/wan/vae.pyindex fd125ceed..7903c7690 100644--- a/comfy/ldm/wan/vae.py+++ b/comfy/ldm/wan/vae.py@@ -459,6 +459,7 @@ class WanVAE(nn.Module):attn_scales=[],temperal_downsample=[True, True, False],image_channels=3,+ conv_out_channels=3,dropout=0.0):super().__init__()self.dim = dim@@ -474,7 +475,7 @@ class WanVAE(nn.Module):attn_scales, self.temperal_downsample, dropout)self.conv1 = CausalConv3d(z_dim * 2, z_dim * 2, 1)self.conv2 = CausalConv3d(z_dim, z_dim, 1)- self.decoder = Decoder3d(dim, z_dim, image_channels, dim_mult, num_res_blocks,+ self.decoder = Decoder3d(dim, z_dim, conv_out_channels, dim_mult, num_res_blocks,attn_scales, self.temperal_upsample, dropout)def encode(self, x):diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 2f49578f6..4e2096d4b 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -76,6 +76,7 @@ class ModelType(Enum):FLUX = 8IMG_TO_IMG = 9FLOW_COSMOS = 10+ IMG_TO_IMG_FLOW = 11def model_sampling(model_config, model_type):@@ -108,6 +109,8 @@ def model_sampling(model_config, model_type):elif model_type == ModelType.FLOW_COSMOS:c = comfy.model_sampling.COSMOS_RFLOWs = comfy.model_sampling.ModelSamplingCosmosRFlow+ elif model_type == ModelType.IMG_TO_IMG_FLOW:+ c = comfy.model_sampling.IMG_TO_IMG_FLOWclass ModelSampling(s, c):pass@@ -1466,6 +1469,12 @@ class WAN22(WAN21):def scale_latent_inpaint(self, sigma, noise, latent_image, **kwargs):return latent_image+class WAN21_FlowRVS(WAN21):+ def __init__(self, model_config, model_type=ModelType.IMG_TO_IMG_FLOW, image_to_video=False, device=None):+ model_config.unet_config["model_type"] = "t2v"+ super(WAN21, self).__init__(model_config, model_type, device=device, unet_model=comfy.ldm.wan.model.WanModel)+ self.image_to_video = image_to_video+class Hunyuan3Dv2(BaseModel):def __init__(self, model_config, model_type=ModelType.FLOW, device=None):super().__init__(model_config, model_type, device=device, unet_model=comfy.ldm.hunyuan3d.model.Hunyuan3Dv2)diff --git a/comfy/model_detection.py b/comfy/model_detection.pyindex 30ea03e8e..030ae6980 100644--- a/comfy/model_detection.py+++ b/comfy/model_detection.py@@ -509,6 +509,9 @@ def detect_unet_config(state_dict, key_prefix, metadata=None):if ref_conv_weight is not None:dit_config["in_dim_ref_conv"] = ref_conv_weight.shape[1]+ if metadata is not None and "config" in metadata:+ dit_config.update(json.loads(metadata["config"]).get("transformer", {}))+return dit_configif '{}latent_in.weight'.format(key_prefix) in state_dict_keys: # Hunyuan 3Ddiff --git a/comfy/model_sampling.py b/comfy/model_sampling.pyindex 2a00ed819..13860e6a2 100644--- a/comfy/model_sampling.py+++ b/comfy/model_sampling.py@@ -83,6 +83,16 @@ class IMG_TO_IMG(X0):def calculate_input(self, sigma, noise):return noise+class IMG_TO_IMG_FLOW(CONST):+ def calculate_denoised(self, sigma, model_output, model_input):+ return model_output++ def noise_scaling(self, sigma, noise, latent_image, max_denoise=False):+ return latent_image++ def inverse_noise_scaling(self, sigma, latent):+ return 1.0 - latent+class COSMOS_RFLOW:def calculate_input(self, sigma, noise):sigma = (sigma / (sigma + 1))diff --git a/comfy/sd.py b/comfy/sd.pyindex 69d4531e3..de119eb8e 100644--- a/comfy/sd.py+++ b/comfy/sd.py@@ -694,8 +694,9 @@ class VAE:self.latent_dim = 3self.latent_channels = 16self.output_channels = sd["encoder.conv1.weight"].shape[1]+ self.conv_out_channels = sd["decoder.head.2.weight"].shape[0]self.pad_channel_value = 1.0- ddconfig = {"dim": dim, "z_dim": self.latent_channels, "dim_mult": [1, 2, 4, 4], "num_res_blocks": 2, "attn_scales": [], "temperal_downsample": [False, True, True], "image_channels": self.output_channels, "dropout": 0.0}+ ddconfig = {"dim": dim, "z_dim": self.latent_channels, "dim_mult": [1, 2, 4, 4], "num_res_blocks": 2, "attn_scales": [], "temperal_downsample": [False, True, True], "image_channels": self.output_channels, "conv_out_channels": self.conv_out_channels, "dropout": 0.0}self.first_stage_model = comfy.ldm.wan.vae.WanVAE(**ddconfig)self.working_dtypes = [torch.bfloat16, torch.float16, torch.float32]self.memory_used_encode = lambda shape, dtype: (1500 if shape[2]<=4 else 6000) * shape[3] * shape[4] * model_management.dtype_size(dtype)diff --git a/comfy/supported_models.py b/comfy/supported_models.pyindex c28be1716..6d08ff0a5 100644--- a/comfy/supported_models.py+++ b/comfy/supported_models.py@@ -1256,6 +1256,16 @@ class WAN22_T2V(WAN21_T2V):out = model_base.WAN22(self, image_to_video=True, device=device)return out+class WAN21_FlowRVS(WAN21_T2V):+ unet_config = {+ "image_model": "wan2.1",+ "model_type": "flow_rvs",+ }++ def get_model(self, state_dict, prefix="", device=None):+ out = model_base.WAN21_FlowRVS(self, image_to_video=True, device=device)+ return out+class Hunyuan3Dv2(supported_models_base.BASE):unet_config = {"image_model": "hunyuan3d2",@@ -1667,6 +1677,6 @@ class ACEStep15(supported_models_base.BASE):return supported_models_base.ClipTarget(comfy.text_encoders.ace15.ACE15Tokenizer, comfy.text_encoders.ace15.te(**detect))-models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, ACEStep15, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5, Anima]+models = [LotusD, Stable_Zero123, SD15_instructpix2pix, SD15, SD20, SD21UnclipL, SD21UnclipH, SDXL_instructpix2pix, SDXLRefiner, SDXL, SSD1B, KOALA_700M, KOALA_1B, Segmind_Vega, SD_X4Upscaler, Stable_Cascade_C, Stable_Cascade_B, SV3D_u, SV3D_p, SD3, StableAudio, AuraFlow, PixArtAlpha, PixArtSigma, HunyuanDiT, HunyuanDiT1, FluxInpaint, Flux, FluxSchnell, GenmoMochi, LTXV, LTXAV, HunyuanVideo15_SR_Distilled, HunyuanVideo15, HunyuanImage21Refiner, HunyuanImage21, HunyuanVideoSkyreelsI2V, HunyuanVideoI2V, HunyuanVideo, CosmosT2V, CosmosI2V, CosmosT2IPredict2, CosmosI2VPredict2, ZImage, Lumina2, WAN22_T2V, WAN21_T2V, WAN21_I2V, WAN21_FunControl2V, WAN21_Vace, WAN21_Camera, WAN22_Camera, WAN22_S2V, WAN21_HuMo, WAN22_Animate, WAN21_FlowRVS, Hunyuan3Dv2mini, Hunyuan3Dv2, Hunyuan3Dv2_1, HiDream, Chroma, ChromaRadiance, ACEStep, ACEStep15, Omnigen2, QwenImage, Flux2, Kandinsky5Image, Kandinsky5, Anima]models += [SVD_img2vid]diff --git a/comfy_extras/nodes_model_advanced.py b/comfy_extras/nodes_model_advanced.pyindex 9601a5f76..8bf6a1afa 100644--- a/comfy_extras/nodes_model_advanced.py+++ b/comfy_extras/nodes_model_advanced.py@@ -52,7 +52,7 @@ class ModelSamplingDiscrete:@classmethoddef INPUT_TYPES(s):return {"required": { "model": ("MODEL",),- "sampling": (["eps", "v_prediction", "lcm", "x0", "img_to_img"],),+ "sampling": (["eps", "v_prediction", "lcm", "x0", "img_to_img", "img_to_img_flow"],),"zsnr": ("BOOLEAN", {"default": False, "advanced": True}),}}@@ -76,6 +76,8 @@ class ModelSamplingDiscrete:sampling_type = comfy.model_sampling.X0elif sampling == "img_to_img":sampling_type = comfy.model_sampling.IMG_TO_IMG+ elif sampling == "img_to_img_flow":+ sampling_type = comfy.model_sampling.IMG_TO_IMG_FLOWclass ModelSamplingAdvanced(sampling_base, sampling_type):pass