The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Allow image_only_indicator to be None.
comfyanonymouscb7c3a2921cf
diff --git a/comfy/ldm/modules/diffusionmodules/openaimodel.py b/comfy/ldm/modules/diffusionmodules/openaimodel.pyindex c54770255..cf89ae017 100644--- a/comfy/ldm/modules/diffusionmodules/openaimodel.py+++ b/comfy/ldm/modules/diffusionmodules/openaimodel.py@@ -484,7 +484,6 @@ class UNetModel(nn.Module):self.predict_codebook_ids = n_embed is not Noneself.default_num_video_frames = None- self.default_image_only_indicator = Nonetime_embed_dim = model_channels * 4self.time_embed = nn.Sequential(@@ -830,7 +829,7 @@ class UNetModel(nn.Module):transformer_patches = transformer_options.get("patches", {})num_video_frames = kwargs.get("num_video_frames", self.default_num_video_frames)- image_only_indicator = kwargs.get("image_only_indicator", self.default_image_only_indicator)+ image_only_indicator = kwargs.get("image_only_indicator", None)time_context = kwargs.get("time_context", None)assert (y is not None) == (diff --git a/comfy/ldm/modules/diffusionmodules/util.py b/comfy/ldm/modules/diffusionmodules/util.pyindex 5a6aa7d77..ce14ad5e1 100644--- a/comfy/ldm/modules/diffusionmodules/util.py+++ b/comfy/ldm/modules/diffusionmodules/util.py@@ -46,23 +46,25 @@ class AlphaBlender(nn.Module):else:raise ValueError(f"unknown merge strategy {self.merge_strategy}")- def get_alpha(self, image_only_indicator: torch.Tensor) -> torch.Tensor:+ def get_alpha(self, image_only_indicator: torch.Tensor, device) -> torch.Tensor:# skip_time_mix = rearrange(repeat(skip_time_mix, 'b -> (b t) () () ()', t=t), '(b t) 1 ... -> b 1 t ...', t=t)if self.merge_strategy == "fixed":# make shape compatible# alpha = repeat(self.mix_factor, '1 -> b () t () ()', t=t, b=bs)- alpha = self.mix_factor.to(image_only_indicator.device)+ alpha = self.mix_factor.to(device)elif self.merge_strategy == "learned":- alpha = torch.sigmoid(self.mix_factor.to(image_only_indicator.device))+ alpha = torch.sigmoid(self.mix_factor.to(device))# make shape compatible# alpha = repeat(alpha, '1 -> s () ()', s = t * bs)elif self.merge_strategy == "learned_with_images":- assert image_only_indicator is not None, "need image_only_indicator ..."- alpha = torch.where(- image_only_indicator.bool(),- torch.ones(1, 1, device=image_only_indicator.device),- rearrange(torch.sigmoid(self.mix_factor.to(image_only_indicator.device)), "... -> ... 1"),- )+ if image_only_indicator is None:+ alpha = rearrange(torch.sigmoid(self.mix_factor.to(device)), "... -> ... 1")+ else:+ alpha = torch.where(+ image_only_indicator.bool(),+ torch.ones(1, 1, device=image_only_indicator.device),+ rearrange(torch.sigmoid(self.mix_factor.to(image_only_indicator.device)), "... -> ... 1"),+ )alpha = rearrange(alpha, self.rearrange_pattern)# make shape compatible# alpha = repeat(alpha, '1 -> s () ()', s = t * bs)@@ -76,7 +78,7 @@ class AlphaBlender(nn.Module):x_temporal,image_only_indicator=None,) -> torch.Tensor:- alpha = self.get_alpha(image_only_indicator)+ alpha = self.get_alpha(image_only_indicator, x_spatial.device)x = (alpha.to(x_spatial.dtype) * x_spatial+ (1.0 - alpha).to(x_spatial.dtype) * x_temporaldiff --git a/comfy/model_base.py b/comfy/model_base.pyindex 170b1fd44..69117dd0e 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -372,7 +372,6 @@ class SVD_img2vid(BaseModel):if "time_conditioning" in kwargs:out["time_context"] = comfy.conds.CONDCrossAttn(kwargs["time_conditioning"])- out['image_only_indicator'] = comfy.conds.CONDConstant(torch.zeros((1,), device=device))out['num_video_frames'] = comfy.conds.CONDConstant(noise.shape[0])return out