The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add Condition by Mask node
Jacob Segale214c917ae88
diff --git a/comfy/samplers.py b/comfy/samplers.pyindex fc19ddcfc..6fa754b90 100644--- a/comfy/samplers.py+++ b/comfy/samplers.py@@ -6,6 +6,7 @@ import contextlibfrom comfy import model_managementfrom .ldm.models.diffusion.ddim import DDIMSamplerfrom .ldm.modules.diffusionmodules.util import make_ddim_timesteps+from torchvision.ops import masks_to_boxes#The main sampling function shared by all the samplers#Returns predicted noise@@ -23,21 +24,34 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, conadm_cond = cond[1]['adm_encoded']input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]- mult = torch.ones_like(input_x) * strength-- rr = 8- if area[2] != 0:- for t in range(rr):- mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))- if (area[0] + area[2]) < x_in.shape[2]:- for t in range(rr):- mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))- if area[3] != 0:- for t in range(rr):- mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))- if (area[1] + area[3]) < x_in.shape[3]:- for t in range(rr):- mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))+ if 'mask' in cond[1]:+ # Scale the mask to the size of the input+ # The mask should have been resized as we began the sampling process+ mask = cond[1]['mask']+ assert(mask.shape[1] == x_in.shape[2])+ assert(mask.shape[2] == x_in.shape[3])+ mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]+ if mask.shape[0] != input_x.shape[0]:+ mask = mask.repeat(input_x.shape[0], 1, 1)+ else:+ mask = torch.ones_like(input_x)+ mult = mask * strength++ if 'mask' not in cond[1]:+ rr = 8+ if area[2] != 0:+ for t in range(rr):+ mult[:,:,t:1+t,:] *= ((1.0/rr) * (t + 1))+ if (area[0] + area[2]) < x_in.shape[2]:+ for t in range(rr):+ mult[:,:,area[0] - 1 - t:area[0] - t,:] *= ((1.0/rr) * (t + 1))+ if area[3] != 0:+ for t in range(rr):+ mult[:,:,:,t:1+t] *= ((1.0/rr) * (t + 1))+ if (area[1] + area[3]) < x_in.shape[3]:+ for t in range(rr):+ mult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))+conditionning = {}conditionning['c_crossattn'] = cond[0]if cond_concat_in is not None and len(cond_concat_in) > 0:@@ -301,6 +315,47 @@ def blank_inpaint_image_like(latent_image):blank_image[:,3] *= 0.1380return blank_image+def resolve_cond_masks(conditions, h, w, device):+ # We need to decide on an area outside the sampling loop in order to properly generate opposite areas of equal sizes.+ # While we're doing this, we can also resolve the mask device and scaling for performance reasons+ for i in range(len(conditions)):+ c = conditions[i]+ if 'mask' in c[1]:+ mask = c[1]['mask']+ mask = mask.to(device=device)+ modified = c[1].copy()+ if len(mask.shape) == 2:+ mask = mask.unsqueeze(0)+ if mask.shape[2] != h or mask.shape[3] != w:+ mask = torch.nn.functional.interpolate(mask.unsqueeze(1), size=(h, w), mode='bilinear', align_corners=False).squeeze(1)++ if 'area' not in modified:+ bounds = torch.max(torch.abs(mask),dim=0).values.unsqueeze(0)+ if torch.max(bounds) == 0:+ # Handle the edge-case of an all black mask (where masks_to_boxes would error)+ area = (0, 0, 0, 0)+ else:+ box = masks_to_boxes(bounds)[0].type(torch.int)+ H, W, Y, X = (box[3] - box[1] + 1, box[2] - box[0] + 1, box[1], box[0])+ # Make sure the height and width are divisible by 8+ if X % 8 != 0:+ newx = X // 8 * 8+ W = W + (X - newx)+ X = newx+ if Y % 8 != 0:+ newy = Y // 8 * 8+ H = H + (Y - newy)+ Y = newy+ if H % 8 != 0:+ H = H + (8 - (H % 8))+ if W % 8 != 0:+ W = W + (8 - (W % 8))+ area = (int(H), int(W), int(Y), (X))+ modified['area'] = area++ modified['mask'] = mask+ conditions[i] = [c[0], modified]+def create_cond_with_same_area_if_none(conds, c):if 'area' not in c[1]:return@@ -461,7 +516,6 @@ class KSampler:sigmas = self.calculate_sigmas(new_steps).to(self.device)self.sigmas = sigmas[-(steps + 1):]-def sample(self, noise, positive, negative, cfg, latent_image=None, start_step=None, last_step=None, force_full_denoise=False, denoise_mask=None, sigmas=None, callback=None):if sigmas is None:sigmas = self.sigmas@@ -484,6 +538,10 @@ class KSampler:positive = positive[:]negative = negative[:]++ resolve_cond_masks(positive, noise.shape[2], noise.shape[3], self.device)+ resolve_cond_masks(negative, noise.shape[2], noise.shape[3], self.device)+#make sure each cond area has an opposite one with the same areafor c in positive:create_cond_with_same_area_if_none(negative, c)diff --git a/nodes.py b/nodes.pyindex 0a9513bed..be02f4676 100644--- a/nodes.py+++ b/nodes.py@@ -85,6 +85,32 @@ class ConditioningSetArea:c.append(n)return (c, )+class ConditioningSetMask:+ @classmethod+ def INPUT_TYPES(s):+ return {"required": {"conditioning": ("CONDITIONING", ),+ "mask": ("MASK", ),+ "strength": ("FLOAT", {"default": 1.0, "min": 0.0, "max": 10.0, "step": 0.01}),+ }}+ RETURN_TYPES = ("CONDITIONING",)+ FUNCTION = "append"++ CATEGORY = "conditioning"++ def append(self, conditioning, mask, strength, min_sigma=0.0, max_sigma=99.0):+ c = []+ if len(mask.shape) < 3:+ mask = mask.unsqueeze(0)+ for t in conditioning:+ n = [t[0], t[1].copy()]+ _, h, w = mask.shape+ n[1]['mask'] = mask+ n[1]['strength'] = strength+ n[1]['min_sigma'] = min_sigma+ n[1]['max_sigma'] = max_sigma+ c.append(n)+ return (c, )+class VAEDecode:def __init__(self, device="cpu"):self.device = device@@ -1115,6 +1141,7 @@ NODE_CLASS_MAPPINGS = {"ImagePadForOutpaint": ImagePadForOutpaint,"ConditioningCombine": ConditioningCombine,"ConditioningSetArea": ConditioningSetArea,+ "ConditioningSetMask": ConditioningSetMask,"KSamplerAdvanced": KSamplerAdvanced,"SetLatentNoiseMask": SetLatentNoiseMask,"LatentComposite": LatentComposite,@@ -1164,6 +1191,7 @@ NODE_DISPLAY_NAME_MAPPINGS = {"CLIPSetLastLayer": "CLIP Set Last Layer","ConditioningCombine": "Conditioning (Combine)","ConditioningSetArea": "Conditioning (Set Area)",+ "ConditioningSetMask": "Conditioning (Set Mask)","ControlNetApply": "Apply ControlNet",# Latent"VAEEncodeForInpaint": "VAE Encode (for Inpainting)",