The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Refactor to make it easier to add custom conds to models.
comfyanonymous036f88c62166
diff --git a/comfy/conds.py b/comfy/conds.pynew file mode 100644index 000000000..1e3111baf--- /dev/null+++ b/comfy/conds.py@@ -0,0 +1,64 @@+import enum+import torch+import math+import comfy.utils+++def lcm(a, b): #TODO: eventually replace by math.lcm (added in python3.9)+ return abs(a*b) // math.gcd(a, b)++class CONDRegular:+ def __init__(self, cond):+ self.cond = cond++ def _copy_with(self, cond):+ return self.__class__(cond)++ def process_cond(self, batch_size, device, **kwargs):+ return self._copy_with(comfy.utils.repeat_to_batch_size(self.cond, batch_size).to(device))++ def can_concat(self, other):+ if self.cond.shape != other.cond.shape:+ return False+ return True++ def concat(self, others):+ conds = [self.cond]+ for x in others:+ conds.append(x.cond)+ return torch.cat(conds)++class CONDNoiseShape(CONDRegular):+ def process_cond(self, batch_size, device, area, **kwargs):+ data = self.cond[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]+ return self._copy_with(comfy.utils.repeat_to_batch_size(data, batch_size).to(device))+++class CONDCrossAttn(CONDRegular):+ def can_concat(self, other):+ s1 = self.cond.shape+ s2 = other.cond.shape+ if s1 != s2:+ if s1[0] != s2[0] or s1[2] != s2[2]: #these 2 cases should not happen+ return False++ mult_min = lcm(s1[1], s2[1])+ diff = mult_min // min(s1[1], s2[1])+ if diff > 4: #arbitrary limit on the padding because it's probably going to impact performance negatively if it's too much+ return False+ return True++ def concat(self, others):+ conds = [self.cond]+ crossattn_max_len = self.cond.shape[1]+ for x in others:+ c = x.cond+ crossattn_max_len = lcm(crossattn_max_len, c.shape[1])+ conds.append(c)++ out = []+ for c in conds:+ if c.shape[1] < crossattn_max_len:+ c = c.repeat(1, crossattn_max_len // c.shape[1], 1) #padding with repeat doesn't change result+ out.append(c)+ return torch.cat(out)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex cda6765e4..edc246f8c 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -4,6 +4,7 @@ from comfy.ldm.modules.encoders.noise_aug_modules import CLIPEmbeddingNoiseAugmefrom comfy.ldm.modules.diffusionmodules.util import make_beta_schedulefrom comfy.ldm.modules.diffusionmodules.openaimodel import Timestepimport comfy.model_management+import comfy.condsimport numpy as npfrom enum import Enumfrom . import utils@@ -49,7 +50,7 @@ class BaseModel(torch.nn.Module):self.register_buffer('alphas_cumprod', torch.tensor(alphas_cumprod, dtype=torch.float32))self.register_buffer('alphas_cumprod_prev', torch.tensor(alphas_cumprod_prev, dtype=torch.float32))- def apply_model(self, x, t, c_concat=None, c_crossattn=None, c_adm=None, control=None, transformer_options={}):+ def apply_model(self, x, t, c_concat=None, c_crossattn=None, c_adm=None, control=None, transformer_options={}, **kwargs):if c_concat is not None:xc = torch.cat([x] + [c_concat], dim=1)else:@@ -72,7 +73,8 @@ class BaseModel(torch.nn.Module):def encode_adm(self, **kwargs):return None- def cond_concat(self, **kwargs):+ def extra_conds(self, **kwargs):+ out = {}if self.inpaint_model:concat_keys = ("mask", "masked_image")cond_concat = []@@ -101,8 +103,12 @@ class BaseModel(torch.nn.Module):cond_concat.append(torch.ones_like(noise)[:,:1])elif ck == "masked_image":cond_concat.append(blank_inpaint_image_like(noise))- return cond_concat- return None+ data = torch.cat(cond_concat, dim=1)+ out['c_concat'] = comfy.conds.CONDNoiseShape(data)+ adm = self.encode_adm(**kwargs)+ if adm is not None:+ out['c_adm'] = comfy.conds.CONDRegular(adm)+ return outdef load_model_weights(self, sd, unet_prefix=""):to_load = {}diff --git a/comfy/sample.py b/comfy/sample.pyindex e6a69973d..b3fcd1658 100644--- a/comfy/sample.py+++ b/comfy/sample.py@@ -1,6 +1,7 @@import torchimport comfy.model_managementimport comfy.samplers+import comfy.condsimport comfy.utilsimport mathimport numpy as np@@ -33,22 +34,24 @@ def prepare_mask(noise_mask, shape, device):noise_mask = noise_mask.to(device)return noise_mask-def broadcast_cond(cond, batch, device):- """broadcasts conditioning to the batch size"""- copy = []- for p in cond:- t = comfy.utils.repeat_to_batch_size(p[0], batch)- t = t.to(device)- copy += [[t] + p[1:]]- return copy-def get_models_from_cond(cond, model_type):models = []for c in cond:- if model_type in c[1]:- models += [c[1][model_type]]+ if model_type in c:+ models += [c[model_type]]return models+def convert_cond(cond):+ out = []+ for c in cond:+ temp = c[1].copy()+ model_conds = temp.get("model_conds", {})+ if c[0] is not None:+ model_conds["c_crossattn"] = comfy.conds.CONDCrossAttn(c[0])+ temp["model_conds"] = model_conds+ out.append(temp)+ return out+def get_additional_models(positive, negative, dtype):"""loads additional models in positive and negative conditioning"""control_nets = set(get_models_from_cond(positive, "control") + get_models_from_cond(negative, "control"))@@ -72,6 +75,8 @@ def cleanup_additional_models(models):def prepare_sampling(model, noise_shape, positive, negative, noise_mask):device = model.load_device+ positive = convert_cond(positive)+ negative = convert_cond(negative)if noise_mask is not None:noise_mask = prepare_mask(noise_mask, noise_shape, device)@@ -81,9 +86,7 @@ def prepare_sampling(model, noise_shape, positive, negative, noise_mask):comfy.model_management.load_models_gpu([model] + models, comfy.model_management.batch_area_memory(noise_shape[0] * noise_shape[2] * noise_shape[3]) + inference_memory)real_model = model.model- positive_copy = broadcast_cond(positive, noise_shape[0], device)- negative_copy = broadcast_cond(negative, noise_shape[0], device)- return real_model, positive_copy, negative_copy, noise_mask, models+ return real_model, positive, negative, noise_mask, modelsdef sample(model, noise, steps, cfg, sampler_name, scheduler, positive, negative, latent_image, denoise=1.0, disable_noise=False, start_step=None, last_step=None, force_full_denoise=False, noise_mask=None, sigmas=None, callback=None, disable_pbar=False, seed=None):diff --git a/comfy/samplers.py b/comfy/samplers.pyindex f88b790d8..f930aa39b 100644--- a/comfy/samplers.py+++ b/comfy/samplers.py@@ -2,96 +2,44 @@ from .k_diffusion import sampling as k_diffusion_samplingfrom .k_diffusion import external as k_diffusion_externalfrom .extra_samplers import uni_pcimport torch+import enumfrom comfy import model_managementfrom .ldm.models.diffusion.ddim import DDIMSamplerfrom .ldm.modules.diffusionmodules.util import make_ddim_timestepsimport mathfrom comfy import model_baseimport comfy.utils---def lcm(a, b): #TODO: eventually replace by math.lcm (added in python3.9)- return abs(a*b) // math.gcd(a, b)--class CONDRegular:- def __init__(self, cond):- self.cond = cond-- def can_concat(self, other):- if self.cond.shape != other.cond.shape:- return False- return True-- def concat(self, others):- conds = [self.cond]- for x in others:- conds.append(x.cond)- return torch.cat(conds)--class CONDCrossAttn:- def __init__(self, cond):- self.cond = cond-- def can_concat(self, other):- s1 = self.cond.shape- s2 = other.cond.shape- if s1 != s2:- if s1[0] != s2[0] or s1[2] != s2[2]: #these 2 cases should not happen- return False-- mult_min = lcm(s1[1], s2[1])- diff = mult_min // min(s1[1], s2[1])- if diff > 4: #arbitrary limit on the padding because it's probably going to impact performance negatively if it's too much- return False- return True-- def concat(self, others):- conds = [self.cond]- crossattn_max_len = self.cond.shape[1]- for x in others:- c = x.cond- crossattn_max_len = lcm(crossattn_max_len, c.shape[1])- conds.append(c)-- out = []- for c in conds:- if c.shape[1] < crossattn_max_len:- c = c.repeat(1, crossattn_max_len // c.shape[1], 1) #padding with repeat doesn't change result- out.append(c)- return torch.cat(out)+import comfy.conds#The main sampling function shared by all the samplers#Returns predicted noisedef sampling_function(model_function, x, timestep, uncond, cond, cond_scale, model_options={}, seed=None):- def get_area_and_mult(cond, x_in, timestep_in):+ def get_area_and_mult(conds, x_in, timestep_in):area = (x_in.shape[2], x_in.shape[3], 0, 0)strength = 1.0- if 'timestep_start' in cond[1]:- timestep_start = cond[1]['timestep_start']++ if 'timestep_start' in conds:+ timestep_start = conds['timestep_start']if timestep_in[0] > timestep_start:return None- if 'timestep_end' in cond[1]:- timestep_end = cond[1]['timestep_end']+ if 'timestep_end' in conds:+ timestep_end = conds['timestep_end']if timestep_in[0] < timestep_end:return None- if 'area' in cond[1]:- area = cond[1]['area']- if 'strength' in cond[1]:- strength = cond[1]['strength']-- adm_cond = None- if 'adm_encoded' in cond[1]:- adm_cond = cond[1]['adm_encoded']+ if 'area' in conds:+ area = conds['area']+ if 'strength' in conds:+ strength = conds['strength']input_x = x_in[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]- if 'mask' in cond[1]:+ if 'mask' in conds:# Scale the mask to the size of the input# The mask should have been resized as we began the sampling processmask_strength = 1.0- if "mask_strength" in cond[1]:- mask_strength = cond[1]["mask_strength"]- mask = cond[1]['mask']+ if "mask_strength" in conds:+ mask_strength = conds["mask_strength"]+ mask = conds['mask']assert(mask.shape[1] == x_in.shape[2])assert(mask.shape[2] == x_in.shape[3])mask = mask[:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]] * mask_strength@@ -100,7 +48,7 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, modmask = torch.ones_like(input_x)mult = mask * strength- if 'mask' not in cond[1]:+ if 'mask' not in conds:rr = 8if area[2] != 0:for t in range(rr):@@ -116,27 +64,17 @@ def sampling_function(model_function, x, timestep, uncond, cond, cond_scale, modmult[:,:,:,area[1] - 1 - t:area[1] - t] *= ((1.0/rr) * (t + 1))conditionning = {}- conditionning['c_crossattn'] = CONDCrossAttn(cond[0])-- if 'concat' in cond[1]:- cond_concat_in = cond[1]['concat']- if cond_concat_in is not None and len(cond_concat_in) > 0:- cropped = []- for x in cond_concat_in:- cr = x[:,:,area[2]:area[0] + area[2],area[3]:area[1] + area[3]]- cropped.append(cr)- conditionning['c_concat'] = CONDRegular(torch.cat(cropped, dim=1))-- if adm_cond is not None:- conditionning['c_adm'] = CONDRegular(adm_cond)+ model_conds = conds["model_conds"]+ for c in model_conds:+ conditionning[c] = model_conds[c].process_cond(batch_size=x_in.shape[0], device=x_in.device, area=area)control = None- if 'control' in cond[1]:- control = cond[1]['control']+ if 'control' in conds:+ control = conds['control']patches = None- if 'gligen' in cond[1]:- gligen = cond[1]['gligen']+ if 'gligen' in conds:+ gligen = conds['gligen']patches = {}gligen_type = gligen[0]gligen_model = gligen[1]@@ -412,19 +350,19 @@ def resolve_areas_and_cond_masks(conditions, h, w, device):# While we're doing this, we can also resolve the mask device and scaling for performance reasonsfor i in range(len(conditions)):c = conditions[i]- if 'area' in c[1]:- area = c[1]['area']+ if 'area' in c:+ area = c['area']if area[0] == "percentage":- modified = c[1].copy()+ modified = c.copy()area = (max(1, round(area[1] * h)), max(1, round(area[2] * w)), round(area[3] * h), round(area[4] * w))modified['area'] = area- c = [c[0], modified]+ c = modifiedconditions[i] = c- if 'mask' in c[1]:- mask = c[1]['mask']+ if 'mask' in c:+ mask = c['mask']mask = mask.to(device=device)- modified = c[1].copy()+ modified = c.copy()if len(mask.shape) == 2:mask = mask.unsqueeze(0)if mask.shape[1] != h or mask.shape[2] != w:@@ -445,37 +383,39 @@ def resolve_areas_and_cond_masks(conditions, h, w, device):modified['area'] = areamodified['mask'] = mask- conditions[i] = [c[0], modified]+ conditions[i] = modifieddef create_cond_with_same_area_if_none(conds, c):- if 'area' not in c[1]:+ if 'area' not in c:return- c_area = c[1]['area']+ c_area = c['area']smallest = Nonefor x in conds:- if 'area' in x[1]:- a = x[1]['area']+ if 'area' in x:+ a = x['area']if c_area[2] >= a[2] and c_area[3] >= a[3]:if a[0] + a[2] >= c_area[0] + c_area[2]:if a[1] + a[3] >= c_area[1] + c_area[3]:if smallest is None:smallest = x- elif 'area' not in smallest[1]:+ elif 'area' not in smallest:smallest = xelse:- if smallest[1]['area'][0] * smallest[1]['area'][1] > a[0] * a[1]:+ if smallest['area'][0] * smallest['area'][1] > a[0] * a[1]:smallest = xelse:if smallest is None:smallest = xif smallest is None:return- if 'area' in smallest[1]:- if smallest[1]['area'] == c_area:+ if 'area' in smallest:+ if smallest['area'] == c_area:return- n = c[1].copy()- conds += [[smallest[0], n]]++ out = c.copy()+ out['model_conds'] = smallest['model_conds'].copy() #TODO: which fields should be copied?+ conds += [out]def calculate_start_end_timesteps(model, conds):for t in range(len(conds)):@@ -483,18 +423,18 @@ def calculate_start_end_timesteps(model, conds):timestep_start = Nonetimestep_end = None- if 'start_percent' in x[1]:- timestep_start = model.sigma_to_t(model.t_to_sigma(torch.tensor(x[1]['start_percent'] * 999.0)))- if 'end_percent' in x[1]:- timestep_end = model.sigma_to_t(model.t_to_sigma(torch.tensor(x[1]['end_percent'] * 999.0)))+ if 'start_percent' in x:+ timestep_start = model.sigma_to_t(model.t_to_sigma(torch.tensor(x['start_percent'] * 999.0)))+ if 'end_percent' in x:+ timestep_end = model.sigma_to_t(model.t_to_sigma(torch.tensor(x['end_percent'] * 999.0)))if (timestep_start is not None) or (timestep_end is not None):- n = x[1].copy()+ n = x.copy()if (timestep_start is not None):n['timestep_start'] = timestep_startif (timestep_end is not None):n['timestep_end'] = timestep_end- conds[t] = [x[0], n]+ conds[t] = ndef pre_run_control(model, conds):for t in range(len(conds)):@@ -503,8 +443,8 @@ def pre_run_control(model, conds):timestep_start = Nonetimestep_end = Nonepercent_to_timestep_function = lambda a: model.sigma_to_t(model.t_to_sigma(torch.tensor(a) * 999.0))- if 'control' in x[1]:- x[1]['control'].pre_run(model.inner_model.inner_model, percent_to_timestep_function)+ if 'control' in x:+ x['control'].pre_run(model.inner_model.inner_model, percent_to_timestep_function)def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func):cond_cnets = []@@ -513,16 +453,16 @@ def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func):uncond_other = []for t in range(len(conds)):x = conds[t]- if 'area' not in x[1]:- if name in x[1] and x[1][name] is not None:- cond_cnets.append(x[1][name])+ if 'area' not in x:+ if name in x and x[name] is not None:+ cond_cnets.append(x[name])else:cond_other.append((x, t))for t in range(len(uncond)):x = uncond[t]- if 'area' not in x[1]:- if name in x[1] and x[1][name] is not None:- uncond_cnets.append(x[1][name])+ if 'area' not in x:+ if name in x and x[name] is not None:+ uncond_cnets.append(x[name])else:uncond_other.append((x, t))@@ -532,47 +472,35 @@ def apply_empty_x_to_equal_area(conds, uncond, name, uncond_fill_func):for x in range(len(cond_cnets)):temp = uncond_other[x % len(uncond_other)]o = temp[0]- if name in o[1] and o[1][name] is not None:- n = o[1].copy()+ if name in o and o[name] is not None:+ n = o.copy()n[name] = uncond_fill_func(cond_cnets, x)- uncond += [[o[0], n]]+ uncond += [n]else:- n = o[1].copy()+ n = o.copy()n[name] = uncond_fill_func(cond_cnets, x)- uncond[temp[1]] = [o[0], n]--def encode_adm(model, conds, batch_size, width, height, device, prompt_type):- for t in range(len(conds)):- x = conds[t]- adm_out = None- if 'adm' in x[1]:- adm_out = x[1]["adm"]- else:- params = x[1].copy()- params["width"] = params.get("width", width * 8)- params["height"] = params.get("height", height * 8)- params["prompt_type"] = params.get("prompt_type", prompt_type)- adm_out = model.encode_adm(device=device, **params)-- if adm_out is not None:- x[1] = x[1].copy()- x[1]["adm_encoded"] = comfy.utils.repeat_to_batch_size(adm_out, batch_size).to(device)+ uncond[temp[1]] = n- return conds--def encode_cond(model_function, key, conds, device, **kwargs):+def encode_model_conds(model_function, conds, noise, device, prompt_type, **kwargs):for t in range(len(conds)):x = conds[t]- params = x[1].copy()+ params = x.copy()params["device"] = device+ params["noise"] = noise+ params["width"] = params.get("width", noise.shape[3] * 8)+ params["height"] = params.get("height", noise.shape[2] * 8)+ params["prompt_type"] = params.get("prompt_type", prompt_type)for k in kwargs:if k not in params:params[k] = kwargs[k]out = model_function(**params)- if out is not None:- x[1] = x[1].copy()- x[1][key] = out+ x = x.copy()+ model_conds = x['model_conds'].copy()+ for k in out:+ model_conds[k] = out[k]+ x['model_conds'] = model_conds+ conds[t] = xreturn condsclass Sampler:@@ -690,19 +618,15 @@ def sample(model, noise, positive, negative, cfg, device, sampler, sigmas, modelpre_run_control(model_wrap, negative + positive)- apply_empty_x_to_equal_area(list(filter(lambda c: c[1].get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])+ apply_empty_x_to_equal_area(list(filter(lambda c: c.get('control_apply_to_uncond', False) == True, positive)), negative, 'control', lambda cond_cnets, x: cond_cnets[x])apply_empty_x_to_equal_area(positive, negative, 'gligen', lambda cond_cnets, x: cond_cnets[x])if latent_image is not None:latent_image = model.process_latent_in(latent_image)- if model.is_adm():- positive = encode_adm(model, positive, noise.shape[0], noise.shape[3], noise.shape[2], device, "positive")- negative = encode_adm(model, negative, noise.shape[0], noise.shape[3], noise.shape[2], device, "negative")-- if hasattr(model, 'cond_concat'):- positive = encode_cond(model.cond_concat, "concat", positive, device, noise=noise, latent_image=latent_image, denoise_mask=denoise_mask)- negative = encode_cond(model.cond_concat, "concat", negative, device, noise=noise, latent_image=latent_image, denoise_mask=denoise_mask)+ if hasattr(model, 'extra_conds'):+ positive = encode_model_conds(model.extra_conds, positive, noise, device, "positive", latent_image=latent_image, denoise_mask=denoise_mask)+ negative = encode_model_conds(model.extra_conds, negative, noise, device, "negative", latent_image=latent_image, denoise_mask=denoise_mask)extra_args = {"cond":positive, "uncond":negative, "cond_scale": cfg, "model_options": model_options, "seed":seed}