The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Controlnet refactor.
comfyanonymous66aaa14001be
diff --git a/comfy/cldm/cldm.py b/comfy/cldm/cldm.pyindex 28076dd92..f8a161594 100644--- a/comfy/cldm/cldm.py+++ b/comfy/cldm/cldm.py@@ -289,7 +289,8 @@ class ControlNet(nn.Module):guided_hint = self.input_hint_block(hint, emb, context)- outs = []+ out_output = []+ out_middle = []hs = []if self.num_classes is not None:@@ -304,10 +305,10 @@ class ControlNet(nn.Module):guided_hint = Noneelse:h = module(h, emb, context)- outs.append(zero_conv(h, emb, context))+ out_output.append(zero_conv(h, emb, context))h = self.middle_block(h, emb, context)- outs.append(self.middle_block_out(h, emb, context))+ out_middle.append(self.middle_block_out(h, emb, context))- return outs+ return {"middle": out_middle, "output": out_output}diff --git a/comfy/controlnet.py b/comfy/controlnet.pyindex 8cf4a61a6..f50df6835 100644--- a/comfy/controlnet.py+++ b/comfy/controlnet.py@@ -89,27 +89,12 @@ class ControlBase:return self.previous_controlnet.inference_memory_requirements(dtype)return 0- def control_merge(self, control_input, control_output, control_prev, output_dtype):+ def control_merge(self, control, control_prev, output_dtype):out = {'input':[], 'middle':[], 'output': []}- if control_input is not None:- for i in range(len(control_input)):- key = 'input'- x = control_input[i]- if x is not None:- x *= self.strength- if x.dtype != output_dtype:- x = x.to(output_dtype)- out[key].insert(0, x)-- if control_output is not None:+ for key in control:+ control_output = control[key]for i in range(len(control_output)):- if i == (len(control_output) - 1):- key = 'middle'- index = 0- else:- key = 'output'- index = ix = control_output[i]if x is not None:if self.global_average_pooling:@@ -120,6 +105,7 @@ class ControlBase:x = x.to(output_dtype)out[key].append(x)+if control_prev is not None:for x in ['input', 'middle', 'output']:o = out[x]@@ -182,7 +168,7 @@ class ControlNet(ControlBase):x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.float(), context=context.to(dtype), y=y)- return self.control_merge(None, control, control_prev, output_dtype)+ return self.control_merge(control, control_prev, output_dtype)def copy(self):c = ControlNet(None, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)@@ -490,12 +476,11 @@ class T2IAdapter(ControlBase):self.control_input = self.t2i_model(self.cond_hint.to(x_noisy.dtype))self.t2i_model.cpu()- control_input = list(map(lambda a: None if a is None else a.clone(), self.control_input))- mid = None- if self.t2i_model.xl == True:- mid = control_input[-1:]- control_input = control_input[:-1]- return self.control_merge(control_input, mid, control_prev, x_noisy.dtype)+ control_input = {}+ for k in self.control_input:+ control_input[k] = list(map(lambda a: None if a is None else a.clone(), self.control_input[k]))++ return self.control_merge(control_input, control_prev, x_noisy.dtype)def copy(self):c = T2IAdapter(self.t2i_model, self.channels_in, self.compression_ratio, self.upscale_algorithm)diff --git a/comfy/ldm/cascade/controlnet.py b/comfy/ldm/cascade/controlnet.pyindex 5dac59394..7a52c3c26 100644--- a/comfy/ldm/cascade/controlnet.py+++ b/comfy/ldm/cascade/controlnet.py@@ -90,4 +90,4 @@ class ControlNet(nn.Module):proj_outputs = [None for _ in range(max(self.proj_blocks) + 1)]for i, idx in enumerate(self.proj_blocks):proj_outputs[idx] = self.projections[i](x)- return proj_outputs+ return {"input": proj_outputs[::-1]}diff --git a/comfy/t2i_adapter/adapter.py b/comfy/t2i_adapter/adapter.pyindex e9a606b1c..10ea18e32 100644--- a/comfy/t2i_adapter/adapter.py+++ b/comfy/t2i_adapter/adapter.py@@ -153,7 +153,13 @@ class Adapter(nn.Module):features.append(None)features.append(x)- return features+ features = features[::-1]++ if self.xl:+ return {"input": features[1:], "middle": features[:1]}+ else:+ return {"input": features}+class LayerNorm(nn.LayerNorm):@@ -290,4 +296,4 @@ class Adapter_light(nn.Module):features.append(None)features.append(x)- return features+ return {"input": features[::-1]}