The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
InstantX depth flux controlnet.
comfyanonymousea3f39bd6906
diff --git a/comfy/controlnet.py b/comfy/controlnet.pyindex 0c8cd30c4..7b202b7a4 100644--- a/comfy/controlnet.py+++ b/comfy/controlnet.py@@ -148,7 +148,7 @@ class ControlBase:elif self.strength_type == StrengthType.LINEAR_UP:x *= (self.strength ** float(len(control_output) - i))- if x.dtype != output_dtype:+ if output_dtype is not None and x.dtype != output_dtype:x = x.to(output_dtype)out[key].append(x)@@ -206,7 +206,6 @@ class ControlNet(ControlBase):if self.manual_cast_dtype is not None:dtype = self.manual_cast_dtype- output_dtype = x_noisy.dtypeif self.cond_hint is None or x_noisy.shape[2] * self.compression_ratio != self.cond_hint.shape[2] or x_noisy.shape[3] * self.compression_ratio != self.cond_hint.shape[3]:if self.cond_hint is not None:del self.cond_hint@@ -236,7 +235,7 @@ class ControlNet(ControlBase):x_noisy = self.model_sampling_current.calculate_input(t, x_noisy)control = self.control_model(x=x_noisy.to(dtype), hint=self.cond_hint, timesteps=timestep.to(dtype), context=context.to(dtype), **extra)- return self.control_merge(control, control_prev, output_dtype)+ return self.control_merge(control, control_prev, output_dtype=None)def copy(self):c = ControlNet(None, global_average_pooling=self.global_average_pooling, load_device=self.load_device, manual_cast_dtype=self.manual_cast_dtype)diff --git a/comfy/ldm/flux/controlnet.py b/comfy/ldm/flux/controlnet.pyindex 0e160b075..2c658a4b1 100644--- a/comfy/ldm/flux/controlnet.py+++ b/comfy/ldm/flux/controlnet.py@@ -23,8 +23,12 @@ class ControlNetFlux(Flux):self.controlnet_blocks = nn.ModuleList([])for _ in range(self.params.depth):controlnet_block = operations.Linear(self.hidden_size, self.hidden_size, dtype=dtype, device=device)- # controlnet_block = zero_module(controlnet_block)self.controlnet_blocks.append(controlnet_block)++ self.controlnet_single_blocks = nn.ModuleList([])+ for _ in range(self.params.depth_single_blocks):+ self.controlnet_single_blocks.append(operations.Linear(self.hidden_size, self.hidden_size, dtype=dtype, device=device))+self.gradient_checkpointing = Falseself.latent_input = latent_inputself.pos_embed_input = operations.Linear(self.in_channels, self.hidden_size, bias=True, dtype=dtype, device=device)@@ -78,26 +82,39 @@ class ControlNetFlux(Flux):ids = torch.cat((txt_ids, img_ids), dim=1)pe = self.pe_embedder(ids)- block_res_samples = ()+ controlnet_double = ()++ for i in range(len(self.double_blocks)):+ img, txt = self.double_blocks[i](img=img, txt=txt, vec=vec, pe=pe)+ controlnet_double = controlnet_double + (self.controlnet_blocks[i](img),)- for block in self.double_blocks:- img, txt = block(img=img, txt=txt, vec=vec, pe=pe)- block_res_samples = block_res_samples + (img,)+ img = torch.cat((txt, img), 1)- controlnet_block_res_samples = ()- for block_res_sample, controlnet_block in zip(block_res_samples, self.controlnet_blocks):- block_res_sample = controlnet_block(block_res_sample)- controlnet_block_res_samples = controlnet_block_res_samples + (block_res_sample,)+ controlnet_single = ()+ for i in range(len(self.single_blocks)):+ img = self.single_blocks[i](img, vec=vec, pe=pe)+ controlnet_single = controlnet_single + (self.controlnet_single_blocks[i](img[:, txt.shape[1] :, ...]),)- repeat = math.ceil(self.main_model_double / len(controlnet_block_res_samples))+ repeat = math.ceil(self.main_model_double / len(controlnet_double))if self.latent_input:out_input = ()- for x in controlnet_block_res_samples:+ for x in controlnet_double:out_input += (x,) * repeatelse:- out_input = (controlnet_block_res_samples * repeat)- return {"input": out_input[:self.main_model_double]}+ out_input = (controlnet_double * repeat)++ out = {"input": out_input[:self.main_model_double]}+ if len(controlnet_single) > 0:+ repeat = math.ceil(self.main_model_single / len(controlnet_single))+ out_output = ()+ if self.latent_input:+ for x in controlnet_single:+ out_output += (x,) * repeat+ else:+ out_output = (controlnet_single * repeat)+ out["output"] = out_output[:self.main_model_single]+ return outdef forward(self, x, timesteps, context, y, guidance=None, hint=None, **kwargs):patch_size = 2