The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Allow using references with MiniMax-H3 Fun Union and fix prefetch race condition (#16020)
Jukka Seppänen02aa70781a8c
diff --git a/comfy/ldm/minimax/controlnet.py b/comfy/ldm/minimax/controlnet.pyindex 86df2e7ee..5037c8c7e 100644--- a/comfy/ldm/minimax/controlnet.py+++ b/comfy/ldm/minimax/controlnet.py@@ -42,8 +42,6 @@ class MiniMaxH3FunControl(torch.nn.Module):for i in range(len(self.injection_layers))])def init_stream(self, h, control_latent, layout, t_emb):- if any(kind not in ("text", "audio", "video") for _, _, kind in layout.segments):- raise ValueError("MiniMax H3 Fun ControlNet does not support keyframe or reference conditioning")adaln_in = self.control_blocks[0].adaln_proj.linear.in_featuresif t_emb.shape[-1] != adaln_in:raise RuntimeError(@@ -59,8 +57,13 @@ class MiniMaxH3FunControl(torch.nn.Module):elif target_rows.shape[1] > patch_dim:raise ValueError("MiniMax H3 control input has {} columns but the model patch expects {}".format(target_rows.shape[1], patch_dim))+ # keyframe/reference conditioning rows get a zero control row+ img_update = layout.img_update.to(h.device)+ rows = torch.zeros(img_update.shape[0], patch_dim, dtype=torch.float32, device=h.device)+ rows[img_update] = target_rows+c = h.clone()- c[layout.img_pos.to(h.device)] = self.control_proj_in(target_rows).to(h.dtype)+ c[layout.img_pos.to(h.device)] = self.control_proj_in(rows).to(h.dtype)return self.control_blocks[0].before_proj(c).add_(h)def step(self, index, c, t_emb, mod_segments, rope_freqs, transformer_options):diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.pyindex e7544db41..9ac827b0a 100644--- a/comfy_extras/nodes_minimax_h3.py+++ b/comfy_extras/nodes_minimax_h3.py@@ -414,6 +414,7 @@ class MiniMaxH3FunControlPatch:self.control_latent = Noneself.control_latent_shape = Noneself.control_stream = None+ self.pristine_stream = Noneself.active = Falsedef _fit_frames(self, frames, frame_count, width, height):@@ -472,26 +473,28 @@ class MiniMaxH3FunControlPatch:self.active = self.sigma_end <= sigma <= self.sigma_startself.control_stream = Noneif self.active:- payload = kwargs.get("minimax_payload") or {}- if payload.get("keyframes") or payload.get("refs"):- raise ValueError("MiniMax H3 Fun ControlNet does not support keyframe or reference conditioning")self.prepare_control_latent(x[0].shape)try:return executor(x, timestep, context, transformer_options, **kwargs)finally:self.control_stream = None+ self.pristine_stream = Nonedef before_block(self, block_index, args):if not self.active or block_index != self.model_patch.model.injection_layers[0]:return- self.control_latent = self.control_latent.to(args["img"].device)- self.control_stream = self.model_patch.model.init_stream(- args["img"], self.control_latent, args["layout"], args["t_emb"])+ # stash only: control weight loads here would clobber the base block's freshly staged weights+ self.pristine_stream = args["img"].clone()def after_block(self, block_index, args, out):if not self.active:return outcontrol_index = self.model_patch.model.injection_layers.index(block_index)+ if control_index == 0:+ self.control_latent = self.control_latent.to(out["img"].device)+ self.control_stream = self.model_patch.model.init_stream(+ self.pristine_stream, self.control_latent, args["layout"], args["t_emb"])+ self.pristine_stream = Noneself.control_stream, skip = self.model_patch.model.step(control_index, self.control_stream, args["t_emb"], args["mod_segments"], args["rope_freqs"],transformer_options=args["transformer_options"])@@ -510,6 +513,7 @@ class MiniMaxH3FunControlPatch:self.control_latent = Noneself.control_latent_shape = Noneself.control_stream = None+ self.pristine_stream = Noneself.active = Falsedef models(self):