The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Support context windows for MiniMax-H3 (#17425)
Jukka Seppänen6651247a8b50
diff --git a/comfy/context_windows.py b/comfy/context_windows.pyindex 5f9899c67..eccc3a50b 100644--- a/comfy/context_windows.py+++ b/comfy/context_windows.py@@ -220,6 +220,7 @@ class WindowingState:guide_entries: list[list[dict] | None] # per-modality guide_attention_entry metadatakeyframe_idxs: list[torch.Tensor | None] # per-modality keyframe_idxs tensor for guide latent_start derivationlatent_shapes: list | None # original packed shapes for unpack/pack (None if not multimodal)+ modality_dims: list[int] # per-modality temporal dim, modality_dims[0] == dimdim: int = 0 # primary modality temporal dim for context windowingis_multimodal: bool = Falsetemporal_downscale_ratio: int = 1 # model's pixel-to-latent temporal compression ratio@@ -248,11 +249,12 @@ class WindowingState:f"{type(model).__name__} must implement map_context_window_to_modalities for multimodal context windows.")modality_windows = {}for mod_idx in range(1, len(self.latents)):- modality_total_frames = self.latents[mod_idx].shape[self.dim]+ mod_dim = self.modality_dims[mod_idx]+ modality_total_frames = self.latents[mod_idx].shape[mod_dim]ratio = modality_total_frames / primary_total if primary_total > 0 else 1modality_overlap = max(round(primary_overlap * ratio), 0)modality_windows[mod_idx] = IndexListContextWindow(- per_modality_indices[mod_idx], dim=self.dim,+ per_modality_indices[mod_idx], dim=mod_dim,total_frames=modality_total_frames,context_overlap=modality_overlap)return IndexListContextWindow(@@ -393,7 +395,7 @@ class IndexListContextHandler(ContextHandlerABC):return kf.condreturn None- def _apply_freenoise(self, noise: torch.Tensor, conds: list[list[dict]], seed: int) -> torch.Tensor:+ def _apply_freenoise(self, noise: torch.Tensor, conds: list[list[dict]], seed: int, model: BaseModel) -> torch.Tensor:"""Apply FreeNoise shuffling, scaling context length/overlap per-modality by frame ratio.If guide frames are present on the primary modality, only the video portion is shuffled."""@@ -402,16 +404,17 @@ class IndexListContextHandler(ContextHandlerABC):latent_shapes = self._get_latent_shapes(conds)if latent_shapes is not None and len(latent_shapes) > 1:+ modality_dims = model.context_window_modality_dims(latent_shapes, self.dim)modalities = comfy.utils.unpack_latents(noise, latent_shapes)primary_total = latent_shapes[0][self.dim]primary_video_count = modalities[0].size(self.dim) - guide_countapply_freenoise(modalities[0].narrow(self.dim, 0, primary_video_count), self.dim, self.context_length, self.context_overlap, seed)for i in range(1, len(modalities)):- mod_total = latent_shapes[i][self.dim]+ mod_total = latent_shapes[i][modality_dims[i]]ratio = mod_total / primary_total if primary_total > 0 else 1mod_ctx_len = max(round(self.context_length * ratio), 1)mod_ctx_overlap = max(round(self.context_overlap * ratio), 0)- modalities[i] = apply_freenoise(modalities[i], self.dim, mod_ctx_len, mod_ctx_overlap, seed)+ modalities[i] = apply_freenoise(modalities[i], modality_dims[i], mod_ctx_len, mod_ctx_overlap, seed)noise, _ = comfy.utils.pack_latents(modalities)return noisevideo_count = noise.size(self.dim) - guide_count@@ -450,6 +453,7 @@ class IndexListContextHandler(ContextHandlerABC):guide_entries=guide_entries_list,keyframe_idxs=keyframe_idxs_list,latent_shapes=latent_shapes,+ modality_dims=model.context_window_modality_dims(latent_shapes, self.dim) if is_multimodal else [self.dim],dim=self.dim,is_multimodal=is_multimodal,temporal_downscale_ratio=model.latent_format.temporal_downscale_ratio)@@ -582,11 +586,12 @@ class IndexListContextHandler(ContextHandlerABC):# Initialize per-modality accumulators (length 1 for single-modality)accum = [[torch.zeros_like(m) for _ in conds] for m in window_state.latents]+ modalities = list(zip(window_state.latents, window_state.modality_dims))if self.fuse_method.name == ContextFuseMethods.RELATIVE:- counts = [[torch.ones(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents]+ counts = [[torch.ones(get_shape_for_dim(m, d), device=m.device) for _ in conds] for m, d in modalities]else:- counts = [[torch.zeros(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents]- biases = [[([0.0] * m.shape[self.dim]) for _ in conds] for m in window_state.latents]+ counts = [[torch.zeros(get_shape_for_dim(m, d), device=m.device) for _ in conds] for m, d in modalities]+ biases = [[([0.0] * m.shape[d]) for _ in conds] for m, d in modalities]for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_START, self.callbacks):callback(self, model, x_in, conds, timestep, model_options)@@ -721,15 +726,15 @@ class IndexListContextHandler(ContextHandlerABC):prev_weight = (bias_total / (bias_total + bias))new_weight = (bias / (bias_total + bias))# account for dims of tensors- idx_window = tuple([slice(None)] * self.dim + [idx])- pos_window = tuple([slice(None)] * self.dim + [pos])+ idx_window = tuple([slice(None)] * window.dim + [idx])+ pos_window = tuple([slice(None)] * window.dim + [pos])# apply new valuesconds_final[i][idx_window] = conds_final[i][idx_window] * prev_weight + sub_conds_out[i][pos_window] * new_weightbiases_final[i][idx] = bias_total + biaselse:# add conds and counts based on weights of fuse method- weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep, context_overlap=window.context_overlap)- weights_tensor = match_weights_to_dim(weights, x_in, self.dim, device=x_in.device)+ weights = get_context_weights(window.context_length, x_in.shape[window.dim], window.index_list, self, sigma=timestep, context_overlap=window.context_overlap)+ weights_tensor = match_weights_to_dim(weights, x_in, window.dim, device=x_in.device)for i in range(len(sub_conds_out)):window.add_window(conds_final[i], sub_conds_out[i] * weights_tensor)window.add_window(counts_final[i], weights_tensor)@@ -775,7 +780,7 @@ def _sampler_sample_wrapper(executor, guider, sigmas, extra_args, callback, noisreturn executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs)conds = [guider.conds.get('positive', guider.conds.get('negative', []))]- noise = handler._apply_freenoise(noise, conds, extra_args["seed"])+ noise = handler._apply_freenoise(noise, conds, extra_args["seed"], guider.model_patcher.model)return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 0461db1fe..42ccd5dbf 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -342,6 +342,10 @@ class BaseModel(torch.nn.Module):Use comfy.context_windows.slice_cond() for common cases."""return None+ def context_window_modality_dims(self, latent_shapes, dim):+ """Temporal dim of each stream of a packed multimodal latent for context windows."""+ return [dim] * len(latent_shapes)+def extra_conds(self, **kwargs):out = {}concat_cond = self.concat_cond(**kwargs)@@ -2238,13 +2242,10 @@ class MiniMaxH3(BaseModel):keyframes = kwargs.get("minimax_keyframes", None)if keyframes is not None:payload["keyframes"] = keyframes- payload["cond_video_latents"] = [kf["latent"] for kf in keyframes if kf.get("latent") is not None]- payload["cond_audio_latents"] = [kf["audio_latent"] for kf in keyframes if kf.get("audio_latent") is not None]refs = kwargs.get("minimax_refs", None)if refs is not None:payload["refs"] = refs- payload["cond_video_latents"] = payload.get("cond_video_latents", []) + [r["latent"] for r in refs if "latent" in r]- payload["cond_audio_latents"] = payload.get("cond_audio_latents", []) + [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]+ payload.update(self._cond_latents(keyframes, refs))if kwargs.get("minimax_visual_cond_noise_aug", None) is not None:payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"]if kwargs.get("minimax_audio_cond_noise_aug", None) is not None:@@ -2267,6 +2268,74 @@ class MiniMaxH3(BaseModel):out['minimax_payload'] = comfy.conds.CONDConstant(payload)return out+ @staticmethod+ def _cond_latents(keyframes, refs):+ # condition latents in packed-layout order: keyframes, then references+ keyframes, refs = keyframes or [], refs or []+ return {"cond_video_latents": [kf["latent"] for kf in keyframes if kf.get("latent") is not None] + [r["latent"] for r in refs if "latent" in r],+ "cond_audio_latents": [kf["audio_latent"] for kf in keyframes if kf.get("audio_latent") is not None] + [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]}++ def context_window_modality_dims(self, latent_shapes, dim):+ # audio is [B, C, 2, T], time on the last axis+ return [dim] + [len(s) - 1 for s in latent_shapes[1:]]++ @staticmethod+ def _latent_pixel_starts(latent_t):+ # pixel frame each video latent starts at, plus the end of the last one+ starts = [0]+ for k in range(latent_t):+ starts.append(starts[-1] + comfy.ldm.minimax.model.FRAME_PER_TOKEN[k % 5])+ return starts++ def map_context_window_to_modalities(self, primary_indices, latent_shapes, dim):+ # the streams share one time axis: split the audio at the video latents' pixel frame boundaries+ starts = self._latent_pixel_starts(latent_shapes[0][dim])+ edges = [round(s * latent_shapes[1][-1] / starts[-1]) for s in starts]+ return [primary_indices, [a for k in primary_indices for a in range(edges[k], edges[k + 1])]]++ def _window_payload(self, payload, window):+ # keyframes anchor the full video's timeline: keep the part of each guide inside the window, cropped and re-anchored to it+ starts = self._latent_pixel_starts(window.total_frames)+ first, last = window.index_list[0], window.index_list[-1]+ origin = starts[first]+ if getattr(window, "causal_anchor_index", None) is not None:+ origin -= comfy.ldm.minimax.model.FRAME_PER_TOKEN[0] # the anchor latent takes the window's first time slot+ audio_window = window.get_window_for_modality(1)+ a0, a1 = audio_window.index_list[0], audio_window.index_list[-1] + 1+ rescale = comfy.ldm.minimax.model.FRAME_RESCALE+ keyframes = []+ for kf in payload["keyframes"]:+ f = kf["resolved_frame_index"]+ latent = kf.get("latent")+ if latent is not None:+ # crop at the guide's 5-latent chunk starts so the clip keeps its 1,4,4,4,4 frame pattern; a chunk+ # overlapping the window's start is kept and may begin a few frames before its origin+ vt = latent.shape[2]+ guide_starts = self._latent_pixel_starts(vt)+ chunks = [g for g in range(0, vt, 5)+ if f + guide_starts[g] < starts[last + 1] and f + guide_starts[min(g + 5, vt)] > starts[first]]+ if chunks:+ keyframes.append({"resolved_frame_index": f + guide_starts[chunks[0]] - origin,+ "latent": latent[:, :, chunks[0]:chunks[-1] + 5]})+ audio = kf.get("audio_latent")+ if audio is not None:+ # audio guide frame j sits at rescale * f + j on the shared time axis; the window's audio starts at a0+ t = rescale * f+ j0, j1 = max(0, math.ceil(a0 - t)), min(audio.shape[-1], math.ceil(a1 - t))+ if j1 > j0:+ keyframes.append({"resolved_frame_index": (t + j0 - a0) / rescale, "audio_latent": audio[..., j0:j1]})+ payload = {k: v for k, v in payload.items() if k != "layout"} # forward rebuilds it for the window+ payload["keyframes"] = keyframes+ payload.update(self._cond_latents(keyframes, payload.get("refs")))+ return payload++ def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]):+ if cond_key == "audio_denoise_mask":+ return cond_value._copy_with(window.get_window_for_modality(1).get_tensor(cond_value.cond, device))+ if cond_key == "minimax_payload" and cond_value.cond.get("keyframes"):+ return cond_value._copy_with(self._window_payload(cond_value.cond, window))+ return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list)+def _pool_masks_to_token_grid(self, masks):# pool the per-pixel masks to the label grid with amax: video per 2x2 DiT patch, audio per latent framevideo_mask = masks[0]diff --git a/comfy_extras/nodes_context_windows.py b/comfy_extras/nodes_context_windows.pyindex c7aad1c6f..e1dd84449 100644--- a/comfy_extras/nodes_context_windows.py+++ b/comfy_extras/nodes_context_windows.py@@ -1,6 +1,7 @@from comfy_api.latest import ComfyExtension, ioimport comfy.context_windowsimport nodes+from comfy_extras.nodes_minimax_h3 import video_latent_tclass ContextWindowsManualNode(io.ComfyNode):@@ -135,12 +136,41 @@ class LTXVContextWindowsNode(ContextWindowsManualNode):cond_retain_index_list=retain_index_list, latent_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows)+class MiniMaxH3ContextWindowsNode(ContextWindowsManualNode):+ @classmethod+ def define_schema(cls) -> io.Schema:+ schema = super().define_schema()+ schema.node_id = "MiniMaxH3ContextWindows"+ schema.display_name = "MiniMax H3 Context Windows"+ schema.description = "Set context windows for MiniMax H3. Windows start on the model's 17-frame chunks to keep video and audio in sync."+ schema.category = "model/patch/minimax"+ schema.inputs = [+ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."),+ io.Int.Input("context_length", min=22, max=nodes.MAX_RESOLUTION, step=17, default=124, tooltip="The length of the context window in real frames. Must be 17*n + 5."),+ io.Int.Input("context_overlap", min=5, step=17, default=39, tooltip="The overlap of the context window in real frames. Must be 17*n + 5."),+ io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."),+ io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True),+ io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True),+ ]+ return schema++ @classmethod+ def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, fuse_method: str, freenoise: bool,+ split_conds_to_windows: bool=False) -> io.Model:+ # length and overlap of 5n + 2 latents start every static window on a chunk boundary+ context_length = video_latent_t(context_length)+ context_overlap = min(video_latent_t(context_overlap), context_length - 5)+ return super().execute(model, context_length, context_overlap, comfy.context_windows.ContextSchedules.STATIC_STANDARD, 1, False, fuse_method, dim=2, freenoise=freenoise,+ split_conds_to_windows=split_conds_to_windows, causal_window_fix=False)++class ContextWindowsExtension(ComfyExtension):async def get_node_list(self) -> list[type[io.ComfyNode]]:return [ContextWindowsManualNode,WanContextWindowsManualNode,LTXVContextWindowsNode,+ MiniMaxH3ContextWindowsNode,]def comfy_entrypoint():