Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Support context windows for MiniMax-H3 (#17425)

Jukka Seppänen6651247a8b50
diff --git a/comfy/context_windows.py b/comfy/context_windows.py
index 5f9899c67..eccc3a50b 100644
--- a/comfy/context_windows.py
+++ b/comfy/context_windows.py
@@ -220,6 +220,7 @@ class WindowingState:
guide_entries: list[list[dict] | None] # per-modality guide_attention_entry metadata
keyframe_idxs: list[torch.Tensor | None] # per-modality keyframe_idxs tensor for guide latent_start derivation
latent_shapes: list | None # original packed shapes for unpack/pack (None if not multimodal)
+ modality_dims: list[int] # per-modality temporal dim, modality_dims[0] == dim
dim: int = 0 # primary modality temporal dim for context windowing
is_multimodal: bool = False
temporal_downscale_ratio: int = 1 # model's pixel-to-latent temporal compression ratio
@@ -248,11 +249,12 @@ class WindowingState:
f"{type(model).__name__} must implement map_context_window_to_modalities for multimodal context windows.")
modality_windows = {}
for mod_idx in range(1, len(self.latents)):
- modality_total_frames = self.latents[mod_idx].shape[self.dim]
+ mod_dim = self.modality_dims[mod_idx]
+ modality_total_frames = self.latents[mod_idx].shape[mod_dim]
ratio = modality_total_frames / primary_total if primary_total > 0 else 1
modality_overlap = max(round(primary_overlap * ratio), 0)
modality_windows[mod_idx] = IndexListContextWindow(
- per_modality_indices[mod_idx], dim=self.dim,
+ per_modality_indices[mod_idx], dim=mod_dim,
total_frames=modality_total_frames,
context_overlap=modality_overlap)
return IndexListContextWindow(
@@ -393,7 +395,7 @@ class IndexListContextHandler(ContextHandlerABC):
return kf.cond
return None
- def _apply_freenoise(self, noise: torch.Tensor, conds: list[list[dict]], seed: int) -> torch.Tensor:
+ def _apply_freenoise(self, noise: torch.Tensor, conds: list[list[dict]], seed: int, model: BaseModel) -> torch.Tensor:
"""Apply FreeNoise shuffling, scaling context length/overlap per-modality by frame ratio.
If guide frames are present on the primary modality, only the video portion is shuffled.
"""
@@ -402,16 +404,17 @@ class IndexListContextHandler(ContextHandlerABC):
latent_shapes = self._get_latent_shapes(conds)
if latent_shapes is not None and len(latent_shapes) > 1:
+ modality_dims = model.context_window_modality_dims(latent_shapes, self.dim)
modalities = comfy.utils.unpack_latents(noise, latent_shapes)
primary_total = latent_shapes[0][self.dim]
primary_video_count = modalities[0].size(self.dim) - guide_count
apply_freenoise(modalities[0].narrow(self.dim, 0, primary_video_count), self.dim, self.context_length, self.context_overlap, seed)
for i in range(1, len(modalities)):
- mod_total = latent_shapes[i][self.dim]
+ mod_total = latent_shapes[i][modality_dims[i]]
ratio = mod_total / primary_total if primary_total > 0 else 1
mod_ctx_len = max(round(self.context_length * ratio), 1)
mod_ctx_overlap = max(round(self.context_overlap * ratio), 0)
- modalities[i] = apply_freenoise(modalities[i], self.dim, mod_ctx_len, mod_ctx_overlap, seed)
+ modalities[i] = apply_freenoise(modalities[i], modality_dims[i], mod_ctx_len, mod_ctx_overlap, seed)
noise, _ = comfy.utils.pack_latents(modalities)
return noise
video_count = noise.size(self.dim) - guide_count
@@ -450,6 +453,7 @@ class IndexListContextHandler(ContextHandlerABC):
guide_entries=guide_entries_list,
keyframe_idxs=keyframe_idxs_list,
latent_shapes=latent_shapes,
+ modality_dims=model.context_window_modality_dims(latent_shapes, self.dim) if is_multimodal else [self.dim],
dim=self.dim,
is_multimodal=is_multimodal,
temporal_downscale_ratio=model.latent_format.temporal_downscale_ratio)
@@ -582,11 +586,12 @@ class IndexListContextHandler(ContextHandlerABC):
# Initialize per-modality accumulators (length 1 for single-modality)
accum = [[torch.zeros_like(m) for _ in conds] for m in window_state.latents]
+ modalities = list(zip(window_state.latents, window_state.modality_dims))
if self.fuse_method.name == ContextFuseMethods.RELATIVE:
- counts = [[torch.ones(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents]
+ counts = [[torch.ones(get_shape_for_dim(m, d), device=m.device) for _ in conds] for m, d in modalities]
else:
- counts = [[torch.zeros(get_shape_for_dim(m, self.dim), device=m.device) for _ in conds] for m in window_state.latents]
- biases = [[([0.0] * m.shape[self.dim]) for _ in conds] for m in window_state.latents]
+ counts = [[torch.zeros(get_shape_for_dim(m, d), device=m.device) for _ in conds] for m, d in modalities]
+ biases = [[([0.0] * m.shape[d]) for _ in conds] for m, d in modalities]
for callback in comfy.patcher_extension.get_all_callbacks(IndexListCallbacks.EXECUTE_START, self.callbacks):
callback(self, model, x_in, conds, timestep, model_options)
@@ -721,15 +726,15 @@ class IndexListContextHandler(ContextHandlerABC):
prev_weight = (bias_total / (bias_total + bias))
new_weight = (bias / (bias_total + bias))
# account for dims of tensors
- idx_window = tuple([slice(None)] * self.dim + [idx])
- pos_window = tuple([slice(None)] * self.dim + [pos])
+ idx_window = tuple([slice(None)] * window.dim + [idx])
+ pos_window = tuple([slice(None)] * window.dim + [pos])
# apply new values
conds_final[i][idx_window] = conds_final[i][idx_window] * prev_weight + sub_conds_out[i][pos_window] * new_weight
biases_final[i][idx] = bias_total + bias
else:
# add conds and counts based on weights of fuse method
- weights = get_context_weights(window.context_length, x_in.shape[self.dim], window.index_list, self, sigma=timestep, context_overlap=window.context_overlap)
- weights_tensor = match_weights_to_dim(weights, x_in, self.dim, device=x_in.device)
+ weights = get_context_weights(window.context_length, x_in.shape[window.dim], window.index_list, self, sigma=timestep, context_overlap=window.context_overlap)
+ weights_tensor = match_weights_to_dim(weights, x_in, window.dim, device=x_in.device)
for i in range(len(sub_conds_out)):
window.add_window(conds_final[i], sub_conds_out[i] * weights_tensor)
window.add_window(counts_final[i], weights_tensor)
@@ -775,7 +780,7 @@ def _sampler_sample_wrapper(executor, guider, sigmas, extra_args, callback, nois
return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs)
conds = [guider.conds.get('positive', guider.conds.get('negative', []))]
- noise = handler._apply_freenoise(noise, conds, extra_args["seed"])
+ noise = handler._apply_freenoise(noise, conds, extra_args["seed"], guider.model_patcher.model)
return executor(guider, sigmas, extra_args, callback, noise, *args, **kwargs)
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 0461db1fe..42ccd5dbf 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -342,6 +342,10 @@ class BaseModel(torch.nn.Module):
Use comfy.context_windows.slice_cond() for common cases."""
return None
+ def context_window_modality_dims(self, latent_shapes, dim):
+ """Temporal dim of each stream of a packed multimodal latent for context windows."""
+ return [dim] * len(latent_shapes)
+
def extra_conds(self, **kwargs):
out = {}
concat_cond = self.concat_cond(**kwargs)
@@ -2238,13 +2242,10 @@ class MiniMaxH3(BaseModel):
keyframes = kwargs.get("minimax_keyframes", None)
if keyframes is not None:
payload["keyframes"] = keyframes
- payload["cond_video_latents"] = [kf["latent"] for kf in keyframes if kf.get("latent") is not None]
- payload["cond_audio_latents"] = [kf["audio_latent"] for kf in keyframes if kf.get("audio_latent") is not None]
refs = kwargs.get("minimax_refs", None)
if refs is not None:
payload["refs"] = refs
- payload["cond_video_latents"] = payload.get("cond_video_latents", []) + [r["latent"] for r in refs if "latent" in r]
- payload["cond_audio_latents"] = payload.get("cond_audio_latents", []) + [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]
+ payload.update(self._cond_latents(keyframes, refs))
if kwargs.get("minimax_visual_cond_noise_aug", None) is not None:
payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"]
if kwargs.get("minimax_audio_cond_noise_aug", None) is not None:
@@ -2267,6 +2268,74 @@ class MiniMaxH3(BaseModel):
out['minimax_payload'] = comfy.conds.CONDConstant(payload)
return out
+ @staticmethod
+ def _cond_latents(keyframes, refs):
+ # condition latents in packed-layout order: keyframes, then references
+ keyframes, refs = keyframes or [], refs or []
+ return {"cond_video_latents": [kf["latent"] for kf in keyframes if kf.get("latent") is not None] + [r["latent"] for r in refs if "latent" in r],
+ "cond_audio_latents": [kf["audio_latent"] for kf in keyframes if kf.get("audio_latent") is not None] + [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]}
+
+ def context_window_modality_dims(self, latent_shapes, dim):
+ # audio is [B, C, 2, T], time on the last axis
+ return [dim] + [len(s) - 1 for s in latent_shapes[1:]]
+
+ @staticmethod
+ def _latent_pixel_starts(latent_t):
+ # pixel frame each video latent starts at, plus the end of the last one
+ starts = [0]
+ for k in range(latent_t):
+ starts.append(starts[-1] + comfy.ldm.minimax.model.FRAME_PER_TOKEN[k % 5])
+ return starts
+
+ def map_context_window_to_modalities(self, primary_indices, latent_shapes, dim):
+ # the streams share one time axis: split the audio at the video latents' pixel frame boundaries
+ starts = self._latent_pixel_starts(latent_shapes[0][dim])
+ edges = [round(s * latent_shapes[1][-1] / starts[-1]) for s in starts]
+ return [primary_indices, [a for k in primary_indices for a in range(edges[k], edges[k + 1])]]
+
+ def _window_payload(self, payload, window):
+ # keyframes anchor the full video's timeline: keep the part of each guide inside the window, cropped and re-anchored to it
+ starts = self._latent_pixel_starts(window.total_frames)
+ first, last = window.index_list[0], window.index_list[-1]
+ origin = starts[first]
+ if getattr(window, "causal_anchor_index", None) is not None:
+ origin -= comfy.ldm.minimax.model.FRAME_PER_TOKEN[0] # the anchor latent takes the window's first time slot
+ audio_window = window.get_window_for_modality(1)
+ a0, a1 = audio_window.index_list[0], audio_window.index_list[-1] + 1
+ rescale = comfy.ldm.minimax.model.FRAME_RESCALE
+ keyframes = []
+ for kf in payload["keyframes"]:
+ f = kf["resolved_frame_index"]
+ latent = kf.get("latent")
+ if latent is not None:
+ # crop at the guide's 5-latent chunk starts so the clip keeps its 1,4,4,4,4 frame pattern; a chunk
+ # overlapping the window's start is kept and may begin a few frames before its origin
+ vt = latent.shape[2]
+ guide_starts = self._latent_pixel_starts(vt)
+ chunks = [g for g in range(0, vt, 5)
+ if f + guide_starts[g] < starts[last + 1] and f + guide_starts[min(g + 5, vt)] > starts[first]]
+ if chunks:
+ keyframes.append({"resolved_frame_index": f + guide_starts[chunks[0]] - origin,
+ "latent": latent[:, :, chunks[0]:chunks[-1] + 5]})
+ audio = kf.get("audio_latent")
+ if audio is not None:
+ # audio guide frame j sits at rescale * f + j on the shared time axis; the window's audio starts at a0
+ t = rescale * f
+ j0, j1 = max(0, math.ceil(a0 - t)), min(audio.shape[-1], math.ceil(a1 - t))
+ if j1 > j0:
+ keyframes.append({"resolved_frame_index": (t + j0 - a0) / rescale, "audio_latent": audio[..., j0:j1]})
+ payload = {k: v for k, v in payload.items() if k != "layout"} # forward rebuilds it for the window
+ payload["keyframes"] = keyframes
+ payload.update(self._cond_latents(keyframes, payload.get("refs")))
+ return payload
+
+ def resize_cond_for_context_window(self, cond_key, cond_value, window, x_in, device, retain_index_list=[]):
+ if cond_key == "audio_denoise_mask":
+ return cond_value._copy_with(window.get_window_for_modality(1).get_tensor(cond_value.cond, device))
+ if cond_key == "minimax_payload" and cond_value.cond.get("keyframes"):
+ return cond_value._copy_with(self._window_payload(cond_value.cond, window))
+ return super().resize_cond_for_context_window(cond_key, cond_value, window, x_in, device, retain_index_list=retain_index_list)
+
def _pool_masks_to_token_grid(self, masks):
# pool the per-pixel masks to the label grid with amax: video per 2x2 DiT patch, audio per latent frame
video_mask = masks[0]
diff --git a/comfy_extras/nodes_context_windows.py b/comfy_extras/nodes_context_windows.py
index c7aad1c6f..e1dd84449 100644
--- a/comfy_extras/nodes_context_windows.py
+++ b/comfy_extras/nodes_context_windows.py
@@ -1,6 +1,7 @@
from comfy_api.latest import ComfyExtension, io
import comfy.context_windows
import nodes
+from comfy_extras.nodes_minimax_h3 import video_latent_t
class ContextWindowsManualNode(io.ComfyNode):
@@ -135,12 +136,41 @@ class LTXVContextWindowsNode(ContextWindowsManualNode):
cond_retain_index_list=retain_index_list, latent_retain_index_list=retain_index_list, split_conds_to_windows=split_conds_to_windows)
+class MiniMaxH3ContextWindowsNode(ContextWindowsManualNode):
+ @classmethod
+ def define_schema(cls) -> io.Schema:
+ schema = super().define_schema()
+ schema.node_id = "MiniMaxH3ContextWindows"
+ schema.display_name = "MiniMax H3 Context Windows"
+ schema.description = "Set context windows for MiniMax H3. Windows start on the model's 17-frame chunks to keep video and audio in sync."
+ schema.category = "model/patch/minimax"
+ schema.inputs = [
+ io.Model.Input("model", tooltip="The model to apply context windows to during sampling."),
+ io.Int.Input("context_length", min=22, max=nodes.MAX_RESOLUTION, step=17, default=124, tooltip="The length of the context window in real frames. Must be 17*n + 5."),
+ io.Int.Input("context_overlap", min=5, step=17, default=39, tooltip="The overlap of the context window in real frames. Must be 17*n + 5."),
+ io.Combo.Input("fuse_method", options=comfy.context_windows.ContextFuseMethods.LIST_STATIC, default=comfy.context_windows.ContextFuseMethods.PYRAMID, tooltip="The method to use to fuse the context windows."),
+ io.Boolean.Input("freenoise", default=True, tooltip="Whether to apply FreeNoise noise shuffling, improves window blending.", advanced=True),
+ io.Boolean.Input("split_conds_to_windows", default=False, tooltip="Whether to split multiple conditionings (created by ConditionCombine) to each window based on region index.", advanced=True),
+ ]
+ return schema
+
+ @classmethod
+ def execute(cls, model: io.Model.Type, context_length: int, context_overlap: int, fuse_method: str, freenoise: bool,
+ split_conds_to_windows: bool=False) -> io.Model:
+ # length and overlap of 5n + 2 latents start every static window on a chunk boundary
+ context_length = video_latent_t(context_length)
+ context_overlap = min(video_latent_t(context_overlap), context_length - 5)
+ return super().execute(model, context_length, context_overlap, comfy.context_windows.ContextSchedules.STATIC_STANDARD, 1, False, fuse_method, dim=2, freenoise=freenoise,
+ split_conds_to_windows=split_conds_to_windows, causal_window_fix=False)
+
+
class ContextWindowsExtension(ComfyExtension):
async def get_node_list(self) -> list[type[io.ComfyNode]]:
return [
ContextWindowsManualNode,
WanContextWindowsManualNode,
LTXVContextWindowsNode,
+ MiniMaxH3ContextWindowsNode,
]
def comfy_entrypoint():