Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add MiniMaxH3AddGuide for anchoring image and audio guides at any frame (#15439)

Barish Ozbaye01fb4c56b7a
diff --git a/comfy/ldm/minimax/model.py b/comfy/ldm/minimax/model.py
index f745db884..b6feb8860 100644
--- a/comfy/ldm/minimax/model.py
+++ b/comfy/ldm/minimax/model.py
@@ -91,6 +91,18 @@ def _video_t_grid(n, origin):
return float(origin) + torch.cat([torch.zeros(1, dtype=torch.float64), spans[:-1].cumsum(0)])
+def _ref_t_span(blk):
+ # time-axis span a reference block occupies ahead of the target streams
+ kind = blk["kind"]
+ if kind == "image":
+ return 1.0
+ if kind == "audio":
+ return float(blk["ref_audio_t"])
+ if kind in ("video", "video_audio"):
+ return max(float(blk["ref_audio_t"]), sum(_video_t_spans(blk["latent_t"])))
+ return 0.0
+
+
def _audio_grid(cursor, t, w_low, w_high):
# channel-major stereo rows: t advances per latent frame, w pinned to the grid extremes per stereo channel, h stays 0
g = torch.zeros(t * 2, 3, dtype=torch.float64)
@@ -288,7 +300,7 @@ class FinalLayer(nn.Module):
class PackedLayout:
"""Static packed-sequence structure for one shape/conditioning signature."""
- def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None, frame_count=None):
+ def __init__(self, text_len, latent_t, latent_h, latent_w, audio_t, keyframes=None, refs=None):
frame, w_grid = _frame_grid(latent_h, latent_w)
frame_rows = frame.shape[0]
@@ -299,29 +311,37 @@ class PackedLayout:
img_pos, img_update = [], []
audio_pos, audio_update = [], []
- cursor = text_len
row = text_len
+ target_audio_w = (float(w_grid[0]), float(w_grid[-1]))
+ # refs pack between text and the targets, so the target timeline starts after their spans
+ cursor = float(text_len)
+ for blk in refs or ():
+ cursor += _ref_t_span(blk)
+
if keyframes:
- # fl2va: keyframe cond rows right after text, sharing the target spatial grid
+ # fl2va: keyframe cond rows right after text, sharing the target spatial grid;
+ # anchors count from the target timeline origin, FRAME_RESCALE per pixel frame, 1.0 per audio latent frame
for kf in keyframes:
- pixel_index = kf["resolved_frame_index"]
- if pixel_index == 0:
- cond_t = float(text_len)
- elif frame_count is not None and pixel_index == frame_count - 1:
- cond_t = float(text_len) + sum(_video_t_spans(latent_t)) - FRAME_RESCALE
- else:
- raise ValueError("only first/last keyframe anchors are supported")
- g = torch.empty(frame_rows, 3, dtype=torch.float64)
- g[:, 0] = cond_t
- g[:, 1:] = frame
- segments.append(("cond", frame_rows))
- pos.append(g)
- img_pos.append(torch.arange(row, row + frame_rows))
- img_update.append(torch.zeros(frame_rows, dtype=torch.bool))
- row += frame_rows
+ cond_t = cursor + FRAME_RESCALE * kf["resolved_frame_index"]
+ video_latent = kf.get("latent")
+ if video_latent is not None:
+ vt = video_latent.shape[2]
+ n = vt * frame_rows
+ segments.append(("cond", n))
+ pos.append(_video_grid(vt, frame, cond_t))
+ img_pos.append(torch.arange(row, row + n))
+ img_update.append(torch.zeros(n, dtype=torch.bool))
+ row += n
+ audio_latent = kf.get("audio_latent")
+ if audio_latent is not None:
+ rt = audio_latent.shape[-1]
+ segments.append(("cond_audio", rt * 2))
+ pos.append(_audio_grid(cond_t, rt, *target_audio_w))
+ audio_pos.append(torch.arange(row, row + rt * 2))
+ audio_update.append(torch.zeros(rt * 2, dtype=torch.bool))
+ row += rt * 2
- target_audio_w = (float(w_grid[0]), float(w_grid[-1]))
if refs:
cursor = float(text_len)
for blk in refs:
@@ -389,7 +409,7 @@ class PackedLayout:
self.audio_update = torch.cat(audio_update)
self.signature = (text_len, latent_t, latent_h, latent_w, audio_t)
# contiguous segment table (start, stop, kind)
- # kinds: text / cond / ref_img / ref_audio / audio / video
+ # kinds: text / cond / cond_audio / ref_img / ref_audio / audio / video
# the packed sequence is uniform per segment in (modality tag, timestep class),
# except the text span (tag runs resolved at forward time from the presentation tags)
seg_abs = []
@@ -529,8 +549,7 @@ class MiniMaxH3Model(nn.Module):
if layout is None or layout.signature != (text_len, latent_t, lat_h, lat_w, audio_t):
layout = PackedLayout(text_len, latent_t, lat_h, lat_w, audio_t,
keyframes=payload.get("keyframes"),
- refs=payload.get("refs"),
- frame_count=payload.get("frame_count"))
+ refs=payload.get("refs"))
# model_base passes model_sampling.timestep(sigma) = sigma * 1000
shift_v = float(transformer_options.get("minimax_h3_sigma_shift_video", self.sigma_shift_video))
@@ -543,14 +562,14 @@ class MiniMaxH3Model(nn.Module):
vis_aug = float(payload.get("visual_cond_noise_aug", VISUAL_COND_TIMESTEP))
aud_aug = float(payload.get("audio_cond_noise_aug", AUDIO_COND_TIMESTEP))
has_vis_cond = any(k in ("cond", "ref_img") for _, _, k in layout.segments)
- has_aud_cond = any(k == "ref_audio" for _, _, k in layout.segments)
+ has_aud_cond = any(k in ("cond_audio", "ref_audio") for _, _, k in layout.segments)
seg_t = {"text": t_v, "video": t_v, "audio": t_a,
"cond": max(t_v, vis_aug), "ref_img": max(t_v, vis_aug),
- "ref_audio": max(t_a, aud_aug)}
+ "cond_audio": max(t_a, aud_aug), "ref_audio": max(t_a, aud_aug)}
unique_t = sorted({t_v, t_a} | ({seg_t["cond"]} if has_vis_cond else set())
| ({seg_t["ref_audio"]} if has_aud_cond else set()))
t_row = {t: i for i, t in enumerate(unique_t)}
- seg_tag = {"text": 1, "video": 0, "audio": 2, "cond": 0, "ref_img": 0, "ref_audio": 2}
+ seg_tag = {"text": 1, "video": 0, "audio": 2, "cond": 0, "ref_img": 0, "cond_audio": 2, "ref_audio": 2}
text_tags = payload.get("text_token_tags")
mod_segments = []
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 90cab7ac0..6705eb6c3 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -2165,13 +2165,13 @@ class MiniMaxH3(BaseModel):
keyframes = kwargs.get("minimax_keyframes", None)
if keyframes is not None:
payload["keyframes"] = keyframes
- payload["frame_count"] = kwargs.get("minimax_frame_count", None)
- payload["cond_video_latents"] = [kf["latent"] for kf in keyframes]
+ payload["cond_video_latents"] = [kf["latent"] for kf in keyframes if kf.get("latent") is not None]
+ payload["cond_audio_latents"] = [kf["audio_latent"] for kf in keyframes if kf.get("audio_latent") is not None]
refs = kwargs.get("minimax_refs", None)
if refs is not None:
payload["refs"] = refs
- payload["cond_video_latents"] = [r["latent"] for r in refs if "latent" in r]
- payload["cond_audio_latents"] = [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]
+ payload["cond_video_latents"] = payload.get("cond_video_latents", []) + [r["latent"] for r in refs if "latent" in r]
+ payload["cond_audio_latents"] = payload.get("cond_audio_latents", []) + [r["audio_latent"] for r in refs if r.get("audio_latent") is not None]
if kwargs.get("minimax_visual_cond_noise_aug", None) is not None:
payload["visual_cond_noise_aug"] = kwargs["minimax_visual_cond_noise_aug"]
if kwargs.get("minimax_audio_cond_noise_aug", None) is not None:
@@ -2185,7 +2185,7 @@ class MiniMaxH3(BaseModel):
payload["layout"] = comfy.ldm.minimax.model.PackedLayout(
cross_attn.shape[1], vs[2], (vs[3] + 1) // 2 * 2, (vs[4] + 1) // 2 * 2,
latent_shapes[1][-1], keyframes=payload.get("keyframes"),
- refs=payload.get("refs"), frame_count=payload.get("frame_count"))
+ refs=payload.get("refs"))
out['minimax_payload'] = comfy.conds.CONDConstant(payload)
return out
diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.py
index 0b1840e85..0a08f185f 100644
--- a/comfy_extras/nodes_minimax_h3.py
+++ b/comfy_extras/nodes_minimax_h3.py
@@ -20,6 +20,7 @@ import comfy.model_sampling
import comfy.nested_tensor
import comfy.utils
import node_helpers
+from comfy.ldm.minimax.model import FRAME_PER_TOKEN, FRAME_RESCALE
from comfy_api.latest import ComfyExtension, io
CANVAS_MULTIPLE = 32
@@ -67,6 +68,16 @@ def _resize(image, width, height, crop):
return samples.movedim(1, -1)
+def _encode_ref_audio(audio_vae, audio):
+ waveform = audio["waveform"] # [B, C, L]
+ sr = audio["sample_rate"]
+ vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)
+ if sr != vae_sr:
+ waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
+ z = audio_vae.encode(waveform[:1].movedim(1, -1)) # [1, 32, 2, T]
+ return z, z.shape[-1]
+
+
def _empty_av_latent(width, height, length, batch_size=1):
frame_count, latent_t, audio_t = temporal_shape(length)
video = torch.zeros([batch_size, 24, latent_t, height // 16, width // 16],
@@ -144,13 +155,87 @@ class MiniMaxH3ImageToVideo(io.ComfyNode):
if keyframes:
for kf in keyframes:
kf["latent"] = vae.encode(kf.pop("image"))
- cond = node_helpers.conditioning_set_values(cond, {
- "minimax_keyframes": keyframes,
- "minimax_frame_count": frame_count,
- })
+ cond = node_helpers.conditioning_set_values(cond, {"minimax_keyframes": keyframes})
return io.NodeOutput(cond, latent)
+class MiniMaxH3AddGuide(io.ComfyNode):
+ """Anchor image and/or audio guides at an arbitrary pixel frame of the target video."""
+
+ @classmethod
+ def define_schema(cls):
+ return io.Schema(
+ node_id="MiniMaxH3AddGuide",
+ display_name="Add Guide for MiniMax H3",
+ category="model/conditioning/minimax",
+ description="Anchor an image, a short clip, audio, or a clip with its soundtrack at any frame of a MiniMax H3 video. Chain several nodes to anchor several frames.",
+ inputs=[
+ io.Conditioning.Input("positive"),
+ io.Vae.Input("vae", optional=True, tooltip="Video VAE, needed when an image is connected."),
+ io.Vae.Input("audio_vae", optional=True, tooltip="Audio VAE, needed when an audio is connected."),
+ io.Latent.Input("latent"),
+ io.Image.Input("image", optional=True, tooltip="Image or video frames to anchor. Multi-frame batches are anchored as a clip and cropped down to the model's valid clip lengths: 5, 22, 39... (17k + 5) frames. Batches shorter than 5 frames use only the first image."),
+ io.Audio.Input("audio", optional=True,
+ tooltip="Soundtrack to anchor starting at the same frame index, cropped to the video's remaining duration."),
+ io.Int.Input("frame_idx", default=0, min=-9999, max=9999,
+ tooltip="Frame index to anchor the image or the clip's first frame at. Negative values are counted from the end of the video."),
+ ],
+ outputs=[io.Conditioning.Output(display_name="positive")],
+ )
+
+ @classmethod
+ def execute(cls, positive, latent, frame_idx, vae=None, audio_vae=None, image=None, audio=None) -> io.NodeOutput:
+ samples = latent["samples"]
+ if not samples.is_nested or len(samples.tensors) != 2 or samples.tensors[0].ndim != 5 or samples.tensors[0].shape[1] != 24:
+ raise ValueError("MiniMaxH3AddGuide expects a MiniMax H3 AV latent")
+ if image is None and audio is None:
+ raise ValueError("MiniMaxH3AddGuide needs an image or an audio to anchor")
+ video = samples.tensors[0]
+ height = video.shape[3] * 16
+ width = video.shape[4] * 16
+ frame_count = sum(FRAME_PER_TOKEN[k % 5] for k in range(video.shape[2]))
+
+ guide_frames = 1
+ if image is not None:
+ if vae is None:
+ raise ValueError("anchoring guide frames needs the vae input")
+ guide_frames = image.shape[0]
+ if guide_frames < 5:
+ guide_frames = 1
+ else:
+ while guide_frames % 17 != 5:
+ guide_frames -= 1
+
+ resolved_frame_index = frame_idx if frame_idx >= 0 else frame_count + frame_idx
+ if resolved_frame_index < 0 or resolved_frame_index + guide_frames > frame_count:
+ if guide_frames == 1:
+ raise ValueError("frame_idx {} is outside the video's {} frames".format(frame_idx, frame_count))
+ raise ValueError("a {} frame guide clip at frame_idx {} does not fit in the video's {} frames".format(
+ guide_frames, frame_idx, frame_count))
+
+ keyframe = {"resolved_frame_index": resolved_frame_index}
+ if image is not None:
+ frames = _resize(image[:guide_frames], width, height, "center")
+ keyframe["latent"] = vae.encode(frames)
+
+ if audio is not None:
+ if audio_vae is None:
+ raise ValueError("anchoring guide audio needs the audio_vae input")
+ audio_latent, audio_rt = _encode_ref_audio(audio_vae, audio)
+ # the streams share one time axis: FRAME_RESCALE per pixel frame, 1.0 per audio latent frame
+ max_rt = math.floor(samples.tensors[1].shape[-1] - FRAME_RESCALE * resolved_frame_index)
+ if max_rt < 1:
+ raise ValueError("frame_idx {} is past the end of the video's audio track".format(frame_idx))
+ if audio_rt > max_rt:
+ audio_latent = audio_latent[..., :max_rt].clone()
+ keyframe["audio_latent"] = audio_latent
+
+ keyframes = list(positive[0][1].get("minimax_keyframes", []))
+ keyframes.append(keyframe)
+ positive = node_helpers.conditioning_set_values(positive, {"minimax_keyframes": keyframes})
+ return io.NodeOutput(positive)
+
+
class MiniMaxH3ReferenceToVideo(io.ComfyNode):
"""ref2va: prompt + reference images / videos / audio -> conditioning + AV latent.
@@ -197,16 +282,6 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):
outputs=[io.Conditioning.Output(display_name="positive"), io.Latent.Output()],
)
- @staticmethod
- def _encode_ref_audio(audio_vae, audio):
- waveform = audio["waveform"] # [B, C, L]
- sr = audio["sample_rate"]
- vae_sr = getattr(audio_vae, "audio_sample_rate", 32000)
- if sr != vae_sr:
- waveform = torchaudio.functional.resample(waveform, sr, vae_sr)
- z = audio_vae.encode(waveform[:1].movedim(1, -1)) # [1, 32, 2, T]
- return z, z.shape[-1]
-
@classmethod
def execute(cls, clip, vae, audio_vae, prompt, width, height, length, ref_image_size="match",
ref_images=None, ref_videos=None, ref_video_audios=None, ref_audios=None) -> io.NodeOutput:
@@ -254,7 +329,7 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):
z = vae.encode(frames)
audio_latent, ref_audio_t = (None, 0)
if soundtrack is not None:
- audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, soundtrack)
+ audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, soundtrack)
# the soundtrack gets its own <Audio j> label, emitted before <Video k>
ref_items.append({"type": "audio"})
# Qwen sees the video at 2 fps with timestamps
@@ -269,7 +344,7 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):
for audio in (ref_audios or {}).values():
if audio is None:
continue
- audio_latent, ref_audio_t = cls._encode_ref_audio(audio_vae, audio)
+ audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, audio)
ref_items.append({"type": "audio"})
ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t, "audio_latent": audio_latent})
@@ -329,6 +404,7 @@ class MiniMaxH3Extension(ComfyExtension):
return [
EmptyMiniMaxH3LatentAV,
MiniMaxH3ImageToVideo,
+ MiniMaxH3AddGuide,
MiniMaxH3ReferenceToVideo,
MiniMaxH3SigmaShift,
]