The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
MiniMax-H3: Make VAE optional, allow using text encoder only references (#16065)
Jukka Seppänen1aec3a1351f7
diff --git a/comfy_extras/nodes_minimax_h3.py b/comfy_extras/nodes_minimax_h3.pyindex 9ac827b0a..108dee13c 100644--- a/comfy_extras/nodes_minimax_h3.py+++ b/comfy_extras/nodes_minimax_h3.py@@ -256,8 +256,8 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):category="model/conditioning/minimax",inputs=[io.Clip.Input("clip"),- io.Vae.Input("vae"),- io.Vae.Input("audio_vae"),+ io.Vae.Input("vae", optional=True, tooltip="Video VAE. Without it reference images/videos only condition the text encoder."),+ io.Vae.Input("audio_vae", optional=True, tooltip="Audio VAE. Without it reference audio only conditions the text encoder."),io.String.Input("prompt", multiline=True, dynamic_prompts=True),io.Int.Input("width", default=1344, min=32, max=nodes.MAX_RESOLUTION, step=32),io.Int.Input("height", default=768, min=32, max=nodes.MAX_RESOLUTION, step=32),@@ -285,7 +285,7 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):)@classmethod- def execute(cls, clip, vae, audio_vae, prompt, width, height, length, ref_image_size="match",+ def execute(cls, clip, prompt, width, height, length, ref_image_size="match", vae=None, audio_vae=None,ref_images=None, ref_videos=None, ref_video_audios=None, ref_audios=None) -> io.NodeOutput:latent, frame_count = _empty_av_latent(width, height, length)@@ -304,9 +304,10 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):tw = max(CANVAS_MULTIPLE, round(w * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)th = max(CANVAS_MULTIPLE, round(h * scale / CANVAS_MULTIPLE) * CANVAS_MULTIPLE)resized = _resize(img[:1], tw, th, "disabled")- z = vae.encode(resized)ref_items.append({"type": "image", "data": resized})- ref_blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, "latent": z})+ if vae is not None:+ z = vae.encode(resized)+ ref_blocks.append({"kind": "image", "latent_h": th // 16, "latent_w": tw // 16, "latent": z})ref_video_audios = ref_video_audios or {}for name, video_frames in (ref_videos or {}).items():@@ -328,10 +329,7 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):while n % 17 != 5:n -= 1frames = frames[:n]- z = vae.encode(frames)- audio_latent, ref_audio_t = (None, 0)if soundtrack is not None:- audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, soundtrack)# the soundtrack gets its own <Audio j> label, emitted before <Video k>ref_items.append({"type": "audio"})# Qwen sees the video at 2 fps with timestamps@@ -339,6 +337,12 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):qwen_frames = frames[sample_idx]ref_items.append({"type": "video", "data": qwen_frames,"timestamps": [i / 2.0 for i in range(len(sample_idx))]})+ if vae is None:+ continue+ z = vae.encode(frames)+ audio_latent, ref_audio_t = (None, 0)+ if soundtrack is not None and audio_vae is not None:+ audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, soundtrack)ref_blocks.append({"kind": "video_audio" if ref_audio_t else "video","latent_t": z.shape[2], "latent_h": ch // 16, "latent_w": cw // 16,"ref_audio_t": ref_audio_t, "latent": z, "audio_latent": audio_latent})@@ -346,9 +350,10 @@ class MiniMaxH3ReferenceToVideo(io.ComfyNode):for audio in (ref_audios or {}).values():if audio is None:continue- audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, audio)ref_items.append({"type": "audio"})- ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t, "audio_latent": audio_latent})+ if audio_vae is not None:+ audio_latent, ref_audio_t = _encode_ref_audio(audio_vae, audio)+ ref_blocks.append({"kind": "audio", "ref_audio_t": ref_audio_t, "audio_latent": audio_latent})tokens = clip.tokenize(prompt, minimax_ref_items=ref_items)cond = clip.encode_from_tokens_scheduled(tokens)