The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(Minimax-H3): add the Max model to the Reference node (#16041)
Alexander Piskunec803fc97ee3
diff --git a/comfy_api_nodes/apis/minimax.py b/comfy_api_nodes/apis/minimax.pyindex e7c128f25..46a13330d 100644--- a/comfy_api_nodes/apis/minimax.py+++ b/comfy_api_nodes/apis/minimax.py@@ -242,6 +242,9 @@ class Hailuo03MaxVideoRequest(BaseModel):aspect_ratio: str | None = Field(None)image_url: str | None = Field(None)end_image_url: str | None = Field(None)+ reference_image_urls: list[str] | None = Field(None)+ reference_video_urls: list[str] | None = Field(None)+ reference_audio_urls: list[str] | None = Field(None)class Hailuo03MaxVideoResult(BaseModel):diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.pyindex 3663d7ec0..8a182fee4 100644--- a/comfy_api_nodes/nodes_minimax.py+++ b/comfy_api_nodes/nodes_minimax.py@@ -31,6 +31,7 @@ from comfy_api_nodes.apis.minimax import (from comfy_api_nodes.util import (ApiEndpoint,download_url_to_video_output,+ downscale_image_tensor_by_max_sides,poll_op,sync_op,upload_audio_to_comfyapi,@@ -468,6 +469,7 @@ HAILUO_03_REGENERATION_ENDPOINT = "/proxy/minimax/v2/video_regeneration"HAILUO_03_MAX_MODEL = "MiniMax H3 Max"HAILUO_03_MAX_ENDPOINT = "/proxy/fal/minimax/h3-max"HAILUO_03_MAX_PROMPT_MAX_LENGTH = 50000+HAILUO_03_MAX_REFERENCE_IMAGE_MAX_SIDES = {"high": (5120, 2048), "standard": (2048, 1024)}def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = True):@@ -549,7 +551,7 @@ async def _hailuo03_run_task(return IO.NodeOutput(await download_url_to_video_output(video_url))-def _hailuo03_max_model_inputs(include_ratio: bool = True):+def _hailuo03_max_model_inputs(include_ratio: bool = True, allow_adaptive: bool = False):inputs = [IO.String.Input("prompt",@@ -565,11 +567,14 @@ def _hailuo03_max_model_inputs(include_ratio: bool = True):),]if include_ratio:+ ratio_options = ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"]+ if allow_adaptive:+ ratio_options.insert(0, "adaptive")inputs.append(IO.Combo.Input("ratio",- options=["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],- default="16:9",+ options=ratio_options,+ default=ratio_options[0],tooltip="Aspect ratio of the output video.",))@@ -866,6 +871,54 @@ class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode):)+def _hailuo03_reference_inputs():+ return [+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("reference_image"),+ names=[+ "image_1",+ "image_2",+ "image_3",+ "image_4",+ "image_5",+ "image_6",+ "image_7",+ "image_8",+ "image_9",+ ],+ min=0,+ ),+ tooltip="Subject or style reference images, referred to in the prompt "+ "as 'Image 1'..'Image 9' in connection order. Up to 9 images.",+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=["video_1", "video_2", "video_3"],+ min=0,+ ),+ tooltip="Motion or scene reference videos, referred to in the prompt "+ "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, "+ "2-15 seconds each, 15 seconds in total.",+ ),+ IO.Autogrow.Input(+ "reference_audios",+ template=IO.Autogrow.TemplateNames(+ IO.Audio.Input("reference_audio"),+ names=["audio_1", "audio_2", "audio_3"],+ min=0,+ ),+ tooltip="Audio references, referred to in the prompt as "+ "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, "+ "2-15 seconds each, 15 seconds in total. Cannot be used without "+ "a reference image or video.",+ ),+ ]++class MinimaxHailuo03ReferenceNode(IO.ComfyNode):@classmethoddef define_schema(cls):@@ -874,7 +927,7 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):display_name="MiniMax H3 Reference to Video",category="partner/video/MiniMax",description="Generate video conditioned on reference images, videos, and audio using the "- "MiniMax H3 model. Refer to the references in the prompt by their order: "+ "MiniMax H3 models. Refer to the references in the prompt by their order: ""'Image 1', 'Image 2', 'Video 1', 'Audio 1', and so on.",inputs=[IO.DynamicCombo.Input(@@ -882,53 +935,23 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):options=[IO.DynamicCombo.Option("MiniMax H3",+ [*_hailuo03_model_inputs(), *_hailuo03_reference_inputs()],+ ),+ IO.DynamicCombo.Option(+ HAILUO_03_MAX_MODEL,[- *_hailuo03_model_inputs(),- IO.Autogrow.Input(- "reference_images",- template=IO.Autogrow.TemplateNames(- IO.Image.Input("reference_image"),- names=[- "image_1",- "image_2",- "image_3",- "image_4",- "image_5",- "image_6",- "image_7",- "image_8",- "image_9",- ],- min=0,- ),- tooltip="Subject or style reference images, referred to in the prompt "- "as 'Image 1'..'Image 9' in connection order. Up to 9 images.",- ),- IO.Autogrow.Input(- "reference_videos",- template=IO.Autogrow.TemplateNames(- IO.Video.Input("reference_video"),- names=["video_1", "video_2", "video_3"],- min=0,- ),- tooltip="Motion or scene reference videos, referred to in the prompt "- "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, "- "2-15 seconds each, 15 seconds in total.",- ),- IO.Autogrow.Input(- "reference_audios",- template=IO.Autogrow.TemplateNames(- IO.Audio.Input("reference_audio"),- names=["audio_1", "audio_2", "audio_3"],- min=0,- ),- tooltip="Audio references, referred to in the prompt as "- "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, "- "2-15 seconds each, 15 seconds in total. Cannot be used without "- "a reference image or video.",+ *_hailuo03_max_model_inputs(allow_adaptive=True),+ IO.Combo.Input(+ "reference_detail",+ options=["high", "standard"],+ default="standard",+ tooltip="Detail level at which reference images are sent. 'high' sends them at "+ "the largest size the model uses (up to a 2048 pixel short side); "+ "'standard' downsizes them to at most 2048x1024 to reduce the reference cost.",),+ *_hailuo03_reference_inputs(),],- )+ ),],tooltip="Model to use for video generation.",),@@ -961,22 +984,44 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):is_api_node=True,price_badge=IO.PriceBadge(depends_on=IO.PriceBadgeDepends(- widgets=["model.resolution", "model.duration"],- input_groups=["model.reference_images", "model.reference_videos"],+ widgets=["model", "model.resolution", "model.duration", "model.reference_detail"],+ input_groups=["model.reference_images", "model.reference_videos", "model.reference_audios"],),expr="""($dur := $lookup(widgets, "model.duration");- $rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859;+ $res := $lookup(widgets, "model.resolution");$imgsRaw := $lookup(inputGroups, "model.reference_images");$imgs := $imgsRaw ? $imgsRaw : 0;$vidsRaw := $lookup(inputGroups, "model.reference_videos");$vids := $vidsRaw ? $vidsRaw : 0;- $base := $dur * $rate + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0);- $vids > 0- ? {"type": "range_usd", "min_usd": $base + $vids * 2 * $rate,- "max_usd": $base + 15 * $rate, "format": {"approximate": true}}- : {"type": "usd", "usd": $base}+ $audsRaw := $lookup(inputGroups, "model.reference_audios");+ $auds := $audsRaw ? $audsRaw : 0;+ $lookup(widgets, "model") = "minimax h3 max"+ ? (+ $unitRate := $res = "480p" ? 1 : 1.6;+ $frameTokens := $res = "480p" ? 390 : 1008;+ $base := $dur * $unitRate;+ $minTokens := $imgs * 576 + $vids * 12 * $frameTokens + $auds * 160;+ $capFrames := $vids * (7 * $dur + 2);+ $maxFrames := $capFrames > 265 ? 265 : $capFrames;+ $imgMax := $lookup(widgets, "model.reference_detail") = "standard" ? 2560 : 10240;+ $maxTokens := $imgs * $imgMax + $maxFrames * $frameTokens + ($auds > 0 ? 1200 : 0);+ $minUnits := $base + ($minTokens > 4096 ? ($minTokens - 4096) * 0.0004 : 0);+ $maxUnits := $base + ($maxTokens > 4096 ? ($maxTokens - 4096) * 0.0004 : 0);+ $minUnits = $maxUnits+ ? {"type": "usd", "usd": $base * 0.0715}+ : {"type": "range_usd", "min_usd": $minUnits * 0.0715, "max_usd": $maxUnits * 0.0715,+ "format": {"approximate": true}}+ )+ : (+ $rate := $res = "768p" ? 0.1287 : 0.1859;+ $base := $dur * $rate + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0);+ $vids > 0+ ? {"type": "range_usd", "min_usd": $base + $vids * 2 * $rate,+ "max_usd": $base + 15 * $rate, "format": {"approximate": true}}+ : {"type": "usd", "usd": $base}+ ))""",),@@ -997,6 +1042,11 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):if not reference_images and not reference_videos:raise ValueError("At least one reference image or video is required.")+ is_max = model["model"] == HAILUO_03_MAX_MODEL+ min_clip_duration = 2.0 if is_max else 1.8+ max_total_duration = 15.0 if is_max else 15.1+ max_fps = 60.19 if is_max else 60.5+for image in reference_images.values():validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5validate_image_dimensions(image, min_width=256, min_height=256)@@ -1007,29 +1057,80 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):fps = float(video.get_frame_rate())except Exception:fps = 0.0- if fps and not (23.9 <= fps <= 60.5):+ if fps and not (23.9 <= fps <= max_fps):raise ValueError(f"Reference video {i} is {fps:.2f} FPS. Supported range is 23.976-60 FPS.")try:dur = video.get_duration()except Exception:continue- if dur < 1.8:+ if dur < min_clip_duration:raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")total_video_duration += dur- if total_video_duration > 15.1:+ if total_video_duration > max_total_duration:raise ValueError(- f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds."+ f"Total reference video duration is {total_video_duration:.2f}s. Maximum is 15 seconds.")total_audio_duration = 0.0for i, audio in enumerate(reference_audios.values(), 1):dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"])- if dur < 1.8:+ if dur < min_clip_duration:raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")total_audio_duration += dur- if total_audio_duration > 15.1:+ if total_audio_duration > max_total_duration:raise ValueError(- f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds."+ f"Total reference audio duration is {total_audio_duration:.2f}s. Maximum is 15 seconds."+ )++ if is_max:+ if watermark:+ raise ValueError("Watermark is only supported by MiniMax H3.")+ validate_string(model["prompt"], strip_whitespace=False, max_length=HAILUO_03_MAX_PROMPT_MAX_LENGTH)+ if len(reference_images) + len(reference_videos) + len(reference_audios) > 12:+ raise ValueError("MiniMax H3 Max accepts at most 12 reference files in total.")+ max_long_side, max_short_side = HAILUO_03_MAX_REFERENCE_IMAGE_MAX_SIDES[model["reference_detail"]]+ reference_image_urls = [+ (+ await upload_images_to_comfyapi(+ cls,+ downscale_image_tensor_by_max_sides(+ image, max_long_side=max_long_side, max_short_side=max_short_side+ ),+ max_images=1,+ total_pixels=None,+ wait_label=f"Uploading image {i}",+ )+ )[0]+ for i, image in enumerate(reference_images.values(), 1)+ ]+ reference_video_urls = [+ await upload_video_to_comfyapi(cls, video, wait_label=f"Uploading video {i}")+ for i, video in enumerate(reference_videos.values(), 1)+ ]+ reference_audio_urls = [+ await upload_audio_to_comfyapi(+ cls,+ audio,+ container_format="mp3",+ codec_name="libmp3lame",+ mime_type="audio/mpeg",+ )+ for audio in reference_audios.values()+ ]+ return await _hailuo03_max_run_task(+ cls,+ endpoint="reference-to-video",+ request=Hailuo03MaxVideoRequest(+ prompt=model["prompt"],+ duration=model["duration"],+ resolution=model["resolution"],+ prompt_expansion_mode=model["prompt_expansion_mode"],+ seed=seed,+ aspect_ratio=model["ratio"],+ reference_image_urls=reference_image_urls or None,+ reference_video_urls=reference_video_urls or None,+ reference_audio_urls=reference_audio_urls or None,+ ),)content: list = [Hailuo03TextContent(text=model["prompt"])]diff --git a/comfy_api_nodes/util/__init__.py b/comfy_api_nodes/util/__init__.pyindex 2bb4a1b04..e351c50be 100644--- a/comfy_api_nodes/util/__init__.py+++ b/comfy_api_nodes/util/__init__.py@@ -16,6 +16,7 @@ from .conversions import (convert_mask_to_image,downscale_image_tensor,downscale_image_tensor_by_max_side,+ downscale_image_tensor_by_max_sides,downscale_video_to_max_pixels,image_tensor_pair_to_batch,pad_images_to_common_channels,@@ -91,6 +92,7 @@ __all__ = ["convert_mask_to_image","downscale_image_tensor","downscale_image_tensor_by_max_side",+ "downscale_image_tensor_by_max_sides","downscale_video_to_max_pixels","image_tensor_pair_to_batch","pad_images_to_common_channels",diff --git a/comfy_api_nodes/util/conversions.py b/comfy_api_nodes/util/conversions.pyindex 80c36eaff..a39c4e470 100644--- a/comfy_api_nodes/util/conversions.py+++ b/comfy_api_nodes/util/conversions.py@@ -165,7 +165,8 @@ def _compute_downscale_dims(src_w: int, src_h: int, total_pixels: int) -> tuple[def downscale_image_tensor(image: torch.Tensor, total_pixels: int = 1536 * 1024) -> torch.Tensor:"""Downscale input image tensor to roughly the specified total pixels.- Output dimensions are even and guaranteed to fit within ``total_pixels``+ Resized output has even dimensions and always fits within ``total_pixels``;+ an image that already fits is returned unchanged."""samples = image.movedim(-1, 1)dims = _compute_downscale_dims(samples.shape[3], samples.shape[2], int(total_pixels))@@ -175,6 +176,23 @@ def downscale_image_tensor(image: torch.Tensor, total_pixels: int = 1536 * 1024)return common_upscale(samples, new_w, new_h, "lanczos", "disabled").movedim(1, -1)+def downscale_image_tensor_by_max_sides(+ image: torch.Tensor, *, max_long_side: int, max_short_side: int+) -> torch.Tensor:+ """Downscale input image tensor so the long side is at most max_long_side and the short side at most max_short_side."""+ samples = image.movedim(-1, 1)+ height, width = samples.shape[2], samples.shape[3]+ long_side, short_side = max(width, height), min(width, height)+ scale_by = min(1.0, max_long_side / long_side, max_short_side / short_side)+ if scale_by >= 1.0:+ return image+ long_new = max(1, math.floor(long_side * scale_by))+ short_new = max(1, min(long_new, math.ceil(short_side * scale_by)))+ new_width, new_height = (long_new, short_new) if width >= height else (short_new, long_new)+ s = common_upscale(samples, new_width, new_height, "lanczos", "disabled")+ return s.movedim(1, -1)++def downscale_image_tensor_by_max_side(image: torch.Tensor, *, max_side: int) -> torch.Tensor:"""Downscale input image tensor so the largest dimension is at most max_side pixels."""samples = image.movedim(-1, 1)diff --git a/tests-unit/comfy_api_nodes_test/image_conversions_test.py b/tests-unit/comfy_api_nodes_test/image_conversions_test.pyindex adfcaa886..d54243923 100644--- a/tests-unit/comfy_api_nodes_test/image_conversions_test.py+++ b/tests-unit/comfy_api_nodes_test/image_conversions_test.py@@ -10,6 +10,7 @@ if not torch.cuda.is_available():args.cpu = Truefrom comfy_api_nodes.util.conversions import ( # noqa: E402+ downscale_image_tensor_by_max_sides,bytesio_to_image_tensor,downscale_image_tensor,pad_images_to_common_channels,@@ -126,3 +127,23 @@ def test_downscale_never_makes_aspect_more_elongated(width, height, total_pixelsdef test_downscale_leaves_fitting_images_untouched():image = torch.zeros(1, 300, 700, 3)assert downscale_image_tensor(image, total_pixels=700 * 300) is image+++ "src, expected",+ [+ ((5120, 2048), (2048, 820)),+ ((2048, 5120), (820, 2048)),+ ((4096, 4096), (1024, 1024)),+ ((2048, 1024), (2048, 1024)),+ ((768, 432), (768, 432)),+ ((1000, 3000), (683, 2048)),+ ],+)+def test_downscale_by_max_sides(src, expected):+ w, h = src+ out = downscale_image_tensor_by_max_sides(torch.zeros(1, h, w, 3), max_long_side=2048, max_short_side=1024)+ assert (out.shape[2], out.shape[1]) == expected+ src_ar = max(w, h) / min(w, h)+ out_ar = max(out.shape[1], out.shape[2]) / min(out.shape[1], out.shape[2])+ assert out_ar <= src_ar + 1e-9