Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(Minimax-H3): add the Max model to the Reference node (#16041)

Alexander Piskunec803fc97ee3
diff --git a/comfy_api_nodes/apis/minimax.py b/comfy_api_nodes/apis/minimax.py
index e7c128f25..46a13330d 100644
--- a/comfy_api_nodes/apis/minimax.py
+++ b/comfy_api_nodes/apis/minimax.py
@@ -242,6 +242,9 @@ class Hailuo03MaxVideoRequest(BaseModel):
aspect_ratio: str | None = Field(None)
image_url: str | None = Field(None)
end_image_url: str | None = Field(None)
+ reference_image_urls: list[str] | None = Field(None)
+ reference_video_urls: list[str] | None = Field(None)
+ reference_audio_urls: list[str] | None = Field(None)
class Hailuo03MaxVideoResult(BaseModel):
diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.py
index 3663d7ec0..8a182fee4 100644
--- a/comfy_api_nodes/nodes_minimax.py
+++ b/comfy_api_nodes/nodes_minimax.py
@@ -31,6 +31,7 @@ from comfy_api_nodes.apis.minimax import (
from comfy_api_nodes.util import (
ApiEndpoint,
download_url_to_video_output,
+ downscale_image_tensor_by_max_sides,
poll_op,
sync_op,
upload_audio_to_comfyapi,
@@ -468,6 +469,7 @@ HAILUO_03_REGENERATION_ENDPOINT = "/proxy/minimax/v2/video_regeneration"
HAILUO_03_MAX_MODEL = "MiniMax H3 Max"
HAILUO_03_MAX_ENDPOINT = "/proxy/fal/minimax/h3-max"
HAILUO_03_MAX_PROMPT_MAX_LENGTH = 50000
+HAILUO_03_MAX_REFERENCE_IMAGE_MAX_SIDES = {"high": (5120, 2048), "standard": (2048, 1024)}
def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = True):
@@ -549,7 +551,7 @@ async def _hailuo03_run_task(
return IO.NodeOutput(await download_url_to_video_output(video_url))
-def _hailuo03_max_model_inputs(include_ratio: bool = True):
+def _hailuo03_max_model_inputs(include_ratio: bool = True, allow_adaptive: bool = False):
inputs = [
IO.String.Input(
"prompt",
@@ -565,11 +567,14 @@ def _hailuo03_max_model_inputs(include_ratio: bool = True):
),
]
if include_ratio:
+ ratio_options = ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"]
+ if allow_adaptive:
+ ratio_options.insert(0, "adaptive")
inputs.append(
IO.Combo.Input(
"ratio",
- options=["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],
- default="16:9",
+ options=ratio_options,
+ default=ratio_options[0],
tooltip="Aspect ratio of the output video.",
)
)
@@ -866,6 +871,54 @@ class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode):
)
+def _hailuo03_reference_inputs():
+ return [
+ IO.Autogrow.Input(
+ "reference_images",
+ template=IO.Autogrow.TemplateNames(
+ IO.Image.Input("reference_image"),
+ names=[
+ "image_1",
+ "image_2",
+ "image_3",
+ "image_4",
+ "image_5",
+ "image_6",
+ "image_7",
+ "image_8",
+ "image_9",
+ ],
+ min=0,
+ ),
+ tooltip="Subject or style reference images, referred to in the prompt "
+ "as 'Image 1'..'Image 9' in connection order. Up to 9 images.",
+ ),
+ IO.Autogrow.Input(
+ "reference_videos",
+ template=IO.Autogrow.TemplateNames(
+ IO.Video.Input("reference_video"),
+ names=["video_1", "video_2", "video_3"],
+ min=0,
+ ),
+ tooltip="Motion or scene reference videos, referred to in the prompt "
+ "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, "
+ "2-15 seconds each, 15 seconds in total.",
+ ),
+ IO.Autogrow.Input(
+ "reference_audios",
+ template=IO.Autogrow.TemplateNames(
+ IO.Audio.Input("reference_audio"),
+ names=["audio_1", "audio_2", "audio_3"],
+ min=0,
+ ),
+ tooltip="Audio references, referred to in the prompt as "
+ "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, "
+ "2-15 seconds each, 15 seconds in total. Cannot be used without "
+ "a reference image or video.",
+ ),
+ ]
+
+
class MinimaxHailuo03ReferenceNode(IO.ComfyNode):
@classmethod
def define_schema(cls):
@@ -874,7 +927,7 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):
display_name="MiniMax H3 Reference to Video",
category="partner/video/MiniMax",
description="Generate video conditioned on reference images, videos, and audio using the "
- "MiniMax H3 model. Refer to the references in the prompt by their order: "
+ "MiniMax H3 models. Refer to the references in the prompt by their order: "
"'Image 1', 'Image 2', 'Video 1', 'Audio 1', and so on.",
inputs=[
IO.DynamicCombo.Input(
@@ -882,53 +935,23 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):
options=[
IO.DynamicCombo.Option(
"MiniMax H3",
+ [*_hailuo03_model_inputs(), *_hailuo03_reference_inputs()],
+ ),
+ IO.DynamicCombo.Option(
+ HAILUO_03_MAX_MODEL,
[
- *_hailuo03_model_inputs(),
- IO.Autogrow.Input(
- "reference_images",
- template=IO.Autogrow.TemplateNames(
- IO.Image.Input("reference_image"),
- names=[
- "image_1",
- "image_2",
- "image_3",
- "image_4",
- "image_5",
- "image_6",
- "image_7",
- "image_8",
- "image_9",
- ],
- min=0,
- ),
- tooltip="Subject or style reference images, referred to in the prompt "
- "as 'Image 1'..'Image 9' in connection order. Up to 9 images.",
- ),
- IO.Autogrow.Input(
- "reference_videos",
- template=IO.Autogrow.TemplateNames(
- IO.Video.Input("reference_video"),
- names=["video_1", "video_2", "video_3"],
- min=0,
- ),
- tooltip="Motion or scene reference videos, referred to in the prompt "
- "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, "
- "2-15 seconds each, 15 seconds in total.",
- ),
- IO.Autogrow.Input(
- "reference_audios",
- template=IO.Autogrow.TemplateNames(
- IO.Audio.Input("reference_audio"),
- names=["audio_1", "audio_2", "audio_3"],
- min=0,
- ),
- tooltip="Audio references, referred to in the prompt as "
- "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, "
- "2-15 seconds each, 15 seconds in total. Cannot be used without "
- "a reference image or video.",
+ *_hailuo03_max_model_inputs(allow_adaptive=True),
+ IO.Combo.Input(
+ "reference_detail",
+ options=["high", "standard"],
+ default="standard",
+ tooltip="Detail level at which reference images are sent. 'high' sends them at "
+ "the largest size the model uses (up to a 2048 pixel short side); "
+ "'standard' downsizes them to at most 2048x1024 to reduce the reference cost.",
),
+ *_hailuo03_reference_inputs(),
],
- )
+ ),
],
tooltip="Model to use for video generation.",
),
@@ -961,22 +984,44 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):
is_api_node=True,
price_badge=IO.PriceBadge(
depends_on=IO.PriceBadgeDepends(
- widgets=["model.resolution", "model.duration"],
- input_groups=["model.reference_images", "model.reference_videos"],
+ widgets=["model", "model.resolution", "model.duration", "model.reference_detail"],
+ input_groups=["model.reference_images", "model.reference_videos", "model.reference_audios"],
),
expr="""
(
$dur := $lookup(widgets, "model.duration");
- $rate := $lookup(widgets, "model.resolution") = "768p" ? 0.1287 : 0.1859;
+ $res := $lookup(widgets, "model.resolution");
$imgsRaw := $lookup(inputGroups, "model.reference_images");
$imgs := $imgsRaw ? $imgsRaw : 0;
$vidsRaw := $lookup(inputGroups, "model.reference_videos");
$vids := $vidsRaw ? $vidsRaw : 0;
- $base := $dur * $rate + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0);
- $vids > 0
- ? {"type": "range_usd", "min_usd": $base + $vids * 2 * $rate,
- "max_usd": $base + 15 * $rate, "format": {"approximate": true}}
- : {"type": "usd", "usd": $base}
+ $audsRaw := $lookup(inputGroups, "model.reference_audios");
+ $auds := $audsRaw ? $audsRaw : 0;
+ $lookup(widgets, "model") = "minimax h3 max"
+ ? (
+ $unitRate := $res = "480p" ? 1 : 1.6;
+ $frameTokens := $res = "480p" ? 390 : 1008;
+ $base := $dur * $unitRate;
+ $minTokens := $imgs * 576 + $vids * 12 * $frameTokens + $auds * 160;
+ $capFrames := $vids * (7 * $dur + 2);
+ $maxFrames := $capFrames > 265 ? 265 : $capFrames;
+ $imgMax := $lookup(widgets, "model.reference_detail") = "standard" ? 2560 : 10240;
+ $maxTokens := $imgs * $imgMax + $maxFrames * $frameTokens + ($auds > 0 ? 1200 : 0);
+ $minUnits := $base + ($minTokens > 4096 ? ($minTokens - 4096) * 0.0004 : 0);
+ $maxUnits := $base + ($maxTokens > 4096 ? ($maxTokens - 4096) * 0.0004 : 0);
+ $minUnits = $maxUnits
+ ? {"type": "usd", "usd": $base * 0.0715}
+ : {"type": "range_usd", "min_usd": $minUnits * 0.0715, "max_usd": $maxUnits * 0.0715,
+ "format": {"approximate": true}}
+ )
+ : (
+ $rate := $res = "768p" ? 0.1287 : 0.1859;
+ $base := $dur * $rate + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0);
+ $vids > 0
+ ? {"type": "range_usd", "min_usd": $base + $vids * 2 * $rate,
+ "max_usd": $base + 15 * $rate, "format": {"approximate": true}}
+ : {"type": "usd", "usd": $base}
+ )
)
""",
),
@@ -997,6 +1042,11 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):
if not reference_images and not reference_videos:
raise ValueError("At least one reference image or video is required.")
+ is_max = model["model"] == HAILUO_03_MAX_MODEL
+ min_clip_duration = 2.0 if is_max else 1.8
+ max_total_duration = 15.0 if is_max else 15.1
+ max_fps = 60.19 if is_max else 60.5
+
for image in reference_images.values():
validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5
validate_image_dimensions(image, min_width=256, min_height=256)
@@ -1007,29 +1057,80 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):
fps = float(video.get_frame_rate())
except Exception:
fps = 0.0
- if fps and not (23.9 <= fps <= 60.5):
+ if fps and not (23.9 <= fps <= max_fps):
raise ValueError(f"Reference video {i} is {fps:.2f} FPS. Supported range is 23.976-60 FPS.")
try:
dur = video.get_duration()
except Exception:
continue
- if dur < 1.8:
+ if dur < min_clip_duration:
raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")
total_video_duration += dur
- if total_video_duration > 15.1:
+ if total_video_duration > max_total_duration:
raise ValueError(
- f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds."
+ f"Total reference video duration is {total_video_duration:.2f}s. Maximum is 15 seconds."
)
total_audio_duration = 0.0
for i, audio in enumerate(reference_audios.values(), 1):
dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"])
- if dur < 1.8:
+ if dur < min_clip_duration:
raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")
total_audio_duration += dur
- if total_audio_duration > 15.1:
+ if total_audio_duration > max_total_duration:
raise ValueError(
- f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds."
+ f"Total reference audio duration is {total_audio_duration:.2f}s. Maximum is 15 seconds."
+ )
+
+ if is_max:
+ if watermark:
+ raise ValueError("Watermark is only supported by MiniMax H3.")
+ validate_string(model["prompt"], strip_whitespace=False, max_length=HAILUO_03_MAX_PROMPT_MAX_LENGTH)
+ if len(reference_images) + len(reference_videos) + len(reference_audios) > 12:
+ raise ValueError("MiniMax H3 Max accepts at most 12 reference files in total.")
+ max_long_side, max_short_side = HAILUO_03_MAX_REFERENCE_IMAGE_MAX_SIDES[model["reference_detail"]]
+ reference_image_urls = [
+ (
+ await upload_images_to_comfyapi(
+ cls,
+ downscale_image_tensor_by_max_sides(
+ image, max_long_side=max_long_side, max_short_side=max_short_side
+ ),
+ max_images=1,
+ total_pixels=None,
+ wait_label=f"Uploading image {i}",
+ )
+ )[0]
+ for i, image in enumerate(reference_images.values(), 1)
+ ]
+ reference_video_urls = [
+ await upload_video_to_comfyapi(cls, video, wait_label=f"Uploading video {i}")
+ for i, video in enumerate(reference_videos.values(), 1)
+ ]
+ reference_audio_urls = [
+ await upload_audio_to_comfyapi(
+ cls,
+ audio,
+ container_format="mp3",
+ codec_name="libmp3lame",
+ mime_type="audio/mpeg",
+ )
+ for audio in reference_audios.values()
+ ]
+ return await _hailuo03_max_run_task(
+ cls,
+ endpoint="reference-to-video",
+ request=Hailuo03MaxVideoRequest(
+ prompt=model["prompt"],
+ duration=model["duration"],
+ resolution=model["resolution"],
+ prompt_expansion_mode=model["prompt_expansion_mode"],
+ seed=seed,
+ aspect_ratio=model["ratio"],
+ reference_image_urls=reference_image_urls or None,
+ reference_video_urls=reference_video_urls or None,
+ reference_audio_urls=reference_audio_urls or None,
+ ),
)
content: list = [Hailuo03TextContent(text=model["prompt"])]
diff --git a/comfy_api_nodes/util/__init__.py b/comfy_api_nodes/util/__init__.py
index 2bb4a1b04..e351c50be 100644
--- a/comfy_api_nodes/util/__init__.py
+++ b/comfy_api_nodes/util/__init__.py
@@ -16,6 +16,7 @@ from .conversions import (
convert_mask_to_image,
downscale_image_tensor,
downscale_image_tensor_by_max_side,
+ downscale_image_tensor_by_max_sides,
downscale_video_to_max_pixels,
image_tensor_pair_to_batch,
pad_images_to_common_channels,
@@ -91,6 +92,7 @@ __all__ = [
"convert_mask_to_image",
"downscale_image_tensor",
"downscale_image_tensor_by_max_side",
+ "downscale_image_tensor_by_max_sides",
"downscale_video_to_max_pixels",
"image_tensor_pair_to_batch",
"pad_images_to_common_channels",
diff --git a/comfy_api_nodes/util/conversions.py b/comfy_api_nodes/util/conversions.py
index 80c36eaff..a39c4e470 100644
--- a/comfy_api_nodes/util/conversions.py
+++ b/comfy_api_nodes/util/conversions.py
@@ -165,7 +165,8 @@ def _compute_downscale_dims(src_w: int, src_h: int, total_pixels: int) -> tuple[
def downscale_image_tensor(image: torch.Tensor, total_pixels: int = 1536 * 1024) -> torch.Tensor:
"""Downscale input image tensor to roughly the specified total pixels.
- Output dimensions are even and guaranteed to fit within ``total_pixels``
+ Resized output has even dimensions and always fits within ``total_pixels``;
+ an image that already fits is returned unchanged.
"""
samples = image.movedim(-1, 1)
dims = _compute_downscale_dims(samples.shape[3], samples.shape[2], int(total_pixels))
@@ -175,6 +176,23 @@ def downscale_image_tensor(image: torch.Tensor, total_pixels: int = 1536 * 1024)
return common_upscale(samples, new_w, new_h, "lanczos", "disabled").movedim(1, -1)
+def downscale_image_tensor_by_max_sides(
+ image: torch.Tensor, *, max_long_side: int, max_short_side: int
+) -> torch.Tensor:
+ """Downscale input image tensor so the long side is at most max_long_side and the short side at most max_short_side."""
+ samples = image.movedim(-1, 1)
+ height, width = samples.shape[2], samples.shape[3]
+ long_side, short_side = max(width, height), min(width, height)
+ scale_by = min(1.0, max_long_side / long_side, max_short_side / short_side)
+ if scale_by >= 1.0:
+ return image
+ long_new = max(1, math.floor(long_side * scale_by))
+ short_new = max(1, min(long_new, math.ceil(short_side * scale_by)))
+ new_width, new_height = (long_new, short_new) if width >= height else (short_new, long_new)
+ s = common_upscale(samples, new_width, new_height, "lanczos", "disabled")
+ return s.movedim(1, -1)
+
+
def downscale_image_tensor_by_max_side(image: torch.Tensor, *, max_side: int) -> torch.Tensor:
"""Downscale input image tensor so the largest dimension is at most max_side pixels."""
samples = image.movedim(-1, 1)
diff --git a/tests-unit/comfy_api_nodes_test/image_conversions_test.py b/tests-unit/comfy_api_nodes_test/image_conversions_test.py
index adfcaa886..d54243923 100644
--- a/tests-unit/comfy_api_nodes_test/image_conversions_test.py
+++ b/tests-unit/comfy_api_nodes_test/image_conversions_test.py
@@ -10,6 +10,7 @@ if not torch.cuda.is_available():
args.cpu = True
from comfy_api_nodes.util.conversions import ( # noqa: E402
+ downscale_image_tensor_by_max_sides,
bytesio_to_image_tensor,
downscale_image_tensor,
pad_images_to_common_channels,
@@ -126,3 +127,23 @@ def test_downscale_never_makes_aspect_more_elongated(width, height, total_pixels
def test_downscale_leaves_fitting_images_untouched():
image = torch.zeros(1, 300, 700, 3)
assert downscale_image_tensor(image, total_pixels=700 * 300) is image
+
+
+ "src, expected",
+ [
+ ((5120, 2048), (2048, 820)),
+ ((2048, 5120), (820, 2048)),
+ ((4096, 4096), (1024, 1024)),
+ ((2048, 1024), (2048, 1024)),
+ ((768, 432), (768, 432)),
+ ((1000, 3000), (683, 2048)),
+ ],
+)
+def test_downscale_by_max_sides(src, expected):
+ w, h = src
+ out = downscale_image_tensor_by_max_sides(torch.zeros(1, h, w, 3), max_long_side=2048, max_short_side=1024)
+ assert (out.shape[2], out.shape[1]) == expected
+ src_ar = max(w, h) / min(w, h)
+ out_ar = max(out.shape[1], out.shape[2]) / min(out.shape[1], out.shape[2])
+ assert out_ar <= src_ar + 1e-9