The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(MiniMax): add ContextIR and Regenerate nodes (#15471)
Alexander Piskun12666983cba9
diff --git a/comfy_api_nodes/apis/minimax.py b/comfy_api_nodes/apis/minimax.pyindex bac4572d4..12a0853ac 100644--- a/comfy_api_nodes/apis/minimax.py+++ b/comfy_api_nodes/apis/minimax.py@@ -161,12 +161,30 @@ class Hailuo03TaskCreationRequest(BaseModel):..., min_length=1)resolution: str = Field(...)- duration: int = Field(..., ge=5, le=15)+ duration: int = Field(..., ge=4, le=15)ratio: str | None = Field(None)seed: int | None = Field(None, ge=0, le=4294967295)aigc_watermark: bool | None = Field(None)+class Hailuo03ContextIRRequest(BaseModel):+ model: str = Field(...)+ content: list[Hailuo03TextContent | Hailuo03ImageContent | Hailuo03VideoContent | Hailuo03AudioContent] = Field(+ ..., min_length=1+ )+ duration: int = Field(..., ge=4, le=15)+ ratio: str | None = Field(None)+++class Hailuo03RegenerationRequest(BaseModel):+ model: str = Field(...)+ content: list[Hailuo03TextContent | Hailuo03ImageContent | Hailuo03VideoContent | Hailuo03AudioContent] = Field(+ ..., min_length=1+ )+ resolution: str = Field(...)+ aigc_watermark: bool | None = Field(None)++class Hailuo03TaskCreationResponse(BaseModel):task_id: str = Field(...)@@ -178,6 +196,7 @@ class Hailuo03TaskError(BaseModel):class Hailuo03TaskContent(BaseModel):url: str | None = Field(None)+ prompt: str | None = Field(None)class Hailuo03TaskUsage(BaseModel):diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.pyindex 3c1d29257..de3895221 100644--- a/comfy_api_nodes/nodes_minimax.py+++ b/comfy_api_nodes/nodes_minimax.py@@ -3,12 +3,14 @@ from typing import Optionalimport torchfrom typing_extensions import override-from comfy_api.latest import IO, ComfyExtension+from comfy_api.latest import IO, ComfyExtension, Inputfrom comfy_api_nodes.apis.minimax import (Hailuo03AudioContent,Hailuo03AudioContentUrl,+ Hailuo03ContextIRRequest,Hailuo03ImageContent,Hailuo03ImageContentUrl,+ Hailuo03RegenerationRequest,Hailuo03TaskCreationRequest,Hailuo03TaskCreationResponse,Hailuo03TaskQueryResponse,@@ -456,6 +458,9 @@ HAILUO_03_QUERY_ENDPOINT = "/proxy/minimax/v2/query/video_generation" # + /{tasHAILUO_03_MODELS = {"MiniMax H3": "MiniMax-H3"}HAILUO_03_FAILED_STATUSES = ["failed", "cancelled", "expired"]+HAILUO_03_CONTEXT_IR_ENDPOINT = "/proxy/minimax/v2/h3_context_ir"+HAILUO_03_REGENERATION_ENDPOINT = "/proxy/minimax/v2/video_regeneration"+def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = True):inputs = [@@ -487,10 +492,10 @@ def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = TrIO.Int.Input("duration",default=5,- min=5,+ min=4,max=15,step=1,- tooltip="Duration of the output video in seconds (5-15).",+ tooltip="Duration of the output video in seconds (4-15).",display_mode=IO.NumberDisplay.slider,))@@ -939,6 +944,592 @@ class MinimaxHailuo03ReferenceNode(IO.ComfyNode):)+class MinimaxHailuo03ContextIRNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="MinimaxHailuo03ContextIRNode",+ display_name="MiniMax H3 Context IR (Prompt Enhancer)",+ category="partner/video/MiniMax",+ description="Analyze text and media context with MiniMax H3 Context IR and produce an enhanced, "+ "structured video prompt. Feed the output into the prompt of a MiniMax H3 video node and attach "+ "the same media there in the same order, because the enhanced prompt refers to the attached "+ "media by position.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "MiniMax H3",+ [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Description of the video you intend to generate.",+ ),+ IO.Int.Input(+ "duration",+ default=5,+ min=4,+ max=15,+ step=1,+ tooltip="Duration of the video you intend to generate, in seconds (4-15).",+ display_mode=IO.NumberDisplay.slider,+ ),+ IO.Combo.Input(+ "ratio",+ options=["adaptive", "16:9", "4:3", "1:1", "3:4", "9:16", "21:9"],+ default="adaptive",+ tooltip="Aspect ratio of the video you intend to generate. 'adaptive' "+ "requires at least one image, video, or audio input.",+ ),+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("reference_image"),+ names=[+ "image_1",+ "image_2",+ "image_3",+ "image_4",+ "image_5",+ "image_6",+ "image_7",+ "image_8",+ "image_9",+ ],+ min=0,+ ),+ tooltip="Subject or style reference images, referred to in the prompt "+ "as 'Image 1'..'Image 9' in connection order. Up to 9 images.",+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=["video_1", "video_2", "video_3"],+ min=0,+ ),+ tooltip="Motion or scene reference videos, referred to in the prompt "+ "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, "+ "2-15 seconds each, 15 seconds in total.",+ ),+ IO.Autogrow.Input(+ "reference_audios",+ template=IO.Autogrow.TemplateNames(+ IO.Audio.Input("reference_audio"),+ names=["audio_1", "audio_2", "audio_3"],+ min=0,+ ),+ tooltip="Audio references, referred to in the prompt as "+ "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, "+ "2-15 seconds each, 15 seconds in total. Cannot be used without "+ "a reference image or video.",+ ),+ ],+ )+ ],+ tooltip="Model to use for prompt enhancement.",+ ),+ IO.Image.Input(+ "first_frame",+ tooltip="First frame of the video you intend to generate. Cannot be combined with "+ "reference media.",+ optional=True,+ ),+ IO.Image.Input(+ "last_frame",+ tooltip="Last frame of the video you intend to generate. Cannot be combined with "+ "reference media.",+ optional=True,+ ),+ ],+ outputs=[+ IO.String.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(+ inputs=["first_frame", "last_frame"],+ input_groups=["model.reference_images", "model.reference_videos", "model.reference_audios"],+ ),+ expr="""+ (+ $imgsRaw := $lookup(inputGroups, "model.reference_images");+ $imgs := $imgsRaw ? $imgsRaw : 0;+ $vidsRaw := $lookup(inputGroups, "model.reference_videos");+ $vids := $vidsRaw ? $vidsRaw : 0;+ $audsRaw := $lookup(inputGroups, "model.reference_audios");+ $auds := $audsRaw ? $audsRaw : 0;+ $frames := (inputs.first_frame.connected ? 1 : 0) + (inputs.last_frame.connected ? 1 : 0);+ ($imgs + $vids + $auds) > 0+ ? {"type": "range_usd", "min_usd": 0.06, "max_usd": 0.11, "format": {"approximate": true}}+ : $frames > 0+ ? {"type": "usd", "usd": 0.05, "format": {"approximate": true}}+ : {"type": "usd", "usd": 0.02, "format": {"approximate": true}}+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ first_frame: torch.Tensor | None = None,+ last_frame: torch.Tensor | None = None,+ ) -> IO.NodeOutput:+ validate_string(model["prompt"], strip_whitespace=True, min_length=1)++ reference_images = {k: v for k, v in (model.get("reference_images") or {}).items() if v is not None}+ reference_videos = {k: v for k, v in (model.get("reference_videos") or {}).items() if v is not None}+ reference_audios = {k: v for k, v in (model.get("reference_audios") or {}).items() if v is not None}+ has_frames = first_frame is not None or last_frame is not None+ has_references = bool(reference_images) or bool(reference_videos) or bool(reference_audios)+ if has_frames and has_references:+ raise ValueError(+ "First/last frame and reference media are mutually exclusive. Use frames for an "+ "image-to-video prompt, or reference media for a reference-to-video prompt."+ )+ if reference_audios and not reference_images and not reference_videos:+ raise ValueError("Reference audio cannot be used without a reference image or video.")+ if not has_frames and not has_references and model["ratio"] == "adaptive":+ raise ValueError(+ "Ratio 'adaptive' is not supported for text-only requests; select an explicit aspect ratio."+ )++ for frame in (first_frame, last_frame):+ if frame is not None:+ validate_image_aspect_ratio(frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5+ validate_image_dimensions(frame, min_width=256, min_height=256)+ for image in reference_images.values():+ validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5+ validate_image_dimensions(image, min_width=256, min_height=256)++ total_video_duration = 0.0+ for i, video in enumerate(reference_videos.values(), 1):+ try:+ fps = float(video.get_frame_rate())+ except Exception:+ fps = 0.0+ if fps and not (23.9 <= fps <= 60.5):+ raise ValueError(f"Reference video {i} is {fps:.2f} FPS. Supported range is 23.976-60 FPS.")+ try:+ dur = video.get_duration()+ except Exception:+ continue+ if dur < 1.8:+ raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")+ total_video_duration += dur+ if total_video_duration > 15.1:+ raise ValueError(+ f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds."+ )++ total_audio_duration = 0.0+ for i, audio in enumerate(reference_audios.values(), 1):+ dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"])+ if dur < 1.8:+ raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")+ total_audio_duration += dur+ if total_audio_duration > 15.1:+ raise ValueError(+ f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds."+ )++ content: list = [Hailuo03TextContent(text=model["prompt"])]+ if first_frame is not None:+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, first_frame, max_images=1, wait_label="Uploading first frame"+ )+ )[0],+ ),+ role="first_frame",+ )+ )+ if last_frame is not None:+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, last_frame, max_images=1, wait_label="Uploading last frame"+ )+ )[0],+ ),+ role="last_frame",+ )+ )+ for i, image in enumerate(reference_images.values(), 1):+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, image, max_images=1, wait_label=f"Uploading image {i}"+ )+ )[0],+ ),+ role="reference_image",+ )+ )+ for i, video in enumerate(reference_videos.values(), 1):+ content.append(+ Hailuo03VideoContent(+ video_url=Hailuo03VideoContentUrl(+ url=await upload_video_to_comfyapi(cls, video, wait_label=f"Uploading video {i}"),+ ),+ )+ )+ for audio in reference_audios.values():+ content.append(+ Hailuo03AudioContent(+ audio_url=Hailuo03AudioContentUrl(+ url=await upload_audio_to_comfyapi(+ cls,+ audio,+ container_format="mp3",+ codec_name="libmp3lame",+ mime_type="audio/mpeg",+ ),+ ),+ )+ )++ response = await sync_op(+ cls,+ ApiEndpoint(path=HAILUO_03_CONTEXT_IR_ENDPOINT, method="POST"),+ response_model=Hailuo03TaskCreationResponse,+ data=Hailuo03ContextIRRequest(+ model=HAILUO_03_MODELS[model["model"]],+ content=content,+ duration=model["duration"],+ ratio=None if model["ratio"] == "adaptive" else model["ratio"],+ ),+ )+ task_result = await poll_op(+ cls,+ ApiEndpoint(path=f"{HAILUO_03_QUERY_ENDPOINT}/{response.task_id}"),+ response_model=Hailuo03TaskQueryResponse,+ status_extractor=lambda r: r.task.status,+ failed_statuses=HAILUO_03_FAILED_STATUSES,+ poll_interval=5,+ )+ prompt = task_result.task.content.prompt if task_result.task.content else None+ if not prompt:+ raise Exception(f"No enhanced prompt in the response: {task_result.model_dump()}")+ return IO.NodeOutput(prompt)+++class MinimaxHailuo03RegenerateNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="MinimaxHailuo03RegenerateNode",+ display_name="MiniMax H3 Regenerate to 2K",+ category="partner/video/MiniMax",+ description="Re-render a MiniMax H3 768P output at 2K resolution. Connect the unmodified 768P "+ "video and the exact prompt used to generate it; if the original generation used first/last "+ "frames or reference media, attach the same inputs.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "MiniMax H3",+ [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="The exact prompt used to generate the source video.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["2K"],+ tooltip="Resolution to re-render the source video at.",+ ),+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("reference_image"),+ names=[+ "image_1",+ "image_2",+ "image_3",+ "image_4",+ "image_5",+ "image_6",+ "image_7",+ "image_8",+ "image_9",+ ],+ min=0,+ ),+ tooltip="Reference images from the original generation, in the same "+ "order. Up to 9 images.",+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=["video_1", "video_2", "video_3"],+ min=0,+ ),+ tooltip="Reference videos from the original generation, in the same "+ "order. Up to 3 videos, 2-15 seconds each, 15 seconds in total.",+ ),+ IO.Autogrow.Input(+ "reference_audios",+ template=IO.Autogrow.TemplateNames(+ IO.Audio.Input("reference_audio"),+ names=["audio_1", "audio_2", "audio_3"],+ min=0,+ ),+ tooltip="Audio references from the original generation, in the same "+ "order. Up to 3 clips, 2-15 seconds each, 15 seconds in total. "+ "Cannot be used without a reference image or video.",+ ),+ ],+ )+ ],+ tooltip="Model to use for video regeneration.",+ ),+ IO.Video.Input(+ "video",+ tooltip="The MiniMax H3 768P output video to re-render. Connect the unmodified output "+ "of a MiniMax H3 video node (24 FPS, 4-15 seconds). 2K outputs cannot be used.",+ ),+ IO.Image.Input(+ "first_frame",+ tooltip="First frame image from the original generation, if one was used.",+ optional=True,+ ),+ IO.Image.Input(+ "last_frame",+ tooltip="Last frame image from the original generation, if one was used.",+ optional=True,+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AIGC watermark to the video.",+ advanced=True,+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ expr="""{"type": "usd", "usd": 0.0715, "format": {"suffix": "/second"}}""",+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ video: Input.Video,+ watermark: bool,+ first_frame: torch.Tensor | None = None,+ last_frame: torch.Tensor | None = None,+ ) -> IO.NodeOutput:+ validate_string(model["prompt"], strip_whitespace=True, min_length=1)++ try:+ fps = float(video.get_frame_rate())+ except Exception:+ fps = 0.0+ if fps and not (23.9 <= fps <= 24.1):+ raise ValueError(+ f"The source video is {fps:.2f} FPS. Regeneration accepts unmodified MiniMax H3 768P "+ "outputs, which are 24 FPS."+ )+ try:+ width, height = video.get_dimensions()+ except Exception:+ width = height = 0+ if width and height and (width % 32 or height % 32 or width * height > 1_032_192):+ raise ValueError(+ f"The source video is {width}x{height}. Regeneration accepts MiniMax H3 768P outputs "+ "(width and height divisible by 32, at most 1,032,192 total pixels); 2K outputs cannot "+ "be used as a source."+ )+ try:+ frame_count = video.get_frame_count()+ except Exception:+ frame_count = 0+ if frame_count and (frame_count < 107 or frame_count > 362 or (frame_count - 107) % 17):+ raise ValueError(+ f"The source video has {frame_count} frames. Regeneration accepts unmodified "+ "MiniMax H3 outputs, whose length is 107 to 362 frames in steps of 17 "+ "(4 to 15 seconds at 24 FPS)."+ )++ reference_images = {k: v for k, v in (model.get("reference_images") or {}).items() if v is not None}+ reference_videos = {k: v for k, v in (model.get("reference_videos") or {}).items() if v is not None}+ reference_audios = {k: v for k, v in (model.get("reference_audios") or {}).items() if v is not None}+ if (first_frame is not None or last_frame is not None) and (+ reference_images or reference_videos or reference_audios+ ):+ raise ValueError(+ "First/last frame and reference media are mutually exclusive. Use frames for an "+ "image-to-video prompt, or reference media for a reference-to-video prompt."+ )+ if reference_audios and not reference_images and not reference_videos:+ raise ValueError("Reference audio cannot be used without a reference image or video.")++ for frame in (first_frame, last_frame):+ if frame is not None:+ validate_image_aspect_ratio(frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5+ validate_image_dimensions(frame, min_width=256, min_height=256)+ for image in reference_images.values():+ validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5+ validate_image_dimensions(image, min_width=256, min_height=256)++ total_video_duration = 0.0+ for i, ref_video in enumerate(reference_videos.values(), 1):+ try:+ ref_fps = float(ref_video.get_frame_rate())+ except Exception:+ ref_fps = 0.0+ if ref_fps and not (23.9 <= ref_fps <= 60.5):+ raise ValueError(f"Reference video {i} is {ref_fps:.2f} FPS. Supported range is 23.976-60 FPS.")+ try:+ dur = ref_video.get_duration()+ except Exception:+ continue+ if dur < 1.8:+ raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")+ total_video_duration += dur+ if total_video_duration > 15.1:+ raise ValueError(+ f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds."+ )++ total_audio_duration = 0.0+ for i, audio in enumerate(reference_audios.values(), 1):+ dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"])+ if dur < 1.8:+ raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")+ total_audio_duration += dur+ if total_audio_duration > 15.1:+ raise ValueError(+ f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds."+ )++ content: list = [+ Hailuo03VideoContent(+ video_url=Hailuo03VideoContentUrl(+ url=await upload_video_to_comfyapi(cls, video, wait_label="Uploading source video"),+ ),+ role="base_video",+ ),+ Hailuo03TextContent(text=model["prompt"]),+ ]+ if first_frame is not None:+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, first_frame, max_images=1, wait_label="Uploading first frame"+ )+ )[0],+ ),+ role="first_frame",+ )+ )+ if last_frame is not None:+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, last_frame, max_images=1, wait_label="Uploading last frame"+ )+ )[0],+ ),+ role="last_frame",+ )+ )+ for i, image in enumerate(reference_images.values(), 1):+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, image, max_images=1, wait_label=f"Uploading image {i}"+ )+ )[0],+ ),+ role="reference_image",+ )+ )+ for i, ref_video in enumerate(reference_videos.values(), 1):+ content.append(+ Hailuo03VideoContent(+ video_url=Hailuo03VideoContentUrl(+ url=await upload_video_to_comfyapi(cls, ref_video, wait_label=f"Uploading video {i}"),+ ),+ )+ )+ for audio in reference_audios.values():+ content.append(+ Hailuo03AudioContent(+ audio_url=Hailuo03AudioContentUrl(+ url=await upload_audio_to_comfyapi(+ cls,+ audio,+ container_format="mp3",+ codec_name="libmp3lame",+ mime_type="audio/mpeg",+ ),+ ),+ )+ )++ response = await sync_op(+ cls,+ ApiEndpoint(path=HAILUO_03_REGENERATION_ENDPOINT, method="POST"),+ response_model=Hailuo03TaskCreationResponse,+ data=Hailuo03RegenerationRequest(+ model=HAILUO_03_MODELS[model["model"]],+ content=content,+ resolution=model["resolution"],+ aigc_watermark=watermark,+ ),+ )+ task_result = await poll_op(+ cls,+ ApiEndpoint(path=f"{HAILUO_03_QUERY_ENDPOINT}/{response.task_id}"),+ response_model=Hailuo03TaskQueryResponse,+ status_extractor=lambda r: r.task.status,+ failed_statuses=HAILUO_03_FAILED_STATUSES,+ poll_interval=10,+ )+ video_url = task_result.task.content.url if task_result.task.content else None+ if not video_url:+ raise Exception(f"No video URL in the response: {task_result.model_dump()}")+ return IO.NodeOutput(await download_url_to_video_output(video_url))++class MinimaxExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[IO.ComfyNode]]:@@ -950,6 +1541,8 @@ class MinimaxExtension(ComfyExtension):MinimaxHailuo03TextToVideoNode,MinimaxHailuo03FirstLastFrameNode,MinimaxHailuo03ReferenceNode,+ MinimaxHailuo03ContextIRNode,+ MinimaxHailuo03RegenerateNode,]