The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Add minimax h3 support (#15167)
Daxiong (Lin)7dd462746012
diff --git a/comfy_api_nodes/apis/minimax.py b/comfy_api_nodes/apis/minimax.pyindex d747e177a..bac4572d4 100644--- a/comfy_api_nodes/apis/minimax.py+++ b/comfy_api_nodes/apis/minimax.py@@ -118,3 +118,81 @@ class MinimaxVideoGenerationResponse(BaseModel):task_id: str = Field(..., description='The task ID for the asynchronous video generation task.')+++class Hailuo03TextContent(BaseModel):+ type: str = Field("text")+ text: str = Field(...)+++class Hailuo03ImageContentUrl(BaseModel):+ url: str = Field(...)+++class Hailuo03ImageContent(BaseModel):+ type: str = Field("image_url")+ image_url: Hailuo03ImageContentUrl = Field(...)+ role: str = Field(...)+++class Hailuo03VideoContentUrl(BaseModel):+ url: str = Field(...)+++class Hailuo03VideoContent(BaseModel):+ type: str = Field("video_url")+ video_url: Hailuo03VideoContentUrl = Field(...)+ role: str = Field("reference_video")+++class Hailuo03AudioContentUrl(BaseModel):+ url: str = Field(...)+++class Hailuo03AudioContent(BaseModel):+ type: str = Field("audio_url")+ audio_url: Hailuo03AudioContentUrl = Field(...)+ role: str = Field("reference_audio")+++class Hailuo03TaskCreationRequest(BaseModel):+ model: str = Field(...)+ content: list[Hailuo03TextContent | Hailuo03ImageContent | Hailuo03VideoContent | Hailuo03AudioContent] = Field(+ ..., min_length=1+ )+ resolution: str = Field(...)+ duration: int = Field(..., ge=5, le=15)+ ratio: str | None = Field(None)+ seed: int | None = Field(None, ge=0, le=4294967295)+ aigc_watermark: bool | None = Field(None)+++class Hailuo03TaskCreationResponse(BaseModel):+ task_id: str = Field(...)+++class Hailuo03TaskError(BaseModel):+ code: int | str | None = Field(None)+ message: str | None = Field(None)+++class Hailuo03TaskContent(BaseModel):+ url: str | None = Field(None)+++class Hailuo03TaskUsage(BaseModel):+ total_seconds: float = Field(0)+ input_seconds: float = Field(0)+ output_seconds: float = Field(0)+++class Hailuo03Task(BaseModel):+ id: str = Field(...)+ status: str = Field(...)+ error: Hailuo03TaskError | None = Field(None)+ content: Hailuo03TaskContent | None = Field(None)+ usage: Hailuo03TaskUsage | None = Field(None)+++class Hailuo03TaskQueryResponse(BaseModel):+ task: Hailuo03Task = Field(...)diff --git a/comfy_api_nodes/nodes_minimax.py b/comfy_api_nodes/nodes_minimax.pyindex 6250af146..2d7aef654 100644--- a/comfy_api_nodes/nodes_minimax.py+++ b/comfy_api_nodes/nodes_minimax.py@@ -5,6 +5,16 @@ from typing_extensions import overridefrom comfy_api.latest import IO, ComfyExtensionfrom comfy_api_nodes.apis.minimax import (+ Hailuo03AudioContent,+ Hailuo03AudioContentUrl,+ Hailuo03ImageContent,+ Hailuo03ImageContentUrl,+ Hailuo03TaskCreationRequest,+ Hailuo03TaskCreationResponse,+ Hailuo03TaskQueryResponse,+ Hailuo03TextContent,+ Hailuo03VideoContent,+ Hailuo03VideoContentUrl,MinimaxFileRetrieveResponse,MiniMaxModel,MinimaxTaskResultResponse,@@ -17,7 +27,11 @@ from comfy_api_nodes.util import (download_url_to_video_output,poll_op,sync_op,+ upload_audio_to_comfyapi,upload_images_to_comfyapi,+ upload_video_to_comfyapi,+ validate_image_aspect_ratio,+ validate_image_dimensions,validate_string,)@@ -293,9 +307,9 @@ class MinimaxHailuoVideoNode(IO.ComfyNode):def define_schema(cls) -> IO.Schema:return IO.Schema(node_id="MinimaxHailuoVideoNode",- display_name="MiniMax Hailuo Video",+ display_name="MiniMax Hailuo 02 Video",category="partner/video/MiniMax",- description="Generates videos from prompt, with optional start frame using the new MiniMax Hailuo-02 model.",+ description="Generates videos from prompt, with optional start frame using the MiniMax Hailuo-02 model.",inputs=[IO.String.Input("prompt_text",@@ -437,6 +451,491 @@ class MinimaxHailuoVideoNode(IO.ComfyNode):return IO.NodeOutput(await download_url_to_video_output(file_url))+HAILUO_03_CREATE_ENDPOINT = "/proxy/minimax/v2/video_generation"+HAILUO_03_QUERY_ENDPOINT = "/proxy/minimax/v2/query/video_generation" # + /{task_id}+HAILUO_03_MODELS = {"MiniMax H3": "MiniMax-H3"}+HAILUO_03_FAILED_STATUSES = ["failed", "cancelled", "expired"]+++def _hailuo03_model_inputs(include_ratio: bool = True, allow_adaptive: bool = True):+ inputs = [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Text prompt for video generation.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["2K"],+ tooltip="Resolution of the output video.",+ ),+ ]+ if include_ratio:+ ratio_options = ["16:9", "4:3", "1:1", "3:4", "9:16", "21:9"]+ if allow_adaptive:+ ratio_options.insert(0, "adaptive")+ inputs.append(+ IO.Combo.Input(+ "ratio",+ options=ratio_options,+ default=ratio_options[0],+ tooltip="Aspect ratio of the output video.",+ )+ )+ inputs.append(+ IO.Int.Input(+ "duration",+ default=5,+ min=5,+ max=15,+ step=1,+ tooltip="Duration of the output video in seconds (5-15).",+ display_mode=IO.NumberDisplay.slider,+ )+ )+ return inputs+++async def _hailuo03_run_task(+ cls: type[IO.ComfyNode],+ *,+ model_id: str,+ content: list,+ resolution: str,+ duration: int,+ ratio: str | None,+ seed: int,+ watermark: bool,+) -> IO.NodeOutput:+ response = await sync_op(+ cls,+ ApiEndpoint(path=HAILUO_03_CREATE_ENDPOINT, method="POST"),+ response_model=Hailuo03TaskCreationResponse,+ data=Hailuo03TaskCreationRequest(+ model=model_id,+ content=content,+ resolution=resolution,+ duration=duration,+ ratio=ratio,+ seed=seed,+ aigc_watermark=watermark,+ ),+ )+ task_result = await poll_op(+ cls,+ ApiEndpoint(path=f"{HAILUO_03_QUERY_ENDPOINT}/{response.task_id}"),+ response_model=Hailuo03TaskQueryResponse,+ status_extractor=lambda r: r.task.status,+ failed_statuses=HAILUO_03_FAILED_STATUSES,+ poll_interval=15,+ )+ video_url = task_result.task.content.url if task_result.task.content else None+ if not video_url:+ raise Exception(f"No video URL in the response: {task_result.model_dump()}")+ return IO.NodeOutput(await download_url_to_video_output(video_url))+++class MinimaxHailuo03TextToVideoNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="MinimaxHailuo03TextToVideoNode",+ display_name="MiniMax H3 Text to Video",+ category="partner/video/MiniMax",+ description="Generate video from a text prompt using the MiniMax H3 model.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[IO.DynamicCombo.Option("MiniMax H3", _hailuo03_model_inputs(allow_adaptive=False))],+ tooltip="Model to use for video generation.",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=4294967295,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Random seed. The same request with the same seed gives similar, "+ "but not guaranteed identical, results.",+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AIGC watermark to the video.",+ advanced=True,+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model.duration"]),+ expr="""+ (+ $dur := $lookup(widgets, "model.duration");+ {"type": "usd", "usd": $dur * 0.1859}+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ seed: int,+ watermark: bool,+ ) -> IO.NodeOutput:+ validate_string(model["prompt"], strip_whitespace=True, min_length=1)+ return await _hailuo03_run_task(+ cls,+ model_id=HAILUO_03_MODELS[model["model"]],+ content=[Hailuo03TextContent(text=model["prompt"])],+ resolution=model["resolution"],+ duration=model["duration"],+ ratio=model["ratio"],+ seed=seed,+ watermark=watermark,+ )+++class MinimaxHailuo03FirstLastFrameNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="MinimaxHailuo03FirstLastFrameNode",+ display_name="MiniMax H3 First-Last-Frame to Video",+ category="partner/video/MiniMax",+ description="Generate video from a first frame image and an optional last frame image "+ "using the MiniMax H3 model. The aspect ratio of the video follows the supplied images.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[IO.DynamicCombo.Option("MiniMax H3", _hailuo03_model_inputs(include_ratio=False))],+ tooltip="Model to use for video generation.",+ ),+ IO.Image.Input(+ "first_frame",+ tooltip="First frame image for the video.",+ ),+ IO.Image.Input(+ "last_frame",+ tooltip="Optional last frame image for the video.",+ optional=True,+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=4294967295,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Random seed. The same request with the same seed gives similar, "+ "but not guaranteed identical, results.",+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AIGC watermark to the video.",+ advanced=True,+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model.duration"]),+ expr="""+ (+ $dur := $lookup(widgets, "model.duration");+ {"type": "usd", "usd": $dur * 0.1859}+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ first_frame: torch.Tensor,+ seed: int,+ watermark: bool,+ last_frame: torch.Tensor | None = None,+ ) -> IO.NodeOutput:+ validate_string(model["prompt"], strip_whitespace=True, min_length=1)+ for frame in (first_frame, last_frame):+ if frame is not None:+ validate_image_aspect_ratio(frame, (2, 5), (5, 2), strict=False) # 0.4 to 2.5+ validate_image_dimensions(frame, min_width=256, min_height=256)++ content: list = [+ Hailuo03TextContent(text=model["prompt"]),+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, first_frame, max_images=1, wait_label="Uploading first frame"+ )+ )[0],+ ),+ role="first_frame",+ ),+ ]+ if last_frame is not None:+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, last_frame, max_images=1, wait_label="Uploading last frame"+ )+ )[0],+ ),+ role="last_frame",+ )+ )+ return await _hailuo03_run_task(+ cls,+ model_id=HAILUO_03_MODELS[model["model"]],+ content=content,+ resolution=model["resolution"],+ duration=model["duration"],+ ratio=None,+ seed=seed,+ watermark=watermark,+ )+++class MinimaxHailuo03ReferenceNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="MinimaxHailuo03ReferenceNode",+ display_name="MiniMax H3 Reference to Video",+ category="partner/video/MiniMax",+ description="Generate video conditioned on reference images, videos, and audio using the "+ "MiniMax H3 model. Refer to the references in the prompt by their order: "+ "'Image 1', 'Image 2', 'Video 1', 'Audio 1', and so on.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "MiniMax H3",+ [+ *_hailuo03_model_inputs(),+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("reference_image"),+ names=[+ "image_1",+ "image_2",+ "image_3",+ "image_4",+ "image_5",+ "image_6",+ "image_7",+ "image_8",+ "image_9",+ ],+ min=0,+ ),+ tooltip="Subject or style reference images, referred to in the prompt "+ "as 'Image 1'..'Image 9' in connection order. Up to 9 images.",+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=["video_1", "video_2", "video_3"],+ min=0,+ ),+ tooltip="Motion or scene reference videos, referred to in the prompt "+ "as 'Video 1'..'Video 3' in connection order. Up to 3 videos, "+ "2-15 seconds each, 15 seconds in total.",+ ),+ IO.Autogrow.Input(+ "reference_audios",+ template=IO.Autogrow.TemplateNames(+ IO.Audio.Input("reference_audio"),+ names=["audio_1", "audio_2", "audio_3"],+ min=0,+ ),+ tooltip="Audio references, referred to in the prompt as "+ "'Audio 1'..'Audio 3' in connection order. Up to 3 clips, "+ "2-15 seconds each, 15 seconds in total. Cannot be used without "+ "a reference image or video.",+ ),+ ],+ )+ ],+ tooltip="Model to use for video generation.",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=4294967295,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Random seed. The same request with the same seed gives similar, "+ "but not guaranteed identical, results.",+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AIGC watermark to the video.",+ advanced=True,+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(+ widgets=["model.duration"],+ input_groups=["model.reference_images", "model.reference_videos"],+ ),+ expr="""+ (+ $dur := $lookup(widgets, "model.duration");+ $imgsRaw := $lookup(inputGroups, "model.reference_images");+ $imgs := $imgsRaw ? $imgsRaw : 0;+ $vidsRaw := $lookup(inputGroups, "model.reference_videos");+ $vids := $vidsRaw ? $vidsRaw : 0;+ $base := $dur * 0.1859 + ($imgs > 5 ? ($imgs - 5) * 0.0572 : 0);+ $vids > 0+ ? {"type": "range_usd", "min_usd": $base + $vids * 2 * 0.1859,+ "max_usd": $base + 15 * 0.1859, "format": {"approximate": true}}+ : {"type": "usd", "usd": $base}+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ seed: int,+ watermark: bool,+ ) -> IO.NodeOutput:+ validate_string(model["prompt"], strip_whitespace=True, min_length=1)++ reference_images = model.get("reference_images", {})+ reference_videos = model.get("reference_videos", {})+ reference_audios = model.get("reference_audios", {})+ if not reference_images and not reference_videos:+ raise ValueError("At least one reference image or video is required.")++ for image in reference_images.values():+ validate_image_aspect_ratio(image, (2, 5), (5, 2), strict=False) # 0.4 to 2.5+ validate_image_dimensions(image, min_width=256, min_height=256)++ total_video_duration = 0.0+ for i, video in enumerate(reference_videos.values(), 1):+ try:+ fps = float(video.get_frame_rate())+ except Exception:+ fps = 0.0+ if fps and not (23.9 <= fps <= 60.5):+ raise ValueError(f"Reference video {i} is {fps:.2f} FPS. Supported range is 23.976-60 FPS.")+ try:+ dur = video.get_duration()+ except Exception:+ continue+ if dur < 1.8:+ raise ValueError(f"Reference video {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")+ total_video_duration += dur+ if total_video_duration > 15.1:+ raise ValueError(+ f"Total reference video duration is {total_video_duration:.1f}s. Maximum is 15 seconds."+ )++ total_audio_duration = 0.0+ for i, audio in enumerate(reference_audios.values(), 1):+ dur = int(audio["waveform"].shape[-1]) / int(audio["sample_rate"])+ if dur < 1.8:+ raise ValueError(f"Reference audio {i} is too short: {dur:.1f}s. Minimum duration is 2 seconds.")+ total_audio_duration += dur+ if total_audio_duration > 15.1:+ raise ValueError(+ f"Total reference audio duration is {total_audio_duration:.1f}s. Maximum is 15 seconds."+ )++ content: list = [Hailuo03TextContent(text=model["prompt"])]+ for i, image in enumerate(reference_images.values(), 1):+ content.append(+ Hailuo03ImageContent(+ image_url=Hailuo03ImageContentUrl(+ url=(+ await upload_images_to_comfyapi(+ cls, image, max_images=1, wait_label=f"Uploading image {i}"+ )+ )[0],+ ),+ role="reference_image",+ )+ )+ for i, video in enumerate(reference_videos.values(), 1):+ content.append(+ Hailuo03VideoContent(+ video_url=Hailuo03VideoContentUrl(+ url=await upload_video_to_comfyapi(cls, video, wait_label=f"Uploading video {i}"),+ ),+ )+ )+ for audio in reference_audios.values():+ content.append(+ Hailuo03AudioContent(+ audio_url=Hailuo03AudioContentUrl(+ url=await upload_audio_to_comfyapi(+ cls,+ audio,+ container_format="mp3",+ codec_name="libmp3lame",+ mime_type="audio/mpeg",+ ),+ ),+ )+ )+ return await _hailuo03_run_task(+ cls,+ model_id=HAILUO_03_MODELS[model["model"]],+ content=content,+ resolution=model["resolution"],+ duration=model["duration"],+ ratio=model["ratio"],+ seed=seed,+ watermark=watermark,+ )++class MinimaxExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[IO.ComfyNode]]:@@ -445,6 +944,9 @@ class MinimaxExtension(ComfyExtension):MinimaxImageToVideoNode,# MinimaxSubjectToVideoNode,MinimaxHailuoVideoNode,+ MinimaxHailuo03TextToVideoNode,+ MinimaxHailuo03FirstLastFrameNode,+ MinimaxHailuo03ReferenceNode,]