The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(Wan): add Wan 3.0 video generation nodes (#15843)
Alexander Piskun41218e487322
diff --git a/comfy_api_nodes/apis/wan.py b/comfy_api_nodes/apis/wan.pyindex c64acae97..523ac68c1 100644--- a/comfy_api_nodes/apis/wan.py+++ b/comfy_api_nodes/apis/wan.py@@ -184,6 +184,32 @@ class Wan27Text2VideoTaskCreationRequest(BaseModel):parameters: Wan27Text2VideoParametersField = Field(...)+class Wan3MediaItem(BaseModel):+ type: str = Field(...)+ url: str = Field(...)+++class Wan3InputField(BaseModel):+ prompt: str | None = Field(None)+ media: list[Wan3MediaItem] | None = Field(None)+++class Wan3ParametersField(BaseModel):+ resolution: str = Field(...)+ ratio: str = Field(...)+ duration: int = Field(..., ge=-1, le=30)+ seed: int = Field(..., ge=0, le=2147483647)+ audio: bool = Field(True)+ prompt_extend: bool = Field(True)+ watermark: bool = Field(False)+++class Wan3TaskCreationRequest(BaseModel):+ model: str = Field(...)+ input: Wan3InputField = Field(...)+ parameters: Wan3ParametersField = Field(...)++class TaskCreationOutputField(BaseModel):task_id: str = Field(...)task_status: str = Field(...)diff --git a/comfy_api_nodes/nodes_wan.py b/comfy_api_nodes/nodes_wan.pyindex 1782739fd..b11c528bc 100644--- a/comfy_api_nodes/nodes_wan.py+++ b/comfy_api_nodes/nodes_wan.py@@ -34,6 +34,10 @@ from comfy_api_nodes.apis.wan import (Wan27VideoEditInputField,Wan27VideoEditParametersField,Wan27VideoEditTaskCreationRequest,+ Wan3InputField,+ Wan3MediaItem,+ Wan3ParametersField,+ Wan3TaskCreationRequest,)from comfy_api_nodes.util import (ApiEndpoint,@@ -53,10 +57,41 @@ from comfy_api_nodes.util import (validate_string,validate_video_duration,)+from comfy_api_nodes.util.client import FAILED_STATUSES, QUEUED_STATUSESRES_IN_PARENS = re.compile(r"\((\d+)\s*[x×]\s*(\d+)\)")+WAN3_QUEUED_STATUSES = [*QUEUED_STATUSES, "pending"]+WAN3_FAILED_STATUSES = [*FAILED_STATUSES, "unknown"]++_WAN3_REF_TAG_RE = re.compile(r"@(image|video|audio)(?P<idx>\d*)(?!\w)", re.IGNORECASE | re.ASCII)+++def _wan3_rewrite_reference_prompt(prompt: str, counts: dict[str, int]) -> str:+ parts = []+ pos = 0+ prev_end = -1+ for match in _WAN3_REF_TAG_RE.finditer(prompt):+ start = match.start()+ before = prompt[start - 1] if start > 0 else ""+ if (before.isascii() and (before.isalnum() or before == "_")) and start != prev_end:+ continue+ kind = match.group(1).lower()+ idx = int(match.group("idx") or 1)+ total = counts[kind]+ if not 1 <= idx <= total:+ raise ValueError(+ f"The prompt references @{kind.capitalize()}{idx}, "+ f"but only {total} reference {kind} inputs are connected."+ )+ parts.append(prompt[pos:start])+ parts.append(f"{kind.capitalize()} {idx}")+ pos = match.end()+ prev_end = match.end()+ parts.append(prompt[pos:])+ return "".join(parts)+class WanTextToImageApi(IO.ComfyNode):@classmethod@@ -1648,6 +1683,390 @@ class Wan2ReferenceVideoApi(IO.ComfyNode):return IO.NodeOutput(await download_url_to_video_output(response.output.video_url))+class Wan3ReferenceToVideoApi(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="Wan3ReferenceToVideoApi",+ display_name="Wan 3.0 Reference to Video",+ category="partner/video/Wan",+ description="Generates a video from a text prompt and optional reference images, videos, and audio "+ "using the Wan 3.0 model. Reference media can be combined freely and mentioned in the prompt "+ "as @Image1, @Video1, @Audio1.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "wan3.0-video",+ [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Prompt describing the elements and visual features. "+ "Supports English and Chinese. Refer to connected reference media "+ "as @Image1, @Video1, @Audio1, numbered per type in input order.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["1080P", "720P", "480P"],+ ),+ IO.Combo.Input(+ "ratio",+ options=["adaptive", "16:9", "9:16", "1:1", "4:3", "3:4"],+ tooltip="Aspect ratio of the output video. With 'adaptive', the output "+ "dimensions are derived from the input media.",+ ),+ IO.Combo.Input(+ "duration",+ options=["auto", *(str(i) for i in range(2, 31))],+ default="5",+ tooltip="Output duration in seconds. With 'auto', the model chooses "+ "a duration that fits the prompt and reference media. The combined "+ "duration of reference videos and output must not exceed 30 seconds.",+ ),+ IO.Boolean.Input(+ "audio",+ default=True,+ tooltip="Whether the output video contains an audio track.",+ ),+ IO.Boolean.Input(+ "prompt_extend",+ default=True,+ tooltip="Whether to enhance the prompt with AI assistance.",+ advanced=True,+ ),+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("reference_image"),+ names=[f"image{i}" for i in range(1, 11)],+ min=0,+ ),+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=[f"video{i}" for i in range(1, 6)],+ min=0,+ ),+ ),+ IO.Autogrow.Input(+ "reference_audios",+ template=IO.Autogrow.TemplateNames(+ IO.Audio.Input("reference_audio"),+ names=[f"audio{i}" for i in range(1, 6)],+ min=0,+ ),+ ),+ ],+ ),+ ],+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=2147483647,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed to use for generation.",+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AI-generated watermark to the result.",+ advanced=True,+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]),+ expr="""+ (+ $ppsTable := { "480p": 0.0715, "720p": 0.143, "1080p": 0.286 };+ $pps := $lookup($ppsTable, $lookup(widgets, "model.resolution"));+ $dur := $lookup(widgets, "model.duration");+ $dur = "auto"+ ? { "type": "usd", "usd": $pps, "format": {"suffix": "/second"} }+ : (+ $minUsd := $round($number($dur) * $pps, 2);+ $maxUsd := $round($min([$number($dur) + 15, 30]) * $pps, 2);+ $minUsd = $maxUsd+ ? { "type": "usd", "usd": $minUsd }+ : { "type": "range_usd", "min_usd": $minUsd, "max_usd": $maxUsd }+ )+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ seed: int,+ watermark: bool,+ ):+ reference_images = model.get("reference_images", {})+ reference_videos = model.get("reference_videos", {})+ reference_audios = model.get("reference_audios", {})+ for key in reference_images:+ if get_number_of_images(reference_images[key]) != 1:+ raise ValueError(f"Reference image input '{key}' must contain exactly one image, not a batch.")+ total_video_seconds = 0.0+ for key in reference_videos:+ validate_video_duration(reference_videos[key], max_duration=15)+ try:+ total_video_seconds += reference_videos[key].get_duration()+ except Exception:+ pass+ if total_video_seconds > 15.0001:+ raise ValueError(+ f"The total duration of reference videos ({total_video_seconds:.2f}s) exceeds the 15s limit."+ )+ total_audio_seconds = 0.0+ for key in reference_audios:+ validate_audio_duration(reference_audios[key], max_duration=15)+ total_audio_seconds += reference_audios[key]["waveform"].shape[-1] / int(+ reference_audios[key]["sample_rate"]+ )+ if total_audio_seconds > 15.0001:+ raise ValueError(+ f"The total duration of reference audios ({total_audio_seconds:.2f}s) exceeds the 15s limit."+ )+ duration = -1 if model["duration"] == "auto" else int(model["duration"])+ if duration != -1 and total_video_seconds + duration > 30.0001:+ raise ValueError(+ f"Reference video duration ({total_video_seconds:.2f}s) plus output duration ({duration}s) "+ "exceeds the 30s combined limit."+ )+ prompt = _wan3_rewrite_reference_prompt(+ model["prompt"],+ {"image": len(reference_images), "video": len(reference_videos), "audio": len(reference_audios)},+ )+ validate_string(prompt, strip_whitespace=False, max_length=20000)+ if not prompt.strip() and not (reference_images or reference_videos or reference_audios):+ raise ValueError("Provide a prompt or at least one reference input.")+ media = []+ for key in reference_images:+ media.append(+ Wan3MediaItem(type="reference_image", url=await upload_image_to_comfyapi(cls, reference_images[key]))+ )+ for key in reference_videos:+ media.append(+ Wan3MediaItem(type="reference_video", url=await upload_video_to_comfyapi(cls, reference_videos[key]))+ )+ for key in reference_audios:+ media.append(+ Wan3MediaItem(+ type="reference_audio",+ url=await upload_audio_to_comfyapi(+ cls,+ reference_audios[key],+ container_format="mp3",+ codec_name="libmp3lame",+ mime_type="audio/mpeg",+ ),+ )+ )+ initial_response = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/wan/api/v1/services/aigc/video-generation/video-synthesis", method="POST"),+ response_model=TaskCreationResponse,+ data=Wan3TaskCreationRequest(+ model=model["model"],+ input=Wan3InputField(prompt=prompt or None, media=media or None),+ parameters=Wan3ParametersField(+ resolution=model["resolution"],+ ratio=model["ratio"],+ duration=duration,+ seed=seed,+ audio=model["audio"],+ prompt_extend=model["prompt_extend"],+ watermark=watermark,+ ),+ ),+ )+ if not initial_response.output:+ raise Exception(f"An unknown error occurred: {initial_response.code} - {initial_response.message}")+ response = await poll_op(+ cls,+ ApiEndpoint(path=f"/proxy/wan/api/v1/tasks/{initial_response.output.task_id}"),+ response_model=VideoTaskStatusResponse,+ status_extractor=lambda x: x.output.task_status,+ queued_statuses=WAN3_QUEUED_STATUSES,+ failed_statuses=WAN3_FAILED_STATUSES,+ poll_interval=10,+ )+ return IO.NodeOutput(await download_url_to_video_output(response.output.video_url))+++class Wan3ImageToVideoApi(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="Wan3ImageToVideoApi",+ display_name="Wan 3.0 Image to Video",+ category="partner/video/Wan",+ description="Generates a video from a first-frame image, with optional last-frame control, "+ "using the Wan 3.0 model.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "wan3.0-video",+ [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Prompt describing the elements and visual features. "+ "Supports English and Chinese.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["1080P", "720P", "480P"],+ ),+ IO.Combo.Input(+ "ratio",+ options=["adaptive", "16:9", "9:16", "1:1", "4:3", "3:4"],+ tooltip="Aspect ratio of the output video. With 'adaptive', the output "+ "dimensions are derived from the first frame.",+ ),+ IO.Combo.Input(+ "duration",+ options=["auto", *(str(i) for i in range(2, 31))],+ default="5",+ tooltip="Output duration in seconds. With 'auto', the model chooses "+ "a duration that fits the prompt.",+ ),+ IO.Boolean.Input(+ "audio",+ default=True,+ tooltip="Whether the output video contains an audio track.",+ ),+ IO.Boolean.Input(+ "prompt_extend",+ default=True,+ tooltip="Whether to enhance the prompt with AI assistance.",+ advanced=True,+ ),+ ],+ ),+ ],+ ),+ IO.Image.Input(+ "first_frame",+ tooltip="First frame image.",+ ),+ IO.Image.Input(+ "last_frame",+ optional=True,+ tooltip="Last frame image. The model generates a video transitioning from first to last frame.",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=2147483647,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed to use for generation.",+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AI-generated watermark to the result.",+ advanced=True,+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution", "model.duration"]),+ expr="""+ (+ $ppsTable := { "480p": 0.0715, "720p": 0.143, "1080p": 0.286 };+ $pps := $lookup($ppsTable, $lookup(widgets, "model.resolution"));+ $dur := $lookup(widgets, "model.duration");+ $dur = "auto"+ ? { "type": "usd", "usd": $pps, "format": {"suffix": "/second"} }+ : { "type": "usd", "usd": $round($number($dur) * $pps, 2) }+ )+ """,+ ),+ )++ @classmethod+ async def execute(+ cls,+ model: dict,+ first_frame: Input.Image,+ seed: int,+ watermark: bool,+ last_frame: Input.Image | None = None,+ ):+ if get_number_of_images(first_frame) != 1:+ raise ValueError("Exactly one first_frame image is required.")+ if last_frame is not None and get_number_of_images(last_frame) != 1:+ raise ValueError("Exactly one last_frame image is required.")+ validate_string(model["prompt"], strip_whitespace=False, max_length=20000)+ media = [Wan3MediaItem(type="first_frame", url=await upload_image_to_comfyapi(cls, first_frame))]+ if last_frame is not None:+ media.append(Wan3MediaItem(type="last_frame", url=await upload_image_to_comfyapi(cls, last_frame)))+ initial_response = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/wan/api/v1/services/aigc/video-generation/video-synthesis", method="POST"),+ response_model=TaskCreationResponse,+ data=Wan3TaskCreationRequest(+ model=model["model"],+ input=Wan3InputField(prompt=model["prompt"] or None, media=media),+ parameters=Wan3ParametersField(+ resolution=model["resolution"],+ ratio=model["ratio"],+ duration=-1 if model["duration"] == "auto" else int(model["duration"]),+ seed=seed,+ audio=model["audio"],+ prompt_extend=model["prompt_extend"],+ watermark=watermark,+ ),+ ),+ )+ if not initial_response.output:+ raise Exception(f"An unknown error occurred: {initial_response.code} - {initial_response.message}")+ response = await poll_op(+ cls,+ ApiEndpoint(path=f"/proxy/wan/api/v1/tasks/{initial_response.output.task_id}"),+ response_model=VideoTaskStatusResponse,+ status_extractor=lambda x: x.output.task_status,+ queued_statuses=WAN3_QUEUED_STATUSES,+ failed_statuses=WAN3_FAILED_STATUSES,+ poll_interval=10,+ )+ return IO.NodeOutput(await download_url_to_video_output(response.output.video_url))++class HappyHorseTextToVideoApi(IO.ComfyNode):@classmethoddef define_schema(cls):@@ -2342,6 +2761,8 @@ class WanApiExtension(ComfyExtension):Wan2VideoContinuationApi,Wan2VideoEditApi,Wan2ReferenceVideoApi,+ Wan3ReferenceToVideoApi,+ Wan3ImageToVideoApi,HappyHorseTextToVideoApi,HappyHorseImageToVideoApi,HappyHorseVideoEditApi,