The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat(api-nodes): add WAN2.6 ReferenceToVideo (#11644)
Alexander Piskun79e94544bd7e
diff --git a/comfy_api_nodes/nodes_wan.py b/comfy_api_nodes/nodes_wan.pyindex 1675fd863..3e04786a9 100644--- a/comfy_api_nodes/nodes_wan.py+++ b/comfy_api_nodes/nodes_wan.py@@ -13,7 +13,9 @@ from comfy_api_nodes.util import (poll_op,sync_op,tensor_to_base64_string,+ upload_video_to_comfyapi,validate_audio_duration,+ validate_video_duration,)@@ -41,6 +43,12 @@ class Image2VideoInputField(BaseModel):audio_url: str | None = Field(None)+class Reference2VideoInputField(BaseModel):+ prompt: str = Field(...)+ negative_prompt: str | None = Field(None)+ reference_video_urls: list[str] = Field(...)++class Txt2ImageParametersField(BaseModel):size: str = Field(...)n: int = Field(1, description="Number of images to generate.") # we support only value=1@@ -76,6 +84,14 @@ class Image2VideoParametersField(BaseModel):shot_type: str = Field("single")+class Reference2VideoParametersField(BaseModel):+ size: str = Field(...)+ duration: int = Field(5, ge=5, le=15)+ shot_type: str = Field("single")+ seed: int = Field(..., ge=0, le=2147483647)+ watermark: bool = Field(False)++class Text2ImageTaskCreationRequest(BaseModel):model: str = Field(...)input: Text2ImageInputField = Field(...)@@ -100,6 +116,12 @@ class Image2VideoTaskCreationRequest(BaseModel):parameters: Image2VideoParametersField = Field(...)+class Reference2VideoTaskCreationRequest(BaseModel):+ model: str = Field(...)+ input: Reference2VideoInputField = Field(...)+ parameters: Reference2VideoParametersField = Field(...)++class TaskCreationOutputField(BaseModel):task_id: str = Field(...)task_status: str = Field(...)@@ -721,6 +743,143 @@ class WanImageToVideoApi(IO.ComfyNode):return IO.NodeOutput(await download_url_to_video_output(response.output.video_url))+class WanReferenceVideoApi(IO.ComfyNode):+ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="WanReferenceVideoApi",+ display_name="Wan Reference to Video",+ category="api node/video/Wan",+ description="Use the character and voice from input videos, combined with a prompt, "+ "to generate a new video that maintains character consistency.",+ inputs=[+ IO.Combo.Input("model", options=["wan2.6-r2v"]),+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Prompt describing the elements and visual features. Supports English and Chinese. "+ "Use identifiers such as `character1` and `character2` to refer to the reference characters.",+ ),+ IO.String.Input(+ "negative_prompt",+ multiline=True,+ default="",+ tooltip="Negative prompt describing what to avoid.",+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=["character1", "character2", "character3"],+ min=1,+ ),+ ),+ IO.Combo.Input(+ "size",+ options=[+ "720p: 1:1 (960x960)",+ "720p: 16:9 (1280x720)",+ "720p: 9:16 (720x1280)",+ "720p: 4:3 (1088x832)",+ "720p: 3:4 (832x1088)",+ "1080p: 1:1 (1440x1440)",+ "1080p: 16:9 (1920x1080)",+ "1080p: 9:16 (1080x1920)",+ "1080p: 4:3 (1632x1248)",+ "1080p: 3:4 (1248x1632)",+ ],+ ),+ IO.Int.Input(+ "duration",+ default=5,+ min=5,+ max=10,+ step=5,+ display_mode=IO.NumberDisplay.slider,+ ),+ IO.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ ),+ IO.Combo.Input(+ "shot_type",+ options=["single", "multi"],+ tooltip="Specifies the shot type for the generated video, that is, whether the video is a "+ "single continuous shot or multiple shots with cuts.",+ ),+ IO.Boolean.Input(+ "watermark",+ default=False,+ tooltip="Whether to add an AI-generated watermark to the result.",+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ model: str,+ prompt: str,+ negative_prompt: str,+ reference_videos: IO.Autogrow.Type,+ size: str,+ duration: int,+ seed: int,+ shot_type: str,+ watermark: bool,+ ):+ reference_video_urls = []+ for i in reference_videos:+ validate_video_duration(reference_videos[i], min_duration=2, max_duration=30)+ for i in reference_videos:+ reference_video_urls.append(await upload_video_to_comfyapi(cls, reference_videos[i]))+ width, height = RES_IN_PARENS.search(size).groups()+ initial_response = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/wan/api/v1/services/aigc/video-generation/video-synthesis", method="POST"),+ response_model=TaskCreationResponse,+ data=Reference2VideoTaskCreationRequest(+ model=model,+ input=Reference2VideoInputField(+ prompt=prompt, negative_prompt=negative_prompt, reference_video_urls=reference_video_urls+ ),+ parameters=Reference2VideoParametersField(+ size=f"{width}*{height}",+ duration=duration,+ shot_type=shot_type,+ watermark=watermark,+ seed=seed,+ ),+ ),+ )+ if not initial_response.output:+ raise Exception(f"An unknown error occurred: {initial_response.code} - {initial_response.message}")+ response = await poll_op(+ cls,+ ApiEndpoint(path=f"/proxy/wan/api/v1/tasks/{initial_response.output.task_id}"),+ response_model=VideoTaskStatusResponse,+ status_extractor=lambda x: x.output.task_status,+ poll_interval=6,+ max_poll_attempts=280,+ )+ return IO.NodeOutput(await download_url_to_video_output(response.output.video_url))++class WanApiExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[IO.ComfyNode]]:@@ -729,6 +888,7 @@ class WanApiExtension(ComfyExtension):WanImageToImageApi,WanTextToVideoApi,WanImageToVideoApi,+ WanReferenceVideoApi,]diff --git a/comfy_api_nodes/util/upload_helpers.py b/comfy_api_nodes/util/upload_helpers.pyindex b8d33f4d1..f1ed7fe9c 100644--- a/comfy_api_nodes/util/upload_helpers.py+++ b/comfy_api_nodes/util/upload_helpers.py@@ -119,7 +119,7 @@ async def upload_video_to_comfyapi(raise ValueError(f"Could not verify video duration from source: {e}") from eupload_mime_type = f"video/{container.value.lower()}"- filename = f"uploaded_video.{container.value.lower()}"+ filename = f"{uuid.uuid4()}.{container.value.lower()}"# Convert VideoInput to BytesIO using specified container/codecvideo_bytes_io = BytesIO()