The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat(api-nodes): add Vidu Video nodes (#9368)
Alexander Piskun54d8fdbed0a7
diff --git a/comfy_api_nodes/nodes_vidu.py b/comfy_api_nodes/nodes_vidu.pynew file mode 100644index 000000000..2f441948c--- /dev/null+++ b/comfy_api_nodes/nodes_vidu.py@@ -0,0 +1,622 @@+import logging+from enum import Enum+from typing import Any, Callable, Optional, Literal, TypeVar+from typing_extensions import override++import torch+from pydantic import BaseModel, Field++from comfy_api.latest import ComfyExtension, io as comfy_io+from comfy_api_nodes.util.validation_utils import (+ validate_aspect_ratio_closeness,+ validate_image_dimensions,+ validate_image_aspect_ratio_range,+ get_number_of_images,+)+from comfy_api_nodes.apis.client import (+ ApiEndpoint,+ HttpMethod,+ SynchronousOperation,+ PollingOperation,+ EmptyRequest,+)+from comfy_api_nodes.apinode_utils import download_url_to_video_output, upload_images_to_comfyapi+++VIDU_TEXT_TO_VIDEO = "/proxy/vidu/text2video"+VIDU_IMAGE_TO_VIDEO = "/proxy/vidu/img2video"+VIDU_REFERENCE_VIDEO = "/proxy/vidu/reference2video"+VIDU_START_END_VIDEO = "/proxy/vidu/start-end2video"+VIDU_GET_GENERATION_STATUS = "/proxy/vidu/tasks/%s/creations"++R = TypeVar("R")++class VideoModelName(str, Enum):+ vidu_q1 = 'viduq1'+++class AspectRatio(str, Enum):+ r_16_9 = "16:9"+ r_9_16 = "9:16"+ r_1_1 = "1:1"+++class Resolution(str, Enum):+ r_1080p = "1080p"+++class MovementAmplitude(str, Enum):+ auto = "auto"+ small = "small"+ medium = "medium"+ large = "large"+++class TaskCreationRequest(BaseModel):+ model: VideoModelName = VideoModelName.vidu_q1+ prompt: Optional[str] = Field(None, max_length=1500)+ duration: Optional[Literal[5]] = 5+ seed: Optional[int] = Field(0, ge=0, le=2147483647)+ aspect_ratio: Optional[AspectRatio] = AspectRatio.r_16_9+ resolution: Optional[Resolution] = Resolution.r_1080p+ movement_amplitude: Optional[MovementAmplitude] = MovementAmplitude.auto+ images: Optional[list[str]] = Field(None, description="Base64 encoded string or image URL")+++class TaskStatus(str, Enum):+ created = "created"+ queueing = "queueing"+ processing = "processing"+ success = "success"+ failed = "failed"+++class TaskCreationResponse(BaseModel):+ task_id: str = Field(...)+ state: TaskStatus = Field(...)+ created_at: str = Field(...)+ code: Optional[int] = Field(None, description="Error code")+++class TaskResult(BaseModel):+ id: str = Field(..., description="Creation id")+ url: str = Field(..., description="The URL of the generated results, valid for one hour")+ cover_url: str = Field(..., description="The cover URL of the generated results, valid for one hour")+++class TaskStatusResponse(BaseModel):+ state: TaskStatus = Field(...)+ err_code: Optional[str] = Field(None)+ creations: list[TaskResult] = Field(..., description="Generated results")+++async def poll_until_finished(+ auth_kwargs: dict[str, str],+ api_endpoint: ApiEndpoint[Any, R],+ result_url_extractor: Optional[Callable[[R], str]] = None,+ estimated_duration: Optional[int] = None,+ node_id: Optional[str] = None,+) -> R:+ return await PollingOperation(+ poll_endpoint=api_endpoint,+ completed_statuses=[TaskStatus.success.value],+ failed_statuses=[TaskStatus.failed.value],+ status_extractor=lambda response: response.state.value,+ auth_kwargs=auth_kwargs,+ result_url_extractor=result_url_extractor,+ estimated_duration=estimated_duration,+ node_id=node_id,+ poll_interval=16.0,+ max_poll_attempts=256,+ ).execute()+++def get_video_url_from_response(response) -> Optional[str]:+ if response.creations:+ return response.creations[0].url+ return None+++def get_video_from_response(response) -> TaskResult:+ if not response.creations:+ error_msg = f"Vidu request does not contain results. State: {response.state}, Error Code: {response.err_code}"+ logging.info(error_msg)+ raise RuntimeError(error_msg)+ logging.info("Vidu task %s succeeded. Video URL: %s", response.creations[0].id, response.creations[0].url)+ return response.creations[0]+++async def execute_task(+ vidu_endpoint: str,+ auth_kwargs: Optional[dict[str, str]],+ payload: TaskCreationRequest,+ estimated_duration: int,+ node_id: str,+) -> R:+ response = await SynchronousOperation(+ endpoint=ApiEndpoint(+ path=vidu_endpoint,+ method=HttpMethod.POST,+ request_model=TaskCreationRequest,+ response_model=TaskCreationResponse,+ ),+ request=payload,+ auth_kwargs=auth_kwargs,+ ).execute()+ if response.state == TaskStatus.failed:+ error_msg = f"Vidu request failed. Code: {response.code}"+ logging.error(error_msg)+ raise RuntimeError(error_msg)+ return await poll_until_finished(+ auth_kwargs,+ ApiEndpoint(+ path=VIDU_GET_GENERATION_STATUS % response.task_id,+ method=HttpMethod.GET,+ request_model=EmptyRequest,+ response_model=TaskStatusResponse,+ ),+ result_url_extractor=get_video_url_from_response,+ estimated_duration=estimated_duration,+ node_id=node_id,+ )+++class ViduTextToVideoNode(comfy_io.ComfyNode):++ @classmethod+ def define_schema(cls):+ return comfy_io.Schema(+ node_id="ViduTextToVideoNode",+ display_name="Vidu Text To Video Generation",+ category="api node/video/Vidu",+ description="Generate video from text prompt",+ inputs=[+ comfy_io.Combo.Input(+ "model",+ options=[model.value for model in VideoModelName],+ default=VideoModelName.vidu_q1.value,+ tooltip="Model name",+ ),+ comfy_io.String.Input(+ "prompt",+ multiline=True,+ tooltip="A textual description for video generation",+ ),+ comfy_io.Int.Input(+ "duration",+ default=5,+ min=5,+ max=5,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ tooltip="Duration of the output video in seconds",+ optional=True,+ ),+ comfy_io.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed for video generation (0 for random)",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "aspect_ratio",+ options=[model.value for model in AspectRatio],+ default=AspectRatio.r_16_9.value,+ tooltip="The aspect ratio of the output video",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "resolution",+ options=[model.value for model in Resolution],+ default=Resolution.r_1080p.value,+ tooltip="Supported values may vary by model & duration",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "movement_amplitude",+ options=[model.value for model in MovementAmplitude],+ default=MovementAmplitude.auto.value,+ tooltip="The movement amplitude of objects in the frame",+ optional=True,+ ),+ ],+ outputs=[+ comfy_io.Video.Output(),+ ],+ hidden=[+ comfy_io.Hidden.auth_token_comfy_org,+ comfy_io.Hidden.api_key_comfy_org,+ comfy_io.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ model: str,+ prompt: str,+ duration: int,+ seed: int,+ aspect_ratio: str,+ resolution: str,+ movement_amplitude: str,+ ) -> comfy_io.NodeOutput:+ if not prompt:+ raise ValueError("The prompt field is required and cannot be empty.")+ payload = TaskCreationRequest(+ model_name=model,+ prompt=prompt,+ duration=duration,+ seed=seed,+ aspect_ratio=aspect_ratio,+ resolution=resolution,+ movement_amplitude=movement_amplitude,+ )+ auth = {+ "auth_token": cls.hidden.auth_token_comfy_org,+ "comfy_api_key": cls.hidden.api_key_comfy_org,+ }+ results = await execute_task(VIDU_TEXT_TO_VIDEO, auth, payload, 320, cls.hidden.unique_id)+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))+++class ViduImageToVideoNode(comfy_io.ComfyNode):++ @classmethod+ def define_schema(cls):+ return comfy_io.Schema(+ node_id="ViduImageToVideoNode",+ display_name="Vidu Image To Video Generation",+ category="api node/video/Vidu",+ description="Generate video from image and optional prompt",+ inputs=[+ comfy_io.Combo.Input(+ "model",+ options=[model.value for model in VideoModelName],+ default=VideoModelName.vidu_q1.value,+ tooltip="Model name",+ ),+ comfy_io.Image.Input(+ "image",+ tooltip="An image to be used as the start frame of the generated video",+ ),+ comfy_io.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="A textual description for video generation",+ optional=True,+ ),+ comfy_io.Int.Input(+ "duration",+ default=5,+ min=5,+ max=5,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ tooltip="Duration of the output video in seconds",+ optional=True,+ ),+ comfy_io.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed for video generation (0 for random)",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "resolution",+ options=[model.value for model in Resolution],+ default=Resolution.r_1080p.value,+ tooltip="Supported values may vary by model & duration",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "movement_amplitude",+ options=[model.value for model in MovementAmplitude],+ default=MovementAmplitude.auto.value,+ tooltip="The movement amplitude of objects in the frame",+ optional=True,+ ),+ ],+ outputs=[+ comfy_io.Video.Output(),+ ],+ hidden=[+ comfy_io.Hidden.auth_token_comfy_org,+ comfy_io.Hidden.api_key_comfy_org,+ comfy_io.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ model: str,+ image: torch.Tensor,+ prompt: str,+ duration: int,+ seed: int,+ resolution: str,+ movement_amplitude: str,+ ) -> comfy_io.NodeOutput:+ if get_number_of_images(image) > 1:+ raise ValueError("Only one input image is allowed.")+ validate_image_aspect_ratio_range(image, (1, 4), (4, 1))+ payload = TaskCreationRequest(+ model_name=model,+ prompt=prompt,+ duration=duration,+ seed=seed,+ resolution=resolution,+ movement_amplitude=movement_amplitude,+ )+ auth = {+ "auth_token": cls.hidden.auth_token_comfy_org,+ "comfy_api_key": cls.hidden.api_key_comfy_org,+ }+ payload.images = await upload_images_to_comfyapi(+ image,+ max_images=1,+ mime_type="image/png",+ auth_kwargs=auth,+ )+ results = await execute_task(VIDU_IMAGE_TO_VIDEO, auth, payload, 120, cls.hidden.unique_id)+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))+++class ViduReferenceVideoNode(comfy_io.ComfyNode):++ @classmethod+ def define_schema(cls):+ return comfy_io.Schema(+ node_id="ViduReferenceVideoNode",+ display_name="Vidu Reference To Video Generation",+ category="api node/video/Vidu",+ description="Generate video from multiple images and prompt",+ inputs=[+ comfy_io.Combo.Input(+ "model",+ options=[model.value for model in VideoModelName],+ default=VideoModelName.vidu_q1.value,+ tooltip="Model name",+ ),+ comfy_io.Image.Input(+ "images",+ tooltip="Images to use as references to generate a video with consistent subjects (max 7 images).",+ ),+ comfy_io.String.Input(+ "prompt",+ multiline=True,+ tooltip="A textual description for video generation",+ ),+ comfy_io.Int.Input(+ "duration",+ default=5,+ min=5,+ max=5,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ tooltip="Duration of the output video in seconds",+ optional=True,+ ),+ comfy_io.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed for video generation (0 for random)",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "aspect_ratio",+ options=[model.value for model in AspectRatio],+ default=AspectRatio.r_16_9.value,+ tooltip="The aspect ratio of the output video",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "resolution",+ options=[model.value for model in Resolution],+ default=Resolution.r_1080p.value,+ tooltip="Supported values may vary by model & duration",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "movement_amplitude",+ options=[model.value for model in MovementAmplitude],+ default=MovementAmplitude.auto.value,+ tooltip="The movement amplitude of objects in the frame",+ optional=True,+ ),+ ],+ outputs=[+ comfy_io.Video.Output(),+ ],+ hidden=[+ comfy_io.Hidden.auth_token_comfy_org,+ comfy_io.Hidden.api_key_comfy_org,+ comfy_io.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ model: str,+ images: torch.Tensor,+ prompt: str,+ duration: int,+ seed: int,+ aspect_ratio: str,+ resolution: str,+ movement_amplitude: str,+ ) -> comfy_io.NodeOutput:+ if not prompt:+ raise ValueError("The prompt field is required and cannot be empty.")+ a = get_number_of_images(images)+ if a > 7:+ raise ValueError("Too many images, maximum allowed is 7.")+ for image in images:+ validate_image_aspect_ratio_range(image, (1, 4), (4, 1))+ validate_image_dimensions(image, min_width=128, min_height=128)+ payload = TaskCreationRequest(+ model_name=model,+ prompt=prompt,+ duration=duration,+ seed=seed,+ aspect_ratio=aspect_ratio,+ resolution=resolution,+ movement_amplitude=movement_amplitude,+ )+ auth = {+ "auth_token": cls.hidden.auth_token_comfy_org,+ "comfy_api_key": cls.hidden.api_key_comfy_org,+ }+ payload.images = await upload_images_to_comfyapi(+ images,+ max_images=7,+ mime_type="image/png",+ auth_kwargs=auth,+ )+ results = await execute_task(VIDU_REFERENCE_VIDEO, auth, payload, 120, cls.hidden.unique_id)+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))+++class ViduStartEndToVideoNode(comfy_io.ComfyNode):++ @classmethod+ def define_schema(cls):+ return comfy_io.Schema(+ node_id="ViduStartEndToVideoNode",+ display_name="Vidu Start End To Video Generation",+ category="api node/video/Vidu",+ description="Generate a video from start and end frames and a prompt",+ inputs=[+ comfy_io.Combo.Input(+ "model",+ options=[model.value for model in VideoModelName],+ default=VideoModelName.vidu_q1.value,+ tooltip="Model name",+ ),+ comfy_io.Image.Input(+ "first_frame",+ tooltip="Start frame",+ ),+ comfy_io.Image.Input(+ "end_frame",+ tooltip="End frame",+ ),+ comfy_io.String.Input(+ "prompt",+ multiline=True,+ tooltip="A textual description for video generation",+ optional=True,+ ),+ comfy_io.Int.Input(+ "duration",+ default=5,+ min=5,+ max=5,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ tooltip="Duration of the output video in seconds",+ optional=True,+ ),+ comfy_io.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed for video generation (0 for random)",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "resolution",+ options=[model.value for model in Resolution],+ default=Resolution.r_1080p.value,+ tooltip="Supported values may vary by model & duration",+ optional=True,+ ),+ comfy_io.Combo.Input(+ "movement_amplitude",+ options=[model.value for model in MovementAmplitude],+ default=MovementAmplitude.auto.value,+ tooltip="The movement amplitude of objects in the frame",+ optional=True,+ ),+ ],+ outputs=[+ comfy_io.Video.Output(),+ ],+ hidden=[+ comfy_io.Hidden.auth_token_comfy_org,+ comfy_io.Hidden.api_key_comfy_org,+ comfy_io.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ model: str,+ first_frame: torch.Tensor,+ end_frame: torch.Tensor,+ prompt: str,+ duration: int,+ seed: int,+ resolution: str,+ movement_amplitude: str,+ ) -> comfy_io.NodeOutput:+ validate_aspect_ratio_closeness(first_frame, end_frame, min_rel=0.8, max_rel=1.25, strict=False)+ payload = TaskCreationRequest(+ model_name=model,+ prompt=prompt,+ duration=duration,+ seed=seed,+ resolution=resolution,+ movement_amplitude=movement_amplitude,+ )+ auth = {+ "auth_token": cls.hidden.auth_token_comfy_org,+ "comfy_api_key": cls.hidden.api_key_comfy_org,+ }+ payload.images = [+ (await upload_images_to_comfyapi(frame, max_images=1, mime_type="image/png", auth_kwargs=auth))[0]+ for frame in (first_frame, end_frame)+ ]+ results = await execute_task(VIDU_START_END_VIDEO, auth, payload, 96, cls.hidden.unique_id)+ return comfy_io.NodeOutput(await download_url_to_video_output(get_video_from_response(results).url))+++class ViduExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:+ return [+ ViduTextToVideoNode,+ ViduImageToVideoNode,+ ViduReferenceVideoNode,+ ViduStartEndToVideoNode,+ ]++async def comfy_entrypoint() -> ViduExtension:+ return ViduExtension()diff --git a/comfy_api_nodes/util/validation_utils.py b/comfy_api_nodes/util/validation_utils.pyindex 031b9fbd3..606b794bf 100644--- a/comfy_api_nodes/util/validation_utils.py+++ b/comfy_api_nodes/util/validation_utils.py@@ -53,6 +53,53 @@ def validate_image_aspect_ratio()+def validate_image_aspect_ratio_range(+ image: torch.Tensor,+ min_ratio: tuple[float, float], # e.g. (1, 4)+ max_ratio: tuple[float, float], # e.g. (4, 1)+ *,+ strict: bool = True, # True -> (min, max); False -> [min, max]+) -> float:+ a1, b1 = min_ratio+ a2, b2 = max_ratio+ if a1 <= 0 or b1 <= 0 or a2 <= 0 or b2 <= 0:+ raise ValueError("Ratios must be positive, like (1, 4) or (4, 1).")+ lo, hi = (a1 / b1), (a2 / b2)+ if lo > hi:+ lo, hi = hi, lo+ a1, b1, a2, b2 = a2, b2, a1, b1 # swap only for error text+ w, h = get_image_dimensions(image)+ if w <= 0 or h <= 0:+ raise ValueError(f"Invalid image dimensions: {w}x{h}")+ ar = w / h+ ok = (lo < ar < hi) if strict else (lo <= ar <= hi)+ if not ok:+ op = "<" if strict else "≤"+ raise ValueError(f"Image aspect ratio {ar:.6g} is outside allowed range: {a1}:{b1} {op} ratio {op} {a2}:{b2}")+ return ar+++def validate_aspect_ratio_closeness(+ start_img,+ end_img,+ min_rel: float,+ max_rel: float,+ *,+ strict: bool = False, # True => exclusive, False => inclusive+) -> None:+ w1, h1 = get_image_dimensions(start_img)+ w2, h2 = get_image_dimensions(end_img)+ if min(w1, h1, w2, h2) <= 0:+ raise ValueError("Invalid image dimensions")+ ar1 = w1 / h1+ ar2 = w2 / h2+ # Normalize so it is symmetric (no need to check both ar1/ar2 and ar2/ar1)+ closeness = max(ar1, ar2) / min(ar1, ar2)+ limit = max(max_rel, 1.0 / min_rel) # for 0.8..1.25 this is 1.25+ if (closeness >= limit) if strict else (closeness > limit):+ raise ValueError(f"Aspect ratios must be close: start/end={ar1/ar2:.4f}, allowed range {min_rel}–{max_rel}.")++def validate_video_dimensions(video: VideoInput,min_width: Optional[int] = None,@@ -98,3 +145,9 @@ def validate_video_duration(raise ValueError(f"Video duration must be at most {max_duration}s, got {duration}s")+++def get_number_of_images(images):+ if isinstance(images, torch.Tensor):+ return images.shape[0] if images.ndim >= 4 else 1+ return len(images)diff --git a/nodes.py b/nodes.pyindex b3fa9c51a..35dda1b19 100644--- a/nodes.py+++ b/nodes.py@@ -2351,6 +2351,7 @@ async def init_builtin_api_nodes():"nodes_moonvalley.py","nodes_rodin.py","nodes_gemini.py",+ "nodes_vidu.py",]if not await load_custom_node(os.path.join(api_nodes_dir, "canary.py"), module_parent="comfy_api_nodes"):