The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(HeyGen): add HeyGen Video 1.0 reference-to-video and image-to-video nodes (#16695)
Alexander Piskun051ddedaae52
diff --git a/comfy_api_nodes/nodes_heygen.py b/comfy_api_nodes/nodes_heygen.pyindex b1375d143..28b7c26d9 100644--- a/comfy_api_nodes/nodes_heygen.py+++ b/comfy_api_nodes/nodes_heygen.py@@ -1,3 +1,5 @@+import re+import torchfrom typing_extensions import override@@ -25,11 +27,13 @@ from comfy_api_nodes.util import (upload_image_to_comfyapi,upload_images_to_comfyapi,upload_video_to_comfyapi,+ validate_image_aspect_ratio,validate_string,)from server import PromptServer_VIDEOS_PATH = "/proxy/heygen/v3/videos"+_MODEL_VIDEOS_PATH = "/proxy/heygen/v3/models/videos"_TRANSLATIONS_PATH = "/proxy/heygen/v3/video-translations"_SPEECH_PATH = "/proxy/heygen/v3/voices/speech"_AVATARS_PATH = "/proxy/heygen/v3/avatars"@@ -42,6 +46,27 @@ _AVATARS_BY_ENGINE = {for e in ("avatar_iv", "avatar_iii", "avatar_v")}+_REFERENCE_TAG_RE = re.compile(r"(?<!\w)@(image|video|audio)(\d*)(?!\w)", re.IGNORECASE | re.ASCII)+_REFERENCE_LABELS = {"image": "Picture", "video": "Video", "audio": "Audio"}+++def _rewrite_reference_tags(prompt: str, counts: dict[str, int]) -> str:+ def repl(match: re.Match) -> str:+ kind = match.group(1).lower()+ idx = int(match.group(2) or 1)+ if not 1 <= idx <= counts[kind]:+ raise ValueError(+ f"The prompt references @{kind.capitalize()}{idx}, "+ f"but only {counts[kind]} reference {kind} inputs are connected."+ )+ return f"<{_REFERENCE_LABELS[kind]} {idx}>"++ prev = None+ while prev != prompt:+ prev = prompt+ prompt = _REFERENCE_TAG_RE.sub(repl, prompt)+ return prompt+async def _apply_speech_source(cls: type[IO.ComfyNode], payload: dict, speech: dict, require_voice: bool) -> None:"""Fill script/audio speech fields of a /v3/videos payload from the DynamicCombo dict."""@@ -64,11 +89,11 @@ async def _apply_speech_source(cls: type[IO.ComfyNode], payload: dict, speech: dpayload["voice_settings"] = {"speed": round(speed, 2)}-async def _create_and_poll_video(cls: type[IO.ComfyNode], payload: dict) -> dict:- """POST a /v3/videos payload, poll until terminal, and return the final video data."""+async def _create_and_poll_video(cls: type[IO.ComfyNode], create_path: str, payload: dict) -> dict:+ """POST a video payload, poll /v3/videos until terminal, and return the final video data."""created = await sync_op_raw(cls,- ApiEndpoint(path=_VIDEOS_PATH, method="POST"),+ ApiEndpoint(path=create_path, method="POST"),data=payload,)video_id = (created.get("data") or {}).get("video_id")@@ -248,7 +273,7 @@ class HeyGenTalkingPhotoNode(IO.ComfyNode):"title": "ComfyUI Talking Photo",}await _apply_speech_source(cls, payload, speech, require_voice=True)- video = await _create_and_poll_video(cls, payload)+ video = await _create_and_poll_video(cls, _VIDEOS_PATH, payload)return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))@@ -449,7 +474,7 @@ class HeyGenAvatarVideoNode(IO.ComfyNode):raise ValueError("background_color must be a hex color code like '#00ff00'.")payload["background"] = {"type": "color", "value": background_color}await _apply_speech_source(cls, payload, speech, require_voice=False)- video = await _create_and_poll_video(cls, payload)+ video = await _create_and_poll_video(cls, _VIDEOS_PATH, payload)return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))@@ -783,6 +808,272 @@ class HeyGenTextToSpeechNode(IO.ComfyNode):return IO.NodeOutput(audio_bytes_to_audio_input(audio_bytes.getvalue()))+class HeyGenReferenceToVideoNode(IO.ComfyNode):++ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="HeyGenReferenceToVideoNode",+ display_name="HeyGen Video 1.0 Reference to Video",+ category="partner/video/HeyGen",+ description="Generate a video with synchronized dialogue and sound from a text prompt using "+ "HeyGen Video 1.0. Optionally connect up to 12 reference images, videos, and audio clips and "+ "mention them in the prompt as @Image1, @Video1, @Audio1.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "heygen-video-1",+ [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Description of the video, including any dialogue. Refer to "+ "connected references as @Image1, @Video1, @Audio1, numbered per type "+ "in input order.",+ ),+ IO.Int.Input(+ "duration",+ default=5,+ min=5,+ max=15,+ step=1,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Duration of the output video in seconds.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["768p", "480p"],+ tooltip="Output resolution.",+ ),+ IO.Combo.Input(+ "aspect_ratio",+ options=["auto", "16:9", "9:16", "1:1", "4:3", "3:4", "21:9"],+ tooltip="Output aspect ratio. 'auto' is 16:9 without references and "+ "otherwise follows the first reference image (or the first reference "+ "video when no images are connected).",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=4294967295,+ control_after_generate=True,+ tooltip="Seed for the generation. Results can still vary between runs "+ "with the same seed.",+ ),+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("reference_image"),+ names=[f"image_{i}" for i in range(1, 10)],+ min=0,+ ),+ tooltip="Up to 9 images of people, products, or places to use in the "+ "video; refer to them as @Image1, @Image2, ...",+ ),+ IO.Autogrow.Input(+ "reference_videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("reference_video"),+ names=[f"video_{i}" for i in range(1, 4)],+ min=0,+ ),+ tooltip="Up to 3 videos to use as references; refer to them as "+ "@Video1, @Video2, ...",+ ),+ IO.Autogrow.Input(+ "reference_audios",+ template=IO.Autogrow.TemplateNames(+ IO.Audio.Input("reference_audio"),+ names=[f"audio_{i}" for i in range(1, 4)],+ min=0,+ ),+ tooltip="Up to 3 audio clips, such as a voice for a speaker; refer to "+ "them as @Audio1, @Audio2, ... Requires at least one reference image or "+ "video. A voice reference needs a few seconds of clean speech; clips "+ "shorter than about 2 seconds are usually ignored.",+ ),+ ],+ ),+ ],+ ),+ ],+ outputs=[IO.Video.Output()],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(+ widgets=["model", "model.duration", "model.resolution"],+ input_groups=["model.reference_images", "model.reference_videos"],+ ),+ expr="""+ (+ $dur := $lookup(widgets, "model.duration");+ $hd := $lookup(widgets, "model.resolution") = "768p";+ $imgsRaw := $lookup(inputGroups, "model.reference_images");+ $imgs := $imgsRaw ? $imgsRaw : 0;+ $vidsRaw := $lookup(inputGroups, "model.reference_videos");+ $vids := $vidsRaw ? $vidsRaw : 0;+ $rate := ($imgs + $vids) > 0 ? ($hd ? 0.0429 : 0.0286) : ($hd ? 0.02145 : 0.0143);+ $vids > 0+ ? {"type":"range_usd","min_usd": $rate * $dur, "max_usd": $rate * ($dur + 5 * $vids)}+ : {"type":"usd","usd": $rate * $dur}+ )+ """,+ ),+ )++ @classmethod+ async def execute(cls, model: dict) -> IO.NodeOutput:+ reference_images = model.get("reference_images", {})+ reference_videos = model.get("reference_videos", {})+ reference_audios = model.get("reference_audios", {})+ if reference_audios and not (reference_images or reference_videos):+ raise ValueError("Reference audio requires at least one reference image or video.")+ total = len(reference_images) + len(reference_videos) + len(reference_audios)+ if total > 12:+ raise ValueError(f"At most 12 references can be connected in total; got {total}.")+ for key, image in reference_images.items():+ if get_number_of_images(image) != 1:+ raise ValueError(f"Reference image input '{key}' must contain exactly one image, not a batch.")+ validate_image_aspect_ratio(image, (1, 4), (4, 1), strict=False)+ prompt = _rewrite_reference_tags(+ model["prompt"],+ {"image": len(reference_images), "video": len(reference_videos), "audio": len(reference_audios)},+ )+ validate_string(prompt, strip_whitespace=True, min_length=1, max_length=32000)+ payload = {+ "model": model["model"],+ "mode": "reference_to_video" if reference_images or reference_videos else "text_to_video",+ "prompt": prompt,+ "duration": model["duration"],+ "resolution": model["resolution"],+ "seed": model["seed"],+ }+ if model["aspect_ratio"] != "auto":+ payload["aspect_ratio"] = model["aspect_ratio"]+ if reference_images:+ urls = await upload_images_to_comfyapi(+ cls, list(reference_images.values()), max_images=9, mime_type="image/png"+ )+ payload["reference_images"] = [{"type": "url", "url": u} for u in urls]+ if reference_videos:+ payload["reference_videos"] = [+ {"type": "url", "url": await upload_video_to_comfyapi(cls, v)} for v in reference_videos.values()+ ]+ if reference_audios:+ payload["reference_audio"] = [+ {+ "type": "url",+ "url": await upload_audio_to_comfyapi(+ cls, a, container_format="mp3", codec_name="libmp3lame", mime_type="audio/mpeg"+ ),+ }+ for a in reference_audios.values()+ ]+ video = await _create_and_poll_video(cls, _MODEL_VIDEOS_PATH, payload)+ return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))+++class HeyGenImageToVideoNode(IO.ComfyNode):++ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="HeyGenImageToVideoNode",+ display_name="HeyGen Video 1.0 Image to Video",+ category="partner/video/HeyGen",+ description="Animate an image into a video with synchronized dialogue and sound using "+ "HeyGen Video 1.0. The image is used as the first frame.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "heygen-video-1",+ [+ IO.Image.Input(+ "image",+ tooltip="First frame of the video. The output keeps the aspect ratio "+ "of this image; crop it to change the shape of the video.",+ ),+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Description of what happens in the video, including any dialogue.",+ ),+ IO.Int.Input(+ "duration",+ default=5,+ min=5,+ max=15,+ step=1,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Duration of the output video in seconds.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["768p", "480p"],+ tooltip="Output resolution.",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=4294967295,+ control_after_generate=True,+ tooltip="Seed for the generation. Results can still vary between runs "+ "with the same seed.",+ ),+ ],+ ),+ ],+ ),+ ],+ outputs=[IO.Video.Output()],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.duration", "model.resolution"]),+ expr="""+ {"type":"usd","usd": ($lookup(widgets, "model.resolution") = "768p" ? 0.02145 : 0.0143)+ * $lookup(widgets, "model.duration")}+ """,+ ),+ )++ @classmethod+ async def execute(cls, model: dict) -> IO.NodeOutput:+ validate_string(model["prompt"], strip_whitespace=True, min_length=1, max_length=32000)+ if get_number_of_images(model["image"]) != 1:+ raise ValueError("The image input must contain exactly one image, not a batch.")+ validate_image_aspect_ratio(model["image"], (1, 4), (4, 1), strict=False)+ image_url = await upload_image_to_comfyapi(cls, model["image"], mime_type="image/png")+ payload = {+ "model": model["model"],+ "mode": "image_to_video",+ "prompt": model["prompt"],+ "image": {"type": "url", "url": image_url},+ "duration": model["duration"],+ "resolution": model["resolution"],+ "seed": model["seed"],+ }+ video = await _create_and_poll_video(cls, _MODEL_VIDEOS_PATH, payload)+ return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))++class HeyGenExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[IO.ComfyNode]]:@@ -792,6 +1083,8 @@ class HeyGenExtension(ComfyExtension):HeyGenCreateAvatarNode,HeyGenVideoTranslateNode,HeyGenTextToSpeechNode,+ HeyGenReferenceToVideoNode,+ HeyGenImageToVideoNode,]