Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(HeyGen): add HeyGen Video 1.0 reference-to-video and image-to-video nodes (#16695)

Alexander Piskun051ddedaae52
diff --git a/comfy_api_nodes/nodes_heygen.py b/comfy_api_nodes/nodes_heygen.py
index b1375d143..28b7c26d9 100644
--- a/comfy_api_nodes/nodes_heygen.py
+++ b/comfy_api_nodes/nodes_heygen.py
@@ -1,3 +1,5 @@
+import re
+
import torch
from typing_extensions import override
@@ -25,11 +27,13 @@ from comfy_api_nodes.util import (
upload_image_to_comfyapi,
upload_images_to_comfyapi,
upload_video_to_comfyapi,
+ validate_image_aspect_ratio,
validate_string,
)
from server import PromptServer
_VIDEOS_PATH = "/proxy/heygen/v3/videos"
+_MODEL_VIDEOS_PATH = "/proxy/heygen/v3/models/videos"
_TRANSLATIONS_PATH = "/proxy/heygen/v3/video-translations"
_SPEECH_PATH = "/proxy/heygen/v3/voices/speech"
_AVATARS_PATH = "/proxy/heygen/v3/avatars"
@@ -42,6 +46,27 @@ _AVATARS_BY_ENGINE = {
for e in ("avatar_iv", "avatar_iii", "avatar_v")
}
+_REFERENCE_TAG_RE = re.compile(r"(?<!\w)@(image|video|audio)(\d*)(?!\w)", re.IGNORECASE | re.ASCII)
+_REFERENCE_LABELS = {"image": "Picture", "video": "Video", "audio": "Audio"}
+
+
+def _rewrite_reference_tags(prompt: str, counts: dict[str, int]) -> str:
+ def repl(match: re.Match) -> str:
+ kind = match.group(1).lower()
+ idx = int(match.group(2) or 1)
+ if not 1 <= idx <= counts[kind]:
+ raise ValueError(
+ f"The prompt references @{kind.capitalize()}{idx}, "
+ f"but only {counts[kind]} reference {kind} inputs are connected."
+ )
+ return f"<{_REFERENCE_LABELS[kind]} {idx}>"
+
+ prev = None
+ while prev != prompt:
+ prev = prompt
+ prompt = _REFERENCE_TAG_RE.sub(repl, prompt)
+ return prompt
+
async def _apply_speech_source(cls: type[IO.ComfyNode], payload: dict, speech: dict, require_voice: bool) -> None:
"""Fill script/audio speech fields of a /v3/videos payload from the DynamicCombo dict."""
@@ -64,11 +89,11 @@ async def _apply_speech_source(cls: type[IO.ComfyNode], payload: dict, speech: d
payload["voice_settings"] = {"speed": round(speed, 2)}
-async def _create_and_poll_video(cls: type[IO.ComfyNode], payload: dict) -> dict:
- """POST a /v3/videos payload, poll until terminal, and return the final video data."""
+async def _create_and_poll_video(cls: type[IO.ComfyNode], create_path: str, payload: dict) -> dict:
+ """POST a video payload, poll /v3/videos until terminal, and return the final video data."""
created = await sync_op_raw(
cls,
- ApiEndpoint(path=_VIDEOS_PATH, method="POST"),
+ ApiEndpoint(path=create_path, method="POST"),
data=payload,
)
video_id = (created.get("data") or {}).get("video_id")
@@ -248,7 +273,7 @@ class HeyGenTalkingPhotoNode(IO.ComfyNode):
"title": "ComfyUI Talking Photo",
}
await _apply_speech_source(cls, payload, speech, require_voice=True)
- video = await _create_and_poll_video(cls, payload)
+ video = await _create_and_poll_video(cls, _VIDEOS_PATH, payload)
return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
@@ -449,7 +474,7 @@ class HeyGenAvatarVideoNode(IO.ComfyNode):
raise ValueError("background_color must be a hex color code like '#00ff00'.")
payload["background"] = {"type": "color", "value": background_color}
await _apply_speech_source(cls, payload, speech, require_voice=False)
- video = await _create_and_poll_video(cls, payload)
+ video = await _create_and_poll_video(cls, _VIDEOS_PATH, payload)
return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
@@ -783,6 +808,272 @@ class HeyGenTextToSpeechNode(IO.ComfyNode):
return IO.NodeOutput(audio_bytes_to_audio_input(audio_bytes.getvalue()))
+class HeyGenReferenceToVideoNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls) -> IO.Schema:
+ return IO.Schema(
+ node_id="HeyGenReferenceToVideoNode",
+ display_name="HeyGen Video 1.0 Reference to Video",
+ category="partner/video/HeyGen",
+ description="Generate a video with synchronized dialogue and sound from a text prompt using "
+ "HeyGen Video 1.0. Optionally connect up to 12 reference images, videos, and audio clips and "
+ "mention them in the prompt as @Image1, @Video1, @Audio1.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option(
+ "heygen-video-1",
+ [
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Description of the video, including any dialogue. Refer to "
+ "connected references as @Image1, @Video1, @Audio1, numbered per type "
+ "in input order.",
+ ),
+ IO.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=15,
+ step=1,
+ display_mode=IO.NumberDisplay.slider,
+ tooltip="Duration of the output video in seconds.",
+ ),
+ IO.Combo.Input(
+ "resolution",
+ options=["768p", "480p"],
+ tooltip="Output resolution.",
+ ),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=["auto", "16:9", "9:16", "1:1", "4:3", "3:4", "21:9"],
+ tooltip="Output aspect ratio. 'auto' is 16:9 without references and "
+ "otherwise follows the first reference image (or the first reference "
+ "video when no images are connected).",
+ ),
+ IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=4294967295,
+ control_after_generate=True,
+ tooltip="Seed for the generation. Results can still vary between runs "
+ "with the same seed.",
+ ),
+ IO.Autogrow.Input(
+ "reference_images",
+ template=IO.Autogrow.TemplateNames(
+ IO.Image.Input("reference_image"),
+ names=[f"image_{i}" for i in range(1, 10)],
+ min=0,
+ ),
+ tooltip="Up to 9 images of people, products, or places to use in the "
+ "video; refer to them as @Image1, @Image2, ...",
+ ),
+ IO.Autogrow.Input(
+ "reference_videos",
+ template=IO.Autogrow.TemplateNames(
+ IO.Video.Input("reference_video"),
+ names=[f"video_{i}" for i in range(1, 4)],
+ min=0,
+ ),
+ tooltip="Up to 3 videos to use as references; refer to them as "
+ "@Video1, @Video2, ...",
+ ),
+ IO.Autogrow.Input(
+ "reference_audios",
+ template=IO.Autogrow.TemplateNames(
+ IO.Audio.Input("reference_audio"),
+ names=[f"audio_{i}" for i in range(1, 4)],
+ min=0,
+ ),
+ tooltip="Up to 3 audio clips, such as a voice for a speaker; refer to "
+ "them as @Audio1, @Audio2, ... Requires at least one reference image or "
+ "video. A voice reference needs a few seconds of clean speech; clips "
+ "shorter than about 2 seconds are usually ignored.",
+ ),
+ ],
+ ),
+ ],
+ ),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(
+ widgets=["model", "model.duration", "model.resolution"],
+ input_groups=["model.reference_images", "model.reference_videos"],
+ ),
+ expr="""
+ (
+ $dur := $lookup(widgets, "model.duration");
+ $hd := $lookup(widgets, "model.resolution") = "768p";
+ $imgsRaw := $lookup(inputGroups, "model.reference_images");
+ $imgs := $imgsRaw ? $imgsRaw : 0;
+ $vidsRaw := $lookup(inputGroups, "model.reference_videos");
+ $vids := $vidsRaw ? $vidsRaw : 0;
+ $rate := ($imgs + $vids) > 0 ? ($hd ? 0.0429 : 0.0286) : ($hd ? 0.02145 : 0.0143);
+ $vids > 0
+ ? {"type":"range_usd","min_usd": $rate * $dur, "max_usd": $rate * ($dur + 5 * $vids)}
+ : {"type":"usd","usd": $rate * $dur}
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ reference_images = model.get("reference_images", {})
+ reference_videos = model.get("reference_videos", {})
+ reference_audios = model.get("reference_audios", {})
+ if reference_audios and not (reference_images or reference_videos):
+ raise ValueError("Reference audio requires at least one reference image or video.")
+ total = len(reference_images) + len(reference_videos) + len(reference_audios)
+ if total > 12:
+ raise ValueError(f"At most 12 references can be connected in total; got {total}.")
+ for key, image in reference_images.items():
+ if get_number_of_images(image) != 1:
+ raise ValueError(f"Reference image input '{key}' must contain exactly one image, not a batch.")
+ validate_image_aspect_ratio(image, (1, 4), (4, 1), strict=False)
+ prompt = _rewrite_reference_tags(
+ model["prompt"],
+ {"image": len(reference_images), "video": len(reference_videos), "audio": len(reference_audios)},
+ )
+ validate_string(prompt, strip_whitespace=True, min_length=1, max_length=32000)
+ payload = {
+ "model": model["model"],
+ "mode": "reference_to_video" if reference_images or reference_videos else "text_to_video",
+ "prompt": prompt,
+ "duration": model["duration"],
+ "resolution": model["resolution"],
+ "seed": model["seed"],
+ }
+ if model["aspect_ratio"] != "auto":
+ payload["aspect_ratio"] = model["aspect_ratio"]
+ if reference_images:
+ urls = await upload_images_to_comfyapi(
+ cls, list(reference_images.values()), max_images=9, mime_type="image/png"
+ )
+ payload["reference_images"] = [{"type": "url", "url": u} for u in urls]
+ if reference_videos:
+ payload["reference_videos"] = [
+ {"type": "url", "url": await upload_video_to_comfyapi(cls, v)} for v in reference_videos.values()
+ ]
+ if reference_audios:
+ payload["reference_audio"] = [
+ {
+ "type": "url",
+ "url": await upload_audio_to_comfyapi(
+ cls, a, container_format="mp3", codec_name="libmp3lame", mime_type="audio/mpeg"
+ ),
+ }
+ for a in reference_audios.values()
+ ]
+ video = await _create_and_poll_video(cls, _MODEL_VIDEOS_PATH, payload)
+ return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
+
+
+class HeyGenImageToVideoNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls) -> IO.Schema:
+ return IO.Schema(
+ node_id="HeyGenImageToVideoNode",
+ display_name="HeyGen Video 1.0 Image to Video",
+ category="partner/video/HeyGen",
+ description="Animate an image into a video with synchronized dialogue and sound using "
+ "HeyGen Video 1.0. The image is used as the first frame.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option(
+ "heygen-video-1",
+ [
+ IO.Image.Input(
+ "image",
+ tooltip="First frame of the video. The output keeps the aspect ratio "
+ "of this image; crop it to change the shape of the video.",
+ ),
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Description of what happens in the video, including any dialogue.",
+ ),
+ IO.Int.Input(
+ "duration",
+ default=5,
+ min=5,
+ max=15,
+ step=1,
+ display_mode=IO.NumberDisplay.slider,
+ tooltip="Duration of the output video in seconds.",
+ ),
+ IO.Combo.Input(
+ "resolution",
+ options=["768p", "480p"],
+ tooltip="Output resolution.",
+ ),
+ IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=4294967295,
+ control_after_generate=True,
+ tooltip="Seed for the generation. Results can still vary between runs "
+ "with the same seed.",
+ ),
+ ],
+ ),
+ ],
+ ),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.duration", "model.resolution"]),
+ expr="""
+ {"type":"usd","usd": ($lookup(widgets, "model.resolution") = "768p" ? 0.02145 : 0.0143)
+ * $lookup(widgets, "model.duration")}
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ validate_string(model["prompt"], strip_whitespace=True, min_length=1, max_length=32000)
+ if get_number_of_images(model["image"]) != 1:
+ raise ValueError("The image input must contain exactly one image, not a batch.")
+ validate_image_aspect_ratio(model["image"], (1, 4), (4, 1), strict=False)
+ image_url = await upload_image_to_comfyapi(cls, model["image"], mime_type="image/png")
+ payload = {
+ "model": model["model"],
+ "mode": "image_to_video",
+ "prompt": model["prompt"],
+ "image": {"type": "url", "url": image_url},
+ "duration": model["duration"],
+ "resolution": model["resolution"],
+ "seed": model["seed"],
+ }
+ video = await _create_and_poll_video(cls, _MODEL_VIDEOS_PATH, payload)
+ return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
+
+
class HeyGenExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
@@ -792,6 +1083,8 @@ class HeyGenExtension(ComfyExtension):
HeyGenCreateAvatarNode,
HeyGenVideoTranslateNode,
HeyGenTextToSpeechNode,
+ HeyGenReferenceToVideoNode,
+ HeyGenImageToVideoNode,
]