Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(HeyGen): add HeyGen Voice 1.0 instant clone and text-to-speech nodes (#17043)

Alexander Piskuna2440bc77cba
diff --git a/comfy_api_nodes/apis/heygen.py b/comfy_api_nodes/apis/heygen.py
index 71b724872..f1def1f24 100644
--- a/comfy_api_nodes/apis/heygen.py
+++ b/comfy_api_nodes/apis/heygen.py
@@ -450,3 +450,45 @@ HEYGEN_TRANSLATE_LANGUAGES = [
"Welsh (United Kingdom)",
"Zulu (South Africa)",
]
+
+HEYGEN_VOICE_LANGUAGES = {
+ "Arabic": "ar",
+ "Belarusian": "be",
+ "Bulgarian": "bg",
+ "Catalan": "ca",
+ "Chinese": "zh",
+ "Croatian": "hr",
+ "Czech": "cs",
+ "Danish": "da",
+ "Dutch": "nl",
+ "English": "en",
+ "Finnish": "fi",
+ "French": "fr",
+ "German": "de",
+ "Greek": "el",
+ "Hebrew": "he",
+ "Hindi": "hi",
+ "Hungarian": "hu",
+ "Indonesian": "id",
+ "Italian": "it",
+ "Japanese": "ja",
+ "Korean": "ko",
+ "Macedonian": "mk",
+ "Malay": "ms",
+ "Persian": "fa",
+ "Polish": "pl",
+ "Portuguese": "pt",
+ "Romanian": "ro",
+ "Russian": "ru",
+ "Serbian": "sr",
+ "Slovak": "sk",
+ "Slovenian": "sl",
+ "Spanish": "es",
+ "Swedish": "sv",
+ "Tagalog": "tl",
+ "Tamil": "ta",
+ "Thai": "th",
+ "Turkish": "tr",
+ "Ukrainian": "uk",
+ "Vietnamese": "vi",
+}
diff --git a/comfy_api_nodes/nodes_heygen.py b/comfy_api_nodes/nodes_heygen.py
index 0a2bba81b..47086c990 100644
--- a/comfy_api_nodes/nodes_heygen.py
+++ b/comfy_api_nodes/nodes_heygen.py
@@ -1,3 +1,4 @@
+import asyncio
import re
import torch
@@ -10,6 +11,7 @@ from comfy_api_nodes.apis.heygen import (
HEYGEN_TRANSLATE_LANGUAGES,
HEYGEN_VOICE_GENERAL_MAP,
HEYGEN_VOICE_GENERAL_OPTIONS,
+ HEYGEN_VOICE_LANGUAGES,
HEYGEN_VOICE_TTS_MAP,
HEYGEN_VOICE_TTS_OPTIONS,
)
@@ -38,8 +40,12 @@ _TRANSLATIONS_PATH = "/proxy/heygen/v3/video-translations"
_SPEECH_PATH = "/proxy/heygen/v3/voices/speech"
_AVATARS_PATH = "/proxy/heygen/v3/avatars"
_LOOKS_PATH = "/proxy/heygen/v3/avatars/looks"
+_MODEL_VOICES_PATH = "/proxy/heygen/v3/models/audio/voices"
+_MODEL_TTS_PATH = "/proxy/heygen/v3/models/audio/tts"
_DEFAULT_VOICE_OPTION = "(avatar's default voice)"
+_VOICE_LANGUAGE_OPTIONS = ["auto", *HEYGEN_VOICE_LANGUAGES]
+_NEW_VOICE_RETRIES = 5
_AVATARS_BY_ENGINE = {
e: [label for label, (_aid, _atype, engines) in HEYGEN_AVATAR_MAP.items() if e in engines]
@@ -91,11 +97,18 @@ async def _apply_speech_source(cls: type[IO.ComfyNode], payload: dict, speech: d
async def _create_and_poll_video(cls: type[IO.ComfyNode], create_path: str, payload: dict) -> dict:
"""POST a video payload, poll /v3/videos until terminal, and return the final video data."""
- created = await sync_op_raw(
- cls,
- ApiEndpoint(path=create_path, method="POST"),
- data=payload,
- )
+ for attempt in range(_NEW_VOICE_RETRIES + 1):
+ try:
+ created = await sync_op_raw(
+ cls,
+ ApiEndpoint(path=create_path, method="POST"),
+ data=payload,
+ )
+ break
+ except Exception as e:
+ if attempt == _NEW_VOICE_RETRIES or "Invalid voice_id" not in str(e):
+ raise
+ await asyncio.sleep(3)
video_id = (created.get("data") or {}).get("video_id")
if not video_id:
raise ValueError(f"HeyGen did not return a video_id: {created}")
@@ -1080,6 +1093,203 @@ class HeyGenImageToVideoNode(IO.ComfyNode):
return IO.NodeOutput(await download_url_to_video_output(video["video_url"]))
+class HeyGenVoiceCloneNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls) -> IO.Schema:
+ return IO.Schema(
+ node_id="HeyGenVoiceCloneNode",
+ display_name="HeyGen Voice 1.0 Instant Clone",
+ category="partner/audio/HeyGen",
+ description="Clone a voice from a recording of one speaker with HeyGen Voice 1.0. Connect the voice_id "
+ "to HeyGen Voice 1.0 Text to Speech, or to custom_voice_id in HeyGen Avatar Video and HeyGen Talking Photo.",
+ inputs=[
+ IO.Audio.Input(
+ "audio",
+ tooltip="Recording of one speaker. Use at least 10 seconds of clear speech; "
+ "only the first 3 minutes are used.",
+ ),
+ IO.Combo.Input(
+ "language",
+ options=_VOICE_LANGUAGE_OPTIONS,
+ tooltip="Language spoken in the recording.",
+ ),
+ IO.Combo.Input(
+ "similarity",
+ options=["medium", "high", "extra_high", "max"],
+ default="max",
+ tooltip="How closely the voice follows the recording.",
+ ),
+ IO.Boolean.Input("remove_background_noise", default=True),
+ IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=2147483647,
+ control_after_generate=IO.ControlAfterGenerate.fixed,
+ tooltip="Change the seed to clone the recording again; each clone is a slightly different voice.",
+ ),
+ ],
+ outputs=[IO.String.Output(display_name="voice_id")],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(expr="""{"type":"usd","usd":0}"""),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ audio: Input.Audio,
+ language: str,
+ similarity: str,
+ remove_background_noise: bool,
+ seed: int,
+ ) -> IO.NodeOutput:
+ sample_rate = audio["sample_rate"]
+ audio_url = await upload_audio_to_comfyapi(
+ cls,
+ {"waveform": audio["waveform"][..., : 180 * sample_rate], "sample_rate": sample_rate},
+ container_format="mp3",
+ codec_name="libmp3lame",
+ mime_type="audio/mpeg",
+ )
+ payload = {
+ "mode": "instant",
+ "name": "ComfyUI Voice",
+ "audio": [{"type": "url", "url": audio_url}],
+ "similarity": similarity,
+ "remove_background_noise": remove_background_noise,
+ }
+ if language != "auto":
+ payload["language"] = HEYGEN_VOICE_LANGUAGES[language]
+ created = await sync_op_raw(
+ cls,
+ ApiEndpoint(path=_MODEL_VOICES_PATH, method="POST"),
+ data=payload,
+ )
+ voice_id = (created.get("data") or {}).get("voice_id")
+ if not voice_id:
+ raise ValueError(f"HeyGen did not return a voice_id: {created}")
+ final = await poll_op_raw(
+ cls,
+ ApiEndpoint(path=f"{_MODEL_VOICES_PATH}/{voice_id}"),
+ status_extractor=lambda r: (r.get("data") or {}).get("status"),
+ completed_statuses=["active", "failed"],
+ poll_interval=2.0,
+ )
+ if final["data"]["status"].strip().lower() == "failed":
+ raise ValueError(
+ f"HeyGen could not create a voice from this recording ({final['data'].get('failure_reason')})."
+ )
+ PromptServer.instance.send_progress_text(
+ f"Please save the voice_id for reuse.\n\nvoice_id: {voice_id}",
+ cls.hidden.unique_id,
+ )
+ return IO.NodeOutput(voice_id)
+
+
+def _voice_tts_inputs() -> list:
+ return [
+ IO.String.Input(
+ "voice_id",
+ default="",
+ tooltip="Voice from HeyGen Voice 1.0 Instant Clone.",
+ ),
+ IO.String.Input(
+ "text",
+ multiline=True,
+ default="",
+ tooltip="Up to 5000 characters. Speech longer than about 12 minutes fails, "
+ "so split very long Chinese or Japanese text.",
+ ),
+ IO.Combo.Input(
+ "language",
+ options=_VOICE_LANGUAGE_OPTIONS,
+ tooltip="Language of the text. Set it for closely related languages such as Serbian or Malay, "
+ "which auto-detection can confuse.",
+ ),
+ IO.Float.Input(
+ "expressiveness_boost",
+ default=0.5,
+ min=0.0,
+ max=1.0,
+ step=0.05,
+ tooltip="Higher values are more expressive; lower values stay closer to the original voice.",
+ ),
+ ]
+
+
+class HeyGenVoiceTextToSpeechNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls) -> IO.Schema:
+ return IO.Schema(
+ node_id="HeyGenVoiceTextToSpeechNode",
+ display_name="HeyGen Voice 1.0 Text to Speech",
+ category="partner/audio/HeyGen",
+ description="Speak text in a voice cloned with HeyGen Voice 1.0 Instant Clone.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option("heygen-voice-1", _voice_tts_inputs()),
+ IO.DynamicCombo.Option("heygen-voice-1-turbo", _voice_tts_inputs()),
+ ],
+ tooltip="heygen-voice-1-turbo is faster and costs half as much.",
+ ),
+ ],
+ outputs=[IO.Audio.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.text"]),
+ expr="""
+ (
+ $t := $lookup(widgets, "model.text");
+ $rate := widgets.model = "heygen-voice-1-turbo" ? 0.010725 : 0.02145;
+ $type($t) = "string" and $length($t) > 0
+ ? {"type":"usd","usd": $length($t) * $rate / 1000}
+ : {"type":"usd","usd": $rate, "format":{"approximate":true,"suffix":"/1K chars"}}
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ voice_id = model["voice_id"].strip()
+ if not voice_id:
+ raise ValueError("Connect a voice_id from HeyGen Voice 1.0 Instant Clone.")
+ text = model["text"].strip()
+ validate_string(text, field_name="text", min_length=1, max_length=5000)
+ payload = {
+ "model": model["model"],
+ "voice_id": voice_id,
+ "text": text,
+ "expressiveness_boost": model["expressiveness_boost"],
+ }
+ if model["language"] != "auto":
+ payload["language"] = HEYGEN_VOICE_LANGUAGES[model["language"]]
+ response = await sync_op_raw(
+ cls,
+ ApiEndpoint(path=_MODEL_TTS_PATH, method="POST"),
+ data=payload,
+ )
+ audio_url = (response.get("data") or {}).get("audio_url")
+ if not audio_url:
+ raise ValueError(f"HeyGen did not return an audio_url: {response}")
+ audio_bytes = await download_url_as_bytesio(audio_url)
+ return IO.NodeOutput(audio_bytes_to_audio_input(audio_bytes.getvalue()))
+
+
class HeyGenExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
@@ -1091,6 +1301,8 @@ class HeyGenExtension(ComfyExtension):
HeyGenTextToSpeechNode,
HeyGenReferenceToVideoNode,
HeyGenImageToVideoNode,
+ HeyGenVoiceCloneNode,
+ HeyGenVoiceTextToSpeechNode,
]