Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(VEED): add VEED Dubbing node (#17388)

Alexander Piskunf49e3b466f63
diff --git a/comfy_api_nodes/apis/veed.py b/comfy_api_nodes/apis/veed.py
index 61efa90f6..a4690bb0b 100644
--- a/comfy_api_nodes/apis/veed.py
+++ b/comfy_api_nodes/apis/veed.py
@@ -40,6 +40,12 @@ class VeedCleanAudioVideoRequest(BaseModel):
target_lufs: float | None = Field(None)
+class VeedDubbingRequest(BaseModel):
+ video_url: str = Field(...)
+ target_language: str = Field(...)
+ source_language: str | None = Field(None)
+
+
class VeedFile(BaseModel):
url: str = Field(...)
@@ -64,3 +70,137 @@ class VeedJob(BaseModel):
class VeedJobResponse(BaseModel):
data: VeedJob = Field(...)
+
+
+VEED_DUBBING_LANGUAGES = {
+ "Afrikaans": "af",
+ "Akan": "ak",
+ "Albanian": "sq",
+ "Amharic": "am",
+ "Arabic": "ar",
+ "Armenian": "hy",
+ "Assamese": "as",
+ "Asturian": "ast",
+ "Azerbaijani": "az",
+ "Basque": "eu",
+ "Belarusian": "be",
+ "Bengali": "bn",
+ "Bosnian": "bs",
+ "Bulgarian": "bg",
+ "Burmese": "my",
+ "Cantonese": "yue",
+ "Catalan": "ca",
+ "Cebuano": "ceb",
+ "Chichewa": "ny",
+ "Chinese (Mandarin)": "zh",
+ "Croatian": "hr",
+ "Czech": "cs",
+ "Danish": "da",
+ "Dutch": "nl",
+ "English": "en",
+ "Estonian": "et",
+ "Filipino": "fil",
+ "Finnish": "fi",
+ "French": "fr",
+ "Galician": "gl",
+ "Ganda": "lg",
+ "Georgian": "ka",
+ "German": "de",
+ "Greek": "el",
+ "Gujarati": "gu",
+ "Hausa": "ha",
+ "Hebrew": "he",
+ "Hindi": "hi",
+ "Hungarian": "hu",
+ "Icelandic": "is",
+ "Indonesian": "id",
+ "Irish": "ga",
+ "Italian": "it",
+ "Japanese": "ja",
+ "Javanese": "jv",
+ "Kannada": "kn",
+ "Kazakh": "kk",
+ "Kikuyu": "ki",
+ "Kinyarwanda": "rw",
+ "Kirundi": "rn",
+ "Korean": "ko",
+ "Kyrgyz": "ky",
+ "Latvian": "lv",
+ "Lingala": "ln",
+ "Lithuanian": "lt",
+ "Luxembourgish": "lb",
+ "Macedonian": "mk",
+ "Malay": "ms",
+ "Malayalam": "ml",
+ "Maltese": "mt",
+ "Maori": "mi",
+ "Marathi": "mr",
+ "Mongolian": "mn",
+ "Nepali": "ne",
+ "Northern Sotho": "nso",
+ "Norwegian": "no",
+ "Occitan": "oc",
+ "Odia": "or",
+ "Pashto": "ps",
+ "Persian": "fa",
+ "Polish": "pl",
+ "Portuguese": "pt",
+ "Punjabi": "pa",
+ "Romanian": "ro",
+ "Russian": "ru",
+ "Serbian": "sr",
+ "Sindhi": "sd",
+ "Slovak": "sk",
+ "Slovenian": "sl",
+ "Somali": "so",
+ "Southern Sotho": "st",
+ "Spanish": "es",
+ "Sundanese": "su",
+ "Swahili": "sw",
+ "Swati": "ss",
+ "Swedish": "sv",
+ "Tajik": "tg",
+ "Tamil": "ta",
+ "Telugu": "te",
+ "Thai": "th",
+ "Tibetan": "bo",
+ "Tsonga": "ts",
+ "Tswana": "tn",
+ "Turkish": "tr",
+ "Ukrainian": "uk",
+ "Urdu": "ur",
+ "Uyghur": "ug",
+ "Uzbek": "uz",
+ "Venda": "ve",
+ "Vietnamese": "vi",
+ "Welsh": "cy",
+ "Wolof": "wo",
+ "Yoruba": "yo",
+ "Zulu": "zu",
+}
+
+VEED_DUBBING_ACCENTS = {
+ "Arabic (Egypt)": "ar-EG",
+ "Chinese (Taiwan)": "zh-TW",
+ "English (Australia)": "en-AU",
+ "English (Canada)": "en-CA",
+ "English (UK)": "en-GB",
+ "English (US)": "en-US",
+ "French (Canada)": "fr-CA",
+ "French (France)": "fr-FR",
+ "Portuguese (Brazil)": "pt-BR",
+ "Portuguese (Portugal)": "pt-PT",
+ "Spanish (Argentina)": "es-AR",
+ "Spanish (Chile)": "es-CL",
+ "Spanish (Mexico)": "es-MX",
+ "Spanish (Spain)": "es-ES",
+}
+
+VEED_DUBBING_STANDARD_ONLY = (
+ "Asturian", "Bengali", "Chichewa", "Irish", "Lingala", "Luxembourgish", "Maltese", "Maori", "Occitan", "Odia",
+ "Pashto", "Serbian", "Somali",
+)
+VEED_DUBBING_PRO_ONLY = (
+ "Akan", "Albanian", "Amharic", "Basque", "Ganda", "Kikuyu", "Kinyarwanda", "Kirundi", "Northern Sotho",
+ "Southern Sotho", "Sundanese", "Swati", "Tibetan", "Tsonga", "Tswana", "Uyghur", "Venda", "Wolof", "Zulu",
+)
diff --git a/comfy_api_nodes/nodes_veed.py b/comfy_api_nodes/nodes_veed.py
index c9b86487a..c2c5a1ee0 100644
--- a/comfy_api_nodes/nodes_veed.py
+++ b/comfy_api_nodes/nodes_veed.py
@@ -4,9 +4,14 @@ from typing_extensions import override
from comfy_api.latest import IO, ComfyExtension, Input
from comfy_api_nodes.apis.veed import (
+ VEED_DUBBING_ACCENTS,
+ VEED_DUBBING_LANGUAGES,
+ VEED_DUBBING_PRO_ONLY,
+ VEED_DUBBING_STANDARD_ONLY,
VeedBackgroundRemovalRequest,
VeedCleanAudioRequest,
VeedCleanAudioVideoRequest,
+ VeedDubbingRequest,
VeedFabricRequest,
VeedGreenScreenRequest,
VeedJob,
@@ -33,6 +38,13 @@ MAX_FABRIC_AUDIO_SECONDS = 300
MAX_LIPSYNC_AUDIO_SECONDS = 600
MAX_BACKGROUND_REMOVAL_SECONDS = 60
MAX_CLEAN_AUDIO_SECONDS = 1799
+MAX_DUBBING_SECONDS = 599
+DUBBING_STANDARD_LANGUAGES = [k for k in VEED_DUBBING_LANGUAGES if k not in VEED_DUBBING_PRO_ONLY]
+DUBBING_PRO_SOURCE_LANGUAGES = [k for k in VEED_DUBBING_LANGUAGES if k not in VEED_DUBBING_STANDARD_ONLY]
+DUBBING_PRO_LANGUAGES = sorted(
+ [k for k in DUBBING_PRO_SOURCE_LANGUAGES if k not in ("English", "French", "Portuguese", "Spanish")]
+ + list(VEED_DUBBING_ACCENTS)
+)
async def _run_job(cls: type[IO.ComfyNode], route: str, request: BaseModel) -> VeedJob:
@@ -431,6 +443,89 @@ class VeedCleanVideoAudioNode(IO.ComfyNode):
return IO.NodeOutput(await download_url_to_video_output(job.result.video.url))
+def _dubbing_option(
+ key: str, target_languages: list[str], default_language: str, source_languages: list[str]
+) -> IO.DynamicCombo.Option:
+ return IO.DynamicCombo.Option(
+ key,
+ [
+ IO.Video.Input(
+ "video",
+ tooltip=f"Video with a visible face and speech, up to {MAX_DUBBING_SECONDS} seconds. "
+ "The output is as long as its audio track and plays at 25 fps.",
+ ),
+ IO.Combo.Input("target_language", options=target_languages, default=default_language),
+ IO.Combo.Input(
+ "source_language",
+ options=["auto", *source_languages],
+ tooltip="Language spoken in the video.",
+ ),
+ IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=2147483647,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed controls whether the node should re-run; "
+ "results are non-deterministic regardless of seed.",
+ ),
+ ],
+ )
+
+
+class VeedDubbingNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="VeedDubbingNode",
+ display_name="VEED Dubbing",
+ category="partner/video/VEED",
+ description="Translate the speech in a video into another language with VEED. Each speaker's voice is "
+ "cloned and their lips are synced to the translation.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ _dubbing_option("standard", DUBBING_STANDARD_LANGUAGES, "English", DUBBING_STANDARD_LANGUAGES),
+ _dubbing_option("pro", DUBBING_PRO_LANGUAGES, "English (US)", DUBBING_PRO_SOURCE_LANGUAGES),
+ ],
+ tooltip="standard: lower price, mono audio; Spanish is Latin American and Portuguese is Brazilian. "
+ "pro: regional accents and stereo audio.",
+ ),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["model"]),
+ expr="""{"type":"usd","usd": widgets.model = "pro" ? 0.2574 : 0.1573, "format":{"suffix":"/second"}}""",
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ validate_video_duration(model["video"], max_duration=MAX_DUBBING_SECONDS)
+ with av.open(model["video"].get_stream_source()) as container:
+ if not container.streams.audio:
+ raise ValueError("The video has no audio track to dub.")
+ source = model["source_language"]
+ job = await _run_job(
+ cls,
+ "dubbing-pro" if model["model"] == "pro" else "dubbing",
+ VeedDubbingRequest(
+ video_url=await upload_video_to_comfyapi(cls, model["video"]),
+ target_language={**VEED_DUBBING_LANGUAGES, **VEED_DUBBING_ACCENTS}[model["target_language"]],
+ source_language=None if source == "auto" else VEED_DUBBING_LANGUAGES[source],
+ ),
+ )
+ return IO.NodeOutput(await download_url_to_video_output(job.result.video.url))
+
+
class VeedExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
@@ -440,6 +535,7 @@ class VeedExtension(ComfyExtension):
VeedRemoveVideoBackgroundNode,
VeedCleanAudioNode,
VeedCleanVideoAudioNode,
+ VeedDubbingNode,
]