Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(VEED): add Fabric 1.0, Lipsync 2.0, Background Removal and Clean Audio nodes (#17242)

Alexander Piskunfaa7e494f3a0
diff --git a/comfy_api_nodes/apis/veed.py b/comfy_api_nodes/apis/veed.py
new file mode 100644
index 000000000..61efa90f6
--- /dev/null
+++ b/comfy_api_nodes/apis/veed.py
@@ -0,0 +1,66 @@
+from pydantic import BaseModel, Field
+
+
+class VeedFabricRequest(BaseModel):
+ image_url: str = Field(...)
+ audio_url: str = Field(...)
+ resolution: str = Field(...)
+
+
+class VeedLipsyncRequest(BaseModel):
+ video_url: str = Field(...)
+ audio_url: str = Field(...)
+
+
+class VeedBackgroundRemovalRequest(BaseModel):
+ video_url: str = Field(...)
+ output_codec: str = Field(...)
+ refine_foreground_edges: bool = Field(...)
+ subject_is_person: bool = Field(...)
+
+
+class VeedGreenScreenRequest(BaseModel):
+ video_url: str = Field(...)
+ output_codec: str = Field(...)
+ spill_suppression_strength: float = Field(...)
+
+
+class VeedCleanAudioRequest(BaseModel):
+ audio_url: str = Field(...)
+ strength: float = Field(...)
+ normalize_loudness: bool = Field(...)
+ target_lufs: float | None = Field(None)
+ output_format: str = Field(...)
+
+
+class VeedCleanAudioVideoRequest(BaseModel):
+ video_url: str = Field(...)
+ strength: float = Field(...)
+ normalize_loudness: bool = Field(...)
+ target_lufs: float | None = Field(None)
+
+
+class VeedFile(BaseModel):
+ url: str = Field(...)
+
+
+class VeedJobResult(BaseModel):
+ video: VeedFile | None = Field(None)
+ audio: VeedFile | None = Field(None)
+ files: list[VeedFile] | None = Field(None)
+
+
+class VeedJobError(BaseModel):
+ code: str = Field(...)
+ message: str = Field(...)
+
+
+class VeedJob(BaseModel):
+ job_id: str = Field(...)
+ status: str = Field(...)
+ result: VeedJobResult | None = Field(None)
+ error: VeedJobError | None = Field(None)
+
+
+class VeedJobResponse(BaseModel):
+ data: VeedJob = Field(...)
diff --git a/comfy_api_nodes/nodes_bria.py b/comfy_api_nodes/nodes_bria.py
index 6673bdcef..6c395ef3e 100644
--- a/comfy_api_nodes/nodes_bria.py
+++ b/comfy_api_nodes/nodes_bria.py
@@ -1,6 +1,4 @@
-import av
import torch
-from av.codec import CodecContext
from pydantic import BaseModel
from typing_extensions import override
@@ -49,6 +47,7 @@ from comfy_api_nodes.util import (
upload_video_to_comfyapi,
validate_string,
validate_video_duration,
+ video_to_images_and_mask,
)
BRIA_MAX_OUTPUT_SIDE = 8192
@@ -1036,28 +1035,6 @@ class BriaVideoReplaceBackground(IO.ComfyNode):
return IO.NodeOutput(await download_url_to_video_output(response.result.video_url))
-def _video_to_images_and_mask(video: Input.Video) -> tuple[Input.Image, Input.Mask]:
- """Decode a transparent webm (VP9 + alpha) into image frames and an alpha mask.
-
- VP9 keeps its alpha in a side layer that PyAV's default vp9 decoder drops, so the frames
- are decoded with libvpx-vp9. Returns RGB images [B,H,W,3] in 0..1 and a mask [B,H,W]
- following the Load Image convention (1 = transparent) for compositing or Save WEBM.
- """
- rgb_frames: list[torch.Tensor] = []
- alpha_frames: list[torch.Tensor] = []
- with av.open(video.get_stream_source(), mode="r") as container:
- stream = container.streams.video[0]
- decoder = CodecContext.create("libvpx-vp9", "r") if stream.codec_context.name == "vp9" else None
- for packet in container.demux(stream):
- for frame in (decoder.decode(packet) if decoder is not None else packet.decode()):
- rgba = torch.from_numpy(frame.to_ndarray(format="rgba")).float() / 255.0
- rgb_frames.append(rgba[..., :3])
- alpha_frames.append(rgba[..., 3])
- images = torch.stack(rgb_frames) if rgb_frames else torch.zeros(0, 0, 0, 3)
- mask = (1.0 - torch.stack(alpha_frames)) if alpha_frames else torch.zeros((images.shape[0], 64, 64))
- return images, mask
-
-
class BriaTransparentVideoBackground(IO.ComfyNode):
@classmethod
@@ -1122,7 +1099,7 @@ class BriaTransparentVideoBackground(IO.ComfyNode):
response_model=BriaRemoveVideoBackgroundResponse,
)
video_out = await download_url_to_video_output(response.result.video_url)
- images, mask = _video_to_images_and_mask(video_out)
+ images, mask = video_to_images_and_mask(video_out)
return IO.NodeOutput(images, mask)
diff --git a/comfy_api_nodes/nodes_veed.py b/comfy_api_nodes/nodes_veed.py
new file mode 100644
index 000000000..c9b86487a
--- /dev/null
+++ b/comfy_api_nodes/nodes_veed.py
@@ -0,0 +1,447 @@
+import av
+from pydantic import BaseModel
+from typing_extensions import override
+
+from comfy_api.latest import IO, ComfyExtension, Input
+from comfy_api_nodes.apis.veed import (
+ VeedBackgroundRemovalRequest,
+ VeedCleanAudioRequest,
+ VeedCleanAudioVideoRequest,
+ VeedFabricRequest,
+ VeedGreenScreenRequest,
+ VeedJob,
+ VeedJobResponse,
+ VeedLipsyncRequest,
+)
+from comfy_api_nodes.util import (
+ ApiEndpoint,
+ audio_bytes_to_audio_input,
+ download_url_as_bytesio,
+ download_url_to_video_output,
+ poll_op,
+ sync_op,
+ upload_audio_to_comfyapi,
+ upload_image_to_comfyapi,
+ upload_video_to_comfyapi,
+ validate_audio_duration,
+ validate_video_duration,
+ video_to_images_and_mask,
+)
+
+VEED_PATH = "/proxy/veed/v1"
+MAX_FABRIC_AUDIO_SECONDS = 300
+MAX_LIPSYNC_AUDIO_SECONDS = 600
+MAX_BACKGROUND_REMOVAL_SECONDS = 60
+MAX_CLEAN_AUDIO_SECONDS = 1799
+
+
+async def _run_job(cls: type[IO.ComfyNode], route: str, request: BaseModel) -> VeedJob:
+ submitted = await sync_op(
+ cls,
+ ApiEndpoint(path=f"{VEED_PATH}/{route}", method="POST"),
+ response_model=VeedJobResponse,
+ data=request,
+ )
+ job = (
+ await poll_op(
+ cls,
+ ApiEndpoint(path=f"{VEED_PATH}/{route}/{submitted.data.job_id}"),
+ response_model=VeedJobResponse,
+ status_extractor=lambda r: r.data.status,
+ completed_statuses=["COMPLETED", "FAILED", "CANCELLED"],
+ failed_statuses=[],
+ poll_interval=10.0,
+ )
+ ).data
+ if job.status != "COMPLETED":
+ if job.error and job.error.code == "insufficient_credits":
+ raise ValueError("VEED is temporarily unavailable. Please try again later.")
+ if job.error:
+ raise ValueError(f"VEED job {job.status.lower()} [{job.error.code}]: {job.error.message}")
+ raise ValueError(f"VEED job {job.status.lower()}.")
+ return job
+
+
+async def _upload_audio(cls: type[IO.ComfyNode], audio: Input.Audio) -> str:
+ return await upload_audio_to_comfyapi(
+ cls, audio, container_format="wav", codec_name="pcm_s16le", mime_type="audio/wav"
+ )
+
+
+class VeedFabricNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="VeedFabricNode",
+ display_name="VEED Fabric 1.0",
+ category="partner/video/VEED",
+ description="Turn a still image and a speech track into a talking, lip-synced video with VEED Fabric. "
+ "Works with photos, illustrations and mascots; the video is as long as the audio.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option(
+ "fabric-1.0",
+ [
+ IO.Image.Input(
+ "image",
+ tooltip="Image with a clearly visible face. The video keeps its aspect ratio.",
+ ),
+ IO.Audio.Input(
+ "audio",
+ tooltip=f"Speech to lip-sync to, up to {MAX_FABRIC_AUDIO_SECONDS} seconds. "
+ "Sets the video length.",
+ ),
+ IO.Combo.Input(
+ "resolution",
+ options=["480p", "720p"],
+ default="720p",
+ tooltip="Output size class: about 0.4 megapixels at 480p (864x480 at 16:9) and "
+ "0.9 megapixels at 720p (1312x736 at 16:9).",
+ ),
+ ],
+ ),
+ ],
+ ),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["model.resolution"]),
+ expr="""
+ {
+ "type":"usd",
+ "usd": $lookup(widgets, "model.resolution") = "480p" ? 0.1144 : 0.2145,
+ "format":{"suffix":"/second"}
+ }
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ validate_audio_duration(model["audio"], max_duration=MAX_FABRIC_AUDIO_SECONDS)
+ job = await _run_job(
+ cls,
+ model["model"],
+ VeedFabricRequest(
+ image_url=await upload_image_to_comfyapi(cls, model["image"], mime_type="image/png"),
+ audio_url=await _upload_audio(cls, model["audio"]),
+ resolution=model["resolution"],
+ ),
+ )
+ return IO.NodeOutput(await download_url_to_video_output(job.result.video.url))
+
+
+class VeedLipsyncNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="VeedLipsyncNode",
+ display_name="VEED Lipsync 2.0",
+ category="partner/video/VEED",
+ description="Re-sync a speaker's mouth in a video to a new speech track with VEED Lipsync. "
+ "One speaker per video; the output keeps the video's resolution at 25 fps and is as long as the audio.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option(
+ "lipsync-2.0",
+ [
+ IO.Video.Input(
+ "video",
+ tooltip="Footage of one front-facing speaker. If it is shorter than the audio, "
+ "it plays forward and backward until the audio ends.",
+ ),
+ IO.Audio.Input(
+ "audio",
+ tooltip=f"New speech, up to {MAX_LIPSYNC_AUDIO_SECONDS} seconds. "
+ "Sets the video length.",
+ ),
+ IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=2147483647,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed controls whether the node should re-run; "
+ "results are non-deterministic regardless of seed.",
+ ),
+ ],
+ ),
+ ],
+ ),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(expr="""{"type":"usd","usd":0.1001,"format":{"suffix":"/second"}}"""),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ validate_audio_duration(model["audio"], max_duration=MAX_LIPSYNC_AUDIO_SECONDS)
+ job = await _run_job(
+ cls,
+ model["model"],
+ VeedLipsyncRequest(
+ video_url=await upload_video_to_comfyapi(cls, model["video"]),
+ audio_url=await _upload_audio(cls, model["audio"]),
+ ),
+ )
+ return IO.NodeOutput(await download_url_to_video_output(job.result.video.url))
+
+
+def _segmentation_option(key: str) -> IO.DynamicCombo.Option:
+ return IO.DynamicCombo.Option(
+ key,
+ [
+ IO.Video.Input("video"),
+ IO.Boolean.Input(
+ "refine_foreground_edges",
+ default=True,
+ tooltip="Cleaner edges on hair, fur and fine detail. Turning it off lowers the price.",
+ ),
+ IO.Boolean.Input(
+ "subject_is_person",
+ default=True,
+ tooltip="Turn off when the subject is an object or animal.",
+ ),
+ ],
+ )
+
+
+class VeedRemoveVideoBackgroundNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="VeedRemoveVideoBackgroundNode",
+ display_name="VEED Remove Video Background",
+ category="partner/video/VEED",
+ description="Cut the subject out of a video with VEED and return the frames plus an alpha mask. "
+ "Use them for compositing, or combine them with Join Image with Alpha and Save WEBM to write a "
+ "transparent video.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ _segmentation_option("standard"),
+ _segmentation_option("fast"),
+ IO.DynamicCombo.Option(
+ "green screen",
+ [
+ IO.Video.Input("video", tooltip="Footage shot against a green screen."),
+ IO.Float.Input(
+ "spill_suppression_strength",
+ default=0.8,
+ min=0.0,
+ max=1.0,
+ step=0.01,
+ tooltip="How strongly green cast is removed from the subject. Raise it when "
+ "green spots remain; lower it when the subject's colors shift.",
+ ),
+ ],
+ ),
+ ],
+ tooltip="standard: any background, best edges, output up to 3840x2160. fast: quicker, output up to "
+ "1920x1080. green screen: keys out a green backdrop, output up to 3840x2160. Larger videos are "
+ "downscaled.",
+ ),
+ ],
+ outputs=[
+ IO.Image.Output("images"),
+ IO.Mask.Output(
+ "mask",
+ tooltip="1 where the background was removed. Use Join Image with Alpha and Save WEBM for a "
+ "transparent video; add Invert Mask before Image Composite Masked.",
+ ),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.refine_foreground_edges"]),
+ expr="""
+ (
+ $unrefined := $lookup(widgets, "model.refine_foreground_edges") = false;
+ $rate := widgets.model = "fast"
+ ? ($unrefined ? 0.01144 : 0.01716)
+ : widgets.model = "green screen"
+ ? 0.03575
+ : ($unrefined ? 0.02145 : 0.032175);
+ {"type":"usd","usd": $rate, "format":{"suffix":"/30 frames"}}
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ validate_video_duration(model["video"], max_duration=MAX_BACKGROUND_REMOVAL_SECONDS)
+ video_url = await upload_video_to_comfyapi(cls, model["video"])
+ if model["model"] == "green screen":
+ route = "video-background-removal-green-screen"
+ request = VeedGreenScreenRequest(
+ video_url=video_url,
+ output_codec="vp9",
+ spill_suppression_strength=model["spill_suppression_strength"],
+ )
+ else:
+ route = "video-background-removal-fast" if model["model"] == "fast" else "video-background-removal"
+ request = VeedBackgroundRemovalRequest(
+ video_url=video_url,
+ output_codec="vp9",
+ refine_foreground_edges=model["refine_foreground_edges"],
+ subject_is_person=model["subject_is_person"],
+ )
+ job = await _run_job(cls, route, request)
+ video = await download_url_to_video_output(job.result.files[0].url)
+ return IO.NodeOutput(*video_to_images_and_mask(video))
+
+
+def _clean_audio_inputs() -> list:
+ return [
+ IO.Float.Input(
+ "strength",
+ default=0.874,
+ min=0.0,
+ max=1.0,
+ step=0.001,
+ tooltip="How much noise is removed under speech. Lower keeps more room tone behind the voice; "
+ "pauses between words are always fully cleaned, and values above the default change little.",
+ ),
+ IO.DynamicCombo.Input(
+ "loudness",
+ options=[
+ IO.DynamicCombo.Option(
+ "normalize",
+ [
+ IO.Float.Input(
+ "target_lufs",
+ default=-19.0,
+ min=-40.0,
+ max=-8.0,
+ step=0.1,
+ tooltip="Integrated loudness of the output in LUFS. Peaks are capped at -1.1 dBTP, so loud "
+ "targets stop early (about -15 LUFS for typical speech).",
+ ),
+ ],
+ ),
+ IO.DynamicCombo.Option("keep input level", []),
+ ],
+ ),
+ ]
+
+
+class VeedCleanAudioNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="VeedCleanAudioNode",
+ display_name="VEED Clean Audio",
+ category="partner/audio/VEED",
+ description="Remove background noise from speech with VEED. Built for speech in any language: "
+ "music and sound effects are removed too. Returns 48 kHz mono audio.",
+ inputs=[
+ IO.Audio.Input("audio", tooltip="Speech recording, shorter than 30 minutes."),
+ *_clean_audio_inputs(),
+ ],
+ outputs=[IO.Audio.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(expr="""{"type":"usd","usd":0.017875,"format":{"suffix":"/minute"}}"""),
+ )
+
+ @classmethod
+ async def execute(cls, audio: Input.Audio, strength: float, loudness: dict) -> IO.NodeOutput:
+ validate_audio_duration(audio, max_duration=MAX_CLEAN_AUDIO_SECONDS)
+ job = await _run_job(
+ cls,
+ "clean-audio",
+ VeedCleanAudioRequest(
+ audio_url=await _upload_audio(cls, audio),
+ strength=strength,
+ normalize_loudness=loudness["loudness"] == "normalize",
+ target_lufs=loudness.get("target_lufs"),
+ output_format="flac",
+ ),
+ )
+ cleaned = await download_url_as_bytesio(job.result.audio.url)
+ return IO.NodeOutput(audio_bytes_to_audio_input(cleaned.getvalue()))
+
+
+class VeedCleanVideoAudioNode(IO.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="VeedCleanVideoAudioNode",
+ display_name="VEED Clean Audio (in Video)",
+ category="partner/video/VEED",
+ description="Remove background noise from the speech in a video with VEED. Only the soundtrack changes: "
+ "it is replaced with the cleaned 48 kHz mono speech.",
+ inputs=[
+ IO.Video.Input("video", tooltip="Video with a speech soundtrack, shorter than 30 minutes."),
+ *_clean_audio_inputs(),
+ ],
+ outputs=[IO.Video.Output()],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(expr="""{"type":"usd","usd":0.017875,"format":{"suffix":"/minute"}}"""),
+ )
+
+ @classmethod
+ async def execute(cls, video: Input.Video, strength: float, loudness: dict) -> IO.NodeOutput:
+ validate_video_duration(video, max_duration=MAX_CLEAN_AUDIO_SECONDS)
+ with av.open(video.get_stream_source()) as container:
+ if not container.streams.audio:
+ raise ValueError("The video has no audio track to clean.")
+ job = await _run_job(
+ cls,
+ "clean-audio-video",
+ VeedCleanAudioVideoRequest(
+ video_url=await upload_video_to_comfyapi(cls, video),
+ strength=strength,
+ normalize_loudness=loudness["loudness"] == "normalize",
+ target_lufs=loudness.get("target_lufs"),
+ ),
+ )
+ return IO.NodeOutput(await download_url_to_video_output(job.result.video.url))
+
+
+class VeedExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:
+ return [
+ VeedFabricNode,
+ VeedLipsyncNode,
+ VeedRemoveVideoBackgroundNode,
+ VeedCleanAudioNode,
+ VeedCleanVideoAudioNode,
+ ]
+
+
+async def comfy_entrypoint() -> VeedExtension:
+ return VeedExtension()
diff --git a/comfy_api_nodes/util/__init__.py b/comfy_api_nodes/util/__init__.py
index 034090a5b..595d4f964 100644
--- a/comfy_api_nodes/util/__init__.py
+++ b/comfy_api_nodes/util/__init__.py
@@ -31,6 +31,7 @@ from .conversions import (
upscale_image_tensor_to_min_pixels,
upscale_video_to_min_pixels,
video_to_base64_string,
+ video_to_images_and_mask,
)
from .download_helpers import (
download_url_as_bytesio,
@@ -107,6 +108,7 @@ __all__ = [
"upscale_image_tensor_to_min_pixels",
"upscale_video_to_min_pixels",
"video_to_base64_string",
+ "video_to_images_and_mask",
# Validation utilities
"get_image_dimensions",
"get_number_of_images",
diff --git a/comfy_api_nodes/util/conversions.py b/comfy_api_nodes/util/conversions.py
index a39c4e470..8879e7b69 100644
--- a/comfy_api_nodes/util/conversions.py
+++ b/comfy_api_nodes/util/conversions.py
@@ -8,6 +8,7 @@ from io import BytesIO
import av
import numpy as np
import torch
+from av.codec import CodecContext
from PIL import Image
from comfy.utils import common_upscale
@@ -590,6 +591,26 @@ def _apply_video_scale(video: Input.Video, scale_dims: tuple[int, int]) -> Input
raise RuntimeError(f"Failed to resize video: {str(e)}") from e
+def video_to_images_and_mask(video: Input.Video) -> tuple[Input.Image, Input.Mask]:
+ frames: list[torch.Tensor | None] = []
+ with av.open(video.get_stream_source(), mode="r") as container:
+ stream = container.streams.video[0]
+ decoder = CodecContext.create("libvpx-vp9", "r") if stream.codec_context.name == "vp9" else None
+ for packet in container.demux(stream):
+ for frame in decoder.decode(packet) if decoder is not None else packet.decode():
+ frames.append(torch.from_numpy(frame.to_ndarray(format="rgba")))
+ if not frames:
+ raise ValueError("The video contains no frames.")
+ height, width = frames[0].shape[:2]
+ images = torch.empty((len(frames), height, width, 3))
+ mask = torch.empty((len(frames), height, width))
+ for i, rgba in enumerate(frames):
+ images[i] = rgba[..., :3] / 255.0
+ mask[i] = 1.0 - rgba[..., 3] / 255.0
+ frames[i] = None
+ return images, mask
+
+
def _f32_pcm(wav: torch.Tensor) -> torch.Tensor:
"""Convert audio to float 32 bits PCM format. Copy-paste from nodes_audio.py file."""
if wav.dtype.is_floating_point: