The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(sync.so): add support for "sync-3" model (#14928)
Alexander Piskun26537080cb1d
diff --git a/comfy_api_nodes/apis/sync_so.py b/comfy_api_nodes/apis/sync_so.pynew file mode 100644index 000000000..af9419580--- /dev/null+++ b/comfy_api_nodes/apis/sync_so.py@@ -0,0 +1,49 @@+from pydantic import BaseModel, Field+++class SyncInputItem(BaseModel):+ type: str = Field(..., description="Input kind: 'video', 'image' or 'audio'.")+ url: str = Field(...)+++class SyncActiveSpeakerDetection(BaseModel):+ auto_detect: bool | None = Field(+ None, description="Detect the active speaker automatically. Video input only; rejected for images."+ )+ frame_number: int | None = Field(+ None, description="Frame used for manual speaker selection. Must be 0 for image inputs."+ )+ coordinates: list[int] | None = Field(+ None, description="Pixel [x, y] of the speaker's face in the frame selected by frame_number."+ )+++class SyncGenerationOptions(BaseModel):+ sync_mode: str | None = Field(+ None,+ description="How to resolve an audio/video duration mismatch: "+ "cut_off, bounce, loop, silence or remap. Ignored for image inputs.",+ )+ i2v_prompt: str | None = Field(+ None, description="Motion prompt for image-to-video generation. Image input only."+ )+ active_speaker_detection: SyncActiveSpeakerDetection | None = Field(None)+++class SyncGenerationRequest(BaseModel):+ model: str = Field(..., description="Generation model, e.g. 'sync-3'.")+ input: list[SyncInputItem] = Field(+ ..., description="Exactly one visual input (video or image) plus one audio input."+ )+ options: SyncGenerationOptions | None = Field(None)+++class SyncGeneration(BaseModel):+ """Subset of the Generation object returned by POST /v2/generate and GET /v2/generate/{id}."""++ id: str = Field(...)+ status: str = Field(..., description="PENDING | PROCESSING | COMPLETED | FAILED | REJECTED")+ outputUrl: str | None = Field(None)+ outputDuration: float | None = Field(None)+ error: str | None = Field(None, description="Human-readable failure message.")+ errorCode: str | None = Field(None, description="Stable machine-readable code from the GET /v2/errors catalog.")diff --git a/comfy_api_nodes/nodes_sync_so.py b/comfy_api_nodes/nodes_sync_so.pynew file mode 100644index 000000000..27382b399--- /dev/null+++ b/comfy_api_nodes/nodes_sync_so.py@@ -0,0 +1,391 @@+from typing_extensions import override++from comfy_api.latest import IO, ComfyExtension, Input+from comfy_api_nodes.apis.sync_so import (+ SyncActiveSpeakerDetection,+ SyncGeneration,+ SyncGenerationOptions,+ SyncGenerationRequest,+ SyncInputItem,+)+from comfy_api_nodes.util import (+ ApiEndpoint,+ download_url_to_video_output,+ downscale_image_tensor,+ downscale_image_tensor_by_max_side,+ get_image_dimensions,+ get_number_of_images,+ poll_op,+ sync_op,+ upload_audio_to_comfyapi,+ upload_image_to_comfyapi,+ upload_video_to_comfyapi,+ validate_audio_duration,+)+++class SyncLipSyncNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="SyncLipSyncNode",+ display_name="sync.so Lip Sync",+ category="partner/video/sync.so",+ description=(+ "Re-sync mouth movement in a video to new speech audio using sync.so. "+ "Handles close-ups, profiles and obstructions automatically while preserving "+ "the speaker's expression. Cost scales with output duration."+ ),+ inputs=[+ IO.Video.Input(+ "video",+ tooltip="Footage of the speaker to re-sync. Up to 4K (4096x2160); "+ "a constant frame rate of 24/25/30 fps works best.",+ ),+ IO.Audio.Input(+ "audio",+ tooltip="Speech audio to sync the mouth to.",+ ),+ IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=2147483647,+ control_after_generate=True,+ tooltip="Seed controls whether the node should re-run; "+ "results are non-deterministic regardless of seed.",+ ),+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "sync-3",+ [+ IO.Combo.Input(+ "sync_mode",+ options=["bounce", "cut_off", "loop", "silence", "remap"],+ default="bounce",+ tooltip=(+ "How to handle a duration mismatch between video and audio; "+ "this also sets the output length. "+ "bounce: video plays forward then backward until the audio ends "+ "(output = audio length). "+ "loop: video restarts until the audio ends (output = audio length). "+ "remap: video is time-stretched to match the audio (output = audio length). "+ "cut_off: the longer track is trimmed (output = shorter length). "+ "silence: nothing is trimmed; the shorter track is padded "+ "(output = longer length)."+ ),+ ),+ IO.Combo.Input(+ "speaker_selection",+ options=["default", "auto-detect", "coordinates"],+ default="default",+ tooltip=(+ "Which face to lipsync when several people are visible. "+ "default: let the model decide. "+ "auto-detect: detect and follow the active speaker. "+ "coordinates: target the face at pixel (speaker_x, speaker_y) "+ "in the frame chosen by speaker_frame."+ ),+ ),+ IO.Int.Input(+ "speaker_frame",+ default=0,+ min=0,+ max=1_000_000,+ advanced=True,+ tooltip="Video frame used to locate the speaker. "+ "Only used when speaker_selection is 'coordinates'.",+ ),+ IO.Int.Input(+ "speaker_x",+ default=0,+ min=0,+ max=4096,+ advanced=True,+ tooltip="X pixel coordinate of the speaker's face. "+ "Only used when speaker_selection is 'coordinates'.",+ ),+ IO.Int.Input(+ "speaker_y",+ default=0,+ min=0,+ max=4096,+ advanced=True,+ tooltip="Y pixel coordinate of the speaker's face. "+ "Only used when speaker_selection is 'coordinates'.",+ ),+ ],+ )+ ],+ tooltip="sync.so generation model.",+ ),+ ],+ outputs=[IO.Video.Output()],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ expr="""{"type":"usd","usd":0.19019,"format":{"approximate":true,"suffix":"/second"}}""",+ ),+ )++ @classmethod+ async def execute(+ cls,+ video: Input.Video,+ audio: Input.Audio,+ seed: int,+ model: dict,+ ) -> IO.NodeOutput:+ try:+ width, height = video.get_dimensions()+ except Exception:+ width = height = None+ if width and height and (max(width, height) > 4096 or width * height > 4096 * 2160):+ raise ValueError(+ f"sync.so rejects videos above 4K (4096x2160); got {width}x{height}. Downscale the video first."+ )+ validate_audio_duration(audio, max_duration=600)++ if model["speaker_selection"] == "auto-detect":+ speaker_detection = SyncActiveSpeakerDetection(auto_detect=True)+ elif model["speaker_selection"] == "coordinates":+ speaker_detection = SyncActiveSpeakerDetection(+ frame_number=model["speaker_frame"],+ coordinates=[model["speaker_x"], model["speaker_y"]],+ )+ else:+ speaker_detection = None++ video_url = await upload_video_to_comfyapi(cls, video, max_duration=600)+ audio_url = await upload_audio_to_comfyapi(cls, audio)++ generation = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/synclabs/v2/generate", method="POST"),+ response_model=SyncGeneration,+ data=SyncGenerationRequest(+ model=model["model"],+ input=[+ SyncInputItem(type="video", url=video_url),+ SyncInputItem(type="audio", url=audio_url),+ ],+ options=SyncGenerationOptions(+ sync_mode=model["sync_mode"],+ active_speaker_detection=speaker_detection,+ ),+ ),+ )+ generation = await poll_op(+ cls,+ ApiEndpoint(path=f"/proxy/synclabs/v2/generate/{generation.id}"),+ response_model=SyncGeneration,+ status_extractor=lambda g: g.status,+ completed_statuses=["COMPLETED", "FAILED", "REJECTED"],+ failed_statuses=[],+ queued_statuses=["PENDING"],+ poll_interval=10.0,+ )+ if generation.status != "COMPLETED":+ code = f" [{generation.errorCode}]" if generation.errorCode else ""+ raise ValueError(+ f"sync.so generation {generation.status.lower()}{code}: "+ f"{generation.error or 'no error details provided'}"+ )+ if not generation.outputUrl:+ raise ValueError("sync.so generation completed but no output URL was returned.")+ return IO.NodeOutput(await download_url_to_video_output(generation.outputUrl))+++class SyncTalkingImageNode(IO.ComfyNode):+ @classmethod+ def define_schema(cls) -> IO.Schema:+ return IO.Schema(+ node_id="SyncTalkingImageNode",+ display_name="sync.so Talking Image",+ category="partner/video/sync.so",+ description=(+ "Animate a still portrait into a talking video driven by speech audio, "+ "using sync.so's sync-3 model. The output duration matches the audio. "+ "Cost scales with output duration."+ ),+ inputs=[+ IO.Image.Input(+ "image",+ tooltip="A single image with a clearly visible face, up to 4K (4096x2160).",+ ),+ IO.Audio.Input(+ "audio",+ tooltip="Speech audio driving the talking video; the output duration matches it. "+ "Chain any TTS node here to drive the animation from text.",+ ),+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Optional guidance for how the portrait comes to life, e.g. "+ "'make the subject smile and look at the camera'. "+ "Leave empty for natural talking motion.",+ ),+ IO.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ control_after_generate=True,+ tooltip="Seed controls whether the node should re-run; "+ "results are non-deterministic regardless of seed.",+ ),+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option(+ "sync-3",+ [+ IO.Combo.Input(+ "speaker_selection",+ options=["default", "coordinates"],+ default="default",+ tooltip=(+ "Which face to animate when several people are visible. "+ "default: let the model decide. "+ "coordinates: target the face at pixel (speaker_x, speaker_y) "+ "in the image. Auto-detection is not supported for images."+ ),+ ),+ IO.Int.Input(+ "speaker_x",+ default=0,+ min=0,+ max=4096,+ advanced=True,+ tooltip="X pixel coordinate of the speaker's face. "+ "Only used when speaker_selection is 'coordinates'.",+ ),+ IO.Int.Input(+ "speaker_y",+ default=0,+ min=0,+ max=4096,+ advanced=True,+ tooltip="Y pixel coordinate of the speaker's face. "+ "Only used when speaker_selection is 'coordinates'.",+ ),+ IO.Boolean.Input(+ "auto_downscale",+ default=True,+ advanced=True,+ tooltip="Automatically downscale the image if it exceeds the 4K "+ "(4096x2160) input limit; speaker coordinates are scaled to match. "+ "When disabled, an oversized image raises an error instead.",+ ),+ ],+ )+ ],+ tooltip="sync.so generation model. Image input is exclusive to sync-3.",+ ),+ ],+ outputs=[IO.Video.Output()],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ expr="""{"type":"usd","usd":0.19019,"format":{"approximate":true,"suffix":"/second"}}""",+ ),+ )++ @classmethod+ async def execute(+ cls,+ image: Input.Image,+ audio: Input.Audio,+ prompt: str,+ seed: int,+ model: dict,+ ) -> IO.NodeOutput:+ if get_number_of_images(image) != 1:+ raise ValueError("Exactly one image is required; got a batch. Pick one frame first.")+ validate_audio_duration(audio, max_duration=600)++ height, width = get_image_dimensions(image)+ speaker_x, speaker_y = model["speaker_x"], model["speaker_y"]+ if max(width, height) > 4096 or width * height > 4096 * 2160:+ if not model["auto_downscale"]:+ raise ValueError(+ f"sync.so rejects images above 4K (4096x2160); got {width}x{height}. "+ "Downscale the image first or enable auto_downscale."+ )+ image = downscale_image_tensor(image, total_pixels=4096 * 2160)+ image = downscale_image_tensor_by_max_side(image, max_side=4096)+ new_height, new_width = get_image_dimensions(image)+ # speaker coordinates are given in the original image's pixel space+ speaker_x = min(new_width - 1, round(speaker_x * new_width / width))+ speaker_y = min(new_height - 1, round(speaker_y * new_height / height))++ if model["speaker_selection"] == "coordinates":+ speaker_detection = SyncActiveSpeakerDetection(+ frame_number=0, # images have a single frame; auto_detect is rejected by the API+ coordinates=[speaker_x, speaker_y],+ )+ else:+ speaker_detection = None++ image_url = await upload_image_to_comfyapi(cls, image, mime_type="image/png", total_pixels=None)+ audio_url = await upload_audio_to_comfyapi(cls, audio)++ generation = await sync_op(+ cls,+ ApiEndpoint(path="/proxy/synclabs/v2/generate", method="POST"),+ response_model=SyncGeneration,+ data=SyncGenerationRequest(+ model=model["model"],+ input=[+ SyncInputItem(type="image", url=image_url),+ SyncInputItem(type="audio", url=audio_url),+ ],+ options=SyncGenerationOptions(+ i2v_prompt=prompt.strip() or None,+ active_speaker_detection=speaker_detection,+ ),+ ),+ )+ generation = await poll_op(+ cls,+ ApiEndpoint(path=f"/proxy/synclabs/v2/generate/{generation.id}"),+ response_model=SyncGeneration,+ status_extractor=lambda g: g.status,+ completed_statuses=["COMPLETED", "FAILED", "REJECTED"],+ failed_statuses=[],+ queued_statuses=["PENDING"],+ poll_interval=10.0,+ )+ if generation.status != "COMPLETED":+ code = f" [{generation.errorCode}]" if generation.errorCode else ""+ raise ValueError(+ f"sync.so generation {generation.status.lower()}{code}: "+ f"{generation.error or 'no error details provided'}"+ )+ if not generation.outputUrl:+ raise ValueError("sync.so generation completed but no output URL was returned.")+ return IO.NodeOutput(await download_url_to_video_output(generation.outputUrl))+++class SyncExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:+ return [+ SyncLipSyncNode,+ SyncTalkingImageNode,+ ]+++async def comfy_entrypoint() -> SyncExtension:+ return SyncExtension()