Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

add Veo3 First-Last-Frame node (#10878)

Alexander Piskun8938aa3f3064
diff --git a/comfy_api_nodes/apis/veo_api.py b/comfy_api_nodes/apis/veo_api.py
index a55137afb..8328d1aa4 100644
--- a/comfy_api_nodes/apis/veo_api.py
+++ b/comfy_api_nodes/apis/veo_api.py
@@ -1,34 +1,21 @@
-from typing import Optional, Union
-from enum import Enum
+from typing import Optional
from pydantic import BaseModel, Field
-class Image2(BaseModel):
- bytesBase64Encoded: str
- gcsUri: Optional[str] = None
- mimeType: Optional[str] = None
+class VeoRequestInstanceImage(BaseModel):
+ bytesBase64Encoded: str | None = Field(None)
+ gcsUri: str | None = Field(None)
+ mimeType: str | None = Field(None)
-class Image3(BaseModel):
- bytesBase64Encoded: Optional[str] = None
- gcsUri: str
- mimeType: Optional[str] = None
-
-
-class Instance1(BaseModel):
- image: Optional[Union[Image2, Image3]] = Field(
- None, description='Optional image to guide video generation'
- )
+class VeoRequestInstance(BaseModel):
+ image: VeoRequestInstanceImage | None = Field(None)
+ lastFrame: VeoRequestInstanceImage | None = Field(None)
prompt: str = Field(..., description='Text description of the video')
-class PersonGeneration1(str, Enum):
- ALLOW = 'ALLOW'
- BLOCK = 'BLOCK'
-
-
-class Parameters1(BaseModel):
+class VeoRequestParameters(BaseModel):
aspectRatio: Optional[str] = Field(None, examples=['16:9'])
durationSeconds: Optional[int] = None
enhancePrompt: Optional[bool] = None
@@ -37,17 +24,18 @@ class Parameters1(BaseModel):
description='Generate audio for the video. Only supported by veo 3 models.',
)
negativePrompt: Optional[str] = None
- personGeneration: Optional[PersonGeneration1] = None
+ personGeneration: str | None = Field(None, description="ALLOW or BLOCK")
sampleCount: Optional[int] = None
seed: Optional[int] = None
storageUri: Optional[str] = Field(
None, description='Optional Cloud Storage URI to upload the video'
)
+ resolution: str | None = Field(None)
class VeoGenVidRequest(BaseModel):
- instances: Optional[list[Instance1]] = None
- parameters: Optional[Parameters1] = None
+ instances: list[VeoRequestInstance] | None = Field(None)
+ parameters: VeoRequestParameters | None = Field(None)
class VeoGenVidResponse(BaseModel):
diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.py
index d37e9e9b4..a54dc13ab 100644
--- a/comfy_api_nodes/nodes_veo2.py
+++ b/comfy_api_nodes/nodes_veo2.py
@@ -1,6 +1,7 @@
import base64
from io import BytesIO
+import torch
from typing_extensions import override
from comfy_api.input_impl.video_types import VideoFromFile
@@ -10,6 +11,9 @@ from comfy_api_nodes.apis.veo_api import (
VeoGenVidPollResponse,
VeoGenVidRequest,
VeoGenVidResponse,
+ VeoRequestInstance,
+ VeoRequestInstanceImage,
+ VeoRequestParameters,
)
from comfy_api_nodes.util import (
ApiEndpoint,
@@ -346,12 +350,163 @@ class Veo3VideoGenerationNode(VeoVideoGenerationNode):
)
+class Veo3FirstLastFrameNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="Veo3FirstLastFrameNode",
+ display_name="Google Veo 3 First-Last-Frame to Video",
+ category="api node/video/Veo",
+ description="Generate video using prompt and first and last frames.",
+ inputs=[
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Text description of the video",
+ ),
+ IO.String.Input(
+ "negative_prompt",
+ multiline=True,
+ default="",
+ tooltip="Negative text prompt to guide what to avoid in the video",
+ ),
+ IO.Combo.Input("resolution", options=["720p", "1080p"]),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=["16:9", "9:16"],
+ default="16:9",
+ tooltip="Aspect ratio of the output video",
+ ),
+ IO.Int.Input(
+ "duration",
+ default=8,
+ min=4,
+ max=8,
+ step=2,
+ display_mode=IO.NumberDisplay.slider,
+ tooltip="Duration of the output video in seconds",
+ ),
+ IO.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=0xFFFFFFFF,
+ step=1,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed for video generation",
+ ),
+ IO.Image.Input("first_frame", tooltip="Start frame"),
+ IO.Image.Input("last_frame", tooltip="End frame"),
+ IO.Combo.Input(
+ "model",
+ options=["veo-3.1-generate", "veo-3.1-fast-generate"],
+ default="veo-3.1-fast-generate",
+ ),
+ IO.Boolean.Input(
+ "generate_audio",
+ default=True,
+ tooltip="Generate audio for the video.",
+ ),
+ ],
+ outputs=[
+ IO.Video.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ prompt: str,
+ negative_prompt: str,
+ resolution: str,
+ aspect_ratio: str,
+ duration: int,
+ seed: int,
+ first_frame: torch.Tensor,
+ last_frame: torch.Tensor,
+ model: str,
+ generate_audio: bool,
+ ):
+ model = MODELS_MAP[model]
+ initial_response = await sync_op(
+ cls,
+ ApiEndpoint(path=f"/proxy/veo/{model}/generate", method="POST"),
+ response_model=VeoGenVidResponse,
+ data=VeoGenVidRequest(
+ instances=[
+ VeoRequestInstance(
+ prompt=prompt,
+ image=VeoRequestInstanceImage(
+ bytesBase64Encoded=tensor_to_base64_string(first_frame), mimeType="image/png"
+ ),
+ lastFrame=VeoRequestInstanceImage(
+ bytesBase64Encoded=tensor_to_base64_string(last_frame), mimeType="image/png"
+ ),
+ ),
+ ],
+ parameters=VeoRequestParameters(
+ aspectRatio=aspect_ratio,
+ personGeneration="ALLOW",
+ durationSeconds=duration,
+ enhancePrompt=True, # cannot be False for Veo3
+ seed=seed,
+ generateAudio=generate_audio,
+ negativePrompt=negative_prompt,
+ resolution=resolution,
+ ),
+ ),
+ )
+ poll_response = await poll_op(
+ cls,
+ ApiEndpoint(path=f"/proxy/veo/{model}/poll", method="POST"),
+ response_model=VeoGenVidPollResponse,
+ status_extractor=lambda r: "completed" if r.done else "pending",
+ data=VeoGenVidPollRequest(
+ operationName=initial_response.name,
+ ),
+ poll_interval=5.0,
+ estimated_duration=AVERAGE_DURATION_VIDEO_GEN,
+ )
+
+ if poll_response.error:
+ raise Exception(f"Veo API error: {poll_response.error.message} (code: {poll_response.error.code})")
+
+ response = poll_response.response
+ filtered_count = response.raiMediaFilteredCount
+ if filtered_count:
+ reasons = response.raiMediaFilteredReasons or []
+ reason_part = f": {reasons[0]}" if reasons else ""
+ raise Exception(
+ f"Content blocked by Google's Responsible AI filters{reason_part} "
+ f"({filtered_count} video{'s' if filtered_count != 1 else ''} filtered)."
+ )
+
+ if response.videos:
+ video = response.videos[0]
+ if video.bytesBase64Encoded:
+ return IO.NodeOutput(VideoFromFile(BytesIO(base64.b64decode(video.bytesBase64Encoded))))
+ if video.gcsUri:
+ return IO.NodeOutput(await download_url_to_video_output(video.gcsUri))
+ raise Exception("Video returned but no data or URL was provided")
+ raise Exception("Video generation completed but no video was returned")
+
+
class VeoExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
return [
VeoVideoGenerationNode,
Veo3VideoGenerationNode,
+ Veo3FirstLastFrameNode,
]