The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
add Veo3 First-Last-Frame node (#10878)
Alexander Piskun8938aa3f3064
diff --git a/comfy_api_nodes/apis/veo_api.py b/comfy_api_nodes/apis/veo_api.pyindex a55137afb..8328d1aa4 100644--- a/comfy_api_nodes/apis/veo_api.py+++ b/comfy_api_nodes/apis/veo_api.py@@ -1,34 +1,21 @@-from typing import Optional, Union-from enum import Enum+from typing import Optionalfrom pydantic import BaseModel, Field-class Image2(BaseModel):- bytesBase64Encoded: str- gcsUri: Optional[str] = None- mimeType: Optional[str] = None+class VeoRequestInstanceImage(BaseModel):+ bytesBase64Encoded: str | None = Field(None)+ gcsUri: str | None = Field(None)+ mimeType: str | None = Field(None)-class Image3(BaseModel):- bytesBase64Encoded: Optional[str] = None- gcsUri: str- mimeType: Optional[str] = None---class Instance1(BaseModel):- image: Optional[Union[Image2, Image3]] = Field(- None, description='Optional image to guide video generation'- )+class VeoRequestInstance(BaseModel):+ image: VeoRequestInstanceImage | None = Field(None)+ lastFrame: VeoRequestInstanceImage | None = Field(None)prompt: str = Field(..., description='Text description of the video')-class PersonGeneration1(str, Enum):- ALLOW = 'ALLOW'- BLOCK = 'BLOCK'---class Parameters1(BaseModel):+class VeoRequestParameters(BaseModel):aspectRatio: Optional[str] = Field(None, examples=['16:9'])durationSeconds: Optional[int] = NoneenhancePrompt: Optional[bool] = None@@ -37,17 +24,18 @@ class Parameters1(BaseModel):description='Generate audio for the video. Only supported by veo 3 models.',)negativePrompt: Optional[str] = None- personGeneration: Optional[PersonGeneration1] = None+ personGeneration: str | None = Field(None, description="ALLOW or BLOCK")sampleCount: Optional[int] = Noneseed: Optional[int] = NonestorageUri: Optional[str] = Field(None, description='Optional Cloud Storage URI to upload the video')+ resolution: str | None = Field(None)class VeoGenVidRequest(BaseModel):- instances: Optional[list[Instance1]] = None- parameters: Optional[Parameters1] = None+ instances: list[VeoRequestInstance] | None = Field(None)+ parameters: VeoRequestParameters | None = Field(None)class VeoGenVidResponse(BaseModel):diff --git a/comfy_api_nodes/nodes_veo2.py b/comfy_api_nodes/nodes_veo2.pyindex d37e9e9b4..a54dc13ab 100644--- a/comfy_api_nodes/nodes_veo2.py+++ b/comfy_api_nodes/nodes_veo2.py@@ -1,6 +1,7 @@import base64from io import BytesIO+import torchfrom typing_extensions import overridefrom comfy_api.input_impl.video_types import VideoFromFile@@ -10,6 +11,9 @@ from comfy_api_nodes.apis.veo_api import (VeoGenVidPollResponse,VeoGenVidRequest,VeoGenVidResponse,+ VeoRequestInstance,+ VeoRequestInstanceImage,+ VeoRequestParameters,)from comfy_api_nodes.util import (ApiEndpoint,@@ -346,12 +350,163 @@ class Veo3VideoGenerationNode(VeoVideoGenerationNode):)+class Veo3FirstLastFrameNode(IO.ComfyNode):++ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="Veo3FirstLastFrameNode",+ display_name="Google Veo 3 First-Last-Frame to Video",+ category="api node/video/Veo",+ description="Generate video using prompt and first and last frames.",+ inputs=[+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Text description of the video",+ ),+ IO.String.Input(+ "negative_prompt",+ multiline=True,+ default="",+ tooltip="Negative text prompt to guide what to avoid in the video",+ ),+ IO.Combo.Input("resolution", options=["720p", "1080p"]),+ IO.Combo.Input(+ "aspect_ratio",+ options=["16:9", "9:16"],+ default="16:9",+ tooltip="Aspect ratio of the output video",+ ),+ IO.Int.Input(+ "duration",+ default=8,+ min=4,+ max=8,+ step=2,+ display_mode=IO.NumberDisplay.slider,+ tooltip="Duration of the output video in seconds",+ ),+ IO.Int.Input(+ "seed",+ default=0,+ min=0,+ max=0xFFFFFFFF,+ step=1,+ display_mode=IO.NumberDisplay.number,+ control_after_generate=True,+ tooltip="Seed for video generation",+ ),+ IO.Image.Input("first_frame", tooltip="Start frame"),+ IO.Image.Input("last_frame", tooltip="End frame"),+ IO.Combo.Input(+ "model",+ options=["veo-3.1-generate", "veo-3.1-fast-generate"],+ default="veo-3.1-fast-generate",+ ),+ IO.Boolean.Input(+ "generate_audio",+ default=True,+ tooltip="Generate audio for the video.",+ ),+ ],+ outputs=[+ IO.Video.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ prompt: str,+ negative_prompt: str,+ resolution: str,+ aspect_ratio: str,+ duration: int,+ seed: int,+ first_frame: torch.Tensor,+ last_frame: torch.Tensor,+ model: str,+ generate_audio: bool,+ ):+ model = MODELS_MAP[model]+ initial_response = await sync_op(+ cls,+ ApiEndpoint(path=f"/proxy/veo/{model}/generate", method="POST"),+ response_model=VeoGenVidResponse,+ data=VeoGenVidRequest(+ instances=[+ VeoRequestInstance(+ prompt=prompt,+ image=VeoRequestInstanceImage(+ bytesBase64Encoded=tensor_to_base64_string(first_frame), mimeType="image/png"+ ),+ lastFrame=VeoRequestInstanceImage(+ bytesBase64Encoded=tensor_to_base64_string(last_frame), mimeType="image/png"+ ),+ ),+ ],+ parameters=VeoRequestParameters(+ aspectRatio=aspect_ratio,+ personGeneration="ALLOW",+ durationSeconds=duration,+ enhancePrompt=True, # cannot be False for Veo3+ seed=seed,+ generateAudio=generate_audio,+ negativePrompt=negative_prompt,+ resolution=resolution,+ ),+ ),+ )+ poll_response = await poll_op(+ cls,+ ApiEndpoint(path=f"/proxy/veo/{model}/poll", method="POST"),+ response_model=VeoGenVidPollResponse,+ status_extractor=lambda r: "completed" if r.done else "pending",+ data=VeoGenVidPollRequest(+ operationName=initial_response.name,+ ),+ poll_interval=5.0,+ estimated_duration=AVERAGE_DURATION_VIDEO_GEN,+ )++ if poll_response.error:+ raise Exception(f"Veo API error: {poll_response.error.message} (code: {poll_response.error.code})")++ response = poll_response.response+ filtered_count = response.raiMediaFilteredCount+ if filtered_count:+ reasons = response.raiMediaFilteredReasons or []+ reason_part = f": {reasons[0]}" if reasons else ""+ raise Exception(+ f"Content blocked by Google's Responsible AI filters{reason_part} "+ f"({filtered_count} video{'s' if filtered_count != 1 else ''} filtered)."+ )++ if response.videos:+ video = response.videos[0]+ if video.bytesBase64Encoded:+ return IO.NodeOutput(VideoFromFile(BytesIO(base64.b64decode(video.bytesBase64Encoded))))+ if video.gcsUri:+ return IO.NodeOutput(await download_url_to_video_output(video.gcsUri))+ raise Exception("Video returned but no data or URL was provided")+ raise Exception("Video generation completed but no video was returned")++class VeoExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[IO.ComfyNode]]:return [VeoVideoGenerationNode,Veo3VideoGenerationNode,+ Veo3FirstLastFrameNode,]