The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(Google-Omni): add support for Omni 1.1 model (#15929)
Alexander Piskun4c6359f2e39d
diff --git a/comfy_api_nodes/apis/gemini.py b/comfy_api_nodes/apis/gemini.pyindex ce89928cb..4cc7dd6f9 100644--- a/comfy_api_nodes/apis/gemini.py+++ b/comfy_api_nodes/apis/gemini.py@@ -256,15 +256,32 @@ class GeminiInteractionMediaPart(BaseModel):mime_type: str | None = Field(None)+class GeminiInteractionVideoConfig(BaseModel):+ task: str | None = Field(+ None, description="One of: text_to_video, image_to_video, reference_to_video, edit, extend."+ )++class GeminiInteractionGenerationConfig(BaseModel):temperature: float | None = Field(None, ge=0.0, le=2.0)top_p: float | None = Field(None, ge=0.0, le=1.0)+ video_config: GeminiInteractionVideoConfig | None = Field(None)+++class GeminiInteractionResponseFormat(BaseModel):+ type: Literal["video"] = "video"+ resolution: str | None = Field(None, description="One of: 360p, 720p, 1080p, 4k.")+ aspect_ratio: str | None = Field(None, description="One of: 16:9, 9:16.")+ delivery: str | None = Field(+ None, description="Set to 'uri' to receive a Files API URI instead of inline base64 data."+ )class GeminiInteractionRequest(BaseModel):model: str = Field(...)input: list[GeminiInteractionTextPart | GeminiInteractionMediaPart] = Field(...)generation_config: GeminiInteractionGenerationConfig | None = Field(None)+ response_format: GeminiInteractionResponseFormat | None = Field(None)class GeminiInteractionModalityTokens(BaseModel):@@ -299,3 +316,9 @@ class GeminiInteraction(BaseModel):)steps: list[GeminiInteractionStep] | None = Field(None)usage: GeminiInteractionUsage | None = Field(None)+++class GeminiFile(BaseModel):+ name: str | None = Field(None, description="Resource name of the file, in the form 'files/<id>'.")+ uri: str | None = Field(None)+ state: str | None = Field(None, description="One of: PROCESSING, ACTIVE, FAILED.")diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.pyindex ebad67950..48c42bdda 100644--- a/comfy_api_nodes/nodes_gemini.py+++ b/comfy_api_nodes/nodes_gemini.py@@ -16,6 +16,7 @@ import folder_pathsfrom comfy_api.latest import IO, ComfyExtension, Input, InputImpl, Typesfrom comfy_api_nodes.apis.gemini import (GeminiContent,+ GeminiFile,GeminiFileData,GeminiGenerateContentRequest,GeminiGenerationConfig,@@ -28,7 +29,9 @@ from comfy_api_nodes.apis.gemini import (GeminiInteractionGenerationConfig,GeminiInteractionMediaPart,GeminiInteractionRequest,+ GeminiInteractionResponseFormat,GeminiInteractionTextPart,+ GeminiInteractionVideoConfig,GeminiMimeType,GeminiPart,GeminiRole,@@ -44,7 +47,9 @@ from comfy_api_nodes.util import (download_url_to_video_output,get_number_of_images,pad_images_to_common_channels,+ poll_op,sync_op,+ sync_op_raw,tensor_to_base64_string,upload_audio_to_comfyapi,upload_image_to_comfyapi,@@ -262,7 +267,7 @@ async def get_video_from_interaction(if content.data:return InputImpl.VideoFromFile(BytesIO(base64.b64decode(content.data)))if content.uri:- return await download_url_to_video_output(content.uri, cls=cls)+ return await download_interaction_video(content.uri, cls=cls)model_message = get_text_from_interaction(interaction).strip()if model_message:raise ValueError(f"Gemini did not generate a video. Model response: {model_message}")@@ -272,6 +277,30 @@ async def get_video_from_interaction()+async def download_interaction_video(uri: str, cls: type[IO.ComfyNode] | None = None) -> InputImpl.VideoFromFile:+ if "/files/" not in uri:+ return await download_url_to_video_output(uri, cls=cls)+ name = uri.split("?", 1)[0].rsplit("/files/", 1)[-1].split(":", 1)[0]+ await poll_op(+ cls,+ ApiEndpoint(path=f"{GEMINI_INTERACTIONS_ENDPOINT}/files/{name}"),+ response_model=GeminiFile,+ status_extractor=lambda file: file.state,+ completed_statuses=["ACTIVE"],+ failed_statuses=["FAILED"],+ queued_statuses=["PROCESSING"],+ poll_interval=3.0,+ max_poll_attempts=200,+ )+ video_bytes = await sync_op_raw(+ cls,+ ApiEndpoint(path=f"{GEMINI_INTERACTIONS_ENDPOINT}/files/{name}:download", query_params={"alt": "media"}),+ as_binary=True,+ wait_label="Downloading video",+ )+ return InputImpl.VideoFromFile(BytesIO(video_bytes))++def create_video_parts(video_input: Input.Video) -> list[GeminiPart]:"""Convert a single video input to Gemini API compatible parts (inline MP4/H.264)."""base_64_string = video_to_base64_string(@@ -1553,9 +1582,11 @@ class GeminiNanoBanana2V2(IO.ComfyNode):OMNI_MAX_IMAGES = 14OMNI_MAX_VIDEOS = 3+OMNI_URI_DELIVERY_RESOLUTIONS = ("1080p", "4k")OMNI_MODELS: dict[str, str] = {"Omni Flash": "gemini-omni-flash-preview",+ "Omni Flash 1.1": "gemini-omni-1.1-flash",}@@ -1650,8 +1681,9 @@ class GeminiVideoOmni(IO.ComfyNode):IO.Hidden.unique_id,],is_api_node=True,+ is_deprecated=True,price_badge=IO.PriceBadge(- expr='{"type":"usd","usd":0.101,"format":{"suffix":"/second","approximate":true}}'+ expr='{"type":"usd","usd":0.1449,"format":{"suffix":"/second","approximate":true}}'),)@@ -1707,6 +1739,266 @@ class GeminiVideoOmni(IO.ComfyNode):)+def _omni_task_input(with_extend: bool) -> Input:+ return IO.Combo.Input(+ "task_type",+ options=["auto", "text_to_video", "image_to_video", "reference_to_video", "edit"]+ + (["extend"] if with_extend else []),+ default="auto",+ tooltip="What to do with the prompt and the attached media. With 'auto' the model decides. "+ "'text_to_video' generates from the prompt alone and rejects attached media. 'image_to_video' "+ "animates one image, or interpolates from a starting frame to an ending frame when two are "+ "attached. 'reference_to_video' treats the attached media as subject references. "+ "'edit' rewrites exactly one attached video"+ + (", and 'extend' appends new footage to it, so the output starts with the input video." if with_extend else "."),+ )+++def _omni_seed_input() -> Input:+ return IO.Int.Input(+ "seed",+ default=42,+ min=0,+ max=2147483647,+ control_after_generate=True,+ tooltip="Seed controls whether the node should re-run; results are non-deterministic regardless of seed.",+ )+++def _omni_v2_flash_inputs() -> list[Input]:+ return [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Describe the video to generate, or the edit to apply to an attached video. Specify the "+ 'length directly in the prompt, e.g. "a 6-second clip"; length may be 3-10 seconds. '+ "The output is 720p, 24 FPS, with audio.",+ ),+ IO.Combo.Input(+ "aspect_ratio",+ options=["16:9", "9:16"],+ default="16:9",+ tooltip="Output aspect ratio: 16:9 (landscape) or 9:16 (portrait). "+ "The 'edit' task keeps the aspect ratio of the input video instead.",+ ),+ _omni_task_input(with_extend=False),+ IO.Autogrow.Input(+ "images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("image"),+ names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)],+ min=0,+ ),+ tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images.",+ ),+ IO.Autogrow.Input(+ "videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("video"),+ names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)],+ min=0,+ ),+ tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, "+ f"each up to 10 seconds long.",+ ),+ IO.Float.Input(+ "temperature",+ default=1.0,+ min=0.0,+ max=2.0,+ step=0.01,+ tooltip="Controls randomness. Lower is more focused/deterministic, higher is more varied.",+ advanced=True,+ ),+ IO.Float.Input(+ "top_p",+ default=0.95,+ min=0.0,+ max=1.0,+ step=0.01,+ tooltip="Nucleus sampling: sample from the smallest token set whose cumulative probability reaches top_p.",+ advanced=True,+ ),+ _omni_seed_input(),+ ]+++def _omni_v2_flash_1_1_inputs() -> list[Input]:+ return [+ IO.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Describe the video to generate, or the edit to apply to an attached video. Specify the "+ 'length directly in the prompt, e.g. "a 6-second clip" or, for the \'extend\' task, "extend by 5 seconds"; '+ "the generated length may be 3-10 seconds and defaults to 10. The output has audio.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["360p", "720p", "1080p", "4k"],+ default="720p",+ tooltip="Output resolution.",+ ),+ IO.Combo.Input(+ "aspect_ratio",+ options=["16:9", "9:16"],+ default="16:9",+ tooltip="Output aspect ratio: 16:9 (landscape) or 9:16 (portrait). "+ "The 'edit' and 'extend' tasks keep the aspect ratio of the input video instead.",+ ),+ _omni_task_input(with_extend=True),+ IO.Autogrow.Input(+ "images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("image"),+ names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)],+ min=0,+ ),+ tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images; "+ "with the 'image_to_video' task the first one is the starting frame and an optional second one "+ "is the ending frame.",+ ),+ IO.Autogrow.Input(+ "videos",+ template=IO.Autogrow.TemplateNames(+ IO.Video.Input("video"),+ names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)],+ min=0,+ ),+ tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, "+ f"each up to 10 seconds long.",+ ),+ _omni_seed_input(),+ ]++++class GeminiVideoOmniV2(IO.ComfyNode):++ @classmethod+ def define_schema(cls):+ return IO.Schema(+ node_id="GeminiVideoOmniV2",+ display_name="Google Gemini Omni (Video)",+ category="partner/video/Gemini",+ essentials_category="Video Generation",+ description="Generate a video with audio from a text prompt using Google's Gemini Omni Flash models. "+ "Optionally provide reference images and/or videos to guide or edit the result. Describe the desired "+ "length (3-10s) directly in the prompt.",+ inputs=[+ IO.DynamicCombo.Input(+ "model",+ options=[+ IO.DynamicCombo.Option("Omni Flash 1.1", _omni_v2_flash_1_1_inputs()),+ IO.DynamicCombo.Option("Omni Flash", _omni_v2_flash_inputs()),+ ],+ tooltip="The Gemini video model used to generate the video.",+ ),+ ],+ outputs=[+ IO.Video.Output(),+ IO.String.Output(),+ ],+ hidden=[+ IO.Hidden.auth_token_comfy_org,+ IO.Hidden.api_key_comfy_org,+ IO.Hidden.unique_id,+ ],+ is_api_node=True,+ price_badge=IO.PriceBadge(+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution"]),+ expr="""+ (+ $prices := {"360p": 0.0483, "720p": 0.1449, "1080p": 0.2174, "4k": 0.4349};+ $r := $lookup(widgets, "model.resolution");+ {"type":"usd","usd": $r ? $lookup($prices, $r) : 0.1449,+ "format":{"suffix":"/second","approximate":true}}+ )+ """,+ ),+ )++ @classmethod+ async def execute(cls, model: dict) -> IO.NodeOutput:+ prompt = model.get("prompt") or ""+ validate_string(prompt, strip_whitespace=True, min_length=1)+ model_id = OMNI_MODELS[model["model"]]+ task = model["task_type"]++ images = [t for t in (model.get("images") or {}).values() if t is not None]+ videos = [v for v in (model.get("videos") or {}).values() if v is not None]+ total_images = sum(get_number_of_images(t) for t in images)+ if total_images > OMNI_MAX_IMAGES:+ raise ValueError(f"The current maximum number of supported images is {OMNI_MAX_IMAGES}.")+ if len(videos) > OMNI_MAX_VIDEOS:+ raise ValueError(f"The current maximum number of supported videos is {OMNI_MAX_VIDEOS}.")+ for video in videos:+ validate_video_duration(video, max_duration=10.1)+ if task == "text_to_video" and (images or videos):+ raise ValueError("The 'text_to_video' task generates from the prompt alone; detach the reference media.")+ if task == "image_to_video" and videos:+ raise ValueError(+ "The 'image_to_video' task takes images only; detach the video(s) or use 'reference_to_video'."+ )+ if task == "image_to_video" and not 1 <= total_images <= 2:+ raise ValueError(+ "The 'image_to_video' task takes one image as the starting frame, "+ "and an optional second one as the ending frame."+ )+ if task in ("edit", "extend") and len(videos) != 1:+ raise ValueError(f"The '{task}' task requires exactly one input video.")++ parts: list[GeminiInteractionTextPart | GeminiInteractionMediaPart] = []+ if images or videos:+ # The Interactions API accepts video only inline or as a Files API URI, not as an HTTP URL.+ media_parts = await build_gemini_media_parts(+ cls, [], [], videos, url_budget=0, max_inline_bytes=GEMINI_INTERACTIONS_MAX_INLINE_BYTES+ )+ video_inline_bytes = sum(len(p.inlineData.data) for p in media_parts)+ media_parts += await build_gemini_media_parts(+ cls, images, [], [], max_inline_bytes=GEMINI_INTERACTIONS_MAX_INLINE_BYTES - video_inline_bytes+ )+ parts.extend(to_interaction_media_part(p) for p in media_parts)+ parts.append(GeminiInteractionTextPart(text=prompt))++ resolution = model.get("resolution")+ response_format = GeminiInteractionResponseFormat(+ resolution=resolution,+ aspect_ratio=None if task in ("edit", "extend") else model["aspect_ratio"],+ delivery="uri" if resolution in OMNI_URI_DELIVERY_RESOLUTIONS else None,+ )+ generation_config = None+ if task != "auto" or "temperature" in model:+ generation_config = GeminiInteractionGenerationConfig(+ temperature=model.get("temperature"),+ top_p=model.get("top_p"),+ video_config=GeminiInteractionVideoConfig(task=task) if task != "auto" else None,+ )++ interaction = await sync_op(+ cls,+ ApiEndpoint(path=GEMINI_INTERACTIONS_ENDPOINT, method="POST"),+ data=GeminiInteractionRequest(+ model=model_id,+ input=parts,+ generation_config=generation_config,+ response_format=response_format,+ ),+ response_model=GeminiInteraction,+ )+ if interaction.status != "completed":+ model_message = get_text_from_interaction(interaction).strip()+ raise ValueError(+ f"Gemini interaction did not complete (status: {interaction.status})."+ + (f" Model response: {model_message}" if model_message else "")+ )+ return IO.NodeOutput(+ await get_video_from_interaction(interaction, cls=cls),+ get_text_from_interaction(interaction),+ )++class GeminiExtension(ComfyExtension):@overrideasync def get_node_list(self) -> list[type[IO.ComfyNode]]:@@ -1718,6 +2010,7 @@ class GeminiExtension(ComfyExtension):GeminiNanoBanana2,GeminiNanoBanana2V2,GeminiVideoOmni,+ GeminiVideoOmniV2,GeminiInputFiles,]