Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

[Partner Nodes] feat(Google-Omni): add support for Omni 1.1 model (#15929)

Alexander Piskun4c6359f2e39d
diff --git a/comfy_api_nodes/apis/gemini.py b/comfy_api_nodes/apis/gemini.py
index ce89928cb..4cc7dd6f9 100644
--- a/comfy_api_nodes/apis/gemini.py
+++ b/comfy_api_nodes/apis/gemini.py
@@ -256,15 +256,32 @@ class GeminiInteractionMediaPart(BaseModel):
mime_type: str | None = Field(None)
+class GeminiInteractionVideoConfig(BaseModel):
+ task: str | None = Field(
+ None, description="One of: text_to_video, image_to_video, reference_to_video, edit, extend."
+ )
+
+
class GeminiInteractionGenerationConfig(BaseModel):
temperature: float | None = Field(None, ge=0.0, le=2.0)
top_p: float | None = Field(None, ge=0.0, le=1.0)
+ video_config: GeminiInteractionVideoConfig | None = Field(None)
+
+
+class GeminiInteractionResponseFormat(BaseModel):
+ type: Literal["video"] = "video"
+ resolution: str | None = Field(None, description="One of: 360p, 720p, 1080p, 4k.")
+ aspect_ratio: str | None = Field(None, description="One of: 16:9, 9:16.")
+ delivery: str | None = Field(
+ None, description="Set to 'uri' to receive a Files API URI instead of inline base64 data."
+ )
class GeminiInteractionRequest(BaseModel):
model: str = Field(...)
input: list[GeminiInteractionTextPart | GeminiInteractionMediaPart] = Field(...)
generation_config: GeminiInteractionGenerationConfig | None = Field(None)
+ response_format: GeminiInteractionResponseFormat | None = Field(None)
class GeminiInteractionModalityTokens(BaseModel):
@@ -299,3 +316,9 @@ class GeminiInteraction(BaseModel):
)
steps: list[GeminiInteractionStep] | None = Field(None)
usage: GeminiInteractionUsage | None = Field(None)
+
+
+class GeminiFile(BaseModel):
+ name: str | None = Field(None, description="Resource name of the file, in the form 'files/<id>'.")
+ uri: str | None = Field(None)
+ state: str | None = Field(None, description="One of: PROCESSING, ACTIVE, FAILED.")
diff --git a/comfy_api_nodes/nodes_gemini.py b/comfy_api_nodes/nodes_gemini.py
index ebad67950..48c42bdda 100644
--- a/comfy_api_nodes/nodes_gemini.py
+++ b/comfy_api_nodes/nodes_gemini.py
@@ -16,6 +16,7 @@ import folder_paths
from comfy_api.latest import IO, ComfyExtension, Input, InputImpl, Types
from comfy_api_nodes.apis.gemini import (
GeminiContent,
+ GeminiFile,
GeminiFileData,
GeminiGenerateContentRequest,
GeminiGenerationConfig,
@@ -28,7 +29,9 @@ from comfy_api_nodes.apis.gemini import (
GeminiInteractionGenerationConfig,
GeminiInteractionMediaPart,
GeminiInteractionRequest,
+ GeminiInteractionResponseFormat,
GeminiInteractionTextPart,
+ GeminiInteractionVideoConfig,
GeminiMimeType,
GeminiPart,
GeminiRole,
@@ -44,7 +47,9 @@ from comfy_api_nodes.util import (
download_url_to_video_output,
get_number_of_images,
pad_images_to_common_channels,
+ poll_op,
sync_op,
+ sync_op_raw,
tensor_to_base64_string,
upload_audio_to_comfyapi,
upload_image_to_comfyapi,
@@ -262,7 +267,7 @@ async def get_video_from_interaction(
if content.data:
return InputImpl.VideoFromFile(BytesIO(base64.b64decode(content.data)))
if content.uri:
- return await download_url_to_video_output(content.uri, cls=cls)
+ return await download_interaction_video(content.uri, cls=cls)
model_message = get_text_from_interaction(interaction).strip()
if model_message:
raise ValueError(f"Gemini did not generate a video. Model response: {model_message}")
@@ -272,6 +277,30 @@ async def get_video_from_interaction(
)
+async def download_interaction_video(uri: str, cls: type[IO.ComfyNode] | None = None) -> InputImpl.VideoFromFile:
+ if "/files/" not in uri:
+ return await download_url_to_video_output(uri, cls=cls)
+ name = uri.split("?", 1)[0].rsplit("/files/", 1)[-1].split(":", 1)[0]
+ await poll_op(
+ cls,
+ ApiEndpoint(path=f"{GEMINI_INTERACTIONS_ENDPOINT}/files/{name}"),
+ response_model=GeminiFile,
+ status_extractor=lambda file: file.state,
+ completed_statuses=["ACTIVE"],
+ failed_statuses=["FAILED"],
+ queued_statuses=["PROCESSING"],
+ poll_interval=3.0,
+ max_poll_attempts=200,
+ )
+ video_bytes = await sync_op_raw(
+ cls,
+ ApiEndpoint(path=f"{GEMINI_INTERACTIONS_ENDPOINT}/files/{name}:download", query_params={"alt": "media"}),
+ as_binary=True,
+ wait_label="Downloading video",
+ )
+ return InputImpl.VideoFromFile(BytesIO(video_bytes))
+
+
def create_video_parts(video_input: Input.Video) -> list[GeminiPart]:
"""Convert a single video input to Gemini API compatible parts (inline MP4/H.264)."""
base_64_string = video_to_base64_string(
@@ -1553,9 +1582,11 @@ class GeminiNanoBanana2V2(IO.ComfyNode):
OMNI_MAX_IMAGES = 14
OMNI_MAX_VIDEOS = 3
+OMNI_URI_DELIVERY_RESOLUTIONS = ("1080p", "4k")
OMNI_MODELS: dict[str, str] = {
"Omni Flash": "gemini-omni-flash-preview",
+ "Omni Flash 1.1": "gemini-omni-1.1-flash",
}
@@ -1650,8 +1681,9 @@ class GeminiVideoOmni(IO.ComfyNode):
IO.Hidden.unique_id,
],
is_api_node=True,
+ is_deprecated=True,
price_badge=IO.PriceBadge(
- expr='{"type":"usd","usd":0.101,"format":{"suffix":"/second","approximate":true}}'
+ expr='{"type":"usd","usd":0.1449,"format":{"suffix":"/second","approximate":true}}'
),
)
@@ -1707,6 +1739,266 @@ class GeminiVideoOmni(IO.ComfyNode):
)
+def _omni_task_input(with_extend: bool) -> Input:
+ return IO.Combo.Input(
+ "task_type",
+ options=["auto", "text_to_video", "image_to_video", "reference_to_video", "edit"]
+ + (["extend"] if with_extend else []),
+ default="auto",
+ tooltip="What to do with the prompt and the attached media. With 'auto' the model decides. "
+ "'text_to_video' generates from the prompt alone and rejects attached media. 'image_to_video' "
+ "animates one image, or interpolates from a starting frame to an ending frame when two are "
+ "attached. 'reference_to_video' treats the attached media as subject references. "
+ "'edit' rewrites exactly one attached video"
+ + (", and 'extend' appends new footage to it, so the output starts with the input video." if with_extend else "."),
+ )
+
+
+def _omni_seed_input() -> Input:
+ return IO.Int.Input(
+ "seed",
+ default=42,
+ min=0,
+ max=2147483647,
+ control_after_generate=True,
+ tooltip="Seed controls whether the node should re-run; results are non-deterministic regardless of seed.",
+ )
+
+
+def _omni_v2_flash_inputs() -> list[Input]:
+ return [
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Describe the video to generate, or the edit to apply to an attached video. Specify the "
+ 'length directly in the prompt, e.g. "a 6-second clip"; length may be 3-10 seconds. '
+ "The output is 720p, 24 FPS, with audio.",
+ ),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=["16:9", "9:16"],
+ default="16:9",
+ tooltip="Output aspect ratio: 16:9 (landscape) or 9:16 (portrait). "
+ "The 'edit' task keeps the aspect ratio of the input video instead.",
+ ),
+ _omni_task_input(with_extend=False),
+ IO.Autogrow.Input(
+ "images",
+ template=IO.Autogrow.TemplateNames(
+ IO.Image.Input("image"),
+ names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)],
+ min=0,
+ ),
+ tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images.",
+ ),
+ IO.Autogrow.Input(
+ "videos",
+ template=IO.Autogrow.TemplateNames(
+ IO.Video.Input("video"),
+ names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)],
+ min=0,
+ ),
+ tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, "
+ f"each up to 10 seconds long.",
+ ),
+ IO.Float.Input(
+ "temperature",
+ default=1.0,
+ min=0.0,
+ max=2.0,
+ step=0.01,
+ tooltip="Controls randomness. Lower is more focused/deterministic, higher is more varied.",
+ advanced=True,
+ ),
+ IO.Float.Input(
+ "top_p",
+ default=0.95,
+ min=0.0,
+ max=1.0,
+ step=0.01,
+ tooltip="Nucleus sampling: sample from the smallest token set whose cumulative probability reaches top_p.",
+ advanced=True,
+ ),
+ _omni_seed_input(),
+ ]
+
+
+def _omni_v2_flash_1_1_inputs() -> list[Input]:
+ return [
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Describe the video to generate, or the edit to apply to an attached video. Specify the "
+ 'length directly in the prompt, e.g. "a 6-second clip" or, for the \'extend\' task, "extend by 5 seconds"; '
+ "the generated length may be 3-10 seconds and defaults to 10. The output has audio.",
+ ),
+ IO.Combo.Input(
+ "resolution",
+ options=["360p", "720p", "1080p", "4k"],
+ default="720p",
+ tooltip="Output resolution.",
+ ),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=["16:9", "9:16"],
+ default="16:9",
+ tooltip="Output aspect ratio: 16:9 (landscape) or 9:16 (portrait). "
+ "The 'edit' and 'extend' tasks keep the aspect ratio of the input video instead.",
+ ),
+ _omni_task_input(with_extend=True),
+ IO.Autogrow.Input(
+ "images",
+ template=IO.Autogrow.TemplateNames(
+ IO.Image.Input("image"),
+ names=[f"image_{i}" for i in range(1, OMNI_MAX_IMAGES + 1)],
+ min=0,
+ ),
+ tooltip=f"Optional reference image(s) to guide or animate the video. Up to {OMNI_MAX_IMAGES} images; "
+ "with the 'image_to_video' task the first one is the starting frame and an optional second one "
+ "is the ending frame.",
+ ),
+ IO.Autogrow.Input(
+ "videos",
+ template=IO.Autogrow.TemplateNames(
+ IO.Video.Input("video"),
+ names=[f"video_{i}" for i in range(1, OMNI_MAX_VIDEOS + 1)],
+ min=0,
+ ),
+ tooltip=f"Optional reference video(s) to guide or edit. Up to {OMNI_MAX_VIDEOS} videos, "
+ f"each up to 10 seconds long.",
+ ),
+ _omni_seed_input(),
+ ]
+
+
+
+class GeminiVideoOmniV2(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="GeminiVideoOmniV2",
+ display_name="Google Gemini Omni (Video)",
+ category="partner/video/Gemini",
+ essentials_category="Video Generation",
+ description="Generate a video with audio from a text prompt using Google's Gemini Omni Flash models. "
+ "Optionally provide reference images and/or videos to guide or edit the result. Describe the desired "
+ "length (3-10s) directly in the prompt.",
+ inputs=[
+ IO.DynamicCombo.Input(
+ "model",
+ options=[
+ IO.DynamicCombo.Option("Omni Flash 1.1", _omni_v2_flash_1_1_inputs()),
+ IO.DynamicCombo.Option("Omni Flash", _omni_v2_flash_inputs()),
+ ],
+ tooltip="The Gemini video model used to generate the video.",
+ ),
+ ],
+ outputs=[
+ IO.Video.Output(),
+ IO.String.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["model", "model.resolution"]),
+ expr="""
+ (
+ $prices := {"360p": 0.0483, "720p": 0.1449, "1080p": 0.2174, "4k": 0.4349};
+ $r := $lookup(widgets, "model.resolution");
+ {"type":"usd","usd": $r ? $lookup($prices, $r) : 0.1449,
+ "format":{"suffix":"/second","approximate":true}}
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(cls, model: dict) -> IO.NodeOutput:
+ prompt = model.get("prompt") or ""
+ validate_string(prompt, strip_whitespace=True, min_length=1)
+ model_id = OMNI_MODELS[model["model"]]
+ task = model["task_type"]
+
+ images = [t for t in (model.get("images") or {}).values() if t is not None]
+ videos = [v for v in (model.get("videos") or {}).values() if v is not None]
+ total_images = sum(get_number_of_images(t) for t in images)
+ if total_images > OMNI_MAX_IMAGES:
+ raise ValueError(f"The current maximum number of supported images is {OMNI_MAX_IMAGES}.")
+ if len(videos) > OMNI_MAX_VIDEOS:
+ raise ValueError(f"The current maximum number of supported videos is {OMNI_MAX_VIDEOS}.")
+ for video in videos:
+ validate_video_duration(video, max_duration=10.1)
+ if task == "text_to_video" and (images or videos):
+ raise ValueError("The 'text_to_video' task generates from the prompt alone; detach the reference media.")
+ if task == "image_to_video" and videos:
+ raise ValueError(
+ "The 'image_to_video' task takes images only; detach the video(s) or use 'reference_to_video'."
+ )
+ if task == "image_to_video" and not 1 <= total_images <= 2:
+ raise ValueError(
+ "The 'image_to_video' task takes one image as the starting frame, "
+ "and an optional second one as the ending frame."
+ )
+ if task in ("edit", "extend") and len(videos) != 1:
+ raise ValueError(f"The '{task}' task requires exactly one input video.")
+
+ parts: list[GeminiInteractionTextPart | GeminiInteractionMediaPart] = []
+ if images or videos:
+ # The Interactions API accepts video only inline or as a Files API URI, not as an HTTP URL.
+ media_parts = await build_gemini_media_parts(
+ cls, [], [], videos, url_budget=0, max_inline_bytes=GEMINI_INTERACTIONS_MAX_INLINE_BYTES
+ )
+ video_inline_bytes = sum(len(p.inlineData.data) for p in media_parts)
+ media_parts += await build_gemini_media_parts(
+ cls, images, [], [], max_inline_bytes=GEMINI_INTERACTIONS_MAX_INLINE_BYTES - video_inline_bytes
+ )
+ parts.extend(to_interaction_media_part(p) for p in media_parts)
+ parts.append(GeminiInteractionTextPart(text=prompt))
+
+ resolution = model.get("resolution")
+ response_format = GeminiInteractionResponseFormat(
+ resolution=resolution,
+ aspect_ratio=None if task in ("edit", "extend") else model["aspect_ratio"],
+ delivery="uri" if resolution in OMNI_URI_DELIVERY_RESOLUTIONS else None,
+ )
+ generation_config = None
+ if task != "auto" or "temperature" in model:
+ generation_config = GeminiInteractionGenerationConfig(
+ temperature=model.get("temperature"),
+ top_p=model.get("top_p"),
+ video_config=GeminiInteractionVideoConfig(task=task) if task != "auto" else None,
+ )
+
+ interaction = await sync_op(
+ cls,
+ ApiEndpoint(path=GEMINI_INTERACTIONS_ENDPOINT, method="POST"),
+ data=GeminiInteractionRequest(
+ model=model_id,
+ input=parts,
+ generation_config=generation_config,
+ response_format=response_format,
+ ),
+ response_model=GeminiInteraction,
+ )
+ if interaction.status != "completed":
+ model_message = get_text_from_interaction(interaction).strip()
+ raise ValueError(
+ f"Gemini interaction did not complete (status: {interaction.status})."
+ + (f" Model response: {model_message}" if model_message else "")
+ )
+ return IO.NodeOutput(
+ await get_video_from_interaction(interaction, cls=cls),
+ get_text_from_interaction(interaction),
+ )
+
+
class GeminiExtension(ComfyExtension):
@override
async def get_node_list(self) -> list[type[IO.ComfyNode]]:
@@ -1718,6 +2010,7 @@ class GeminiExtension(ComfyExtension):
GeminiNanoBanana2,
GeminiNanoBanana2V2,
GeminiVideoOmni,
+ GeminiVideoOmniV2,
GeminiInputFiles,
]