The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
[Partner Nodes] feat(xAI): update nodes for grok-imagine-video-1.5 model (#15197)
Alexander Piskund8e6aa55f377
diff --git a/comfy_api_nodes/apis/grok.py b/comfy_api_nodes/apis/grok.pyindex fbedb53e0..526d8c8ab 100644--- a/comfy_api_nodes/apis/grok.py+++ b/comfy_api_nodes/apis/grok.py@@ -15,6 +15,10 @@ class InputUrlObject(BaseModel):url: str = Field(...)+class VoiceReferenceObject(BaseModel):+ voice_id: str = Field(...)++class ImageEditRequest(BaseModel):model: str = Field(...)images: list[InputUrlObject] = Field(...)@@ -31,6 +35,7 @@ class VideoGenerationRequest(BaseModel):prompt: str = Field(...)image: InputUrlObject | None = Field(None)reference_images: list[InputUrlObject] | None = Field(None)+ reference_audios: list[VoiceReferenceObject] | None = Field(None)duration: int = Field(...)aspect_ratio: str | None = Field(...)resolution: str = Field(...)diff --git a/comfy_api_nodes/nodes_grok.py b/comfy_api_nodes/nodes_grok.pyindex a95b35917..672a3e537 100644--- a/comfy_api_nodes/nodes_grok.py+++ b/comfy_api_nodes/nodes_grok.py@@ -1,3 +1,5 @@+import re+import torchfrom typing_extensions import override@@ -12,6 +14,7 @@ from comfy_api_nodes.apis.grok import (VideoGenerationRequest,VideoGenerationResponse,VideoStatusResponse,+ VoiceReferenceObject,)from comfy_api_nodes.util import (ApiEndpoint,@@ -33,6 +36,75 @@ _GROK_VIDEO_MODEL_API_IDS = {"grok-imagine-video-1.5": "grok-imagine-video-1.5",}+_GROK_VOICE_OPTIONS = [+ "none",+ "ara",+ "eve",+ "leo",+ "rex",+ "sal",+ "carina",+ "zagan",+ "helix",+ "orion",+ "luna",+ "iris",+ "altair",+ "zenith",+ "perseus",+ "helios",+ "lux",+ "kepler",+ "rigel",+ "cosmo",+ "celeste",+ "ursa",+ "sirius",+ "lumen",+ "castor",+ "naksh",+ "atlas",+]+++_GROK_REF_TAG_RE = re.compile(r"(?<!\w)@(image|audio)(?P<idx>\d*)(?!\w)", re.IGNORECASE | re.ASCII)+++def _normalize_grok_reference_prompt(prompt: str, total_images: int, voices: list[str]) -> str:+ """Rewrite @Image1/@Audio1 style references (1-based, shared partner-node syntax)+ into Grok's native <IMAGE_0>/<AUDIO_0> tags; an unnumbered @image/@audio means the first one.+ Native tags pass through untouched. @ImageN refers to the Nth reference image overall, in+ input order — a batched input contributes one number per image. @AudioN refers to the+ 'voice_N' widget; the API only accepts compact arrays, so voices are remapped to array+ positions and 'none' slots between selected voices are harmless. Substitution repeats until+ stable so adjacent tags like '@Image1@Image2' all resolve."""+ audio_indices: dict[int, int] = {}+ for slot, voice in enumerate(voices, start=1):+ if voice != "none":+ audio_indices[slot] = len(audio_indices)++ def repl(match: re.Match) -> str:+ kind = match.group(1).lower()+ idx = int(match.group("idx") or 1)+ if kind == "image":+ if not 1 <= idx <= total_images:+ raise ValueError(+ f"The prompt references @Image{idx}, but only {total_images} "+ f"reference images are connected (a batched input counts once per image)."+ )+ return f"<IMAGE_{idx - 1}>"+ if idx not in audio_indices:+ if 1 <= idx <= len(voices):+ raise ValueError(f"The prompt references @Audio{idx}, but 'voice_{idx}' is set to 'none'.")+ raise ValueError(f"The prompt references @Audio{idx}, but only voices 1..{len(voices)} exist.")+ return f"<AUDIO_{audio_indices[idx]}>"++ prev = None+ while prev != prompt:+ prev = prompt+ prompt = _GROK_REF_TAG_RE.sub(repl, prompt)+ return prompt+def _extract_grok_price(response) -> float | None:if response.usage and response.usage.cost_in_usd_ticks is not None:@@ -509,12 +581,13 @@ class GrokVideoNode(IO.ComfyNode):IO.Combo.Input("model",options=["grok-imagine-video", "grok-imagine-video-1.5"],- tooltip="grok-imagine-video-1.5 currently always requires an input image.",+ tooltip="The model to use for video generation.",),IO.String.Input("prompt",multiline=True,- tooltip="Text description of the desired video.",+ tooltip="Text description of the desired video. "+ "Optional for grok-imagine-video-1.5 when an input image is provided.",),IO.Combo.Input("resolution",@@ -549,7 +622,7 @@ class GrokVideoNode(IO.ComfyNode):IO.Image.Input("image",optional=True,- tooltip="Optional starting image for grok-imagine-video. Required for grok-imagine-video-1.5.",+ tooltip="Optional starting image. If omitted, the video is generated from the text prompt alone.",),],outputs=[@@ -589,8 +662,6 @@ class GrokVideoNode(IO.ComfyNode):seed: int,image: Input.Image | None = None,) -> IO.NodeOutput:- if image is None and model == "grok-imagine-video-1.5":- raise ValueError(f"The '{model}' model requires an input image; connect one to the 'image' input.")if resolution == "1080p" and model != "grok-imagine-video-1.5":raise ValueError(f"1080p resolution is only available for grok-imagine-video-1.5, not '{model}'.")image_url = None@@ -598,7 +669,8 @@ class GrokVideoNode(IO.ComfyNode):if get_number_of_images(image) != 1:raise ValueError("Only one input image is supported.")image_url = InputUrlObject(url=f"data:image/png;base64,{tensor_to_base64_string(image)}")- validate_string(prompt, strip_whitespace=True, min_length=1)+ if image is None or model != "grok-imagine-video-1.5":+ validate_string(prompt, strip_whitespace=True, min_length=1)initial_response = await sync_op(cls,ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"),@@ -709,7 +781,7 @@ class GrokVideoReferenceNode(IO.ComfyNode):node_id="GrokVideoReferenceNode",display_name="Grok Reference-to-Video",category="partner/video/Grok",- description="Generate video guided by reference images as style and content references.",+ description="Generate video guided by reference images, with optional preset voice references.",inputs=[IO.String.Input("prompt",@@ -719,6 +791,57 @@ class GrokVideoReferenceNode(IO.ComfyNode):IO.DynamicCombo.Input("model",options=[+ IO.DynamicCombo.Option(+ "grok-imagine-video-1.5",+ [+ IO.Autogrow.Input(+ "reference_images",+ template=IO.Autogrow.TemplateNames(+ IO.Image.Input("image"),+ names=[f"reference_{i}" for i in range(1, 8)],+ min=1,+ ),+ tooltip="Up to 7 reference images to guide the video generation. "+ "Refer to them in the prompt as @Image1 ... @Image7, numbered "+ "in input order; a batched input counts once per image.",+ ),+ IO.Combo.Input(+ "voice_1",+ options=_GROK_VOICE_OPTIONS,+ tooltip="Optional preset voice reference; refer to it in the prompt as @Audio1. "+ "The API supports only these preset voices, not custom audio.",+ ),+ IO.Combo.Input(+ "voice_2",+ options=_GROK_VOICE_OPTIONS,+ tooltip="Optional second voice reference; @Audio2 in the prompt.",+ ),+ IO.Combo.Input(+ "voice_3",+ options=_GROK_VOICE_OPTIONS,+ tooltip="Optional third voice reference; @Audio3 in the prompt.",+ ),+ IO.Combo.Input(+ "resolution",+ options=["480p", "720p"],+ tooltip="The resolution of the output video.",+ ),+ IO.Combo.Input(+ "aspect_ratio",+ options=["16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"],+ tooltip="The aspect ratio of the output video.",+ ),+ IO.Int.Input(+ "duration",+ default=6,+ min=1,+ max=15,+ step=1,+ tooltip="The duration of the output video in seconds.",+ display_mode=IO.NumberDisplay.slider,+ ),+ ],+ ),IO.DynamicCombo.Option("grok-imagine-video",[@@ -779,16 +902,20 @@ class GrokVideoReferenceNode(IO.ComfyNode):is_api_node=True,price_badge=IO.PriceBadge(depends_on=IO.PriceBadgeDepends(- widgets=["model.duration", "model.resolution"],+ widgets=["model", "model.duration", "model.resolution"],input_groups=["model.reference_images"],),expr="""(+ $is15 := $contains(widgets.model, "1.5");$res := $lookup(widgets, "model.resolution");$dur := $lookup(widgets, "model.duration");$refs := $lookup(inputGroups, "model.reference_images");- $rate := $res = "720p" ? 0.07 : 0.05;- $price := ($rate * $dur + 0.002 * $refs) * 1.43;+ $rate := $is15+ ? ($res = "720p" ? 0.14 : 0.08)+ : ($res = "720p" ? 0.07 : 0.05);+ $imgCost := $is15 ? 0.01 : 0.002;+ $price := ($rate * $dur + $imgCost * $refs) * 1.43;{"type":"usd","usd": $price})""",@@ -803,6 +930,14 @@ class GrokVideoReferenceNode(IO.ComfyNode):seed: int,) -> IO.NodeOutput:validate_string(prompt, strip_whitespace=True, min_length=1)+ total_images = sum(get_number_of_images(t) for t in model["reference_images"].values())+ if total_images > 7:+ raise ValueError(f"A maximum of 7 reference images is supported; {total_images} are connected.")+ reference_audios = None+ if model["model"] == "grok-imagine-video-1.5":+ voices = [model.get(f"voice_{i}", "none") for i in range(1, 4)]+ reference_audios = [VoiceReferenceObject(voice_id=v) for v in voices if v != "none"] or None+ prompt = _normalize_grok_reference_prompt(prompt, total_images=total_images, voices=voices)ref_image_urls = await upload_images_to_comfyapi(cls,list(model["reference_images"].values()),@@ -814,8 +949,9 @@ class GrokVideoReferenceNode(IO.ComfyNode):cls,ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"),data=VideoGenerationRequest(- model=model["model"],+ model=_GROK_VIDEO_MODEL_API_IDS.get(model["model"], model["model"]),reference_images=[InputUrlObject(url=i) for i in ref_image_urls],+ reference_audios=reference_audios,prompt=prompt,resolution=model["resolution"],duration=model["duration"],