Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat(api-nodes): add Grok Imagine nodes (#12136)

Alexander Piskun6c5f906bf244
diff --git a/comfy_api_nodes/apis/grok.py b/comfy_api_nodes/apis/grok.py
new file mode 100644
index 000000000..8e3c79ab9
--- /dev/null
+++ b/comfy_api_nodes/apis/grok.py
@@ -0,0 +1,67 @@
+from pydantic import BaseModel, Field
+
+
+class ImageGenerationRequest(BaseModel):
+ model: str = Field(...)
+ prompt: str = Field(...)
+ aspect_ratio: str = Field(...)
+ n: int = Field(...)
+ seed: int = Field(...)
+ response_for: str = Field("url")
+
+
+class InputUrlObject(BaseModel):
+ url: str = Field(...)
+
+
+class ImageEditRequest(BaseModel):
+ model: str = Field(...)
+ image: InputUrlObject = Field(...)
+ prompt: str = Field(...)
+ resolution: str = Field(...)
+ n: int = Field(...)
+ seed: int = Field(...)
+ response_for: str = Field("url")
+
+
+class VideoGenerationRequest(BaseModel):
+ model: str = Field(...)
+ prompt: str = Field(...)
+ image: InputUrlObject | None = Field(...)
+ duration: int = Field(...)
+ aspect_ratio: str | None = Field(...)
+ resolution: str = Field(...)
+ seed: int = Field(...)
+
+
+class VideoEditRequest(BaseModel):
+ model: str = Field(...)
+ prompt: str = Field(...)
+ video: InputUrlObject = Field(...)
+ seed: int = Field(...)
+
+
+class ImageResponseObject(BaseModel):
+ url: str | None = Field(None)
+ b64_json: str | None = Field(None)
+ revised_prompt: str | None = Field(None)
+
+
+class ImageGenerationResponse(BaseModel):
+ data: list[ImageResponseObject] = Field(...)
+
+
+class VideoGenerationResponse(BaseModel):
+ request_id: str = Field(...)
+
+
+class VideoResponseObject(BaseModel):
+ url: str = Field(...)
+ upsampled_prompt: str | None = Field(None)
+ duration: int = Field(...)
+
+
+class VideoStatusResponse(BaseModel):
+ status: str | None = Field(None)
+ video: VideoResponseObject | None = Field(None)
+ model: str | None = Field(None)
diff --git a/comfy_api_nodes/nodes_grok.py b/comfy_api_nodes/nodes_grok.py
new file mode 100644
index 000000000..da15e97ea
--- /dev/null
+++ b/comfy_api_nodes/nodes_grok.py
@@ -0,0 +1,417 @@
+import torch
+from typing_extensions import override
+
+from comfy_api.latest import IO, ComfyExtension, Input
+from comfy_api_nodes.apis.grok import (
+ ImageEditRequest,
+ ImageGenerationRequest,
+ ImageGenerationResponse,
+ InputUrlObject,
+ VideoEditRequest,
+ VideoGenerationRequest,
+ VideoGenerationResponse,
+ VideoStatusResponse,
+)
+from comfy_api_nodes.util import (
+ ApiEndpoint,
+ download_url_to_image_tensor,
+ download_url_to_video_output,
+ get_fs_object_size,
+ get_number_of_images,
+ poll_op,
+ sync_op,
+ tensor_to_base64_string,
+ upload_video_to_comfyapi,
+ validate_string,
+ validate_video_duration,
+)
+
+
+class GrokImageNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="GrokImageNode",
+ display_name="Grok Image",
+ category="api node/image/Grok",
+ description="Generate images using Grok based on a text prompt",
+ inputs=[
+ IO.Combo.Input("model", options=["grok-imagine-image-beta"]),
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="The text prompt used to generate the image",
+ ),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=[
+ "1:1",
+ "2:3",
+ "3:2",
+ "3:4",
+ "4:3",
+ "9:16",
+ "16:9",
+ "9:19.5",
+ "19.5:9",
+ "9:20",
+ "20:9",
+ "1:2",
+ "2:1",
+ ],
+ ),
+ IO.Int.Input(
+ "number_of_images",
+ default=1,
+ min=1,
+ max=10,
+ step=1,
+ tooltip="Number of images to generate",
+ display_mode=IO.NumberDisplay.number,
+ ),
+ IO.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed to determine if node should re-run; "
+ "actual results are nondeterministic regardless of seed.",
+ ),
+ ],
+ outputs=[
+ IO.Image.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["number_of_images"]),
+ expr="""{"type":"usd","usd":0.033 * widgets.number_of_images}""",
+ ),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ aspect_ratio: str,
+ number_of_images: int,
+ seed: int,
+ ) -> IO.NodeOutput:
+ validate_string(prompt, strip_whitespace=True, min_length=1)
+ response = await sync_op(
+ cls,
+ ApiEndpoint(path="/proxy/xai/v1/images/generations", method="POST"),
+ data=ImageGenerationRequest(
+ model=model,
+ prompt=prompt,
+ aspect_ratio=aspect_ratio,
+ n=number_of_images,
+ seed=seed,
+ ),
+ response_model=ImageGenerationResponse,
+ )
+ if len(response.data) == 1:
+ return IO.NodeOutput(await download_url_to_image_tensor(response.data[0].url))
+ return IO.NodeOutput(
+ torch.cat(
+ [await download_url_to_image_tensor(i) for i in [str(d.url) for d in response.data if d.url]],
+ )
+ )
+
+
+class GrokImageEditNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="GrokImageEditNode",
+ display_name="Grok Image Edit",
+ category="api node/image/Grok",
+ description="Modify an existing image based on a text prompt",
+ inputs=[
+ IO.Combo.Input("model", options=["grok-imagine-image-beta"]),
+ IO.Image.Input("image"),
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="The text prompt used to generate the image",
+ ),
+ IO.Combo.Input("resolution", options=["1K"]),
+ IO.Int.Input(
+ "number_of_images",
+ default=1,
+ min=1,
+ max=10,
+ step=1,
+ tooltip="Number of edited images to generate",
+ display_mode=IO.NumberDisplay.number,
+ ),
+ IO.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed to determine if node should re-run; "
+ "actual results are nondeterministic regardless of seed.",
+ ),
+ ],
+ outputs=[
+ IO.Image.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["number_of_images"]),
+ expr="""{"type":"usd","usd":0.002 + 0.033 * widgets.number_of_images}""",
+ ),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ image: Input.Image,
+ prompt: str,
+ resolution: str,
+ number_of_images: int,
+ seed: int,
+ ) -> IO.NodeOutput:
+ validate_string(prompt, strip_whitespace=True, min_length=1)
+ if get_number_of_images(image) != 1:
+ raise ValueError("Only one input image is supported.")
+ response = await sync_op(
+ cls,
+ ApiEndpoint(path="/proxy/xai/v1/images/edits", method="POST"),
+ data=ImageEditRequest(
+ model=model,
+ image=InputUrlObject(url=f"data:image/png;base64,{tensor_to_base64_string(image)}"),
+ prompt=prompt,
+ resolution=resolution.lower(),
+ n=number_of_images,
+ seed=seed,
+ ),
+ response_model=ImageGenerationResponse,
+ )
+ if len(response.data) == 1:
+ return IO.NodeOutput(await download_url_to_image_tensor(response.data[0].url))
+ return IO.NodeOutput(
+ torch.cat(
+ [await download_url_to_image_tensor(i) for i in [str(d.url) for d in response.data if d.url]],
+ )
+ )
+
+
+class GrokVideoNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="GrokVideoNode",
+ display_name="Grok Video",
+ category="api node/video/Grok",
+ description="Generate video from a prompt or an image",
+ inputs=[
+ IO.Combo.Input("model", options=["grok-imagine-video-beta"]),
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="Text description of the desired video.",
+ ),
+ IO.Combo.Input(
+ "resolution",
+ options=["480p", "720p"],
+ tooltip="The resolution of the output video.",
+ ),
+ IO.Combo.Input(
+ "aspect_ratio",
+ options=["auto", "16:9", "4:3", "3:2", "1:1", "2:3", "3:4", "9:16"],
+ tooltip="The aspect ratio of the output video.",
+ ),
+ IO.Int.Input(
+ "duration",
+ default=6,
+ min=1,
+ max=15,
+ step=1,
+ tooltip="The duration of the output video in seconds.",
+ display_mode=IO.NumberDisplay.slider,
+ ),
+ IO.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed to determine if node should re-run; "
+ "actual results are nondeterministic regardless of seed.",
+ ),
+ IO.Image.Input("image", optional=True),
+ ],
+ outputs=[
+ IO.Video.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ depends_on=IO.PriceBadgeDepends(widgets=["duration"], inputs=["image"]),
+ expr="""
+ (
+ $base := 0.181 * widgets.duration;
+ {"type":"usd","usd": inputs.image.connected ? $base + 0.002 : $base}
+ )
+ """,
+ ),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ resolution: str,
+ aspect_ratio: str,
+ duration: int,
+ seed: int,
+ image: Input.Image | None = None,
+ ) -> IO.NodeOutput:
+ image_url = None
+ if image is not None:
+ if get_number_of_images(image) != 1:
+ raise ValueError("Only one input image is supported.")
+ image_url = InputUrlObject(url=f"data:image/png;base64,{tensor_to_base64_string(image)}")
+ validate_string(prompt, strip_whitespace=True, min_length=1)
+ initial_response = await sync_op(
+ cls,
+ ApiEndpoint(path="/proxy/xai/v1/videos/generations", method="POST"),
+ data=VideoGenerationRequest(
+ model=model,
+ image=image_url,
+ prompt=prompt,
+ resolution=resolution,
+ duration=duration,
+ aspect_ratio=None if aspect_ratio == "auto" else aspect_ratio,
+ seed=seed,
+ ),
+ response_model=VideoGenerationResponse,
+ )
+ response = await poll_op(
+ cls,
+ ApiEndpoint(path=f"/proxy/xai/v1/videos/{initial_response.request_id}"),
+ status_extractor=lambda r: r.status if r.status is not None else "complete",
+ response_model=VideoStatusResponse,
+ )
+ return IO.NodeOutput(await download_url_to_video_output(response.video.url))
+
+
+class GrokVideoEditNode(IO.ComfyNode):
+
+ @classmethod
+ def define_schema(cls):
+ return IO.Schema(
+ node_id="GrokVideoEditNode",
+ display_name="Grok Video Edit",
+ category="api node/video/Grok",
+ description="Edit an existing video based on a text prompt.",
+ inputs=[
+ IO.Combo.Input("model", options=["grok-imagine-video-beta"]),
+ IO.String.Input(
+ "prompt",
+ multiline=True,
+ tooltip="Text description of the desired video.",
+ ),
+ IO.Video.Input("video", tooltip="Maximum supported duration is 8.7 seconds and 50MB file size."),
+ IO.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=IO.NumberDisplay.number,
+ control_after_generate=True,
+ tooltip="Seed to determine if node should re-run; "
+ "actual results are nondeterministic regardless of seed.",
+ ),
+ ],
+ outputs=[
+ IO.Video.Output(),
+ ],
+ hidden=[
+ IO.Hidden.auth_token_comfy_org,
+ IO.Hidden.api_key_comfy_org,
+ IO.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ price_badge=IO.PriceBadge(
+ expr="""{"type":"usd","usd": 0.191, "format": {"suffix": "/sec", "approximate": true}}""",
+ ),
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ video: Input.Video,
+ seed: int,
+ ) -> IO.NodeOutput:
+ validate_string(prompt, strip_whitespace=True, min_length=1)
+ validate_video_duration(video, min_duration=1, max_duration=8.7)
+ video_stream = video.get_stream_source()
+ video_size = get_fs_object_size(video_stream)
+ if video_size > 50 * 1024 * 1024:
+ raise ValueError(f"Video size ({video_size / 1024 / 1024:.1f}MB) exceeds 50MB limit.")
+ initial_response = await sync_op(
+ cls,
+ ApiEndpoint(path="/proxy/xai/v1/videos/edits", method="POST"),
+ data=VideoEditRequest(
+ model=model,
+ video=InputUrlObject(url=await upload_video_to_comfyapi(cls, video)),
+ prompt=prompt,
+ seed=seed,
+ ),
+ response_model=VideoGenerationResponse,
+ )
+ response = await poll_op(
+ cls,
+ ApiEndpoint(path=f"/proxy/xai/v1/videos/{initial_response.request_id}"),
+ status_extractor=lambda r: r.status if r.status is not None else "complete",
+ response_model=VideoStatusResponse,
+ )
+ return IO.NodeOutput(await download_url_to_video_output(response.video.url))
+
+
+class GrokExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[IO.ComfyNode]]:
+ return [
+ GrokImageNode,
+ GrokImageEditNode,
+ GrokVideoNode,
+ GrokVideoEditNode,
+ ]
+
+
+async def comfy_entrypoint() -> GrokExtension:
+ return GrokExtension()