Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat(api-nodes): add Sora2 API node (#10249)

Alexander Piskun9e984c48bc6a
diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.py
index 5ac3b92aa..571c74286 100644
--- a/comfy_api_nodes/apinode_utils.py
+++ b/comfy_api_nodes/apinode_utils.py
@@ -18,7 +18,7 @@ from comfy_api_nodes.apis.client import (
UploadResponse,
)
from server import PromptServer
-
+from comfy.cli_args import args
import numpy as np
from PIL import Image
@@ -30,7 +30,9 @@ from io import BytesIO
import av
-async def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile:
+async def download_url_to_video_output(
+ video_url: str, timeout: int = None, auth_kwargs: Optional[dict[str, str]] = None
+) -> VideoFromFile:
"""Downloads a video from a URL and returns a `VIDEO` output.
Args:
@@ -39,7 +41,7 @@ async def download_url_to_video_output(video_url: str, timeout: int = None) -> V
Returns:
A Comfy node `VIDEO` output.
"""
- video_io = await download_url_to_bytesio(video_url, timeout)
+ video_io = await download_url_to_bytesio(video_url, timeout, auth_kwargs=auth_kwargs)
if video_io is None:
error_msg = f"Failed to download video from {video_url}"
logging.error(error_msg)
@@ -164,7 +166,9 @@ def mimetype_to_extension(mime_type: str) -> str:
return mime_type.split("/")[-1].lower()
-async def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
+async def download_url_to_bytesio(
+ url: str, timeout: int = None, auth_kwargs: Optional[dict[str, str]] = None
+) -> BytesIO:
"""Downloads content from a URL using requests and returns it as BytesIO.
Args:
@@ -174,9 +178,18 @@ async def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:
Returns:
BytesIO object containing the downloaded content.
"""
+ headers = {}
+ if url.startswith("/proxy/"):
+ url = str(args.comfy_api_base).rstrip("/") + url
+ auth_token = auth_kwargs.get("auth_token")
+ comfy_api_key = auth_kwargs.get("comfy_api_key")
+ if auth_token:
+ headers["Authorization"] = f"Bearer {auth_token}"
+ elif comfy_api_key:
+ headers["X-API-KEY"] = comfy_api_key
timeout_cfg = aiohttp.ClientTimeout(total=timeout) if timeout else None
async with aiohttp.ClientSession(timeout=timeout_cfg) as session:
- async with session.get(url) as resp:
+ async with session.get(url, headers=headers) as resp:
resp.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)
return BytesIO(await resp.read())
diff --git a/comfy_api_nodes/nodes_sora.py b/comfy_api_nodes/nodes_sora.py
new file mode 100644
index 000000000..2d532d637
--- /dev/null
+++ b/comfy_api_nodes/nodes_sora.py
@@ -0,0 +1,175 @@
+from typing import Optional
+from typing_extensions import override
+
+import torch
+from pydantic import BaseModel, Field
+from comfy_api.latest import ComfyExtension, io as comfy_io
+from comfy_api_nodes.apis.client import (
+ ApiEndpoint,
+ HttpMethod,
+ SynchronousOperation,
+ PollingOperation,
+ EmptyRequest,
+)
+from comfy_api_nodes.util.validation_utils import get_number_of_images
+
+from comfy_api_nodes.apinode_utils import (
+ download_url_to_video_output,
+ tensor_to_bytesio,
+)
+
+class Sora2GenerationRequest(BaseModel):
+ prompt: str = Field(...)
+ model: str = Field(...)
+ seconds: str = Field(...)
+ size: str = Field(...)
+
+
+class Sora2GenerationResponse(BaseModel):
+ id: str = Field(...)
+ error: Optional[dict] = Field(None)
+ status: Optional[str] = Field(None)
+
+
+class OpenAIVideoSora2(comfy_io.ComfyNode):
+ @classmethod
+ def define_schema(cls):
+ return comfy_io.Schema(
+ node_id="OpenAIVideoSora2",
+ display_name="OpenAI Sora - Video",
+ category="api node/video/Sora",
+ description="OpenAI video and audio generation.",
+ inputs=[
+ comfy_io.Combo.Input(
+ "model",
+ options=["sora-2", "sora-2-pro"],
+ default="sora-2",
+ ),
+ comfy_io.String.Input(
+ "prompt",
+ multiline=True,
+ default="",
+ tooltip="Guiding text; may be empty if an input image is present.",
+ ),
+ comfy_io.Combo.Input(
+ "size",
+ options=[
+ "720x1280",
+ "1280x720",
+ "1024x1792",
+ "1792x1024",
+ ],
+ default="1280x720",
+ ),
+ comfy_io.Combo.Input(
+ "duration",
+ options=[4, 8, 12],
+ default=8,
+ ),
+ comfy_io.Image.Input(
+ "image",
+ optional=True,
+ ),
+ comfy_io.Int.Input(
+ "seed",
+ default=0,
+ min=0,
+ max=2147483647,
+ step=1,
+ display_mode=comfy_io.NumberDisplay.number,
+ control_after_generate=True,
+ optional=True,
+ tooltip="Seed to determine if node should re-run; "
+ "actual results are nondeterministic regardless of seed.",
+ ),
+ ],
+ outputs=[
+ comfy_io.Video.Output(),
+ ],
+ hidden=[
+ comfy_io.Hidden.auth_token_comfy_org,
+ comfy_io.Hidden.api_key_comfy_org,
+ comfy_io.Hidden.unique_id,
+ ],
+ is_api_node=True,
+ )
+
+ @classmethod
+ async def execute(
+ cls,
+ model: str,
+ prompt: str,
+ size: str = "1280x720",
+ duration: int = 8,
+ seed: int = 0,
+ image: Optional[torch.Tensor] = None,
+ ):
+ if model == "sora-2" and size not in ("720x1280", "1280x720"):
+ raise ValueError("Invalid size for sora-2 model, only 720x1280 and 1280x720 are supported.")
+ files_input = None
+ if image is not None:
+ if get_number_of_images(image) != 1:
+ raise ValueError("Currently only one input image is supported.")
+ files_input = {"input_reference": ("image.png", tensor_to_bytesio(image), "image/png")}
+ auth = {
+ "auth_token": cls.hidden.auth_token_comfy_org,
+ "comfy_api_key": cls.hidden.api_key_comfy_org,
+ }
+ payload = Sora2GenerationRequest(
+ model=model,
+ prompt=prompt,
+ seconds=str(duration),
+ size=size,
+ )
+ initial_operation = SynchronousOperation(
+ endpoint=ApiEndpoint(
+ path="/proxy/openai/v1/videos",
+ method=HttpMethod.POST,
+ request_model=Sora2GenerationRequest,
+ response_model=Sora2GenerationResponse
+ ),
+ request=payload,
+ files=files_input,
+ auth_kwargs=auth,
+ content_type="multipart/form-data",
+ )
+ initial_response = await initial_operation.execute()
+ if initial_response.error:
+ raise Exception(initial_response.error.message)
+
+ model_time_multiplier = 1 if model == "sora-2" else 2
+ poll_operation = PollingOperation(
+ poll_endpoint=ApiEndpoint(
+ path=f"/proxy/openai/v1/videos/{initial_response.id}",
+ method=HttpMethod.GET,
+ request_model=EmptyRequest,
+ response_model=Sora2GenerationResponse
+ ),
+ completed_statuses=["completed"],
+ failed_statuses=["failed"],
+ status_extractor=lambda x: x.status,
+ auth_kwargs=auth,
+ poll_interval=8.0,
+ max_poll_attempts=160,
+ node_id=cls.hidden.unique_id,
+ estimated_duration=45 * (duration / 4) * model_time_multiplier,
+ )
+ await poll_operation.execute()
+ return comfy_io.NodeOutput(
+ await download_url_to_video_output(
+ f"/proxy/openai/v1/videos/{initial_response.id}/content",
+ auth_kwargs=auth,
+ )
+ )
+
+
+class OpenAISoraExtension(ComfyExtension):
+ @override
+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:
+ return [
+ OpenAIVideoSora2,
+ ]
+
+
+async def comfy_entrypoint() -> OpenAISoraExtension:
+ return OpenAISoraExtension()
diff --git a/nodes.py b/nodes.py
index 88d712993..2a2a5f2ad 100644
--- a/nodes.py
+++ b/nodes.py
@@ -2357,6 +2357,7 @@ async def init_builtin_api_nodes():
"nodes_stability.py",
"nodes_pika.py",
"nodes_runway.py",
+ "nodes_sora.py",
"nodes_tripo.py",
"nodes_moonvalley.py",
"nodes_rodin.py",