The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat(api-nodes): add Sora2 API node (#10249)
Alexander Piskun9e984c48bc6a
diff --git a/comfy_api_nodes/apinode_utils.py b/comfy_api_nodes/apinode_utils.pyindex 5ac3b92aa..571c74286 100644--- a/comfy_api_nodes/apinode_utils.py+++ b/comfy_api_nodes/apinode_utils.py@@ -18,7 +18,7 @@ from comfy_api_nodes.apis.client import (UploadResponse,)from server import PromptServer-+from comfy.cli_args import argsimport numpy as npfrom PIL import Image@@ -30,7 +30,9 @@ from io import BytesIOimport av-async def download_url_to_video_output(video_url: str, timeout: int = None) -> VideoFromFile:+async def download_url_to_video_output(+ video_url: str, timeout: int = None, auth_kwargs: Optional[dict[str, str]] = None+) -> VideoFromFile:"""Downloads a video from a URL and returns a `VIDEO` output.Args:@@ -39,7 +41,7 @@ async def download_url_to_video_output(video_url: str, timeout: int = None) -> VReturns:A Comfy node `VIDEO` output."""- video_io = await download_url_to_bytesio(video_url, timeout)+ video_io = await download_url_to_bytesio(video_url, timeout, auth_kwargs=auth_kwargs)if video_io is None:error_msg = f"Failed to download video from {video_url}"logging.error(error_msg)@@ -164,7 +166,9 @@ def mimetype_to_extension(mime_type: str) -> str:return mime_type.split("/")[-1].lower()-async def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:+async def download_url_to_bytesio(+ url: str, timeout: int = None, auth_kwargs: Optional[dict[str, str]] = None+) -> BytesIO:"""Downloads content from a URL using requests and returns it as BytesIO.Args:@@ -174,9 +178,18 @@ async def download_url_to_bytesio(url: str, timeout: int = None) -> BytesIO:Returns:BytesIO object containing the downloaded content."""+ headers = {}+ if url.startswith("/proxy/"):+ url = str(args.comfy_api_base).rstrip("/") + url+ auth_token = auth_kwargs.get("auth_token")+ comfy_api_key = auth_kwargs.get("comfy_api_key")+ if auth_token:+ headers["Authorization"] = f"Bearer {auth_token}"+ elif comfy_api_key:+ headers["X-API-KEY"] = comfy_api_keytimeout_cfg = aiohttp.ClientTimeout(total=timeout) if timeout else Noneasync with aiohttp.ClientSession(timeout=timeout_cfg) as session:- async with session.get(url) as resp:+ async with session.get(url, headers=headers) as resp:resp.raise_for_status() # Raises HTTPError for bad responses (4XX or 5XX)return BytesIO(await resp.read())diff --git a/comfy_api_nodes/nodes_sora.py b/comfy_api_nodes/nodes_sora.pynew file mode 100644index 000000000..2d532d637--- /dev/null+++ b/comfy_api_nodes/nodes_sora.py@@ -0,0 +1,175 @@+from typing import Optional+from typing_extensions import override++import torch+from pydantic import BaseModel, Field+from comfy_api.latest import ComfyExtension, io as comfy_io+from comfy_api_nodes.apis.client import (+ ApiEndpoint,+ HttpMethod,+ SynchronousOperation,+ PollingOperation,+ EmptyRequest,+)+from comfy_api_nodes.util.validation_utils import get_number_of_images++from comfy_api_nodes.apinode_utils import (+ download_url_to_video_output,+ tensor_to_bytesio,+)++class Sora2GenerationRequest(BaseModel):+ prompt: str = Field(...)+ model: str = Field(...)+ seconds: str = Field(...)+ size: str = Field(...)+++class Sora2GenerationResponse(BaseModel):+ id: str = Field(...)+ error: Optional[dict] = Field(None)+ status: Optional[str] = Field(None)+++class OpenAIVideoSora2(comfy_io.ComfyNode):+ @classmethod+ def define_schema(cls):+ return comfy_io.Schema(+ node_id="OpenAIVideoSora2",+ display_name="OpenAI Sora - Video",+ category="api node/video/Sora",+ description="OpenAI video and audio generation.",+ inputs=[+ comfy_io.Combo.Input(+ "model",+ options=["sora-2", "sora-2-pro"],+ default="sora-2",+ ),+ comfy_io.String.Input(+ "prompt",+ multiline=True,+ default="",+ tooltip="Guiding text; may be empty if an input image is present.",+ ),+ comfy_io.Combo.Input(+ "size",+ options=[+ "720x1280",+ "1280x720",+ "1024x1792",+ "1792x1024",+ ],+ default="1280x720",+ ),+ comfy_io.Combo.Input(+ "duration",+ options=[4, 8, 12],+ default=8,+ ),+ comfy_io.Image.Input(+ "image",+ optional=True,+ ),+ comfy_io.Int.Input(+ "seed",+ default=0,+ min=0,+ max=2147483647,+ step=1,+ display_mode=comfy_io.NumberDisplay.number,+ control_after_generate=True,+ optional=True,+ tooltip="Seed to determine if node should re-run; "+ "actual results are nondeterministic regardless of seed.",+ ),+ ],+ outputs=[+ comfy_io.Video.Output(),+ ],+ hidden=[+ comfy_io.Hidden.auth_token_comfy_org,+ comfy_io.Hidden.api_key_comfy_org,+ comfy_io.Hidden.unique_id,+ ],+ is_api_node=True,+ )++ @classmethod+ async def execute(+ cls,+ model: str,+ prompt: str,+ size: str = "1280x720",+ duration: int = 8,+ seed: int = 0,+ image: Optional[torch.Tensor] = None,+ ):+ if model == "sora-2" and size not in ("720x1280", "1280x720"):+ raise ValueError("Invalid size for sora-2 model, only 720x1280 and 1280x720 are supported.")+ files_input = None+ if image is not None:+ if get_number_of_images(image) != 1:+ raise ValueError("Currently only one input image is supported.")+ files_input = {"input_reference": ("image.png", tensor_to_bytesio(image), "image/png")}+ auth = {+ "auth_token": cls.hidden.auth_token_comfy_org,+ "comfy_api_key": cls.hidden.api_key_comfy_org,+ }+ payload = Sora2GenerationRequest(+ model=model,+ prompt=prompt,+ seconds=str(duration),+ size=size,+ )+ initial_operation = SynchronousOperation(+ endpoint=ApiEndpoint(+ path="/proxy/openai/v1/videos",+ method=HttpMethod.POST,+ request_model=Sora2GenerationRequest,+ response_model=Sora2GenerationResponse+ ),+ request=payload,+ files=files_input,+ auth_kwargs=auth,+ content_type="multipart/form-data",+ )+ initial_response = await initial_operation.execute()+ if initial_response.error:+ raise Exception(initial_response.error.message)++ model_time_multiplier = 1 if model == "sora-2" else 2+ poll_operation = PollingOperation(+ poll_endpoint=ApiEndpoint(+ path=f"/proxy/openai/v1/videos/{initial_response.id}",+ method=HttpMethod.GET,+ request_model=EmptyRequest,+ response_model=Sora2GenerationResponse+ ),+ completed_statuses=["completed"],+ failed_statuses=["failed"],+ status_extractor=lambda x: x.status,+ auth_kwargs=auth,+ poll_interval=8.0,+ max_poll_attempts=160,+ node_id=cls.hidden.unique_id,+ estimated_duration=45 * (duration / 4) * model_time_multiplier,+ )+ await poll_operation.execute()+ return comfy_io.NodeOutput(+ await download_url_to_video_output(+ f"/proxy/openai/v1/videos/{initial_response.id}/content",+ auth_kwargs=auth,+ )+ )+++class OpenAISoraExtension(ComfyExtension):+ @override+ async def get_node_list(self) -> list[type[comfy_io.ComfyNode]]:+ return [+ OpenAIVideoSora2,+ ]+++async def comfy_entrypoint() -> OpenAISoraExtension:+ return OpenAISoraExtension()diff --git a/nodes.py b/nodes.pyindex 88d712993..2a2a5f2ad 100644--- a/nodes.py+++ b/nodes.py@@ -2357,6 +2357,7 @@ async def init_builtin_api_nodes():"nodes_stability.py","nodes_pika.py","nodes_runway.py",+ "nodes_sora.py","nodes_tripo.py","nodes_moonvalley.py","nodes_rodin.py",