World's first open-source, agentic video production system. 12 production pipelines, 100+ tools, 700+ agent skill and production-knowledge files. Turn your AI coding assistant into a full video production studio.
Python agentagentic-aiaiclaudecopilotcursorelevenlabsffmpegfluximage-generationopen-sourceopenaipythonremotionstable-diffusiontext-to-speechtext-to-videovideo-generationvideo-production
"""Thin REST client for a running ComfyUI server.
Handles the full generation cycle: submit workflow, poll for completion,
download artifacts. Used by comfyui_image, comfyui_video, and comfyui_music.
"""
from __future__ import annotations
import copy
import json
import os
import random
import time
import uuid
from pathlib import Path
from typing import Any, Callable
import requests
class ComfyUIError(Exception):
"""Raised when ComfyUI returns an error or times out.
``prompt_id`` is set when the error follows a successful ``submit()``,
so callers can recover a timed-out-but-still-running job instead of
losing track of it: poll ``GET /history/{prompt_id}`` directly, or
pass ``resume_prompt_id`` back into ``ComfyUIVideo.execute()``.
"""
def __init__(self, message: str, prompt_id: str | None = None) -> None:
super().__init__(message)
self.prompt_id = prompt_id
class ComfyUIClient:
"""Client for the ComfyUI REST API.
The protocol is simple and battle-tested:
1. POST /prompt → queue a workflow, get a prompt_id
2. GET /history/{id} → poll until outputs appear
3. GET /view?filename=… → download the generated artifact
4. POST /upload/image → stage a local image for I2V workflows
"""
def __init__(
self, server_url: str | None = None, capability: str | None = None
) -> None:
"""*capability*, if given (e.g. ``"image"``, ``"video"``), lets a
per-capability env var (``COMFYUI_{CAPABILITY}_SERVER_URL``) point
this client at its own ComfyUI instance -- useful when image and
video generation are split across separate servers/GPUs. Falls back
to the shared ``COMFYUI_SERVER_URL`` when the capability-specific
var isn't set, so single-server setups need no extra configuration.
"""
self.capability = capability
self._capability_env_var = (
f"COMFYUI_{capability.upper()}_SERVER_URL" if capability else None
)
resolved = (
server_url or self._capability_url() or os.environ.get("COMFYUI_SERVER_URL")
)
self.server_url = (resolved or "http://localhost:8188").rstrip("/")
# Scopes websocket execution events to this client (see wait_ws) and
# is echoed back on /prompt so the server targets messages to us.
self.client_id = str(uuid.uuid4())
def _capability_url(self) -> str | None:
if self._capability_env_var:
return os.environ.get(self._capability_env_var)
return None
# ------------------------------------------------------------------
# Health
# ------------------------------------------------------------------
@property
def is_default_url(self) -> bool:
"""True if neither the capability-specific nor shared env var is set."""
return not (self._capability_url() or os.environ.get("COMFYUI_SERVER_URL"))
def unavailable_reason(self) -> str:
"""Human-readable explanation of why the server can't be reached."""
env_var_hint = self._capability_env_var or "COMFYUI_SERVER_URL"
if self._capability_env_var:
env_var_hint += " (or the shared COMFYUI_SERVER_URL)"
if self.is_default_url:
return (
f"No ComfyUI server found at {self.server_url} "
f"(default — no server URL configured).\n"
f"Set {env_var_hint} in your .env file to the address of "
f"your ComfyUI server (e.g. http://localhost:8188)."
)
return (
f"ComfyUI server not reachable at {self.server_url}.\n"
f"Check that ComfyUI is running and the URL is correct."
)
def is_available(self) -> bool:
"""Return True if the ComfyUI server is reachable."""
try:
resp = requests.get(f"{self.server_url}/system_stats", timeout=5)
return resp.status_code == 200
except Exception:
return False
# ------------------------------------------------------------------
# Model discovery
# ------------------------------------------------------------------
def list_models(self) -> dict[str, list[str]]:
"""Query ComfyUI for available models, grouped by type.
Returns a dict like::
{
"checkpoints": ["sd_xl_base.safetensors", ...],
"diffusion_models": ["flux2-dev-nvfp4.safetensors", ...],
"vae": ["ae.safetensors", ...],
"clip": ["clip_l.safetensors", ...],
"loras": ["my_lora.safetensors", ...],
}
"""
node_to_key = {
"CheckpointLoaderSimple": ("ckpt_name", "checkpoints"),
"UNETLoader": ("unet_name", "diffusion_models"),
"VAELoader": ("vae_name", "vae"),
"CLIPLoader": ("clip_name", "clip"),
"LoraLoaderModelOnly": ("lora_name", "loras"),
}
result: dict[str, list[str]] = {}
for node_class, (field, group) in node_to_key.items():
try:
resp = requests.get(
f"{self.server_url}/object_info/{node_class}", timeout=10
)
resp.raise_for_status()
data = resp.json()
options = (
data.get(node_class, {})
.get("input", {})
.get("required", {})
.get(field, [[]])[0]
)
if isinstance(options, list):
result[group] = options
except Exception:
result[group] = []
return result
def check_models(self, required: list[str]) -> tuple[list[str], list[str]]:
"""Check which of *required* model filenames are available.
Returns ``(found, missing)`` — two lists of filenames.
"""
all_models: set[str] = set()
for names in self.list_models().values():
all_models.update(names)
found = [m for m in required if m in all_models]
missing = [m for m in required if m not in all_models]
return found, missing
def has_node(self, node_class: str) -> bool:
"""Return whether the connected server exposes a node class."""
try:
response = requests.get(
f"{self.server_url}/object_info/{node_class}", timeout=10
)
response.raise_for_status()
return node_class in response.json()
except Exception:
return False
# ------------------------------------------------------------------
# Core cycle
# ------------------------------------------------------------------
def submit(self, workflow: dict) -> str:
"""Queue a workflow for execution. Returns the ``prompt_id``."""
resp = requests.post(
f"{self.server_url}/prompt",
json={"prompt": workflow, "client_id": self.client_id},
timeout=30,
)
try:
data = resp.json()
except ValueError:
data = {}
if data.get("node_errors"):
raise ComfyUIError(f"Node errors: {json.dumps(data['node_errors'])}")
if data.get("error"):
raise ComfyUIError(f"Prompt error: {json.dumps(data['error'])}")
resp.raise_for_status()
prompt_id = data.get("prompt_id")
if not prompt_id:
raise ComfyUIError(f"No prompt_id in response: {data}")
return prompt_id
def poll(
self,
prompt_id: str,
*,
timeout: int = 600,
interval: int = 5,
) -> dict:
"""Block until *prompt_id* finishes. Returns the history entry."""
deadline = time.time() + timeout
while time.time() < deadline:
entry = self._history_entry(prompt_id)
if entry is not None:
return entry
time.sleep(interval)
raise ComfyUIError(
f"Prompt {prompt_id} did not complete within {timeout}s. "
f"The job is very likely still running on the ComfyUI server "
f"(local/custom workflows on modest GPUs routinely exceed the "
f"client wait) — it was not cancelled. Poll "
f"GET {{server_url}}/history/{prompt_id} directly, or call "
f"generate()/execute() again with a longer timeout and this "
f"prompt_id to resume waiting without resubmitting.",
prompt_id=prompt_id,
)
def _history_entry(self, prompt_id: str) -> dict | None:
"""Return a completed history entry, or ``None`` while it is absent."""
resp = requests.get(f"{self.server_url}/history/{prompt_id}", timeout=10)
resp.raise_for_status()
entry = resp.json().get(prompt_id)
if entry is None:
return None
status = entry.get("status", {})
if status.get("status_str") == "error":
msgs = status.get("messages", [])
raise ComfyUIError(f"Execution error: {msgs}", prompt_id=prompt_id)
return entry
def _history_entry_if_reachable(self, prompt_id: str) -> dict | None:
"""Best-effort history probe while the websocket remains usable."""
try:
return self._history_entry(prompt_id)
except ComfyUIError:
raise
except Exception:
return None
def wait_ws(
self,
prompt_id: str,
*,
timeout: int = 600,
interval: int = 5,
on_progress: Callable[[dict], None] | None = None,
) -> dict:
"""Block until *prompt_id* finishes, watching ComfyUI's websocket feed.
Reacts to server-pushed ``executing``/``progress``/``execution_error``
events instead of sleeping between REST polls, so completion and
errors are detected immediately rather than up to *interval* seconds
late. *on_progress*, if given, is called with each ``progress``
message's ``data`` dict (``value``, ``max``, ``node``, ``prompt_id``).
Requires the optional ``websocket-client`` package. Any transport
failure (missing dependency, connection refused, dropped socket,
malformed frame) propagates as a plain exception — callers should
catch it and fall back to :meth:`poll`, which is what :meth:`generate`
does. A genuine ComfyUI-side execution error or an unmet deadline is
raised as :class:`ComfyUIError` with ``prompt_id`` set, exactly like
:meth:`poll`, so ``resume_prompt_id`` recovery works the same way
regardless of which wait strategy was used.
"""
import websocket # websocket-client; optional, see docstring
# History is authoritative and websocket events are not replayed. The
# job may already have finished between submit() and this wait call.
entry = self._history_entry_if_reachable(prompt_id)
if entry is not None:
return entry
ws_url = self.server_url.replace("http://", "ws://", 1).replace(
"https://", "wss://", 1
)
conn = websocket.create_connection(
f"{ws_url}/ws?clientId={self.client_id}", timeout=10
)
try:
conn.settimeout(interval)
deadline = time.time() + timeout
finished = False
# Close the remaining race between the first history probe and
# websocket connection establishment. Events after this point are
# queued on the open socket; earlier completion is in history.
entry = self._history_entry_if_reachable(prompt_id)
if entry is not None:
return entry
while time.time() < deadline:
try:
raw = conn.recv()
except websocket.WebSocketTimeoutException:
entry = self._history_entry_if_reachable(prompt_id)
if entry is not None:
return entry
continue
if not isinstance(raw, str):
continue # binary preview-image frame, not a status message
try:
message = json.loads(raw)
except json.JSONDecodeError:
continue
data = message.get("data", {})
if data.get("prompt_id") not in (None, prompt_id):
continue # another job sharing this connection
msg_type = message.get("type")
if msg_type == "progress":
if on_progress:
on_progress(data)
elif msg_type == "execution_error":
raise ComfyUIError(f"Execution error: {data}", prompt_id=prompt_id)
elif msg_type == "executing" and data.get("node") is None:
finished = True
break
finally:
conn.close()
if not finished:
entry = self._history_entry_if_reachable(prompt_id)
if entry is not None:
return entry
raise ComfyUIError(
f"Prompt {prompt_id} did not complete within {timeout}s "
f"(websocket wait). The job was not cancelled — resume with "
f"resume_prompt_id={prompt_id!r} and a longer timeout.",
prompt_id=prompt_id,
)
entry = self._history_entry(prompt_id)
if entry is None:
raise ComfyUIError(
f"No history entry for {prompt_id} after completion",
prompt_id=prompt_id,
)
return entry
def _wait(
self,
prompt_id: str,
*,
timeout: int,
interval: int,
on_progress: Callable[[dict], None] | None = None,
) -> dict:
"""Wait for *prompt_id*, preferring the websocket feed over polling.
Falls back to :meth:`poll` when ``websocket-client`` isn't installed
or the websocket can't be established/maintained. A genuine
:class:`ComfyUIError` (execution error or deadline reached) is never
swallowed by the fallback — only transport-level failures are. The
fallback gets whatever's left of *timeout*, not a fresh budget, so a
mid-wait websocket drop can't double the caller's worst-case wait.
"""
started = time.time()
try:
return self.wait_ws(
prompt_id, timeout=timeout, interval=interval, on_progress=on_progress
)
except ComfyUIError:
raise
except Exception:
remaining = max(timeout - (time.time() - started), 0)
return self.poll(prompt_id, timeout=remaining, interval=interval)
def download(
self,
filename: str,
subfolder: str,
dest: Path,
folder_type: str = "output",
) -> Path:
"""Download an output artifact from the ComfyUI server."""
resp = requests.get(
f"{self.server_url}/view",
params={
"filename": filename,
"subfolder": subfolder,
"type": folder_type,
},
timeout=120,
)
resp.raise_for_status()
dest.parent.mkdir(parents=True, exist_ok=True)
dest.write_bytes(resp.content)
return dest
def upload_image(self, local_path: Path, name: str) -> str:
"""Upload a local image so it can be referenced by LoadImage nodes.
Returns the server-side filename.
"""
with open(local_path, "rb") as f:
resp = requests.post(
f"{self.server_url}/upload/image",
files={"image": (name, f, "image/png")},
timeout=30,
)
resp.raise_for_status()
return resp.json()["name"]
# ------------------------------------------------------------------
# High-level helper
# ------------------------------------------------------------------
def generate(
self,
workflow: dict,
output_node: str,
dest: Path,
*,
timeout: int = 600,
interval: int = 5,
resume_prompt_id: str | None = None,
on_progress: Callable[[dict], None] | None = None,
) -> list[Path]:
"""Submit → wait → download. Returns list of artifact paths.
Pass ``resume_prompt_id`` (from a previous ``ComfyUIError.prompt_id``)
to skip re-submitting an already-queued/running job and just resume
waiting on it — the common recovery path after a timeout.
Waiting prefers ComfyUI's websocket feed (immediate completion/error
detection, optional live ``on_progress`` callback) and transparently
falls back to REST polling if ``websocket-client`` isn't installed or
the connection can't be used. See :meth:`_wait`.
"""
prompt_id = resume_prompt_id or self.submit(workflow)
if resume_prompt_id:
# A prompt resumed by a new client instance was submitted with the
# original instance's client_id, so its websocket events are not
# guaranteed to reach this socket. Poll authoritative history.
entry = self.poll(prompt_id, timeout=timeout, interval=interval)
else:
entry = self._wait(
prompt_id, timeout=timeout, interval=interval, on_progress=on_progress
)
outputs = entry.get("outputs", {})
node_output = outputs.get(output_node, {})
# ComfyUI stores images/video frames under "images", legacy GIFs
# under "gifs", and the native SaveAudio node's output under "audio".
items = (
node_output.get("images", [])
or node_output.get("gifs", [])
or node_output.get("audio", [])
or node_output.get("video", [])
)
if not items:
raise ComfyUIError(
f"No output artifacts on node {output_node}. "
f"Available nodes: {list(outputs.keys())}"
)
paths: list[Path] = []
for i, item in enumerate(items):
suffix = Path(item["filename"]).suffix
if len(items) == 1:
target = dest
else:
target = dest.with_stem(f"{dest.stem}_{i:03d}").with_suffix(suffix)
self.download(
item["filename"],
item.get("subfolder", ""),
target,
item.get("type", "output"),
)
paths.append(target)
return paths
# ------------------------------------------------------------------
# Workflow helpers
# ------------------------------------------------------------------
@staticmethod
def load_workflow(path: Path) -> dict:
"""Load a workflow JSON template from disk."""
with open(path) as f:
return json.load(f)
@staticmethod
def patch_workflow(workflow: dict, patches: dict[str, dict[str, Any]]) -> dict:
"""Deep-copy *workflow* and apply *patches*.
*patches* maps ``node_id`` → ``{input_name: value, ...}``.
"""
w = copy.deepcopy(workflow)
for node_id, values in patches.items():
if node_id not in w:
raise ComfyUIError(
f"Node {node_id!r} not found in workflow. "
f"Available: {list(w.keys())}"
)
for key, val in values.items():
w[node_id]["inputs"][key] = val
return w
@staticmethod
def random_seed() -> int:
"""Return a random seed suitable for ComfyUI noise nodes."""
return random.randint(0, 2**32 - 1)