The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Load the optional local agent through comfy-agent's v1 host (#17241)
Simon Pinfold7a28332047b6
diff --git a/agent_requirements.txt b/agent_requirements.txtnew file mode 100644index 000000000..a4cca7397--- /dev/null+++ b/agent_requirements.txt@@ -0,0 +1 @@+comfy-agent==0.0.0.dev0diff --git a/app/agent_host.py b/app/agent_host.pynew file mode 100644index 000000000..9db531b0f--- /dev/null+++ b/app/agent_host.py@@ -0,0 +1,176 @@+"""Loads the optional comfy-agent package and fronts it to ComfyUI.++ComfyUI owns a thin v1 contract and nothing more: it discovers the+``comfyui.agent`` "v1" entry point, creates the host, hands it the live aiohttp+app once (``register(app)``) so the host mounts its own routes on ComfyUI's+origin, starts it after the server is listening, stops it on shutdown, and+publishes the host's state as the ``agent`` feature flag plus an ``agent_status``+websocket message on every change. Everything else -- the agent process, its+storage, restarts, the proxy and the loopback/same-origin trust boundary --+lives in the package.++A v1 host must provide:+ register(app) -> None+ start(*, comfy_url: str, comfy_path: str, logger, on_change=None) -> None+ stop(timeout: float = ...) -> None+ status() -> {"state": "starting"|"ready"|"failed", "reason": str | None, ...}+"""++from __future__ import annotations++import importlib.metadata+import logging+import os++import folder_paths+from comfy_api import feature_flags++ENTRY_POINT_GROUP = "comfyui.agent"+PROTOCOL = "v1"++_host = None+# Set when loading or starting the host failed, so the flag reports why rather+# than leaving the agent looking switched off.+_failure = None+# feature_flag() can be read on every connection; log a raising status() once.+_status_logged = False+++def emit_event(event, /, **fields) -> None:+ logging.info(" ".join([f"[agent-event] {event}", *(f"{k}={v}" for k, v in fields.items() if v is not None)]))+++def load() -> bool:+ """Find and create the agent host. Reports why when it can't; nothing about+ the installed package can stop ComfyUI from starting."""+ global _host, _failure+ try:+ entry_points = {ep.name: ep for ep in importlib.metadata.entry_points(group=ENTRY_POINT_GROUP)}+ if PROTOCOL not in entry_points:+ # Some comfy-agent is installed (other protocols present) -> "update";+ # nothing there -> "install".+ if entry_points:+ logging.warning(f"The installed comfy-agent offers agent protocols {sorted(entry_points)}; this ComfyUI needs {PROTOCOL}. Update comfy-agent to use the agent.")+ _failure = {"state": "failed", "reason": "outdated"}+ emit_event("agent_error", reason="outdated")+ else:+ req = os.path.join(os.path.dirname(os.path.abspath(folder_paths.__file__)), "agent_requirements.txt")+ logging.warning(f"`--enable-agent` was passed but the comfy-agent package is not installed, so the agent is disabled. Its requirements are in {req}.")+ _failure = {"state": "failed", "reason": "package_missing"}+ emit_event("package_missing")+ _publish_flag()+ return False+ host = entry_points[PROTOCOL].load()()+ if not all(callable(getattr(host, name, None)) for name in ("start", "stop", "status", "register")):+ raise TypeError(f"the {PROTOCOL} entry point returned {type(host).__name__}, not an agent host")+ _host = host+ except (Exception, SystemExit): # a broken wheel may exit; that must not end ComfyUI's startup+ logging.exception("Could not load the local agent; it is disabled.")+ _failure = {"state": "failed", "reason": "unknown"}+ emit_event("agent_error", reason="unknown")+ _publish_flag()+ return False+ _publish_flag() # reads status(), which disables a host that exits+ return _failure is None+++def register(app) -> None:+ """Mount the agent's own API routes onto ComfyUI's aiohttp ``app``.++ ComfyUI calls this once while it registers its routes (before the router+ freezes); the host mounts /api/agent and /api/workflows onto the live app.+ No-op when the package did not load; a failure here disables the agent, not+ ComfyUI's startup."""+ global _failure+ if _host is None or _failure is not None:+ return+ try:+ _host.register(app)+ except (Exception, SystemExit): # a broken wheel must not end ComfyUI's startup+ logging.exception("The local agent's routes could not be registered; it is disabled.")+ _failure = {"state": "failed", "reason": "unknown"}+ emit_event("agent_error", reason="unknown")+ _publish_flag()+++def start(server, scheme: str, address: str, port: int) -> None:+ """Start the loaded host once the server is listening. Main thread only."""+ # register() may have failed after load(): _host stays set but _failure is+ # now recorded. Starting then would spawn an agent with no routes mounted+ # (claiming its data dir) while the flag reports failed, so bail out too.+ if _host is None or _failure is not None:+ return+ host = "127.0.0.1" if address in ("", "0.0.0.0") else "::1" if address == "::" else address+ comfy_url = f"{scheme}://[{host}]:{port}" if ":" in host else f"{scheme}://{host}:{port}"++ last_sent = None++ def on_change():+ # Called from the host's threads; last_sent is unsynchronized, so a race+ # costs at most one duplicate message. The host also reports changes the+ # published flag doesn't carry (e.g. its restart count), so only send+ # when the flag itself moves.+ nonlocal last_sent+ flag = _publish_flag()+ if flag != last_sent:+ last_sent = flag+ server.send_sync("agent_status", flag)++ def run():+ global _failure+ try:+ _host.start(comfy_url=comfy_url, comfy_path=os.path.dirname(os.path.abspath(folder_paths.__file__)), logger=logging.getLogger(), on_change=on_change)+ except (Exception, SystemExit):+ logging.exception("The local agent could not be started.")+ _failure = {"state": "failed", "reason": "spawn_failed"}+ emit_event("agent_error", reason="spawn_failed")+ on_change()++ # start() spawns and waits on the agent child, so keep it off the event loop.+ if server.loop is not None:+ server.loop.run_in_executor(None, run)+ else:+ run()+++def stop() -> None:+ if _host is not None:+ try:+ _host.stop()+ except (Exception, SystemExit):+ logging.exception("The local agent could not be stopped.")+++def _publish_flag():+ """Mirror the current agent flag into the shared server feature-flag dict so+ new connections read it via ``get_server_features()``; omitted when there is+ no value. Returns the flag so callers can detect transitions."""+ flag = feature_flag()+ if flag is None:+ feature_flags.SERVER_FEATURE_FLAGS.pop("agent", None)+ else:+ feature_flags.SERVER_FEATURE_FLAGS["agent"] = flag+ return flag+++def feature_flag():+ """The `agent` feature flag: the host's state, and its reason when it has one."""+ global _failure, _status_logged+ if _failure is not None or _host is None:+ return _failure+ try:+ status = _host.status()+ flag = {"state": status["state"]}+ if status.get("reason"):+ flag["reason"] = status["reason"]+ return flag+ except SystemExit: # a wheel that exits is broken, not briefly unhealthy, so disable it+ logging.exception("The local agent's status could not be read; it is disabled.")+ _failure = {"state": "failed", "reason": "unknown"}+ emit_event("agent_error", reason="unknown")+ return _failure+ except Exception:+ if not _status_logged:+ _status_logged = True+ logging.exception("The local agent's status could not be read.")+ return {"state": "failed", "reason": "unknown"}diff --git a/comfy/cli_args.py b/comfy/cli_args.pyindex f42dfbb4d..135a89900 100644--- a/comfy/cli_args.py+++ b/comfy/cli_args.py@@ -160,6 +160,7 @@ upcast.add_argument("--dont-upcast-attention", action="store_true", help="Disablparser.add_argument("--enable-manager", action="store_true", help="Enable the ComfyUI-Manager feature.")+parser.add_argument("--enable-agent", action="store_true", help="Enable the ComfyUI agent feature.")manager_group = parser.add_mutually_exclusive_group()manager_group.add_argument("--disable-manager-ui", action="store_true", help="Disables only the ComfyUI-Manager UI and endpoints. Scheduled installations and similar background tasks will still operate.")manager_group.add_argument("--enable-manager-legacy-ui", action="store_true", help="Enables the legacy UI of ComfyUI-Manager. Implies --enable-manager.")diff --git a/main.py b/main.pyindex 25dd02fc5..8494d5839 100644--- a/main.py+++ b/main.py@@ -37,6 +37,7 @@ import sysfrom comfy_execution.progress import get_progress_statefrom comfy_execution.utils import get_executing_contextfrom comfy_api import feature_flags+import app.agent_hostif __name__ == "__main__":#NOTE: These do not do anything on core ComfyUI, they are for custom nodes.@@ -127,6 +128,11 @@ def handle_comfyui_manager_unavailable():args.enable_manager = False+if args.enable_agent:+ logging.info("[agent-event] flag_enabled")+ app.agent_host.load() # a failure stays reported: the agent's paths answer 503 with the reason++def apply_custom_paths():# extra model pathsextra_model_paths_config_path = os.path.join(os.path.dirname(os.path.realpath(__file__)), "extra_model_paths.yaml")@@ -633,6 +639,15 @@ def start_comfyui(asyncio_loop=None):webbrowser.open(f"{scheme}://{address}:{port}")call_on_start = startup_server+ if args.enable_agent:+ open_browser = call_on_start++ def start_agent(scheme, address, port):+ if open_browser is not None:+ open_browser(scheme, address, port)+ app.agent_host.start(prompt_server, scheme, address, port)+ call_on_start = start_agent+async def start_all():await prompt_server.setup()await run(prompt_server, address=args.listen, port=args.port, verbose=not args.dont_print_server, call_on_start=call_on_start)@@ -672,4 +687,5 @@ if __name__ == "__main__":except KeyboardInterrupt:logging.info("\nStopped server")finally:+ app.agent_host.stop()prompt_server.asset_manager.shutdown()diff --git a/server.py b/server.pyindex 29d0b1942..6af7adf31 100644--- a/server.py+++ b/server.py@@ -59,6 +59,7 @@ if dependencies_available():from app.assets.services.preview_rules import preview_fieldsfrom app.asset_export import AssetExportManager+from app import agent_hostfrom app.user_manager import UserManagerfrom app.model_manager import ModelFileManagerfrom app.custom_node_manager import CustomNodeManager@@ -1314,6 +1315,12 @@ class PromptServer():self.client_session = aiohttp.ClientSession(timeout=timeout)def add_routes(self):+ # The comfy-agent host mounts its own /api/agent and /api/workflows+ # routes (the proxy + trust boundary now live in the wheel). Do it before+ # the '/' static catch-all below so those prefixes win, and before the+ # router freezes in start_multi_address().+ if args.enable_agent:+ agent_host.register(self.app)self.user_manager.add_routes(self.routes)self.model_file_manager.add_routes(self.routes)self.custom_node_manager.add_routes(self.routes, self.app, nodes.LOADED_MODULE_DIRS.items())diff --git a/tests-unit/app_test/agent_host_test.py b/tests-unit/app_test/agent_host_test.pynew file mode 100644index 000000000..aa463d8a6--- /dev/null+++ b/tests-unit/app_test/agent_host_test.py@@ -0,0 +1,299 @@+"""Contract test for app/agent_host.py against a fake v1 host."""++from __future__ import annotations++import asyncio+import importlib.metadata+import logging+import os+import threading+import time+from types import SimpleNamespace++import pytest++import folder_paths+from app import agent_host+from comfy_api import feature_flags+++class FakeHost:+ def __init__(self):+ self.state = {"state": "starting", "url": None, "reason": None, "restarts": 0, "max_restarts": 3}+ self.started = None+ self.on_change = None+ self.stopped = 0+ self.registered = []++ def start(self, *, comfy_url, comfy_path, logger, on_change=None):+ self.started = {"comfy_url": comfy_url, "comfy_path": comfy_path, "logger": logger}+ self.on_change = on_change++ def stop(self, timeout=10.0):+ self.stopped += 1++ def status(self):+ return dict(self.state)++ def register(self, app):+ # The real host mounts its /api/agent + /api/workflows subapps here; the+ # proxy + trust boundary live in the wheel, so core only hands over app.+ self.registered.append(app)++[email protected](autouse=True)+def _reset(monkeypatch):+ monkeypatch.setattr(agent_host, "_host", None)+ monkeypatch.setattr(agent_host, "_failure", None)+ monkeypatch.setattr(agent_host, "_status_logged", False)+ # The loader publishes the agent flag by mutating this shared dict; keep that+ # leakage out of other tests.+ feature_flags.SERVER_FEATURE_FLAGS.pop("agent", None)+ yield+ feature_flags.SERVER_FEATURE_FLAGS.pop("agent", None)+++def _entry_points(monkeypatch, eps: dict):+ points = [SimpleNamespace(name=name, load=lambda f=factory: f) for name, factory in eps.items()]+ monkeypatch.setattr(importlib.metadata, "entry_points", lambda group: points if group == "comfyui.agent" else [])+++def _raise(*args, **kwargs):+ raise RuntimeError("broken install")+++def _exit(*args, **kwargs):+ raise SystemExit("wrong-architecture wheel")+++async def _until(predicate):+ while not predicate():+ await asyncio.sleep(0.02)+++def _agent_records(caplog):+ return [r.getMessage() for r in caplog.records if r.getMessage().startswith("[agent-event]")]+++def _comfy_path():+ return os.path.dirname(os.path.abspath(folder_paths.__file__))+++ "eps, loaded, record",+ [+ ({"v1": FakeHost}, True, None),+ ({}, False, "[agent-event] package_missing"),+ ({"v9": FakeHost}, False, "[agent-event] agent_error reason=outdated"),+ ({"v1": _raise}, False, "[agent-event] agent_error reason=unknown"),+ ({"v1": _exit}, False, "[agent-event] agent_error reason=unknown"),+ ],+)+def test_load_finds_v1_or_reports_why_not(monkeypatch, caplog, eps, loaded, record):+ _entry_points(monkeypatch, eps)+ with caplog.at_level(logging.INFO):+ assert agent_host.load() is loaded+ assert _agent_records(caplog) == ([record] if record else [])+++def test_broken_entry_point_metadata_does_not_stop_comfyui(monkeypatch, caplog):+ monkeypatch.setattr(importlib.metadata, "entry_points", _raise)+ with caplog.at_level(logging.INFO):+ assert agent_host.load() is False+ assert _agent_records(caplog) == ["[agent-event] agent_error reason=unknown"]+++def test_status_reaches_the_flag_and_the_websocket(monkeypatch):+ _entry_points(monkeypatch, {"v1": FakeHost})+ assert agent_host.feature_flag() is None+ agent_host.load()+ sent = []+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda event, data: sent.append((event, data))), "http", "0.0.0.0", 8188)+ # Records must leave through ComfyUI's own logger; the host is told where+ # ComfyUI lives and the loopback URL to reach it on.+ assert agent_host._host.started == {"comfy_url": "http://127.0.0.1:8188", "comfy_path": _comfy_path(), "logger": logging.getLogger()}+ assert agent_host.feature_flag() == {"state": "starting"}+ agent_host._host.state = {"state": "ready", "url": None, "reason": None}+ agent_host._host.on_change()+ assert sent == [("agent_status", {"state": "ready"})]+ assert feature_flags.get_server_features()["agent"] == sent[0][1]++[email protected]("url", ["http://127.0.0.1:5555", "/agent", None])[email protected]("state", ["starting", "ready"])+def test_the_flag_never_publishes_a_url(monkeypatch, url, state):+ # The proxy is mandatory: the frontend always reaches the agent at /api/agent.+ monkeypatch.setattr(agent_host, "_host", FakeHost())+ agent_host._host.state = {"state": state, "url": url, "reason": None}+ assert agent_host.feature_flag() == {"state": state}+++def test_the_flag_carries_a_reason_only_when_set(monkeypatch):+ monkeypatch.setattr(agent_host, "_host", FakeHost())+ agent_host._host.state = {"state": "starting", "url": None, "reason": "crashed"}+ assert agent_host.feature_flag() == {"state": "starting", "reason": "crashed"}+ agent_host._host.state = {"state": "ready", "url": None, "reason": ""}+ assert agent_host.feature_flag() == {"state": "ready"}++[email protected](("eps", "reason"), [({}, "package_missing"), ({"v9": FakeHost}, "outdated"), ({"v1": _raise}, "unknown")])+def test_a_package_that_does_not_load_publishes_failed(monkeypatch, eps, reason):+ _entry_points(monkeypatch, eps)+ agent_host.load()+ assert feature_flags.get_server_features()["agent"] == {"state": "failed", "reason": reason}+++def test_status_changes_the_flag_does_not_carry_send_nothing(monkeypatch):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ sent = []+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda event, data: sent.append(data)), "http", "127.0.0.1", 8188)+ agent_host._host.on_change()+ agent_host._host.state = {**agent_host._host.state, "restarts": 1}+ agent_host._host.on_change() # only the restart count moved+ assert sent == [{"state": "starting"}]+++def test_a_raising_status_is_logged_once(monkeypatch, caplog):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ monkeypatch.setattr(agent_host._host, "status", _raise)+ for _ in range(3):+ assert agent_host.feature_flag()["state"] == "failed"+ assert sum("status could not be read" in r.getMessage() for r in caplog.records) == 1++[email protected]("error", [SystemExit])+def test_any_exception_from_the_package_fails_the_agent_not_comfyui(monkeypatch, error):+ def create():+ raise error()++ _entry_points(monkeypatch, {"v1": create})+ assert agent_host.load() is False+++def test_a_status_that_exits_during_load_fails_the_agent_not_comfyui(monkeypatch):+ class ExitingHost(FakeHost):+ status = _exit++ _entry_points(monkeypatch, {"v1": ExitingHost})+ assert agent_host.load() is False+ assert feature_flags.get_server_features()["agent"] == {"state": "failed", "reason": "unknown"}+ agent_host.register(object())+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda *a: None), "http", "127.0.0.1", 8188)+ assert agent_host._host.registered == [] and agent_host._host.started is None, "a host that exits is disabled"+++def test_a_factory_that_returns_no_host_fails_the_agent(monkeypatch):+ _entry_points(monkeypatch, {"v1": object})+ assert agent_host.load() is False+ assert agent_host.feature_flag() == {"state": "failed", "reason": "unknown"}+++def test_a_start_that_exits_fails_the_agent_not_comfyui(monkeypatch):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()++ def start(**kwargs):+ raise SystemExit(3) # what a corrupt or wrong-architecture wheel does++ monkeypatch.setattr(agent_host._host, "start", start)+ sent = []+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda event, data: sent.append(data)), "http", "127.0.0.1", 8188)+ assert sent == [{"state": "failed", "reason": "spawn_failed"}]++[email protected](("address", "url"), [("0.0.0.0", "http://127.0.0.1:8188"), ("", "http://127.0.0.1:8188"), ("::", "http://[::1]:8188"), ("::1", "http://[::1]:8188"), ("192.168.1.5", "http://192.168.1.5:8188")])+def test_comfyui_tells_the_agent_a_url_it_can_reach(monkeypatch, address, url):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda event, data: None), "http", address, 8188)+ assert agent_host._host.started["comfy_url"] == url+++def test_a_host_that_fails_to_start_reports_failed(monkeypatch, caplog):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ monkeypatch.setattr(agent_host._host, "start", _raise)+ sent = []+ with caplog.at_level(logging.INFO):+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda event, data: sent.append((event, data))), "http", "127.0.0.1", 8188)+ failed = {"state": "failed", "reason": "spawn_failed"}+ assert agent_host.feature_flag() == failed+ assert sent == [("agent_status", failed)]+ assert _agent_records(caplog) == ["[agent-event] agent_error reason=spawn_failed"]+++def test_register_hands_the_host_comfyuis_app(monkeypatch):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ app = object()+ agent_host.register(app)+ assert agent_host._host.registered == [app], "register() hands the host ComfyUI's app to mount its own routes on"+++def test_register_without_a_loaded_package_is_a_noop():+ # package_missing / outdated / broken: load() left _host None, so there is+ # nothing to mount and add_routes must carry on.+ agent_host.register(object())+++def test_a_register_failure_fails_the_agent_not_comfyui(monkeypatch, caplog):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ monkeypatch.setattr(agent_host._host, "register", _raise)+ with caplog.at_level(logging.INFO):+ agent_host.register(object()) # a broken wheel must not end add_routes+ assert agent_host.feature_flag() == {"state": "failed", "reason": "unknown"}+ assert feature_flags.get_server_features()["agent"] == {"state": "failed", "reason": "unknown"}+ assert "[agent-event] agent_error reason=unknown" in _agent_records(caplog)+++def test_start_does_not_run_the_host_after_a_register_failure(monkeypatch):+ # register() raising sets _failure while _host stays set. start() must bail+ # out instead of spawning an agent whose routes were never mounted.+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ monkeypatch.setattr(agent_host._host, "register", _raise)+ agent_host.register(object())+ sent = []+ agent_host.start(SimpleNamespace(loop=None, send_sync=lambda event, data: sent.append(data)), "http", "127.0.0.1", 8188)+ assert agent_host._host.started is None, "start() must not call the host after register() failed"+ assert sent == []+ assert agent_host.feature_flag() == {"state": "failed", "reason": "unknown"}++[email protected]("raises", [None, RuntimeError, SystemExit])+def test_stop_stops_the_host(monkeypatch, raises):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ if raises:+ def stop(timeout=10.0):+ agent_host._host.stopped += 1+ raise raises()++ monkeypatch.setattr(agent_host._host, "stop", stop)+ agent_host.stop()+ assert agent_host._host.stopped == 1+++def test_start_runs_the_host_off_the_event_loop(monkeypatch):+ _entry_points(monkeypatch, {"v1": FakeHost})+ agent_host.load()+ real_start = agent_host._host.start+ released = threading.Event()++ def slow_start(**kwargs):+ released.wait(5)+ real_start(**kwargs)++ monkeypatch.setattr(agent_host._host, "start", slow_start)+ loop = asyncio.new_event_loop()+ try:+ began = time.monotonic()+ agent_host.start(SimpleNamespace(loop=loop, send_sync=lambda event, data: None), "http", "127.0.0.1", 8188)+ assert time.monotonic() - began < 0.5, "start() doesn't run the host on the event loop"+ released.set()+ loop.run_until_complete(asyncio.wait_for(_until(lambda: agent_host._host.started), 5))+ finally:+ loop.close()