Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Add a way to attach metadata to the websocket messages of a prompt. (#16763)

Christian Byrne6806341a5876
diff --git a/main.py b/main.py
index 9a29a55d3..4365fe120 100644
--- a/main.py
+++ b/main.py
@@ -357,6 +357,7 @@ def prompt_worker(q, server_instance, asset_manager):
execution_start_time = time.perf_counter()
prompt_id = item[1]
server_instance.last_prompt_id = prompt_id
+ server_instance.workflow_metadata = item[3].get("workflow_metadata", {})
sensitive = item[5]
extra_data = item[3].copy()
@@ -381,6 +382,7 @@ def prompt_worker(q, server_instance, asset_manager):
asset_manager.resume_background_scan()
if server_instance.client_id is not None:
server_instance.send_sync("executing", {"node": None, "prompt_id": prompt_id}, server_instance.client_id)
+ server_instance.workflow_metadata = {}
current_time = time.perf_counter()
execution_time = current_time - execution_start_time
diff --git a/openapi.yaml b/openapi.yaml
index d1146ffe7..806a94ffc 100644
--- a/openapi.yaml
+++ b/openapi.yaml
@@ -1067,6 +1067,11 @@ components:
workflow_id:
description: UUID identifying the cloud workflow entity to associate with this job
type: string
+ workflow_metadata:
+ additionalProperties:
+ type: string
+ description: 'Key/values echoed on the json websocket messages of this prompt that carry a `prompt_id`, so a client can tell which of its open workflows a message belongs to. Messages without a `prompt_id`, such as `status`, do not carry it, because a single socket serves every open workflow and such a message has no one owner. The server does not interpret the contents. Limited to 256 bytes serialised; a larger or non-object value is ignored. When this field is omitted, the server falls back to `{workflow_id: extra_data.extra_pnginfo.workflow.id}` if the submitted workflow carries an id, so a client that sends only the workflow gets the same messages. Note the `workflow_id` key a client typically sends here is the graph id from `extra_pnginfo.workflow.id`, which is not the same thing as the `workflow_id` field above.'
+ type: object
workflow_version_id:
description: UUID identifying the workflow version to associate with this job
type: string
diff --git a/server.py b/server.py
index 547bb5242..eaf49f64a 100644
--- a/server.py
+++ b/server.py
@@ -75,6 +75,37 @@ if args.enable_manager:
import comfyui_manager
+def valid_workflow_metadata(json_data: dict) -> Optional[dict]:
+ """The metadata a client may attach to a prompt's websocket messages.
+
+ Returns None when absent, not a dict, or over 256 bytes serialised. The
+ value is merged into outgoing messages, so it is validated here rather
+ than at the point of use.
+ """
+ metadata = json_data.get("workflow_metadata")
+ if isinstance(metadata, dict) and len(json.dumps(metadata)) <= 256:
+ return metadata
+ return None
+
+
+def workflow_metadata_from_prompt(extra_data: dict) -> Optional[dict]:
+ """Fallback for a client that sends a workflow but no explicit metadata.
+
+ Reads the same id /api/jobs reports, so the websocket messages of a prompt
+ carry it whether or not the client knows about workflow_metadata.
+ """
+ extra_pnginfo = extra_data.get("extra_pnginfo")
+ if not isinstance(extra_pnginfo, dict):
+ return None
+ workflow = extra_pnginfo.get("workflow")
+ if not isinstance(workflow, dict):
+ return None
+ workflow_id = workflow.get("id")
+ if isinstance(workflow_id, str) and workflow_id:
+ return valid_workflow_metadata({"workflow_metadata": {"workflow_id": workflow_id}})
+ return None
+
+
def _remove_sensitive_from_queue(queue: list) -> list:
"""Remove sensitive data (index 5) from queue item tuples."""
return [item[:5] for item in queue]
@@ -292,6 +323,7 @@ class PromptServer():
self.routes = routes
self.last_node_id = None
self.client_id = None
+ self.workflow_metadata = {}
self.on_prompt_handlers = []
@@ -1151,6 +1183,13 @@ class PromptServer():
if "client_id" in json_data:
extra_data["client_id"] = json_data["client_id"]
+ extra_data.pop("workflow_metadata", None)
+ metadata = valid_workflow_metadata(json_data)
+ if metadata is None:
+ metadata = workflow_metadata_from_prompt(extra_data)
+ if metadata is not None:
+ extra_data["workflow_metadata"] = metadata
+
if "comfy_usage_source" not in extra_data:
usage_source = request.headers.get("Comfy-Usage-Source")
if usage_source:
@@ -1424,6 +1463,9 @@ class PromptServer():
await send_socket_catch_exception(self.sockets[sid].send_json, message)
def send_sync(self, event, data, sid=None):
+ if self.workflow_metadata and isinstance(data, dict) and "prompt_id" in data:
+ data = {**self.workflow_metadata, **data}
+
self.loop.call_soon_threadsafe(
self.messages.put_nowait, (event, data, sid))
diff --git a/tests-unit/server_test/test_ws_workflow_metadata.py b/tests-unit/server_test/test_ws_workflow_metadata.py
new file mode 100644
index 000000000..0e3294b99
--- /dev/null
+++ b/tests-unit/server_test/test_ws_workflow_metadata.py
@@ -0,0 +1,164 @@
+"""Tests for the workflow_metadata key/values added to outgoing websocket messages"""
+
+import json
+
+import pytest
+
+import server
+from comfy_api.feature_flags import SERVER_FEATURE_FLAGS # noqa: F401
+
+
+class FakeLoop:
+ def call_soon_threadsafe(self, callback, *args):
+ callback(*args)
+
+
+def prompt_server():
+ instance = server.PromptServer.__new__(server.PromptServer)
+ instance.loop = FakeLoop()
+ instance.workflow_metadata = {}
+ instance.messages = type("Queue", (), {"sent": [], "put_nowait": lambda self, msg: self.sent.append(msg)})()
+ return instance
+
+
+def sent_data(prompt_server):
+ return [data for _, data, _ in prompt_server.messages.sent]
+
+
+def test_metadata_is_added_to_messages(prompt_server):
+ prompt_server.workflow_metadata = {"workflow_id": "abc"}
+ prompt_server.send_sync("executing", {"node": "1", "prompt_id": "p1"})
+ assert sent_data(prompt_server) == [{"workflow_id": "abc", "node": "1", "prompt_id": "p1"}]
+
+
+def test_no_metadata_leaves_messages_unchanged(prompt_server):
+ prompt_server.send_sync("executing", {"node": "1"})
+ assert sent_data(prompt_server) == [{"node": "1"}]
+ assert "workflow_id" not in sent_data(prompt_server)[0]
+
+
+def test_message_fields_win_over_metadata(prompt_server):
+ prompt_server.workflow_metadata = {"node": "spoofed", "workflow_id": "abc"}
+ prompt_server.send_sync("executed", {"node": "1"})
+ assert sent_data(prompt_server)[0]["node"] == "1"
+
+
+def test_status_messages_are_not_touched(prompt_server):
+ prompt_server.workflow_metadata = {"workflow_id": "abc"}
+ prompt_server.send_sync("status", {"status": {"exec_info": {}}})
+ assert sent_data(prompt_server) == [{"status": {"exec_info": {}}}]
+
+
+def test_binary_messages_are_not_touched(prompt_server):
+ prompt_server.workflow_metadata = {"workflow_id": "abc"}
+ prompt_server.send_sync(server.BinaryEventTypes.TEXT, "some text")
+ assert sent_data(prompt_server) == ["some text"]
+
+
+def test_events_without_a_prompt_id_are_not_touched(prompt_server):
+ prompt_server.workflow_metadata = {"workflow_id": "abc"}
+ prompt_server.send_sync("assets.seed.paused", {"reason": "prompt_running"})
+ prompt_server.send_sync("logs", {"entries": [], "size": 0})
+ assert sent_data(prompt_server) == [
+ {"reason": "prompt_running"},
+ {"entries": [], "size": 0},
+ ]
+
+
+def test_metadata_is_captured_when_the_message_is_queued(prompt_server):
+ prompt_server.workflow_metadata = {"workflow_id": "first"}
+ prompt_server.send_sync("execution_success", {"prompt_id": "p1"})
+ prompt_server.workflow_metadata = {"workflow_id": "second"}
+ assert sent_data(prompt_server)[0]["workflow_id"] == "first"
+
+
+class TestValidWorkflowMetadata:
+ """The dict is merged into outgoing messages, so it is validated at the
+ request boundary. A client can also put a value straight into extra_data,
+ which the route strips before applying the validated one."""
+
+ def test_dict_within_the_limit_is_accepted(self):
+ metadata = {"workflow_id": "abc"}
+ assert server.valid_workflow_metadata(
+ {"workflow_metadata": metadata}
+ ) == metadata
+
+ def test_absent_field(self):
+ assert server.valid_workflow_metadata({}) is None
+
+ @pytest.mark.parametrize("value", ["abc", 7, ["abc"], None, True])
+ def test_non_dict_is_rejected(self, value):
+ assert server.valid_workflow_metadata({"workflow_metadata": value}) is None
+
+ def test_oversized_dict_is_rejected(self):
+ assert (
+ server.valid_workflow_metadata({"workflow_metadata": {"k": "v" * 300}})
+ is None
+ )
+
+ def test_dict_at_the_limit_is_accepted(self):
+ metadata = {"k": "v" * (256 - len('{"k": ""}'))}
+ assert len(json.dumps(metadata)) == 256
+ assert server.valid_workflow_metadata({"workflow_metadata": metadata}) == metadata
+
+ def test_empty_dict_is_accepted_and_stamps_nothing(self, prompt_server):
+ assert server.valid_workflow_metadata({"workflow_metadata": {}}) == {}
+ prompt_server.workflow_metadata = {}
+ prompt_server.send_sync("executing", {"prompt_id": "p1"})
+ assert sent_data(prompt_server) == [{"prompt_id": "p1"}]
+
+
+class TestWorkflowMetadataFromPrompt:
+ """Cloud stamps workflow_id for any client because it reads the id from the
+ job record. Core reads the same id out of the submitted workflow, so a
+ client that knows nothing about workflow_metadata gets the same contract."""
+
+ @staticmethod
+ def pnginfo(workflow):
+ return {"extra_pnginfo": {"workflow": workflow}}
+
+ def test_id_in_the_workflow_is_used(self):
+ assert server.workflow_metadata_from_prompt(
+ self.pnginfo({"id": "abc", "nodes": []})
+ ) == {"workflow_id": "abc"}
+
+ def test_absent_when_the_workflow_has_no_id(self):
+ assert server.workflow_metadata_from_prompt(self.pnginfo({"nodes": []})) is None
+
+ def test_absent_when_there_is_no_workflow(self):
+ assert server.workflow_metadata_from_prompt({}) is None
+ assert server.workflow_metadata_from_prompt({"extra_pnginfo": {}}) is None
+
+ @pytest.mark.parametrize("value", ["abc", 7, [], None])
+ def test_non_dict_extra_pnginfo_is_ignored(self, value):
+ assert server.workflow_metadata_from_prompt({"extra_pnginfo": value}) is None
+
+ @pytest.mark.parametrize("value", ["abc", 7, [], None])
+ def test_non_dict_workflow_is_ignored(self, value):
+ assert server.workflow_metadata_from_prompt(self.pnginfo(value)) is None
+
+ def test_oversized_id_is_ignored(self):
+ # The id comes from the submitted workflow, so it is as client-controlled
+ # as the explicit field and has to meet the same size limit.
+ huge = "x" * 300
+ assert (
+ server.workflow_metadata_from_prompt(self.pnginfo({"id": huge, "nodes": []}))
+ is None
+ )
+
+ @pytest.mark.parametrize("value", ["", 7, None, {}])
+ def test_non_string_or_empty_id_is_ignored(self, value):
+ assert (
+ server.workflow_metadata_from_prompt(self.pnginfo({"id": value})) is None
+ )
+
+ def test_explicit_metadata_wins_over_the_workflow_id(self):
+ # The route prefers the validated field and only falls back, so a client
+ # that sends both gets the one it asked for.
+ json_data = {"workflow_metadata": {"workflow_id": "explicit"}}
+ extra_data = self.pnginfo({"id": "from-workflow"})
+ metadata = server.valid_workflow_metadata(json_data)
+ if metadata is None:
+ metadata = server.workflow_metadata_from_prompt(extra_data)
+ assert metadata == {"workflow_id": "explicit"}