The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Fix progress update crossover between users (#9706)
guilla9f1bb10a52c
diff --git a/.github/workflows/test-execution.yml b/.github/workflows/test-execution.ymlnew file mode 100644index 000000000..00ef07ebf--- /dev/null+++ b/.github/workflows/test-execution.yml@@ -0,0 +1,30 @@+name: Execution Tests++on:+ push:+ branches: [ main, master ]+ pull_request:+ branches: [ main, master ]++jobs:+ test:+ strategy:+ matrix:+ os: [ubuntu-latest, windows-latest, macos-latest]+ runs-on: ${{ matrix.os }}+ continue-on-error: true+ steps:+ - uses: actions/checkout@v4+ - name: Set up Python+ uses: actions/setup-python@v4+ with:+ python-version: '3.12'+ - name: Install requirements+ run: |+ python -m pip install --upgrade pip+ pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/cpu+ pip install -r requirements.txt+ pip install -r tests-unit/requirements.txt+ - name: Run Execution Tests+ run: |+ python -m pytest tests/execution -v --skip-timing-checksdiff --git a/comfy_execution/progress.py b/comfy_execution/progress.pyindex e8f5ede1e..f951a3350 100644--- a/comfy_execution/progress.py+++ b/comfy_execution/progress.py@@ -181,8 +181,9 @@ class WebUIProgressHandler(ProgressHandler):}# Send a combined progress_state message with all node states+ # Include client_id to ensure message is only sent to the initiating clientself.server_instance.send_sync(- "progress_state", {"prompt_id": prompt_id, "nodes": active_nodes}+ "progress_state", {"prompt_id": prompt_id, "nodes": active_nodes}, self.server_instance.client_id)@overridediff --git a/tests/conftest.py b/tests/conftest.pyindex 4e30eb581..290e3a5c0 100644--- a/tests/conftest.py+++ b/tests/conftest.py@@ -6,6 +6,7 @@ def pytest_addoption(parser):parser.addoption('--output_dir', action="store", default='tests/inference/samples', help='Output directory for generated images')parser.addoption("--listen", type=str, default="127.0.0.1", metavar="IP", nargs="?", const="0.0.0.0", help="Specify the IP address to listen on (default: 127.0.0.1). If --listen is provided without an argument, it defaults to 0.0.0.0. (listens on all)")parser.addoption("--port", type=int, default=8188, help="Set the listen port.")+ parser.addoption("--skip-timing-checks", action="store_true", default=False, help="Skip timing-related assertions in tests (useful for CI environments with variable performance)")# This initializes args at the beginning of the test session@pytest.fixture(scope="session", autouse=True)@@ -19,6 +20,11 @@ def args_pytest(pytestconfig):return args[email protected](scope="session")+def skip_timing_checks(pytestconfig):+ """Fixture that returns whether timing checks should be skipped."""+ return pytestconfig.getoption("--skip-timing-checks")+def pytest_collection_modifyitems(items):# Modifies items so tests run in the correct orderdiff --git a/tests/inference/extra_model_paths.yaml b/tests/execution/extra_model_paths.yamlsimilarity index 100%rename from tests/inference/extra_model_paths.yamlrename to tests/execution/extra_model_paths.yamldiff --git a/tests/inference/test_async_nodes.py b/tests/execution/test_async_nodes.pysimilarity index 95%rename from tests/inference/test_async_nodes.pyrename to tests/execution/test_async_nodes.pyindex f029953dd..c771b4b36 100644--- a/tests/inference/test_async_nodes.py+++ b/tests/execution/test_async_nodes.py@@ -7,7 +7,7 @@ import subprocessfrom pytest import fixturefrom comfy_execution.graph_utils import GraphBuilder-from tests.inference.test_execution import ComfyClient, run_warmup+from tests.execution.test_execution import ComfyClient, run_warmup@pytest.mark.execution@@ -23,7 +23,7 @@ class TestAsyncNodes:'--output-directory', args_pytest["output_dir"],'--listen', args_pytest["listen"],'--port', str(args_pytest["port"]),- '--extra-model-paths-config', 'tests/inference/extra_model_paths.yaml',+ '--extra-model-paths-config', 'tests/execution/extra_model_paths.yaml','--cpu',]use_lru, lru_size = request.param@@ -81,7 +81,7 @@ class TestAsyncNodes:assert len(result_images) == 1, "Should have 1 image"assert np.array(result_images[0]).min() == 0 and np.array(result_images[0]).max() == 0, "Image should be black"- def test_multiple_async_parallel_execution(self, client: ComfyClient, builder: GraphBuilder):+ def test_multiple_async_parallel_execution(self, client: ComfyClient, builder: GraphBuilder, skip_timing_checks):"""Test that multiple async nodes execute in parallel."""# Warmup execution to ensure server is fully initializedrun_warmup(client)@@ -104,7 +104,8 @@ class TestAsyncNodes:elapsed_time = time.time() - start_time# Should take ~0.5s (max duration) not 1.2s (sum of durations)- assert elapsed_time < 0.8, f"Parallel execution took {elapsed_time}s, expected < 0.8s"+ if not skip_timing_checks:+ assert elapsed_time < 0.8, f"Parallel execution took {elapsed_time}s, expected < 0.8s"# Verify all nodes executedassert result.did_run(sleep1) and result.did_run(sleep2) and result.did_run(sleep3)@@ -150,7 +151,7 @@ class TestAsyncNodes:with pytest.raises(urllib.error.HTTPError):client.run(g)- def test_async_lazy_evaluation(self, client: ComfyClient, builder: GraphBuilder):+ def test_async_lazy_evaluation(self, client: ComfyClient, builder: GraphBuilder, skip_timing_checks):"""Test async nodes with lazy evaluation."""# Warmup execution to ensure server is fully initializedrun_warmup(client, prefix="warmup_lazy")@@ -173,7 +174,8 @@ class TestAsyncNodes:elapsed_time = time.time() - start_time# Should only execute sleep1, not sleep2- assert elapsed_time < 0.5, f"Should skip sleep2, took {elapsed_time}s"+ if not skip_timing_checks:+ assert elapsed_time < 0.5, f"Should skip sleep2, took {elapsed_time}s"assert result.did_run(sleep1), "Sleep1 should have executed"assert not result.did_run(sleep2), "Sleep2 should have been skipped"@@ -310,7 +312,7 @@ class TestAsyncNodes:images = result.get_images(output)assert len(images) == 1, "Should have blocked second image"- def test_async_caching_behavior(self, client: ComfyClient, builder: GraphBuilder):+ def test_async_caching_behavior(self, client: ComfyClient, builder: GraphBuilder, skip_timing_checks):"""Test that async nodes are properly cached."""# Warmup execution to ensure server is fully initializedrun_warmup(client, prefix="warmup_cache")@@ -330,9 +332,10 @@ class TestAsyncNodes:elapsed_time = time.time() - start_timeassert not result2.did_run(sleep_node), "Should be cached"- assert elapsed_time < 0.1, f"Cached run took {elapsed_time}s, should be instant"+ if not skip_timing_checks:+ assert elapsed_time < 0.1, f"Cached run took {elapsed_time}s, should be instant"- def test_async_with_dynamic_prompts(self, client: ComfyClient, builder: GraphBuilder):+ def test_async_with_dynamic_prompts(self, client: ComfyClient, builder: GraphBuilder, skip_timing_checks):"""Test async nodes within dynamically generated prompts."""# Warmup execution to ensure server is fully initializedrun_warmup(client, prefix="warmup_dynamic")@@ -345,8 +348,8 @@ class TestAsyncNodes:dynamic_async = g.node("TestDynamicAsyncGeneration",image1=image1.out(0),image2=image2.out(0),- num_async_nodes=3,- sleep_duration=0.2)+ num_async_nodes=5,+ sleep_duration=0.4)g.node("SaveImage", images=dynamic_async.out(0))start_time = time.time()@@ -354,7 +357,8 @@ class TestAsyncNodes:elapsed_time = time.time() - start_time# Should execute async nodes in parallel within dynamic prompt- assert elapsed_time < 0.5, f"Dynamic async execution took {elapsed_time}s"+ if not skip_timing_checks:+ assert elapsed_time < 1.0, f"Dynamic async execution took {elapsed_time}s"assert result.did_run(dynamic_async)def test_async_resource_cleanup(self, client: ComfyClient, builder: GraphBuilder):diff --git a/tests/inference/test_execution.py b/tests/execution/test_execution.pysimilarity index 98%rename from tests/inference/test_execution.pyrename to tests/execution/test_execution.pyindex e7b29302e..8ea05fdd8 100644--- a/tests/inference/test_execution.py+++ b/tests/execution/test_execution.py@@ -149,7 +149,7 @@ class TestExecution:'--output-directory', args_pytest["output_dir"],'--listen', args_pytest["listen"],'--port', str(args_pytest["port"]),- '--extra-model-paths-config', 'tests/inference/extra_model_paths.yaml',+ '--extra-model-paths-config', 'tests/execution/extra_model_paths.yaml','--cpu',]use_lru, lru_size = request.param@@ -518,7 +518,7 @@ class TestExecution:assert numpy.array(images[0]).min() == 63 and numpy.array(images[0]).max() == 63, "Image should have value 0.25"assert not result.did_run(test_node), "The execution should have been cached"- def test_parallel_sleep_nodes(self, client: ComfyClient, builder: GraphBuilder):+ def test_parallel_sleep_nodes(self, client: ComfyClient, builder: GraphBuilder, skip_timing_checks):# Warmup execution to ensure server is fully initializedrun_warmup(client)@@ -541,14 +541,15 @@ class TestExecution:# The test should take around 3.0 seconds (the longest sleep duration)# plus some overhead, but definitely less than the sum of all sleeps (9.0s)- assert elapsed_time < 8.9, f"Parallel execution took {elapsed_time}s, expected less than 8.9s"+ if not skip_timing_checks:+ assert elapsed_time < 8.9, f"Parallel execution took {elapsed_time}s, expected less than 8.9s"# Verify that all nodes executedassert result.did_run(sleep_node1), "Sleep node 1 should have run"assert result.did_run(sleep_node2), "Sleep node 2 should have run"assert result.did_run(sleep_node3), "Sleep node 3 should have run"- def test_parallel_sleep_expansion(self, client: ComfyClient, builder: GraphBuilder):+ def test_parallel_sleep_expansion(self, client: ComfyClient, builder: GraphBuilder, skip_timing_checks):# Warmup execution to ensure server is fully initializedrun_warmup(client)@@ -574,7 +575,9 @@ class TestExecution:# Similar to the previous test, expect parallel execution of the sleep nodes# which should complete in less than the sum of all sleeps- assert elapsed_time < 10.0, f"Expansion execution took {elapsed_time}s, expected less than 5.5s"+ # Lots of leeway here since Windows CI is slow+ if not skip_timing_checks:+ assert elapsed_time < 13.0, f"Expansion execution took {elapsed_time}s"# Verify the parallel sleep node executedassert result.did_run(parallel_sleep), "ParallelSleep node should have run"diff --git a/tests/execution/test_progress_isolation.py b/tests/execution/test_progress_isolation.pynew file mode 100644index 000000000..93dc0d41b--- /dev/null+++ b/tests/execution/test_progress_isolation.py@@ -0,0 +1,233 @@+"""Test that progress updates are properly isolated between WebSocket clients."""++import json+import pytest+import time+import threading+import uuid+import websocket+from typing import List, Dict, Any+from comfy_execution.graph_utils import GraphBuilder+from tests.execution.test_execution import ComfyClient+++class ProgressTracker:+ """Tracks progress messages received by a WebSocket client."""++ def __init__(self, client_id: str):+ self.client_id = client_id+ self.progress_messages: List[Dict[str, Any]] = []+ self.lock = threading.Lock()++ def add_message(self, message: Dict[str, Any]):+ """Thread-safe addition of progress messages."""+ with self.lock:+ self.progress_messages.append(message)++ def get_messages_for_prompt(self, prompt_id: str) -> List[Dict[str, Any]]:+ """Get all progress messages for a specific prompt_id."""+ with self.lock:+ return [+ msg for msg in self.progress_messages+ if msg.get('data', {}).get('prompt_id') == prompt_id+ ]++ def has_cross_contamination(self, own_prompt_id: str) -> bool:+ """Check if this client received progress for other prompts."""+ with self.lock:+ for msg in self.progress_messages:+ msg_prompt_id = msg.get('data', {}).get('prompt_id')+ if msg_prompt_id and msg_prompt_id != own_prompt_id:+ return True+ return False+++class IsolatedClient(ComfyClient):+ """Extended ComfyClient that tracks all WebSocket messages."""++ def __init__(self):+ super().__init__()+ self.progress_tracker = None+ self.all_messages: List[Dict[str, Any]] = []++ def connect(self, listen='127.0.0.1', port=8188, client_id=None):+ """Connect with a specific client_id and set up message tracking."""+ if client_id is None:+ client_id = str(uuid.uuid4())+ super().connect(listen, port, client_id)+ self.progress_tracker = ProgressTracker(client_id)++ def listen_for_messages(self, duration: float = 5.0):+ """Listen for WebSocket messages for a specified duration."""+ end_time = time.time() + duration+ self.ws.settimeout(0.5) # Non-blocking with timeout++ while time.time() < end_time:+ try:+ out = self.ws.recv()+ if isinstance(out, str):+ message = json.loads(out)+ self.all_messages.append(message)++ # Track progress_state messages+ if message.get('type') == 'progress_state':+ self.progress_tracker.add_message(message)+ except websocket.WebSocketTimeoutException:+ continue+ except Exception:+ # Log error silently in test context+ break+++class TestProgressIsolation:+ """Test suite for verifying progress update isolation between clients."""++ @pytest.fixture(scope="class", autouse=True)+ def _server(self, args_pytest):+ """Start the ComfyUI server for testing."""+ import subprocess+ pargs = [+ 'python', 'main.py',+ '--output-directory', args_pytest["output_dir"],+ '--listen', args_pytest["listen"],+ '--port', str(args_pytest["port"]),+ '--extra-model-paths-config', 'tests/execution/extra_model_paths.yaml',+ '--cpu',+ ]+ p = subprocess.Popen(pargs)+ yield+ p.kill()++ def start_client_with_retry(self, listen: str, port: int, client_id: str = None):+ """Start client with connection retries."""+ client = IsolatedClient()+ # Connect to server (with retries)+ n_tries = 5+ for i in range(n_tries):+ time.sleep(4)+ try:+ client.connect(listen, port, client_id)+ return client+ except ConnectionRefusedError as e:+ print(e) # noqa: T201+ print(f"({i+1}/{n_tries}) Retrying...") # noqa: T201+ raise ConnectionRefusedError(f"Failed to connect after {n_tries} attempts")++ def test_progress_isolation_between_clients(self, args_pytest):+ """Test that progress updates are isolated between different clients."""+ listen = args_pytest["listen"]+ port = args_pytest["port"]++ # Create two separate clients with unique IDs+ client_a_id = "client_a_" + str(uuid.uuid4())+ client_b_id = "client_b_" + str(uuid.uuid4())++ try:+ # Connect both clients with retries+ client_a = self.start_client_with_retry(listen, port, client_a_id)+ client_b = self.start_client_with_retry(listen, port, client_b_id)++ # Create simple workflows for both clients+ graph_a = GraphBuilder(prefix="client_a")+ image_a = graph_a.node("StubImage", content="BLACK", height=256, width=256, batch_size=1)+ graph_a.node("PreviewImage", images=image_a.out(0))++ graph_b = GraphBuilder(prefix="client_b")+ image_b = graph_b.node("StubImage", content="WHITE", height=256, width=256, batch_size=1)+ graph_b.node("PreviewImage", images=image_b.out(0))++ # Submit workflows from both clients+ prompt_a = graph_a.finalize()+ prompt_b = graph_b.finalize()++ response_a = client_a.queue_prompt(prompt_a)+ prompt_id_a = response_a['prompt_id']++ response_b = client_b.queue_prompt(prompt_b)+ prompt_id_b = response_b['prompt_id']++ # Start threads to listen for messages on both clients+ def listen_client_a():+ client_a.listen_for_messages(duration=10.0)++ def listen_client_b():+ client_b.listen_for_messages(duration=10.0)++ thread_a = threading.Thread(target=listen_client_a)+ thread_b = threading.Thread(target=listen_client_b)++ thread_a.start()+ thread_b.start()++ # Wait for threads to complete+ thread_a.join()+ thread_b.join()++ # Verify isolation+ # Client A should only receive progress for prompt_id_a+ assert not client_a.progress_tracker.has_cross_contamination(prompt_id_a), \+ f"Client A received progress updates for other clients' workflows. " \+ f"Expected only {prompt_id_a}, but got messages for multiple prompts."++ # Client B should only receive progress for prompt_id_b+ assert not client_b.progress_tracker.has_cross_contamination(prompt_id_b), \+ f"Client B received progress updates for other clients' workflows. " \+ f"Expected only {prompt_id_b}, but got messages for multiple prompts."++ # Verify each client received their own progress updates+ client_a_messages = client_a.progress_tracker.get_messages_for_prompt(prompt_id_a)+ client_b_messages = client_b.progress_tracker.get_messages_for_prompt(prompt_id_b)++ assert len(client_a_messages) > 0, \+ "Client A did not receive any progress updates for its own workflow"+ assert len(client_b_messages) > 0, \+ "Client B did not receive any progress updates for its own workflow"++ # Ensure no cross-contamination+ client_a_other = client_a.progress_tracker.get_messages_for_prompt(prompt_id_b)+ client_b_other = client_b.progress_tracker.get_messages_for_prompt(prompt_id_a)++ assert len(client_a_other) == 0, \+ f"Client A incorrectly received {len(client_a_other)} progress updates for Client B's workflow"+ assert len(client_b_other) == 0, \+ f"Client B incorrectly received {len(client_b_other)} progress updates for Client A's workflow"++ finally:+ # Clean up connections+ if hasattr(client_a, 'ws'):+ client_a.ws.close()+ if hasattr(client_b, 'ws'):+ client_b.ws.close()++ def test_progress_with_missing_client_id(self, args_pytest):+ """Test that progress updates handle missing client_id gracefully."""+ listen = args_pytest["listen"]+ port = args_pytest["port"]++ try:+ # Connect client with retries+ client = self.start_client_with_retry(listen, port)++ # Create a simple workflow+ graph = GraphBuilder(prefix="test_missing_id")+ image = graph.node("StubImage", content="BLACK", height=128, width=128, batch_size=1)+ graph.node("PreviewImage", images=image.out(0))++ # Submit workflow+ prompt = graph.finalize()+ response = client.queue_prompt(prompt)+ prompt_id = response['prompt_id']++ # Listen for messages+ client.listen_for_messages(duration=5.0)++ # Should still receive progress updates for own workflow+ messages = client.progress_tracker.get_messages_for_prompt(prompt_id)+ assert len(messages) > 0, \+ "Client did not receive progress updates even though it initiated the workflow"++ finally:+ if hasattr(client, 'ws'):+ client.ws.close()+diff --git a/tests/inference/testing_nodes/testing-pack/__init__.py b/tests/execution/testing_nodes/testing-pack/__init__.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/__init__.pyrename to tests/execution/testing_nodes/testing-pack/__init__.pydiff --git a/tests/inference/testing_nodes/testing-pack/api_test_nodes.py b/tests/execution/testing_nodes/testing-pack/api_test_nodes.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/api_test_nodes.pyrename to tests/execution/testing_nodes/testing-pack/api_test_nodes.pydiff --git a/tests/inference/testing_nodes/testing-pack/async_test_nodes.py b/tests/execution/testing_nodes/testing-pack/async_test_nodes.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/async_test_nodes.pyrename to tests/execution/testing_nodes/testing-pack/async_test_nodes.pydiff --git a/tests/inference/testing_nodes/testing-pack/conditions.py b/tests/execution/testing_nodes/testing-pack/conditions.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/conditions.pyrename to tests/execution/testing_nodes/testing-pack/conditions.pydiff --git a/tests/inference/testing_nodes/testing-pack/flow_control.py b/tests/execution/testing_nodes/testing-pack/flow_control.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/flow_control.pyrename to tests/execution/testing_nodes/testing-pack/flow_control.pydiff --git a/tests/inference/testing_nodes/testing-pack/specific_tests.py b/tests/execution/testing_nodes/testing-pack/specific_tests.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/specific_tests.pyrename to tests/execution/testing_nodes/testing-pack/specific_tests.pydiff --git a/tests/inference/testing_nodes/testing-pack/stubs.py b/tests/execution/testing_nodes/testing-pack/stubs.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/stubs.pyrename to tests/execution/testing_nodes/testing-pack/stubs.pydiff --git a/tests/inference/testing_nodes/testing-pack/tools.py b/tests/execution/testing_nodes/testing-pack/tools.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/tools.pyrename to tests/execution/testing_nodes/testing-pack/tools.pydiff --git a/tests/inference/testing_nodes/testing-pack/util.py b/tests/execution/testing_nodes/testing-pack/util.pysimilarity index 100%rename from tests/inference/testing_nodes/testing-pack/util.pyrename to tests/execution/testing_nodes/testing-pack/util.py