Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

feat(npu): support async weight offload streams (#16057)

yulun78368eafee72
diff --git a/comfy/model_management.py b/comfy/model_management.py
index d6856009b..4ea7cbae7 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1391,6 +1391,8 @@ def current_stream(device):
return torch.cuda.current_stream()
elif is_device_xpu(device):
return torch.xpu.current_stream()
+ elif is_device_npu(device):
+ return torch.npu.current_stream(device)
else:
return None
@@ -1523,6 +1525,18 @@ def get_offload_stream(device):
s = ss[stream_counter]
stream_counters[device] = stream_counter
return s
+ elif is_device_npu(device):
+ ss = []
+ # Match the CUDA/XPU stream interface used by the shared offload path.
+ # torch.npu.stream provides the context manager for torch-npu streams.
+ for k in range(NUM_STREAMS):
+ s1 = torch.npu.Stream(device=device, priority=0)
+ s1.as_context = torch.npu.stream
+ ss.append(s1)
+ STREAMS[device] = ss
+ s = ss[stream_counter]
+ stream_counters[device] = stream_counter
+ return s
return None
def sync_stream(device, stream):
@@ -1857,6 +1871,9 @@ def is_device_xpu(device):
def is_device_cuda(device):
return is_device_type(device, 'cuda')
+def is_device_npu(device):
+ return is_device_type(device, 'npu')
+
def set_torch_device(device):
"""Set the current device for the given torch device. Supports CUDA and XPU."""
if is_device_cuda(device):
@@ -2097,6 +2114,8 @@ def synchronize():
return
if is_intel_xpu():
torch.xpu.synchronize()
+ elif is_ascend_npu():
+ torch.npu.synchronize()
elif torch.cuda.is_available():
torch.cuda.synchronize()
diff --git a/tests-unit/comfy_test/model_management_test.py b/tests-unit/comfy_test/model_management_test.py
new file mode 100644
index 000000000..7c912057d
--- /dev/null
+++ b/tests-unit/comfy_test/model_management_test.py
@@ -0,0 +1,81 @@
+from unittest.mock import Mock, call
+
+import comfy.model_management as model_management
+
+
+class NPUDevice:
+ type = "npu"
+
+
+class FakeStream:
+ def __init__(self):
+ self.waited_for = None
+
+ def wait_stream(self, stream):
+ self.waited_for = stream
+
+
+def test_npu_current_stream(monkeypatch):
+ device = NPUDevice()
+ current_stream = object()
+ npu = Mock()
+ npu.current_stream.return_value = current_stream
+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)
+
+ assert model_management.is_device_npu(device)
+ assert model_management.current_stream(device) is current_stream
+ npu.current_stream.assert_called_once_with(device)
+
+
+def test_npu_offload_streams(monkeypatch):
+ device = NPUDevice()
+ current_stream = object()
+ first_stream = FakeStream()
+ second_stream = FakeStream()
+ stream_context = object()
+ npu = Mock()
+ npu.current_stream.return_value = current_stream
+ npu.Stream.side_effect = [first_stream, second_stream]
+ npu.stream = stream_context
+
+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)
+ monkeypatch.setattr(model_management.torch.compiler, "is_compiling", lambda: False)
+ monkeypatch.setattr(model_management, "NUM_STREAMS", 2)
+ monkeypatch.setattr(model_management, "STREAMS", {})
+ monkeypatch.setattr(model_management, "stream_counters", {})
+
+ assert model_management.get_offload_stream(device) is first_stream
+ assert model_management.get_offload_stream(device) is second_stream
+ assert model_management.get_offload_stream(device) is first_stream
+ assert model_management.STREAMS[device] == [first_stream, second_stream]
+ assert first_stream.as_context is stream_context
+ assert second_stream.as_context is stream_context
+ assert first_stream.waited_for is current_stream
+ assert second_stream.waited_for is current_stream
+ assert model_management.stream_counters[device] == 0
+ assert npu.Stream.call_count == 2
+ assert npu.Stream.call_args_list == [
+ call(device=device, priority=0),
+ call(device=device, priority=0),
+ ]
+
+
+def test_npu_offload_streams_disabled(monkeypatch):
+ npu = Mock()
+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)
+ monkeypatch.setattr(model_management, "NUM_STREAMS", 0)
+
+ assert model_management.get_offload_stream(NPUDevice()) is None
+ npu.Stream.assert_not_called()
+
+
+def test_npu_synchronize(monkeypatch):
+ npu = Mock()
+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)
+ monkeypatch.setattr(model_management, "cpu_mode", lambda: False)
+ monkeypatch.setattr(model_management, "is_intel_xpu", lambda: False)
+ monkeypatch.setattr(model_management, "is_ascend_npu", lambda: True)
+
+ model_management.synchronize()
+
+ npu.synchronize.assert_called_once_with()