The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
feat(npu): support async weight offload streams (#16057)
yulun78368eafee72
diff --git a/comfy/model_management.py b/comfy/model_management.pyindex d6856009b..4ea7cbae7 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1391,6 +1391,8 @@ def current_stream(device):return torch.cuda.current_stream()elif is_device_xpu(device):return torch.xpu.current_stream()+ elif is_device_npu(device):+ return torch.npu.current_stream(device)else:return None@@ -1523,6 +1525,18 @@ def get_offload_stream(device):s = ss[stream_counter]stream_counters[device] = stream_counterreturn s+ elif is_device_npu(device):+ ss = []+ # Match the CUDA/XPU stream interface used by the shared offload path.+ # torch.npu.stream provides the context manager for torch-npu streams.+ for k in range(NUM_STREAMS):+ s1 = torch.npu.Stream(device=device, priority=0)+ s1.as_context = torch.npu.stream+ ss.append(s1)+ STREAMS[device] = ss+ s = ss[stream_counter]+ stream_counters[device] = stream_counter+ return sreturn Nonedef sync_stream(device, stream):@@ -1857,6 +1871,9 @@ def is_device_xpu(device):def is_device_cuda(device):return is_device_type(device, 'cuda')+def is_device_npu(device):+ return is_device_type(device, 'npu')+def set_torch_device(device):"""Set the current device for the given torch device. Supports CUDA and XPU."""if is_device_cuda(device):@@ -2097,6 +2114,8 @@ def synchronize():returnif is_intel_xpu():torch.xpu.synchronize()+ elif is_ascend_npu():+ torch.npu.synchronize()elif torch.cuda.is_available():torch.cuda.synchronize()diff --git a/tests-unit/comfy_test/model_management_test.py b/tests-unit/comfy_test/model_management_test.pynew file mode 100644index 000000000..7c912057d--- /dev/null+++ b/tests-unit/comfy_test/model_management_test.py@@ -0,0 +1,81 @@+from unittest.mock import Mock, call++import comfy.model_management as model_management+++class NPUDevice:+ type = "npu"+++class FakeStream:+ def __init__(self):+ self.waited_for = None++ def wait_stream(self, stream):+ self.waited_for = stream+++def test_npu_current_stream(monkeypatch):+ device = NPUDevice()+ current_stream = object()+ npu = Mock()+ npu.current_stream.return_value = current_stream+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)++ assert model_management.is_device_npu(device)+ assert model_management.current_stream(device) is current_stream+ npu.current_stream.assert_called_once_with(device)+++def test_npu_offload_streams(monkeypatch):+ device = NPUDevice()+ current_stream = object()+ first_stream = FakeStream()+ second_stream = FakeStream()+ stream_context = object()+ npu = Mock()+ npu.current_stream.return_value = current_stream+ npu.Stream.side_effect = [first_stream, second_stream]+ npu.stream = stream_context++ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)+ monkeypatch.setattr(model_management.torch.compiler, "is_compiling", lambda: False)+ monkeypatch.setattr(model_management, "NUM_STREAMS", 2)+ monkeypatch.setattr(model_management, "STREAMS", {})+ monkeypatch.setattr(model_management, "stream_counters", {})++ assert model_management.get_offload_stream(device) is first_stream+ assert model_management.get_offload_stream(device) is second_stream+ assert model_management.get_offload_stream(device) is first_stream+ assert model_management.STREAMS[device] == [first_stream, second_stream]+ assert first_stream.as_context is stream_context+ assert second_stream.as_context is stream_context+ assert first_stream.waited_for is current_stream+ assert second_stream.waited_for is current_stream+ assert model_management.stream_counters[device] == 0+ assert npu.Stream.call_count == 2+ assert npu.Stream.call_args_list == [+ call(device=device, priority=0),+ call(device=device, priority=0),+ ]+++def test_npu_offload_streams_disabled(monkeypatch):+ npu = Mock()+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)+ monkeypatch.setattr(model_management, "NUM_STREAMS", 0)++ assert model_management.get_offload_stream(NPUDevice()) is None+ npu.Stream.assert_not_called()+++def test_npu_synchronize(monkeypatch):+ npu = Mock()+ monkeypatch.setattr(model_management.torch, "npu", npu, raising=False)+ monkeypatch.setattr(model_management, "cpu_mode", lambda: False)+ monkeypatch.setattr(model_management, "is_intel_xpu", lambda: False)+ monkeypatch.setattr(model_management, "is_ascend_npu", lambda: True)++ model_management.synchronize()++ npu.synchronize.assert_called_once_with()