The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
model_patcher: Improve dynamic offload heuristic (#12759)
rattusd531e3fb2a88
diff --git a/comfy/model_patcher.py b/comfy/model_patcher.pyindex 70f78a089..168ce8430 100644--- a/comfy/model_patcher.py+++ b/comfy/model_patcher.py@@ -699,7 +699,7 @@ class ModelPatcher:for key in list(self.pinned):self.unpin_weight(key)- def _load_list(self, prio_comfy_cast_weights=False, default_device=None):+ def _load_list(self, for_dynamic=False, default_device=None):loading = []for n, m in self.model.named_modules():default = False@@ -727,8 +727,13 @@ class ModelPatcher:return 0module_offload_mem += check_module_offload_mem("{}.weight".format(n))module_offload_mem += check_module_offload_mem("{}.bias".format(n))- prepend = (not hasattr(m, "comfy_cast_weights"),) if prio_comfy_cast_weights else ()- loading.append(prepend + (module_offload_mem, module_mem, n, m, params))+ # Dynamic: small weights (<64KB) first, then larger weights prioritized by size.+ # Non-dynamic: prioritize by module offload cost.+ if for_dynamic:+ sort_criteria = (module_offload_mem >= 64 * 1024, -module_offload_mem)+ else:+ sort_criteria = (module_offload_mem,)+ loading.append(sort_criteria + (module_mem, n, m, params))return loadingdef load(self, device_to=None, lowvram_model_memory=0, force_patch_weights=False, full_load=False):@@ -1508,11 +1513,11 @@ class ModelPatcherDynamic(ModelPatcher):if vbar is not None:vbar.prioritize()- loading = self._load_list(prio_comfy_cast_weights=True, default_device=device_to)- loading.sort(reverse=True)+ loading = self._load_list(for_dynamic=True, default_device=device_to)+ loading.sort()for x in loading:- _, _, _, n, m, params = x+ *_, module_mem, n, m, params = xdef set_dirty(item, dirty):if dirty or not hasattr(item, "_v_signature"):@@ -1627,9 +1632,9 @@ class ModelPatcherDynamic(ModelPatcher):return freeddef partially_unload_ram(self, ram_to_unload):- loading = self._load_list(prio_comfy_cast_weights=True, default_device=self.offload_device)+ loading = self._load_list(for_dynamic=True, default_device=self.offload_device)for x in loading:- _, _, _, _, m, _ = x+ *_, m, _ = xram_to_unload -= comfy.pinned_memory.unpin_memory(m)if ram_to_unload <= 0:return