The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.
Improve qwen 2.1 KV cache location logic (#16429)
rattus2f7c6d47a841
diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.pyindex 013d2391b..793b66c8a 100644--- a/comfy/ldm/qwen_image21/model.py+++ b/comfy/ldm/qwen_image21/model.py@@ -249,14 +249,17 @@ class QwenImage21Transformer2DModel(nn.Module):if cache is None:store = options.get("device", "auto")if store == "auto":- # spare VRAM first, then pinned RAM prefetched behind compute; room for cond and uncond- store = next((d for d in (device, torch.device("cpu")) if comfy.model_management.get_free_memory(d) > 4 * cache_bytes), None)- if store is None:+ # spare VRAM first, then reclaim inactive model RAM for a host cache+ if self.current_patcher.get_free_memory(device) > 4 * cache_bytes:+ store = device+ elif comfy.model_management.ensure_pin_budget(cache_bytes, evict_active=False):+ store = torch.device("cpu")+ else:return None, Falseelse:store = device if store == "gpu" else torch.device("cpu")cache = self.prefix_cache = PoseBranchCache(store_device=store, dtype=dtype)- if comfy.model_management.get_free_memory(cache.store_device) < 2 * cache_bytes:+ if not (self.current_patcher.get_free_memory(device) > 2 * cache_bytes if cache.store_device == device else comfy.model_management.ensure_pin_budget(cache_bytes, evict_active=False)):# no room for this slot: recompute rather than evict the other cond's slot every stepreturn None, Falsecache.select(key)diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.pyindex a8a487670..8bc55f4e9 100644--- a/comfy/ldm/wan/model_animate2.py+++ b/comfy/ldm/wan/model_animate2.py@@ -185,7 +185,7 @@ class PoseBranchCache:t, params = comfy.quant_ops.TensorWiseINT8Layout.quantize(t.reshape(-1, t.shape[-1]), is_weight=True, per_channel=True, convrot=True, convrot_groupsize=g)t = t.to(self.store_device, copy=True)- if comfy.model_management.pin_memory(t):+ if comfy.model_management.pin_memory(t, evict_active=False):self.slot["pinned"].append(t)self.slot["blocks"][i] = t# the scales follow the blocks off the GPU: per-window slots would otherwise pile them up in VRAM (~200 MB per window at 480p int4)diff --git a/comfy/model_base.py b/comfy/model_base.pyindex 0bed59cc5..6f1430991 100644--- a/comfy/model_base.py+++ b/comfy/model_base.py@@ -2671,6 +2671,7 @@ class QwenImage21(QwenImage):self._current_patcher = patcherdiffusion_model = getattr(self, "diffusion_model", None)if diffusion_model is not None:+ diffusion_model.current_patcher = patcherdiffusion_model.reset_prefix_cache(patcher is not None and len(patcher.hook_patches) == 0)def extra_conds(self, **kwargs):diff --git a/comfy/model_management.py b/comfy/model_management.pyindex 9aa0a9de0..c050441ca 100644--- a/comfy/model_management.py+++ b/comfy/model_management.py@@ -1637,7 +1637,7 @@ def discard_cuda_async_error():#Dump it! We already know about it from the synchronous returnpass-def pin_memory(tensor):+def pin_memory(tensor, evict_active=True):global TOTAL_PINNED_MEMORYif MAX_PINNED_MEMORY <= 0:return False@@ -1659,7 +1659,8 @@ def pin_memory(tensor):size = tensor.nbytescomfy.memory_management.extra_ram_release(comfy.memory_management.RAM_CACHE_HEADROOM)- ensure_pin_registerable(size)+ if not ensure_pin_registerable(size, evict_active=evict_active):+ return Falseptr = tensor.data_ptr()if ptr == 0:@@ -1778,6 +1779,12 @@ def force_upcast_attention_dtype():else:return None+#Developers and agents: You almost never want to call this function from Model code as it does+#not account for ComfyUIs smart memory feature combining with Dynamic VRAM, where inactive models+#are preserved in VRAM right up until there is higher priority demand (I.E whatever you want to do+#that makes you meansure VRAM from model code). Instead call get_free_memory() on the ModelPatcher+#for your BaseModel object (.current_patcher) instead to count this VRAM as free and then Dynamic+#VRAM will evict that extra VRAM for you when you use it.def get_free_memory(dev=None, torch_free_too=False):global directml_enabledif dev is None: