Skip to content

The most powerful and modular diffusion model GUI, api and backend with a graph/nodes interface. The fastest local inference engine in the world.

Improve qwen 2.1 KV cache location logic (#16429)

rattus2f7c6d47a841
diff --git a/comfy/ldm/qwen_image21/model.py b/comfy/ldm/qwen_image21/model.py
index 013d2391b..793b66c8a 100644
--- a/comfy/ldm/qwen_image21/model.py
+++ b/comfy/ldm/qwen_image21/model.py
@@ -249,14 +249,17 @@ class QwenImage21Transformer2DModel(nn.Module):
if cache is None:
store = options.get("device", "auto")
if store == "auto":
- # spare VRAM first, then pinned RAM prefetched behind compute; room for cond and uncond
- store = next((d for d in (device, torch.device("cpu")) if comfy.model_management.get_free_memory(d) > 4 * cache_bytes), None)
- if store is None:
+ # spare VRAM first, then reclaim inactive model RAM for a host cache
+ if self.current_patcher.get_free_memory(device) > 4 * cache_bytes:
+ store = device
+ elif comfy.model_management.ensure_pin_budget(cache_bytes, evict_active=False):
+ store = torch.device("cpu")
+ else:
return None, False
else:
store = device if store == "gpu" else torch.device("cpu")
cache = self.prefix_cache = PoseBranchCache(store_device=store, dtype=dtype)
- if comfy.model_management.get_free_memory(cache.store_device) < 2 * cache_bytes:
+ if not (self.current_patcher.get_free_memory(device) > 2 * cache_bytes if cache.store_device == device else comfy.model_management.ensure_pin_budget(cache_bytes, evict_active=False)):
# no room for this slot: recompute rather than evict the other cond's slot every step
return None, False
cache.select(key)
diff --git a/comfy/ldm/wan/model_animate2.py b/comfy/ldm/wan/model_animate2.py
index a8a487670..8bc55f4e9 100644
--- a/comfy/ldm/wan/model_animate2.py
+++ b/comfy/ldm/wan/model_animate2.py
@@ -185,7 +185,7 @@ class PoseBranchCache:
t, params = comfy.quant_ops.TensorWiseINT8Layout.quantize(t.reshape(-1, t.shape[-1]), is_weight=True, per_channel=True, convrot=True, convrot_groupsize=g)
t = t.to(self.store_device, copy=True)
- if comfy.model_management.pin_memory(t):
+ if comfy.model_management.pin_memory(t, evict_active=False):
self.slot["pinned"].append(t)
self.slot["blocks"][i] = t
# the scales follow the blocks off the GPU: per-window slots would otherwise pile them up in VRAM (~200 MB per window at 480p int4)
diff --git a/comfy/model_base.py b/comfy/model_base.py
index 0bed59cc5..6f1430991 100644
--- a/comfy/model_base.py
+++ b/comfy/model_base.py
@@ -2671,6 +2671,7 @@ class QwenImage21(QwenImage):
self._current_patcher = patcher
diffusion_model = getattr(self, "diffusion_model", None)
if diffusion_model is not None:
+ diffusion_model.current_patcher = patcher
diffusion_model.reset_prefix_cache(patcher is not None and len(patcher.hook_patches) == 0)
def extra_conds(self, **kwargs):
diff --git a/comfy/model_management.py b/comfy/model_management.py
index 9aa0a9de0..c050441ca 100644
--- a/comfy/model_management.py
+++ b/comfy/model_management.py
@@ -1637,7 +1637,7 @@ def discard_cuda_async_error():
#Dump it! We already know about it from the synchronous return
pass
-def pin_memory(tensor):
+def pin_memory(tensor, evict_active=True):
global TOTAL_PINNED_MEMORY
if MAX_PINNED_MEMORY <= 0:
return False
@@ -1659,7 +1659,8 @@ def pin_memory(tensor):
size = tensor.nbytes
comfy.memory_management.extra_ram_release(comfy.memory_management.RAM_CACHE_HEADROOM)
- ensure_pin_registerable(size)
+ if not ensure_pin_registerable(size, evict_active=evict_active):
+ return False
ptr = tensor.data_ptr()
if ptr == 0:
@@ -1778,6 +1779,12 @@ def force_upcast_attention_dtype():
else:
return None
+#Developers and agents: You almost never want to call this function from Model code as it does
+#not account for ComfyUIs smart memory feature combining with Dynamic VRAM, where inactive models
+#are preserved in VRAM right up until there is higher priority demand (I.E whatever you want to do
+#that makes you meansure VRAM from model code). Instead call get_free_memory() on the ModelPatcher
+#for your BaseModel object (.current_patcher) instead to count this VRAM as free and then Dynamic
+#VRAM will evict that extra VRAM for you when you use it.
def get_free_memory(dev=None, torch_free_too=False):
global directml_enabled
if dev is None: