Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: free the Qwen-Image-2.1 prefix KV cache before VAE decode (#15094)

Daniel Han91e388570ebd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.py
index dadb5f1bf8e..7f164b4ca42 100644
--- a/studio/backend/core/inference/diffusion.py
+++ b/studio/backend/core/inference/diffusion.py
@@ -124,9 +124,11 @@ from .diffusion_zimage_cfg_split import (
zimage_cfg_split_status,
)
from .diffusion_zimage_cfg_split import render as zimage_cfg_split_render
+from .diffusion_prefix_kv import release_prefix_kv, track_prefix_kv
from .diffusion_multigpu import (
AUTOMATIC_ORDINALS,
end_cfg_render,
+ release_cfg_prefix_kv,
drop_displaced_cfg_router,
install_cfg_parallel,
multigpu_mode,
@@ -1128,6 +1130,17 @@ def _is_source_sized(workflow: str, family: Any = None) -> bool:
return workflow in ("img2img", "inpaint", "upscale")
+def _release_prefix_kv(pipe: Any) -> None:
+ """Drop the denoiser's prefix KV caches (and a CFG-parallel replica's copies) before VAE decode."""
+ try:
+ released = release_prefix_kv(getattr(pipe, "transformer", None))
+ if released:
+ release_cfg_prefix_kv(pipe)
+ logger.debug("diffusion.prefix_kv: %d MiB released before decode", released >> 20)
+ except Exception as exc: # noqa: BLE001 - a memory saving only
+ logger.debug("diffusion.prefix_kv: release failed: %s", exc)
+
+
def _compile_shape_dims(
workflow: str,
init_pil: Any,
@@ -13862,10 +13875,15 @@ class DiffusionBackend:
if cancel.is_set():
pipe._interrupt = True
cross_forward_step(pipe, step_index)
+ # Last denoise step issued: the prefix KV cache (Qwen-Image-2.1) is dead weight through decode.
+ num_steps = getattr(pipe, "_num_timesteps", None)
+ if isinstance(num_steps, int) and step_index + 1 >= num_steps:
+ _release_prefix_kv(pipe)
return callback_kwargs
if "callback_on_step_end" in call_params:
kwargs["callback_on_step_end"] = _on_step
+ track_prefix_kv(getattr(pipe, "transformer", None))
# Hires fix latent upscaler (ComfyUI LatentUpscale): the base pass hands the refine pass its latents,
# so it skips the VAE decode and the image post-processing that only the discarded pixels needed.
latent_output = bool(
diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.py
index 57d774fbd68..d72e3ccc179 100644
--- a/studio/backend/core/inference/diffusion_multigpu.py
+++ b/studio/backend/core/inference/diffusion_multigpu.py
@@ -3254,6 +3254,17 @@ def end_cfg_render(pipe: Any) -> None:
router.end_render()
+def release_cfg_prefix_kv(pipe: Any) -> None:
+ """Drop the replica's prefix-KV copies (and their hold on the primary's tensors) after the last denoise step."""
+ try:
+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None
+ except TypeError:
+ return
+ if router is not None:
+ router.kv_mirrors.clear()
+ router.kv_declined.clear()
+
+
def uninstall_cfg_parallel(pipe: Any) -> None:
try:
router = _CFG_ROUTERS.pop(pipe, None) if pipe is not None else None
diff --git a/studio/backend/core/inference/diffusion_prefix_kv.py b/studio/backend/core/inference/diffusion_prefix_kv.py
index 9a86123a735..4acb7eaef7a 100644
--- a/studio/backend/core/inference/diffusion_prefix_kv.py
+++ b/studio/backend/core/inference/diffusion_prefix_kv.py
@@ -6,6 +6,7 @@ it for the render; a hook copies the cached tensors out after the prefill forwar
from __future__ import annotations
+import contextlib
import inspect
from typing import Any
@@ -53,6 +54,71 @@ def _returned_cache(output: Any) -> Any:
return cache
+_SEEN_ATTR = "_unsloth_prefix_kv_seen"
+_TRACK_HOOK_ATTR = "_unsloth_prefix_kv_track"
+
+
+def track_prefix_kv(transformer: Any) -> bool:
+ """Weakly record the denoiser's prefix KV caches: Qwen-Image-2.1 keeps them as ``__call__`` locals through
+ VAE decode (~2 GiB per guidance branch at 1 MP)."""
+ if transformer is None or getattr(transformer, _TRACK_HOOK_ATTR, None) is not None:
+ return False
+ if not _takes_prefix_kv(transformer):
+ return False
+ import weakref
+
+ seen: list = []
+
+ def _record(_module, _args, kwargs):
+ if kwargs.get("kv_cache_mode") != "extract":
+ return None
+ cache = kwargs.get("kv_cache")
+ if cache is None:
+ return None
+ with contextlib.suppress(TypeError):
+ if not any(ref() is cache for ref in seen):
+ seen[:] = [ref for ref in seen if ref() is not None]
+ seen.append(weakref.ref(cache))
+ return None
+
+ try:
+ handle = transformer.register_forward_pre_hook(_record, with_kwargs = True)
+ except Exception: # noqa: BLE001 - a memory saving only
+ return False
+ setattr(transformer, _SEEN_ATTR, seen)
+ setattr(transformer, _TRACK_HOOK_ATTR, handle)
+ return True
+
+
+def release_prefix_kv(transformer: Any) -> int:
+ """Set tracked caches' per-layer tensors to None; returns bytes freed. Only safe after the last denoiser call."""
+ import torch
+
+ seen = getattr(transformer, _SEEN_ATTR, None) if transformer is not None else None
+ if not seen:
+ return 0
+ released = 0
+ caches = [ref() for ref in seen]
+ seen.clear()
+ for cache in caches:
+ if cache is None:
+ continue
+ try:
+ groups = [v for v in vars(cache).values() if isinstance(v, (list, tuple))]
+ except TypeError:
+ continue
+ for group in groups:
+ for layer in group:
+ slots = getattr(layer, "__dict__", None)
+ if not slots:
+ continue
+ for name, tensor in list(slots.items()):
+ if isinstance(tensor, torch.Tensor):
+ released += tensor.untyped_storage().nbytes()
+ setattr(layer, name, None)
+ return released
+
+
def install_prefix_kv_compaction(transformer: Any, logger: Any = None) -> bool:
if getattr(transformer, _HOOK_ATTR, None) is not None or not _takes_prefix_kv(transformer):
return False
diff --git a/studio/backend/tests/test_diffusion_prefix_kv.py b/studio/backend/tests/test_diffusion_prefix_kv.py
index c04eff609f4..4edb13d6a00 100644
--- a/studio/backend/tests/test_diffusion_prefix_kv.py
+++ b/studio/backend/tests/test_diffusion_prefix_kv.py
@@ -16,6 +16,8 @@ torch = pytest.importorskip("torch")
from core.inference.diffusion_prefix_kv import ( # noqa: E402
compact_prefix_kv_cache,
install_prefix_kv_compaction,
+ release_prefix_kv,
+ track_prefix_kv,
)
@@ -233,3 +235,98 @@ def test_a_regional_compile_installs_the_hook(monkeypatch):
pipe.transformer(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")
k = cache.layer_caches[0].k
assert k.untyped_storage().nbytes() == k.numel() * k.element_size()
+
+
+def test_tracked_caches_are_emptied_on_release_and_only_once():
+ module = _PrefixKVDenoiser()
+ assert track_prefix_kv(module) is True
+ assert track_prefix_kv(module) is False
+ cond, neg = _Cache(2), _Cache(2)
+ x = torch.zeros(1)
+ for cache in (cond, neg):
+ module(x, kv_cache = cache, kv_cache_mode = "extract")
+ module(x, kv_cache = cond, kv_cache_mode = "cached")
+
+ released = release_prefix_kv(module)
+
+ assert released == 2 * 2 * 2 * 64 * 4 * 8 * 4
+ for cache in (cond, neg):
+ assert all(layer.k is None and layer.v is None for layer in cache.layer_caches)
+ assert release_prefix_kv(module) == 0
+
+
+def test_release_frees_the_cache_storage_and_does_not_keep_dead_caches_alive():
+ import gc
+ import weakref
+
+ module = _PrefixKVDenoiser()
+ track_prefix_kv(module)
+ cache = _Cache(1)
+ module(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")
+ k = weakref.ref(cache.layer_caches[0].k)
+ release_prefix_kv(module)
+ gc.collect()
+ assert k() is None
+
+ other = _Cache(1)
+ module(torch.zeros(1), kv_cache = other, kv_cache_mode = "extract")
+ ref = weakref.ref(other)
+ del other
+ gc.collect()
+ assert ref() is None
+ assert release_prefix_kv(module) == 0
+
+
+def test_tracking_skips_denoisers_without_a_prefix_cache():
+ class _Plain(torch.nn.Module):
+ def forward(self, hidden_states):
+ return hidden_states
+
+ assert track_prefix_kv(_Plain()) is False
+ assert track_prefix_kv(None) is False
+ assert release_prefix_kv(None) == 0
+
+
+def test_the_real_qwen_image_21_cache_is_released():
+ tq = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")
+
+ class _Q21Like(torch.nn.Module):
+ def forward(
+ self,
+ hidden_states,
+ kv_cache = None,
+ kv_cache_mode = None,
+ ):
+ if kv_cache_mode == "extract":
+ for index in range(2):
+ kv_cache.get_layer(index).store(
+ torch.randn(1, 3, 4, 8), torch.randn(1, 3, 4, 8)
+ )
+ return hidden_states
+
+ module = _Q21Like()
+ track_prefix_kv(module)
+ cache = tq.QwenImage21KVCache(2)
+ module(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")
+ assert release_prefix_kv(module) == 2 * 2 * 3 * 4 * 8 * 4
+ assert all(layer.k is None and layer.v is None for layer in cache.layer_caches)
+
+
+def test_the_render_helper_also_drops_a_cfg_parallel_replica_copy(monkeypatch):
+ from core.inference import diffusion as diffusion_mod
+ from core.inference import diffusion_multigpu as mg
+
+ pipe = type("P", (), {})()
+ pipe.transformer = _PrefixKVDenoiser()
+ track_prefix_kv(pipe.transformer)
+ cache = _Cache(1)
+ pipe.transformer(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")
+ router = types.SimpleNamespace(
+ kv_mirrors = {id(cache): (cache, [], None)}, kv_declined = {1: (cache, [])}
+ )
+ monkeypatch.setattr(mg, "_CFG_ROUTERS", {pipe: router})
+
+ diffusion_mod._release_prefix_kv(pipe)
+
+ assert cache.layer_caches[0].k is None
+ assert router.kv_mirrors == {} and router.kv_declined == {}