Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: free the Qwen-Image-2.1 prefix KV cache before VAE decode (#15094)
Daniel Han91e388570ebd
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex dadb5f1bf8e..7f164b4ca42 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -124,9 +124,11 @@ from .diffusion_zimage_cfg_split import (zimage_cfg_split_status,)from .diffusion_zimage_cfg_split import render as zimage_cfg_split_render+from .diffusion_prefix_kv import release_prefix_kv, track_prefix_kvfrom .diffusion_multigpu import (AUTOMATIC_ORDINALS,end_cfg_render,+ release_cfg_prefix_kv,drop_displaced_cfg_router,install_cfg_parallel,multigpu_mode,@@ -1128,6 +1130,17 @@ def _is_source_sized(workflow: str, family: Any = None) -> bool:return workflow in ("img2img", "inpaint", "upscale")+def _release_prefix_kv(pipe: Any) -> None:+ """Drop the denoiser's prefix KV caches (and a CFG-parallel replica's copies) before VAE decode."""+ try:+ released = release_prefix_kv(getattr(pipe, "transformer", None))+ if released:+ release_cfg_prefix_kv(pipe)+ logger.debug("diffusion.prefix_kv: %d MiB released before decode", released >> 20)+ except Exception as exc: # noqa: BLE001 - a memory saving only+ logger.debug("diffusion.prefix_kv: release failed: %s", exc)++def _compile_shape_dims(workflow: str,init_pil: Any,@@ -13862,10 +13875,15 @@ class DiffusionBackend:if cancel.is_set():pipe._interrupt = Truecross_forward_step(pipe, step_index)+ # Last denoise step issued: the prefix KV cache (Qwen-Image-2.1) is dead weight through decode.+ num_steps = getattr(pipe, "_num_timesteps", None)+ if isinstance(num_steps, int) and step_index + 1 >= num_steps:+ _release_prefix_kv(pipe)return callback_kwargsif "callback_on_step_end" in call_params:kwargs["callback_on_step_end"] = _on_step+ track_prefix_kv(getattr(pipe, "transformer", None))# Hires fix latent upscaler (ComfyUI LatentUpscale): the base pass hands the refine pass its latents,# so it skips the VAE decode and the image post-processing that only the discarded pixels needed.latent_output = bool(diff --git a/studio/backend/core/inference/diffusion_multigpu.py b/studio/backend/core/inference/diffusion_multigpu.pyindex 57d774fbd68..d72e3ccc179 100644--- a/studio/backend/core/inference/diffusion_multigpu.py+++ b/studio/backend/core/inference/diffusion_multigpu.py@@ -3254,6 +3254,17 @@ def end_cfg_render(pipe: Any) -> None:router.end_render()+def release_cfg_prefix_kv(pipe: Any) -> None:+ """Drop the replica's prefix-KV copies (and their hold on the primary's tensors) after the last denoise step."""+ try:+ router = _CFG_ROUTERS.get(pipe) if pipe is not None else None+ except TypeError:+ return+ if router is not None:+ router.kv_mirrors.clear()+ router.kv_declined.clear()++def uninstall_cfg_parallel(pipe: Any) -> None:try:router = _CFG_ROUTERS.pop(pipe, None) if pipe is not None else Nonediff --git a/studio/backend/core/inference/diffusion_prefix_kv.py b/studio/backend/core/inference/diffusion_prefix_kv.pyindex 9a86123a735..4acb7eaef7a 100644--- a/studio/backend/core/inference/diffusion_prefix_kv.py+++ b/studio/backend/core/inference/diffusion_prefix_kv.py@@ -6,6 +6,7 @@ it for the render; a hook copies the cached tensors out after the prefill forwarfrom __future__ import annotations+import contextlibimport inspectfrom typing import Any@@ -53,6 +54,71 @@ def _returned_cache(output: Any) -> Any:return cache+_SEEN_ATTR = "_unsloth_prefix_kv_seen"+_TRACK_HOOK_ATTR = "_unsloth_prefix_kv_track"+++def track_prefix_kv(transformer: Any) -> bool:+ """Weakly record the denoiser's prefix KV caches: Qwen-Image-2.1 keeps them as ``__call__`` locals through+ VAE decode (~2 GiB per guidance branch at 1 MP)."""+ if transformer is None or getattr(transformer, _TRACK_HOOK_ATTR, None) is not None:+ return False+ if not _takes_prefix_kv(transformer):+ return False+ import weakref++ seen: list = []++ def _record(_module, _args, kwargs):+ if kwargs.get("kv_cache_mode") != "extract":+ return None+ cache = kwargs.get("kv_cache")+ if cache is None:+ return None+ with contextlib.suppress(TypeError):+ if not any(ref() is cache for ref in seen):+ seen[:] = [ref for ref in seen if ref() is not None]+ seen.append(weakref.ref(cache))+ return None++ try:+ handle = transformer.register_forward_pre_hook(_record, with_kwargs = True)+ except Exception: # noqa: BLE001 - a memory saving only+ return False+ setattr(transformer, _SEEN_ATTR, seen)+ setattr(transformer, _TRACK_HOOK_ATTR, handle)+ return True+++def release_prefix_kv(transformer: Any) -> int:+ """Set tracked caches' per-layer tensors to None; returns bytes freed. Only safe after the last denoiser call."""+ import torch++ seen = getattr(transformer, _SEEN_ATTR, None) if transformer is not None else None+ if not seen:+ return 0+ released = 0+ caches = [ref() for ref in seen]+ seen.clear()+ for cache in caches:+ if cache is None:+ continue+ try:+ groups = [v for v in vars(cache).values() if isinstance(v, (list, tuple))]+ except TypeError:+ continue+ for group in groups:+ for layer in group:+ slots = getattr(layer, "__dict__", None)+ if not slots:+ continue+ for name, tensor in list(slots.items()):+ if isinstance(tensor, torch.Tensor):+ released += tensor.untyped_storage().nbytes()+ setattr(layer, name, None)+ return released++def install_prefix_kv_compaction(transformer: Any, logger: Any = None) -> bool:if getattr(transformer, _HOOK_ATTR, None) is not None or not _takes_prefix_kv(transformer):return Falsediff --git a/studio/backend/tests/test_diffusion_prefix_kv.py b/studio/backend/tests/test_diffusion_prefix_kv.pyindex c04eff609f4..4edb13d6a00 100644--- a/studio/backend/tests/test_diffusion_prefix_kv.py+++ b/studio/backend/tests/test_diffusion_prefix_kv.py@@ -16,6 +16,8 @@ torch = pytest.importorskip("torch")from core.inference.diffusion_prefix_kv import ( # noqa: E402compact_prefix_kv_cache,install_prefix_kv_compaction,+ release_prefix_kv,+ track_prefix_kv,)@@ -233,3 +235,98 @@ def test_a_regional_compile_installs_the_hook(monkeypatch):pipe.transformer(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")k = cache.layer_caches[0].kassert k.untyped_storage().nbytes() == k.numel() * k.element_size()+++def test_tracked_caches_are_emptied_on_release_and_only_once():+ module = _PrefixKVDenoiser()+ assert track_prefix_kv(module) is True+ assert track_prefix_kv(module) is False+ cond, neg = _Cache(2), _Cache(2)+ x = torch.zeros(1)+ for cache in (cond, neg):+ module(x, kv_cache = cache, kv_cache_mode = "extract")+ module(x, kv_cache = cond, kv_cache_mode = "cached")++ released = release_prefix_kv(module)++ assert released == 2 * 2 * 2 * 64 * 4 * 8 * 4+ for cache in (cond, neg):+ assert all(layer.k is None and layer.v is None for layer in cache.layer_caches)+ assert release_prefix_kv(module) == 0+++def test_release_frees_the_cache_storage_and_does_not_keep_dead_caches_alive():+ import gc+ import weakref++ module = _PrefixKVDenoiser()+ track_prefix_kv(module)+ cache = _Cache(1)+ module(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")+ k = weakref.ref(cache.layer_caches[0].k)+ release_prefix_kv(module)+ gc.collect()+ assert k() is None++ other = _Cache(1)+ module(torch.zeros(1), kv_cache = other, kv_cache_mode = "extract")+ ref = weakref.ref(other)+ del other+ gc.collect()+ assert ref() is None+ assert release_prefix_kv(module) == 0+++def test_tracking_skips_denoisers_without_a_prefix_cache():+ class _Plain(torch.nn.Module):+ def forward(self, hidden_states):+ return hidden_states++ assert track_prefix_kv(_Plain()) is False+ assert track_prefix_kv(None) is False+ assert release_prefix_kv(None) == 0+++def test_the_real_qwen_image_21_cache_is_released():+ tq = pytest.importorskip("diffusers.models.transformers.transformer_qwenimage21")++ class _Q21Like(torch.nn.Module):+ def forward(+ self,+ hidden_states,+ kv_cache = None,+ kv_cache_mode = None,+ ):+ if kv_cache_mode == "extract":+ for index in range(2):+ kv_cache.get_layer(index).store(+ torch.randn(1, 3, 4, 8), torch.randn(1, 3, 4, 8)+ )+ return hidden_states++ module = _Q21Like()+ track_prefix_kv(module)+ cache = tq.QwenImage21KVCache(2)+ module(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")+ assert release_prefix_kv(module) == 2 * 2 * 3 * 4 * 8 * 4+ assert all(layer.k is None and layer.v is None for layer in cache.layer_caches)+++def test_the_render_helper_also_drops_a_cfg_parallel_replica_copy(monkeypatch):+ from core.inference import diffusion as diffusion_mod+ from core.inference import diffusion_multigpu as mg++ pipe = type("P", (), {})()+ pipe.transformer = _PrefixKVDenoiser()+ track_prefix_kv(pipe.transformer)+ cache = _Cache(1)+ pipe.transformer(torch.zeros(1), kv_cache = cache, kv_cache_mode = "extract")+ router = types.SimpleNamespace(+ kv_mirrors = {id(cache): (cache, [], None)}, kv_declined = {1: (cache, [])}+ )+ monkeypatch.setattr(mg, "_CFG_ROUTERS", {pipe: router})++ diffusion_mod._release_prefix_kv(pipe)++ assert cache.layer_caches[0].k is None+ assert router.kv_mirrors == {} and router.kv_declined == {}