Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: let an explicit FBCache request engage on Qwen-Image-2.1 and other prefix-KV models (#11713)
Daniel Han2be14e0ce4cf
diff --git a/studio/backend/core/inference/diffusion.py b/studio/backend/core/inference/diffusion.pyindex ba6ad5e9062..8ceb290d2a4 100644--- a/studio/backend/core/inference/diffusion.py+++ b/studio/backend/core/inference/diffusion.py@@ -5510,6 +5510,9 @@ class DiffusionBackend:threshold = transformer_cache_threshold,# GGUF transformers are quantized too, so the cache needs the higher threshold.quant_active = cache_quant_active,+ # Prefix-KV families (Qwen-Image-2.1) cache only when asked: at 40 steps the default+ # threshold skips about half the steps, too lossy for an automatic default.+ length_changes_ok = not cache_auto,logger = logger,)self._raise_if_load_cancelled(_load_token)@@ -5530,6 +5533,8 @@ class DiffusionBackend:f"auto: {default_steps}-step default schedule is below "f"{FBCACHE_MIN_STEPS}; re-checked per generation")+ elif cache_request is not None and not cache_engaged:+ cache_reason = "requested, but this model does not support step caching; running uncached"else:cache_reason = "requested"# The dense fast path sets gguf_filename, but its transformer is dense.diff --git a/studio/backend/core/inference/diffusion_cache.py b/studio/backend/core/inference/diffusion_cache.pyindex 902665e81c2..4e6f6504ce9 100644--- a/studio/backend/core/inference/diffusion_cache.py+++ b/studio/backend/core/inference/diffusion_cache.py@@ -301,6 +301,46 @@ def _pipeline_opens_cache_context(pipe: Any) -> bool:return "cache_context(" in src+def install_fbcache_length_guard() -> bool:+ """Make First-Block-Cache recompute, instead of raise, when the block sequence length changes.++ FBCache decides per step by subtracting the previous step's head-block residual from this one's.+ A prefix-KV transformer (Qwen-Image-2.1) runs step 0 over prompt + target tokens and every later+ step over the target alone, so that subtraction raises on step 1. A length change means the+ stored residuals describe a different sequence, so the only correct answer is "compute": the+ full pass then stores residuals at the new length and later steps cache normally. Same-length+ calls take the original path unchanged. Process-wide and idempotent; False when this diffusers+ has no FBCache head hook to guard."""+ try:+ from diffusers.hooks.first_block_cache import FBCHeadBlockHook+ except Exception: # noqa: BLE001 - no FBCache in this diffusers+ return False+ original = getattr(FBCHeadBlockHook, "_should_compute_remaining_blocks", None)+ if original is None:+ return False+ if getattr(original, "_unsloth_length_guard", False):+ return True+ import torch++ @torch.compiler.disable+ def _should_compute_remaining_blocks(self, hidden_states_residual):+ state = self.state_manager.get_state()+ previous = state.head_block_residual+ if previous is not None and previous.shape != hidden_states_residual.shape:+ return True+ tail = state.tail_block_residuals+ if tail is not None and getattr(tail[0], "shape", None) not in (+ None,+ hidden_states_residual.shape,+ ):+ return True+ return original(self, hidden_states_residual)++ _should_compute_remaining_blocks._unsloth_length_guard = True+ FBCHeadBlockHook._should_compute_remaining_blocks = _should_compute_remaining_blocks+ return True++def _reuses_prefix_kv(pipe: Any, transformer: Any) -> bool:"""Whether the denoise loop feeds the blocks a SHORTER sequence after the first step.@@ -353,11 +393,14 @@ def apply_step_cache(mode: Optional[str],threshold: Optional[float] = None,quant_active: bool = False,+ length_changes_ok: bool = False,logger: Any = None,) -> Optional[str]:"""Engage step caching on ``pipe.transformer``. Returns the mode engaged, or None whendisabled / unsupported (runs uncached). ``threshold`` overrides the default; ``quant_active``- raises it so the cache triggers on a quantised transformer. Best-effort."""+ raises it so the cache triggers on a quantised transformer. ``length_changes_ok`` lets a+ prefix-KV transformer engage through the length guard; only an explicit request sets it, since+ those families skip far more steps at the default threshold. Best-effort."""mode = normalize_transformer_cache(mode)if mode is None or mode == TC_AUTO:# AUTO is resolved by the loader before this; treat a stray auto as off.@@ -387,8 +430,11 @@ def apply_step_cache(return None# Prefix KV reuse shortens the block sequence after the first step, which the cache's stored# residuals cannot be subtracted from. Checked before enable_cache: engaging here does not fail- # at load, it fails at step 2 of the user's generation.- if _reuses_prefix_kv(pipe, transformer):+ # at load, it fails at step 2 of the user's generation. On an explicit request the length guard+ # makes the head block recompute whenever the length moved, so the cache engages; otherwise refuse.+ if _reuses_prefix_kv(pipe, transformer) and not (+ length_changes_ok and install_fbcache_length_guard()+ ):_warn(logger,mode,diff --git a/studio/backend/tests/test_diffusion_cache.py b/studio/backend/tests/test_diffusion_cache.pyindex 7227bcccae1..afdc408f5c3 100644--- a/studio/backend/tests/test_diffusion_cache.py+++ b/studio/backend/tests/test_diffusion_cache.py@@ -16,6 +16,8 @@ import typesimport pytest+import core.inference.diffusion_cache as dc+from core.inference.diffusion_cache import (DEFAULT_FBCACHE_THRESHOLD,QUANT_FBCACHE_THRESHOLD,@@ -221,17 +223,59 @@ def test_pipeline_without_cache_context_runs_uncached(monkeypatch):assert t.enabled_with is None # enable_cache was never called-def test_prefix_kv_transformer_runs_uncached(monkeypatch):+def test_prefix_kv_transformer_runs_uncached_without_the_length_guard(monkeypatch):# Qwen-Image-2.1 / FLUX.2 klein KV / Wan-Animate-2 cache the prompt prefix, so the blocks see# the whole joint sequence on step 0 and the target tokens alone from step 1. FBCache subtracts# the stored residual elementwise, so engaging here raises "The size of tensor a (4096) must# match the size of tensor b (4297)" at step 2 of a user's generation, not at load._stub_diffusers(monkeypatch)+ monkeypatch.setattr(dc, "install_fbcache_length_guard", lambda: False)t = _PrefixKVTransformer()- assert apply_step_cache(_PrefixKVPipe(t), mode = "fbcache") is None+ assert apply_step_cache(_PrefixKVPipe(t), mode = "fbcache", length_changes_ok = True) is Noneassert t.enabled_with is None # enable_cache was never called+def test_prefix_kv_transformer_is_cached_once_the_length_guard_is_in(monkeypatch):+ _stub_diffusers(monkeypatch)+ monkeypatch.setattr(dc, "install_fbcache_length_guard", lambda: True)+ t = _PrefixKVTransformer()+ # An automatic decision (the default) still refuses; only an explicit request opts in.+ assert apply_step_cache(_PrefixKVPipe(t), mode = "fbcache") is None+ assert t.enabled_with is None+ assert apply_step_cache(_PrefixKVPipe(t), mode = "fbcache", length_changes_ok = True) == "fbcache"+ assert t.enabled_with is not None+++def test_length_guard_recomputes_on_a_length_change_and_defers_otherwise(monkeypatch):+ torch = pytest.importorskip("torch")+ fbc = pytest.importorskip("diffusers.hooks.first_block_cache")+ monkeypatch.setattr(+ fbc.FBCHeadBlockHook,+ "_should_compute_remaining_blocks",+ fbc.FBCHeadBlockHook._should_compute_remaining_blocks,+ )+ assert dc.install_fbcache_length_guard() is True+ guarded = fbc.FBCHeadBlockHook._should_compute_remaining_blocks+ assert dc.install_fbcache_length_guard() is True # idempotent: no second wrapper+ assert fbc.FBCHeadBlockHook._should_compute_remaining_blocks is guarded++ state = types.SimpleNamespace(+ head_block_residual = torch.ones(1, 4297, 8), tail_block_residuals = None+ )+ hook = fbc.FBCHeadBlockHook.__new__(fbc.FBCHeadBlockHook)+ hook.state_manager = types.SimpleNamespace(get_state = lambda: state)+ hook.threshold = 0.1+ # Step 1 of a prefix-KV render: target tokens only, so the stored residual no longer lines up.+ assert hook._should_compute_remaining_blocks(torch.ones(1, 4096, 8)) is True+ # Same length: the original relative-change test decides (identical residual -> reuse).+ state.head_block_residual = torch.ones(1, 4096, 8)+ assert hook._should_compute_remaining_blocks(torch.ones(1, 4096, 8)) is False+ assert hook._should_compute_remaining_blocks(torch.full((1, 4096, 8), 2.0)) is True+ # A tail residual stored at another length can never be added back, so that also recomputes.+ state.tail_block_residuals = (torch.ones(1, 4297, 8), None)+ assert hook._should_compute_remaining_blocks(torch.ones(1, 4096, 8)) is True++def test_prefix_kv_guard_does_not_catch_an_ordinary_transformer(monkeypatch):# The control for the guard above: a CacheMixin transformer whose forward takes no# kv_cache_mode (Flux, Qwen-Image) keeps a constant sequence length and must still be cached.