Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: reuse the MLX prompt cache on hybrid recurrent models (#10062)
Long Yixing68348bf90f16
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex a1580a58b4e..2d45532dda3 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -1018,11 +1018,20 @@ def _flatten_kv_entries(cache):def _kv_prefix_coverage(cache):+ """Tokens the whole cache holds, or None when no entry can attest to it.++ A recurrent entry has no offset: it holds fixed-size state, not a token+ sequence. An attention sibling attests for it, since unrewindable state only+ ever serves the prefix it was built from. Nothing attesting keeps mamba out.+ """covered = Nonefor entry in _flatten_kv_entries(cache):offset = getattr(entry, "offset", None)if offset is None:- return None+ # Upstream trims once all entries agree, so only unrewindable counts.+ if getattr(entry, "is_trimmable", lambda: False)():+ return None+ continueif getattr(entry, "start_position", 0):return Nonewindow = getattr(entry, "max_size", None)diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex fa2065a2279..880b29bc777 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -1597,6 +1597,72 @@ def test_mlx_prompt_cache_only_stores_verifiable_prefix_coverage(monkeypatch):assert tuple(range(30)) in history._lru.entries["key"]+def test_mlx_prompt_cache_covers_hybrid_recurrent_layouts(monkeypatch):+ mx = pytest.importorskip("mlx.core")+ from mlx_lm.models.cache import (+ ArraysCache,+ CacheList,+ KVCache,+ RotatingKVCache,+ can_trim_prompt_cache,+ )++ _install_fake_prompt_cache_api(monkeypatch)+ from core.inference.mlx_inference import _kv_prefix_coverage, _MLXPromptCacheHistory++ def attention(entry, n):+ for _ in range(n):+ block = mx.zeros((1, 2, 1, 4), dtype = mx.float16)+ entry.update_and_fetch(block, block)+ mx.eval(entry.state)+ return entry++ def recurrent():+ # A GatedDeltaNet layer: conv + recurrent state, neither grows with the prompt.+ entry = ArraysCache(size = 2)+ entry[0] = mx.zeros((1, 4, 4), dtype = mx.float16)+ entry[1] = mx.zeros((1, 2, 4, 4), dtype = mx.float16)+ mx.eval(entry.state)+ return entry++ assert getattr(recurrent(), "offset", None) is None++ # qwen3_5/qwen3_next: a full-attention layer every fourth layer.+ hybrid = [recurrent(), recurrent(), recurrent(), attention(KVCache(), 30)]+ assert _kv_prefix_coverage(hybrid) == 30+ # falcon_h1: both halves inside one CacheList.+ nested = [CacheList(recurrent(), attention(KVCache(), 30))]+ assert _kv_prefix_coverage(nested) == 30++ # Pin it: an mlx-lm that made these trimmable must fail here, not reuse stale state.+ assert can_trim_prompt_cache(hybrid) is False+ assert can_trim_prompt_cache(nested) is False++ class _TrimmableOpaqueState:+ def is_trimmable(self):+ return True++ assert _kv_prefix_coverage([_TrimmableOpaqueState(), attention(KVCache(), 30)]) is None++ # mamba/rwkv: nothing attests to a token count.+ assert _kv_prefix_coverage([recurrent(), recurrent()]) is None++ # A recurrent entry does not excuse an attention sibling that cannot attest.+ windowed = attention(RotatingKVCache(max_size = 10, keep = 2), 30)+ assert _kv_prefix_coverage([recurrent(), windowed]) is None+ assert (+ _kv_prefix_coverage([recurrent(), attention(KVCache(), 30), attention(KVCache(), 29)])+ is None+ )++ history = _MLXPromptCacheHistory(6, 1 << 40)+ history.insert("recurrent", list(range(40)), [recurrent(), recurrent()])+ assert "recurrent" not in history._lru.entries++ history.insert("hybrid", list(range(40)), hybrid)+ assert tuple(range(30)) in history._lru.entries["hybrid"]++# ── Tests: audio-input capability + generation ───────────────────────