Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: stop long MLX chats from re-reading the whole chat every turn (#14574)
Nilaya981fa88e2d7
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex ef35446d0bd..aa1ec9f97a1 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -2991,8 +2991,35 @@ class _MLXPromptCacheHistory:self._trim = trimself._max_bytes = max_bytesself._lru = lru_cls(max_size = max_entries, max_bytes = max_bytes)+ self._oversized = None++ def _take_oversized(self, key, tokens):+ held, self._oversized = self._oversized, None+ if held is None:+ return None+ held_key, held_tokens, cache, nbytes = held+ if held_key != key:+ return None+ common = next(+ (i for i, (a, b) in enumerate(zip(held_tokens, tokens)) if a != b),+ min(len(held_tokens), len(tokens)),+ )+ prefix = min(common, len(tokens) - 1)+ # A prefix within budget is the LRU's to serve, so chats under it behave as before.+ if prefix * nbytes <= self._max_bytes * len(held_tokens):+ return None+ excess = len(held_tokens) - prefix+ if excess and not (self._can_trim(cache) and self._trim(cache, excess) == excess):+ return None+ return cache, list(tokens[prefix:])++ def release_oversized(self):+ self._oversized = Nonedef fetch(self, model, key, tokens):+ held = self._take_oversized(key, tokens)+ if held is not None:+ return heldcache, rest = self._lru.fetch_nearest_cache(key, list(tokens))if cache is not None:if rest:@@ -3012,19 +3039,11 @@ class _MLXPromptCacheHistory:return cache if self._prepare is None else self._prepare(cache), list(tokens)def insert(self, key, tokens, cache):- # An over-budget entry evicts itself and every other conversation.sizes = [_kv_entry_nbytes(entry) for entry in cache]if any(size is None for size in sizes):logger.debug("MLX prompt cache: skipping state whose size is unmeasurable")returnnbytes = sum(sizes)- if nbytes > self._max_bytes:- logger.debug(- "MLX prompt cache: skipping %.2f GB entry over the %.2f GB budget",- nbytes / 1e9,- self._max_bytes / 1e9,- )- returncovered = _kv_prefix_coverage(cache)if covered is None:logger.debug("MLX prompt cache: skipping cache with unverifiable prefix coverage")@@ -3040,6 +3059,15 @@ class _MLXPromptCacheHistory:tokens = tokens[:covered]if not tokens:return+ if nbytes > self._max_bytes:+ # In the LRU it would evict itself and every other conversation.+ logger.debug(+ "MLX prompt cache: holding %.2f GB entry over the %.2f GB budget for its next turn",+ nbytes / 1e9,+ self._max_bytes / 1e9,+ )+ self._oversized = (key, tokens, cache, nbytes)+ returnself._lru.insert_cache(key, tokens, cache)@@ -3873,6 +3901,7 @@ class _TextBatchSession:if row.ready_at is None:row.ready_at = nowyield from self._feed(row, event)+ self._release_held_cache_beside_live_rows()def _feed(self, row, event):stream = row.plan.stream@@ -3923,6 +3952,14 @@ class _TextBatchSession:row.plan.stream,)yield from self._retire(row, cancelled = True)+ self._release_held_cache_beside_live_rows()++ def _release_held_cache_beside_live_rows(self):+ # Decided once every row that ended together is retired: holding it beside a row still+ # decoding would raise peak memory.+ history = self.backend._prompt_cache_history+ if self._rows and history is not None:+ history.release_oversized()def _retire(self, row, *, cancelled):self._rows.pop(row.handle, None)diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 1a434cd77ff..a1438b5dd58 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -3170,7 +3170,7 @@ def test_mlx_prompt_cache_survives_reset_but_not_unload(monkeypatch):assert backend._prompt_cache_history is None-def test_mlx_prompt_cache_skips_entries_over_budget(monkeypatch):+def test_mlx_prompt_cache_keeps_entries_over_budget_out_of_the_lru(monkeypatch):from core.inference.mlx_inference import _MLXPromptCacheHistory_install_fake_prompt_cache_api(monkeypatch)@@ -3185,6 +3185,132 @@ def test_mlx_prompt_cache_skips_entries_over_budget(monkeypatch):assert tuple(range(50)) not in stored+def test_mlx_prompt_cache_keeps_reusing_a_conversation_over_budget(monkeypatch):+ from core.inference.mlx_inference import _MLXPromptCacheHistory++ _install_fake_prompt_cache_api(monkeypatch)++ history = _MLXPromptCacheHistory(6, 1000)+ turn = list(range(50))+ history.insert("key", turn[:3], [_FakeCacheEntry(offset = 3, nbytes = 400)])+ long_cache = [_FakeCacheEntry(offset = 50, nbytes = 5000)]+ history.insert("key", turn, long_cache)+ assert tuple(turn) not in history._lru.entries["key"]++ cache, rest = history.fetch(object(), "key", turn + [50, 51])+ assert cache is long_cache+ assert rest == [50, 51]++ cache[0].offset = 52+ history.insert("key", turn + [50, 51], cache)+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])+ assert cache is long_cache+ assert cache[0].offset == 45+ assert rest == [99]++ history.insert("key", turn[:45], cache)+ cache, rest = history.fetch(object(), "key", turn[:5] + [99])+ assert cache is not long_cache+ assert rest == turn[3:5] + [99]+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])+ assert cache is not long_cache+ assert rest == turn[3:45] + [99]++ history.insert("key", turn[:45], long_cache)+ cache, rest = history.fetch(object(), "key", turn[:45])+ assert cache is long_cache+ assert (cache[0].offset, rest) == (44, turn[44:45])++ history.insert("key", turn[:44], cache)+ cache, rest = history.fetch(object(), "other-adapter", turn[:44] + [99])+ assert cache is not long_cache+ assert rest == turn[:44] + [99]+++def test_mlx_prompt_cache_never_serves_an_over_budget_cache_it_cannot_rewind(monkeypatch):+ from core.inference.mlx_inference import _MLXPromptCacheHistory++ _install_fake_prompt_cache_api(monkeypatch, trimmable = False)+ history = _MLXPromptCacheHistory(6, 1000)+ turn = list(range(50))+ long_cache = [_FakeCacheEntry(offset = 50, nbytes = 5000)]++ history.insert("key", turn, long_cache)+ cache, rest = history.fetch(object(), "key", turn + [50])+ assert (cache, rest) == (long_cache, [50])++ history.insert("key", turn, long_cache)+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])+ assert cache is not long_cache+ assert rest == turn[:45] + [99]++ _install_fake_prompt_cache_api(monkeypatch)+ history = _MLXPromptCacheHistory(6, 1000)+ history._trim = lambda cache, num: num - 1+ history.insert("key", turn, long_cache)+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])+ assert cache is not long_cache+++def test_mlx_batch_releases_an_over_budget_cache_while_another_row_decodes(monkeypatch):+ from core.inference.mlx_inference import _MLXPromptCacheHistory, _TextBatchSession++ _install_fake_prompt_cache_api(monkeypatch)++ history = _MLXPromptCacheHistory(6, 1000)+ turn = list(range(50))+ long_cache = [_FakeCacheEntry(offset = 50, nbytes = 5000)]+ banked = {"long": (turn, long_cache), "short": ([7, 8, 9], [_FakeCacheEntry(offset = 3)])}+ session = object.__new__(_TextBatchSession)+ session.backend = SimpleNamespace(+ _prompt_cache_history = history,+ _bank_row_cache = lambda key, *_: history.insert("key", *banked[key]),+ )+ session._settled = {}+ stream = SimpleNamespace(settle = lambda: [], drain = lambda cancelled: [])++ def feed(row, event):+ if event.finish_reason is not None:+ session.backend._bank_row_cache(row.cache_key)+ yield from session._retire(row, cancelled = False)++ session._feed = feed++ def held_after(ended, withdraw = False):+ rows = {+ handle: SimpleNamespace(+ handle = handle,+ uid = handle,+ cache_key = handle,+ all_tokens = [],+ plan = SimpleNamespace(stream = stream),+ ready_at = None,+ processing_at = None,+ prompt_len = 1,+ generated = 1,+ cached = 0,+ reason = "stop",+ )+ for handle in banked+ }+ session._rows, session._by_uid = rows, dict(rows)+ events = [+ SimpleNamespace(uid = handle, finish_reason = "stop" if handle in ended else None)+ for handle in banked+ ]+ session.generator = SimpleNamespace(+ next = lambda: ([], events),+ remove = lambda uids, return_prompt_caches: {},+ )+ list(session.withdraw(ended) if withdraw else session.step())+ return history.fetch(object(), "key", turn + [50])[0] is long_cache++ assert not held_after(["long"])+ assert held_after(["long", "short"])+ assert not held_after(["long"], withdraw = True)+ assert held_after(["long", "short"], withdraw = True)++def test_mlx_prompt_cache_keys_on_what_the_kv_covers(monkeypatch):from core.inference.mlx_inference import _MLXPromptCacheHistory