Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: stop long MLX chats from re-reading the whole chat every turn (#14574)

Nilaya981fa88e2d7
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index ef35446d0bd..aa1ec9f97a1 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -2991,8 +2991,35 @@ class _MLXPromptCacheHistory:
self._trim = trim
self._max_bytes = max_bytes
self._lru = lru_cls(max_size = max_entries, max_bytes = max_bytes)
+ self._oversized = None
+
+ def _take_oversized(self, key, tokens):
+ held, self._oversized = self._oversized, None
+ if held is None:
+ return None
+ held_key, held_tokens, cache, nbytes = held
+ if held_key != key:
+ return None
+ common = next(
+ (i for i, (a, b) in enumerate(zip(held_tokens, tokens)) if a != b),
+ min(len(held_tokens), len(tokens)),
+ )
+ prefix = min(common, len(tokens) - 1)
+ # A prefix within budget is the LRU's to serve, so chats under it behave as before.
+ if prefix * nbytes <= self._max_bytes * len(held_tokens):
+ return None
+ excess = len(held_tokens) - prefix
+ if excess and not (self._can_trim(cache) and self._trim(cache, excess) == excess):
+ return None
+ return cache, list(tokens[prefix:])
+
+ def release_oversized(self):
+ self._oversized = None
def fetch(self, model, key, tokens):
+ held = self._take_oversized(key, tokens)
+ if held is not None:
+ return held
cache, rest = self._lru.fetch_nearest_cache(key, list(tokens))
if cache is not None:
if rest:
@@ -3012,19 +3039,11 @@ class _MLXPromptCacheHistory:
return cache if self._prepare is None else self._prepare(cache), list(tokens)
def insert(self, key, tokens, cache):
- # An over-budget entry evicts itself and every other conversation.
sizes = [_kv_entry_nbytes(entry) for entry in cache]
if any(size is None for size in sizes):
logger.debug("MLX prompt cache: skipping state whose size is unmeasurable")
return
nbytes = sum(sizes)
- if nbytes > self._max_bytes:
- logger.debug(
- "MLX prompt cache: skipping %.2f GB entry over the %.2f GB budget",
- nbytes / 1e9,
- self._max_bytes / 1e9,
- )
- return
covered = _kv_prefix_coverage(cache)
if covered is None:
logger.debug("MLX prompt cache: skipping cache with unverifiable prefix coverage")
@@ -3040,6 +3059,15 @@ class _MLXPromptCacheHistory:
tokens = tokens[:covered]
if not tokens:
return
+ if nbytes > self._max_bytes:
+ # In the LRU it would evict itself and every other conversation.
+ logger.debug(
+ "MLX prompt cache: holding %.2f GB entry over the %.2f GB budget for its next turn",
+ nbytes / 1e9,
+ self._max_bytes / 1e9,
+ )
+ self._oversized = (key, tokens, cache, nbytes)
+ return
self._lru.insert_cache(key, tokens, cache)
@@ -3873,6 +3901,7 @@ class _TextBatchSession:
if row.ready_at is None:
row.ready_at = now
yield from self._feed(row, event)
+ self._release_held_cache_beside_live_rows()
def _feed(self, row, event):
stream = row.plan.stream
@@ -3923,6 +3952,14 @@ class _TextBatchSession:
row.plan.stream,
)
yield from self._retire(row, cancelled = True)
+ self._release_held_cache_beside_live_rows()
+
+ def _release_held_cache_beside_live_rows(self):
+ # Decided once every row that ended together is retired: holding it beside a row still
+ # decoding would raise peak memory.
+ history = self.backend._prompt_cache_history
+ if self._rows and history is not None:
+ history.release_oversized()
def _retire(self, row, *, cancelled):
self._rows.pop(row.handle, None)
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 1a434cd77ff..a1438b5dd58 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -3170,7 +3170,7 @@ def test_mlx_prompt_cache_survives_reset_but_not_unload(monkeypatch):
assert backend._prompt_cache_history is None
-def test_mlx_prompt_cache_skips_entries_over_budget(monkeypatch):
+def test_mlx_prompt_cache_keeps_entries_over_budget_out_of_the_lru(monkeypatch):
from core.inference.mlx_inference import _MLXPromptCacheHistory
_install_fake_prompt_cache_api(monkeypatch)
@@ -3185,6 +3185,132 @@ def test_mlx_prompt_cache_skips_entries_over_budget(monkeypatch):
assert tuple(range(50)) not in stored
+def test_mlx_prompt_cache_keeps_reusing_a_conversation_over_budget(monkeypatch):
+ from core.inference.mlx_inference import _MLXPromptCacheHistory
+
+ _install_fake_prompt_cache_api(monkeypatch)
+
+ history = _MLXPromptCacheHistory(6, 1000)
+ turn = list(range(50))
+ history.insert("key", turn[:3], [_FakeCacheEntry(offset = 3, nbytes = 400)])
+ long_cache = [_FakeCacheEntry(offset = 50, nbytes = 5000)]
+ history.insert("key", turn, long_cache)
+ assert tuple(turn) not in history._lru.entries["key"]
+
+ cache, rest = history.fetch(object(), "key", turn + [50, 51])
+ assert cache is long_cache
+ assert rest == [50, 51]
+
+ cache[0].offset = 52
+ history.insert("key", turn + [50, 51], cache)
+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])
+ assert cache is long_cache
+ assert cache[0].offset == 45
+ assert rest == [99]
+
+ history.insert("key", turn[:45], cache)
+ cache, rest = history.fetch(object(), "key", turn[:5] + [99])
+ assert cache is not long_cache
+ assert rest == turn[3:5] + [99]
+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])
+ assert cache is not long_cache
+ assert rest == turn[3:45] + [99]
+
+ history.insert("key", turn[:45], long_cache)
+ cache, rest = history.fetch(object(), "key", turn[:45])
+ assert cache is long_cache
+ assert (cache[0].offset, rest) == (44, turn[44:45])
+
+ history.insert("key", turn[:44], cache)
+ cache, rest = history.fetch(object(), "other-adapter", turn[:44] + [99])
+ assert cache is not long_cache
+ assert rest == turn[:44] + [99]
+
+
+def test_mlx_prompt_cache_never_serves_an_over_budget_cache_it_cannot_rewind(monkeypatch):
+ from core.inference.mlx_inference import _MLXPromptCacheHistory
+
+ _install_fake_prompt_cache_api(monkeypatch, trimmable = False)
+ history = _MLXPromptCacheHistory(6, 1000)
+ turn = list(range(50))
+ long_cache = [_FakeCacheEntry(offset = 50, nbytes = 5000)]
+
+ history.insert("key", turn, long_cache)
+ cache, rest = history.fetch(object(), "key", turn + [50])
+ assert (cache, rest) == (long_cache, [50])
+
+ history.insert("key", turn, long_cache)
+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])
+ assert cache is not long_cache
+ assert rest == turn[:45] + [99]
+
+ _install_fake_prompt_cache_api(monkeypatch)
+ history = _MLXPromptCacheHistory(6, 1000)
+ history._trim = lambda cache, num: num - 1
+ history.insert("key", turn, long_cache)
+ cache, rest = history.fetch(object(), "key", turn[:45] + [99])
+ assert cache is not long_cache
+
+
+def test_mlx_batch_releases_an_over_budget_cache_while_another_row_decodes(monkeypatch):
+ from core.inference.mlx_inference import _MLXPromptCacheHistory, _TextBatchSession
+
+ _install_fake_prompt_cache_api(monkeypatch)
+
+ history = _MLXPromptCacheHistory(6, 1000)
+ turn = list(range(50))
+ long_cache = [_FakeCacheEntry(offset = 50, nbytes = 5000)]
+ banked = {"long": (turn, long_cache), "short": ([7, 8, 9], [_FakeCacheEntry(offset = 3)])}
+ session = object.__new__(_TextBatchSession)
+ session.backend = SimpleNamespace(
+ _prompt_cache_history = history,
+ _bank_row_cache = lambda key, *_: history.insert("key", *banked[key]),
+ )
+ session._settled = {}
+ stream = SimpleNamespace(settle = lambda: [], drain = lambda cancelled: [])
+
+ def feed(row, event):
+ if event.finish_reason is not None:
+ session.backend._bank_row_cache(row.cache_key)
+ yield from session._retire(row, cancelled = False)
+
+ session._feed = feed
+
+ def held_after(ended, withdraw = False):
+ rows = {
+ handle: SimpleNamespace(
+ handle = handle,
+ uid = handle,
+ cache_key = handle,
+ all_tokens = [],
+ plan = SimpleNamespace(stream = stream),
+ ready_at = None,
+ processing_at = None,
+ prompt_len = 1,
+ generated = 1,
+ cached = 0,
+ reason = "stop",
+ )
+ for handle in banked
+ }
+ session._rows, session._by_uid = rows, dict(rows)
+ events = [
+ SimpleNamespace(uid = handle, finish_reason = "stop" if handle in ended else None)
+ for handle in banked
+ ]
+ session.generator = SimpleNamespace(
+ next = lambda: ([], events),
+ remove = lambda uids, return_prompt_caches: {},
+ )
+ list(session.withdraw(ended) if withdraw else session.step())
+ return history.fetch(object(), "key", turn + [50])[0] is long_cache
+
+ assert not held_after(["long"])
+ assert held_after(["long", "short"])
+ assert not held_after(["long"], withdraw = True)
+ assert held_after(["long", "short"], withdraw = True)
+
+
def test_mlx_prompt_cache_keys_on_what_the_kv_covers(monkeypatch):
from core.inference.mlx_inference import _MLXPromptCacheHistory