Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(studio): keep a long MLX vision chat's prompt snapshot past the cache budget (#16079)

Long Yixing80a660ae9dfd
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index 1c9d7cadcdb..05219dc6f10 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -7,6 +7,7 @@ import functools
import hashlib
import importlib
import inspect
+import math
import os
import re
import shutil
@@ -454,6 +455,47 @@ def share_kv_rows(entries, base):
return shared
+def kv_skeleton(entries):
+ """``entries`` with in-order KV rows left out (``offset`` kept) and recurrent state copied;
+ None for any other entry (quantized, wrapped)."""
+ import mlx.core as mx
+
+ classes = (_cache_classes("KVCache"), _cache_classes("RotatingKVCache"))
+ recurrent = _cache_classes("ArraysCache")
+ skeleton = []
+ for entry in entries:
+ if type(entry) in classes[0] + classes[1] and _rows_in_order(entry, classes):
+ entry = copy.copy(entry)
+ entry.keys = entry.values = None
+ elif isinstance(entry, recurrent):
+ entry = _copy_value(entry, mx)
+ else:
+ return None
+ skeleton.append(entry)
+ mx.eval(list(_arrays(skeleton, mx)))
+ return skeleton
+
+
+def fill_kv_skeleton(skeleton, final):
+ """Point ``skeleton``'s KV entries at the leading rows of the same row's ``final`` cache;
+ False when ``final`` does not hold them from row 0 (a wrapped ring)."""
+ kv = _cache_classes("KVCache") + _cache_classes("RotatingKVCache")
+ if len(final) != len(skeleton):
+ return False
+ pairs = [(entry, source) for entry, source in zip(skeleton, final) if type(entry) in kv]
+ if not all(
+ type(source) is type(entry)
+ and source.keys is not None
+ and getattr(source, "_idx", source.offset) == source.offset >= entry.offset
+ for entry, source in pairs
+ ):
+ return False
+ for entry, source in pairs:
+ entry.keys = source.keys[..., : entry.offset, :]
+ entry.values = source.values[..., : entry.offset, :]
+ return True
+
+
def compact_sliding_windows(entries):
"""Drop ring rows past the window after a multi-row update; no later update reads them."""
rings = _cache_classes("RotatingKVCache")
@@ -478,12 +520,16 @@ def compact_sliding_windows(entries):
class VLMPromptSnapshotStore:
"""LRU boundary snapshots under a byte budget; a shorter prefix reads KV rows from its base.
- A replay (one per key) serves only a prompt of that exact length and reads through no base."""
+ A replay (one per key) serves only a prompt of that exact length and reads through no base.
+
+ Those over the budget are held for one chat's next lookup only; a boundary that serves is
+ handed over uncopied, since that request banks successors a kept copy would sit beside."""
def __init__(
self,
max_bytes,
max_entries = VLM_PROMPT_CACHE_ENTRIES,
+ holds = True,
):
self._max_bytes = max_bytes
self._max_entries = max_entries
@@ -491,12 +537,55 @@ class VLMPromptSnapshotStore:
self._bases = {}
self._replays = {}
self.nbytes = 0
+ # One request's boundary and replay.
+ self._held = VLMPromptSnapshotStore(math.inf, 2, holds = False) if holds else None
def __len__(self):
return len(self._entries)
- def lookup(self, key, token_ids, limit):
- """Longest stored prefix ending by ``limit``, or the replay holding all but the last token."""
+ def __contains__(self, item):
+ return item in self._entries or (self._held is not None and item in self._held)
+
+ def lookup(
+ self,
+ key,
+ token_ids,
+ limit,
+ takes = False,
+ ):
+ """Longest stored prefix ending by ``limit``, or the replay holding all but the last token.
+ A held replay that serves stays held unless the caller ``takes`` it, as a boundary is."""
+ entries, n = self._lookup(key, token_ids, limit)
+ if not self._held:
+ return entries, n
+ held, held_n = self._held._lookup(key, token_ids, limit)
+ if held_n <= n:
+ self._held.clear()
+ return entries, n
+ if takes or not self._held._is_replay((key, tuple(token_ids[:held_n]))):
+ self._held.clear()
+ return held, held_n
+
+ def release_held(self):
+ if self._held is not None:
+ self._held.clear()
+
+ def over_budget(self, entries):
+ return cache_entries_nbytes(entries) > self._max_bytes
+
+ def held_entries(self, key):
+ """``key``'s held snapshots as (prefix ids, entries, replay), shortest first."""
+ held = self._held._entries.items() if self._held else ()
+ return sorted(
+ (
+ (ids, entries, self._held._is_replay((stored, ids)))
+ for (stored, ids), (entries, _nbytes) in held
+ if stored == key
+ ),
+ key = lambda item: len(item[0]),
+ )
+
+ def _lookup(self, key, token_ids, limit):
best = None
for item, (entries, _nbytes) in self._entries.items():
stored_key, prefix = item
@@ -519,16 +608,23 @@ class VLMPromptSnapshotStore:
prefix_ids,
entries,
replay = False,
+ hold = True,
):
compact_sliding_windows(entries)
nbytes = cache_entries_nbytes(entries)
if nbytes > self._max_bytes:
+ held = self._held is not None and hold
logger.debug(
- "MLX VLM prompt cache: skipping %.2f GB snapshot over the %.2f GB budget",
+ "MLX VLM prompt cache: %s %.2f GB snapshot over the %.2f GB budget",
+ "holding" if held else "skipping",
nbytes / 1e9,
self._max_bytes / 1e9,
)
- return False
+ if not held:
+ return False
+ if any(other[0] != key for other in self._held._entries):
+ self._held.clear()
+ return self._held.store(key, prefix_ids, entries, replay)
item = (key, tuple(prefix_ids))
replay = replay or self._is_replay(item)
readers = [other for other, base in self._bases.items() if base == item]
@@ -619,15 +715,21 @@ class VLMPromptSnapshotStore:
self._pop(item)
for other in [other for other, base in self._bases.items() if base == item]:
self.discard(other)
+ if self._held is not None:
+ self._held.discard(item)
def retain(self, item):
keep = (item, self._bases.get(item))
for other in [other for other in self._entries if other not in keep]:
self.discard(other)
+ if self._held is not None:
+ self._held.retain(item)
def retain_key(self, key):
+ """Held snapshots go too: no prompt has matched them yet."""
for other in [other for other in self._entries if other[0] != key]:
self.discard(other)
+ self.release_held()
def resume_entries(self):
"""The newest turn's longest boundary snapshot, then its replay."""
@@ -652,6 +754,7 @@ class VLMPromptSnapshotStore:
self._bases.clear()
self._replays.clear()
self.nbytes = 0
+ self.release_held()
def _extends(item, other):
@@ -701,6 +804,8 @@ class VLMPromptCacheSession:
self.reused_tokens = 0
self._token_ids = None
self._media_end = 0
+ self._handed_over = False
+ self._rehold = None
def __enter__(self):
self._forward.__enter__()
@@ -735,6 +840,17 @@ class VLMPromptCacheSession:
self._keep(None)
return 0
entries, prefix_len = self._store.lookup(self._key, token_ids, boundary)
+ self._handed_over = (
+ entries is not None and (self._key, tuple(token_ids[:prefix_len])) not in self._store
+ )
+ if self._handed_over and prefix_len == boundary:
+ # Banking no boundary past it, the request holds again what served it, from its final cache.
+ try:
+ self._rehold = kv_skeleton(entries)
+ except Exception as exc:
+ logger.info(
+ "MLX VLM prompt cache: served snapshot not kept to hold again (%s)", exc
+ )
if entries is None and origin:
# Produced here instead, in the one square forward the vision tokens need.
self._keep(None)
@@ -772,6 +888,7 @@ class VLMPromptCacheSession:
def _detach_served(self, offset):
prefix_ids = self._token_ids[: self.reused_tokens]
if offset != self.reused_tokens:
+ self._rehold = None
# Declined: freed now, before the fresh-cache request's media pass evaluates.
self._store.discard((self._key, tuple(prefix_ids)))
try:
@@ -779,6 +896,8 @@ class VLMPromptCacheSession:
except Exception as exc:
logger.info("MLX VLM prompt cache: declined snapshot not released (%s)", exc)
return
+ if self._handed_over:
+ return
# About to be advanced by this generation: the store keeps a copy.
try:
copy = copy_cache_entries(self.cache)
@@ -787,7 +906,9 @@ class VLMPromptCacheSession:
logger.info("MLX VLM prompt cache: served snapshot not kept (%s)", exc)
self._store.discard((self._key, tuple(prefix_ids)))
return
- self._store.store(self._key, prefix_ids, copy)
+ # Held only in place of what served it, never beside the generation that produced it.
+ kept = (self._key, tuple(prefix_ids)) in self._store
+ self._store.store(self._key, prefix_ids, copy, hold = kept)
def update(self, _token_ids, _cache):
"""mlx-vlm's after-generation hook; ``finish`` stores, as it also runs on a consumer that
@@ -798,6 +919,7 @@ class VLMPromptCacheSession:
return False
record = self._forward.record
stored = self._store_boundary(record.snapshot)
+ stored = self._store_rehold() or stored
replay = len(self._token_ids) - 1
if (
record.tail is not None
@@ -810,6 +932,18 @@ class VLMPromptCacheSession:
)
return stored
+ def _store_rehold(self):
+ skeleton, self._rehold = self._rehold, None
+ try:
+ if not skeleton or not fill_kv_skeleton(skeleton, self.cache):
+ return False
+ import mlx.core as mx
+ mx.eval(list(_arrays(skeleton, mx)))
+ except Exception as exc:
+ logger.info("MLX VLM prompt cache: served snapshot not held again (%s)", exc)
+ return False
+ return self._store.store(self._key, self._token_ids[: self.reused_tokens], skeleton)
+
def _store_boundary(self, snapshot):
if snapshot is None:
return False
@@ -832,7 +966,9 @@ class VLMBatchRowCache:
and store: ``open`` resumes the longest snapshot ending by the row's boundary and names the
chunk ends to bank, the boundary and the replay; ``checkpoint`` stores each. With no
``store`` the row prefills cold on the same cache layout; ``releases_unserved`` frees what
- does not serve it, as the single path does."""
+ does not serve it, as the single path does. A row takes what is held uncopied. Over the
+ budget (with ``alone``) a row keeps only a skeleton, filled in ``retire`` from the cache
+ unsloth-zoo hands back if ``alone()`` then; a copy would sit beside its decode."""
def __init__(
self,
@@ -842,6 +978,7 @@ class VLMBatchRowCache:
media_token_ids = (),
releases_unserved = False,
step = VLM_PREFILL_STEP,
+ alone = None,
):
self._store = store
self._key = key
@@ -849,8 +986,10 @@ class VLMBatchRowCache:
self._media_token_ids = tuple(media_token_ids)
self._releases_unserved = releases_unserved
self._step = step
+ self._alone = alone
self._token_ids = []
self._replay = None
+ self._skeletons = []
def open(self, token_ids):
self._token_ids = token_ids = list(token_ids)
@@ -861,20 +1000,35 @@ class VLMBatchRowCache:
if boundary < media_end:
self._keep(None)
return self._make_cache(), ()
- cache = None
- entries, prefix = self._store.lookup(self._key, token_ids, boundary)
+ cache, handed = None, False
+ held = self._store.held_entries(self._key) if self._alone is not None else ()
+ entries, prefix = self._store.lookup(self._key, token_ids, boundary, takes = True)
if entries is not None and prefix >= media_end:
- try:
- # The row advances its cache in place; the store keeps its own.
- cache = copy_cache_entries(entries)
- except Exception as exc:
- logger.info("MLX VLM prompt cache: snapshot not copied for a batched row (%s)", exc)
+ if (self._key, tuple(token_ids[:prefix])) not in self._store:
+ # Handed over by the store: the row's only copy.
+ cache, handed = entries, True
+ else:
+ try:
+ # The row advances its cache in place; the store keeps its own.
+ cache = copy_cache_entries(entries)
+ except Exception as exc:
+ logger.info(
+ "MLX VLM prompt cache: snapshot not copied for a batched row (%s)", exc
+ )
if cache is None:
cache, prefix = self._make_cache(), 0
self._keep((self._key, tuple(token_ids[:prefix])) if prefix else None)
replay = len(token_ids) - 1
self._replay = replay if replay != boundary else None
- return cache, [length for length in (boundary, replay) if length > prefix]
+ lengths = [length for length in (boundary, replay) if length > prefix]
+ if handed and boundary <= prefix:
+ # Banking no boundary past it, the row holds again what served it and the prefixes it extends.
+ self._skeletons = [
+ (len(ids), skeleton, is_replay)
+ for ids, held_entries, is_replay in held
+ if len(ids) <= prefix and (skeleton := kv_skeleton(held_entries))
+ ]
+ return cache, lengths
def _keep(self, item):
if self._releases_unserved:
@@ -883,15 +1037,39 @@ class VLMBatchRowCache:
def checkpoint(self, token_count, cache):
# Runs inside the batch's prefill: a failure costs a later turn its reuse, never this row.
try:
+ replay = token_count == self._replay
+ skeleton = (
+ self._alone is not None and self._store.over_budget(cache) and kv_skeleton(cache)
+ )
+ if skeleton:
+ self._skeletons.append((token_count, skeleton, replay))
+ return
self._store.store(
self._key,
self._token_ids[:token_count],
copy_cache_entries(cache),
- replay = token_count == self._replay,
+ replay = replay,
+ hold = False,
)
except Exception as exc:
logger.info("MLX VLM prompt cache: batched row snapshot not stored (%s)", exc)
+ def retire(self, take):
+ skeletons, self._skeletons = self._skeletons, []
+ if not skeletons or not self._alone():
+ return
+ try:
+ import mlx.core as mx
+
+ final = take()
+ if not all(fill_kv_skeleton(entries, final) for _count, entries, _replay in skeletons):
+ return
+ mx.eval([list(_arrays(entries, mx)) for _count, entries, _replay in skeletons])
+ for token_count, entries, replay in skeletons:
+ self._store.store(self._key, self._token_ids[:token_count], entries, replay = replay)
+ except Exception as exc:
+ logger.info("MLX VLM prompt cache: batched row snapshot not held (%s)", exc)
+
def _mlx_adapter_modules(model):
"""Return bypassable adapter entries and unsupported wrapper paths."""
@@ -4667,6 +4845,14 @@ def _row_prompt_cache_gap():
return row_prompt_cache_unavailable_reason()
+def _row_retire_gap():
+ try:
+ from unsloth_zoo.mlx.generate import row_retire_unavailable_reason
+ except ImportError:
+ return "the installed unsloth-zoo cannot hand a batched row's cache back"
+ return row_retire_unavailable_reason()
+
+
def _row_quantized_cache_gap():
try:
from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason
@@ -4691,6 +4877,7 @@ class _VisionBatchSession:
self._adapter_state = adapter_state
self._resumes_rows = _row_prompt_cache_gap() is None
self._rows = {}
+ self._leaving = set()
self._by_row = {}
self._settled = {}
self._held = ExitStack()
@@ -4718,6 +4905,7 @@ class _VisionBatchSession:
except BaseException:
self._held.close()
raise
+ self._retires_rows = self._resumes_rows and _row_retire_gap() is None
@property
def rows_in_flight(self) -> int:
@@ -4744,8 +4932,9 @@ class _VisionBatchSession:
backend = self.backend
plan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)
row = _VisionBatchRow(handle = handle, plan = plan)
+ alone = self._alone(handle) if self._retires_rows else None
state = (
- backend._vlm_batch_row_cache(self._adapter_state, plan.images)
+ backend._vlm_batch_row_cache(self._adapter_state, plan.images, alone = alone)
if self._resumes_rows
else None
)
@@ -4754,6 +4943,8 @@ class _VisionBatchSession:
raise RowRefused("this row cannot carry the load's quantized KV cache")
if state is None and plan.images:
backend._release_vlm_snapshots()
+ elif state is None:
+ backend._release_held_vlm_snapshots()
resume = {} if state is None else {"prompt_cache_state": state}
admitted_at = time.perf_counter()
try:
@@ -4784,6 +4975,12 @@ class _VisionBatchSession:
)
return plan.think_prefix
+ def _alone(self, handle):
+ """No other row decodes beside ``handle``'s; rows ending this step count until it ends,
+ as the speculative stream frees their caches only then."""
+ rows, leaving = self._rows, self._leaving
+ return lambda: all(other == handle for other in (*rows, *leaving))
+
def step(self, waiting = None):
now, stopped = None, {}
# Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.
@@ -4826,9 +5023,11 @@ class _VisionBatchSession:
if not row.cancelled:
row.reason = result.finish_reason
self._record(row, result, len(result.token_ids) + (result.finish_reason == "stop"))
+ self._leaving.add(row.handle)
yield from self._retire(row, cancelled = row.cancelled)
elif row.cancelled:
stopped[row.handle] = row
+ self._leaving.clear()
yield from self._fail(list(failing.values()))
for row in stopped.values():
if row.handle in self._rows:
@@ -5097,6 +5296,12 @@ class MLXInferenceBackend:
if store is not None:
store.clear()
+ def _release_held_vlm_snapshots(self):
+ """Before a request that looks nothing up, which they would sit beside."""
+ store = getattr(self, "_vlm_snapshot_store", None)
+ if store is not None:
+ store.release_held()
+
def _vlm_prompt_cache_store(self):
if self._vlm_snapshot_store is not None or self._vlm_snapshot_store_unavailable:
return self._vlm_snapshot_store
@@ -5205,6 +5410,7 @@ class MLXInferenceBackend:
self,
adapter_state,
images = None,
+ alone = None,
):
"""A batched row's ``prompt_cache_state``, or None while the store is off on an unquantized
load. Otherwise every row gets one, so all rows' caches share the factory's layout."""
@@ -5235,6 +5441,7 @@ class MLXInferenceBackend:
media_ids,
releases_unserved = bool(images),
step = vlm_prefill_step(),
+ alone = alone,
)
@staticmethod
@@ -7743,6 +7950,8 @@ class MLXInferenceBackend:
if session is None and (images or videos):
# Free headroom for the vision pass under the lock; a video turn resumes no snapshot, so retained ones are pure occupancy
self._release_vlm_snapshots()
+ elif session is None:
+ self._release_held_vlm_snapshots()
generation_scope.enter_context(
self._generation_scopes("vision", *(() if draft is None else ("speculative",)))
)
@@ -8176,6 +8385,8 @@ class MLXInferenceBackend:
]
if any(plan.images or plan.videos for plan in plans):
self._release_vlm_snapshots()
+ else:
+ self._release_held_vlm_snapshots()
logger.info(
"Generating %d vision replies as one batch: images=%d, videos=%d, max_tokens=%s, model=%s",
len(plans),
diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.py
index 7610c7a7f71..0d97510b709 100644
--- a/studio/backend/tests/test_mlx_inference_backend.py
+++ b/studio/backend/tests/test_mlx_inference_backend.py
@@ -8444,6 +8444,7 @@ def _open_vision_session(rows):
session._rows = {row.handle: row for row in rows}
session._by_row, session._settled = {row.row: row for row in rows}, {}
session._speculative, session._decodes_ids = None, False
+ session._retires_rows, session._leaving = False, set()
return session
@@ -8518,6 +8519,51 @@ def test_a_resident_vision_tool_row_decodes_its_ids_and_keeps_the_stop_that_clos
assert list(session.step())[-2:] == [("r", "".join(pieces)), ("r", None)]
+def test_rows_ending_in_one_step_are_never_alone_until_it_ends(monkeypatch):
+ """The speculative stream frees the caches of rows it reported finished only as its step ends."""
+ from core.inference import mlx_inference
+
+ backend = _vlm_backend(monkeypatch, markers = None)
+ session = _open_vision_session(
+ [
+ mlx_inference._VisionBatchRow(
+ handle = handle,
+ plan = SimpleNamespace(stream = _vlm_row(backend), streamed = None),
+ row = number,
+ )
+ for number, handle in enumerate("ab")
+ ]
+ )
+ alone, seen = session._alone("a"), []
+ result = SimpleNamespace(finish_reason = "length", prompt_token_count = 3, token_ids = [1])
+
+ def step():
+ for index in (0, 1):
+ yield SimpleNamespace(index = index, delta = "", result = result)
+ seen.append(alone())
+
+ session.stream = SimpleNamespace(step = step)
+ list(session.step())
+ assert seen == [False] and not session._rows and alone()
+
+
+def _store_holding_over_budget():
+ from core.inference import mlx_inference
+
+ store = mlx_inference.VLMPromptSnapshotStore(max_bytes = 10**9)
+ store._entries[("m", (1,))], store._held._entries[("m", (1, 2))] = ([], 0), ([], 0)
+ return store
+
+
+def test_a_vision_reply_without_a_prompt_cache_session_frees_what_is_held(monkeypatch):
+ from core.inference import mlx_inference
+
+ backend = mlx_inference.MLXInferenceBackend()
+ backend._vlm_snapshot_store = store = _store_holding_over_budget()
+ _drive_vlm_generation(backend, monkeypatch)
+ assert not store._held and len(store) == 1
+
+
def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatch):
engine = _batch_engine(monkeypatch)
monkeypatch.setattr(engine, "GenerationRequest", SimpleNamespace, raising = False)
@@ -8553,8 +8599,11 @@ def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypat
monkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)
monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
backend._reads_vision, backend._generation_lock = False, threading.Lock()
+ backend._vlm_snapshot_store = store = _store_holding_over_budget()
with pytest.raises(LookupError):
next(backend._generate_vlm_batch([{}]))
+ # Its rows look nothing up, so what is held over the budget would sit beside them.
+ assert not store._held and len(store) == 1
from dataclasses import make_dataclass
fields = (
@@ -8726,11 +8775,15 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
added = []
session.stream = SimpleNamespace(add = lambda request: added.append(request) or len(added) - 1)
+ session._retires_rows = True
session.admit({}, "text")
assert added[0].prompt_cache_state._store is store and len(store) == 2
+ alone = added[0].prompt_cache_state._alone
+ assert alone()
backend._model = SimpleNamespace(config = SimpleNamespace(image_token_id = 9))
+ store._held._entries[("m|img", (1, 2))] = ([], 0)
session.admit({"images": [object()]}, "image")
- assert list(store._entries) == [("m|img", (1,))]
+ assert list(store._entries) == [("m|img", (1,))] and not store._held and not alone()
assert added[1].prompt_cache_state._store is store
backend._model.config.use_bidirectional_attention = "vision"
session.admit({"images": [object()]}, "block")
@@ -8747,12 +8800,22 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
session.stream.withdraw = lambda row: result
list(session.withdraw(["image"]))
assert session.take_stats("image")["usage"]["prompt_tokens_details"]["cached_tokens"] == 6
+ # A row already reported finished is alone while no other row is in flight.
+ assert not alone()
+ block = session._rows.pop("block")
+ assert alone()
+ session._rows["block"] = block
store._entries[("m|img", (1,))] = ([], 0)
monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: "old zoo")
session._resumes_rows = mlx_inference._row_prompt_cache_gap() is None
session.admit({"images": [object()]}, "old")
assert not hasattr(added[3], "prompt_cache_state") and len(store) == 0
+ # A text row that looks nothing up still frees what is held over the budget.
+ store._entries[("m", (1,))] = ([], 0)
+ store._held._entries[("m", (1, 2))] = ([], 0)
+ session.admit({}, "old text")
+ assert not store._held and len(store) == 1
backend._kv_quant = {"kv_bits": 4}
with pytest.raises(mlx_inference.RowRefused):
@@ -8760,7 +8823,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mo
session._resumes_rows = True
backend._vlm_prompt_cache_store = lambda: None
session.admit({"images": [object()]}, "quantized")
- assert added[4].prompt_cache_state._store is None
+ assert added[5].prompt_cache_state._store is None
def _batch_engine(monkeypatch):
@@ -8948,6 +9011,34 @@ def test_a_turboquant_vision_batch_quantizes_rows_as_the_single_path_does(monkey
assert not hasattr(mlx_inference._VisionBatchSession(backend, width = 2).stream, "kv_bits")
+def test_a_vision_row_holds_its_snapshot_only_where_its_cache_is_handed_back(monkeypatch):
+ from core.inference import mlx_inference
+
+ engine = _batch_engine(monkeypatch)
+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)
+ monkeypatch.setattr(engine, "BatchStream", lambda *a, **k: None, raising = False)
+ monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: None, raising = False)
+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)
+ backend._model = backend._processor = object()
+ backend._kv_quant, backend._turboquant = {}, False
+ held = []
+ for resumes, gap, draft in (
+ (None, None, None),
+ (None, None, object()),
+ (None, "old mlx-vlm", None),
+ ("old zoo", None, None),
+ (None, ImportError, None),
+ ):
+ monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: resumes)
+ if gap is ImportError:
+ monkeypatch.delattr(engine, "row_retire_unavailable_reason")
+ else:
+ monkeypatch.setattr(engine, "row_retire_unavailable_reason", lambda: gap, raising = False)
+ backend._speculative_draft = draft
+ held.append(mlx_inference._VisionBatchSession(backend, width = 2)._retires_rows)
+ assert held == [True, True, False, False, False]
+
+
def test_a_batched_vision_row_keeps_to_the_context_budget(monkeypatch):
from core.inference import context_refusal
diff --git a/studio/backend/tests/test_mlx_vlm_prompt_cache.py b/studio/backend/tests/test_mlx_vlm_prompt_cache.py
index 5b17401a5da..29a5bda5af2 100644
--- a/studio/backend/tests/test_mlx_vlm_prompt_cache.py
+++ b/studio/backend/tests/test_mlx_vlm_prompt_cache.py
@@ -400,7 +400,8 @@ def test_store_serves_the_longest_prefix_and_evicts_to_fit(fake_mx):
assert store.lookup("m", b + [0], limit = 256) == (None, 0)
assert store.lookup("m", a + [0], limit = 256)[1] == 256
assert store.nbytes == 4 * 512 and len(store) == 2
- assert not store.store("m", [4] * 800, _snapshot([4] * 800))
+ assert not store.store("m", [4] * 800, _snapshot([4] * 800), hold = False)
+ assert store.store("m", [4] * 800, _snapshot([4] * 800)) # held apart
assert len(store) == 2 and store.nbytes == 4 * 512
store = VLMPromptSnapshotStore(max_bytes = 10**9, max_entries = 2)
@@ -422,6 +423,97 @@ def test_store_serves_the_longest_prefix_and_evicts_to_fit(fake_mx):
assert len(store) == 0 and store.nbytes == 0
+def _over_budget_store():
+ """256 rows fit the budget; one request's 512 boundary and 699 replay are held."""
+ store, ids = VLMPromptSnapshotStore(max_bytes = 4 * 300), list(range(1000))
+ store.store("m", ids[:256], _snapshot(ids[:256]))
+ assert store.store("m", ids[:512], _snapshot(ids[:512]))
+ assert store.store("m", ids[:699], _snapshot(ids[:699]), replay = True)
+ return store, ids
+
+
+def test_snapshots_over_the_budget_are_held_for_the_chats_next_lookup_only(fake_mx):
+ store, ids = _over_budget_store()
+ assert len(store) == 1 and store.nbytes == 4 * 256 and len(store._held) == 2
+ assert ("m", tuple(ids[:512])) in store and ("m", tuple(ids[:699])) in store
+
+ # Another key, another prefix, an earlier boundary: served as before, and released.
+ for key, prompt, limit, reused in (
+ ("t", ids, 768, 0),
+ ("m", [-1] + ids[1:], 768, 0),
+ ("m", ids, 511, 256),
+ ("m", ids[:701], 511, 256),
+ ):
+ store, ids = _over_budget_store()
+ assert store.lookup(key, prompt, limit)[1] == reused and not store._held
+ store, ids = _over_budget_store()
+ store._entries[("m", tuple(ids[:768]))] = (_snapshot(ids[:768]), 0)
+ assert store.lookup("m", ids, 768)[1] == 768 and not store._held
+
+ # The boundary is handed over, uncopied, and its replay goes with it.
+ store, ids = _over_budget_store()
+ held = store._held._entries[("m", tuple(ids[:512]))][0]
+ assert store.lookup("m", ids[:800], limit = 768) == (held, 512)
+ assert ("m", tuple(ids[:512])) not in store and not store._held and len(store) == 1
+ assert not fake_mx
+
+ # The replay serves the same prompt again and stays, with the boundary reading through it.
+ store, ids = _over_budget_store()
+ assert store.lookup("m", ids[:700], limit = 512)[1] == 699 and len(store._held) == 2
+ assert store.lookup("m", ids[:700], limit = 512)[1] == 699
+
+ # One chat at a time; release follows every way the store frees.
+ assert store.store("t", ids[:400], _snapshot(ids[:400]))
+ assert list(store._held._entries) == [("t", tuple(ids[:400]))]
+ for free in (
+ lambda s: s.retain(("m", tuple(ids[:256]))),
+ lambda s: s.retain_key("m"),
+ lambda s: s.discard(("m", tuple(ids[:512]))),
+ lambda s: s.release_held(),
+ lambda s: s.clear(),
+ ):
+ store, ids = _over_budget_store()
+ free(store)
+ assert ("m", tuple(ids[:512])) not in store
+
+
+def test_a_long_chat_resumes_from_its_held_snapshot_without_a_copy(fake_mx):
+ # 256 rows of make_cache fit; 512 do not.
+ store, language_model, ids = (
+ VLMPromptSnapshotStore(2400),
+ FakeLanguageModel(),
+ list(range(1100)),
+ )
+ _generate(store, language_model, ids[:700])
+ assert len(store) == 0 and len(store._held) == 2
+ held = store._held._entries[("m", tuple(ids[:512]))][0]
+ fake_mx.clear()
+ session, cache, stored = _generate(store, language_model, ids[:900])
+ assert session.reused_tokens == 512 and cache is held and stored
+ assert len(fake_mx) == 2 # the two captures; nothing kept of what it resumed from
+ assert [len(item[1]) for item in store._held._entries] == [768, 899]
+
+ fake_mx.clear()
+ session, cache, stored = _generate(store, language_model, ids[:900])
+ assert session.reused_tokens == 899 and cache[0].offset == 900 and len(fake_mx) == 1
+ assert ("m", tuple(ids[:899])) in store and ("m", tuple(ids[:768])) in store
+ assert store._held._entries[("m", tuple(ids[:899]))][0][0].offset == 899 # never advanced
+ session, _cache, _stored = _generate(store, language_model, ids[:1] + ids[2:900])
+ assert session.reused_tokens == 0 and ("m", tuple(ids[:768])) not in store
+
+
[email protected]("served", (512, 699))
+def test_a_batched_row_takes_what_is_held_and_holds_nothing(fake_mx, served):
+ store, ids = VLMPromptSnapshotStore(2400), list(range(1100))
+ _generate(store, FakeLanguageModel(), ids[:700])
+ held = store._held._entries[("m", tuple(ids[:served]))][0]
+ fake_mx.clear()
+ row = VLMBatchRowCache(store, "m", make_cache, step = STEP)
+ prefix, cache = _prefill_row(row, ids[: 900 if served == 512 else 700])
+ assert prefix == served and cache is held and not store._held and len(store) == 0
+ assert len(fake_mx) == (2 if served == 512 else 0) # its own checkpoints, then dropped
+
+
def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mx):
store = VLMPromptSnapshotStore(max_bytes = 10**9)
language_model = FakeLanguageModel()
@@ -613,6 +705,110 @@ def test_session_serves_and_stores_only_prefixes_past_the_last_media_token(fake_
assert len(store) == 0 and store.nbytes == 0
[email protected]("ring", (False, True))
+def test_a_lone_batched_row_holds_its_snapshots_from_the_cache_it_hands_back(ring):
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+
+ def positions(start, n):
+ return mx.arange(start, start + n, dtype = mx.float32).reshape(1, 1, n, 1) * mx.ones(
+ (1, 2, 1, 64)
+ )
+
+ def advance(entries, start, n):
+ for entry in entries:
+ if isinstance(entry, cache.ArraysCache):
+ entry[0] = mx.full((1, 64), float(start + n))
+ else:
+ entry.update_and_fetch(positions(start, n), -positions(start, n))
+ mx.eval([entry.state for entry in entries])
+
+ def prefill(
+ row,
+ ids,
+ decoded = 6,
+ ):
+ entries, lengths = row.open(ids)
+ done = cache_entries_offset(entries)
+ while done < len(ids) - 1:
+ take = min(STEP - done % STEP, len(ids) - 1 - done)
+ advance(entries, done, take)
+ done += take
+ if done in lengths:
+ row.checkpoint(done, entries)
+ for step in range(decoded):
+ advance(entries, done + step, 1)
+ # What unsloth-zoo hands back for a lone row: its own entries.
+ return lambda: entries
+
+ ids, alone = list(range(1000)), [True]
+ kv = (lambda: cache.RotatingKVCache(max_size = 4096)) if ring else cache.KVCache
+ plain = lambda: [kv(), cache.ArraysCache(size = 1), kv()]
+ window = lambda: cache.RotatingKVCache(max_size = 256)
+ wrapped = lambda: [window(), cache.ArraysCache(size = 1), window()]
+ store = VLMPromptSnapshotStore(max_bytes = 200_000)
+
+ def holds(lengths):
+ assert [len(item[1]) for item in store._held._entries] == lengths
+ for (_key, prefix), (entries, _nbytes) in store._held._entries.items():
+ n = len(prefix)
+ assert [entry.offset for entry in entries[::2]] == [n, n]
+ assert entries[1][0][0, 0].item() == n
+ for entry in entries[::2]:
+ assert mx.array_equal(entry.keys, positions(0, n)).item()
+ assert mx.array_equal(entry.values, -positions(0, n)).item()
+
+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])
+ take = prefill(row, ids[:700])
+ assert not store._held and len(store) == 0
+ assert [e.keys for _n, skeleton, _r in row._skeletons for e in skeleton[::2]] == [None] * 4
+ row.retire(take)
+ holds([512, 699])
+ # A regenerate banks nothing, so it holds again what served it and the boundary it extends.
+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])
+ take = prefill(row, ids[:700])
+ assert not store._held and cache_entries_offset(take()) == 705
+ row.retire(take)
+ holds([512, 699])
+ # A prompt one past the boundary resumes it whole and holds it again, not the longer replay.
+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])
+ row.retire(prefill(row, ids[:513]))
+ holds([512])
+ # The next turn resumes the boundary and holds only what it banks itself.
+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])
+ take = prefill(row, ids[:900])
+ assert [n for n, _skeleton, _replay in row._skeletons] == [768, 899]
+ row.retire(take)
+ holds([768, 899])
+ # A short turn inside that boundary's chunk banks only its replay and holds the boundary again.
+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])
+ row.retire(prefill(row, ids[:950]))
+ holds([768, 949])
+
+ # Held only by a row alone in the batch, from a cache holding its rows in order, and only
+ # where unsloth-zoo hands that cache back.
+ short = prefill(VLMBatchRowCache(None, "m", plain), ids[:600], decoded = 0)
+ slid = prefill(VLMBatchRowCache(None, "m", wrapped), ids[:700], decoded = 0)
+ for layout, alone[0], handed in (
+ (plain, False, lambda: pytest.fail("taken")),
+ (plain, True, short),
+ (plain, True, slid),
+ (wrapped, True, lambda: wrapped()),
+ ):
+ row = VLMBatchRowCache(store, "m", layout, step = STEP, alone = lambda: alone[0])
+ prefill(row, ids[:700])
+ assert bool(row._skeletons) is (layout is plain)
+ row.retire(handed)
+ assert not store._held and len(store) == 0 and not row._skeletons
+ prefill(VLMBatchRowCache(store, "m", plain, step = STEP), ids[:700])
+ assert not store._held and len(store) == 0
+ # Within the budget a row banks copies as it prefills.
+ store = VLMPromptSnapshotStore(max_bytes = 10**9)
+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: True)
+ prefill(row, ids[:700])
+ assert len(store) == 2 and not row._skeletons
+
+
def _prefill_row(state, token_ids):
"""unsloth-zoo's batched row: resume the opened cache, checkpoint where a grid chunk ends."""
cache, lengths = state.open(token_ids)
@@ -704,6 +900,15 @@ def test_session_prefills_the_media_block_and_chains_from_it(fake_mx, caplog):
session, _cache, stored = generate(200, honour_reuse = False)
assert session.reused_tokens == 100 and not stored
+ # Over the budget the block is not held beside the generation that produced it.
+ small, prompt = VLMPromptSnapshotStore(max_bytes = 100), _block_prompt(500)
+ block.store = small
+ with Session(small, "m", language_model, make_cache, **kwargs) as session:
+ assert session.find_prefix_length(prompt) == 100
+ run_generation(language_model, prompt, session.cache, 100)
+ assert not small._held
+ assert session.finish() and [len(item[1]) for item in small._held._entries] == [356, 499]
+
store, failing = VLMPromptSnapshotStore(max_bytes = 10**9), FakeMediaBlock(100)
failing.fail = True
fake_mx.clear()
@@ -1091,3 +1296,54 @@ def test_a_compacted_window_updates_exactly_as_the_full_one(keep, next_rows):
want = full.update_and_fetch(rows(start, n), -rows(start, n))
for a, b in zip(got, want):
assert a.shape == b.shape and mx.array_equal(a, b).item()
+
+
+def test_a_short_turn_holds_again_the_boundary_it_resumed_from():
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+
+ def positions(start, n):
+ return mx.arange(start, start + n, dtype = mx.float32).reshape(1, 1, n, 1) * mx.ones(
+ (1, 2, 1, 8)
+ )
+
+ class Model:
+ def __call__(
+ self,
+ inputs,
+ cache = None,
+ **_kwargs,
+ ):
+ start = cache_entries_offset(cache)
+ for entry in cache:
+ entry.update_and_fetch(
+ positions(start, len(inputs)), -positions(start, len(inputs))
+ )
+ mx.eval([entry.state for entry in cache])
+
+ model, ids = Model(), list(range(1000))
+ make = lambda: [cache.KVCache(), cache.KVCache()]
+
+ def turn(store, n):
+ with Session(store, "m", model, make) as session:
+ prefix = session.find_prefix_length(ids[:n])
+ run_generation(model, ids[:n], session.cache, prefix)
+ for step in range(4):
+ model([0], cache = session.cache)
+ session.finish()
+ return prefix
+
+ def held(store):
+ for (_key, prefix), (entries, _nbytes) in store._held._entries.items():
+ n = len(prefix)
+ assert [entry.offset for entry in entries] == [n, n]
+ assert mx.array_equal(entries[0].keys[..., :n, :], positions(0, n)).item()
+ return [len(item[1]) for item in store._held._entries]
+
+ store = VLMPromptSnapshotStore(max_bytes = 1)
+ assert turn(store, 700) == 0 and held(store) == [512, 699]
+ # Each short turn stays inside the 512 boundary's chunk, so it banks only its replay.
+ for n in (720, 740, 760):
+ assert turn(store, n) == 512
+ assert held(store) == [512, n - 1]
+ assert turn(store, 800) == 512 and turn(store, 800) == 799