Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(studio): keep a long MLX vision chat's prompt snapshot past the cache budget (#16079)
Long Yixing80a660ae9dfd
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex 1c9d7cadcdb..05219dc6f10 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -7,6 +7,7 @@ import functoolsimport hashlibimport importlibimport inspect+import mathimport osimport reimport shutil@@ -454,6 +455,47 @@ def share_kv_rows(entries, base):return shared+def kv_skeleton(entries):+ """``entries`` with in-order KV rows left out (``offset`` kept) and recurrent state copied;+ None for any other entry (quantized, wrapped)."""+ import mlx.core as mx++ classes = (_cache_classes("KVCache"), _cache_classes("RotatingKVCache"))+ recurrent = _cache_classes("ArraysCache")+ skeleton = []+ for entry in entries:+ if type(entry) in classes[0] + classes[1] and _rows_in_order(entry, classes):+ entry = copy.copy(entry)+ entry.keys = entry.values = None+ elif isinstance(entry, recurrent):+ entry = _copy_value(entry, mx)+ else:+ return None+ skeleton.append(entry)+ mx.eval(list(_arrays(skeleton, mx)))+ return skeleton+++def fill_kv_skeleton(skeleton, final):+ """Point ``skeleton``'s KV entries at the leading rows of the same row's ``final`` cache;+ False when ``final`` does not hold them from row 0 (a wrapped ring)."""+ kv = _cache_classes("KVCache") + _cache_classes("RotatingKVCache")+ if len(final) != len(skeleton):+ return False+ pairs = [(entry, source) for entry, source in zip(skeleton, final) if type(entry) in kv]+ if not all(+ type(source) is type(entry)+ and source.keys is not None+ and getattr(source, "_idx", source.offset) == source.offset >= entry.offset+ for entry, source in pairs+ ):+ return False+ for entry, source in pairs:+ entry.keys = source.keys[..., : entry.offset, :]+ entry.values = source.values[..., : entry.offset, :]+ return True++def compact_sliding_windows(entries):"""Drop ring rows past the window after a multi-row update; no later update reads them."""rings = _cache_classes("RotatingKVCache")@@ -478,12 +520,16 @@ def compact_sliding_windows(entries):class VLMPromptSnapshotStore:"""LRU boundary snapshots under a byte budget; a shorter prefix reads KV rows from its base.- A replay (one per key) serves only a prompt of that exact length and reads through no base."""+ A replay (one per key) serves only a prompt of that exact length and reads through no base.++ Those over the budget are held for one chat's next lookup only; a boundary that serves is+ handed over uncopied, since that request banks successors a kept copy would sit beside."""def __init__(self,max_bytes,max_entries = VLM_PROMPT_CACHE_ENTRIES,+ holds = True,):self._max_bytes = max_bytesself._max_entries = max_entries@@ -491,12 +537,55 @@ class VLMPromptSnapshotStore:self._bases = {}self._replays = {}self.nbytes = 0+ # One request's boundary and replay.+ self._held = VLMPromptSnapshotStore(math.inf, 2, holds = False) if holds else Nonedef __len__(self):return len(self._entries)- def lookup(self, key, token_ids, limit):- """Longest stored prefix ending by ``limit``, or the replay holding all but the last token."""+ def __contains__(self, item):+ return item in self._entries or (self._held is not None and item in self._held)++ def lookup(+ self,+ key,+ token_ids,+ limit,+ takes = False,+ ):+ """Longest stored prefix ending by ``limit``, or the replay holding all but the last token.+ A held replay that serves stays held unless the caller ``takes`` it, as a boundary is."""+ entries, n = self._lookup(key, token_ids, limit)+ if not self._held:+ return entries, n+ held, held_n = self._held._lookup(key, token_ids, limit)+ if held_n <= n:+ self._held.clear()+ return entries, n+ if takes or not self._held._is_replay((key, tuple(token_ids[:held_n]))):+ self._held.clear()+ return held, held_n++ def release_held(self):+ if self._held is not None:+ self._held.clear()++ def over_budget(self, entries):+ return cache_entries_nbytes(entries) > self._max_bytes++ def held_entries(self, key):+ """``key``'s held snapshots as (prefix ids, entries, replay), shortest first."""+ held = self._held._entries.items() if self._held else ()+ return sorted(+ (+ (ids, entries, self._held._is_replay((stored, ids)))+ for (stored, ids), (entries, _nbytes) in held+ if stored == key+ ),+ key = lambda item: len(item[0]),+ )++ def _lookup(self, key, token_ids, limit):best = Nonefor item, (entries, _nbytes) in self._entries.items():stored_key, prefix = item@@ -519,16 +608,23 @@ class VLMPromptSnapshotStore:prefix_ids,entries,replay = False,+ hold = True,):compact_sliding_windows(entries)nbytes = cache_entries_nbytes(entries)if nbytes > self._max_bytes:+ held = self._held is not None and holdlogger.debug(- "MLX VLM prompt cache: skipping %.2f GB snapshot over the %.2f GB budget",+ "MLX VLM prompt cache: %s %.2f GB snapshot over the %.2f GB budget",+ "holding" if held else "skipping",nbytes / 1e9,self._max_bytes / 1e9,)- return False+ if not held:+ return False+ if any(other[0] != key for other in self._held._entries):+ self._held.clear()+ return self._held.store(key, prefix_ids, entries, replay)item = (key, tuple(prefix_ids))replay = replay or self._is_replay(item)readers = [other for other, base in self._bases.items() if base == item]@@ -619,15 +715,21 @@ class VLMPromptSnapshotStore:self._pop(item)for other in [other for other, base in self._bases.items() if base == item]:self.discard(other)+ if self._held is not None:+ self._held.discard(item)def retain(self, item):keep = (item, self._bases.get(item))for other in [other for other in self._entries if other not in keep]:self.discard(other)+ if self._held is not None:+ self._held.retain(item)def retain_key(self, key):+ """Held snapshots go too: no prompt has matched them yet."""for other in [other for other in self._entries if other[0] != key]:self.discard(other)+ self.release_held()def resume_entries(self):"""The newest turn's longest boundary snapshot, then its replay."""@@ -652,6 +754,7 @@ class VLMPromptSnapshotStore:self._bases.clear()self._replays.clear()self.nbytes = 0+ self.release_held()def _extends(item, other):@@ -701,6 +804,8 @@ class VLMPromptCacheSession:self.reused_tokens = 0self._token_ids = Noneself._media_end = 0+ self._handed_over = False+ self._rehold = Nonedef __enter__(self):self._forward.__enter__()@@ -735,6 +840,17 @@ class VLMPromptCacheSession:self._keep(None)return 0entries, prefix_len = self._store.lookup(self._key, token_ids, boundary)+ self._handed_over = (+ entries is not None and (self._key, tuple(token_ids[:prefix_len])) not in self._store+ )+ if self._handed_over and prefix_len == boundary:+ # Banking no boundary past it, the request holds again what served it, from its final cache.+ try:+ self._rehold = kv_skeleton(entries)+ except Exception as exc:+ logger.info(+ "MLX VLM prompt cache: served snapshot not kept to hold again (%s)", exc+ )if entries is None and origin:# Produced here instead, in the one square forward the vision tokens need.self._keep(None)@@ -772,6 +888,7 @@ class VLMPromptCacheSession:def _detach_served(self, offset):prefix_ids = self._token_ids[: self.reused_tokens]if offset != self.reused_tokens:+ self._rehold = None# Declined: freed now, before the fresh-cache request's media pass evaluates.self._store.discard((self._key, tuple(prefix_ids)))try:@@ -779,6 +896,8 @@ class VLMPromptCacheSession:except Exception as exc:logger.info("MLX VLM prompt cache: declined snapshot not released (%s)", exc)return+ if self._handed_over:+ return# About to be advanced by this generation: the store keeps a copy.try:copy = copy_cache_entries(self.cache)@@ -787,7 +906,9 @@ class VLMPromptCacheSession:logger.info("MLX VLM prompt cache: served snapshot not kept (%s)", exc)self._store.discard((self._key, tuple(prefix_ids)))return- self._store.store(self._key, prefix_ids, copy)+ # Held only in place of what served it, never beside the generation that produced it.+ kept = (self._key, tuple(prefix_ids)) in self._store+ self._store.store(self._key, prefix_ids, copy, hold = kept)def update(self, _token_ids, _cache):"""mlx-vlm's after-generation hook; ``finish`` stores, as it also runs on a consumer that@@ -798,6 +919,7 @@ class VLMPromptCacheSession:return Falserecord = self._forward.recordstored = self._store_boundary(record.snapshot)+ stored = self._store_rehold() or storedreplay = len(self._token_ids) - 1if (record.tail is not None@@ -810,6 +932,18 @@ class VLMPromptCacheSession:)return stored+ def _store_rehold(self):+ skeleton, self._rehold = self._rehold, None+ try:+ if not skeleton or not fill_kv_skeleton(skeleton, self.cache):+ return False+ import mlx.core as mx+ mx.eval(list(_arrays(skeleton, mx)))+ except Exception as exc:+ logger.info("MLX VLM prompt cache: served snapshot not held again (%s)", exc)+ return False+ return self._store.store(self._key, self._token_ids[: self.reused_tokens], skeleton)+def _store_boundary(self, snapshot):if snapshot is None:return False@@ -832,7 +966,9 @@ class VLMBatchRowCache:and store: ``open`` resumes the longest snapshot ending by the row's boundary and names thechunk ends to bank, the boundary and the replay; ``checkpoint`` stores each. With no``store`` the row prefills cold on the same cache layout; ``releases_unserved`` frees what- does not serve it, as the single path does."""+ does not serve it, as the single path does. A row takes what is held uncopied. Over the+ budget (with ``alone``) a row keeps only a skeleton, filled in ``retire`` from the cache+ unsloth-zoo hands back if ``alone()`` then; a copy would sit beside its decode."""def __init__(self,@@ -842,6 +978,7 @@ class VLMBatchRowCache:media_token_ids = (),releases_unserved = False,step = VLM_PREFILL_STEP,+ alone = None,):self._store = storeself._key = key@@ -849,8 +986,10 @@ class VLMBatchRowCache:self._media_token_ids = tuple(media_token_ids)self._releases_unserved = releases_unservedself._step = step+ self._alone = aloneself._token_ids = []self._replay = None+ self._skeletons = []def open(self, token_ids):self._token_ids = token_ids = list(token_ids)@@ -861,20 +1000,35 @@ class VLMBatchRowCache:if boundary < media_end:self._keep(None)return self._make_cache(), ()- cache = None- entries, prefix = self._store.lookup(self._key, token_ids, boundary)+ cache, handed = None, False+ held = self._store.held_entries(self._key) if self._alone is not None else ()+ entries, prefix = self._store.lookup(self._key, token_ids, boundary, takes = True)if entries is not None and prefix >= media_end:- try:- # The row advances its cache in place; the store keeps its own.- cache = copy_cache_entries(entries)- except Exception as exc:- logger.info("MLX VLM prompt cache: snapshot not copied for a batched row (%s)", exc)+ if (self._key, tuple(token_ids[:prefix])) not in self._store:+ # Handed over by the store: the row's only copy.+ cache, handed = entries, True+ else:+ try:+ # The row advances its cache in place; the store keeps its own.+ cache = copy_cache_entries(entries)+ except Exception as exc:+ logger.info(+ "MLX VLM prompt cache: snapshot not copied for a batched row (%s)", exc+ )if cache is None:cache, prefix = self._make_cache(), 0self._keep((self._key, tuple(token_ids[:prefix])) if prefix else None)replay = len(token_ids) - 1self._replay = replay if replay != boundary else None- return cache, [length for length in (boundary, replay) if length > prefix]+ lengths = [length for length in (boundary, replay) if length > prefix]+ if handed and boundary <= prefix:+ # Banking no boundary past it, the row holds again what served it and the prefixes it extends.+ self._skeletons = [+ (len(ids), skeleton, is_replay)+ for ids, held_entries, is_replay in held+ if len(ids) <= prefix and (skeleton := kv_skeleton(held_entries))+ ]+ return cache, lengthsdef _keep(self, item):if self._releases_unserved:@@ -883,15 +1037,39 @@ class VLMBatchRowCache:def checkpoint(self, token_count, cache):# Runs inside the batch's prefill: a failure costs a later turn its reuse, never this row.try:+ replay = token_count == self._replay+ skeleton = (+ self._alone is not None and self._store.over_budget(cache) and kv_skeleton(cache)+ )+ if skeleton:+ self._skeletons.append((token_count, skeleton, replay))+ returnself._store.store(self._key,self._token_ids[:token_count],copy_cache_entries(cache),- replay = token_count == self._replay,+ replay = replay,+ hold = False,)except Exception as exc:logger.info("MLX VLM prompt cache: batched row snapshot not stored (%s)", exc)+ def retire(self, take):+ skeletons, self._skeletons = self._skeletons, []+ if not skeletons or not self._alone():+ return+ try:+ import mlx.core as mx++ final = take()+ if not all(fill_kv_skeleton(entries, final) for _count, entries, _replay in skeletons):+ return+ mx.eval([list(_arrays(entries, mx)) for _count, entries, _replay in skeletons])+ for token_count, entries, replay in skeletons:+ self._store.store(self._key, self._token_ids[:token_count], entries, replay = replay)+ except Exception as exc:+ logger.info("MLX VLM prompt cache: batched row snapshot not held (%s)", exc)+def _mlx_adapter_modules(model):"""Return bypassable adapter entries and unsupported wrapper paths."""@@ -4667,6 +4845,14 @@ def _row_prompt_cache_gap():return row_prompt_cache_unavailable_reason()+def _row_retire_gap():+ try:+ from unsloth_zoo.mlx.generate import row_retire_unavailable_reason+ except ImportError:+ return "the installed unsloth-zoo cannot hand a batched row's cache back"+ return row_retire_unavailable_reason()++def _row_quantized_cache_gap():try:from unsloth_zoo.mlx.generate import row_quantized_prompt_cache_unavailable_reason@@ -4691,6 +4877,7 @@ class _VisionBatchSession:self._adapter_state = adapter_stateself._resumes_rows = _row_prompt_cache_gap() is Noneself._rows = {}+ self._leaving = set()self._by_row = {}self._settled = {}self._held = ExitStack()@@ -4718,6 +4905,7 @@ class _VisionBatchSession:except BaseException:self._held.close()raise+ self._retires_rows = self._resumes_rows and _row_retire_gap() is None@propertydef rows_in_flight(self) -> int:@@ -4744,8 +4932,9 @@ class _VisionBatchSession:backend = self.backendplan = backend._plan_vlm_request(request, decodes_ids = self._decodes_ids)row = _VisionBatchRow(handle = handle, plan = plan)+ alone = self._alone(handle) if self._retires_rows else Nonestate = (- backend._vlm_batch_row_cache(self._adapter_state, plan.images)+ backend._vlm_batch_row_cache(self._adapter_state, plan.images, alone = alone)if self._resumes_rowselse None)@@ -4754,6 +4943,8 @@ class _VisionBatchSession:raise RowRefused("this row cannot carry the load's quantized KV cache")if state is None and plan.images:backend._release_vlm_snapshots()+ elif state is None:+ backend._release_held_vlm_snapshots()resume = {} if state is None else {"prompt_cache_state": state}admitted_at = time.perf_counter()try:@@ -4784,6 +4975,12 @@ class _VisionBatchSession:)return plan.think_prefix+ def _alone(self, handle):+ """No other row decodes beside ``handle``'s; rows ending this step count until it ends,+ as the speculative stream frees their caches only then."""+ rows, leaving = self._rows, self._leaving+ return lambda: all(other == handle for other in (*rows, *leaving))+def step(self, waiting = None):now, stopped = None, {}# Only a speculative step spans many tokens, reported as they are decoded; only its unsloth-zoo has the call.@@ -4826,9 +5023,11 @@ class _VisionBatchSession:if not row.cancelled:row.reason = result.finish_reasonself._record(row, result, len(result.token_ids) + (result.finish_reason == "stop"))+ self._leaving.add(row.handle)yield from self._retire(row, cancelled = row.cancelled)elif row.cancelled:stopped[row.handle] = row+ self._leaving.clear()yield from self._fail(list(failing.values()))for row in stopped.values():if row.handle in self._rows:@@ -5097,6 +5296,12 @@ class MLXInferenceBackend:if store is not None:store.clear()+ def _release_held_vlm_snapshots(self):+ """Before a request that looks nothing up, which they would sit beside."""+ store = getattr(self, "_vlm_snapshot_store", None)+ if store is not None:+ store.release_held()+def _vlm_prompt_cache_store(self):if self._vlm_snapshot_store is not None or self._vlm_snapshot_store_unavailable:return self._vlm_snapshot_store@@ -5205,6 +5410,7 @@ class MLXInferenceBackend:self,adapter_state,images = None,+ alone = None,):"""A batched row's ``prompt_cache_state``, or None while the store is off on an unquantizedload. Otherwise every row gets one, so all rows' caches share the factory's layout."""@@ -5235,6 +5441,7 @@ class MLXInferenceBackend:media_ids,releases_unserved = bool(images),step = vlm_prefill_step(),+ alone = alone,)@staticmethod@@ -7743,6 +7950,8 @@ class MLXInferenceBackend:if session is None and (images or videos):# Free headroom for the vision pass under the lock; a video turn resumes no snapshot, so retained ones are pure occupancyself._release_vlm_snapshots()+ elif session is None:+ self._release_held_vlm_snapshots()generation_scope.enter_context(self._generation_scopes("vision", *(() if draft is None else ("speculative",))))@@ -8176,6 +8385,8 @@ class MLXInferenceBackend:]if any(plan.images or plan.videos for plan in plans):self._release_vlm_snapshots()+ else:+ self._release_held_vlm_snapshots()logger.info("Generating %d vision replies as one batch: images=%d, videos=%d, max_tokens=%s, model=%s",len(plans),diff --git a/studio/backend/tests/test_mlx_inference_backend.py b/studio/backend/tests/test_mlx_inference_backend.pyindex 7610c7a7f71..0d97510b709 100644--- a/studio/backend/tests/test_mlx_inference_backend.py+++ b/studio/backend/tests/test_mlx_inference_backend.py@@ -8444,6 +8444,7 @@ def _open_vision_session(rows):session._rows = {row.handle: row for row in rows}session._by_row, session._settled = {row.row: row for row in rows}, {}session._speculative, session._decodes_ids = None, False+ session._retires_rows, session._leaving = False, set()return session@@ -8518,6 +8519,51 @@ def test_a_resident_vision_tool_row_decodes_its_ids_and_keeps_the_stop_that_closassert list(session.step())[-2:] == [("r", "".join(pieces)), ("r", None)]+def test_rows_ending_in_one_step_are_never_alone_until_it_ends(monkeypatch):+ """The speculative stream frees the caches of rows it reported finished only as its step ends."""+ from core.inference import mlx_inference++ backend = _vlm_backend(monkeypatch, markers = None)+ session = _open_vision_session(+ [+ mlx_inference._VisionBatchRow(+ handle = handle,+ plan = SimpleNamespace(stream = _vlm_row(backend), streamed = None),+ row = number,+ )+ for number, handle in enumerate("ab")+ ]+ )+ alone, seen = session._alone("a"), []+ result = SimpleNamespace(finish_reason = "length", prompt_token_count = 3, token_ids = [1])++ def step():+ for index in (0, 1):+ yield SimpleNamespace(index = index, delta = "", result = result)+ seen.append(alone())++ session.stream = SimpleNamespace(step = step)+ list(session.step())+ assert seen == [False] and not session._rows and alone()+++def _store_holding_over_budget():+ from core.inference import mlx_inference++ store = mlx_inference.VLMPromptSnapshotStore(max_bytes = 10**9)+ store._entries[("m", (1,))], store._held._entries[("m", (1, 2))] = ([], 0), ([], 0)+ return store+++def test_a_vision_reply_without_a_prompt_cache_session_frees_what_is_held(monkeypatch):+ from core.inference import mlx_inference++ backend = mlx_inference.MLXInferenceBackend()+ backend._vlm_snapshot_store = store = _store_holding_over_budget()+ _drive_vlm_generation(backend, monkeypatch)+ assert not store._held and len(store) == 1++def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatch):engine = _batch_engine(monkeypatch)monkeypatch.setattr(engine, "GenerationRequest", SimpleNamespace, raising = False)@@ -8553,8 +8599,11 @@ def test_a_text_load_batch_row_carries_one_bos_as_the_single_path_does(monkeypatmonkeypatch.setattr(engine, "stream_batch", stream_batch, raising = False)monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)backend._reads_vision, backend._generation_lock = False, threading.Lock()+ backend._vlm_snapshot_store = store = _store_holding_over_budget()with pytest.raises(LookupError):next(backend._generate_vlm_batch([{}]))+ # Its rows look nothing up, so what is held over the budget would sit beside them.+ assert not store._held and len(store) == 1from dataclasses import make_dataclassfields = (@@ -8726,11 +8775,15 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(moadded = []session.stream = SimpleNamespace(add = lambda request: added.append(request) or len(added) - 1)+ session._retires_rows = Truesession.admit({}, "text")assert added[0].prompt_cache_state._store is store and len(store) == 2+ alone = added[0].prompt_cache_state._alone+ assert alone()backend._model = SimpleNamespace(config = SimpleNamespace(image_token_id = 9))+ store._held._entries[("m|img", (1, 2))] = ([], 0)session.admit({"images": [object()]}, "image")- assert list(store._entries) == [("m|img", (1,))]+ assert list(store._entries) == [("m|img", (1,))] and not store._held and not alone()assert added[1].prompt_cache_state._store is storebackend._model.config.use_bidirectional_attention = "vision"session.admit({"images": [object()]}, "block")@@ -8747,12 +8800,22 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mosession.stream.withdraw = lambda row: resultlist(session.withdraw(["image"]))assert session.take_stats("image")["usage"]["prompt_tokens_details"]["cached_tokens"] == 6+ # A row already reported finished is alone while no other row is in flight.+ assert not alone()+ block = session._rows.pop("block")+ assert alone()+ session._rows["block"] = blockstore._entries[("m|img", (1,))] = ([], 0)monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: "old zoo")session._resumes_rows = mlx_inference._row_prompt_cache_gap() is Nonesession.admit({"images": [object()]}, "old")assert not hasattr(added[3], "prompt_cache_state") and len(store) == 0+ # A text row that looks nothing up still frees what is held over the budget.+ store._entries[("m", (1,))] = ([], 0)+ store._held._entries[("m", (1, 2))] = ([], 0)+ session.admit({}, "old text")+ assert not store._held and len(store) == 1backend._kv_quant = {"kv_bits": 4}with pytest.raises(mlx_inference.RowRefused):@@ -8760,7 +8823,7 @@ def test_a_resident_vision_row_resumes_from_the_snapshot_store_and_reports_it(mosession._resumes_rows = Truebackend._vlm_prompt_cache_store = lambda: Nonesession.admit({"images": [object()]}, "quantized")- assert added[4].prompt_cache_state._store is None+ assert added[5].prompt_cache_state._store is Nonedef _batch_engine(monkeypatch):@@ -8948,6 +9011,34 @@ def test_a_turboquant_vision_batch_quantizes_rows_as_the_single_path_does(monkeyassert not hasattr(mlx_inference._VisionBatchSession(backend, width = 2).stream, "kv_bits")+def test_a_vision_row_holds_its_snapshot_only_where_its_cache_is_handed_back(monkeypatch):+ from core.inference import mlx_inference++ engine = _batch_engine(monkeypatch)+ monkeypatch.setattr(engine, "GenerationDefaults", SimpleNamespace, raising = False)+ monkeypatch.setattr(engine, "BatchStream", lambda *a, **k: None, raising = False)+ monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: None, raising = False)+ backend = mlx_inference.MLXInferenceBackend.__new__(mlx_inference.MLXInferenceBackend)+ backend._model = backend._processor = object()+ backend._kv_quant, backend._turboquant = {}, False+ held = []+ for resumes, gap, draft in (+ (None, None, None),+ (None, None, object()),+ (None, "old mlx-vlm", None),+ ("old zoo", None, None),+ (None, ImportError, None),+ ):+ monkeypatch.setattr(engine, "row_prompt_cache_unavailable_reason", lambda: resumes)+ if gap is ImportError:+ monkeypatch.delattr(engine, "row_retire_unavailable_reason")+ else:+ monkeypatch.setattr(engine, "row_retire_unavailable_reason", lambda: gap, raising = False)+ backend._speculative_draft = draft+ held.append(mlx_inference._VisionBatchSession(backend, width = 2)._retires_rows)+ assert held == [True, True, False, False, False]++def test_a_batched_vision_row_keeps_to_the_context_budget(monkeypatch):from core.inference import context_refusaldiff --git a/studio/backend/tests/test_mlx_vlm_prompt_cache.py b/studio/backend/tests/test_mlx_vlm_prompt_cache.pyindex 5b17401a5da..29a5bda5af2 100644--- a/studio/backend/tests/test_mlx_vlm_prompt_cache.py+++ b/studio/backend/tests/test_mlx_vlm_prompt_cache.py@@ -400,7 +400,8 @@ def test_store_serves_the_longest_prefix_and_evicts_to_fit(fake_mx):assert store.lookup("m", b + [0], limit = 256) == (None, 0)assert store.lookup("m", a + [0], limit = 256)[1] == 256assert store.nbytes == 4 * 512 and len(store) == 2- assert not store.store("m", [4] * 800, _snapshot([4] * 800))+ assert not store.store("m", [4] * 800, _snapshot([4] * 800), hold = False)+ assert store.store("m", [4] * 800, _snapshot([4] * 800)) # held apartassert len(store) == 2 and store.nbytes == 4 * 512store = VLMPromptSnapshotStore(max_bytes = 10**9, max_entries = 2)@@ -422,6 +423,97 @@ def test_store_serves_the_longest_prefix_and_evicts_to_fit(fake_mx):assert len(store) == 0 and store.nbytes == 0+def _over_budget_store():+ """256 rows fit the budget; one request's 512 boundary and 699 replay are held."""+ store, ids = VLMPromptSnapshotStore(max_bytes = 4 * 300), list(range(1000))+ store.store("m", ids[:256], _snapshot(ids[:256]))+ assert store.store("m", ids[:512], _snapshot(ids[:512]))+ assert store.store("m", ids[:699], _snapshot(ids[:699]), replay = True)+ return store, ids+++def test_snapshots_over_the_budget_are_held_for_the_chats_next_lookup_only(fake_mx):+ store, ids = _over_budget_store()+ assert len(store) == 1 and store.nbytes == 4 * 256 and len(store._held) == 2+ assert ("m", tuple(ids[:512])) in store and ("m", tuple(ids[:699])) in store++ # Another key, another prefix, an earlier boundary: served as before, and released.+ for key, prompt, limit, reused in (+ ("t", ids, 768, 0),+ ("m", [-1] + ids[1:], 768, 0),+ ("m", ids, 511, 256),+ ("m", ids[:701], 511, 256),+ ):+ store, ids = _over_budget_store()+ assert store.lookup(key, prompt, limit)[1] == reused and not store._held+ store, ids = _over_budget_store()+ store._entries[("m", tuple(ids[:768]))] = (_snapshot(ids[:768]), 0)+ assert store.lookup("m", ids, 768)[1] == 768 and not store._held++ # The boundary is handed over, uncopied, and its replay goes with it.+ store, ids = _over_budget_store()+ held = store._held._entries[("m", tuple(ids[:512]))][0]+ assert store.lookup("m", ids[:800], limit = 768) == (held, 512)+ assert ("m", tuple(ids[:512])) not in store and not store._held and len(store) == 1+ assert not fake_mx++ # The replay serves the same prompt again and stays, with the boundary reading through it.+ store, ids = _over_budget_store()+ assert store.lookup("m", ids[:700], limit = 512)[1] == 699 and len(store._held) == 2+ assert store.lookup("m", ids[:700], limit = 512)[1] == 699++ # One chat at a time; release follows every way the store frees.+ assert store.store("t", ids[:400], _snapshot(ids[:400]))+ assert list(store._held._entries) == [("t", tuple(ids[:400]))]+ for free in (+ lambda s: s.retain(("m", tuple(ids[:256]))),+ lambda s: s.retain_key("m"),+ lambda s: s.discard(("m", tuple(ids[:512]))),+ lambda s: s.release_held(),+ lambda s: s.clear(),+ ):+ store, ids = _over_budget_store()+ free(store)+ assert ("m", tuple(ids[:512])) not in store+++def test_a_long_chat_resumes_from_its_held_snapshot_without_a_copy(fake_mx):+ # 256 rows of make_cache fit; 512 do not.+ store, language_model, ids = (+ VLMPromptSnapshotStore(2400),+ FakeLanguageModel(),+ list(range(1100)),+ )+ _generate(store, language_model, ids[:700])+ assert len(store) == 0 and len(store._held) == 2+ held = store._held._entries[("m", tuple(ids[:512]))][0]+ fake_mx.clear()+ session, cache, stored = _generate(store, language_model, ids[:900])+ assert session.reused_tokens == 512 and cache is held and stored+ assert len(fake_mx) == 2 # the two captures; nothing kept of what it resumed from+ assert [len(item[1]) for item in store._held._entries] == [768, 899]++ fake_mx.clear()+ session, cache, stored = _generate(store, language_model, ids[:900])+ assert session.reused_tokens == 899 and cache[0].offset == 900 and len(fake_mx) == 1+ assert ("m", tuple(ids[:899])) in store and ("m", tuple(ids[:768])) in store+ assert store._held._entries[("m", tuple(ids[:899]))][0][0].offset == 899 # never advanced+ session, _cache, _stored = _generate(store, language_model, ids[:1] + ids[2:900])+ assert session.reused_tokens == 0 and ("m", tuple(ids[:768])) not in store++[email protected]("served", (512, 699))+def test_a_batched_row_takes_what_is_held_and_holds_nothing(fake_mx, served):+ store, ids = VLMPromptSnapshotStore(2400), list(range(1100))+ _generate(store, FakeLanguageModel(), ids[:700])+ held = store._held._entries[("m", tuple(ids[:served]))][0]+ fake_mx.clear()+ row = VLMBatchRowCache(store, "m", make_cache, step = STEP)+ prefix, cache = _prefill_row(row, ids[: 900 if served == 512 else 700])+ assert prefix == served and cache is held and not store._held and len(store) == 0+ assert len(fake_mx) == (2 if served == 512 else 0) # its own checkpoints, then dropped++def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mx):store = VLMPromptSnapshotStore(max_bytes = 10**9)language_model = FakeLanguageModel()@@ -613,6 +705,110 @@ def test_session_serves_and_stores_only_prefixes_past_the_last_media_token(fake_assert len(store) == 0 and store.nbytes == 0[email protected]("ring", (False, True))+def test_a_lone_batched_row_holds_its_snapshots_from_the_cache_it_hands_back(ring):+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")++ def positions(start, n):+ return mx.arange(start, start + n, dtype = mx.float32).reshape(1, 1, n, 1) * mx.ones(+ (1, 2, 1, 64)+ )++ def advance(entries, start, n):+ for entry in entries:+ if isinstance(entry, cache.ArraysCache):+ entry[0] = mx.full((1, 64), float(start + n))+ else:+ entry.update_and_fetch(positions(start, n), -positions(start, n))+ mx.eval([entry.state for entry in entries])++ def prefill(+ row,+ ids,+ decoded = 6,+ ):+ entries, lengths = row.open(ids)+ done = cache_entries_offset(entries)+ while done < len(ids) - 1:+ take = min(STEP - done % STEP, len(ids) - 1 - done)+ advance(entries, done, take)+ done += take+ if done in lengths:+ row.checkpoint(done, entries)+ for step in range(decoded):+ advance(entries, done + step, 1)+ # What unsloth-zoo hands back for a lone row: its own entries.+ return lambda: entries++ ids, alone = list(range(1000)), [True]+ kv = (lambda: cache.RotatingKVCache(max_size = 4096)) if ring else cache.KVCache+ plain = lambda: [kv(), cache.ArraysCache(size = 1), kv()]+ window = lambda: cache.RotatingKVCache(max_size = 256)+ wrapped = lambda: [window(), cache.ArraysCache(size = 1), window()]+ store = VLMPromptSnapshotStore(max_bytes = 200_000)++ def holds(lengths):+ assert [len(item[1]) for item in store._held._entries] == lengths+ for (_key, prefix), (entries, _nbytes) in store._held._entries.items():+ n = len(prefix)+ assert [entry.offset for entry in entries[::2]] == [n, n]+ assert entries[1][0][0, 0].item() == n+ for entry in entries[::2]:+ assert mx.array_equal(entry.keys, positions(0, n)).item()+ assert mx.array_equal(entry.values, -positions(0, n)).item()++ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])+ take = prefill(row, ids[:700])+ assert not store._held and len(store) == 0+ assert [e.keys for _n, skeleton, _r in row._skeletons for e in skeleton[::2]] == [None] * 4+ row.retire(take)+ holds([512, 699])+ # A regenerate banks nothing, so it holds again what served it and the boundary it extends.+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])+ take = prefill(row, ids[:700])+ assert not store._held and cache_entries_offset(take()) == 705+ row.retire(take)+ holds([512, 699])+ # A prompt one past the boundary resumes it whole and holds it again, not the longer replay.+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])+ row.retire(prefill(row, ids[:513]))+ holds([512])+ # The next turn resumes the boundary and holds only what it banks itself.+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])+ take = prefill(row, ids[:900])+ assert [n for n, _skeleton, _replay in row._skeletons] == [768, 899]+ row.retire(take)+ holds([768, 899])+ # A short turn inside that boundary's chunk banks only its replay and holds the boundary again.+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: alone[0])+ row.retire(prefill(row, ids[:950]))+ holds([768, 949])++ # Held only by a row alone in the batch, from a cache holding its rows in order, and only+ # where unsloth-zoo hands that cache back.+ short = prefill(VLMBatchRowCache(None, "m", plain), ids[:600], decoded = 0)+ slid = prefill(VLMBatchRowCache(None, "m", wrapped), ids[:700], decoded = 0)+ for layout, alone[0], handed in (+ (plain, False, lambda: pytest.fail("taken")),+ (plain, True, short),+ (plain, True, slid),+ (wrapped, True, lambda: wrapped()),+ ):+ row = VLMBatchRowCache(store, "m", layout, step = STEP, alone = lambda: alone[0])+ prefill(row, ids[:700])+ assert bool(row._skeletons) is (layout is plain)+ row.retire(handed)+ assert not store._held and len(store) == 0 and not row._skeletons+ prefill(VLMBatchRowCache(store, "m", plain, step = STEP), ids[:700])+ assert not store._held and len(store) == 0+ # Within the budget a row banks copies as it prefills.+ store = VLMPromptSnapshotStore(max_bytes = 10**9)+ row = VLMBatchRowCache(store, "m", plain, step = STEP, alone = lambda: True)+ prefill(row, ids[:700])+ assert len(store) == 2 and not row._skeletons++def _prefill_row(state, token_ids):"""unsloth-zoo's batched row: resume the opened cache, checkpoint where a grid chunk ends."""cache, lengths = state.open(token_ids)@@ -704,6 +900,15 @@ def test_session_prefills_the_media_block_and_chains_from_it(fake_mx, caplog):session, _cache, stored = generate(200, honour_reuse = False)assert session.reused_tokens == 100 and not stored+ # Over the budget the block is not held beside the generation that produced it.+ small, prompt = VLMPromptSnapshotStore(max_bytes = 100), _block_prompt(500)+ block.store = small+ with Session(small, "m", language_model, make_cache, **kwargs) as session:+ assert session.find_prefix_length(prompt) == 100+ run_generation(language_model, prompt, session.cache, 100)+ assert not small._held+ assert session.finish() and [len(item[1]) for item in small._held._entries] == [356, 499]+store, failing = VLMPromptSnapshotStore(max_bytes = 10**9), FakeMediaBlock(100)failing.fail = Truefake_mx.clear()@@ -1091,3 +1296,54 @@ def test_a_compacted_window_updates_exactly_as_the_full_one(keep, next_rows):want = full.update_and_fetch(rows(start, n), -rows(start, n))for a, b in zip(got, want):assert a.shape == b.shape and mx.array_equal(a, b).item()+++def test_a_short_turn_holds_again_the_boundary_it_resumed_from():+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")++ def positions(start, n):+ return mx.arange(start, start + n, dtype = mx.float32).reshape(1, 1, n, 1) * mx.ones(+ (1, 2, 1, 8)+ )++ class Model:+ def __call__(+ self,+ inputs,+ cache = None,+ **_kwargs,+ ):+ start = cache_entries_offset(cache)+ for entry in cache:+ entry.update_and_fetch(+ positions(start, len(inputs)), -positions(start, len(inputs))+ )+ mx.eval([entry.state for entry in cache])++ model, ids = Model(), list(range(1000))+ make = lambda: [cache.KVCache(), cache.KVCache()]++ def turn(store, n):+ with Session(store, "m", model, make) as session:+ prefix = session.find_prefix_length(ids[:n])+ run_generation(model, ids[:n], session.cache, prefix)+ for step in range(4):+ model([0], cache = session.cache)+ session.finish()+ return prefix++ def held(store):+ for (_key, prefix), (entries, _nbytes) in store._held._entries.items():+ n = len(prefix)+ assert [entry.offset for entry in entries] == [n, n]+ assert mx.array_equal(entries[0].keys[..., :n, :], positions(0, n)).item()+ return [len(item[1]) for item in store._held._entries]++ store = VLMPromptSnapshotStore(max_bytes = 1)+ assert turn(store, 700) == 0 and held(store) == [512, 699]+ # Each short turn stays inside the 512 boundary's chunk, so it banks only its replay.+ for n in (720, 740, 760):+ assert turn(store, n) == 512+ assert held(store) == [512, n - 1]+ assert turn(store, 800) == 512 and turn(store, 800) == 799