Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Studio: share MLX VLM prompt-cache snapshot buffers and replay exact prompts (#11659)
Long Yixing32a35ae6dc89
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.pyindex d8d7c39cb84..179ed360f50 100644--- a/studio/backend/core/inference/mlx_inference.py+++ b/studio/backend/core/inference/mlx_inference.py@@ -106,7 +106,8 @@ def _is_opaque_cache(value):def _copy_value(value, mx):if isinstance(value, mx.array):- return value + 0+ # Safe to share: MLX writes in place only into an unreferenced buffer; slices are copied out.+ return mx.contiguous(value)if isinstance(value, list):return [_copy_value(item, mx) for item in value]if isinstance(value, tuple):@@ -159,7 +160,6 @@ def release_cache_entries(entries):def copy_cache_entries(entries):- """Copies, object-shallow and array-deep, evaluated so they own their data."""import mlx.core as mxfor entry in entries:@@ -225,12 +225,22 @@ def _place_per_layer_inputs(kwargs, rows, start):class _ForwardRecord:- __slots__ = ("forwards", "capture_at", "snapshot", "resume_offset", "on_resume")+ __slots__ = (+ "forwards",+ "capture_at",+ "snapshot",+ "tail_at",+ "tail",+ "resume_offset",+ "on_resume",+ )def __init__(self):self.forwards = 0self.capture_at = 0self.snapshot = None+ self.tail_at = 0+ self.tail = Noneself.resume_offset = Noneself.on_resume = None@@ -278,14 +288,19 @@ def _recording_class(base):# two (KV quantization).if (record is not None- and record.capture_at- and record.forwards == record.capture_at+ and record.forwards+ and record.forwards in (record.capture_at, record.tail_at)and cache is not None):try:- record.snapshot = copy_cache_entries(cache)+ copied = copy_cache_entries(cache)except Exception as exc:logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)+ else:+ if record.forwards == record.capture_at:+ record.snapshot = copied+ else:+ record.tail = copiedoutput = base.__call__(self, *args, **kwargs)if record is not None:record.forwards += 1@@ -344,8 +359,84 @@ class RecordingForward:return self._base.__call__(self._language_model, *args, **kwargs)+def _cache_classes(name):+ modules = (sys.modules.get(m) for m in ("mlx_vlm.models.cache", "mlx_lm.models.cache"))+ return tuple(getattr(m, name) for m in modules if hasattr(m, name))+++def _rows_in_order(entry, classes):+ """Rows are positions 0..offset-1: plain KV, or a ring that has not dropped a row yet."""+ kv, rings = classes+ if entry.keys is None:+ return False+ return type(entry) in kv or (+ type(entry) in rings and entry.offset == entry._idx == entry.keys.shape[2]+ )+++def _shared_kv_pairs(entries, base, classes, pairs):+ """All or none: False when ``base`` cannot serve every in-order entry."""+ if isinstance(entries, (list, tuple)):+ if not isinstance(base, (list, tuple)) or len(entries) != len(base):+ return False+ return all(_shared_kv_pairs(e, b, classes, pairs) for e, b in zip(entries, base))+ if type(entries) is not type(base):+ return False+ if type(entries) in classes[0] + classes[1]:+ if not _rows_in_order(entries, classes):+ return True+ if not _rows_in_order(base, classes) or entries.offset > base.offset:+ return False+ if entries.keys.shape[2] > base.keys.shape[2]:+ return False+ pairs.append((entries, base))+ return True+ nested = getattr(entries, "caches", None)+ if isinstance(nested, (list, tuple)):+ return _shared_kv_pairs(nested, base.caches, classes, pairs)+ return True+++def share_kv_rows(entries, base):+ """Point in-order entries at ``base``'s leading rows; returns bytes now read through ``base``."""+ pairs = []+ classes = (_cache_classes("KVCache"), _cache_classes("RotatingKVCache"))+ if not _shared_kv_pairs(entries, base, classes, pairs):+ return 0+ shared = 0+ for entry, source in pairs:+ rows = entry.keys.shape[2]+ entry.keys = source.keys[..., :rows, :]+ entry.values = source.values[..., :rows, :]+ shared += entry.keys.nbytes + entry.values.nbytes+ return shared+++def compact_sliding_windows(entries):+ """Drop ring rows past the window after a multi-row update; no later update reads them."""+ rings = _cache_classes("RotatingKVCache")+ pending = list(entries)+ compacted = []+ while pending:+ entry = pending.pop()+ nested = entry if isinstance(entry, (list, tuple)) else getattr(entry, "caches", None)+ if isinstance(nested, (list, tuple)):+ pending.extend(nested)+ elif type(entry) in rings and entry.keys is not None:+ excess = entry.keys.shape[2] - entry.max_size+ if excess > 0 and entry._idx == entry.keys.shape[2]:+ entry.keys = entry._trim(excess, entry.keys)+ entry.values = entry._trim(excess, entry.values)+ entry._idx = entry.max_size+ compacted.extend((entry.keys, entry.values))+ if compacted:+ import mlx.core as mx+ mx.eval(compacted)++class VLMPromptSnapshotStore:- """Boundary snapshots, most recently used last, under a byte budget."""+ """LRU boundary snapshots under a byte budget; a shorter prefix reads KV rows from its base.+ A replay (one per key) serves only a prompt of that exact length and reads through no base."""def __init__(self,@@ -355,17 +446,21 @@ class VLMPromptSnapshotStore:self._max_bytes = max_bytesself._max_entries = max_entriesself._entries = OrderedDict()+ self._bases = {}+ self._replays = {}self.nbytes = 0def __len__(self):return len(self._entries)def lookup(self, key, token_ids, limit):- """Longest stored prefix of ``token_ids`` under ``key`` ending by ``limit``."""+ """Longest stored prefix ending by ``limit``, or the replay holding all but the last token."""best = None- for (stored_key, prefix), (entries, _nbytes) in self._entries.items():+ for item, (entries, _nbytes) in self._entries.items():+ stored_key, prefix = itemn = len(prefix)- if stored_key != key or n == 0 or n > limit:+ fits = n == len(token_ids) - 1 if self._is_replay(item) else n <= limit+ if stored_key != key or n == 0 or not fits:continueif best is not None and n <= len(best[0]):continue@@ -373,10 +468,17 @@ class VLMPromptSnapshotStore:best = (prefix, entries)if best is None:return None, 0- self._entries.move_to_end((key, best[0]))+ self._touch((key, best[0]))return best[1], len(best[0])- def store(self, key, prefix_ids, entries):+ def store(+ self,+ key,+ prefix_ids,+ entries,+ replay = False,+ ):+ compact_sliding_windows(entries)nbytes = cache_entries_nbytes(entries)if nbytes > self._max_bytes:logger.debug(@@ -386,29 +488,117 @@ class VLMPromptSnapshotStore:)return Falseitem = (key, tuple(prefix_ids))- self.discard(item)+ replay = replay or self._is_replay(item)+ readers = [other for other, base in self._bases.items() if base == item]+ self._pop(item)+ self._entries[item] = (entries, nbytes)+ self.nbytes += nbytes+ replaced = self._replays.get(key)+ if replay:+ self._replays[key] = item+ longer = [other for other in reversed(self._entries) if _extends(other, item)]+ base = self._bases.get(longer[0], longer[0]) if longer and not replay else item+ rebased = {+ other+ for other in list(self._entries)+ if other != base+ and other not in (replaced, self._replays.get(key))+ and (other == item or _extends(item, other))+ and self._rebase(other, base)+ }+ for other in readers:+ if other not in rebased:+ self.discard(other)+ if replay and replaced not in (None, item):+ self._hand_down(replaced)+ self.discard(replaced)+ self._touch(item)while self._entries and (- self.nbytes + nbytes > self._max_bytes or len(self._entries) >= self._max_entries+ self.nbytes > self._max_bytes or len(self._entries) > self._max_entries):self.discard(next(iter(self._entries)))- self._entries[item] = (entries, nbytes)- self.nbytes += nbytes- return True-- def discard(self, item):+ return item in self._entries++ def _rebase(self, item, base):+ entries, nbytes = self._entries[item]+ shared = share_kv_rows(entries, self._entries[base][0])+ if shared:+ owned = cache_entries_nbytes(entries) - shared+ self._entries[item] = (entries, owned)+ self.nbytes += owned - nbytes+ self._bases[item] = base+ return bool(shared)++ def _hand_down(self, item):+ """Re-home ``item``'s readers first, else an edit's new replay drops the old branch."""+ readers = [other for other, base in self._bases.items() if base == item]+ for other in sorted(readers, key = lambda other: len(other[1]), reverse = True):+ del self._bases[other]+ longer = [+ longest+ for longest in reversed(self._entries)+ if longest != item and self._bases.get(longest) != item and _extends(longest, other)+ ]+ longer.sort(key = lambda longest: len(longest[1]), reverse = True)+ if longer and self._rebase(other, self._bases.get(longer[0], longer[0])):+ self._touch(other)+ continue+ entries, nbytes = self._entries[other]+ try:+ entries = copy_cache_entries(entries)+ except Exception as exc:+ logger.info("MLX VLM prompt cache: snapshot not handed down (%s)", exc)+ self.discard(other)+ continue+ owned = cache_entries_nbytes(entries)+ self._entries[other] = (entries, owned)+ self.nbytes += owned - nbytes+ self._entries.move_to_end(other)++ def _touch(self, item):+ self._entries.move_to_end(item)+ base = self._bases.get(item)+ if base is not None:+ # Base is evicted last.+ self._entries.move_to_end(base)++ def _is_replay(self, item):+ return self._replays.get(item[0]) == item++ def _pop(self, item):dropped = self._entries.pop(item, None)+ self._bases.pop(item, None)+ if self._is_replay(item):+ del self._replays[item[0]]if dropped is not None:self.nbytes -= dropped[1]+ def discard(self, item):+ self._pop(item)+ for other in [other for other, base in self._bases.items() if base == item]:+ self.discard(other)+def retain(self, item):- for other in [other for other in self._entries if other != item]:+ keep = (item, self._bases.get(item))+ for other in [other for other in self._entries if other not in keep]:self.discard(other)def clear(self):self._entries.clear()+ self._bases.clear()+ self._replays.clear()self.nbytes = 0+def _extends(item, other):+ (key, prefix), (other_key, other_prefix) = item, other+ return (+ key == other_key+ and len(prefix) > len(other_prefix)+ and prefix[: len(other_prefix)] == other_prefix+ )++class VLMPromptCacheSession:"""The ``prompt_cache_state`` one generation hands to mlx-vlm: ``find_prefix_length`` picks thesnapshot to resume from, swaps ``cache`` to it and arms the capture, and ``finish`` stores@@ -500,6 +690,8 @@ class VLMPromptCacheSession:record.on_resume = self._detach_servedself._keep((self._key, tuple(token_ids[:prefix_len])) if prefix_len else None)record.capture_at = (boundary - prefix_len) // self.step+ if prefix_len <= boundary < len(token_ids) - 1:+ record.tail_at = record.capture_at + 1self.reused_tokens = prefix_lenreturn prefix_len@@ -533,8 +725,24 @@ class VLMPromptCacheSession:stops reading early."""def finish(self):- snapshot = self._forward.record.snapshot- if snapshot is None or self._token_ids is None:+ if self._token_ids is None:+ return False+ record = self._forward.record+ stored = self._store_boundary(record.snapshot)+ replay = len(self._token_ids) - 1+ if (+ record.tail is not None+ and record.resume_offset == self.reused_tokens+ and cache_entries_offset(record.tail) == replay+ ):+ stored = (+ self._store.store(self._key, self._token_ids[:replay], record.tail, replay = True)+ or stored+ )+ return stored++ def _store_boundary(self, snapshot):+ if snapshot is None:return False# Read off the snapshot: a declined offer captures an earlier boundary.held = cache_entries_offset(snapshot)diff --git a/studio/backend/tests/test_mlx_vlm_prompt_cache.py b/studio/backend/tests/test_mlx_vlm_prompt_cache.pyindex e4786cb02f4..6b1836f5cf6 100644--- a/studio/backend/tests/test_mlx_vlm_prompt_cache.py+++ b/studio/backend/tests/test_mlx_vlm_prompt_cache.py@@ -55,6 +55,7 @@ def fake_mx(monkeypatch):mlx_core = types.ModuleType("mlx.core")mlx_core.array = FakeArraymlx_core.eval = lambda arrays: evaluated.append(list(arrays))+ mlx_core.contiguous = lambda array: FakeArray(array.rows)mlx_pkg = types.ModuleType("mlx")mlx_pkg.core = mlx_coremonkeypatch.setitem(sys.modules, "mlx", mlx_pkg)@@ -299,7 +300,7 @@ def test_recording_forward_copies_what_generation_converted_after_the_boundary_fsession.find_prefix_length(ids)run_generation(session._forward._language_model, ids, session.cache, 0, between = quantize)assert session.finish()- entries, prefix = store.lookup("m", ids, limit = 512)+ entries, prefix = store.lookup("m", ids + [-1], limit = 512)assert prefix == 512 and type(entries[0]) is FakeQuantizedKVassert entries[0].keys.rows == ids[:512]@@ -426,7 +427,7 @@ def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mxids = list(range(700))session, cache, stored = _generate(store, language_model, ids)assert session.reused_tokens == 0 and stored and cache[0].offset == 700- stored_entries, prefix = store.lookup("m", ids, limit = 512)+ stored_entries, prefix = store.lookup("m", ids + [-1], limit = 512)assert prefix == 512 and stored_entries[0].keys.rows == ids[:512]assert stored_entries[1].cache[0].rows == [511]store.store("m", ids[:256], _snapshot(range(9000, 9256)))@@ -441,10 +442,25 @@ def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mxassert kept[0].offset == 512 and kept[0].keys.rows == ids[:512] # the copy never movedassert kept[0].keys in fake_mx[0] # the copy, evaluated at the first forwardassert cache[0].offset == 800 and cache[0].keys.rows == longer- assert stored and store.lookup("m", longer, limit = 768)[1] == 768+ assert stored and store.lookup("m", longer + [-1], limit = 768)[1] == 768session, _cache, stored = _generate(store, language_model, longer + [1])- assert session.reused_tokens == 768 and not stored and len(store) == 4+ assert session.reused_tokens == 768 and stored and len(store) == 5+++def test_session_replays_a_prompt_from_where_its_prefill_ended(fake_mx):+ store = VLMPromptSnapshotStore(max_bytes = 10**9)+ language_model = FakeLanguageModel()+ ids = list(range(700))+ _generate(store, language_model, ids)+ assert [len(item[1]) for item in store._entries] == [512, 699]+ assert store.lookup("m", ids + [1] * 300, limit = 768)[1] == 512+ session, cache, stored = _generate(store, language_model, ids)+ assert session.reused_tokens == 699 and cache[0].offset == 700 and not stored+ assert session._forward.record.snapshot is None and session._forward.record.tail is None+ assert store.lookup("m", ids + [1] * 300, limit = 768)[1] == 512+ _generate(store, language_model, ids + [1])+ assert [len(item[1]) for item in store._entries] == [512, 700]def test_session_stores_what_the_snapshot_holds_when_reuse_was_declined(fake_mx, monkeypatch):@@ -452,17 +468,17 @@ def test_session_stores_what_the_snapshot_holds_when_reuse_was_declined(fake_mx,language_model = FakeLanguageModel()ids = list(range(1100))_generate(store, language_model, ids) # boundary 1024- original = store.lookup("m", ids, limit = 1024)[0]+ original = store.lookup("m", ids + [-1], limit = 1024)[0]store.store("t", ids[:512], _snapshot(range(9000, 9512))) # a decoyfake_mx.clear()session, _cache, stored = _generate(store, language_model, ids + [1] * 300, honour_reuse = False)assert session.reused_tokens == 1024 and stored- assert store.lookup("m", ids, limit = 256)[1] == 256+ assert store.lookup("m", ids + [-1], limit = 256)[1] == 256# The declined offer is released, not copied: it would be declined again.- assert len(store) == 2 and store.lookup("m", ids, limit = 1024)[1] == 256+ assert len(store) == 3 and store.lookup("m", ids + [-1], limit = 1024)[1] == 256assert store.lookup("t", ids, limit = 512)[1] == 512assert not list(snapshots._arrays(original, sys.modules["mlx.core"]))- assert original[0].offset == 1024 and len(fake_mx) == 1+ assert original[0].offset == 1024 and len(fake_mx) == 2# A copy that cannot be made drops the snapshot it was taken from, too.store, ids = VLMPromptSnapshotStore(max_bytes = 10**9), list(range(700))@@ -475,7 +491,7 @@ def test_session_stores_what_the_snapshot_holds_when_reuse_was_declined(fake_mx,with Session(store, "m", language_model, make_cache) as session:assert session.find_prefix_length(ids + [1]) == 512run_generation(language_model, ids + [1], session.cache, 512)- assert session.cache[0].offset == 701 and len(store) == 0+ assert session.cache[0].offset == 701 and list(store._entries) == [("m", tuple(ids[:699]))]assert not session.finish()@@ -488,7 +504,8 @@ def test_session_stores_only_snapshots_that_sit_on_the_grid(fake_mx):language_model(ids[:150], cache = session.cache)language_model(ids[150:300], cache = session.cache)language_model(ids[300:], cache = session.cache)- assert session._forward.record.snapshot is not None+ language_model([0], cache = session.cache)+ assert session._forward.record.snapshot is not None and session._forward.record.tailassert not session.finish()with Session(store, "m", language_model, lambda: [FakeState()]) as session:session.find_prefix_length(ids[:300])@@ -503,9 +520,9 @@ def test_session_stores_only_snapshots_that_sit_on_the_grid(fake_mx):session.find_prefix_length(ids)run_generation(language_model, ids, session.cache, 0)assert session.finish()- entries, prefix = store.lookup("m", ids, limit = 512)+ entries, prefix = store.lookup("m", ids + [-1], limit = 512)assert prefix == 512 and entries[1].caches[0].offset == 512- assert store.nbytes == 2 * 4 * (512 + 1 + 1)+ assert store.nbytes == 2 * 4 * (512 + 1 + 1) + 2 * 4 * (599 + 1 + 1)def _media_prompt(n):@@ -522,7 +539,7 @@ def test_session_serves_and_stores_only_prefixes_past_the_last_media_token(fake_)assert not stored and len(store) == 0 and session.reused_tokens == 0 and not fake_mx_generate(store, language_model, _media_prompt(900), media_token_ids = (9,))- assert store.lookup("m", _media_prompt(900), limit = 768)[1] == 768+ assert store.lookup("m", _media_prompt(900) + [-1], limit = 768)[1] == 768session, _cache, _stored = _generate(store, language_model, _media_prompt(1000), media_token_ids = (9,))@@ -530,16 +547,18 @@ def test_session_serves_and_stores_only_prefixes_past_the_last_media_token(fake_session, _cache, stored = _generate(store, language_model, _media_prompt(1100), honour_reuse = False, media_token_ids = (9,))- assert session.reused_tokens == 768 and not stored and len(store) == 0+ assert session.reused_tokens == 768 and not stored and len(store) == 1+ store.clear()prompt = _media_prompt(1000)_generate(store, language_model, prompt, media_token_ids = (9,))+ prompt.append(5)store.store("t", prompt[:768], _snapshot(prompt[:768]))store.store("m", [5] + prompt[1:768], _snapshot(range(768)))session = lambda **kw: Session(store, "m", language_model, make_cache, media_token_ids = (9,), **kw)- assert session().find_prefix_length(prompt) == 768 and len(store) == 3+ assert session().find_prefix_length(prompt) == 768 and len(store) == 4# A media request keeps only what serves it, and these three serve nothing.assert session(releases_unserved = True).find_prefix_length(prompt) == 768assert list(store._entries) == [("m", tuple(prompt[:768]))]@@ -585,20 +604,22 @@ def test_session_prefills_the_media_block_and_chains_from_it(fake_mx, caplog):session, cache, stored = generate(500)assert session.reused_tokens == 100 and block.prefilled == 1 and storedassert block.entries_at_prefill == 0 and session.produced_seconds > 0- assert session.produced_tokens == 100 and cache[0].offset == 500 and len(fake_mx) == 2- assert [len(item[1]) for item in store._entries] == [100, 356]+ assert session.produced_tokens == 100 and cache[0].offset == 500 and len(fake_mx) == 3+ assert [len(item[1]) for item in store._entries] == [100, 356, 499]session, _cache, stored = generate(700)assert session.reused_tokens == 356 and block.prefilled == 1 and storedassert session.produced_tokens == 0- assert [len(item[1]) for item in store._entries] == [356, 612]+ assert [len(item[1]) for item in store._entries] == [356, 612, 699]store.clear()- session, _cache, stored = generate(300) # no whole chunk past the block- assert session.reused_tokens == 100 and block.prefilled == 2 and not stored- assert [len(item[1]) for item in store._entries] == [100]+ session, _cache, stored = generate(300)+ assert session.reused_tokens == 100 and block.prefilled == 2 and stored+ assert [len(item[1]) for item in store._entries] == [100, 299]# Declined and run from zero: the capture lands at 256, off the grid; block dropped.session, _cache, stored = generate(500, honour_reuse = False)assert session.reused_tokens == 100 and block.prefilled == 2assert not stored and len(store) == 0+ session, _cache, stored = generate(200, honour_reuse = False)+ assert session.reused_tokens == 100 and not stored# A block that fails: prefilled by the caller as before, nothing captured.store, failing = VLMPromptSnapshotStore(max_bytes = 10**9), FakeMediaBlock(100)@@ -674,7 +695,7 @@ def test_session_defaults_to_mlx_vlm_prefill_step(fake_mx):assert session.find_prefix_length(ids) == 0run_generation(language_model, ids, session.cache, 0, step = session.step)assert session.finish()- assert store.lookup("m", ids, limit = len(ids))[1] == 2048+ assert store.lookup("m", ids + [-1], limit = len(ids))[1] == 2048assert shape_stable_prefix(2048) == 0 and shape_stable_prefix(2049) == 2048@@ -757,6 +778,48 @@ def test_copy_refuses_a_cache_it_cannot_walk_rather_than_sharing_it(fake_mx):copy_cache_entries([FakeCacheList(FakeSlottedKV())])+def test_copy_shares_buffers_yet_keeps_what_the_live_cache_overwrites():+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")+ np = pytest.importorskip("numpy")++ def rows(start, n):+ return mx.arange(start * 128, (start + n) * 128, dtype = mx.float32).reshape(1, 2, n, 64)++ kv, ring = cache.KVCache(), cache.RotatingKVCache(max_size = 16)+ for entry in (kv, ring):+ entry.update_and_fetch(rows(0, 16), -rows(0, 16))+ live = [kv, ring]+ mx.eval([entry.state for entry in live])+ held = [np.array(entry.keys) for entry in live]++ mx.synchronize()+ before = mx.get_active_memory()+ saved = copy_cache_entries(live)+ mx.synchronize()+ assert mx.get_active_memory() - before < cache_entries_nbytes(live) // 2++ for entry in live:+ entry.update_and_fetch(rows(16, 1), -rows(16, 1))+ mx.eval([entry.state for entry in live])+ assert ring.keys[0, 0, 0, 0].item() == 16 * 128+ for entry, keys in zip(saved, held):+ assert entry.offset == 16 and np.array_equal(np.array(entry.keys), keys)++ # A state that is a slice of a larger buffer, as a convolution window can be.+ mx.synchronize()+ before = mx.get_active_memory()+ conv = cache.ArraysCache(size = 1)+ conv[0] = mx.arange(4096 * 256, dtype = mx.float32).reshape(1, 4096, 256)[:, -3:, :]+ mx.eval(conv[0])+ window = np.array(conv[0])+ (saved,) = copy_cache_entries([conv])+ del conv+ mx.synchronize()+ assert mx.get_active_memory() - before < 4096 * 256 * 4 // 2+ assert np.array_equal(np.array(saved[0]), window)++def test_session_restores_the_model_when_one_host_is_named_twice(fake_mx):"""``policy_hosts`` is (model, language_model), and the second falls back to thefirst when the wrapper exposes no language model, so the same object can be listed@@ -781,3 +844,169 @@ def test_session_restores_the_model_even_if_unpatching_fails(fake_mx):del host.chunked_prefill_policysession.__exit__(None, None, None)assert type(host) is before+++def test_store_holds_the_kv_rows_nested_snapshots_share_once():+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")++ def advance(entries, start, n):+ for entry in entries[:2]:+ rows = mx.arange(start * 128, (start + n) * 128, dtype = mx.float32).reshape(1, 2, n, 64)+ entry.update_and_fetch(rows, -rows)+ entries[2][0] = mx.full((1, 64), float(start + n))+ mx.eval([entry.state for entry in entries])++ short = [cache.KVCache(), cache.RotatingKVCache(max_size = 4096, keep = 4)]+ short.append(cache.ArraysCache(size = 1))+ advance(short, 0, 256)+ served = [entry.state[0] * 1 for entry in short[:2]]+ mx.eval(served)+ extended = copy_cache_entries(short)+ advance(extended, 256, 256)+ ids = list(range(1000, 1512))+ shared = sum(entry.keys.nbytes + entry.values.nbytes for entry in short[:2])++ store = VLMPromptSnapshotStore(1 << 30)+ store.store("m", ids[:256], short)+ mx.synchronize()+ before = mx.get_active_memory()+ store.store("m", ids, extended)+ mx.eval([entry.state for entry in short])+ mx.synchronize()+ assert before - mx.get_active_memory() >= shared+ assert store.nbytes == cache_entries_nbytes(extended) + cache_entries_nbytes(short) - shared++ entries, reused = store.lookup("m", ids[:300], 300)+ assert reused == 256 and entries is short+ for entry, rows in zip(entries, served):+ assert mx.array_equal(entry.state[0], rows).item()+ assert entries[2][0][0, 0].item() == 256.0+ base = ("m", tuple(ids))+ assert list(store._entries)[-1] == base+ store.retain(("m", tuple(ids[:256])))+ assert len(store) == 2+ store.discard(base)+ assert len(store) == 0 and store.nbytes == 0++ store.store("m", ids[:256], short)+ store.store("m", ids, extended)+ store.store("m", ids, copy_cache_entries(extended))+ assert len(store) == 2+ lone = [cache.ArraysCache(size = 1)]+ lone[0][0] = mx.zeros((1, 64))+ store.store("m", ids, lone)+ assert len(store) == 1 and store.nbytes == cache_entries_nbytes(lone)++ windows = [[cache.RotatingKVCache(max_size = 16)] for _ in range(2)]+ for entries, n in zip(windows, (32, 64)):+ entries.append(cache.KVCache())+ entries.append(cache.ArraysCache(size = 1))+ advance(entries, 0, n // 2)+ advance(entries, n // 2, n // 2)+ assert snapshots.share_kv_rows(*windows) == windows[0][1].keys.nbytes * 2+ nested = [[cache.KVCache(), cache.RotatingKVCache(max_size = 16)] for _ in range(3)]+ for entries, n in zip(nested, (8, 12, 32)):+ entries.append(cache.ArraysCache(size = 1))+ advance(entries, 0, min(n, 16))+ if n > 16:+ advance(entries, 16, n - 16)+ store.store("w", ids[:n], entries)+ store.discard(("w", tuple(ids[:12])))+ assert len(store) == 2+ assert store.nbytes == cache_entries_nbytes(lone) + cache_entries_nbytes(nested[2])+++def test_a_replay_reads_through_no_snapshot_yet_serves_as_a_base():+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")++ def entries(n, last_chunk):+ rows = mx.arange(n * 64, dtype = mx.float32).reshape(1, 1, n, 64)+ rows = rows + last_chunk * (mx.arange(n) >= 512).reshape(1, 1, n, 1)+ kv = cache.KVCache()+ kv.update_and_fetch(rows, -rows)+ mx.eval(kv.keys, kv.values)+ return [kv]++ ids = list(range(1000))+ for order in ((600, 768), (768, 600)):+ store = VLMPromptSnapshotStore(1 << 30)+ for n in order:+ store.store("m", ids[:n], entries(n, 0.5 if n == 600 else 0.0), replay = n == 600)+ replay, n = store.lookup("m", ids[:601], limit = 512)+ assert n == 600 and replay[0].keys[0, 0, 599, 0].item() == 599 * 64 + 0.5+ grid = ("m", tuple(ids[:512]))+ store.store("m", ids[:512], entries(512, 0.0))+ assert store._bases[grid] == ("m", tuple(ids[:600])) and store._entries[grid][1] == 0+ store.store("m", ids[:580], entries(580, 0.25), replay = True)+ assert ("m", tuple(ids[:600])) not in store._entries+ assert store._bases[grid] == ("m", tuple(ids[:580])) and store._entries[grid][1] == 0+ divergent = ids[:300] + [-1] * 700+ store.store("m", divergent[:590], entries(590, 0.75), replay = True)+ longer = ("m", tuple(ids[:768]))+ assert set(store._entries) == {grid, longer, ("m", tuple(divergent[:590]))}+ assert store._bases[grid] == longer and store._entries[grid][1] == 0+ assert store._entries[grid][0][0].keys[0, 0, 511, 0].item() == 511 * 64+ assert store.nbytes == cache_entries_nbytes(store._entries[longer][0]) + cache_entries_nbytes(+ store._entries[("m", tuple(divergent[:590]))][0]+ )+ store.discard(("m", tuple(divergent[:590])))+ assert not store._replays+++def test_an_edit_keeps_the_old_branch_to_return_to():+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")++ def entries(ids):+ rows = mx.array(ids, dtype = mx.float32).reshape(1, 1, len(ids), 1)+ kv = cache.KVCache()+ kv.update_and_fetch(rows, -rows)+ mx.eval(kv.keys, kv.values)+ return [kv]++ def request(ids):+ _, reused = store.lookup("m", ids, len(ids) - 1)+ for n in range(256, len(ids), 256):+ if n > reused:+ store.store("m", ids[:n], entries(ids[:n]))+ store.store("m", ids[:-1], entries(ids[:-1]), replay = True)+ return reused++ store = VLMPromptSnapshotStore(1 << 30)+ turns = [list(range(n)) for n in (557, 931, 1311, 1674)]+ for ids in turns:+ request(ids)+ assert request(turns[0] + [-1] * 393) == 512+ kept, reused = store.lookup("m", turns[3], 1673)+ assert reused == 1536+ assert mx.array_equal(kept[0].keys[0, 0, :, 0], mx.arange(1536, dtype = mx.float32)).item()+ assert store.nbytes == sum(nbytes for _, nbytes in store._entries.values())+ assert all(base in store._entries for base in store._bases.values())+ store.clear()++[email protected]("keep", [0, 4])[email protected]("next_rows", [1, 5])+def test_a_compacted_window_updates_exactly_as_the_full_one(keep, next_rows):+ mx = pytest.importorskip("mlx.core")+ cache = pytest.importorskip("mlx_vlm.models.cache")++ def rows(start, n):+ return mx.arange(start * 8, (start + n) * 8, dtype = mx.float32).reshape(1, 1, n, 8)++ def window():+ ring = cache.RotatingKVCache(max_size = 16, keep = keep)+ for start in (0, 24):+ ring.update_and_fetch(rows(start, 24), -rows(start, 24))+ return ring++ full, compact = window(), window()+ VLMPromptSnapshotStore(1 << 30).store("m", [1], [[compact]])+ assert compact.keys.shape[2] == 16 < full.keys.shape[2]+ for start, n in ((48, next_rows), (48 + next_rows, 1)):+ got = compact.update_and_fetch(rows(start, n), -rows(start, n))+ want = full.update_and_fetch(rows(start, n), -rows(start, n))+ for a, b in zip(got, want):+ assert a.shape == b.shape and mx.array_equal(a, b).item()