Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Studio: share MLX VLM prompt-cache snapshot buffers and replay exact prompts (#11659)

Long Yixing32a35ae6dc89
diff --git a/studio/backend/core/inference/mlx_inference.py b/studio/backend/core/inference/mlx_inference.py
index d8d7c39cb84..179ed360f50 100644
--- a/studio/backend/core/inference/mlx_inference.py
+++ b/studio/backend/core/inference/mlx_inference.py
@@ -106,7 +106,8 @@ def _is_opaque_cache(value):
def _copy_value(value, mx):
if isinstance(value, mx.array):
- return value + 0
+ # Safe to share: MLX writes in place only into an unreferenced buffer; slices are copied out.
+ return mx.contiguous(value)
if isinstance(value, list):
return [_copy_value(item, mx) for item in value]
if isinstance(value, tuple):
@@ -159,7 +160,6 @@ def release_cache_entries(entries):
def copy_cache_entries(entries):
- """Copies, object-shallow and array-deep, evaluated so they own their data."""
import mlx.core as mx
for entry in entries:
@@ -225,12 +225,22 @@ def _place_per_layer_inputs(kwargs, rows, start):
class _ForwardRecord:
- __slots__ = ("forwards", "capture_at", "snapshot", "resume_offset", "on_resume")
+ __slots__ = (
+ "forwards",
+ "capture_at",
+ "snapshot",
+ "tail_at",
+ "tail",
+ "resume_offset",
+ "on_resume",
+ )
def __init__(self):
self.forwards = 0
self.capture_at = 0
self.snapshot = None
+ self.tail_at = 0
+ self.tail = None
self.resume_offset = None
self.on_resume = None
@@ -278,14 +288,19 @@ def _recording_class(base):
# two (KV quantization).
if (
record is not None
- and record.capture_at
- and record.forwards == record.capture_at
+ and record.forwards
+ and record.forwards in (record.capture_at, record.tail_at)
and cache is not None
):
try:
- record.snapshot = copy_cache_entries(cache)
+ copied = copy_cache_entries(cache)
except Exception as exc:
logger.info("MLX VLM prompt cache: cache layout not copied (%s)", exc)
+ else:
+ if record.forwards == record.capture_at:
+ record.snapshot = copied
+ else:
+ record.tail = copied
output = base.__call__(self, *args, **kwargs)
if record is not None:
record.forwards += 1
@@ -344,8 +359,84 @@ class RecordingForward:
return self._base.__call__(self._language_model, *args, **kwargs)
+def _cache_classes(name):
+ modules = (sys.modules.get(m) for m in ("mlx_vlm.models.cache", "mlx_lm.models.cache"))
+ return tuple(getattr(m, name) for m in modules if hasattr(m, name))
+
+
+def _rows_in_order(entry, classes):
+ """Rows are positions 0..offset-1: plain KV, or a ring that has not dropped a row yet."""
+ kv, rings = classes
+ if entry.keys is None:
+ return False
+ return type(entry) in kv or (
+ type(entry) in rings and entry.offset == entry._idx == entry.keys.shape[2]
+ )
+
+
+def _shared_kv_pairs(entries, base, classes, pairs):
+ """All or none: False when ``base`` cannot serve every in-order entry."""
+ if isinstance(entries, (list, tuple)):
+ if not isinstance(base, (list, tuple)) or len(entries) != len(base):
+ return False
+ return all(_shared_kv_pairs(e, b, classes, pairs) for e, b in zip(entries, base))
+ if type(entries) is not type(base):
+ return False
+ if type(entries) in classes[0] + classes[1]:
+ if not _rows_in_order(entries, classes):
+ return True
+ if not _rows_in_order(base, classes) or entries.offset > base.offset:
+ return False
+ if entries.keys.shape[2] > base.keys.shape[2]:
+ return False
+ pairs.append((entries, base))
+ return True
+ nested = getattr(entries, "caches", None)
+ if isinstance(nested, (list, tuple)):
+ return _shared_kv_pairs(nested, base.caches, classes, pairs)
+ return True
+
+
+def share_kv_rows(entries, base):
+ """Point in-order entries at ``base``'s leading rows; returns bytes now read through ``base``."""
+ pairs = []
+ classes = (_cache_classes("KVCache"), _cache_classes("RotatingKVCache"))
+ if not _shared_kv_pairs(entries, base, classes, pairs):
+ return 0
+ shared = 0
+ for entry, source in pairs:
+ rows = entry.keys.shape[2]
+ entry.keys = source.keys[..., :rows, :]
+ entry.values = source.values[..., :rows, :]
+ shared += entry.keys.nbytes + entry.values.nbytes
+ return shared
+
+
+def compact_sliding_windows(entries):
+ """Drop ring rows past the window after a multi-row update; no later update reads them."""
+ rings = _cache_classes("RotatingKVCache")
+ pending = list(entries)
+ compacted = []
+ while pending:
+ entry = pending.pop()
+ nested = entry if isinstance(entry, (list, tuple)) else getattr(entry, "caches", None)
+ if isinstance(nested, (list, tuple)):
+ pending.extend(nested)
+ elif type(entry) in rings and entry.keys is not None:
+ excess = entry.keys.shape[2] - entry.max_size
+ if excess > 0 and entry._idx == entry.keys.shape[2]:
+ entry.keys = entry._trim(excess, entry.keys)
+ entry.values = entry._trim(excess, entry.values)
+ entry._idx = entry.max_size
+ compacted.extend((entry.keys, entry.values))
+ if compacted:
+ import mlx.core as mx
+ mx.eval(compacted)
+
+
class VLMPromptSnapshotStore:
- """Boundary snapshots, most recently used last, under a byte budget."""
+ """LRU boundary snapshots under a byte budget; a shorter prefix reads KV rows from its base.
+ A replay (one per key) serves only a prompt of that exact length and reads through no base."""
def __init__(
self,
@@ -355,17 +446,21 @@ class VLMPromptSnapshotStore:
self._max_bytes = max_bytes
self._max_entries = max_entries
self._entries = OrderedDict()
+ self._bases = {}
+ self._replays = {}
self.nbytes = 0
def __len__(self):
return len(self._entries)
def lookup(self, key, token_ids, limit):
- """Longest stored prefix of ``token_ids`` under ``key`` ending by ``limit``."""
+ """Longest stored prefix ending by ``limit``, or the replay holding all but the last token."""
best = None
- for (stored_key, prefix), (entries, _nbytes) in self._entries.items():
+ for item, (entries, _nbytes) in self._entries.items():
+ stored_key, prefix = item
n = len(prefix)
- if stored_key != key or n == 0 or n > limit:
+ fits = n == len(token_ids) - 1 if self._is_replay(item) else n <= limit
+ if stored_key != key or n == 0 or not fits:
continue
if best is not None and n <= len(best[0]):
continue
@@ -373,10 +468,17 @@ class VLMPromptSnapshotStore:
best = (prefix, entries)
if best is None:
return None, 0
- self._entries.move_to_end((key, best[0]))
+ self._touch((key, best[0]))
return best[1], len(best[0])
- def store(self, key, prefix_ids, entries):
+ def store(
+ self,
+ key,
+ prefix_ids,
+ entries,
+ replay = False,
+ ):
+ compact_sliding_windows(entries)
nbytes = cache_entries_nbytes(entries)
if nbytes > self._max_bytes:
logger.debug(
@@ -386,29 +488,117 @@ class VLMPromptSnapshotStore:
)
return False
item = (key, tuple(prefix_ids))
- self.discard(item)
+ replay = replay or self._is_replay(item)
+ readers = [other for other, base in self._bases.items() if base == item]
+ self._pop(item)
+ self._entries[item] = (entries, nbytes)
+ self.nbytes += nbytes
+ replaced = self._replays.get(key)
+ if replay:
+ self._replays[key] = item
+ longer = [other for other in reversed(self._entries) if _extends(other, item)]
+ base = self._bases.get(longer[0], longer[0]) if longer and not replay else item
+ rebased = {
+ other
+ for other in list(self._entries)
+ if other != base
+ and other not in (replaced, self._replays.get(key))
+ and (other == item or _extends(item, other))
+ and self._rebase(other, base)
+ }
+ for other in readers:
+ if other not in rebased:
+ self.discard(other)
+ if replay and replaced not in (None, item):
+ self._hand_down(replaced)
+ self.discard(replaced)
+ self._touch(item)
while self._entries and (
- self.nbytes + nbytes > self._max_bytes or len(self._entries) >= self._max_entries
+ self.nbytes > self._max_bytes or len(self._entries) > self._max_entries
):
self.discard(next(iter(self._entries)))
- self._entries[item] = (entries, nbytes)
- self.nbytes += nbytes
- return True
-
- def discard(self, item):
+ return item in self._entries
+
+ def _rebase(self, item, base):
+ entries, nbytes = self._entries[item]
+ shared = share_kv_rows(entries, self._entries[base][0])
+ if shared:
+ owned = cache_entries_nbytes(entries) - shared
+ self._entries[item] = (entries, owned)
+ self.nbytes += owned - nbytes
+ self._bases[item] = base
+ return bool(shared)
+
+ def _hand_down(self, item):
+ """Re-home ``item``'s readers first, else an edit's new replay drops the old branch."""
+ readers = [other for other, base in self._bases.items() if base == item]
+ for other in sorted(readers, key = lambda other: len(other[1]), reverse = True):
+ del self._bases[other]
+ longer = [
+ longest
+ for longest in reversed(self._entries)
+ if longest != item and self._bases.get(longest) != item and _extends(longest, other)
+ ]
+ longer.sort(key = lambda longest: len(longest[1]), reverse = True)
+ if longer and self._rebase(other, self._bases.get(longer[0], longer[0])):
+ self._touch(other)
+ continue
+ entries, nbytes = self._entries[other]
+ try:
+ entries = copy_cache_entries(entries)
+ except Exception as exc:
+ logger.info("MLX VLM prompt cache: snapshot not handed down (%s)", exc)
+ self.discard(other)
+ continue
+ owned = cache_entries_nbytes(entries)
+ self._entries[other] = (entries, owned)
+ self.nbytes += owned - nbytes
+ self._entries.move_to_end(other)
+
+ def _touch(self, item):
+ self._entries.move_to_end(item)
+ base = self._bases.get(item)
+ if base is not None:
+ # Base is evicted last.
+ self._entries.move_to_end(base)
+
+ def _is_replay(self, item):
+ return self._replays.get(item[0]) == item
+
+ def _pop(self, item):
dropped = self._entries.pop(item, None)
+ self._bases.pop(item, None)
+ if self._is_replay(item):
+ del self._replays[item[0]]
if dropped is not None:
self.nbytes -= dropped[1]
+ def discard(self, item):
+ self._pop(item)
+ for other in [other for other, base in self._bases.items() if base == item]:
+ self.discard(other)
+
def retain(self, item):
- for other in [other for other in self._entries if other != item]:
+ keep = (item, self._bases.get(item))
+ for other in [other for other in self._entries if other not in keep]:
self.discard(other)
def clear(self):
self._entries.clear()
+ self._bases.clear()
+ self._replays.clear()
self.nbytes = 0
+def _extends(item, other):
+ (key, prefix), (other_key, other_prefix) = item, other
+ return (
+ key == other_key
+ and len(prefix) > len(other_prefix)
+ and prefix[: len(other_prefix)] == other_prefix
+ )
+
+
class VLMPromptCacheSession:
"""The ``prompt_cache_state`` one generation hands to mlx-vlm: ``find_prefix_length`` picks the
snapshot to resume from, swaps ``cache`` to it and arms the capture, and ``finish`` stores
@@ -500,6 +690,8 @@ class VLMPromptCacheSession:
record.on_resume = self._detach_served
self._keep((self._key, tuple(token_ids[:prefix_len])) if prefix_len else None)
record.capture_at = (boundary - prefix_len) // self.step
+ if prefix_len <= boundary < len(token_ids) - 1:
+ record.tail_at = record.capture_at + 1
self.reused_tokens = prefix_len
return prefix_len
@@ -533,8 +725,24 @@ class VLMPromptCacheSession:
stops reading early."""
def finish(self):
- snapshot = self._forward.record.snapshot
- if snapshot is None or self._token_ids is None:
+ if self._token_ids is None:
+ return False
+ record = self._forward.record
+ stored = self._store_boundary(record.snapshot)
+ replay = len(self._token_ids) - 1
+ if (
+ record.tail is not None
+ and record.resume_offset == self.reused_tokens
+ and cache_entries_offset(record.tail) == replay
+ ):
+ stored = (
+ self._store.store(self._key, self._token_ids[:replay], record.tail, replay = True)
+ or stored
+ )
+ return stored
+
+ def _store_boundary(self, snapshot):
+ if snapshot is None:
return False
# Read off the snapshot: a declined offer captures an earlier boundary.
held = cache_entries_offset(snapshot)
diff --git a/studio/backend/tests/test_mlx_vlm_prompt_cache.py b/studio/backend/tests/test_mlx_vlm_prompt_cache.py
index e4786cb02f4..6b1836f5cf6 100644
--- a/studio/backend/tests/test_mlx_vlm_prompt_cache.py
+++ b/studio/backend/tests/test_mlx_vlm_prompt_cache.py
@@ -55,6 +55,7 @@ def fake_mx(monkeypatch):
mlx_core = types.ModuleType("mlx.core")
mlx_core.array = FakeArray
mlx_core.eval = lambda arrays: evaluated.append(list(arrays))
+ mlx_core.contiguous = lambda array: FakeArray(array.rows)
mlx_pkg = types.ModuleType("mlx")
mlx_pkg.core = mlx_core
monkeypatch.setitem(sys.modules, "mlx", mlx_pkg)
@@ -299,7 +300,7 @@ def test_recording_forward_copies_what_generation_converted_after_the_boundary_f
session.find_prefix_length(ids)
run_generation(session._forward._language_model, ids, session.cache, 0, between = quantize)
assert session.finish()
- entries, prefix = store.lookup("m", ids, limit = 512)
+ entries, prefix = store.lookup("m", ids + [-1], limit = 512)
assert prefix == 512 and type(entries[0]) is FakeQuantizedKV
assert entries[0].keys.rows == ids[:512]
@@ -426,7 +427,7 @@ def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mx
ids = list(range(700))
session, cache, stored = _generate(store, language_model, ids)
assert session.reused_tokens == 0 and stored and cache[0].offset == 700
- stored_entries, prefix = store.lookup("m", ids, limit = 512)
+ stored_entries, prefix = store.lookup("m", ids + [-1], limit = 512)
assert prefix == 512 and stored_entries[0].keys.rows == ids[:512]
assert stored_entries[1].cache[0].rows == [511]
store.store("m", ids[:256], _snapshot(range(9000, 9256)))
@@ -441,10 +442,25 @@ def test_session_captures_the_boundary_and_resumes_leaving_a_copy_behind(fake_mx
assert kept[0].offset == 512 and kept[0].keys.rows == ids[:512] # the copy never moved
assert kept[0].keys in fake_mx[0] # the copy, evaluated at the first forward
assert cache[0].offset == 800 and cache[0].keys.rows == longer
- assert stored and store.lookup("m", longer, limit = 768)[1] == 768
+ assert stored and store.lookup("m", longer + [-1], limit = 768)[1] == 768
session, _cache, stored = _generate(store, language_model, longer + [1])
- assert session.reused_tokens == 768 and not stored and len(store) == 4
+ assert session.reused_tokens == 768 and stored and len(store) == 5
+
+
+def test_session_replays_a_prompt_from_where_its_prefill_ended(fake_mx):
+ store = VLMPromptSnapshotStore(max_bytes = 10**9)
+ language_model = FakeLanguageModel()
+ ids = list(range(700))
+ _generate(store, language_model, ids)
+ assert [len(item[1]) for item in store._entries] == [512, 699]
+ assert store.lookup("m", ids + [1] * 300, limit = 768)[1] == 512
+ session, cache, stored = _generate(store, language_model, ids)
+ assert session.reused_tokens == 699 and cache[0].offset == 700 and not stored
+ assert session._forward.record.snapshot is None and session._forward.record.tail is None
+ assert store.lookup("m", ids + [1] * 300, limit = 768)[1] == 512
+ _generate(store, language_model, ids + [1])
+ assert [len(item[1]) for item in store._entries] == [512, 700]
def test_session_stores_what_the_snapshot_holds_when_reuse_was_declined(fake_mx, monkeypatch):
@@ -452,17 +468,17 @@ def test_session_stores_what_the_snapshot_holds_when_reuse_was_declined(fake_mx,
language_model = FakeLanguageModel()
ids = list(range(1100))
_generate(store, language_model, ids) # boundary 1024
- original = store.lookup("m", ids, limit = 1024)[0]
+ original = store.lookup("m", ids + [-1], limit = 1024)[0]
store.store("t", ids[:512], _snapshot(range(9000, 9512))) # a decoy
fake_mx.clear()
session, _cache, stored = _generate(store, language_model, ids + [1] * 300, honour_reuse = False)
assert session.reused_tokens == 1024 and stored
- assert store.lookup("m", ids, limit = 256)[1] == 256
+ assert store.lookup("m", ids + [-1], limit = 256)[1] == 256
# The declined offer is released, not copied: it would be declined again.
- assert len(store) == 2 and store.lookup("m", ids, limit = 1024)[1] == 256
+ assert len(store) == 3 and store.lookup("m", ids + [-1], limit = 1024)[1] == 256
assert store.lookup("t", ids, limit = 512)[1] == 512
assert not list(snapshots._arrays(original, sys.modules["mlx.core"]))
- assert original[0].offset == 1024 and len(fake_mx) == 1
+ assert original[0].offset == 1024 and len(fake_mx) == 2
# A copy that cannot be made drops the snapshot it was taken from, too.
store, ids = VLMPromptSnapshotStore(max_bytes = 10**9), list(range(700))
@@ -475,7 +491,7 @@ def test_session_stores_what_the_snapshot_holds_when_reuse_was_declined(fake_mx,
with Session(store, "m", language_model, make_cache) as session:
assert session.find_prefix_length(ids + [1]) == 512
run_generation(language_model, ids + [1], session.cache, 512)
- assert session.cache[0].offset == 701 and len(store) == 0
+ assert session.cache[0].offset == 701 and list(store._entries) == [("m", tuple(ids[:699]))]
assert not session.finish()
@@ -488,7 +504,8 @@ def test_session_stores_only_snapshots_that_sit_on_the_grid(fake_mx):
language_model(ids[:150], cache = session.cache)
language_model(ids[150:300], cache = session.cache)
language_model(ids[300:], cache = session.cache)
- assert session._forward.record.snapshot is not None
+ language_model([0], cache = session.cache)
+ assert session._forward.record.snapshot is not None and session._forward.record.tail
assert not session.finish()
with Session(store, "m", language_model, lambda: [FakeState()]) as session:
session.find_prefix_length(ids[:300])
@@ -503,9 +520,9 @@ def test_session_stores_only_snapshots_that_sit_on_the_grid(fake_mx):
session.find_prefix_length(ids)
run_generation(language_model, ids, session.cache, 0)
assert session.finish()
- entries, prefix = store.lookup("m", ids, limit = 512)
+ entries, prefix = store.lookup("m", ids + [-1], limit = 512)
assert prefix == 512 and entries[1].caches[0].offset == 512
- assert store.nbytes == 2 * 4 * (512 + 1 + 1)
+ assert store.nbytes == 2 * 4 * (512 + 1 + 1) + 2 * 4 * (599 + 1 + 1)
def _media_prompt(n):
@@ -522,7 +539,7 @@ def test_session_serves_and_stores_only_prefixes_past_the_last_media_token(fake_
)
assert not stored and len(store) == 0 and session.reused_tokens == 0 and not fake_mx
_generate(store, language_model, _media_prompt(900), media_token_ids = (9,))
- assert store.lookup("m", _media_prompt(900), limit = 768)[1] == 768
+ assert store.lookup("m", _media_prompt(900) + [-1], limit = 768)[1] == 768
session, _cache, _stored = _generate(
store, language_model, _media_prompt(1000), media_token_ids = (9,)
)
@@ -530,16 +547,18 @@ def test_session_serves_and_stores_only_prefixes_past_the_last_media_token(fake_
session, _cache, stored = _generate(
store, language_model, _media_prompt(1100), honour_reuse = False, media_token_ids = (9,)
)
- assert session.reused_tokens == 768 and not stored and len(store) == 0
+ assert session.reused_tokens == 768 and not stored and len(store) == 1
+ store.clear()
prompt = _media_prompt(1000)
_generate(store, language_model, prompt, media_token_ids = (9,))
+ prompt.append(5)
store.store("t", prompt[:768], _snapshot(prompt[:768]))
store.store("m", [5] + prompt[1:768], _snapshot(range(768)))
session = lambda **kw: Session(
store, "m", language_model, make_cache, media_token_ids = (9,), **kw
)
- assert session().find_prefix_length(prompt) == 768 and len(store) == 3
+ assert session().find_prefix_length(prompt) == 768 and len(store) == 4
# A media request keeps only what serves it, and these three serve nothing.
assert session(releases_unserved = True).find_prefix_length(prompt) == 768
assert list(store._entries) == [("m", tuple(prompt[:768]))]
@@ -585,20 +604,22 @@ def test_session_prefills_the_media_block_and_chains_from_it(fake_mx, caplog):
session, cache, stored = generate(500)
assert session.reused_tokens == 100 and block.prefilled == 1 and stored
assert block.entries_at_prefill == 0 and session.produced_seconds > 0
- assert session.produced_tokens == 100 and cache[0].offset == 500 and len(fake_mx) == 2
- assert [len(item[1]) for item in store._entries] == [100, 356]
+ assert session.produced_tokens == 100 and cache[0].offset == 500 and len(fake_mx) == 3
+ assert [len(item[1]) for item in store._entries] == [100, 356, 499]
session, _cache, stored = generate(700)
assert session.reused_tokens == 356 and block.prefilled == 1 and stored
assert session.produced_tokens == 0
- assert [len(item[1]) for item in store._entries] == [356, 612]
+ assert [len(item[1]) for item in store._entries] == [356, 612, 699]
store.clear()
- session, _cache, stored = generate(300) # no whole chunk past the block
- assert session.reused_tokens == 100 and block.prefilled == 2 and not stored
- assert [len(item[1]) for item in store._entries] == [100]
+ session, _cache, stored = generate(300)
+ assert session.reused_tokens == 100 and block.prefilled == 2 and stored
+ assert [len(item[1]) for item in store._entries] == [100, 299]
# Declined and run from zero: the capture lands at 256, off the grid; block dropped.
session, _cache, stored = generate(500, honour_reuse = False)
assert session.reused_tokens == 100 and block.prefilled == 2
assert not stored and len(store) == 0
+ session, _cache, stored = generate(200, honour_reuse = False)
+ assert session.reused_tokens == 100 and not stored
# A block that fails: prefilled by the caller as before, nothing captured.
store, failing = VLMPromptSnapshotStore(max_bytes = 10**9), FakeMediaBlock(100)
@@ -674,7 +695,7 @@ def test_session_defaults_to_mlx_vlm_prefill_step(fake_mx):
assert session.find_prefix_length(ids) == 0
run_generation(language_model, ids, session.cache, 0, step = session.step)
assert session.finish()
- assert store.lookup("m", ids, limit = len(ids))[1] == 2048
+ assert store.lookup("m", ids + [-1], limit = len(ids))[1] == 2048
assert shape_stable_prefix(2048) == 0 and shape_stable_prefix(2049) == 2048
@@ -757,6 +778,48 @@ def test_copy_refuses_a_cache_it_cannot_walk_rather_than_sharing_it(fake_mx):
copy_cache_entries([FakeCacheList(FakeSlottedKV())])
+def test_copy_shares_buffers_yet_keeps_what_the_live_cache_overwrites():
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+ np = pytest.importorskip("numpy")
+
+ def rows(start, n):
+ return mx.arange(start * 128, (start + n) * 128, dtype = mx.float32).reshape(1, 2, n, 64)
+
+ kv, ring = cache.KVCache(), cache.RotatingKVCache(max_size = 16)
+ for entry in (kv, ring):
+ entry.update_and_fetch(rows(0, 16), -rows(0, 16))
+ live = [kv, ring]
+ mx.eval([entry.state for entry in live])
+ held = [np.array(entry.keys) for entry in live]
+
+ mx.synchronize()
+ before = mx.get_active_memory()
+ saved = copy_cache_entries(live)
+ mx.synchronize()
+ assert mx.get_active_memory() - before < cache_entries_nbytes(live) // 2
+
+ for entry in live:
+ entry.update_and_fetch(rows(16, 1), -rows(16, 1))
+ mx.eval([entry.state for entry in live])
+ assert ring.keys[0, 0, 0, 0].item() == 16 * 128
+ for entry, keys in zip(saved, held):
+ assert entry.offset == 16 and np.array_equal(np.array(entry.keys), keys)
+
+ # A state that is a slice of a larger buffer, as a convolution window can be.
+ mx.synchronize()
+ before = mx.get_active_memory()
+ conv = cache.ArraysCache(size = 1)
+ conv[0] = mx.arange(4096 * 256, dtype = mx.float32).reshape(1, 4096, 256)[:, -3:, :]
+ mx.eval(conv[0])
+ window = np.array(conv[0])
+ (saved,) = copy_cache_entries([conv])
+ del conv
+ mx.synchronize()
+ assert mx.get_active_memory() - before < 4096 * 256 * 4 // 2
+ assert np.array_equal(np.array(saved[0]), window)
+
+
def test_session_restores_the_model_when_one_host_is_named_twice(fake_mx):
"""``policy_hosts`` is (model, language_model), and the second falls back to the
first when the wrapper exposes no language model, so the same object can be listed
@@ -781,3 +844,169 @@ def test_session_restores_the_model_even_if_unpatching_fails(fake_mx):
del host.chunked_prefill_policy
session.__exit__(None, None, None)
assert type(host) is before
+
+
+def test_store_holds_the_kv_rows_nested_snapshots_share_once():
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+
+ def advance(entries, start, n):
+ for entry in entries[:2]:
+ rows = mx.arange(start * 128, (start + n) * 128, dtype = mx.float32).reshape(1, 2, n, 64)
+ entry.update_and_fetch(rows, -rows)
+ entries[2][0] = mx.full((1, 64), float(start + n))
+ mx.eval([entry.state for entry in entries])
+
+ short = [cache.KVCache(), cache.RotatingKVCache(max_size = 4096, keep = 4)]
+ short.append(cache.ArraysCache(size = 1))
+ advance(short, 0, 256)
+ served = [entry.state[0] * 1 for entry in short[:2]]
+ mx.eval(served)
+ extended = copy_cache_entries(short)
+ advance(extended, 256, 256)
+ ids = list(range(1000, 1512))
+ shared = sum(entry.keys.nbytes + entry.values.nbytes for entry in short[:2])
+
+ store = VLMPromptSnapshotStore(1 << 30)
+ store.store("m", ids[:256], short)
+ mx.synchronize()
+ before = mx.get_active_memory()
+ store.store("m", ids, extended)
+ mx.eval([entry.state for entry in short])
+ mx.synchronize()
+ assert before - mx.get_active_memory() >= shared
+ assert store.nbytes == cache_entries_nbytes(extended) + cache_entries_nbytes(short) - shared
+
+ entries, reused = store.lookup("m", ids[:300], 300)
+ assert reused == 256 and entries is short
+ for entry, rows in zip(entries, served):
+ assert mx.array_equal(entry.state[0], rows).item()
+ assert entries[2][0][0, 0].item() == 256.0
+ base = ("m", tuple(ids))
+ assert list(store._entries)[-1] == base
+ store.retain(("m", tuple(ids[:256])))
+ assert len(store) == 2
+ store.discard(base)
+ assert len(store) == 0 and store.nbytes == 0
+
+ store.store("m", ids[:256], short)
+ store.store("m", ids, extended)
+ store.store("m", ids, copy_cache_entries(extended))
+ assert len(store) == 2
+ lone = [cache.ArraysCache(size = 1)]
+ lone[0][0] = mx.zeros((1, 64))
+ store.store("m", ids, lone)
+ assert len(store) == 1 and store.nbytes == cache_entries_nbytes(lone)
+
+ windows = [[cache.RotatingKVCache(max_size = 16)] for _ in range(2)]
+ for entries, n in zip(windows, (32, 64)):
+ entries.append(cache.KVCache())
+ entries.append(cache.ArraysCache(size = 1))
+ advance(entries, 0, n // 2)
+ advance(entries, n // 2, n // 2)
+ assert snapshots.share_kv_rows(*windows) == windows[0][1].keys.nbytes * 2
+ nested = [[cache.KVCache(), cache.RotatingKVCache(max_size = 16)] for _ in range(3)]
+ for entries, n in zip(nested, (8, 12, 32)):
+ entries.append(cache.ArraysCache(size = 1))
+ advance(entries, 0, min(n, 16))
+ if n > 16:
+ advance(entries, 16, n - 16)
+ store.store("w", ids[:n], entries)
+ store.discard(("w", tuple(ids[:12])))
+ assert len(store) == 2
+ assert store.nbytes == cache_entries_nbytes(lone) + cache_entries_nbytes(nested[2])
+
+
+def test_a_replay_reads_through_no_snapshot_yet_serves_as_a_base():
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+
+ def entries(n, last_chunk):
+ rows = mx.arange(n * 64, dtype = mx.float32).reshape(1, 1, n, 64)
+ rows = rows + last_chunk * (mx.arange(n) >= 512).reshape(1, 1, n, 1)
+ kv = cache.KVCache()
+ kv.update_and_fetch(rows, -rows)
+ mx.eval(kv.keys, kv.values)
+ return [kv]
+
+ ids = list(range(1000))
+ for order in ((600, 768), (768, 600)):
+ store = VLMPromptSnapshotStore(1 << 30)
+ for n in order:
+ store.store("m", ids[:n], entries(n, 0.5 if n == 600 else 0.0), replay = n == 600)
+ replay, n = store.lookup("m", ids[:601], limit = 512)
+ assert n == 600 and replay[0].keys[0, 0, 599, 0].item() == 599 * 64 + 0.5
+ grid = ("m", tuple(ids[:512]))
+ store.store("m", ids[:512], entries(512, 0.0))
+ assert store._bases[grid] == ("m", tuple(ids[:600])) and store._entries[grid][1] == 0
+ store.store("m", ids[:580], entries(580, 0.25), replay = True)
+ assert ("m", tuple(ids[:600])) not in store._entries
+ assert store._bases[grid] == ("m", tuple(ids[:580])) and store._entries[grid][1] == 0
+ divergent = ids[:300] + [-1] * 700
+ store.store("m", divergent[:590], entries(590, 0.75), replay = True)
+ longer = ("m", tuple(ids[:768]))
+ assert set(store._entries) == {grid, longer, ("m", tuple(divergent[:590]))}
+ assert store._bases[grid] == longer and store._entries[grid][1] == 0
+ assert store._entries[grid][0][0].keys[0, 0, 511, 0].item() == 511 * 64
+ assert store.nbytes == cache_entries_nbytes(store._entries[longer][0]) + cache_entries_nbytes(
+ store._entries[("m", tuple(divergent[:590]))][0]
+ )
+ store.discard(("m", tuple(divergent[:590])))
+ assert not store._replays
+
+
+def test_an_edit_keeps_the_old_branch_to_return_to():
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+
+ def entries(ids):
+ rows = mx.array(ids, dtype = mx.float32).reshape(1, 1, len(ids), 1)
+ kv = cache.KVCache()
+ kv.update_and_fetch(rows, -rows)
+ mx.eval(kv.keys, kv.values)
+ return [kv]
+
+ def request(ids):
+ _, reused = store.lookup("m", ids, len(ids) - 1)
+ for n in range(256, len(ids), 256):
+ if n > reused:
+ store.store("m", ids[:n], entries(ids[:n]))
+ store.store("m", ids[:-1], entries(ids[:-1]), replay = True)
+ return reused
+
+ store = VLMPromptSnapshotStore(1 << 30)
+ turns = [list(range(n)) for n in (557, 931, 1311, 1674)]
+ for ids in turns:
+ request(ids)
+ assert request(turns[0] + [-1] * 393) == 512
+ kept, reused = store.lookup("m", turns[3], 1673)
+ assert reused == 1536
+ assert mx.array_equal(kept[0].keys[0, 0, :, 0], mx.arange(1536, dtype = mx.float32)).item()
+ assert store.nbytes == sum(nbytes for _, nbytes in store._entries.values())
+ assert all(base in store._entries for base in store._bases.values())
+ store.clear()
+
+
[email protected]("keep", [0, 4])
[email protected]("next_rows", [1, 5])
+def test_a_compacted_window_updates_exactly_as_the_full_one(keep, next_rows):
+ mx = pytest.importorskip("mlx.core")
+ cache = pytest.importorskip("mlx_vlm.models.cache")
+
+ def rows(start, n):
+ return mx.arange(start * 8, (start + n) * 8, dtype = mx.float32).reshape(1, 1, n, 8)
+
+ def window():
+ ring = cache.RotatingKVCache(max_size = 16, keep = keep)
+ for start in (0, 24):
+ ring.update_and_fetch(rows(start, 24), -rows(start, 24))
+ return ring
+
+ full, compact = window(), window()
+ VLMPromptSnapshotStore(1 << 30).store("m", [1], [[compact]])
+ assert compact.keys.shape[2] == 16 < full.keys.shape[2]
+ for start, n in ((48, next_rows), (48 + next_rows, 1)):
+ got = compact.update_and_fetch(rows(start, n), -rows(start, n))
+ want = full.update_and_fetch(rows(start, n), -rows(start, n))
+ for a, b in zip(got, want):
+ assert a.shape == b.shape and mx.array_equal(a, b).item()