Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(scheduler): add radix_lookahead keys to the prefix cache (#834)

Xiaoze Fan9221fd7fc2b9
diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.py
index bd675d72..d4982f9e 100644
--- a/python/freetoken/scheduler/cache.py
+++ b/python/freetoken/scheduler/cache.py
@@ -31,7 +31,7 @@ _SWA_RETAIN_GAP = 16
class CacheManager:
def __init__(self, num_pages: int, page_size: int, page_table: torch.Tensor, type: str,
- linear_state_pool=None, swa_pool=None, sliding_window_size=None):
+ linear_state_pool=None, swa_pool=None, sliding_window_size=None, radix_lookahead: bool = False):
# The `_free_slots` follows a page-aligned manner. For example, if page_size = 2,
# the `_free_slots` may look like [0, 2, 4, 6, ...], and each slot represents a page.
device = page_table.device
@@ -61,6 +61,9 @@ class CacheManager:
self.page_table = page_table
self.page_size = page_size
self.cache_type = type
+ # radix_lookahead keys position i by (t_i, t_{i+1}), so a hit also matches the token after it and a draft whose KV slot i depends on t_{i+1} (MTP) never reuses a slot written for a different next token
+ assert not (radix_lookahead and self.is_swa), "the SWA radix cache keeps token keys"
+ self.radix_lookahead = radix_lookahead
# ----- capability hooks (defaults; plugged-in pools may narrow them) -----
supports_runtime_rebuild = True
@@ -98,7 +101,8 @@ class CacheManager:
def match_req(self, req: PendingReq, max_len: int | None = None) -> MatchResult:
input_len = req.input_len
assert input_len > 0, "Input length must be greater than 0."
- ids = req.input_ids[: input_len - 1 if max_len is None else max(0, min(input_len - 1, max_len))]
+ limit = input_len - 1 if max_len is None else max(0, min(input_len - 1, max_len))
+ ids = self._keys(req.input_ids, limit)
if self.is_swa:
from freetoken.kvcache.swa_radix_cache import SWACacheHandle
m = self.prefix_cache.match_prefix(ids)
@@ -301,8 +305,8 @@ class CacheManager:
# We should free it if the request has finished.
page_indices = self.page_table[req.table_idx, : req.cached_len]
old_handle = req.cache_handle
- insert_ids = req.input_ids[: req.cached_len]
- cached_len, new_handle = self.prefix_cache.insert_prefix(insert_ids, page_indices)
+ insert_ids = self._keys(req.input_ids, req.cached_len)
+ cached_len, new_handle = self.prefix_cache.insert_prefix(insert_ids, page_indices[: len(insert_ids)])
# unlock until all operations on handle is done
self.unlock(old_handle)
# this part is already in the prefix cache, free it. A naive-SWA request (swa_paged, no
@@ -356,11 +360,11 @@ class CacheManager:
and 0 < L <= req.cached_len
and align_down(L, self.page_size) == L
and req.mamba_ping_pong is not None
+ and len(keys := self._keys(req.input_ids, L)) == L
):
frozen_idx = 1 - req.mamba_next_track_idx
frozen = req.mamba_ping_pong[frozen_idx]
- prefix_len, mamba_exist = self.prefix_cache.insert(
- req.input_ids[:L], page_indices[:L], frozen)
+ prefix_len, mamba_exist = self.prefix_cache.insert(keys, page_indices[:L], frozen)
pool.free([s for s in req.mamba_ping_pong if mamba_exist or s != frozen])
req.mamba_ping_pong = None
self._free(page_indices[free_upto : max(free_upto, prefix_len)])
@@ -372,9 +376,9 @@ class CacheManager:
# remain as reuse points).
insert_len = align_down(req.cached_len, self.page_size)
keep_live = False
- if insert_len == req.cached_len and insert_len > 0:
- prefix_len, mamba_exist = self.prefix_cache.insert(
- req.input_ids[:insert_len], page_indices[:insert_len], req.linear_slot_idx)
+ keys = self._keys(req.input_ids, insert_len)
+ if insert_len == req.cached_len and insert_len > 0 and len(keys) == insert_len:
+ prefix_len, mamba_exist = self.prefix_cache.insert(keys, page_indices[:insert_len], req.linear_slot_idx)
self.unlock(old_handle)
self._free(page_indices[free_upto : max(free_upto, prefix_len)])
keep_live = not mamba_exist # tree now owns linear_slot_idx
@@ -388,22 +392,22 @@ class CacheManager:
L = req.mamba_last_track_seqlen
if L is None:
return # no ×64 boundary crossed this chunk; req keeps its pages (committed later)
- if align_down(L, self.page_size) != L:
- # page_size>1 only: insert would align the key down, attaching a state that encodes
- # L tokens to a SHORTER node -- a future hit would COW-restore an over-advanced
- # state. Skip; the next aligned boundary (or the finish-donate) commits instead.
+ keys = self._keys(req.input_ids, L)
+ if align_down(L, self.page_size) != L or len(keys) != L:
+ # page_size>1, or a lookahead key short of the token after L: insert would attach a
+ # state that encodes L tokens to a SHORTER node -- a future hit would COW-restore an
+ # over-advanced state. Skip; the next boundary (or the finish-donate) commits instead.
req.mamba_last_track_seqlen = None
return
frozen_idx = 1 - req.mamba_next_track_idx # the slot the forward just wrote
frozen = req.mamba_ping_pong[frozen_idx]
- prefix_len, mamba_exist = self.prefix_cache.insert(
- req.input_ids[:L], page_indices[:L], frozen)
+ prefix_len, mamba_exist = self.prefix_cache.insert(keys, page_indices[:L], frozen)
self.unlock(old_handle)
self._free(page_indices[old_handle.cached_len : prefix_len])
# Lock the committed snapshot node FIRST: the replacement-slot alloc below can trigger
# evict_mamba (via ensure_mamba_slots), which would otherwise reclaim this still-unlocked
# just-donated node -- freeing its KV pages under the still-decoding request.
- m = self.prefix_cache.match_prefix(req.input_ids[:L])
+ m = self.prefix_cache.match_prefix(keys)
# Same re-point as the generic path: the dedup free above returned this request's own
# pages for [old_handle.cached_len, prefix_len) while its row still named them.
if prefix_len > old_handle.cached_len:
@@ -623,6 +627,14 @@ class CacheManager:
if len(indices) > 0:
self.free_slots = torch.cat([self.free_slots, indices[:: self.page_size]])
+ def _keys(self, ids: torch.Tensor, n: int) -> torch.Tensor:
+ """Radix keys of the first ``n`` positions of ``ids``; with radix_lookahead, fewer when the token after them is not known yet."""
+ if not self.radix_lookahead:
+ return ids[:n]
+ m = max(0, min(n, len(ids) - 1))
+ # ids stay below 2^31 (multimodal pseudo ids included), so a pair packs into one int64
+ return (ids[:m].to(torch.int64) << 31) | ids[1 : m + 1].to(torch.int64)
+
def _page_to_token(self, pages: torch.Tensor) -> torch.Tensor:
if self.page_size == 1:
return pages
diff --git a/tests/scheduler/test_radix_lookahead.py b/tests/scheduler/test_radix_lookahead.py
new file mode 100644
index 00000000..081c36f7
--- /dev/null
+++ b/tests/scheduler/test_radix_lookahead.py
@@ -0,0 +1,101 @@
+"""radix_lookahead: a prefix hit also matches the token after it, so a draft KV slot that depends on the next token (MTP) is only reused when that token agrees. CPU, real CacheManager + real trees."""
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from freetoken.core import Req, SamplingParams
+from freetoken.kvcache.linear_state_pool import LinearStatePool
+from freetoken.mm import MM_PAD_SHIFT_VALUE
+from freetoken.models.config import LinearGatedDeltaGroupConfig
+from freetoken.scheduler.cache import CacheManager
+
+
+def _pend(ids):
+ return SimpleNamespace(input_ids=torch.tensor(ids, dtype=torch.int32), input_len=len(ids))
+
+
+def _publish(cm, table_idx, ids, *, finished=True, track=None):
+ """Admit ids at their prefix hit as prefill does, then commit them into the tree with the last id as the sampled token."""
+ handle = cm.match_req(_pend(ids)).cuda_handle
+ req = Req(input_ids=torch.tensor(ids, dtype=torch.int32), table_idx=table_idx, cached_len=handle.cached_len,
+ output_len=0, uid=table_idx, sampling_params=SamplingParams(), cache_handle=handle)
+ cm.lock(handle)
+ if handle.cached_len > 0:
+ cm.page_table[table_idx, : handle.cached_len].copy_(handle.get_matched_indices())
+ req.device_len = len(ids) - 1
+ cm.allocate_paged([req])
+ req.cached_len = len(ids) - 1
+ if cm.is_hybrid:
+ pool = cm.linear_state_pool
+ req.linear_slot_idx = pool.alloc(1)[0]
+ req.mamba_ping_pong = tuple(pool.alloc(2))
+ req.mamba_next_track_idx = 1
+ req.mamba_last_track_seqlen = track
+ with cm.lazy_free_region():
+ cm.cache_req(req, finished=finished)
+ return req
+
+
+def _hybrid(lookahead):
+ g = LinearGatedDeltaGroupConfig(
+ name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4,
+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,
+ )
+ pool = LinearStatePool(group=g, num_slots=32, dtype=torch.bfloat16, device=torch.device("cpu"), tp_size=1)
+ return CacheManager(256, 1, torch.zeros(4, 64, dtype=torch.int32), "hybrid_radix", linear_state_pool=pool,
+ radix_lookahead=lookahead)
+
+
[email protected]("page_size", [1, 4])
+def test_a_hit_takes_the_slot_before_the_fork_from_the_branch_it_follows(page_size):
+ a = list(range(1, 18))
+ b = a[:8] + list(range(100, 109)) # forks from a at position 8
+ query = b[:12] + list(range(200, 205))
+ rows = {}
+ for lookahead in (False, True):
+ page_table = torch.zeros(4, 64, dtype=torch.int32)
+ cm = CacheManager(64, page_size, page_table, "radix", radix_lookahead=lookahead)
+ _publish(cm, 0, a, finished=False)
+ req_b = _publish(cm, 1, b, finished=False)
+ hit = cm.match_req(_pend(query)).cuda_handle
+ rows[lookahead] = (hit.cached_len, hit.get_matched_indices(), page_table[req_b.table_idx].clone())
+ plain_len, plain_slots, _ = rows[False]
+ lookahead_len, lookahead_slots, b_row = rows[True]
+ # the query shares 12 tokens with b; a lookahead key needs the token after, so one position less (page-aligned)
+ assert plain_len == 12 // page_size * page_size
+ assert lookahead_len == 11 // page_size * page_size
+ # slot 7's draft KV depends on the token at 8, where b differs from a: it must be b's slot
+ assert lookahead_slots[7] == b_row[7]
+ assert plain_slots[7] != b_row[7]
+
+
+def test_the_prompt_end_key_uses_the_sampled_token():
+ prompt, sampled = list(range(1, 11)), 77
+ cm = CacheManager(64, 1, torch.zeros(4, 64, dtype=torch.int32), "radix", radix_lookahead=True)
+ _publish(cm, 0, prompt + [sampled], finished=False)
+ assert cm.match_req(_pend(prompt + [sampled, 5])).cuda_handle.cached_len == len(prompt)
+ assert cm.match_req(_pend(prompt + [sampled + 1, 5])).cuda_handle.cached_len == len(prompt) - 1
+
+
+def test_multimodal_pseudo_ids_keep_distinct_keys():
+ top = MM_PAD_SHIFT_VALUE + (1 << 30) - 1
+ image_a, image_b = [top] * 4, [top - 1] * 4
+ cm = CacheManager(64, 1, torch.zeros(4, 64, dtype=torch.int32), "radix", radix_lookahead=True)
+ _publish(cm, 0, [1, 2] + image_a + [3, 4], finished=False)
+ assert cm.match_req(_pend([1, 2] + image_a + [3, 4, 9])).cuda_handle.cached_len == 7
+ # the first image row is keyed with the second, so a different image matches only before it
+ assert cm.match_req(_pend([1, 2] + image_b + [3, 4, 9])).cuda_handle.cached_len == 1
+
+
+def test_hybrid_snapshot_hits_only_when_the_token_after_it_agrees():
+ tokens = list(range(1, 18))
+ for lookahead, expect_same, expect_diff in ((False, 8, 8), (True, 8, None)):
+ cm = _hybrid(lookahead)
+ # the frozen snapshot at 8 (an anchor or a track boundary), then the finish donates the live state
+ _publish(cm, 0, tokens, finished=True, track=8)
+ same = cm.match_req(_pend(tokens[:9] + [500, 501]))
+ diff = cm.match_req(_pend(tokens[:8] + [500, 501]))
+ assert (same.cuda_handle.cached_len if same.mamba_value is not None else None) == expect_same
+ assert (diff.cuda_handle.cached_len if diff.mamba_value is not None else None) == expect_diff