FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(scheduler): add radix_lookahead keys to the prefix cache (#834)
Xiaoze Fan9221fd7fc2b9
diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.pyindex bd675d72..d4982f9e 100644--- a/python/freetoken/scheduler/cache.py+++ b/python/freetoken/scheduler/cache.py@@ -31,7 +31,7 @@ _SWA_RETAIN_GAP = 16class CacheManager:def __init__(self, num_pages: int, page_size: int, page_table: torch.Tensor, type: str,- linear_state_pool=None, swa_pool=None, sliding_window_size=None):+ linear_state_pool=None, swa_pool=None, sliding_window_size=None, radix_lookahead: bool = False):# The `_free_slots` follows a page-aligned manner. For example, if page_size = 2,# the `_free_slots` may look like [0, 2, 4, 6, ...], and each slot represents a page.device = page_table.device@@ -61,6 +61,9 @@ class CacheManager:self.page_table = page_tableself.page_size = page_sizeself.cache_type = type+ # radix_lookahead keys position i by (t_i, t_{i+1}), so a hit also matches the token after it and a draft whose KV slot i depends on t_{i+1} (MTP) never reuses a slot written for a different next token+ assert not (radix_lookahead and self.is_swa), "the SWA radix cache keeps token keys"+ self.radix_lookahead = radix_lookahead# ----- capability hooks (defaults; plugged-in pools may narrow them) -----supports_runtime_rebuild = True@@ -98,7 +101,8 @@ class CacheManager:def match_req(self, req: PendingReq, max_len: int | None = None) -> MatchResult:input_len = req.input_lenassert input_len > 0, "Input length must be greater than 0."- ids = req.input_ids[: input_len - 1 if max_len is None else max(0, min(input_len - 1, max_len))]+ limit = input_len - 1 if max_len is None else max(0, min(input_len - 1, max_len))+ ids = self._keys(req.input_ids, limit)if self.is_swa:from freetoken.kvcache.swa_radix_cache import SWACacheHandlem = self.prefix_cache.match_prefix(ids)@@ -301,8 +305,8 @@ class CacheManager:# We should free it if the request has finished.page_indices = self.page_table[req.table_idx, : req.cached_len]old_handle = req.cache_handle- insert_ids = req.input_ids[: req.cached_len]- cached_len, new_handle = self.prefix_cache.insert_prefix(insert_ids, page_indices)+ insert_ids = self._keys(req.input_ids, req.cached_len)+ cached_len, new_handle = self.prefix_cache.insert_prefix(insert_ids, page_indices[: len(insert_ids)])# unlock until all operations on handle is doneself.unlock(old_handle)# this part is already in the prefix cache, free it. A naive-SWA request (swa_paged, no@@ -356,11 +360,11 @@ class CacheManager:and 0 < L <= req.cached_lenand align_down(L, self.page_size) == Land req.mamba_ping_pong is not None+ and len(keys := self._keys(req.input_ids, L)) == L):frozen_idx = 1 - req.mamba_next_track_idxfrozen = req.mamba_ping_pong[frozen_idx]- prefix_len, mamba_exist = self.prefix_cache.insert(- req.input_ids[:L], page_indices[:L], frozen)+ prefix_len, mamba_exist = self.prefix_cache.insert(keys, page_indices[:L], frozen)pool.free([s for s in req.mamba_ping_pong if mamba_exist or s != frozen])req.mamba_ping_pong = Noneself._free(page_indices[free_upto : max(free_upto, prefix_len)])@@ -372,9 +376,9 @@ class CacheManager:# remain as reuse points).insert_len = align_down(req.cached_len, self.page_size)keep_live = False- if insert_len == req.cached_len and insert_len > 0:- prefix_len, mamba_exist = self.prefix_cache.insert(- req.input_ids[:insert_len], page_indices[:insert_len], req.linear_slot_idx)+ keys = self._keys(req.input_ids, insert_len)+ if insert_len == req.cached_len and insert_len > 0 and len(keys) == insert_len:+ prefix_len, mamba_exist = self.prefix_cache.insert(keys, page_indices[:insert_len], req.linear_slot_idx)self.unlock(old_handle)self._free(page_indices[free_upto : max(free_upto, prefix_len)])keep_live = not mamba_exist # tree now owns linear_slot_idx@@ -388,22 +392,22 @@ class CacheManager:L = req.mamba_last_track_seqlenif L is None:return # no ×64 boundary crossed this chunk; req keeps its pages (committed later)- if align_down(L, self.page_size) != L:- # page_size>1 only: insert would align the key down, attaching a state that encodes- # L tokens to a SHORTER node -- a future hit would COW-restore an over-advanced- # state. Skip; the next aligned boundary (or the finish-donate) commits instead.+ keys = self._keys(req.input_ids, L)+ if align_down(L, self.page_size) != L or len(keys) != L:+ # page_size>1, or a lookahead key short of the token after L: insert would attach a+ # state that encodes L tokens to a SHORTER node -- a future hit would COW-restore an+ # over-advanced state. Skip; the next boundary (or the finish-donate) commits instead.req.mamba_last_track_seqlen = Nonereturnfrozen_idx = 1 - req.mamba_next_track_idx # the slot the forward just wrotefrozen = req.mamba_ping_pong[frozen_idx]- prefix_len, mamba_exist = self.prefix_cache.insert(- req.input_ids[:L], page_indices[:L], frozen)+ prefix_len, mamba_exist = self.prefix_cache.insert(keys, page_indices[:L], frozen)self.unlock(old_handle)self._free(page_indices[old_handle.cached_len : prefix_len])# Lock the committed snapshot node FIRST: the replacement-slot alloc below can trigger# evict_mamba (via ensure_mamba_slots), which would otherwise reclaim this still-unlocked# just-donated node -- freeing its KV pages under the still-decoding request.- m = self.prefix_cache.match_prefix(req.input_ids[:L])+ m = self.prefix_cache.match_prefix(keys)# Same re-point as the generic path: the dedup free above returned this request's own# pages for [old_handle.cached_len, prefix_len) while its row still named them.if prefix_len > old_handle.cached_len:@@ -623,6 +627,14 @@ class CacheManager:if len(indices) > 0:self.free_slots = torch.cat([self.free_slots, indices[:: self.page_size]])+ def _keys(self, ids: torch.Tensor, n: int) -> torch.Tensor:+ """Radix keys of the first ``n`` positions of ``ids``; with radix_lookahead, fewer when the token after them is not known yet."""+ if not self.radix_lookahead:+ return ids[:n]+ m = max(0, min(n, len(ids) - 1))+ # ids stay below 2^31 (multimodal pseudo ids included), so a pair packs into one int64+ return (ids[:m].to(torch.int64) << 31) | ids[1 : m + 1].to(torch.int64)+def _page_to_token(self, pages: torch.Tensor) -> torch.Tensor:if self.page_size == 1:return pagesdiff --git a/tests/scheduler/test_radix_lookahead.py b/tests/scheduler/test_radix_lookahead.pynew file mode 100644index 00000000..081c36f7--- /dev/null+++ b/tests/scheduler/test_radix_lookahead.py@@ -0,0 +1,101 @@+"""radix_lookahead: a prefix hit also matches the token after it, so a draft KV slot that depends on the next token (MTP) is only reused when that token agrees. CPU, real CacheManager + real trees."""++from types import SimpleNamespace++import pytest+import torch++from freetoken.core import Req, SamplingParams+from freetoken.kvcache.linear_state_pool import LinearStatePool+from freetoken.mm import MM_PAD_SHIFT_VALUE+from freetoken.models.config import LinearGatedDeltaGroupConfig+from freetoken.scheduler.cache import CacheManager+++def _pend(ids):+ return SimpleNamespace(input_ids=torch.tensor(ids, dtype=torch.int32), input_len=len(ids))+++def _publish(cm, table_idx, ids, *, finished=True, track=None):+ """Admit ids at their prefix hit as prefill does, then commit them into the tree with the last id as the sampled token."""+ handle = cm.match_req(_pend(ids)).cuda_handle+ req = Req(input_ids=torch.tensor(ids, dtype=torch.int32), table_idx=table_idx, cached_len=handle.cached_len,+ output_len=0, uid=table_idx, sampling_params=SamplingParams(), cache_handle=handle)+ cm.lock(handle)+ if handle.cached_len > 0:+ cm.page_table[table_idx, : handle.cached_len].copy_(handle.get_matched_indices())+ req.device_len = len(ids) - 1+ cm.allocate_paged([req])+ req.cached_len = len(ids) - 1+ if cm.is_hybrid:+ pool = cm.linear_state_pool+ req.linear_slot_idx = pool.alloc(1)[0]+ req.mamba_ping_pong = tuple(pool.alloc(2))+ req.mamba_next_track_idx = 1+ req.mamba_last_track_seqlen = track+ with cm.lazy_free_region():+ cm.cache_req(req, finished=finished)+ return req+++def _hybrid(lookahead):+ g = LinearGatedDeltaGroupConfig(+ name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4,+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,+ )+ pool = LinearStatePool(group=g, num_slots=32, dtype=torch.bfloat16, device=torch.device("cpu"), tp_size=1)+ return CacheManager(256, 1, torch.zeros(4, 64, dtype=torch.int32), "hybrid_radix", linear_state_pool=pool,+ radix_lookahead=lookahead)++[email protected]("page_size", [1, 4])+def test_a_hit_takes_the_slot_before_the_fork_from_the_branch_it_follows(page_size):+ a = list(range(1, 18))+ b = a[:8] + list(range(100, 109)) # forks from a at position 8+ query = b[:12] + list(range(200, 205))+ rows = {}+ for lookahead in (False, True):+ page_table = torch.zeros(4, 64, dtype=torch.int32)+ cm = CacheManager(64, page_size, page_table, "radix", radix_lookahead=lookahead)+ _publish(cm, 0, a, finished=False)+ req_b = _publish(cm, 1, b, finished=False)+ hit = cm.match_req(_pend(query)).cuda_handle+ rows[lookahead] = (hit.cached_len, hit.get_matched_indices(), page_table[req_b.table_idx].clone())+ plain_len, plain_slots, _ = rows[False]+ lookahead_len, lookahead_slots, b_row = rows[True]+ # the query shares 12 tokens with b; a lookahead key needs the token after, so one position less (page-aligned)+ assert plain_len == 12 // page_size * page_size+ assert lookahead_len == 11 // page_size * page_size+ # slot 7's draft KV depends on the token at 8, where b differs from a: it must be b's slot+ assert lookahead_slots[7] == b_row[7]+ assert plain_slots[7] != b_row[7]+++def test_the_prompt_end_key_uses_the_sampled_token():+ prompt, sampled = list(range(1, 11)), 77+ cm = CacheManager(64, 1, torch.zeros(4, 64, dtype=torch.int32), "radix", radix_lookahead=True)+ _publish(cm, 0, prompt + [sampled], finished=False)+ assert cm.match_req(_pend(prompt + [sampled, 5])).cuda_handle.cached_len == len(prompt)+ assert cm.match_req(_pend(prompt + [sampled + 1, 5])).cuda_handle.cached_len == len(prompt) - 1+++def test_multimodal_pseudo_ids_keep_distinct_keys():+ top = MM_PAD_SHIFT_VALUE + (1 << 30) - 1+ image_a, image_b = [top] * 4, [top - 1] * 4+ cm = CacheManager(64, 1, torch.zeros(4, 64, dtype=torch.int32), "radix", radix_lookahead=True)+ _publish(cm, 0, [1, 2] + image_a + [3, 4], finished=False)+ assert cm.match_req(_pend([1, 2] + image_a + [3, 4, 9])).cuda_handle.cached_len == 7+ # the first image row is keyed with the second, so a different image matches only before it+ assert cm.match_req(_pend([1, 2] + image_b + [3, 4, 9])).cuda_handle.cached_len == 1+++def test_hybrid_snapshot_hits_only_when_the_token_after_it_agrees():+ tokens = list(range(1, 18))+ for lookahead, expect_same, expect_diff in ((False, 8, 8), (True, 8, None)):+ cm = _hybrid(lookahead)+ # the frozen snapshot at 8 (an anchor or a track boundary), then the finish donates the live state+ _publish(cm, 0, tokens, finished=True, track=8)+ same = cm.match_req(_pend(tokens[:9] + [500, 501]))+ diff = cm.match_req(_pend(tokens[:8] + [500, 501]))+ assert (same.cuda_handle.cached_len if same.mamba_value is not None else None) == expect_same+ assert (diff.cuda_handle.cached_len if diff.mamba_value is not None else None) == expect_diff