FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(kvcache): clone inserted radix keys to release request buffers (#559)
cherry77-cloud555efd894472
diff --git a/python/freetoken/kvcache/hybrid_radix_cache.py b/python/freetoken/kvcache/hybrid_radix_cache.pyindex 1bd9e0b9..0b66253b 100644--- a/python/freetoken/kvcache/hybrid_radix_cache.py+++ b/python/freetoken/kvcache/hybrid_radix_cache.py@@ -98,7 +98,10 @@ class HybridRadixCache:node, prefix_len = self._walk(input_ids)if prefix_len != insert_len:new_node = RadixTreeNode(self.key_fn)- new_node.set_key_value(input_ids[prefix_len:], kv_indices[prefix_len:].clone())+ # A view would keep the request's full output-budget token buffer alive.+ new_node.set_key_value(+ input_ids[prefix_len:].clone(), kv_indices[prefix_len:].clone()+ )new_node.set_parent(node)self.full_evictable += new_node.lengthnode = new_nodediff --git a/python/freetoken/kvcache/radix_cache.py b/python/freetoken/kvcache/radix_cache.pyindex 5437286d..ec0bf75d 100644--- a/python/freetoken/kvcache/radix_cache.py+++ b/python/freetoken/kvcache/radix_cache.py@@ -167,7 +167,10 @@ class RadixPrefixCache(BasePrefixCache):node, prefix_len = self._tree_walk(input_ids)if prefix_len != insert_len: # NOTE: prefix_len < insert_lennew_node = RadixTreeNode(self.key_fn)- new_node.set_key_value(input_ids[prefix_len:], indices[prefix_len:].clone())+ # A view would keep the request's full output-budget token buffer alive.+ new_node.set_key_value(+ input_ids[prefix_len:].clone(), indices[prefix_len:].clone()+ )new_node.set_parent(node)self.evictable_size += new_node.lengthnode = new_nodediff --git a/python/freetoken/kvcache/swa_radix_cache.py b/python/freetoken/kvcache/swa_radix_cache.pyindex 260439ad..d2cc89c2 100644--- a/python/freetoken/kvcache/swa_radix_cache.py+++ b/python/freetoken/kvcache/swa_radix_cache.py@@ -215,7 +215,8 @@ class SWARadixCache:# then a live (non-tombstone) leaf for the in-window remainder (never a tombstone leaf --# the clamp + the free_swa -page_size margin guarantee a live tail).if total < insert_len:- suffix_ids = input_ids[total:]+ # Both suffix nodes can share this copy without retaining the request buffer.+ suffix_ids = input_ids[total:].clone()suffix_kv = kv_indices[total:].clone()boundary = max(0, min(swa_evicted_seqlen, insert_len) - total)boundary = min(boundary, max(0, len(suffix_ids) - self.page_size))