Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(kvcache): clone inserted radix keys to release request buffers (#559)

cherry77-cloud555efd894472
diff --git a/python/freetoken/kvcache/hybrid_radix_cache.py b/python/freetoken/kvcache/hybrid_radix_cache.py
index 1bd9e0b9..0b66253b 100644
--- a/python/freetoken/kvcache/hybrid_radix_cache.py
+++ b/python/freetoken/kvcache/hybrid_radix_cache.py
@@ -98,7 +98,10 @@ class HybridRadixCache:
node, prefix_len = self._walk(input_ids)
if prefix_len != insert_len:
new_node = RadixTreeNode(self.key_fn)
- new_node.set_key_value(input_ids[prefix_len:], kv_indices[prefix_len:].clone())
+ # A view would keep the request's full output-budget token buffer alive.
+ new_node.set_key_value(
+ input_ids[prefix_len:].clone(), kv_indices[prefix_len:].clone()
+ )
new_node.set_parent(node)
self.full_evictable += new_node.length
node = new_node
diff --git a/python/freetoken/kvcache/radix_cache.py b/python/freetoken/kvcache/radix_cache.py
index 5437286d..ec0bf75d 100644
--- a/python/freetoken/kvcache/radix_cache.py
+++ b/python/freetoken/kvcache/radix_cache.py
@@ -167,7 +167,10 @@ class RadixPrefixCache(BasePrefixCache):
node, prefix_len = self._tree_walk(input_ids)
if prefix_len != insert_len: # NOTE: prefix_len < insert_len
new_node = RadixTreeNode(self.key_fn)
- new_node.set_key_value(input_ids[prefix_len:], indices[prefix_len:].clone())
+ # A view would keep the request's full output-budget token buffer alive.
+ new_node.set_key_value(
+ input_ids[prefix_len:].clone(), indices[prefix_len:].clone()
+ )
new_node.set_parent(node)
self.evictable_size += new_node.length
node = new_node
diff --git a/python/freetoken/kvcache/swa_radix_cache.py b/python/freetoken/kvcache/swa_radix_cache.py
index 260439ad..d2cc89c2 100644
--- a/python/freetoken/kvcache/swa_radix_cache.py
+++ b/python/freetoken/kvcache/swa_radix_cache.py
@@ -215,7 +215,8 @@ class SWARadixCache:
# then a live (non-tombstone) leaf for the in-window remainder (never a tombstone leaf --
# the clamp + the free_swa -page_size margin guarantee a live tail).
if total < insert_len:
- suffix_ids = input_ids[total:]
+ # Both suffix nodes can share this copy without retaining the request buffer.
+ suffix_ids = input_ids[total:].clone()
suffix_kv = kv_indices[total:].clone()
boundary = max(0, min(swa_evicted_seqlen, insert_len) - total)
boundary = min(boundary, max(0, len(suffix_ids) - self.page_size))