FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
fix(runtime): correct overlap token limits and cache safety (#546)
Xiaoze Fancee23bf44a86
diff --git a/python/freetoken/kvcache/cache_status.py b/python/freetoken/kvcache/cache_status.pyindex 10169d65..6aa13e7f 100644--- a/python/freetoken/kvcache/cache_status.py+++ b/python/freetoken/kvcache/cache_status.py@@ -120,21 +120,18 @@ def compute_cache_floors(engine: "Engine") -> Dict[str, int]:return int(_linear_pool_min_slots(config) - 1)def _swa() -> int:- # Window-pool floor in tokens (matches the pool's own page unit x count):- # DSV4 -- the working-set floor pages _dsv4_pool_sizes clamps n_win_pages to, x P.- # radix-SWA -- the concurrent working set (+1 slot-0 sentinel) _swa_paged_num_tokens- # floors to.- # 0 for models without a window pool.+ # Report usable tokens, matching num_swa_pages in rebuild requests.+ # DSV4's physical floor includes a dummy page; the SWA floor already excludes slot 0.from .dsv4_cost_model import _dsv4_window_floor_pagesfrom .hybrid_swa_pool import _swa_pool_floormc = config.model_configif mc.dsv4_args is not None:P = mc.dsv4_args.window_size- return int(_dsv4_window_floor_pages(config, P) * P)+ return int((_dsv4_window_floor_pages(config, P) - 1) * P)if not (mc.has_swa_attention and config.cache_type == "swa_radix"):return 0- return int(_swa_pool_floor(config) + 1)+ return int(_swa_pool_floor(config))for key, fn in (("kv_tokens", _kv), ("moe_experts", _moe), ("mamba_slots", _mamba),("swa_tokens", _swa)):diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 7abb9c80..32e0f3cc 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -354,7 +354,7 @@ class OffloadMoeCache:name, layer_id, source.shape, source.dtype,)self.bank_sources[name] = list(per_layer)- self.bank_caches[name] = torch.empty(+ self.bank_caches[name] = torch.zeros((self.cache_size, *head.shape[1:]),dtype=head.dtype,device=self.device,@@ -492,7 +492,7 @@ class OffloadMoeCache:# 3. Reallocate the slot cache from the retained host sources.for name in self.bank_schema:head = self.bank_sources[name][0]- self.bank_caches[name] = torch.empty(+ self.bank_caches[name] = torch.zeros((cache_size, *head.shape[1:]), dtype=head.dtype, device=self.device)self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.pyindex ac1bf322..fa0d96a7 100644--- a/python/freetoken/scheduler/scheduler.py+++ b/python/freetoken/scheduler/scheduler.py@@ -349,7 +349,8 @@ class Scheduler(SchedulerIOMixin):next_token = int(next_token.item())# EOS / stop-string -> "stop", output budget exhausted -> "length";# EOS and stop strings win over length.- hit_length = not req.can_decode+ # Overlap can advance device_len ahead of the token delivered to the host.+ hit_length = req.input_ids.numel() >= req.max_device_lenhit_eos = (not req.sampling_params.ignore_eos and next_token in self.eos_token_ids)diff --git a/tests/kvcache/test_cache_unit_bytes.py b/tests/kvcache/test_cache_unit_bytes.pyindex c0eec34e..496622f9 100644--- a/tests/kvcache/test_cache_unit_bytes.py+++ b/tests/kvcache/test_cache_unit_bytes.py@@ -213,7 +213,7 @@ def test_floors_dsv4_reports_real_window_floor():assert floors["kv_tokens"] > 0# The window (swa) floor is reported too, in tokens (window pages x P) -- not the bogus 0 the# radix-only branch used to give for DSV4.- assert floors["swa_tokens"] == _dsv4_window_floor_pages(cfg, P) * P+ assert floors["swa_tokens"] == (_dsv4_window_floor_pages(cfg, P) - 1) * Pdef test_floors_missing_config_all_zero():