Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

fix(runtime): correct overlap token limits and cache safety (#546)

Xiaoze Fancee23bf44a86
diff --git a/python/freetoken/kvcache/cache_status.py b/python/freetoken/kvcache/cache_status.py
index 10169d65..6aa13e7f 100644
--- a/python/freetoken/kvcache/cache_status.py
+++ b/python/freetoken/kvcache/cache_status.py
@@ -120,21 +120,18 @@ def compute_cache_floors(engine: "Engine") -> Dict[str, int]:
return int(_linear_pool_min_slots(config) - 1)
def _swa() -> int:
- # Window-pool floor in tokens (matches the pool's own page unit x count):
- # DSV4 -- the working-set floor pages _dsv4_pool_sizes clamps n_win_pages to, x P.
- # radix-SWA -- the concurrent working set (+1 slot-0 sentinel) _swa_paged_num_tokens
- # floors to.
- # 0 for models without a window pool.
+ # Report usable tokens, matching num_swa_pages in rebuild requests.
+ # DSV4's physical floor includes a dummy page; the SWA floor already excludes slot 0.
from .dsv4_cost_model import _dsv4_window_floor_pages
from .hybrid_swa_pool import _swa_pool_floor
mc = config.model_config
if mc.dsv4_args is not None:
P = mc.dsv4_args.window_size
- return int(_dsv4_window_floor_pages(config, P) * P)
+ return int((_dsv4_window_floor_pages(config, P) - 1) * P)
if not (mc.has_swa_attention and config.cache_type == "swa_radix"):
return 0
- return int(_swa_pool_floor(config) + 1)
+ return int(_swa_pool_floor(config))
for key, fn in (("kv_tokens", _kv), ("moe_experts", _moe), ("mamba_slots", _mamba),
("swa_tokens", _swa)):
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index 7abb9c80..32e0f3cc 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -354,7 +354,7 @@ class OffloadMoeCache:
name, layer_id, source.shape, source.dtype,
)
self.bank_sources[name] = list(per_layer)
- self.bank_caches[name] = torch.empty(
+ self.bank_caches[name] = torch.zeros(
(self.cache_size, *head.shape[1:]),
dtype=head.dtype,
device=self.device,
@@ -492,7 +492,7 @@ class OffloadMoeCache:
# 3. Reallocate the slot cache from the retained host sources.
for name in self.bank_schema:
head = self.bank_sources[name][0]
- self.bank_caches[name] = torch.empty(
+ self.bank_caches[name] = torch.zeros(
(cache_size, *head.shape[1:]), dtype=head.dtype, device=self.device
)
self.banks = [(self.bank_sources[n], self.bank_caches[n]) for n in self.bank_schema]
diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.py
index ac1bf322..fa0d96a7 100644
--- a/python/freetoken/scheduler/scheduler.py
+++ b/python/freetoken/scheduler/scheduler.py
@@ -349,7 +349,8 @@ class Scheduler(SchedulerIOMixin):
next_token = int(next_token.item())
# EOS / stop-string -> "stop", output budget exhausted -> "length";
# EOS and stop strings win over length.
- hit_length = not req.can_decode
+ # Overlap can advance device_len ahead of the token delivered to the host.
+ hit_length = req.input_ids.numel() >= req.max_device_len
hit_eos = (
not req.sampling_params.ignore_eos and next_token in self.eos_token_ids
)
diff --git a/tests/kvcache/test_cache_unit_bytes.py b/tests/kvcache/test_cache_unit_bytes.py
index c0eec34e..496622f9 100644
--- a/tests/kvcache/test_cache_unit_bytes.py
+++ b/tests/kvcache/test_cache_unit_bytes.py
@@ -213,7 +213,7 @@ def test_floors_dsv4_reports_real_window_floor():
assert floors["kv_tokens"] > 0
# The window (swa) floor is reported too, in tokens (window pages x P) -- not the bogus 0 the
# radix-only branch used to give for DSV4.
- assert floors["swa_tokens"] == _dsv4_window_floor_pages(cfg, P) * P
+ assert floors["swa_tokens"] == (_dsv4_window_floor_pages(cfg, P) - 1) * P
def test_floors_missing_config_all_zero():