FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(kvcache): checkpoint decode state at the tool-call opener token
Xiaoze Fanea5348b4e316
diff --git a/python/freetoken/core.py b/python/freetoken/core.pyindex 897ba4af..ef0a539c 100644--- a/python/freetoken/core.py+++ b/python/freetoken/core.py@@ -53,6 +53,13 @@ class Req:mamba_restore_src: int | None = None # on a prefix hit: tree snapshot slot to COW into the live slot (first chunk only)swa_evicted_seqlen: int = 0 # SWA radix: positions < this had their swa KV freed (slid out of window) during decodedecode_batch_idx: int = 0 # SWA radix: # of decode forwards done; the proactive free_swa skips the first (overlap guard)+ # Set once, at the first sampled tool-call opener token (scheduler detection): the state+ # length just after that token (its index + 1). A client-side rewrite of the echoed tool+ # call diverges strictly after this point, so it is the deepest reuse boundary that+ # survives such a rewrite. GDN: the state is frozen into a ping-pong slot when cached_len+ # reaches it (snapshot_toolcall_anchor) and donated at finish. SWA: caps the proactive+ # out-of-window eviction so the window ending here stays resumable.+ toolcall_anchor_len: int | None = None# Abort arrived while this request's forward was in flight (overlap scheduling). The abort# handler must not free resources under an in-flight forward; it sets this flag and# _process_last_data frees the request when the batch drains (after copy_done.synchronize).diff --git a/python/freetoken/kvcache/hybrid_swa_pool.py b/python/freetoken/kvcache/hybrid_swa_pool.pyindex 7525d331..41c3880e 100644--- a/python/freetoken/kvcache/hybrid_swa_pool.py+++ b/python/freetoken/kvcache/hybrid_swa_pool.py@@ -381,7 +381,10 @@ def _swa_per_req_swa_floor(config) -> int:window = next(g.sliding_window for g in config.model_config.kv_cache_group_specs() if g.is_swa)ps = config.page_sizelocked = ((window + _SWA_RETAIN_GAP + ps - 1) // ps) * ps- return locked + window + _SWA_EVICTION_INTERVAL + 2 * ps+ floor = locked + window + _SWA_EVICTION_INTERVAL + 2 * ps+ if getattr(config, "special_token_ckpt", False):+ floor += window + _SWA_RETAIN_GAP + _SWA_EVICTION_INTERVAL+ return floordef _swa_pool_floor(config) -> int:diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.pyindex 34049e6f..9be235b2 100644--- a/python/freetoken/scheduler/cache.py+++ b/python/freetoken/scheduler/cache.py@@ -141,6 +141,33 @@ class CacheManager:self.linear_state_pool.free(er.mamba_slots)self._free(er.kv_indices)+ def snapshot_toolcall_anchor(self, reqs: List[Req]) -> None:+ """Freeze each decoding request's GDN state at its tool-call anchor, into the ping-pong+ slot that is idle during decode (the kernel-side ×CHUNK track only runs on prefill+ extends). Must run on the engine stream before the current step's kernels: cached_len+ equals the anchor exactly when every enqueued step up to the anchor-consuming one has+ been issued and the next (current) one has not, so the copy lands between them in+ stream order. Reuses ``mamba_last_track_seqlen`` as the pending-donate mark -- the+ prefill track's own pending freeze was consumed by the prefill-commit ``cache_req``+ before any decode drain could set an anchor."""+ if not self.is_hybrid:+ return+ pool = self.linear_state_pool+ for r in reqs:+ a = r.toolcall_anchor_len+ if (+ a is None+ or r.mamba_ping_pong is None+ or r.mamba_last_track_seqlen is not None+ or r.cached_len != a+ or align_down(a, self.page_size) != a+ ):+ continue+ dst = r.mamba_ping_pong[r.mamba_next_track_idx]+ pool.copy_from(r.linear_slot_idx, dst)+ r.mamba_last_track_seqlen = a+ r.mamba_next_track_idx = 1 - r.mamba_next_track_idx+def maybe_free_swa_out_of_window(self, reqs: List[Req], *, forward_iter: int) -> None:"""Proactively free each decoding request's now-out-of-window SWA slots, bounding its swafootprint to ~one window so a smaller-than-full swa pool (swa_full_tokens_ratio<1) stays@@ -158,6 +185,22 @@ class CacheManager:continue # overlap guard: extend forward may still be runningfloor = req.cache_handle.cached_len # reused prefix -> its swa is tree-owned, not oursthreshold = (req.device_len - 1) - window - self.page_size+ if req.toolcall_anchor_len is not None:+ # Keep the window ending at the anchor resumable: a client-side rewrite of the+ # echoed tool call forks after the anchor, and a resume there needs+ # [anchor - window, anchor) live. The finish-insert then adopts (rather than+ # tombstones) these never-evicted slots; they stay unlocked, so real pool+ # pressure can still reclaim them (same soft retention as the prompt-end pin).+ cap = req.toolcall_anchor_len - window - _SWA_RETAIN_GAP+ if threshold - cap > window + _SWA_RETAIN_GAP:+ # The decode ran on far past the anchor (a tool call is normally within+ # tens of tokens of the end). Holding the cap would grow this request's+ # live swa without bound ("SWA pool exhausted" is unhandled) -- drop the+ # anchor and let normal eviction resume. This bound is what the+ # anchor-retention term in _swa_per_req_swa_floor sizes the pool for.+ req.toolcall_anchor_len = None+ else:+ threshold = min(threshold, cap)new_evicted = align_down(threshold, self.page_size)start = max(req.swa_evicted_seqlen, floor)if new_evicted > start:@@ -308,6 +351,29 @@ class CacheManager:returnif finished:+ # A pending freeze (the tool-call anchor, or a prefill ×64 track the request+ # finished too early to chunk-commit) is a strictly shorter prefix than the live+ # donate below: insert it first and advance the dedup-free floor to its boundary+ # -- [prefix_len, L) is now tree-owned by the donated node, so only [old, prefix_len)+ # is this request's dup to free. The frozen slot is consumed either way (taken by+ # the tree or freed here) and both ping-pong refs are dropped before+ # _free_req_slots so nothing double-frees.+ free_upto = old_handle.cached_len+ L = req.mamba_last_track_seqlen+ if (+ L is not None+ and 0 < L <= req.cached_len+ and align_down(L, self.page_size) == L+ and req.mamba_ping_pong is not None+ ):+ frozen_idx = 1 - req.mamba_next_track_idx+ frozen = req.mamba_ping_pong[frozen_idx]+ prefix_len, mamba_exist = self.prefix_cache.insert(+ req.input_ids[:L], page_indices[:L], frozen)+ pool.free([s for s in req.mamba_ping_pong if mamba_exist or s != frozen])+ req.mamba_ping_pong = None+ self._free(page_indices[free_upto : max(free_upto, prefix_len)])+ free_upto = max(free_upto, L)# Donate the live slot (final full-sequence state). The live state is at cached_len;# only attach it when cached_len is itself the page-aligned node boundary (always for# page_size==1). For page_size>1 a non-aligned cached_len would attach an over-advanced@@ -319,11 +385,11 @@ class CacheManager:prefix_len, mamba_exist = self.prefix_cache.insert(req.input_ids[:insert_len], page_indices[:insert_len], req.linear_slot_idx)self.unlock(old_handle)- self._free(page_indices[old_handle.cached_len : prefix_len])+ self._free(page_indices[free_upto : max(free_upto, prefix_len)])keep_live = not mamba_exist # tree now owns linear_slot_idxelse:self.unlock(old_handle)- self._free(page_indices[old_handle.cached_len :])+ self._free(page_indices[free_upto :])self._free_req_slots(req, keep_live=keep_live)returndiff --git a/python/freetoken/scheduler/config.py b/python/freetoken/scheduler/config.pyindex 464b861c..b2bbe0ae 100644--- a/python/freetoken/scheduler/config.py+++ b/python/freetoken/scheduler/config.py@@ -17,6 +17,7 @@ class SchedulerConfig(EngineConfig):cache_type: str = "radix"offline_mode: bool = Falsedecode_log_interval: int = 40+ special_token_ckpt: bool = False# networking config_unique_suffix: str = field(default_factory=_get_pid_suffix)diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.pyindex 1424a70f..35541161 100644--- a/python/freetoken/scheduler/scheduler.py+++ b/python/freetoken/scheduler/scheduler.py@@ -18,7 +18,12 @@ from freetoken.message import (PromptAdmittedMsg,UserMsg,)-from freetoken.utils import init_logger, load_eos_token_ids, load_tokenizer+from freetoken.utils import (+ init_logger,+ load_eos_token_ids,+ load_tokenizer,+ load_toolcall_anchor_id,+)from .cache import CacheManagerfrom .config import SchedulerConfig@@ -106,6 +111,16 @@ class Scheduler(SchedulerIOMixin):self._pending_rebuild: CacheRebuildBackendMsg | None = Noneself.tokenizer = load_tokenizer(config.model_path)self.eos_token_ids = load_eos_token_ids(config.model_path, self.tokenizer)+ self.toolcall_anchor_id = None+ if config.special_token_ckpt and (+ self.cache_manager.is_hybrid or self.cache_manager.is_swa+ ):+ from freetoken.server.function_call_parser import toolcall_opener_for++ self.toolcall_anchor_id = load_toolcall_anchor_id(+ self.tokenizer,+ toolcall_opener_for(getattr(config, "tool_call_parser", "")),+ )self.token_pool = self.table_manager.token_pool# Floor the prefill chunk by the cache manager's cap (DSV4: ~half the window pool) so a# sliding-window cache chunks long prompts and frees out-of-window pages between chunks@@ -336,6 +351,12 @@ class Scheduler(SchedulerIOMixin):if finishedelse None)+ if (+ next_token == self.toolcall_anchor_id+ and req.toolcall_anchor_len is None+ and not finished+ ):+ req.toolcall_anchor_len = req.input_ids.numel()reply.append(DetokenizeMsg(uid=req.uid,@@ -842,6 +863,8 @@ class Scheduler(SchedulerIOMixin):def _forward(self, forward_input: ForwardInput) -> ForwardOutput:batch, sample_args, input_mapping, output_mapping = forward_inputbatch.input_ids = self.token_pool[input_mapping]+ if self.toolcall_anchor_id is not None and not batch.is_prefill:+ self.cache_manager.snapshot_toolcall_anchor(batch.reqs)forward_output = self.engine.forward_batch(batch, sample_args)self.token_pool[output_mapping] = forward_output.next_tokens_gpuself.decode_manager.filter_reqs(forward_input.batch.reqs)diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 689260c1..c2bb118e 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -543,6 +543,21 @@ def parse_args(),)+ parser.add_argument(+ "--enable-special-token-ckpt",+ action="store_true",+ dest="special_token_ckpt",+ default=ServerArgs.special_token_ckpt,+ help=(+ "Checkpoint decode state at special tokens (currently the tool-call opener). "+ "When a GDN-hybrid or SWA model samples its tool-call opener token, the "+ "scheduler preserves a reuse point just after it (GDN: a state snapshot "+ "donated to the prefix cache; SWA: the trailing window is kept resumable), so "+ "a client that rewrites the echoed tool call only invalidates the call body, "+ "not the turn."+ ),+ )+parser.add_argument("--moe-prefill-hit-d2d",action="store_true",diff --git a/python/freetoken/server/function_call_parser.py b/python/freetoken/server/function_call_parser.pyindex 31d4dd22..6981b476 100644--- a/python/freetoken/server/function_call_parser.py+++ b/python/freetoken/server/function_call_parser.py@@ -239,6 +239,14 @@ class BaseFormatDetector(ABC):# when this holds; otherwise they send the full arguments once at close.args_fragments_prefix_stable = True+ # The marker that UNIQUELY opens a tool-call block in this wire format, or None when+ # no such marker exists (gpt-oss: <|channel|> opens every channel header, tool call or+ # not; DSV32/DSV4: the DSML opener is a multi-piece composite). Consumed by the+ # scheduler's special-token checkpoint, which only uses it when the tokenizer encodes+ # it as a single token. Often equals bot_token, but declared separately because+ # bot_token is a parse trigger, not a uniqueness claim.+ toolcall_opener: str | None = None+def __init__(self):# Streaming state management# Buffer for accumulating incomplete patterns that arrive across multiple streaming chunks@@ -899,7 +907,7 @@ class Qwen25Detector(BaseFormatDetector):Reference: https://huggingface.co/Qwen/Qwen2.5-0.5B-Instruct?chat_template=default"""-+ toolcall_opener = "<tool_call>"def __init__(self):"""Initializes the detector with necessary state variables.@@ -1091,7 +1099,7 @@ class Llama32Detector(BaseFormatDetector):<python_tag>{"name":"xxx", "arguments":{...}}```"""-+ toolcall_opener = "<|python_tag|>"def __init__(self):super().__init__()self.bot_token = "<|python_tag|>"@@ -1181,7 +1189,7 @@ class Glm47Detector(BaseFormatDetector):Reference: https://github.com/vllm-project/vllm/blob/main/vllm/tool_parsers/glm4_moe_tool_parser.py"""-+ toolcall_opener = "<tool_call>"def __init__(self):super().__init__()self.bot_token = "<tool_call>"@@ -1859,6 +1867,7 @@ class DeepSeekV32Detector(BaseFormatDetector):class Qwen3CoderDetector(InvokeParamStreamMixin, BaseFormatDetector):+ toolcall_opener = "<tool_call>"_ps_trim = "\n"_ps_trim_single = True_ps_missing_type = "string"@@ -2024,7 +2033,7 @@ class Qwen3CoderDetector(InvokeParamStreamMixin, BaseFormatDetector):class Gemma4Detector(BaseFormatDetector):"""FreeToken serving adapter for Gemma4's compact tool-call format."""-+ toolcall_opener = "<|tool_call>"def __init__(self):super().__init__()self.bot_token = "<|tool_call>"@@ -2313,6 +2322,7 @@ class Gemma4Detector(BaseFormatDetector):class MiniMaxDetector(InvokeParamStreamMixin, BaseFormatDetector):+ toolcall_opener = "<minimax:tool_call>"_ps_trim = "\n"_ps_trim_single = False_ps_missing_type = "loose"@@ -2740,6 +2750,13 @@ class FunctionCallParser:SUPPORTED_TOOL_CALL_PARSERS = list(FunctionCallParser.ToolCallParserEnum.keys())+def toolcall_opener_for(tool_call_parser: str) -> str | None:+ """The configured parser's unique tool-call opening marker, or None when the format has+ no single unique opener (see ``BaseFormatDetector.toolcall_opener``)."""+ detector = FunctionCallParser.ToolCallParserEnum.get(tool_call_parser)+ return detector.toolcall_opener if detector is not None else None++def _coerce_tools(tools: List[Any] | None) -> List[Tool]:return [_coerce_tool(tool) for tool in tools or []]diff --git a/python/freetoken/utils/__init__.py b/python/freetoken/utils/__init__.pyindex 65150f34..2e4ad15f 100644--- a/python/freetoken/utils/__init__.py+++ b/python/freetoken/utils/__init__.py@@ -11,6 +11,7 @@ from .hf import (load_eos_token_ids,load_generation_sampling,load_tokenizer,+ load_toolcall_anchor_id,)from .logger import init_loggerfrom .misc import UNSET, Unset, align_ceil, align_down, call_if_main, div_ceil, div_even, mem_GB@@ -31,6 +32,7 @@ __all__ = ["load_eos_token_ids","load_generation_sampling","load_tokenizer",+ "load_toolcall_anchor_id","init_logger","is_arch_supported","is_sm90_family",diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.pyindex 7456490b..d9cad465 100644--- a/python/freetoken/utils/hf.py+++ b/python/freetoken/utils/hf.py@@ -72,6 +72,23 @@ def load_eos_token_ids(return frozenset(ids)+def load_toolcall_anchor_id(+ tokenizer: PreTrainedTokenizerBase, opener: str | None+) -> int | None:+ """The single token id of ``opener`` -- the wire format's unique tool-call opening+ marker, declared by the model's detector (``BaseFormatDetector.toolcall_opener``).+ None when there is no opener or the tokenizer spells it with more than one token:+ the scheduler's special-token checkpoint matches sampled ids one at a time, so only+ a one-token opener can anchor."""+ if not opener:+ return None+ try:+ ids = tokenizer.encode(opener, add_special_tokens=False)+ except Exception:+ return None+ return int(ids[0]) if len(ids) == 1 else None++def load_generation_sampling(model_path: str) -> dict[str, Any]:"""Recommended sampling defaults from ``generation_config.json`` (sglang's``sampling_defaults='model'``). Returns ``{temperature, top_k, top_p}`` for the keysdiff --git a/tests/scheduler/test_abort_inflight_prefill.py b/tests/scheduler/test_abort_inflight_prefill.pyindex db35848c..1c9a9fba 100644--- a/tests/scheduler/test_abort_inflight_prefill.py+++ b/tests/scheduler/test_abort_inflight_prefill.py@@ -59,6 +59,7 @@ def _setup():prefill_manager=pm,finished_reqs=set(),eos_token_ids=set(),+ toolcall_anchor_id=None,config=SimpleNamespace(page_size=1),status_reporter=SimpleNamespace(report_batch=lambda *_, **__: None),send_result=sent.extend,