FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
refactor: add engine and scheduler seams for multi-row decode steps (#833)
Xiaoze Fane328cf638181
diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.pyindex eda3b61f..8e528450 100644--- a/python/freetoken/attention/__init__.py+++ b/python/freetoken/attention/__init__.py@@ -1,7 +1,7 @@from __future__ import annotationsfrom dataclasses import dataclass-from typing import TYPE_CHECKING, Protocol+from typing import TYPE_CHECKING, Literal, Protocolfrom freetoken.utils import Registry, init_logger@@ -33,6 +33,8 @@ class BackendInfo:# Whether forward() honors a per-call AttentionSpec (window/sm_scale/sinks).# Non-consumers raise on a non-None spec instead of silently dropping it.consumes_attn_spec: bool = False+ # How the backend captures a graph of `rows_per_req` query rows per request: "device" keeps all metadata on device, "host_plan" plans on the host before each replay, "none" cannot.+ uniform_extend_graph: Literal["none", "host_plan", "device"] = "none"SUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend")@@ -58,6 +60,7 @@ def create_trtllm_backend(config: ModelConfig):BackendInfo(supported_types=frozenset({AttnType.FULL}),requires_flashinfer=True,+ uniform_extend_graph="host_plan",),)def create_fi_backend(config: ModelConfig):@@ -71,6 +74,7 @@ def create_fi_backend(config: ModelConfig):BackendInfo(supported_types=frozenset({AttnType.FULL}),requires_sgl_kernel=True,+ uniform_extend_graph="device",),)def create_fa_backend(config: ModelConfig):@@ -84,6 +88,7 @@ def create_fa_backend(config: ModelConfig):BackendInfo(supported_types=frozenset({AttnType.FULL, AttnType.SWA}),consumes_attn_spec=True,+ uniform_extend_graph="device",),)def create_triton_backend(config: ModelConfig):diff --git a/python/freetoken/attention/base.py b/python/freetoken/attention/base.pyindex 846e9944..02172992 100644--- a/python/freetoken/attention/base.py+++ b/python/freetoken/attention/base.py@@ -70,7 +70,8 @@ class BaseAttnBackend(ABC):def prepare_metadata(self, batch: Batch) -> None: ...@abstractmethod- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None: ...+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ """Static buffers for graphs of ``rows_per_req`` query rows per request: 1 for decode, more for a uniform extend."""@abstractmethoddef prepare_for_capture(self, batch: Batch) -> None: ...@@ -116,8 +117,8 @@ class HybridBackend(BaseAttnBackend):backend = self.prefill_backend if batch.is_prefill else self.decode_backendreturn backend.prepare_metadata(batch)- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:- self.decode_backend.init_capture_graph(max_seq_len, bs_list)+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ self.decode_backend.init_capture_graph(max_seq_len, bs_list, rows_per_req)def prepare_for_capture(self, batch: Batch) -> None:self.decode_backend.prepare_for_capture(batch)diff --git a/python/freetoken/attention/dsa.py b/python/freetoken/attention/dsa.pyindex e8c7b3c2..b565b01f 100644--- a/python/freetoken/attention/dsa.py+++ b/python/freetoken/attention/dsa.py@@ -341,7 +341,9 @@ class DSAAttnBackend(DSAIndexerMixin, BaseAttnBackend):return o# ----- CUDA graph (decode) ----------------------------------------------------------- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ if rows_per_req != 1:+ raise NotImplementedError("dsa captures decode graphs only")self.max_seq_len = max_seq_lenself.capture_bs = sorted(bs_list)max_bs = max(bs_list)diff --git a/python/freetoken/attention/dsa_indexer_kpool.py b/python/freetoken/attention/dsa_indexer_kpool.pyindex 2b2492aa..2db80ea8 100644--- a/python/freetoken/attention/dsa_indexer_kpool.py+++ b/python/freetoken/attention/dsa_indexer_kpool.py@@ -56,8 +56,8 @@ class Glm5NextDSABackend(DSAAttnBackend):# ----- CUDA-graph decode staging ------------------------------------------------------# The tail rings are keyed by Req.table_idx; a captured decode step must read it# from a static buffer restaged per replay (rows/kvlen precedent in the parent).- def init_capture_graph(self, max_seq_len: int, bs_list) -> None:- super().init_capture_graph(max_seq_len, bs_list)+ def init_capture_graph(self, max_seq_len: int, bs_list, rows_per_req: int = 1) -> None:+ super().init_capture_graph(max_seq_len, bs_list, rows_per_req)self._slots_buf = torch.zeros(max(bs_list), dtype=torch.int64, device=self.device)def _stage_decode(self, batch: "Batch", bs: int, table_idx: torch.Tensor) -> None:diff --git a/python/freetoken/attention/dsv41_sparse.py b/python/freetoken/attention/dsv41_sparse.pyindex baaa2cf0..0d827ffb 100644--- a/python/freetoken/attention/dsv41_sparse.py+++ b/python/freetoken/attention/dsv41_sparse.py@@ -212,7 +212,9 @@ class DSV41SparseAttnBackend(BaseAttnBackend):decoder_rows = torch.cat(rows).to(self.device, non_blocking=True) if rows else torch.empty(0, dtype=torch.int64, device=self.device)return DSV41AttnMetadata(last_indices=last_indices, segments=segments, decoder_segments=dec, decoder_rows=decoder_rows, decoder_pad=pad)- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ if rows_per_req != 1:+ raise NotImplementedError("dsv41_sparse captures decode graphs only")assert self.capture is None, "Capture already initialized."self.max_graph_bs = max(bs_list)self.capture = DSV41CaptureData.create(self.max_graph_bs, max_seq_len, self.device)diff --git a/python/freetoken/attention/dsv4_sparse.py b/python/freetoken/attention/dsv4_sparse.pyindex 2c740e2a..bb68411a 100644--- a/python/freetoken/attention/dsv4_sparse.py+++ b/python/freetoken/attention/dsv4_sparse.py@@ -190,7 +190,9 @@ class DSV4SparseAttnBackend(BaseAttnBackend, CompressorBackendMixin, IndexerBacklast_indices=last, table_rows=self._table_rows(batch), window_ar=self._window_ar)- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ if rows_per_req != 1:+ raise NotImplementedError("dsv4_sparse captures decode graphs only")assert self.capture is None, "Capture already initialized."self.max_graph_bs = max(bs_list)self.capture = DSV4CaptureData.create(self.max_graph_bs, max_seq_len, self.device)diff --git a/python/freetoken/attention/fa.py b/python/freetoken/attention/fa.pyindex 782ef040..07212e13 100644--- a/python/freetoken/attention/fa.py+++ b/python/freetoken/attention/fa.py@@ -46,6 +46,7 @@ class FlashAttentionBackend(BaseAttnBackend):self.kvcache = ctx.kv_cacheself.page_size = ctx.page_sizeself.capture: FACaptureData | None = None+ self.capture_rows_per_req = 1self.max_graph_bs = 0self.capture_bs: List[int] = []self.scale = config.head_dim**-0.5@@ -120,23 +121,26 @@ class FlashAttentionBackend(BaseAttnBackend):page_table=new_page_table,)- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:assert self.capture is None, "Capture already initialized."max_bs = max(bs_list)- capture = FACaptureData.create(max_bs, max_seq_len // self.page_size, self.kvcache.device)+ capture = FACaptureData.create(max_bs, max_seq_len // self.page_size, self.kvcache.device, rows_per_req=rows_per_req)self.max_graph_bs = max_bsself.capture = captureself.capture_bs = sorted(bs_list)+ self.capture_rows_per_req = rows_per_reqdef prepare_for_capture(self, batch: Batch) -> None:assert (bs := batch.size) in self.capture_bs and self.capturecapture = self.capture+ rows_per_req = self.capture_rows_per_req+ capture.seq_lens[:bs].fill_(rows_per_req)metadata = FAMetadata(cu_seqlens_k=capture.cu_seqlens_k[: bs + 1],cu_seqlens_q=capture.cu_seqlens_q[: bs + 1],cache_seqlens=capture.seq_lens[:bs],max_seqlen_k=capture.page_table.size(1) * self.page_size,- max_seqlen_q=1, # decode only+ max_seqlen_q=rows_per_req,page_table=capture.page_table[:bs, :],)batch.attn_metadata = metadata@@ -145,7 +149,7 @@ class FlashAttentionBackend(BaseAttnBackend):metadata, bs = batch.attn_metadata, batch.padded_sizeassert isinstance(metadata, FAMetadata)assert self.capture is not None and bs in self.capture_bs- # cu_seqlens_q is always [0, 1, 2, ..., bs] for decode (i.e. no-op)+ # cu_seqlens_q is the constant arange * rows_per_req under a graph (i.e. no-op)table_len = metadata.page_table.size(1)self.capture.cu_seqlens_k[: bs + 1].copy_(metadata.cu_seqlens_k)self.capture.seq_lens[:bs].copy_(metadata.cache_seqlens)diff --git a/python/freetoken/attention/fi.py b/python/freetoken/attention/fi.pyindex c9e58538..175c21ff 100644--- a/python/freetoken/attention/fi.py+++ b/python/freetoken/attention/fi.py@@ -139,8 +139,11 @@ class FlashInferBackend(BaseAttnBackend):# for cuda graphself.capture_bs: List[int] = []self.max_graph_bs = 0- self.graph_wrappers: Dict[int, CUDAGraphBatchDecodeWithPagedKVCacheWrapper] = {}+ self.graph_wrappers: Dict[+ int, CUDAGraphBatchDecodeWithPagedKVCacheWrapper | BatchPrefillWithPagedKVCacheWrapper+ ] = {}self.capture: FICaptureData | None = None+ self.capture_rows_per_req = 1self.last_event = torch.cuda.Event()self.last_event.record()@@ -266,14 +269,15 @@ class FlashInferBackend(BaseAttnBackend):super().reset_capture()self.graph_wrappers = {}- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:assert self.capture is None, "Capture already initialized."max_bs = max(bs_list)- capture = FICaptureData.create(max_bs, max_seq_len, self.kvcache.device)+ capture = FICaptureData.create(max_bs, max_seq_len, self.kvcache.device, rows_per_req=rows_per_req)capture.page_table = capture.page_table.view(-1) # use 1D as ragged indicesself.max_graph_bs = max_bsself.capture = captureself.capture_bs = sorted(bs_list)+ self.capture_rows_per_req = rows_per_req@cached_propertydef use_tensor_cores(self) -> bool:@@ -283,22 +287,41 @@ class FlashInferBackend(BaseAttnBackend):GQA = self.config.num_qo_heads // self.config.num_kv_headsreturn GQA >= 4- def prepare_for_capture(self, batch: Batch) -> None:- from flashinfer import CUDAGraphBatchDecodeWithPagedKVCacheWrapper+ def _make_graph_wrapper(+ self, bs: int+ ) -> CUDAGraphBatchDecodeWithPagedKVCacheWrapper | BatchPrefillWithPagedKVCacheWrapper:+ from flashinfer import BatchPrefillWithPagedKVCacheWrapper, CUDAGraphBatchDecodeWithPagedKVCacheWrapper++ capture = self.capture+ assert capture is not None+ if self.capture_rows_per_req > 1:+ wrapper = BatchPrefillWithPagedKVCacheWrapper(+ self.float_workspace_buffer,+ kv_layout="NHD",+ backend="fa2",+ use_cuda_graph=True,+ qo_indptr_buf=capture.cu_seqlens_q[: bs + 1],+ paged_kv_indptr_buf=capture.cu_seqlens_k[: bs + 1],+ paged_kv_indices_buf=capture.indices,+ paged_kv_last_page_len_buf=capture.one_tensor[:bs],+ )+ else:+ wrapper = CUDAGraphBatchDecodeWithPagedKVCacheWrapper(+ self.float_workspace_buffer,+ kv_layout="NHD",+ use_tensor_cores=self.use_tensor_cores,+ indptr_buffer=capture.cu_seqlens_k[: bs + 1],+ indices_buffer=capture.indices,+ last_page_len_buffer=capture.one_tensor[:bs],+ )+ wrapper._backend = "fa2"+ wrapper._int_workspace_buffer = self.int_workspace_buffer+ return wrapper+ def prepare_for_capture(self, batch: Batch) -> None:bs = batch.sizeassert bs in self.capture_bs and bs not in self.graph_wrappers and self.capture- capture = self.capture- self.graph_wrappers[bs] = CUDAGraphBatchDecodeWithPagedKVCacheWrapper(- self.float_workspace_buffer,- kv_layout="NHD",- use_tensor_cores=self.use_tensor_cores,- indptr_buffer=capture.cu_seqlens_k[: bs + 1],- indices_buffer=capture.indices,- last_page_len_buffer=capture.one_tensor[:bs],- )- self.graph_wrappers[bs]._backend = "fa2"- self.graph_wrappers[bs]._int_workspace_buffer = self.int_workspace_buffer+ self.graph_wrappers[bs] = self._make_graph_wrapper(bs)self.prepare_metadata(batch)metadata = batch.attn_metadataassert isinstance(metadata, FIMetadata)diff --git a/python/freetoken/attention/m3_sparse.py b/python/freetoken/attention/m3_sparse.pyindex 82a98a5a..dc1869cb 100644--- a/python/freetoken/attention/m3_sparse.py+++ b/python/freetoken/attention/m3_sparse.py@@ -368,7 +368,9 @@ class M3SparseAttnBackend(BaseAttnBackend):return out# ----- CUDA graph (decode) ------------------------------------------------------------- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ if rows_per_req != 1:+ raise NotImplementedError("m3_sparse captures decode graphs only")self.inner.init_capture_graph(max_seq_len, bs_list)self.capture_bs = sorted(bs_list)max_bs = max(bs_list)diff --git a/python/freetoken/attention/qsa_sparse.py b/python/freetoken/attention/qsa_sparse.pyindex 8ce63c42..3e37d5f8 100644--- a/python/freetoken/attention/qsa_sparse.py+++ b/python/freetoken/attention/qsa_sparse.py@@ -489,7 +489,9 @@ class QSASparseAttnBackend(BaseAttnBackend):return torch.empty((rows, *shape), dtype=dtype, device=self.device)# ----- CUDA graph (decode) --------------------------------------------------------------- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ if rows_per_req != 1:+ raise NotImplementedError("qsa_sparse captures decode graphs only")self.capture_bs = sorted(bs_list)max_bs = max(bs_list)width = get_global_ctx().page_table.shape[1]diff --git a/python/freetoken/attention/triton.py b/python/freetoken/attention/triton.pyindex 7f29c481..071ee0d4 100644--- a/python/freetoken/attention/triton.py+++ b/python/freetoken/attention/triton.py@@ -32,15 +32,16 @@ class TritonCaptureData(BaseCaptureData):num_q_heads: int,max_head_dim: int,scratch_rows: int,+ rows_per_req: int = 1,**kwargs,):return cls(seq_lens=torch.ones((max_bs,), dtype=torch.int32, device=device),- positions=torch.zeros((max_bs,), dtype=torch.int32, device=device),+ positions=torch.zeros((max_bs * rows_per_req,), dtype=torch.int32, device=device),cu_seqlens_k=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device),- cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device),+ cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device) * rows_per_req,page_table=torch.zeros((max_bs, max_seq_len), dtype=torch.int32, device=device),- q_to_req=torch.arange(max_bs, dtype=torch.int32, device=device),+ q_to_req=torch.arange(max_bs, dtype=torch.int32, device=device).repeat_interleave(rows_per_req),attn_logits=torch.empty((scratch_rows * num_q_heads * max_head_dim,), dtype=torch.float32, device=device),attn_lse=torch.empty((scratch_rows * num_q_heads,), dtype=torch.float32, device=device),**kwargs,@@ -71,6 +72,7 @@ class TritonAttentionBackend(BaseAttnBackend):self.kvcache = get_global_ctx().kv_cacheself.device = self.kvcache.deviceself.capture: TritonCaptureData | None = None+ self.capture_rows_per_req = 1self.capture_bs: List[int] = []self.max_graph_bs = 0self.prefill_tile_min_q = 128@@ -253,7 +255,7 @@ class TritonAttentionBackend(BaseAttnBackend):swa_indices=swa_indices,)- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:assert self.capture is None, "Capture already initialized."max_bs = max(bs_list)self.capture = TritonCaptureData.create(@@ -263,7 +265,9 @@ class TritonAttentionBackend(BaseAttnBackend):num_q_heads=max(1, self.num_q_heads),max_head_dim=max(1, self.max_head_dim),scratch_rows=max(bs * self._split_cap(bs) for bs in bs_list),+ rows_per_req=rows_per_req,)+ self.capture_rows_per_req = rows_per_reqif self._swa_capture_enabled():self.capture.swa_page_table = torch.zeros((max_bs, max_seq_len),@@ -290,6 +294,10 @@ class TritonAttentionBackend(BaseAttnBackend):indices = self.capture.page_table.view(-1)self.capture.cu_seqlens_q[: bs + 1].copy_(metadata.cu_seqlens_q_gpu)self.capture.cu_seqlens_k[: bs + 1].copy_(metadata.indptr)+ if not metadata.is_decode:+ # the extend kernel reads prefix_lens; seq_lens is otherwise unused by this backend, so it holds them+ self.capture.seq_lens[:bs].copy_(metadata.prefix_lens)+ metadata.prefix_lens = self.capture.seq_lens[:bs]total = metadata.indices.numel()indices[:total].copy_(metadata.indices)if metadata.swa_indices is not None:@@ -319,6 +327,24 @@ class TritonAttentionBackend(BaseAttnBackend):bs = batch.sizeassert bs in self.capture_bs and self.capture is not Nonecapture = self.capture+ rows_per_req = self.capture_rows_per_req+ if rows_per_req > 1:+ n = bs * rows_per_req+ # placeholders for a fresh request (kv rows == query rows, no prefix); replay overwrites both+ capture.cu_seqlens_k[: bs + 1].copy_(capture.cu_seqlens_q[: bs + 1])+ capture.seq_lens[:bs].fill_(0)+ batch.attn_metadata = TritonMetadata(+ cu_seqlens_q_gpu=capture.cu_seqlens_q[: bs + 1],+ indptr=capture.cu_seqlens_k[: bs + 1],+ indices=capture.page_table.view(-1),+ q_to_req=capture.q_to_req[:n],+ q_positions=capture.positions[:n],+ is_decode=False,+ prefix_lens=capture.seq_lens[:bs],+ max_q_len=rows_per_req,+ swa_indices=self._capture_swa_indices(),+ )+ returnattn_logits, attn_lse = self._capture_scratch(bs)batch.attn_metadata = TritonMetadata(cu_seqlens_q_gpu=capture.cu_seqlens_q[: bs + 1],diff --git a/python/freetoken/attention/trtllm.py b/python/freetoken/attention/trtllm.pyindex 29844156..20eb2fb5 100644--- a/python/freetoken/attention/trtllm.py+++ b/python/freetoken/attention/trtllm.py@@ -138,7 +138,9 @@ class TensorRTLLMBackend(BaseAttnBackend):page_table=new_page_table,)- def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int], rows_per_req: int = 1) -> None:+ if rows_per_req != 1:+ raise NotImplementedError("trtllm captures decode graphs only")assert self.capture is None, "Capture already initialized."max_bs = max(bs_list)capture = TRTLLMCaptureData.create(diff --git a/python/freetoken/attention/utils.py b/python/freetoken/attention/utils.pyindex c45c3941..1de53e70 100644--- a/python/freetoken/attention/utils.py+++ b/python/freetoken/attention/utils.py@@ -12,12 +12,13 @@ class BaseCaptureData:page_table: torch.Tensor@classmethod- def create(cls, max_bs: int, max_seq_len: int, device: torch.device, **kwargs):+ def create(cls, max_bs: int, max_seq_len: int, device: torch.device, rows_per_req: int = 1, **kwargs):return cls(seq_lens=torch.ones((max_bs,), dtype=torch.int32, device=device),- positions=torch.zeros((max_bs,), dtype=torch.int32, device=device),+ positions=torch.zeros((max_bs * rows_per_req,), dtype=torch.int32, device=device),cu_seqlens_k=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device),- cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device),+ # constant under a graph: every request has exactly `rows_per_req` query rows+ cu_seqlens_q=torch.arange(0, max_bs + 1, dtype=torch.int32, device=device) * rows_per_req,page_table=torch.zeros((max_bs, max_seq_len), dtype=torch.int32, device=device),**kwargs,)diff --git a/python/freetoken/core.py b/python/freetoken/core.pyindex 812f9a43..f37ef84f 100644--- a/python/freetoken/core.py+++ b/python/freetoken/core.py@@ -11,7 +11,7 @@ if TYPE_CHECKING:from freetoken.attention.linear import FLAMetadatafrom freetoken.kvcache import BaseCacheHandle, BaseKVCachePoolfrom freetoken.kvcache.linear_state_pool import LinearStatePool- from freetoken.moe.offload_cache import OffloadMoeCache+ from freetoken.models.blocks import FeatureSink@dataclass@@ -95,6 +95,15 @@ class Req:self.cached_len = self.device_lenself.device_len += 1+ def widen_pending(self, k: int) -> None:+ """Extend the pending token at cached_len by k provisional draft tokens."""+ self.device_len += k++ def commit_pending(self, n: int) -> None:+ """Make n tokens final and leave the last committed one pending at cached_len."""+ self.cached_len += n+ self.device_len = self.cached_len + 1+def append_host(self, next_token: torch.Tensor) -> None:n = self.input_ids.numel()m = n + next_token.numel()@@ -118,7 +127,9 @@ class Req:@dataclassclass Batch:reqs: List[Req]- phase: Literal["prefill", "decode"]+ phase: Literal["prefill", "decode", "verify"]+ # query rows per request: 1 on decode, more on verify, None on prefill (ragged)+ rows_per_req: int | None = None# these fields should be set by schedulerinput_ids: torch.Tensor = field(init=False)positions: torch.Tensor = field(init=False)@@ -160,6 +171,14 @@ class Batch:# _prepare_batch succeeds. Continuation chunks leave this empty, so accounting is# exactly-once.prompt_admissions: List[Tuple[int, int, int]] = field(default_factory=list, init=False)+ # per-row hidden states a draft model reads from this forward; None when nothing reads them+ features: "FeatureSink | None" = field(default=None, init=False)++ def __post_init__(self) -> None:+ if self.rows_per_req is None and self.phase == "decode":+ self.rows_per_req = 1+ # without it, input_mapping[0][::None] would read every row as its own request+ assert self.phase != "verify" or self.rows_per_req is not None, "a verify batch needs rows_per_req"@propertydef is_prefill(self) -> bool:@@ -187,7 +206,6 @@ class Context:# NOTE: this table always treat page_size = 1page_table: torch.Tensor = field(init=False)attn_backend: BaseAttnBackend = field(init=False)- moe_offload_cache: OffloadMoeCache | None = Nonekv_cache: BaseKVCachePool = field(init=False)# Per-request recurrent state for GatedDeltaNet layers; set by the engine for# hybrid linear-attention models, otherwise None.@@ -200,11 +218,37 @@ class Context:return self._batch@contextmanager- def forward_batch(self, batch: Batch):+ def override(+ self,+ *,+ kv_cache: BaseKVCachePool | None = None,+ attn_backend: BaseAttnBackend | None = None,+ hide_target_state: bool = False,+ ):+ """Swap the KV pool and attention backend for a draft forward; hide_target_state also hides the target's GDN pool."""+ swaps = {}+ if kv_cache is not None:+ swaps["kv_cache"] = kv_cache+ if attn_backend is not None:+ swaps["attn_backend"] = attn_backend+ if hide_target_state:+ swaps["linear_state_pool"] = None+ saved = {name: getattr(self, name) for name in swaps}+ try:+ for name, value in swaps.items():+ setattr(self, name, value)+ yield+ finally:+ for name, value in saved.items():+ setattr(self, name, value)++ @contextmanager+ def forward_batch(self, batch: Batch, **override):assert self._batch is None, "Nested forward_batch is not allowed"try:self._batch = batch- yield+ with self.override(**override):+ yieldfinally:self._batch = Nonediff --git a/python/freetoken/engine/config.py b/python/freetoken/engine/config.pyindex 77aaa3ee..39379c42 100644--- a/python/freetoken/engine/config.py+++ b/python/freetoken/engine/config.py@@ -176,6 +176,11 @@ class EngineConfig:def max_forward_len(self) -> int:return self.max_seq_len+ @property+ def decode_rows_per_req(self) -> int:+ """Query rows each running request takes per decode step; every per-request decode allowance derives from it."""+ return 1+@propertydef distributed_addr(self) -> str:return "tcp://127.0.0.1:2333"diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 2de7f47a..eca7bddd 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -47,7 +47,8 @@ def _clamp_decode_batch_to_pools(config: EngineConfig, pool_sizes: list[int]) ->small = [s for s in pool_sizes if s < num_experts]if not small:return- limit = min(small) // config.model_config.num_experts_per_tok+ # a decode step routes every row of every request, decode_rows_per_req rows each+ limit = min(small) // (config.model_config.num_experts_per_tok * config.decode_rows_per_req)if config.max_running_req > limit:logger.warning_rank0(f"the smallest MoE slot pool has {min(small)} slots; lowering max_running_req "@@ -111,6 +112,12 @@ def _page_table_width(max_seq_len: int, page_size: int) -> int:return align_ceil(align_ceil(max_seq_len, page_size), 32)+def _seq_limits(config, num_tokens: int) -> tuple[int, int]:+ """(longest request, page-table width). A decode step may widen a request by decode_rows_per_req - 1 provisional rows past its length, so requests get that much less and the table still covers the widened tail."""+ limit = min(config.max_seq_len, num_tokens)+ return limit - (config.decode_rows_per_req - 1), _page_table_width(limit, config.page_size)++def _required_attn_types(model_config) -> frozenset[AttnType]:"""Backend-driving attention types of this model, from the group-spec walk(single source shared with the pool factory and the KV cost model). getattr@@ -524,8 +531,7 @@ class Engine:# ======================= Page table initialization ========================# NOTE: 1. aligned to 128 bytes; 2. store raw locations instead of pages- self.max_seq_len = min(config.max_seq_len, num_tokens)- aligned_max_seq_len = _page_table_width(self.max_seq_len, config.page_size)+ self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)self.ctx.page_table = self.page_table = torch.zeros( # + 1 for dummy request(config.max_running_req + 1, aligned_max_seq_len),dtype=torch.int32,@@ -564,16 +570,16 @@ class Engine:self.graph_runner = GraphRunner(stream=self.stream,device=self.device,- model=self.model,+ forward=self.model.forward,attn_backend=self.attn_backend,cuda_graph_bs=config.cuda_graph_bs,cuda_graph_max_bs=config.cuda_graph_max_bs,free_memory=init_free_memory,max_seq_len=aligned_max_seq_len,- vocab_size=config.model_config.vocab_size,dummy_req=self.dummy_req,moe_offload_cache=self.moe_offload_cache,mrope=config.model_config.model_is_mrope,+ out_dtype=torch.float32,)if config.attention_backend.split(",")[0] == "triton":# Prefill runs on the first comma part; warm its autotune cache.@@ -805,8 +811,8 @@ class Engine:f"(--moe-cache-scope per-layer gives each layer its own)")self._moe_slot_limit = _slot_limit(layer_methods) if layer_methods is not None else (method.slot_limit() if method is not None else None)- # with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one token per step- pools = self._moe_pools(1)+ # with GPU decode, a pool below num_experts slots bounds the batch; the startup split keeps room for one request's decode step+ pools = self._moe_pools(config.decode_rows_per_req if decode_target == "gpu" else 1)size_cap = sum(p.max_num_slots for p in pools)if config.moe_cache_auto:size, pages, overlap = self._resolve_auto_moe_cache_size(config, banks, method, max_slots=size_cap)@@ -902,7 +908,6 @@ class Engine:self._resolve_hybrid_fetch(config, cache)if locked_fetch:attach_locked_fetch(cache, config, staging, self._profile_layer_rows)- self.ctx.moe_offload_cache = cacheself.moe_offload_cache = cachereturn cache@@ -1034,6 +1039,7 @@ class Engine:# Decode batches never exceed max_running_req, but CUDA-graph padding can# round a batch up to the largest captured size; cover both.max_tokens = max(config.max_running_req, config.cuda_graph_max_bs or 0, *(config.cuda_graph_bs or []), 1)+ max_tokens *= config.decode_rows_per_reqby_id = sorted(layers, key=lambda layer: layer.layer_id)layer_formats = [layer.quant_method.cpu_spec() for layer in by_id]executor = CpuMoeExecutor(@@ -1098,8 +1104,7 @@ class Engine:def _refresh_seq_state(self, config) -> None:num_tokens = self.num_pages * config.page_size- self.max_seq_len = min(config.max_seq_len, num_tokens)- aligned_max_seq_len = _page_table_width(self.max_seq_len, config.page_size)+ self.max_seq_len, aligned_max_seq_len = _seq_limits(config, num_tokens)if aligned_max_seq_len != self.page_table.shape[1]:# max_seq_len changed (e.g. KV grew past the startup token budget); the page table# columns must track it or new requests would index out of bounds. The scheduler@@ -1143,7 +1148,7 @@ class Engine:)try:# a rebuild does not lower max_running_req again, so with GPU decode every pool must hold a full batch; hybrid caps its fetches instead- batch = config.max_running_req if self.moe_offload_cache.decode_target == "gpu" else 1+ batch = config.max_running_req * config.decode_rows_per_req if self.moe_offload_cache.decode_target == "gpu" else 1moe_pool_sizes = split_slots(moe_cache_size, self._moe_pools(batch), self._moe_expert_counts)self.moe_offload_cache.validate_rebuild(moe_cache_size, moe_pool_sizes)except ValueError as e:@@ -1244,7 +1249,7 @@ class Engine:self.linear_state_pool.rebuild(num_mamba_slots + 1)# 3. Refresh max_seq_len (+ generic page table) for the new token budget.self._refresh_seq_state(config)- aligned_max_seq_len = _page_table_width(self.max_seq_len, config.page_size)+ aligned_max_seq_len = self.page_table.shape[1]# 4. Re-capture CUDA graphs against the new tensors (reset_capture above re-armed# the backend; _sync_get_memory empties the cache so freed memory is reclaimed).gc.collect()@@ -1252,22 +1257,20 @@ class Engine:self.graph_runner = GraphRunner(stream=self.stream,device=self.device,- model=self.model,+ forward=self.model.forward,attn_backend=self.attn_backend,cuda_graph_bs=prior_graph_bs, # reuse the startup-resolved set (see above)cuda_graph_max_bs=config.cuda_graph_max_bs,free_memory=free_min,max_seq_len=aligned_max_seq_len,- vocab_size=config.model_config.vocab_size,dummy_req=self.dummy_req,moe_offload_cache=self.moe_offload_cache,mrope=config.model_config.model_is_mrope,+ out_dtype=torch.float32,)- def forward_batch(self, batch: Batch, args: BatchSamplingArgs) -> ForwardOutput:- assert torch.cuda.current_stream() == self.stream- if batch.mm_gather_plan:- self._run_mm_encoder(batch)+ def forward_logits(self, batch: Batch) -> torch.Tensor:+ """The batch's logits from its captured graph, or eagerly when none fits."""use_graph = self.graph_runner.can_use_cuda_graph(batch)with self.ctx.forward_batch(batch), self.model.forward_host_ctx(batch, use_graph):logits = self.graph_runner.replay(batch) if use_graph else self.model.forward()@@ -1275,6 +1278,13 @@ class Engine:# One pinned read: surfaces a fired flag-handshake watchdog (dead coordinator# -> stale expert outputs) as a loud error instead of silent corruption.self.cpu_moe_executor.raise_if_unhealthy()+ return logits++ def forward_batch(self, batch: Batch, args: BatchSamplingArgs) -> ForwardOutput:+ assert torch.cuda.current_stream() == self.stream+ if batch.mm_gather_plan:+ self._run_mm_encoder(batch)+ logits = self.forward_logits(batch)for req in batch.reqs:req.complete_one()diff --git a/python/freetoken/engine/graph.py b/python/freetoken/engine/graph.pyindex a995316d..42306918 100644--- a/python/freetoken/engine/graph.py+++ b/python/freetoken/engine/graph.py@@ -5,7 +5,7 @@ import functoolsimport gcimport sysfrom dataclasses import dataclass-from typing import TYPE_CHECKING, Dict, List+from typing import TYPE_CHECKING, Any, Callable, Dict, List, Literalimport torchfrom freetoken.core import Batch, Req, get_global_ctx@@ -16,7 +16,6 @@ from tqdm import tqdmif TYPE_CHECKING:from freetoken.attention import BaseAttnBackend- from freetoken.models import BaseLLMModelfrom freetoken.moe.offload_cache import OffloadMoeCachelogger = init_logger(__name__)@@ -27,56 +26,54 @@ class GraphCaptureBuffer:input_ids: torch.Tensorout_loc: torch.Tensorpositions: torch.Tensor- # [3, bs] t/h/w rope positions; allocated only for mrope models (else None).+ # [3, bs * rows_per_req] t/h/w rope positions; allocated only for mrope models (else None).mrope_positions: torch.Tensor | None- logits: torch.Tensortable_idx: torch.Tensor # per-request slot id for GatedDeltaNet state gather/scatter- # Decode GDN query indptr = arange(bs+1); a constant per captured bs, filled once.+ # GDN query indptr = arange(bs+1) * rows_per_req; a constant per captured bs, filled once.fla_cu_seqlens: torch.Tensor+ rows_per_req: int = 1@classmethod- def init(- cls, bs: int, vocab_size: int, device: torch.device, mrope: bool = False- ) -> GraphCaptureBuffer:+ def init(cls, bs: int, device: torch.device, mrope: bool = False, rows_per_req: int = 1) -> GraphCaptureBuffer:+ n = bs * rows_per_reqreturn GraphCaptureBuffer(- input_ids=torch.zeros(bs, dtype=torch.int32, device=device),- out_loc=torch.zeros(bs, dtype=torch.int32, device=device),- positions=torch.zeros(bs, dtype=torch.int32, device=device),+ input_ids=torch.zeros(n, dtype=torch.int32, device=device),+ out_loc=torch.zeros(n, dtype=torch.int32, device=device),+ positions=torch.zeros(n, dtype=torch.int32, device=device),mrope_positions=(- torch.zeros(3, bs, dtype=torch.int32, device=device) if mrope else None+ torch.zeros(3, n, dtype=torch.int32, device=device) if mrope else None),- logits=torch.empty(bs, vocab_size, dtype=torch.float32, device=device),table_idx=torch.zeros(bs, dtype=torch.int32, device=device),- fla_cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device),+ fla_cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device) * rows_per_req,+ rows_per_req=rows_per_req,)def set_batch(self, batch: Batch) -> None:from freetoken.attention.linear import FLAMetadata- _slice = slice(batch.padded_size)bs = batch.padded_size- batch.input_ids = self.input_ids[_slice]- batch.out_loc = self.out_loc[_slice]- batch.positions = self.positions[_slice]+ _rows = slice(bs * self.rows_per_req)+ batch.input_ids = self.input_ids[_rows]+ batch.out_loc = self.out_loc[_rows]+ batch.positions = self.positions[_rows]if self.mrope_positions is not None:- batch.mrope_positions = self.mrope_positions[:, _slice]- batch.linear_table_idx = self.table_idx[_slice]- # Decode GDN metadata reads the persistent cu_seqlens (constant arange) and the- # persistent table_idx slot map, so the captured kernels see stable addresses.+ batch.mrope_positions = self.mrope_positions[:, _rows]+ batch.linear_table_idx = self.table_idx[:bs]+ # GDN metadata reads the persistent cu_seqlens (constant arange * rows_per_req) and table_idx slot map, so the captured kernels see stable addresses.batch.fla_metadata = FLAMetadata(- cu_seqlens=self.fla_cu_seqlens[: bs + 1], cache_indices=self.table_idx[_slice]+ cu_seqlens=self.fla_cu_seqlens[: bs + 1], cache_indices=self.table_idx[:bs])def copy_from(self, batch: Batch) -> None:- _slice = slice(batch.padded_size)- self.input_ids[_slice] = batch.input_ids+ _rows = slice(batch.padded_size * self.rows_per_req)+ self.input_ids[_rows] = batch.input_idsif batch.out_loc is not None:- self.out_loc[_slice] = batch.out_loc- self.positions[_slice] = batch.positions+ self.out_loc[_rows] = batch.out_loc+ self.positions[_rows] = batch.positionsif self.mrope_positions is not None:- self.mrope_positions[:, _slice] = batch.mrope_positions+ self.mrope_positions[:, _rows] = batch.mrope_positionsif batch.linear_table_idx is not None:- self.table_idx[_slice] = batch.linear_table_idx+ self.table_idx[: batch.padded_size] = batch.linear_table_idxdef _determine_cuda_graph_bs(@@ -130,26 +127,37 @@ def _upload_graph(graph: torch.cuda.CUDAGraph, stream: torch.cuda.Stream) -> Nonclass GraphRunner:+ """CUDA graphs of ``forward`` over padded batches of ``rows_per_req`` rows per request, one graph per batch size; ``replay`` returns the real requests' rows of each output."""+def __init__(self,stream: torch.cuda.Stream,device: torch.device,- model: BaseLLMModel,+ forward: Callable[[], Any],attn_backend: BaseAttnBackend,cuda_graph_bs: List[int] | None,cuda_graph_max_bs: int | None,free_memory: int,max_seq_len: int,- vocab_size: int,dummy_req: Req,moe_offload_cache: OffloadMoeCache | None = None,mrope: bool = False,+ *,+ rows_per_req: int = 1,+ phase: Literal["decode", "verify"] = "decode",+ ctx_override: Dict[str, Any] | None = None,+ batch_hook: Callable[[Batch], None] | None = None,+ pool: Any = None,+ out_dtype: torch.dtype | None = None,) -> None:cuda_graph_bs = _determine_cuda_graph_bs(cuda_graph_bs=cuda_graph_bs,cuda_graph_max_bs=cuda_graph_max_bs,free_memory=free_memory,)+ # the padded rows replay the dummy request, so it must carry rows_per_req rows+ assert rows_per_req == 1 or dummy_req.extend_len == rows_per_req, (rows_per_req, dummy_req.extend_len)+ self.forward = forwardself.attn_backend = attn_backendself.max_graph_bs = max(cuda_graph_bs) if cuda_graph_bs else 0self.graph_bs_list = sorted(cuda_graph_bs)@@ -158,13 +166,33 @@ class GraphRunner:self.mrope = mropeself.stream = streamself.device = device- self._capture_graphs(max_seq_len, vocab_size, model)+ self.rows_per_req = rows_per_req+ self.phase = phase+ self.ctx_override = ctx_override or {}+ self.batch_hook = batch_hook+ self.graph_pool = pool+ self.out_dtype = out_dtype+ self._out: List[torch.Tensor] | None = None+ self._out_is_tuple = False+ self._out_rows: Dict[int, tuple[int, ...]] = {}+ self._capture_graphs(max_seq_len)def _reset_moe_offload_cache(self) -> None:if self.moe_offload_cache is not None:self.moe_offload_cache.reset()- def _capture_graphs(self, max_seq_len: int, vocab_size: int, model: BaseLLMModel):+ def _store_output(self, bs: int, out: Any) -> None:+ self._out_is_tuple = isinstance(out, tuple)+ outs = out if self._out_is_tuple else (out,)+ if self._out is None:+ # the largest batch is captured first, so its outputs size the buffers+ assert self.out_dtype is None or len(outs) == 1, "out_dtype applies to a single output"+ self._out = [torch.empty_like(o, dtype=self.out_dtype or o.dtype) for o in outs]+ for buf, o in zip(self._out, outs, strict=True):+ buf[: o.shape[0]] = o+ self._out_rows[bs] = tuple(o.shape[0] for o in outs)++ def _capture_graphs(self, max_seq_len: int):# Mark the post-weights "warmup" phase for /health: this stretch (graph capture — or the# remaining readiness work when graphs are disabled) moves no bytes, so without this the# loader would sit at 100% (last byte bar) until the ready ack. total=0 ⇒ the desktop@@ -175,7 +203,7 @@ class GraphRunner:if self.max_graph_bs == 0:return logger.info_rank0("CUDA graph is disabled.")- self.attn_backend.init_capture_graph(max_seq_len=max_seq_len, bs_list=self.graph_bs_list)+ self.attn_backend.init_capture_graph(max_seq_len=max_seq_len, bs_list=self.graph_bs_list, rows_per_req=self.rows_per_req)torch.cuda.synchronize(self.device)torch.cuda.empty_cache()@@ -185,9 +213,7 @@ class GraphRunner:free_memory = get_free_memory(self.device)logger.info_rank0(f"Free GPU memory before capturing CUDA graphs: {mem_GB(free_memory)}")- self.buffer = GraphCaptureBuffer.init(- self.max_graph_bs, vocab_size, self.device, mrope=self.mrope- )+ self.buffer = GraphCaptureBuffer.init(self.max_graph_bs, self.device, mrope=self.mrope, rows_per_req=self.rows_per_req)self._reset_moe_offload_cache()pbar = tqdm(@@ -196,14 +222,16 @@ class GraphRunner:unit="batch",disable=not get_tp_info().is_primary(), # disable for non-primary ranks)- pool = None+ pool = self.graph_poolfor bs in pbar:free_memory = get_free_memory(self.device)pbar.desc = f"Capturing graphs: bs = {bs:<3} | avail_mem = {mem_GB(free_memory)}"pbar.refresh()graph = torch.cuda.CUDAGraph()- batch = Batch(reqs=[self.dummy_req] * bs, phase="decode")+ batch = Batch(reqs=[self.dummy_req] * bs, phase=self.phase, rows_per_req=self.rows_per_req)batch.padded_reqs = batch.reqs+ if self.batch_hook is not None:+ self.batch_hook(batch)self.attn_backend.prepare_for_capture(batch)self.buffer.set_batch(batch)# capture on the dummy linear-state slot so GatedDeltaNet gather/scatter@@ -213,32 +241,41 @@ class GraphRunner:if self.dummy_req.linear_slot_idx is not Noneelse self.dummy_req.table_idx)self.buffer.table_idx[:bs].fill_(dummy_slot)- with get_global_ctx().forward_batch(batch):- self.buffer.logits[:bs] = model.forward()+ with get_global_ctx().forward_batch(batch, **self.ctx_override):+ self._store_output(bs, self.forward())# Keep the offload cache warmed for capture. Resetting here forces# CUDA graph capture to replay cold-cache expert copies.with torch.cuda.graph(graph, pool=pool, stream=self.stream):- self.buffer.logits[:bs] = model.forward()+ self._store_output(bs, self.forward())_upload_graph(graph, self.stream)self._reset_moe_offload_cache()if pool is None:pool = graph.pool() # reuse cuda graph handle to reduce memoryself.graph_map[bs] = graph+ self.graph_pool = poolself._reset_moe_offload_cache()free_memory = get_free_memory(self.device)logger.info_rank0(f"Free GPU memory after capturing CUDA graphs: {mem_GB(free_memory)}")def can_use_cuda_graph(self, batch: Batch) -> bool:- return batch.is_decode and batch.size <= self.max_graph_bs+ return (+ batch.phase == self.phase+ and batch.rows_per_req == self.rows_per_req+ and batch.size <= self.max_graph_bs+ )- def replay(self, batch: Batch) -> torch.Tensor:- assert self.can_use_cuda_graph(batch)+ def replay(self, batch: Batch) -> Any:+ assert self.can_use_cuda_graph(batch) and self._out is not Noneself.buffer.copy_from(batch)- g = self.graph_map[batch.padded_size]+ padded = batch.padded_size+ g = self.graph_map[padded]self.attn_backend.prepare_for_replay(batch)g.replay()- return self.buffer.logits[: batch.size]+ outs = tuple(+ buf[: rows // padded * batch.size] for buf, rows in zip(self._out, self._out_rows[padded], strict=True)+ )+ return outs if self._out_is_tuple else outs[0]def pad_batch(self, batch: Batch) -> None:padded_size = ( # choose the first available batch size@@ -251,10 +288,11 @@ class GraphRunner:# NOTE: This must be called before freeing NCCL resources to prevent program hangdef destroy_cuda_graphs(self) -> None:# Drop the CUDAGraph objects (and the shared mempool they hold) AND the static- # GraphCaptureBuffer tensors ([max_bs, vocab] logits + input/out_loc/positions/...).+ # GraphCaptureBuffer tensors (input/out_loc/positions/...) and the output buffers (_out).# Dropping the references is the load-bearing step; without it a runtime rebuild's# free-before-alloc cannot reclaim this GPU memory. empty_cache() is left to the# caller / next capture (GraphRunner._capture_graphs already runs it).self.graph_map = {}self.buffer = None+ self._out = Nonegc.collect()diff --git a/python/freetoken/layers/embedding.py b/python/freetoken/layers/embedding.pyindex f86670b1..469b752d 100644--- a/python/freetoken/layers/embedding.py+++ b/python/freetoken/layers/embedding.py@@ -129,11 +129,16 @@ class ParallelLMHead(VocabParallelEmbedding):ctx = get_global_ctx()batch = ctx.batchbs = batch.size+ if (features := getattr(batch, "features", None)) is not None:+ features.final_hidden(x)if batch.is_prefill:indices = batch.attn_metadata.get_last_indices(bs)x = x[indices].contiguous()del indices+ return self.logits(x)+ def logits(self, x: torch.Tensor) -> torch.Tensor:+ """Full-vocab logits of the rows of ``x``; reads no batch state."""if self.tied_embedding is not None:logits = self.tied_embedding.quant_method.logits(self.tied_embedding, x, self.bias)else:@@ -143,7 +148,8 @@ class ParallelLMHead(VocabParallelEmbedding):input_shape = logits.shapeoutput_tensor = self._comm.all_gather(logits)- if bs == 1:+ # keyed on rows, not batch size: a verify batch has several rows per request+ if input_shape[0] == 1:return output_tensor.view(1, -1)[:, : self.num_embeddings]output_tensor = output_tensor.view((self.tp_size,) + input_shape)diff --git a/python/freetoken/llm/llm.py b/python/freetoken/llm/llm.pyindex 20ae0874..8b6a17a1 100644--- a/python/freetoken/llm/llm.py+++ b/python/freetoken/llm/llm.py@@ -94,8 +94,10 @@ class LLM(Scheduler):continueassert isinstance(msg, DetokenizeMsg)status = self.status_map[msg.uid]- if not (msg.finished and msg.next_token in self.eos_token_ids):- status.output_ids.append(msg.next_token)+ tokens = msg.next_tokens+ if msg.finished and tokens and tokens[-1] in self.eos_token_ids:+ tokens = tokens[:-1]+ status.output_ids.extend(tokens)def generate(self,diff --git a/python/freetoken/message/tokenizer.py b/python/freetoken/message/tokenizer.pyindex f091d0d3..56308d6e 100644--- a/python/freetoken/message/tokenizer.py+++ b/python/freetoken/message/tokenizer.py@@ -27,7 +27,8 @@ class BatchTokenizerMsg(BaseTokenizerMsg):@dataclassclass DetokenizeMsg(BaseTokenizerMsg):uid: int- next_token: int+ # tokens committed this step, in order: one per decode step, up to k+1 per speculative verify+ next_tokens: List[int]finished: boolfinish_reason: str | None = None# The stop string that ended generation (if any), so the detokenizer can trim itdiff --git a/python/freetoken/models/blocks.py b/python/freetoken/models/blocks.pyindex 4f039fd3..a7a095a2 100644--- a/python/freetoken/models/blocks.py+++ b/python/freetoken/models/blocks.py@@ -48,6 +48,21 @@ class SupportsMultimodal(Protocol):...+@runtime_checkable+class FeatureSink(Protocol):+ """Where a forward hands per-row hidden states to a draft model; set on ``batch.features``."""++ wanted_layers: frozenset[int]++ def final_hidden(self, x: torch.Tensor) -> None:+ """Every row's hidden state before the LM head; the LM head calls it once per forward."""+ ...++ def layer_output(self, layer_id: int, x: torch.Tensor, num_streams: int = 1) -> None:+ """A layer's output rows; called only for layers in ``wanted_layers``."""+ ...++def embed_input_ids(embed_tokens, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor:"""Token embeddings of the batch; on a chunk with multimodal rows, batch.mm_embeds' leading columns replace the rows batch.mm_rows."""if batch.mm_embeds is None:diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 4a6a297d..6610b128 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -690,7 +690,7 @@ class OffloadMoeCache:reloads banks. Tears down prefill-overlap buffers first (their views aliasthe old ``bank_caches``), frees the old GPU tensors, then reallocates. Slotscold-start after rebuild. Object identity is preserved so attached layers and- ``ctx.moe_offload_cache`` stay valid.+ ``Engine.moe_offload_cache`` stay valid."""assert self.bank_sources, "set_bank_sources must run before rebuild"sizes = [cache_size] if pool_sizes is None else list(pool_sizes)diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.pyindex d4982f9e..5a3b4601 100644--- a/python/freetoken/scheduler/cache.py+++ b/python/freetoken/scheduler/cache.py@@ -635,6 +635,19 @@ class CacheManager:# ids stay below 2^31 (multimodal pseudo ids included), so a pair packs into one int64return (ids[:m].to(torch.int64) << 31) | ids[1 : m + 1].to(torch.int64)+ def rollback_tail(self, req: Req, keep_len: int, drop_len: int) -> None:+ """Free the slots allocated for positions [keep_len, drop_len), rounded out to whole pages as allocate_paged allocates them.++ Call inside lazy_free_region, after the forward that wrote them has drained.+ """+ start = div_ceil(keep_len, self.page_size) * self.page_size+ end = div_ceil(drop_len, self.page_size) * self.page_size+ if end > start:+ tail = self.page_table[req.table_idx, start:end]+ if self.swa_paged:+ self._free_swa(tail)+ self._free(tail)+def _page_to_token(self, pages: torch.Tensor) -> torch.Tensor:if self.page_size == 1:return pagesdiff --git a/python/freetoken/scheduler/decode.py b/python/freetoken/scheduler/decode.pyindex 8f74c12c..c5c75ef2 100644--- a/python/freetoken/scheduler/decode.py+++ b/python/freetoken/scheduler/decode.py@@ -9,6 +9,8 @@ from freetoken.core import Batch, Req@dataclassclass DecodeManager:page_size: int+ # slots a decode step may hold past each request's committed length (provisional rows)+ token_headroom: int = 0running_reqs: Set[Req] = field(default_factory=set)def filter_reqs(self, reqs: Iterable[Req]) -> None:@@ -26,7 +28,8 @@ class DecodeManager:@propertydef inflight_tokens(self) -> int:- tokens_reserved = (self.page_size - 1) * len(self.running_reqs) # 1 page reserved+ # each request's partly filled page, plus the provisional rows a decode step may add+ tokens_reserved = (self.page_size - 1 + self.token_headroom) * len(self.running_reqs)return sum(req.remain_len for req in self.running_reqs) + tokens_reserveddef schedule_next_batch(self) -> Batch | None:diff --git a/python/freetoken/scheduler/prefill.py b/python/freetoken/scheduler/prefill.pyindex de4bddfb..94e708fb 100644--- a/python/freetoken/scheduler/prefill.py+++ b/python/freetoken/scheduler/prefill.py@@ -55,6 +55,8 @@ class PrefillAdder:# allocated only in allocate_paged (after the pass), so swa_available_size does not decrement# across the admission loop -- without this, successive admits all see the full pool.reserved_swa: int = 0+ # slots a decode step may hold past a request's own length; reserved with it+ token_headroom: int = 0def __post_init__(self) -> None:if not self.pass_budget:@@ -79,7 +81,7 @@ class PrefillAdder:# TODO: better estimate policyextend_len = req.input_len - cached_lenestimated_size = self._kv_reservation_size(- req.input_len + req.output_len, cached_len+ req.input_len + req.output_len + self.token_headroom, cached_len)if estimated_size + self.reserved_size > self.cache_manager.available_size:@@ -201,7 +203,7 @@ class PrefillAdder:# page span here; charging raw tokens can admit several short requests against one page# even though allocate_paged() needs a separate page for each request.self.reserved_size += self._kv_reservation_size(- pending_req.input_len + pending_req.output_len, cached_len+ pending_req.input_len + pending_req.output_len + self.token_headroom, cached_len)# NOTE: update the tokens ids only; new pages will be allocated in the scheduler_slice = slice(cached_len, cached_len + chunk_size)@@ -277,6 +279,7 @@ class PrefillManager:decode_manager: DecodeManagerencoder_cache: EncoderCache | None = Nonekeep_images_whole: bool = False+ token_headroom: int = 0pending_list: List[PendingReq] = field(default_factory=list)def add_one_req(self, req: UserMsg) -> None:@@ -303,6 +306,7 @@ class PrefillManager:table_manager=self.table_manager,encoder_cache=self.encoder_cache,keep_images_whole=self.keep_images_whole,+ token_headroom=self.token_headroom,)reqs: List[Req] = []chunked_list: List[PendingReq] = []diff --git a/python/freetoken/scheduler/scheduler.py b/python/freetoken/scheduler/scheduler.pyindex f16608cc..21ef19d9 100644--- a/python/freetoken/scheduler/scheduler.py+++ b/python/freetoken/scheduler/scheduler.py@@ -93,7 +93,8 @@ class Scheduler(SchedulerIOMixin):None,) or getattr(self.engine.kv_cache, "sliding_window_size", None),)- self.decode_manager = DecodeManager(config.page_size)+ token_headroom = config.decode_rows_per_req - 1+ self.decode_manager = DecodeManager(config.page_size, token_headroom=token_headroom)self._bidirectional_mm = any(getattr(g, "bidirectional_mm_blocks", False) for g in config.model_config.attention_groups)self.prefill_manager = PrefillManager(self.cache_manager,@@ -101,6 +102,7 @@ class Scheduler(SchedulerIOMixin):self.decode_manager,encoder_cache=self.engine.encoder_cache,keep_images_whole=self._bidirectional_mm,+ token_headroom=token_headroom,)# some alias for easy access@@ -360,43 +362,9 @@ class Scheduler(SchedulerIOMixin):# are freed below/already; shipping this token would append past the# client's terminal reply.continue- next_token = next_tokens_cpu[i]- req.append_host(next_token.unsqueeze(0))- next_token = int(next_token.item())- # EOS / stop-string -> "stop", output budget exhausted -> "length";- # EOS and stop strings win over length.- # Overlap can advance device_len one token ahead of this host reply.- hit_length = req.input_ids.numel() >= req.max_device_len- hit_eos = (- not req.sampling_params.ignore_eos and next_token in self.eos_token_ids- )- matched_stop = (- self._match_stop_str(req)- if not hit_eos and req.sampling_params.stop_strs- else None- )- finished = hit_length or hit_eos or matched_stop is not None- finish_reason = (- ("stop" if (hit_eos or matched_stop is not None) else "length")- if finished- else None- )- if (- next_token == self.toolcall_anchor_id- and req.toolcall_anchor_len is None- and not finished- ):- req.toolcall_anchor_len = req.input_ids.numel()- reply.append(- DetokenizeMsg(- uid=req.uid,- next_token=next_token,- finished=finished,- finish_reason=finish_reason,- matched_stop=matched_stop,- stop_strs=req.sampling_params.stop_strs or None,- )- )+ msg, _ = self._commit_tokens(req, next_tokens_cpu[i : i + 1])+ reply.append(msg)+ finished = msg.finished# NOTE: overlap scheduling may make the request freed twice, skip second freeif finished and req not in self.finished_reqs:@@ -444,6 +412,37 @@ class Scheduler(SchedulerIOMixin):)self.send_result(reply)+ def _commit_tokens(self, req: Req, tokens: torch.Tensor) -> Tuple[DetokenizeMsg, int]:+ """Append sampled tokens in order until one finishes the request; returns the reply and how many tokens were committed."""+ params = req.sampling_params+ committed: List[int] = []+ finished, finish_reason, matched_stop = False, None, None+ for j in range(tokens.numel()):+ req.append_host(tokens[j : j + 1])+ token = int(tokens[j].item())+ committed.append(token)+ # EOS / stop-string -> "stop", output budget exhausted -> "length";+ # EOS and stop strings win over length.+ # Overlap can advance device_len one token ahead of this host reply.+ hit_length = req.input_ids.numel() >= req.max_device_len+ hit_eos = not params.ignore_eos and token in self.eos_token_ids+ matched_stop = self._match_stop_str(req) if not hit_eos and params.stop_strs else None+ if hit_length or hit_eos or matched_stop is not None:+ finished = True+ finish_reason = "stop" if (hit_eos or matched_stop is not None) else "length"+ break+ if token == self.toolcall_anchor_id and req.toolcall_anchor_len is None:+ req.toolcall_anchor_len = req.input_ids.numel()+ msg = DetokenizeMsg(+ uid=req.uid,+ next_tokens=committed,+ finished=finished,+ finish_reason=finish_reason,+ matched_stop=matched_stop,+ stop_strs=params.stop_strs or None,+ )+ return msg, len(committed)+def _match_stop_str(self, req: Req) -> str | None:"""First stop string present in this request's generated tail, else None. Decodesonly a short suffix (bounded by the longest stop string's char length, so a stop of@@ -841,7 +840,7 @@ class Scheduler(SchedulerIOMixin):def _prepare_batch(self, batch: Batch) -> ForwardInput:self.engine.graph_runner.pad_batch(batch)self._forward_iter += 1- if batch.is_decode:+ if not batch.is_prefill:# Free each decoding request's now-out-of-window SWA slots BEFORE the alloc below,# so they can back the new token -- this is what bounds the per-request swa# footprint during decode. (no-op unless the model is SWA / paged swa pool.)@@ -866,7 +865,7 @@ class Scheduler(SchedulerIOMixin):write_mapping = _make_write_tuple(batch, self.device)batch.out_loc = self.engine.page_table[input_mapping]if self.engine.linear_state_pool is not None:- if batch.is_decode:+ if not batch.is_prefill:# GPU GDN-state slot (one per padded request) for the decode gather/scatter;# lands in the CUDA-graph input buffer via copy_from. Gate on the cache mode,# NOT on whether any padded req has a linear_slot_idx -- the persistent dummy@@ -882,15 +881,15 @@ class Scheduler(SchedulerIOMixin):slots, dtype=torch.int32, device="cpu", pin_memory=True).to(self.device, non_blocking=True)else:- batch.linear_table_idx = input_mapping[0].to(torch.int32)+ batch.linear_table_idx = input_mapping[0][:: batch.rows_per_req].to(torch.int32)# Per-forward GDN metadata (cu_seqlens / cache_indices / continuation flags),# built once here instead of rebuilt in each of the 30 GDN layers. For decode# under CUDA graph the persistent cu_seqlens buffer is supplied by set_batch.batch.fla_metadata = build_fla_metadata(batch, self.device)- if batch.is_decode:+ if not batch.is_prefill:# This batch's padded per-row page-table rows. Backends that snapshot the table for# a captured replay (DSV4) read them in prepare_metadata / prepare_for_replay.- batch.active_table_idx = input_mapping[0].view(-1)+ batch.active_table_idx = input_mapping[0][:: batch.rows_per_req].view(-1)self.engine.attn_backend.prepare_metadata(batch)return ForwardInput(batch=batch,diff --git a/python/freetoken/scheduler/status.py b/python/freetoken/scheduler/status.pyindex 76c20cd4..5e9ca0e0 100644--- a/python/freetoken/scheduler/status.py+++ b/python/freetoken/scheduler/status.py@@ -36,7 +36,9 @@ class SchedulerStatusReporter:page_size: int,mamba_slots: tuple[int, int] | None = None,swa_tokens: tuple[int, int] | None = None,+ generated: int | None = None,) -> None:+ """``generated``: tokens a decode step committed; None means one per request."""if batch.is_prefill:self._report_prefill(batch,@@ -47,7 +49,7 @@ class SchedulerStatusReporter:mamba_slots=mamba_slots,swa_tokens=swa_tokens,)- elif batch.is_decode:+ else:self._report_decode(batch,running_reqs=running_reqs,@@ -57,6 +59,7 @@ class SchedulerStatusReporter:page_size=page_size,mamba_slots=mamba_slots,swa_tokens=swa_tokens,+ generated=generated,)def _report_prefill(@@ -103,9 +106,10 @@ class SchedulerStatusReporter:page_size: int,mamba_slots: tuple[int, int] | None = None,swa_tokens: tuple[int, int] | None = None,+ generated: int | None = None,) -> None:self._decode_forward_count += 1- self._decode_generated_tokens += len(batch.reqs)+ self._decode_generated_tokens += len(batch.reqs) if generated is None else generatedif self._decode_forward_count % self.decode_log_interval != 0:returndiff --git a/python/freetoken/tokenizer/detokenize.py b/python/freetoken/tokenizer/detokenize.pyindex d138ac1a..045e3389 100644--- a/python/freetoken/tokenizer/detokenize.py+++ b/python/freetoken/tokenizer/detokenize.py@@ -103,8 +103,10 @@ class DetokenizeManager:sent_offset=0,)s = self.decode_map[msg.uid]- if not (msg.finished and msg.next_token in self.eos_token_ids):- s.decoded_ids.append(msg.next_token)+ tokens = msg.next_tokens+ if msg.finished and tokens and tokens[-1] in self.eos_token_ids:+ tokens = tokens[:-1]+ s.decoded_ids.extend(tokens)read_ids.append(s.decoded_ids[s.surr_offset :])surr_ids.append(s.decoded_ids[s.surr_offset : s.read_offset])diff --git a/python/freetoken/tokenizer/server.py b/python/freetoken/tokenizer/server.pyindex fad66d74..98887160 100644--- a/python/freetoken/tokenizer/server.py+++ b/python/freetoken/tokenizer/server.py@@ -236,7 +236,7 @@ def tokenize_worker(finished=msg.finished,finish_reason=msg.finish_reason,matched_stop=msg.matched_stop,- completion_tokens_delta=1,+ completion_tokens_delta=len(msg.next_tokens),kv_used_pages=msg.kv_used_pages,kv_total_pages=msg.kv_total_pages,mamba_used_slots=msg.mamba_used_slots,diff --git a/tests/attention/test_capture_rows_per_req.py b/tests/attention/test_capture_rows_per_req.pynew file mode 100644index 00000000..a7469f1b--- /dev/null+++ b/tests/attention/test_capture_rows_per_req.py@@ -0,0 +1,105 @@+"""Attention graphs captured for `rows_per_req` query rows per request (1 = decode, more = a uniform extend such as a speculative verify) replay what the eager path computes."""++from types import SimpleNamespace++import pytest+import torch++import freetoken.core as core+from freetoken.attention import attention_backend_info+from freetoken.core import Batch, Context, Req+from freetoken.distributed import set_tp_info, try_get_tp_info++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++HEADS, KV_HEADS, HEAD_DIM = 8, 2, 128+BS, WIDTH = 2, 64+CACHED = (5, 17)+++def _backend(name: str):+ config = SimpleNamespace(num_qo_heads=HEADS, num_kv_heads=KV_HEADS, head_dim=HEAD_DIM)+ if name == "fa":+ pytest.importorskip("sgl_kernel")+ from freetoken.attention.fa import FlashAttentionBackend++ return FlashAttentionBackend(config)+ if name == "fi":+ pytest.importorskip("flashinfer")+ from freetoken.attention.fi import FlashInferBackend++ return FlashInferBackend(config)+ from freetoken.attention.triton import TritonAttentionBackend++ return TritonAttentionBackend(config)+++def _context(monkeypatch) -> Context:+ from freetoken.kvcache.mha_pool import MHAKVCache++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ device = torch.device("cuda")+ ctx = Context(page_size=1)+ # rows 0..BS-1 belong to the requests, row BS is the dummy request's (padded rows write there)+ ctx.page_table = torch.arange((BS + 1) * WIDTH, dtype=torch.int32, device=device).view(BS + 1, WIDTH)+ ctx.kv_cache = MHAKVCache(KV_HEADS, 1, HEAD_DIM, num_pages=(BS + 1) * WIDTH, page_size=1, dtype=torch.bfloat16, device=device)+ n = (BS + 1) * WIDTH+ ctx.kv_cache.store_kv(+ torch.randn(n, KV_HEADS * HEAD_DIM, dtype=torch.bfloat16, device=device),+ torch.randn(n, KV_HEADS * HEAD_DIM, dtype=torch.bfloat16, device=device),+ torch.arange(n, dtype=torch.int32, device=device),+ 0,+ )+ monkeypatch.setattr(core, "_GLOBAL_CTX", ctx)+ return ctx+++def _batch(reqs, rows_per_req: int, ctx: Context) -> Batch:+ batch = Batch(reqs=reqs, phase="decode" if rows_per_req == 1 else "verify", rows_per_req=rows_per_req)+ batch.padded_reqs = reqs+ batch.positions = torch.cat([torch.arange(r.cached_len, r.device_len, dtype=torch.int32) for r in reqs]).cuda()+ batch.out_loc = torch.cat([ctx.page_table[r.table_idx, r.cached_len : r.device_len] for r in reqs])+ return batch+++def _req(table_idx: int, cached: int, rows_per_req: int) -> Req:+ return Req(input_ids=torch.zeros(cached + rows_per_req, dtype=torch.int32), table_idx=table_idx, cached_len=cached,+ output_len=1, uid=table_idx, sampling_params=None, cache_handle=None)++[email protected]("rows_per_req", [1, 4])[email protected]("name", ["fa", "triton", "fi"])+def test_graph_replay_matches_eager_attention(monkeypatch, name, rows_per_req):+ assert attention_backend_info(name).uniform_extend_graph != "none"+ ctx = _context(monkeypatch)+ backend = _backend(name)+ rows = BS * rows_per_req+ q = torch.randn(rows, HEADS, HEAD_DIM, dtype=torch.bfloat16, device="cuda")+ k = torch.randn(rows, KV_HEADS * HEAD_DIM, dtype=torch.bfloat16, device="cuda")+ v = torch.randn(rows, KV_HEADS * HEAD_DIM, dtype=torch.bfloat16, device="cuda")++ eager = _batch([_req(i, CACHED[i], rows_per_req) for i in range(BS)], rows_per_req, ctx)+ backend.prepare_metadata(eager)+ expected = backend.forward(q, k, v, 0, eager)++ backend.init_capture_graph(max_seq_len=WIDTH, bs_list=[BS], rows_per_req=rows_per_req)+ capture = _batch([_req(BS, 0, rows_per_req)] * BS, rows_per_req, ctx)+ backend.prepare_for_capture(capture)+ static_q, static_k, static_v = q.clone(), k.clone(), v.clone()+ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ backend.forward(static_q, static_k, static_v, 0, capture)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph, stream=stream):+ out = backend.forward(static_q, static_k, static_v, 0, capture)+ torch.cuda.current_stream().wait_stream(stream)++ replay = _batch([_req(i, CACHED[i], rows_per_req) for i in range(BS)], rows_per_req, ctx)+ backend.prepare_metadata(replay)+ backend.prepare_for_replay(replay)+ capture.out_loc.copy_(replay.out_loc)+ graph.replay()+ torch.cuda.synchronize()+ torch.testing.assert_close(out, expected, atol=1e-2, rtol=1e-2)diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.pyindex 44130d35..5c1f93b3 100644--- a/tests/engine/test_cache_budget.py+++ b/tests/engine/test_cache_budget.py@@ -569,7 +569,14 @@ def test_moe_pool_flags_and_decode_batch_clamp():# GPU decode needs batch * top_k distinct experts to fit the smallest pool below num_expertsconfig = SimpleNamespace(model_config=SimpleNamespace(num_experts=16, num_experts_per_tok=4),- max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4],+ max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4], decode_rows_per_req=1,)_clamp_decode_batch_to_pools(config, [12, 16, 9])assert (config.max_running_req, config.cuda_graph_max_bs, config.cuda_graph_bs) == (2, 2, [1, 2])+ # two rows per request route twice the experts: 9 // (4 * 2) leaves one request+ config = SimpleNamespace(+ model_config=SimpleNamespace(num_experts=16, num_experts_per_tok=4),+ max_running_req=4, cuda_graph_max_bs=None, cuda_graph_bs=[1, 2, 4], decode_rows_per_req=2,+ )+ _clamp_decode_batch_to_pools(config, [12, 16, 9])+ assert (config.max_running_req, config.cuda_graph_max_bs, config.cuda_graph_bs) == (1, 1, [1])diff --git a/tests/kvcache/test_kv_cache_rebuild.py b/tests/kvcache/test_kv_cache_rebuild.pyindex 0e3b7050..be1e87c3 100644--- a/tests/kvcache/test_kv_cache_rebuild.py+++ b/tests/kvcache/test_kv_cache_rebuild.py@@ -227,7 +227,7 @@ def test_dsv4_refresh_seq_state_tracks_page_table_width():P, mrr = 128, 4config = SimpleNamespace(- max_seq_len=65536, page_size=P, max_running_req=mrr,+ max_seq_len=65536, page_size=P, max_running_req=mrr, decode_rows_per_req=1,model_config=SimpleNamespace(dsv4_args=SimpleNamespace(window_size=P)),)# attach_page_table only re-points the pool's full_loc_map; the decode snapshot belongs todiff --git a/tests/models/qwen4_exp/test_ple_disk.py b/tests/models/qwen4_exp/test_ple_disk.pyindex f71c9e6a..85a98dfd 100644--- a/tests/models/qwen4_exp/test_ple_disk.py+++ b/tests/models/qwen4_exp/test_ple_disk.py@@ -398,10 +398,9 @@ def _wait_sync_large_graph(tmp_path, kernels):stream.wait_stream(torch.cuda.current_stream())with torch.cuda.stream(stream):# Exercise the capture-time upload from #697 as well as the PLE dispatch hook.- runner = GraphRunner(stream=stream, device=torch.device("cuda"), model=SimpleNamespace(forward=forward),+ runner = GraphRunner(stream=stream, device=torch.device("cuda"), forward=forward,attn_backend=attn, cuda_graph_bs=[8], cuda_graph_max_bs=None,- free_memory=1024**3, max_seq_len=16,- vocab_size=args.num_ngram_heads * disk.head_dim, dummy_req=req)+ free_memory=1024**3, max_seq_len=16, dummy_req=req, out_dtype=torch.float32)batch = Batch(reqs=[req] * 8, phase="decode")runner.pad_batch(batch)batch.positions = torch.full((8,), 2, dtype=torch.int32, device="cuda")diff --git a/tests/moe/test_offload.py b/tests/moe/test_offload.pyindex a29d94e9..3baddeb5 100644--- a/tests/moe/test_offload.py+++ b/tests/moe/test_offload.py@@ -475,7 +475,7 @@ def test_graph_capture_reuses_warm_offload_cache_before_capture(monkeypatch):events.append("graph_exit")class FakeAttnBackend:- def init_capture_graph(self, max_seq_len, bs_list):+ def init_capture_graph(self, max_seq_len, bs_list, rows_per_req=1):passdef prepare_for_capture(self, batch):@@ -511,13 +511,12 @@ def test_graph_capture_reuses_warm_offload_cache_before_capture(monkeypatch):GraphRunner(stream=None,device=torch.device("cpu"),- model=FakeModel(),+ forward=FakeModel().forward,attn_backend=FakeAttnBackend(),cuda_graph_bs=[1],cuda_graph_max_bs=None,free_memory=1024,max_seq_len=1,- vocab_size=3,dummy_req=dummy_req,moe_offload_cache=FakeOffloadCache(),)diff --git a/tests/scheduler/test_abort_inflight_prefill.py b/tests/scheduler/test_abort_inflight_prefill.pyindex 85c21785..2ec5508a 100644--- a/tests/scheduler/test_abort_inflight_prefill.py+++ b/tests/scheduler/test_abort_inflight_prefill.py@@ -72,6 +72,7 @@ def _setup():_last_data=None,)stub._free_req_resources = lambda req: Scheduler._free_req_resources(stub, req)+ stub._commit_tokens = lambda req, tokens: Scheduler._commit_tokens(stub, req, tokens)return pool, cm, tm, dm, pm, sent, stubdiff --git a/tests/scheduler/test_cost_accounting_core.py b/tests/scheduler/test_cost_accounting_core.pyindex ed741ef8..67ac53d1 100644--- a/tests/scheduler/test_cost_accounting_core.py+++ b/tests/scheduler/test_cost_accounting_core.py@@ -209,7 +209,7 @@ def test_normal_loop_sends_prior_sample_before_abort_terminal():scheduler._schedule_next_batch = lambda: Nonesent = []scheduler.send_result = lambda messages: sent.append(messages)- late_sample = DetokenizeMsg(uid=5, next_token=99, finished=False)+ late_sample = DetokenizeMsg(uid=5, next_tokens=[99], finished=False)scheduler._process_last_data = lambda _data: scheduler.send_result([late_sample])Scheduler.normal_loop(scheduler)diff --git a/tests/scheduler/test_rollback_tail.py b/tests/scheduler/test_rollback_tail.pynew file mode 100644index 00000000..da74b188--- /dev/null+++ b/tests/scheduler/test_rollback_tail.py@@ -0,0 +1,36 @@+"""rollback_tail after a widened decode step: the request keeps exactly the pages of its committed tokens, the rest go back, and the next step allocates no page twice."""++from types import SimpleNamespace++import pytest+import torch++from freetoken.core import Req+from freetoken.scheduler.cache import CacheManager+from freetoken.utils import div_ceil++K = 3+NUM_PAGES = 64++[email protected]("page_size", [1, 4])[email protected]("cached", [8, 9, 11])[email protected]("n", range(1, K + 2))+def test_rollback_keeps_committed_pages_only(page_size, cached, n):+ page_table = torch.zeros(2, 64, dtype=torch.int32)+ cm = CacheManager(NUM_PAGES, page_size, page_table, "naive")+ req = Req(input_ids=torch.zeros(cached, dtype=torch.int32), table_idx=0, cached_len=0, output_len=32,+ uid=0, sampling_params=SimpleNamespace(), cache_handle=None)+ cm.allocate_paged([req])+ req.complete_one() # cached_len = cached, the pending token sits at position `cached`++ for _ in range(2):+ entry = req.cached_len+ req.widen_pending(K)+ cm.allocate_paged([req])+ req.commit_pending(n)+ with cm.lazy_free_region():+ cm.rollback_tail(req, req.cached_len, entry + K + 1)+ kept = div_ceil(req.cached_len, page_size)+ assert len(cm.free_slots) == NUM_PAGES - kept+ assert not set(cm.free_slots.tolist()) & set(cm.page_table[0, : kept * page_size : page_size].tolist())diff --git a/tests/server/test_message_wire.py b/tests/server/test_message_wire.pyindex fdc09592..76d8ecc3 100644--- a/tests/server/test_message_wire.py+++ b/tests/server/test_message_wire.py@@ -106,7 +106,7 @@ def test_user_reply_token_deltas_round_trip():def test_detokenize_msg_carries_kv_usage_round_trip():msg = DetokenizeMsg(- uid=3, next_token=42, finished=True,+ uid=3, next_tokens=[42], finished=True,kv_used_pages=10, kv_total_pages=256, gpu_mem_bytes=1 << 30,mamba_used_slots=7, mamba_total_slots=64,swa_used_tokens=8448, swa_total_tokens=76800,