FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(qwen4_exp): support Qwen3.8-Flash-Next (#257)
Xiaoze Fanbd8f3d519a48
diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.pyindex 746c04c4..72b01c04 100644--- a/python/freetoken/attention/__init__.py+++ b/python/freetoken/attention/__init__.py@@ -33,10 +33,6 @@ class BackendInfo:# Whether forward() honors a per-call AttentionSpec (window/sm_scale/sinks).# Non-consumers raise on a non-None spec instead of silently dropping it.consumes_attn_spec: bool = False- # Whether this backend coexists with hybrid-linear (GDN/mamba) models. The- # linear layers bypass the backend entirely, but a backend whose metadata or- # graph machinery assumes layer 0 is an attention layer can opt out here.- hybrid_linear_ok: bool = TrueSUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend")@@ -132,6 +128,21 @@ def create_m3_sparse_backend(config: ModelConfig):return M3SparseAttnBackend(config)+@SUPPORTED_ATTENTION_BACKENDS.register(+ "qsa_sparse",+ BackendInfo(+ supported_types=frozenset({AttnType.QSA}),+ # 64-token pages: a 4-token compress group never straddles a page, so the+ # compressed row of a group is page_base // 4 + block-in-page.+ page_sizes=(64,),+ ),+)+def create_qsa_sparse_backend(config: ModelConfig):+ from .qsa_sparse import QSASparseAttnBackend++ return QSASparseAttnBackend(config)++def attention_backend_info(name: str) -> BackendInfo:return SUPPORTED_ATTENTION_BACKENDS.info(name)diff --git a/python/freetoken/attention/base.py b/python/freetoken/attention/base.pyindex eb39d472..ca36fb88 100644--- a/python/freetoken/attention/base.py+++ b/python/freetoken/attention/base.py@@ -24,6 +24,10 @@ class AttnType(str, Enum):# GQA block-sparse (MiniMax-M3): paged GQA K/V + a per-sparse-layer index-key# slab; the indexer picks top-k 128-token blocks per query -> BSAKVCacheBSA = "bsa"+ # QSA compressed-block sparse (Qwen3.8-Flash-Next): paged GQA K/V + a compressed+ # index-key slab (one row per index_ratio tokens, row = slot // index_ratio) ++ # a per-request pending ring for unclosed groups -> QSAKVCache+ QSA = "qsa"@propertydef backend_driven(self) -> bool:diff --git a/python/freetoken/attention/linear.py b/python/freetoken/attention/linear.pyindex 1283ba02..f3717e58 100644--- a/python/freetoken/attention/linear.py+++ b/python/freetoken/attention/linear.py@@ -41,6 +41,7 @@ class FLAMetadata:track_dst: torch.Tensor | None = None # [nt] int64 dst pool slot per tracked reqtrack_h_row: torch.Tensor | None = None # [nt] int64 row into h (boh_i + aligned//CHUNK)track_conv_src: torch.Tensor | None = None # [nt, kernel-1] int64 conv-input token positions+ track_boundary_row: torch.Tensor | None = None # [nt] int64 forward-local row of the track boundary; states with their own left context (qwen4_exp PLE) derive their windows from itdef build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:@@ -55,7 +56,7 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:builder serves the eager scheduler path and direct-op test callers."""reqs = batch.padded_reqs- pin = {"device": "cpu", "pin_memory": True}+ pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()}# GDN state slot per request: the hybrid-radix live slot (decoupled from table_idx) when# allocated, else table_idx (naive / force-naive GDN models keep the old keying).@@ -77,7 +78,7 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:fresh = [gdn_slot(r) for r in reqs if r.cached_len == 0]fresh_host = torch.tensor(fresh, dtype=torch.int64, **pin) if fresh else None- track_dst, track_h_row, track_conv_src = _build_track_metadata(reqs, cu_host, device, pin)+ track = _build_track_metadata(reqs, cu_host, device, pin)return FLAMetadata(cu_seqlens=cu_host.to(device, non_blocking=True),@@ -86,24 +87,29 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:fresh_state_indices=(fresh_host.to(device, non_blocking=True) if fresh_host is not None else None),- track_dst=track_dst, track_h_row=track_h_row, track_conv_src=track_conv_src,+ **track,)def _build_track_metadata(reqs, cu_host, device, pin):"""Hybrid-radix (extra_buffer): for each request that crosses a ×CHUNK boundary thisprefill forward, snapshot its GDN state at the deepest mid-chunk boundary into its current- ping-pong slot. Returns (track_dst, track_h_row, track_conv_src) device int64 tensors, or- (None, None, None) when no request tracks (non-hybrid, or all extends < CHUNK+1)."""+ ping-pong slot. Returns the ``FLAMetadata`` track kwargs, all None when no request+ tracks (non-hybrid, or all extends < CHUNK+1)."""+ empty = dict(track_dst=None, track_h_row=None, track_conv_src=None, track_boundary_row=None)if not any(r.mamba_ping_pong is not None for r in reqs):- return None, None, None+ return emptyfrom freetoken.core import get_global_ctxfrom freetoken.kernel.fla.chunk import CHUNK_SIZEfrom freetoken.kernel.fla.index import prepare_chunk_offsetskm1 = get_global_ctx().linear_state_pool.conv_states.shape[-1] # conv_kernel_dim - 1+ assert km1 <= CHUNK_SIZE, (+ f"conv history {km1} exceeds CHUNK_SIZE {CHUNK_SIZE}: the snapshot window "+ "would reach before this forward's first token"+ )boh = prepare_chunk_offsets(cu_host, CHUNK_SIZE).tolist()- dst, h_row, conv_src = [], [], []+ dst, h_row, conv_src, boundary_rows = [], [], [], []for i, r in enumerate(reqs):if r.mamba_ping_pong is None:continue@@ -117,13 +123,18 @@ def _build_track_metadata(reqs, cu_host, device, pin):dst.append(r.mamba_ping_pong[r.mamba_next_track_idx])h_row.append(boh[i] + c)conv_src.append([off + c * CHUNK_SIZE - km1 + j for j in range(km1)])+ boundary_rows.append(off + c * CHUNK_SIZE)r.mamba_last_track_seqlen = boundaryr.mamba_next_track_idx = 1 - r.mamba_next_track_idxif not dst:- return None, None, None+ return emptyto = lambda xs, **kw: torch.tensor(xs, **pin, **kw).to(device, non_blocking=True)- return (to(dst, dtype=torch.int64), to(h_row, dtype=torch.int64),- to(conv_src, dtype=torch.int64))+ return dict(+ track_dst=to(dst, dtype=torch.int64),+ track_h_row=to(h_row, dtype=torch.int64),+ track_conv_src=to(conv_src, dtype=torch.int64),+ track_boundary_row=to(boundary_rows, dtype=torch.int64),+ )__all__ = ["FLAMetadata", "build_fla_metadata"]diff --git a/python/freetoken/attention/qsa_sparse.py b/python/freetoken/attention/qsa_sparse.pynew file mode 100644index 00000000..4a28dc85--- /dev/null+++ b/python/freetoken/attention/qsa_sparse.py@@ -0,0 +1,507 @@+"""Qwen3.8-Flash-Next QSA compressed-block sparse attention backend.++Serves ``AttnType.QSA`` over ``kvcache/qsa_pool.py``: paged GQA K/V for the 12 full-attention+layers, a compressed index-key slab holding one key per ``index_ratio`` tokens, and a+per-request pending ring for the group a forward leaves open. The 36 GDN layers never reach+this backend, and the model has no dense attention layer, so :meth:`forward` is not served --+the only entry point is :meth:`qsa_forward` (``models/qwen4_exp/attention.py``).++One QSA layer's forward, all ragged over ``[T, ...]`` metadata:++1. store K/V at ``batch.out_loc``;+2. pool each row's closing group (members at positions >= ``cached_len`` come from this+ forward's raw index keys, the older ones from the pending ring), zero-centered rmsnorm it+ and rope it at the group's first position, then scatter it into the slab row+ ``out_loc // index_ratio`` (rows whose group does not close land on the request's scratch+ row and are never read);+3. store this forward's last ``ring_capacity`` raw index keys per request into the ring;+4. norm+rope the indexer queries at their own positions;+5. score every COMPLETE visible block (``sum_h relu(<q_h, k_bar_b>) / sqrt(index_head_dim)``,+ clamped to ``kvlen // index_ratio`` -- slab rows are never cleared, so stale rows must stay+ unreachable), take the top ``index_budget // index_ratio`` blocks, expand them to token+ indices plus the causal tail of the open group;+6. attend to exactly those tokens.++Addressing: the engine pins ``page_size == 64`` (this backend's ``page_sizes``), so a group of+``index_ratio`` tokens never straddles a page and ``block_table[req, p] = page_table[req, p *+64] // 64`` names both the K/V page and, viewed as ``page_size // index_ratio`` compressed+rows, the block's slab page. Decode stages that table plus the live lengths and table_idx into+static buffers (``prepare_for_replay``) so the whole path is CUDA-graph capturable.+"""++from __future__ import annotations++import os+from dataclasses import dataclass+from typing import TYPE_CHECKING, Callable, List++import torch+from freetoken.core import Batch, get_global_ctx+from freetoken.utils import init_logger++from .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadata++logger = init_logger(__name__)++if TYPE_CHECKING:+ from freetoken.models import ModelConfig++_CPU_PINNED = {"device": "cpu", "dtype": torch.int32, "pin_memory": True}+# Block-score transient budget (vLLM's number): the fp32 [rows, n_blocks] logits tile is+# 256 KB per row at a 1M-token context, so a long prefill must be scored in row chunks.+_LOGITS_WORKSPACE_BYTES = 128 << 20+++TORCH_TOPK_ENV = "FREETOKEN_QSA_TORCH_TOPK"+++def _resolve_block_topk() -> Callable | None:+ """The in-repo Triton block top-k, or None to fall back on torch.topk."""+ if os.getenv(TORCH_TOPK_ENV, "0") == "1":+ logger.info(f"qsa_sparse block top-k: torch.topk ({TORCH_TOPK_ENV}=1)")+ return None+ try:+ from freetoken.kernel.triton.qsa import qsa_block_topk+ except Exception as exc:+ logger.info(f"qsa_sparse block top-k: torch.topk (triton unavailable: {exc})")+ return None+ logger.info("qsa_sparse block top-k: triton qsa_block_topk")+ return qsa_block_topk+++@dataclass+class QSASparseMetadata(BaseAttnMetadata):+ # fmt: off+ is_decode: bool+ last_indices: torch.Tensor # gpu+ qo_indptr_cpu: torch.Tensor # cpu pinned int32 [bs+1]+ kv_len_cpu: torch.Tensor # cpu pinned int32 [bs]+ # Ragged per-token / per-request addressing. Decode defers these to the static graph+ # buffers (prepare_for_replay) or to a lazy eager snapshot at the first QSA layer.+ token_to_req: torch.Tensor | None = None # [T] int32+ cu_seqlens: torch.Tensor | None = None # [bs+1] int32+ seq_lens: torch.Tensor | None = None # [bs] int32, device_len+ ring_slots: torch.Tensor | None = None # [bs] int32, Req.table_idx+ block_table: torch.Tensor | None = None # [bs, W//page_size] int32, physical page ids+ # Per-forward scatter plans, built once by the first QSA layer and reused by the rest.+ # positions is bound here (not in prepare_metadata) because a capture batch has none yet.+ cmp_rows: torch.Tensor | None = None # [T] int32, compressed slab destination+ ring_rows: torch.Tensor | None = None # [T] int32, flat ring row or -1+ positions: torch.Tensor | None = None # [T] int32, logical query positions+ # fmt: on++ def get_last_indices(self, bs: int) -> torch.Tensor:+ return self.last_indices[:bs]+++class QSASparseAttnBackend(BaseAttnBackend):+ def __init__(self, config: ModelConfig) -> None:+ from freetoken.kvcache.qsa_pool import QSAKVCache++ args = config.qwen4_args+ assert args is not None, "qsa_sparse backend needs ModelConfig.qwen4_args"+ self.head_dim = config.head_dim+ self.index_heads = args.index_n_heads+ self.token_topk = args.index_budget+ self.kvcache = get_global_ctx().kv_cache+ assert isinstance(self.kvcache, QSAKVCache), (+ f"qsa_sparse backend needs a QSA pool, got {type(self.kvcache).__name__}"+ )+ self.device = self.kvcache.device+ self.dtype = self.kvcache.dtype+ self.index_head_dim = self.kvcache.index_head_dim+ self.ratio = self.kvcache.index_ratio+ self.ring_capacity = self.kvcache.ring_capacity+ self.page_size = get_global_ctx().page_size+ assert self.page_size % self.ratio == 0, (+ f"QSA needs page_size ({self.page_size}) divisible by index_ratio ({self.ratio})"+ )+ self.cmp_page_size = self.page_size // self.ratio+ self.block_topk = self.token_topk // self.ratio+ self.select_width = self.token_topk + self.ratio - 1+ assert self.token_topk % self.ratio == 0, "QSA budget must be a whole number of blocks"+ # The sparse attend kernel bakes 1/sqrt(head_dim) into its exp2 scale.+ assert config.attn_sm_scale in (None, self.head_dim**-0.5), (+ "qsa_sparse serves the default 1/sqrt(head_dim) attention scale only"+ )+ # QSA layer -> index slab slot, in sparse-layer order (the pool's own convention).+ group = self._qsa_group(config)+ self._idx_slot = {lid: i for i, lid in enumerate(group.layer_ids)}+ self.rotary_config = group.rotary_config+ self._index_cos_sin: torch.Tensor | None = None++ self._block_topk_kernel = _resolve_block_topk()+ # decode staging (static buffers under CUDA graphs; eager decode snapshots per step)+ self._graph: dict[str, torch.Tensor] = {}+ self.capture_bs: List[int] = []++ @staticmethod+ def _qsa_group(config: ModelConfig):+ from freetoken.models.config import FullAttentionGroupConfig++ groups = [+ g+ for g in config.attention_groups+ if isinstance(g, FullAttentionGroupConfig) and g.index_ratio > 1+ ]+ assert len(groups) == 1, f"expected one QSA attention group, got {len(groups)}"+ return groups[0]++ # ----- slab views ---------------------------------------------------------------------+ def _cmp_pages(self, slot: int) -> torch.Tensor:+ """The compressed slab as ``[pages, page_size // ratio, 1, dim]``, the score kernel's+ paged layout. The scratch rows past ``cmp_scratch_base`` stay out of the view."""+ rows = self.kvcache.cmp_k_cache(slot)[: self.kvcache.cmp_scratch_base]+ return rows.view(-1, self.cmp_page_size, 1, self.index_head_dim)++ def _index_rope_cache(self) -> torch.Tensor:+ """cos/sin table of the indexer rope: same rotary_dim and frequencies as the main+ attention, ``head_size`` 128 instead of 256, so it is a separate get_rope instance.++ The table itself (not RotaryEmbedding.forward) because the indexer's norm+rope is one+ fused kernel and the compressed keys rope at their group's position, not the query's."""+ if self._index_cos_sin is None:+ from freetoken.layers.rotary import get_rope++ rotary = self.rotary_config+ with torch.device(self.device):+ rope = get_rope(+ head_dim=self.index_head_dim,+ rotary_dim=rotary.rotary_dim,+ max_position=rotary.max_position,+ base=rotary.base,+ rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,+ )+ self._index_cos_sin = rope._cos_sin_cache.to(self.device)+ return self._index_cos_sin++ # ----- metadata -----------------------------------------------------------------------+ def prepare_metadata(self, batch: Batch) -> None:+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs+ seqlens_q = [r.extend_len for r in reqs]+ seqlens_k = [r.device_len for r in reqs]+ is_decode = getattr(batch, "phase", None) == "decode"+ qo_indptr = torch.tensor([0] + seqlens_q, **_CPU_PINNED).cumsum_(0).to(torch.int32)+ kv_len = torch.tensor(seqlens_k, **_CPU_PINNED)+ last = (qo_indptr[1:].to(torch.int32) - 1).to(self.device, non_blocking=True)+ md = QSASparseMetadata(+ is_decode=is_decode,+ last_indices=last,+ qo_indptr_cpu=qo_indptr,+ kv_len_cpu=kv_len,+ )+ batch.attn_metadata = md+ if not is_decode:+ table_idx = torch.tensor([r.table_idx for r in reqs], **_CPU_PINNED)+ token_to_req = torch.repeat_interleave(+ torch.arange(len(reqs), dtype=torch.int32),+ torch.tensor(seqlens_q, dtype=torch.int32),+ ).pin_memory()+ md.cu_seqlens = qo_indptr.to(self.device, non_blocking=True)+ md.token_to_req = token_to_req.to(self.device, non_blocking=True)+ md.seq_lens = kv_len.to(self.device, non_blocking=True)+ md.ring_slots = table_idx.to(self.device, non_blocking=True)+ md.block_table = self._block_table(md.ring_slots.to(torch.int64))+ # Decode addressing is DEFERRED: a graph-bound step stages it into the static+ # buffers (prepare_for_replay), an eager step snapshots at the first QSA layer.++ def _block_base_view(self) -> torch.Tensor:+ """Every-``page_size``-th column of the page table: the per-page base slots. A strided+ VIEW, so gathering rows through it materializes only [bs, W/page_size]."""+ return get_global_ctx().page_table[:, :: self.page_size]++ def _block_table(self, table_idx: torch.Tensor) -> torch.Tensor:+ return (self._block_base_view().index_select(0, table_idx) // self.page_size).to(+ torch.int32+ )++ def _stage_decode(self, md: QSASparseMetadata, bs: int, table_idx: torch.Tensor) -> None:+ """Copy this step's addressing into the static graph buffers and point the metadata+ at them (restage-per-replay, m3/dsa precedent)."""+ self._graph["block_table"][:bs].copy_(+ self._block_base_view().index_select(0, table_idx) // self.page_size+ )+ self._graph["kvlen"][:bs].copy_(md.kv_len_cpu.to(self.device, non_blocking=True))+ self._graph["table_idx"][:bs].copy_(table_idx)+ md.block_table = self._graph["block_table"][:bs]+ md.seq_lens = self._graph["kvlen"][:bs]+ md.ring_slots = self._graph["table_idx"][:bs]+ md.token_to_req = self._graph["token_to_req"][:bs]+ md.cu_seqlens = self._graph["cu_seqlens"][: bs + 1]++ def _snapshot_decode(self, md: QSASparseMetadata, batch: Batch) -> None:+ """Eager decode (not graph-staged): this step's rows, once per forward. The live+ page-table row may mutate for the next batch while this one runs, so gather now."""+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs+ bs = len(reqs)+ table_idx = torch.tensor([r.table_idx for r in reqs], **_CPU_PINNED)+ md.ring_slots = table_idx.to(self.device, non_blocking=True)+ md.block_table = self._block_table(md.ring_slots.to(torch.int64))+ md.seq_lens = md.kv_len_cpu.to(self.device, non_blocking=True)+ md.token_to_req = torch.arange(bs, dtype=torch.int32, device=self.device)+ md.cu_seqlens = torch.arange(bs + 1, dtype=torch.int32, device=self.device)++ # ----- dense layers -------------------------------------------------------------------+ def forward(+ self,+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ layer_id: int,+ batch: Batch,+ attn_spec: AttentionSpec | None = None,+ ) -> torch.Tensor:+ raise NotImplementedError(+ "qsa_sparse serves QSA layers only (Qwen3.8-Flash-Next has no dense attention "+ "layer); the QSA layer calls qsa_forward"+ )++ # ----- QSA layers ---------------------------------------------------------------------+ def qsa_forward(+ self,+ q: torch.Tensor, # [T, HQ, D]+ k: torch.Tensor, # [T, KVH * D]+ v: torch.Tensor, # [T, KVH * D]+ index, # models.qwen4_exp.attention.QSAIndexerInputs+ layer_id: int,+ batch: Batch,+ ) -> torch.Tensor:+ from freetoken.kernel.triton.qsa import qsa_sparse_paged_attention++ md = batch.attn_metadata+ assert isinstance(md, QSASparseMetadata)+ slot = self._idx_slot[layer_id]+ self.kvcache.store_kv(k, v, batch.out_loc, layer_id)+ if md.block_table is None:+ self._snapshot_decode(md, batch)+ if slot == 0 or md.cmp_rows is None:+ # Rebuilt at the first QSA layer of every forward, not cached on the metadata: a+ # capture batch runs its warmup and its capture through ONE metadata object, and a+ # cached plan would bake the warmup's addresses into the graph.+ self._plan_index_writes(md, batch)++ self._update_index_cache(index, md, slot)+ indices = self._select(index, md, slot)+ return qsa_sparse_paged_attention(+ q,+ self.kvcache.k_cache(layer_id),+ self.kvcache.v_cache(layer_id),+ indices,+ md.block_table,+ md.token_to_req,+ torch.empty_like(q),+ )++ def _plan_index_writes(self, md: QSASparseMetadata, batch: Batch) -> None:+ """Per-token slab row and ring row for this forward; the other QSA layers reuse it+ (it is layer-invariant). Pure device arithmetic: no host sync, graph-capturable."""+ md.positions = batch.positions+ out_loc = batch.out_loc.to(torch.int64)+ positions = batch.positions.to(torch.int64)+ rows = torch.arange(out_loc.numel(), device=self.device)+ req = md.token_to_req.to(torch.int64)+ slots = md.ring_slots.to(torch.int64).index_select(0, req)+ # out_loc % page_size == position % page_size and index_ratio divides page_size, so a+ # group closes exactly on out_loc % index_ratio == index_ratio - 1.+ closing = out_loc % self.ratio == self.ratio - 1+ scratch = self.kvcache.cmp_scratch_base + slots+ md.cmp_rows = torch.where(closing, out_loc // self.ratio, scratch).to(torch.int32)+ # Only the last ring_capacity rows of a request survive to the next forward; the rest+ # are masked off instead of dumped somewhere (vLLM rule).+ ends = md.cu_seqlens.to(torch.int64).index_select(0, req + 1)+ keep = rows >= ends - self.ring_capacity+ ring_row = slots * self.ring_capacity + positions % self.ring_capacity+ md.ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to(+ torch.int32+ )++ def _update_index_cache(self, index, md: QSASparseMetadata, slot: int) -> None:+ """Compress each closing group into the slab, then refresh the pending ring."""+ from freetoken.kernel.triton.qsa import (+ qsa_compress_groups,+ qsa_index_norm_rope,+ qsa_store_rows,+ )++ rows = index.k.shape[0]+ ring = self.kvcache.pending_ring(slot)+ pooled = self._scratch("pooled", rows, self.index_head_dim, dtype=self.dtype)+ first = self._scratch("first_pos", rows, dtype=torch.int32)+ qsa_compress_groups(+ index.k,+ ring,+ md.ring_slots,+ md.token_to_req,+ md.cu_seqlens,+ md.positions,+ self.ratio,+ pooled,+ first,+ )+ qsa_index_norm_rope(+ pooled,+ first,+ self._index_rope_cache(),+ index.k_norm_weight,+ index.eps,+ self.kvcache.cmp_k_cache(slot),+ dest_rows=md.cmp_rows,+ )+ # After the compression read: the ring rows this forward overwrites are exactly the+ # ones a straddling group just consumed.+ qsa_store_rows(ring, md.ring_rows, index.k)++ def _select(self, index, md: QSASparseMetadata, slot: int) -> torch.Tensor:+ """Score complete visible blocks, take the top-k, expand them to token indices."""+ from freetoken.kernel.triton.qsa import (+ expand_qsa_block_indices,+ qsa_index_norm_rope,+ qsa_mqa_paged,+ )++ rows = index.q.shape[0]+ positions = md.positions+ q_index = self._scratch(+ "q_index", rows, self.index_heads, self.index_head_dim, dtype=self.dtype+ )+ qsa_index_norm_rope(+ index.q.view(-1, self.index_head_dim),+ positions,+ self._index_rope_cache(),+ index.q_norm_weight,+ index.eps,+ q_index.view(-1, self.index_head_dim),+ heads=self.index_heads,+ )+ cmp_pages = self._cmp_pages(slot)+ columns = md.block_table.shape[1] * self.cmp_page_size+ indices = self._scratch("indices", rows, self.select_width, dtype=torch.int32)+ rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1))+ for start in range(0, rows, rows_per_chunk):+ end = min(start + rows_per_chunk, rows)+ chunk = slice(start, end)+ logits = self._scratch("logits", end - start, columns, dtype=torch.float32)+ visible = self._scratch("visible", end - start, dtype=torch.int32)+ qsa_mqa_paged(+ q_index[chunk],+ cmp_pages,+ md.block_table,+ md.token_to_req[chunk],+ positions[chunk],+ md.seq_lens,+ self.ratio,+ logits,+ visible,+ )+ blocks = self._scratch("blocks", end - start, self.block_topk, dtype=torch.int32)+ self._top_blocks(logits, visible, blocks)+ expand_qsa_block_indices(+ blocks,+ positions[chunk],+ md.seq_lens,+ md.token_to_req[chunk],+ self.ratio,+ self.token_topk,+ indices[chunk],+ )+ return indices++ def _top_blocks(+ self,+ logits: torch.Tensor,+ visible: torch.Tensor,+ blocks: torch.Tensor,+ ) -> None:+ """Top ``block_topk`` complete blocks per row, row-relative, -1 padded."""+ assert blocks.shape == (logits.shape[0], self.block_topk), (+ f"qsa block top-k output must be [rows, {self.block_topk}], got {tuple(blocks.shape)}"+ )+ if self._block_topk_kernel is not None:+ scratch_width = self._topk_scratch_width(logits.shape[1])+ scratch = (+ self._scratch("topk_scratch", logits.shape[0], scratch_width, dtype=torch.int32)+ if scratch_width+ else None+ )+ self._block_topk_kernel(logits, visible, blocks, scratch)+ return+ # The score kernel only writes columns below visible_blocks; mask the rest so a+ # stale row cannot win a slot. Real block scores are relu sums, never -inf.+ columns = logits.shape[1]+ column = torch.arange(columns, dtype=torch.int32, device=logits.device)+ logits.masked_fill_(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf"))+ width = min(self.block_topk, columns)+ values, chosen = torch.topk(logits, width, dim=-1)+ blocks[:, :width] = torch.where(values > -float("inf"), chosen.to(torch.int32), -1)+ if width < self.block_topk:+ blocks[:, width:] = -1++ def _topk_scratch_width(self, columns: int) -> int:+ """int32 columns per row the block top-k wants as scratch, 0 when it wants none."""+ if self._block_topk_kernel is None:+ return 0+ from freetoken.kernel.triton.qsa import qsa_block_topk_scratch_width++ return qsa_block_topk_scratch_width(columns, self.block_topk)++ # ----- scratch ------------------------------------------------------------------------+ def _scratch(self, name: str, rows: int, *shape: int, dtype: torch.dtype) -> torch.Tensor:+ """A per-forward transient: the static decode buffer when it is wide enough (so a+ captured graph keeps one address), otherwise a fresh allocation."""+ buffer = self._graph.get(name)+ if buffer is not None and rows <= buffer.shape[0] and buffer.shape[1:] == shape:+ return buffer[:rows]+ return torch.empty((rows, *shape), dtype=dtype, device=self.device)++ # ----- CUDA graph (decode) --------------------------------------------------------------+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ self.capture_bs = sorted(bs_list)+ max_bs = max(bs_list)+ width = get_global_ctx().page_table.shape[1]+ pages = -(-width // self.page_size)+ columns = pages * self.cmp_page_size+ chunk = max(1, min(max_bs, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1)))+ topk_scratch = self._topk_scratch_width(columns)++ def empty(*shape: int, dtype: torch.dtype) -> torch.Tensor:+ return torch.empty(shape, dtype=dtype, device=self.device)++ self._graph = {+ "block_table": torch.zeros((max_bs, pages), dtype=torch.int32, device=self.device),+ "kvlen": torch.zeros(max_bs, dtype=torch.int32, device=self.device),+ "table_idx": torch.zeros(max_bs, dtype=torch.int32, device=self.device),+ "token_to_req": torch.arange(max_bs, dtype=torch.int32, device=self.device),+ "cu_seqlens": torch.arange(max_bs + 1, dtype=torch.int32, device=self.device),+ "logits": empty(chunk, columns, dtype=torch.float32),+ "visible": empty(max_bs, dtype=torch.int32),+ "blocks": empty(max_bs, self.block_topk, dtype=torch.int32),+ "indices": empty(max_bs, self.select_width, dtype=torch.int32),+ "pooled": empty(max_bs, self.index_head_dim, dtype=self.dtype),+ "first_pos": empty(max_bs, dtype=torch.int32),+ "q_index": empty(max_bs, self.index_heads, self.index_head_dim, dtype=self.dtype),+ }+ if topk_scratch:+ self._graph["topk_scratch"] = empty(chunk, topk_scratch, dtype=torch.int32)++ def prepare_for_capture(self, batch: Batch) -> None:+ self.prepare_metadata(batch)+ md = batch.attn_metadata+ assert isinstance(md, QSASparseMetadata)+ bs = batch.size+ dummy = torch.full(+ (bs,), batch.padded_reqs[0].table_idx, dtype=torch.int64, device=self.device+ )+ self._stage_decode(md, bs, dummy)++ def prepare_for_replay(self, batch: Batch) -> None:+ md = batch.attn_metadata+ assert isinstance(md, QSASparseMetadata)+ assert batch.active_table_idx is not None, "decode batch is missing its page-table rows"+ self._stage_decode(md, batch.padded_size, batch.active_table_idx.to(torch.int64))++ def reset_capture(self) -> None:+ super().reset_capture()+ self._graph = {}+++__all__ = ["QSASparseAttnBackend", "QSASparseMetadata"]diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex cd6505d2..73dc7688 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -114,9 +114,7 @@ def _backend_requirements_met(name: str) -> bool:return True-def _resolve_auto_attention_backend(- required: frozenset[AttnType], hybrid_linear: bool-) -> str:+def _resolve_auto_attention_backend(required: frozenset[AttnType]) -> str:"""First candidate (in per-type priority order) whose arch condition holds,whose packages are installed, and whose every comma part serves ALL requiredtypes. Reproduces the historical hardware tree for FULL-only models:@@ -128,6 +126,8 @@ def _resolve_auto_attention_backend(candidates.append(("dsa", True))if AttnType.BSA in required:candidates.append(("m3_sparse", True))+ if AttnType.QSA in required:+ candidates.append(("qsa_sparse", True))if AttnType.SWA in required:candidates.append(("triton", True))if AttnType.FULL in required:@@ -142,10 +142,6 @@ def _resolve_auto_attention_backend(continueif not _backend_parts_serve(name, required):continue- if hybrid_linear and not all(- attention_backend_info(p).hybrid_linear_ok for p in name.split(",")- ):- continueif not _backend_requirements_met(name):continuereturn name@@ -176,7 +172,10 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Attif missing:valid = [name- for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse")+ for name in (+ "fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse",+ "qsa_sparse",+ )if required <= attention_backend_info(name).supported_types]missing_names = "/".join(sorted(t.value for t in missing))@@ -185,11 +184,6 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Attf"attention, which backend {part!r} does not support; valid backends: "f"{', '.join(valid)} (or auto), got {config.attention_backend!r}.")- if getattr(model_config, "has_linear_attention", False) and not info.hybrid_linear_ok:- raise ValueError(- f"backend {part!r} does not support hybrid-linear (GDN/mamba) models, "- f"got {config.attention_backend!r}."- )if AttnType.SWA in required and not info.consumes_attn_spec:# SWA models drive window/sinks/sm_scale through the per-call AttentionSpec;# a backend that drops it would attend with the wrong window silently.@@ -333,6 +327,12 @@ class Engine:self._post_weights_free = post_weights_freeself.moe_offload_cache = Noneself.cpu_moe_executor = None+ # Host-side auxiliary stores (qwen4_exp's pinned PLE table): after the weights so a+ # load failure is not masked, before the MoE offload cache so the bank residency+ # planning sees the pin quota the table already spent.+ self._host_tables_bytes = 0+ if hasattr(self.model, "load_host_tables"):+ self._host_tables_bytes = int(self.model.load_host_tables(config) or 0)if is_offload_moe_backend(config.moe_backend):self._init_offload_moe_cache(config)if hasattr(self.model, "prepare_for_runtime"):@@ -361,6 +361,7 @@ class Engine:dtype=self.dtype,device=self.device,tp_size=config.tp_info.size,+ slot_states=config.model_config.slot_states,)self.ctx.linear_state_pool = self.linear_state_poolelse:@@ -518,9 +519,11 @@ class Engine:not cpu_layer_idsand config.moe_cpu_layers is Noneand config.moe_backend in ("offload", "hybrid")- and _pin_budget_bytes() is not None+ and _pin_budget_bytes(self._host_tables_bytes) is not None):- cpu_layer_ids = _auto_cpu_layers(config, config.model_config.num_moe_layers)+ cpu_layer_ids = _auto_cpu_layers(+ config, config.model_config.num_moe_layers, reserved=self._host_tables_bytes+ )if config.moe_backend == "hybrid":decode_target = "hybrid"elif cpu_layer_ids:@@ -533,13 +536,13 @@ class Engine:split_residency = (bool(cpu_layer_ids)and config.moe_backend in ("offload", "hybrid")- and _pin_budget_bytes() is not None+ and _pin_budget_bytes(self._host_tables_bytes) is not None)if config.moe_backend == "cpu" and not split_residency:# cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything insteadfrom freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes- budget = _pin_budget_bytes()+ budget = _pin_budget_bytes(self._host_tables_bytes)bank_bytes = Noneif budget is not None:bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)@@ -1158,18 +1161,20 @@ def _cpu_moe_executor_viable(model_config) -> bool:return fmt == "mxfp4" or fmt in _WFMT_IDS-def _pin_budget_bytes() -> int | None:- """Bytes this process can safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).+def _pin_budget_bytes(reserved: int = 0) -> int | None:+ """Bytes this process can still safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).- WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere."""+ WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere. ``reserved`` subtracts host bytes already pinned outside the expert banks (qwen4_exp's PLE table)."""if env := os.environ.get("FREETOKEN_PIN_BUDGET_GB"):- return int(float(env) * 2**30)- if not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag+ cap = int(float(env) * 2**30)+ elif not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tagreturn None- return int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)+ else:+ cap = int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)+ return max(0, cap - reserved)-def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int]:+def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int = 0) -> frozenset[int]:"""Pick CPU (locked) MoE layers automatically when the banks exceed the pin budget.Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""@@ -1178,7 +1183,7 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[intbank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)if not bank_bytes:return frozenset()- budget = _pin_budget_bytes()+ budget = _pin_budget_bytes(reserved)if budget is None or bank_bytes <= budget:return frozenset()if not _cpu_moe_executor_viable(config.model_config):@@ -1292,8 +1297,12 @@ def _adjust_config(config: EngineConfig):# comma part must serve every required type, with packages/arch available.required_attn_types = _required_attn_types(model_config)_dtype = getattr(config, "dtype", None) # duck-typed test configs omit it- if AttnType.BSA in required_attn_types and _dtype is not None and _dtype.itemsize != 2:- # Reject at config time: the BSA pool's own assert only fires after the+ if (+ required_attn_types & {AttnType.BSA, AttnType.QSA}+ and _dtype is not None+ and _dtype.itemsize != 2+ ):+ # Reject at config time: the BSA/QSA pool's own assert only fires after the# model is resident (and not at all under `python -O`).raise ValueError(f"--dtype {config.dtype}: block-sparse attention serves 16-bit "@@ -1314,7 +1323,7 @@ def _adjust_config(config: EngineConfig):if config.attention_backend == "auto":override("attention_backend",- _resolve_auto_attention_backend(required_attn_types, has_linear_attention),+ _resolve_auto_attention_backend(required_attn_types),)logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}")_validate_attention_backend_choice(config, override, required_attn_types)diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex a00154d0..c9c2fb98 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -75,13 +75,16 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int# models/loader.py stream_moe_expert_sources: gate_up [E, 2I, H], down [E, H, I], bf16return {"gate_up": 2 * I * H * 2, "down": H * I * 2}if fmt == "fp8_block":- # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block scales+ # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block+ # scales, trailing scale dim 16B-padded (same helper as the loader)+ from freetoken.moe.offload_cache import fp8_block_scale_pad+B = 128return {"gate_up": 2 * I * H,- "gate_up_scale": (2 * I // B) * (H // B) * 2,+ "gate_up_scale": (2 * I // B) * fp8_block_scale_pad(2 * I // B, H // B) * 2,"down": H * I,- "down_scale": (H // B) * (I // B) * 2,+ "down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,}if fmt == "q4_0":# gemma4/gguf.py _q4_0_expert_specs: GGML Q4_0 rows, 32 elems -> 18 bytes@@ -183,6 +186,19 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (moe_intermediate_size=512,expert_formats=_NVFP4_FORMATS,),+ AotModel(+ # QSA compressed-sparse attention (12 of 48 layers): the QSAKVCache stores K/V+ # through store_cache (2 kv heads x 256 head_dim), the compressed index-key slab+ # and the pending ring write via the vendored qsa triton kernels. Hyper-connections+ # carry the residual, so the embedding row indexing() sees is still hidden_size.+ name="RadixArk/Qwen3.8-Flash-Next-NVFP4",+ architecture="Qwen4ExpForConditionalGeneration",+ hidden_size=2560,+ kv_groups=((2, 256),),+ top_k=10,+ moe_intermediate_size=640,+ expert_formats=(*_NVFP4_FORMATS, "fp8_block"),+ ),AotModel(name="google/gemma-4-26B-A4B-it",architecture="Gemma4ForConditionalGeneration",diff --git a/python/freetoken/kernel/triton/hc.py b/python/freetoken/kernel/triton/hc.pynew file mode 100644index 00000000..45d71bd7--- /dev/null+++ b/python/freetoken/kernel/triton/hc.py@@ -0,0 +1,395 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/hc.py)+"""NVIDIA HyperConnection kernels for Qwen4Exp."""++from __future__ import annotations++import functools++import torch+import triton+import triton.language as tl++from freetoken.utils.arch import is_sm90_supported+++def _pdl_supported() -> bool:+ return is_sm90_supported()+++def _grouped_gemma_rmsnorm_kernel(+ x_ptr,+ w_ptr,+ y_ptr,+ stride_x,+ stride_y,+ DIM: tl.constexpr,+ NUM_GROUPS: tl.constexpr,+ W_SHARED: tl.constexpr,+ EPS: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS+ BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM)++ pid = tl.program_id(0)+ group_id = pid % NUM_GROUPS+ # row * stride can overflow int32 for large token counts.+ row = (pid // NUM_GROUPS).to(tl.int64)++ offs_g = tl.arange(0, BLOCK_SIZE)+ offsets = group_id * GROUP_DIM + offs_g+ mask = offs_g < GROUP_DIM+ # A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped+ # checkpoint layout.+ w_offs = offs_g if W_SHARED else offsets++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32)+ w = tl.load(w_ptr + w_offs, mask, other=0.0)++ rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS)+ # Gemma's (1 + w) affine is written this way to lower to an FMA.+ y = x * rrms+ y += y * w.to(tl.float32)++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()+ tl.store(y_ptr + row * stride_y + offsets, y, mask)+++def grouped_gemma_rmsnorm(+ x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int+) -> torch.Tensor:+ N, DIM = x.shape+ assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride"+ assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous"+ assert DIM % num_groups == 0+ group_dim = DIM // num_groups+ assert weight.numel() in (group_dim, DIM)++ y = x.new_empty(x.shape)+ _grouped_gemma_rmsnorm_kernel[(N * num_groups,)](+ x,+ weight,+ y,+ x.stride(0),+ y.stride(0),+ DIM,+ num_groups,+ W_SHARED=weight.numel() == group_dim,+ EPS=eps,+ launch_pdl=_pdl_supported(),+ )+ return y+++def _hc_silu_kernel(+ x_ptr,+ y_ptr,+ stride_x,+ stride_y,+ DIM: tl.constexpr,+ HC: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM)++ row = tl.program_id(0).to(tl.int64)+ offs = tl.arange(0, BLOCK_SIZE)+ mask = offs < DIM++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC+ y = x * tl.sigmoid(x)++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()+ tl.store(y_ptr + row * stride_y + offs, y, mask)+++def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:+ num_tokens, DIM = x.shape+ assert x.stride(1) == 1++ output = x.new_empty(x.shape)+ _hc_silu_kernel[(num_tokens,)](+ x,+ output,+ x.stride(0),+ output.stride(0),+ DIM=DIM,+ HC=hc_count,+ launch_pdl=_pdl_supported(),+ )+ return output+++def _hc_gate_mix_kernel(+ x_ptr,+ g_ptr,+ y_ptr,+ stride_x,+ stride_g,+ stride_y,+ DIM: tl.constexpr,+ HC: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ HC_DIM: tl.constexpr = DIM // HC++ row = tl.program_id(0).to(tl.int64)+ tile_id = tl.program_id(1)+ offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)+ mask = offs_inner < HC_DIM++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ # The constexpr loop is unrolled and keeps one stream live at a time.+ # Materializing [HC, BLOCK_SIZE] more than doubles latency at large M.+ acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)+ for stream in tl.static_range(HC):+ offsets = stream * HC_DIM + offs_inner+ g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0)+ x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0)+ acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32)+ acc /= HC++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()+ tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)+++def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:+ N, DIM = gate.shape+ assert x.shape == gate.shape+ assert DIM % hc_count == 0+ assert x.stride(1) == 1+ assert gate.stride(1) == 1++ HC_DIM = DIM // hc_count+ out = x.new_empty(N, HC_DIM)+ BLOCK_SIZE = 512+ _hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))](+ x,+ gate,+ out,+ x.stride(0),+ gate.stride(0),+ out.stride(0),+ DIM,+ hc_count,+ BLOCK_SIZE,+ launch_pdl=_pdl_supported(),+ )+ return out+++def _hc_combine_kernel(+ block_ptr,+ res_ptr,+ inj_ptr,+ out_ptr,+ stride_block,+ stride_res,+ stride_inj,+ stride_out,+ HC_DIM: tl.constexpr,+ HC: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ HC_PAD: tl.constexpr = triton.next_power_of_2(HC)++ row = tl.program_id(0).to(tl.int64)+ tile_id = tl.program_id(1)++ offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)+ mask_inner = offs_inner < HC_DIM+ offs_hc = tl.arange(0, HC_PAD)+ mask_hc = offs_hc < HC+ offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :]+ mask = mask_hc[:, None] & mask_inner[None, :]++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)+ block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)+ res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)++ # Keeping HC as a broadcast dimension is faster here than four separate+ # residual load/store sequences.+ inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)+ out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()+ tl.store(out_ptr + row * stride_out + offs, out, mask=mask)+++def hc_combine(+ residual: torch.Tensor,+ block_output: torch.Tensor,+ injection_logits: torch.Tensor,+ hc_count: int,+) -> torch.Tensor:+ N, DIM = residual.shape+ assert DIM % hc_count == 0+ hc_dim = DIM // hc_count+ assert block_output.shape == (N, hc_dim)+ assert injection_logits.shape == (N, hc_count)+ assert residual.stride(1) == 1+ assert block_output.stride(1) == 1+ assert injection_logits.stride(1) == 1++ out = residual.new_empty(residual.shape)+ BLOCK_SIZE = 512+ _hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))](+ block_output,+ residual,+ injection_logits,+ out,+ block_output.stride(0),+ residual.stride(0),+ injection_logits.stride(0),+ out.stride(0),+ hc_dim,+ hc_count,+ BLOCK_SIZE,+ launch_pdl=_pdl_supported(),+ )+ return out+++def _hc_combine_norm_kernel(+ block_ptr,+ res_ptr,+ inj_ptr,+ w_ptr,+ out_ptr,+ y_ptr,+ stride_block,+ stride_res,+ stride_inj,+ stride_out,+ stride_y,+ HC_DIM: tl.constexpr,+ HC: tl.constexpr,+ W_SHARED: tl.constexpr,+ EPS: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ HC_PAD: tl.constexpr = triton.next_power_of_2(HC)+ NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)+ NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)++ row = tl.program_id(0).to(tl.int64)+ stream = tl.program_id(1)+ offs_hc = tl.arange(0, HC_PAD)+ mask_hc = offs_hc < HC+ tile_ids = tl.arange(0, NUM_TILES_PAD)+ offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]+ mask_inner = offs_inner < HC_DIM+ offs = stream * HC_DIM + offs_inner+ # Shared norm weights repeat across streams; per-branch weights use the+ # same flattened HC layout as the residual.+ w_offs = offs_inner if W_SHARED else offs++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ # Start the uncached residual load first, then issue the other combine+ # loads before consuming any of them.+ res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)+ inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)+ block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)+ inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)+ inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))+ # Round the materialized combine result before normalization. This matches+ # the unfused combine -> RMSNorm boundary.+ out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)+ tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)++ out = out.to(tl.float32)+ # Keep the two-axis reduction: flattening the padded tile is ~40% slower+ # at decode sizes.+ sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)+ rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()++ # Loading the weight earlier helps decode but keeps the tile live across+ # the reduction and regresses larger batches, so defer it to the norm.+ w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)+ y = out * rrms+ y += y * w.to(tl.float32)+ tl.store(y_ptr + row * stride_y + offs, y, mask_inner)+++def hc_combine_norm(+ residual: torch.Tensor,+ block_output: torch.Tensor,+ injection_logits: torch.Tensor,+ norm_weight: torch.Tensor,+ eps: float,+ hc_count: int,+) -> tuple[torch.Tensor, torch.Tensor]:+ N, DIM = residual.shape+ assert DIM % hc_count == 0+ hc_dim = DIM // hc_count+ assert block_output.shape == (N, hc_dim)+ assert injection_logits.shape == (N, hc_count)+ assert residual.stride(1) == 1+ assert block_output.stride(1) == 1+ assert injection_logits.stride(1) == 1+ assert norm_weight.is_contiguous()+ assert norm_weight.numel() in (hc_dim, DIM)++ out = residual.new_empty(residual.shape)+ y = residual.new_empty(residual.shape)+ BLOCK_SIZE = 512+ _hc_combine_norm_kernel[(N, hc_count)](+ block_output,+ residual,+ injection_logits,+ norm_weight,+ out,+ y,+ block_output.stride(0),+ residual.stride(0),+ injection_logits.stride(0),+ out.stride(0),+ y.stride(0),+ hc_dim,+ hc_count,+ W_SHARED=norm_weight.numel() == hc_dim,+ EPS=eps,+ BLOCK_SIZE=BLOCK_SIZE,+ launch_pdl=_pdl_supported(),+ )+ return out, y+++__all__ = [+ "grouped_gemma_rmsnorm",+ "hc_combine",+ "hc_combine_norm",+ "hc_gate_mix",+ "hc_silu",+]diff --git a/python/freetoken/kernel/triton/moe_router.py b/python/freetoken/kernel/triton/moe_router.pynew file mode 100644index 00000000..54ecfe32--- /dev/null+++ b/python/freetoken/kernel/triton/moe_router.py@@ -0,0 +1,160 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the SGLang project+# Adapted from SGLang (kernels/ops/moe/moe_fused_gate.py)+"""Fused softmax top-k MoE router (bias-free, ungrouped experts).++The ``num_token_non_padded`` row mask reads the device tensor, so it survives CUDA-graph capture.+"""++from __future__ import annotations++import functools+from typing import Tuple++import torch+import triton+import triton.language as tl++from freetoken.utils.arch import is_sm90_supported+++def _pdl_supported() -> bool:+ return is_sm90_supported()+++def _router_triton_kernel(+ scores_ptr,+ out_weights_ptr,+ out_indices_ptr,+ num_token_non_padded_ptr,+ M,+ stride_sm,+ stride_sn,+ stride_wm,+ stride_wk,+ stride_im,+ stride_ik,+ N: tl.constexpr,+ K: tl.constexpr,+ BLOCK_M: tl.constexpr,+ BLOCK_N: tl.constexpr,+ BLOCK_K: tl.constexpr,+ RENORMALIZE: tl.constexpr,+ HAS_TOKEN_LIMIT: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ # Row-tiled: each program handles BLOCK_M rows; all reductions run along the+ # expert (N) axis. Tiling rows keeps CTAs large enough to stay occupancy-bound+ # rather than launch-bound at small N (many tiny 1-warp CTAs otherwise).+ pid = tl.program_id(0)+ offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)+ offs_n = tl.arange(0, BLOCK_N)+ mask_m = offs_m < M+ mask_n = offs_n < N++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ # offs_m * stride can overflow int32 for large token counts.+ row_ptr = scores_ptr + offs_m[:, None].to(tl.int64) * stride_sm + offs_n[None, :] * stride_sn+ mask2d = mask_m[:, None] & mask_n[None, :]+ logits = tl.load(row_ptr, mask=mask2d, other=0.0).to(tl.float32)++ ranked = tl.where(mask_n[None, :], logits, -float("inf"))+ row_max = tl.max(ranked, axis=1)[:, None]+ exp_row = tl.where(mask_n[None, :], tl.exp(ranked - row_max), 0.0)+ activated = exp_row / tl.sum(exp_row, axis=1)[:, None]++ # Map NaN -> a finite floor+ ranked = tl.where(ranked == ranked, ranked, -1e30)++ offs_k = tl.arange(0, BLOCK_K)+ mask_k = offs_k < K+ selected_vals = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.float32)+ selected_idx = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.int32)++ cur = ranked+ for k in tl.static_range(K):+ max_val = tl.max(cur, axis=1)[:, None]+ lane_id = tl.where(cur == max_val, offs_n[None, :], N + 1) # lowest expert id wins ties+ win_lane = tl.min(lane_id, axis=1)[:, None].to(tl.int32)+ win_activated = tl.sum(+ tl.where(offs_n[None, :] == win_lane, activated, 0.0), axis=1+ )[:, None]+ slot = offs_k[None, :] == k+ selected_vals = tl.where(slot, win_activated, selected_vals)+ selected_idx = tl.where(slot, win_lane, selected_idx)+ cur = tl.where(offs_n[None, :] == win_lane, -float("inf"), cur)++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()++ if RENORMALIZE:+ routed_sum = tl.sum(tl.where(mask_k[None, :], selected_vals, 0.0), axis=1)[:, None]+ selected_vals = selected_vals / tl.where(routed_sum > 0.0, routed_sum, 1.0)++ if HAS_TOKEN_LIMIT:+ limit = tl.load(num_token_non_padded_ptr)+ selected_idx = tl.where(offs_m[:, None] < limit, selected_idx, -1)++ out_w_ptr = out_weights_ptr + offs_m[:, None].to(tl.int64) * stride_wm + offs_k[None, :] * stride_wk+ out_i_ptr = out_indices_ptr + offs_m[:, None].to(tl.int64) * stride_im + offs_k[None, :] * stride_ik+ store_mask = mask_m[:, None] & mask_k[None, :]+ tl.store(out_w_ptr, selected_vals, mask=store_mask)+ tl.store(out_i_ptr, selected_idx, mask=store_mask)+++def fused_topk_softmax(+ gating_output: torch.Tensor,+ topk: int,+ renormalize: bool,+ num_token_non_padded: torch.Tensor | None = None,+) -> Tuple[torch.Tensor, torch.Tensor]:+ """Softmax over all experts, top-k, then renormalize; ties keep the lowest expert id.++ ``num_token_non_padded`` is a device scalar; rows at or past it get expert id -1.+ """+ assert gating_output.ndim == 2, "gating_output must be 2D"+ M, N = gating_output.shape+ weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)+ indices = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device)++ BLOCK_N = triton.next_power_of_2(N)+ BLOCK_K = triton.next_power_of_2(topk)+ # Single warp per program keeps the per-row top-k reductions on cheap warp+ # shuffles; pack a few rows per program only when N is small so tiny launches+ # stay occupancy-bound. Swept on H100/B200; larger tiles / more warps regress+ # (register pressure).+ BLOCK_M = max(1, min(4, 256 // BLOCK_N))+ # For wide rows the K sequential argmax passes dominate and benefit from more+ # warps despite the cross-warp reduction cost.+ num_warps = 1 if BLOCK_N <= 512 else 4++ _router_triton_kernel[(triton.cdiv(M, BLOCK_M),)](+ gating_output,+ weights,+ indices,+ num_token_non_padded,+ M,+ gating_output.stride(0),+ gating_output.stride(1),+ weights.stride(0),+ weights.stride(1),+ indices.stride(0),+ indices.stride(1),+ N=N,+ K=topk,+ BLOCK_M=BLOCK_M,+ BLOCK_N=BLOCK_N,+ BLOCK_K=BLOCK_K,+ RENORMALIZE=renormalize,+ HAS_TOKEN_LIMIT=num_token_non_padded is not None,+ launch_pdl=_pdl_supported(),+ num_warps=num_warps,+ )+ return weights, indices+++__all__ = ["fused_topk_softmax"]diff --git a/python/freetoken/kernel/triton/moe_shared_gate.py b/python/freetoken/kernel/triton/moe_shared_gate.pynew file mode 100644index 00000000..0615c2fa--- /dev/null+++ b/python/freetoken/kernel/triton/moe_shared_gate.py@@ -0,0 +1,137 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the SGLang project+# Adapted from SGLang (kernels/ops/elementwise.py, ``_fused_gate_sigmoid_mul_add``)+"""Gated shared-expert epilogue: the gate reduction and the sigmoid-mul-add.++The routed experts may write into ``hidden_states`` in place, so the gate reduction runs+before them and the mul-add after.+"""++from __future__ import annotations++import functools++import torch+import triton+import triton.language as tl++from freetoken.utils.arch import is_sm90_supported+++def _pdl_supported() -> bool:+ return is_sm90_supported()+++def _reduction_warps(hidden_dim: int, num_tokens: int) -> int:+ warps = max(min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), 32), 4)+ return min(warps, 8) if num_tokens >= 1024 else warps+++def _gate_sigmoid_kernel(+ hidden_ptr,+ weight_ptr,+ gate_ptr,+ stride_h,+ HIDDEN: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ # row * stride can overflow int32 for large token counts.+ row = tl.program_id(0).to(tl.int64)+ offs = tl.arange(0, BLOCK_SIZE)+ mask = offs < HIDDEN++ w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32)++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ h = tl.load(hidden_ptr + row * stride_h + offs, mask=mask, other=0.0).to(tl.float32)++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()++ tl.store(gate_ptr + row, tl.sigmoid(tl.sum(h * w, axis=0)))+++def shared_gate_sigmoid(hidden_states: torch.Tensor, gate_weight: torch.Tensor) -> torch.Tensor:+ """Per-token ``sigmoid(hidden_states @ gate_weight)`` as fp32 [num_tokens]."""+ num_tokens, hidden_dim = hidden_states.shape+ assert hidden_states.stride(1) == 1, "shared gate requires unit inner stride"+ assert gate_weight.shape == (hidden_dim,) and gate_weight.is_contiguous()++ gate = torch.empty(num_tokens, dtype=torch.float32, device=hidden_states.device)+ _gate_sigmoid_kernel[(num_tokens,)](+ hidden_states,+ gate_weight,+ gate,+ hidden_states.stride(0),+ HIDDEN=hidden_dim,+ BLOCK_SIZE=triton.next_power_of_2(hidden_dim),+ launch_pdl=_pdl_supported(),+ num_warps=_reduction_warps(hidden_dim, num_tokens),+ )+ return gate+++def _gate_mul_add_kernel(+ routed_ptr,+ shared_ptr,+ gate_ptr,+ out_ptr,+ stride_r,+ stride_s,+ stride_o,+ HIDDEN: tl.constexpr,+ BLOCK_SIZE: tl.constexpr,+ launch_pdl: tl.constexpr,+) -> None:+ row = tl.program_id(0).to(tl.int64)+ block = tl.program_id(1)+ offs = block * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)+ mask = offs < HIDDEN++ if launch_pdl:+ tl.extra.cuda.gdc_wait()++ gate = tl.load(gate_ptr + row)+ routed = tl.load(routed_ptr + row * stride_r + offs, mask=mask, other=0.0).to(tl.float32)+ shared = tl.load(shared_ptr + row * stride_s + offs, mask=mask, other=0.0).to(tl.float32)++ if launch_pdl:+ tl.extra.cuda.gdc_launch_dependents()++ tl.store(out_ptr + row * stride_o + offs, routed + gate * shared, mask=mask)+++def shared_gate_mul_add(+ routed: torch.Tensor, shared: torch.Tensor, gate: torch.Tensor+) -> torch.Tensor:+ """``routed + gate[:, None] * shared`` into a fresh tensor."""+ num_tokens, hidden_dim = routed.shape+ assert shared.shape == routed.shape+ assert routed.stride(1) == 1 and shared.stride(1) == 1+ assert gate.shape == (num_tokens,)++ out = torch.empty_like(routed)+ block_size = min(triton.next_power_of_2(hidden_dim), 2048)+ _gate_mul_add_kernel[(num_tokens, triton.cdiv(hidden_dim, block_size))](+ routed,+ shared,+ gate,+ out,+ routed.stride(0),+ shared.stride(0),+ out.stride(0),+ HIDDEN=hidden_dim,+ BLOCK_SIZE=block_size,+ launch_pdl=_pdl_supported(),+ num_warps=4,+ )+ return out+++__all__ = ["shared_gate_mul_add", "shared_gate_sigmoid"]diff --git a/python/freetoken/kernel/triton/ple.py b/python/freetoken/kernel/triton/ple.pynew file mode 100644index 00000000..2314e94e--- /dev/null+++ b/python/freetoken/kernel/triton/ple.py@@ -0,0 +1,95 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the SGLang project+# Adapted from SGLang (python/sglang/srt/models/qwen4_exp.py)+"""UVA row gather for the Qwen3.8-Flash-Next PLE n-gram table.++The table (320,001,536 rows x 160, FP8-e4m3 + one scalar scale = 47.7 GiB) stays in pinned+host memory and the GPU dereferences it in place over PCIe -- at its host VA on Linux/UVA, at+the mapped device address on WDDM (``kernel/pinned.device_ptr``). One program per requested+row: read the row, widen to fp32, apply the per-tensor scale, store bf16.++Ids outside the table store zeros.+"""++from __future__ import annotations++import torch+import triton+import triton.language as tl++from freetoken.kernel.triton.e4m3_compat import e4m3_native_cx, e4m3_u8_to_f32++# Latency-bound over PCIe, so keep the block small and let many of them be in flight.+_NUM_WARPS = 1+++def _ple_gather_kernel(+ table_ptr,+ ids_ptr,+ out_ptr,+ scale,+ num_rows,+ EMB_DIM: tl.constexpr,+ IS_FP8: tl.constexpr,+ BLOCK_D: tl.constexpr,+):+ row = tl.program_id(0)+ idx = tl.load(ids_ptr + row).to(tl.int64)+ in_range = (idx >= 0) & (idx < num_rows)+ idx = tl.where(in_range, idx, 0)+ offsets = tl.arange(0, BLOCK_D)+ mask = offsets < EMB_DIM+ # the table is a host allocation: rebuild the typed pointer from the raw address+ if IS_FP8:+ if e4m3_native_cx():+ base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.float8e4nv))+ values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)+ else:+ # pre-sm_89 has no fp8e4nv type: load raw bytes and decode in software+ base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.uint8))+ values = e4m3_u8_to_f32(tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0))+ else:+ base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.bfloat16))+ values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)+ values = tl.where(in_range, values * scale, 0.0)+ tl.store(+ out_ptr + row * EMB_DIM + offsets,+ values.to(out_ptr.dtype.element_ty),+ mask=mask,+ )+++def ple_gather_rows(+ table_ptr: int,+ num_rows: int,+ embed_dim: int,+ row_ids: torch.Tensor,+ out: torch.Tensor,+ scale: float = 1.0,+ is_fp8: bool = True,+) -> torch.Tensor:+ """Gather ``row_ids`` from the host-resident table at ``table_ptr`` into ``out``.++ ``row_ids`` is a flat device int tensor; ``out`` is ``[row_ids.numel(), embed_dim]``+ bf16 on the same device. ``table_ptr`` is the address the GPU must dereference+ (``kernel/pinned.device_ptr``), not necessarily the host ``data_ptr``.+ """+ n = row_ids.numel()+ assert out.shape == (n, embed_dim) and out.is_contiguous(), out.shape+ if n:+ _ple_gather_kernel[(n,)](+ table_ptr,+ row_ids,+ out,+ float(scale),+ num_rows,+ EMB_DIM=embed_dim,+ IS_FP8=is_fp8,+ BLOCK_D=triton.next_power_of_2(embed_dim),+ num_warps=_NUM_WARPS,+ )+ return out+++__all__ = ["ple_gather_rows"]diff --git a/python/freetoken/kernel/triton/qsa/__init__.py b/python/freetoken/kernel/triton/qsa/__init__.pynew file mode 100644index 00000000..91753b79--- /dev/null+++ b/python/freetoken/kernel/triton/qsa/__init__.py@@ -0,0 +1,18 @@+"""Triton kernels for Qwen3.8-Flash-Next QSA sparse attention."""++from .attend import qsa_sparse_paged_attention+from .compress import qsa_compress_groups, qsa_index_norm_rope, qsa_store_rows+from .expand import expand_qsa_block_indices+from .score import qsa_mqa_paged+from .topk import qsa_block_topk, qsa_block_topk_scratch_width++__all__ = [+ "expand_qsa_block_indices",+ "qsa_block_topk",+ "qsa_block_topk_scratch_width",+ "qsa_compress_groups",+ "qsa_index_norm_rope",+ "qsa_mqa_paged",+ "qsa_sparse_paged_attention",+ "qsa_store_rows",+]diff --git a/python/freetoken/kernel/triton/qsa/attend.py b/python/freetoken/kernel/triton/qsa/attend.pynew file mode 100644index 00000000..541e27c6--- /dev/null+++ b/python/freetoken/kernel/triton/qsa/attend.py@@ -0,0 +1,360 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py)+"""Sparse paged GQA over the QSA selection."""++from __future__ import annotations++import torch+import triton+import triton.language as tl+++def _qsa_sparse_paged_gqa_splitk_kernel(+ q_ptr,+ k_cache_ptr,+ v_cache_ptr,+ indices_ptr,+ block_table_ptr,+ token_to_req_ptr,+ partial_output_ptr,+ partial_lse_ptr,+ output_ptr,+ stride_q_row,+ stride_q_head,+ stride_k_block,+ stride_k_token,+ stride_k_head,+ stride_v_block,+ stride_v_token,+ stride_v_head,+ stride_indices_row,+ stride_table_req,+ stride_output_row,+ stride_output_head,+ num_rows,+ num_cache_blocks,+ num_requests,+ TOPK: tl.constexpr,+ PAGE_SIZE: tl.constexpr,+ PAGE_TABLE_WIDTH: tl.constexpr,+ GROUP_SIZE: tl.constexpr,+ HEAD_DIM: tl.constexpr,+ NUM_QUERY_HEADS: tl.constexpr,+ NUM_SPLITS: tl.constexpr,+ NUM_TILES: tl.constexpr,+ BLOCK_M: tl.constexpr,+ BLOCK_N: tl.constexpr,+) -> None:+ # row * stride can overflow int32 for large row counts.+ row = tl.program_id(0).to(tl.int64)+ kv_head = tl.program_id(1)+ split_id = tl.program_id(2)+ request = tl.load(token_to_req_ptr + row)+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)++ head_offsets = tl.arange(0, BLOCK_M)+ dim_offsets = tl.arange(0, HEAD_DIM)+ column_offsets = tl.arange(0, BLOCK_N)+ first_head = kv_head * GROUP_SIZE+ query = tl.load(+ q_ptr+ + row * stride_q_row+ + (first_head + head_offsets[:, None]) * stride_q_head+ + dim_offsets[None, :],+ mask=head_offsets[:, None] < GROUP_SIZE,+ other=0.0,+ )++ max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32)+ normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32)+ accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)+ softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634++ # Dynamic bounds avoid padded main-loop iterations for uneven splits.+ split_tile_start = split_id * NUM_TILES // NUM_SPLITS+ split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS+ for tile in range(split_tile_start, split_tile_end):+ columns = tile * BLOCK_N + column_offsets+ logical_token = tl.load(+ indices_ptr + row * stride_indices_row + columns,+ mask=columns < TOPK,+ other=-1,+ )+ safe_token = tl.maximum(logical_token, 0)+ logical_page = safe_token // PAGE_SIZE+ page_offset = safe_token % PAGE_SIZE+ valid = (+ (request >= 0)+ & (request < num_requests)+ & (logical_token >= 0)+ & (logical_page < PAGE_TABLE_WIDTH)+ )+ physical_page = tl.load(+ block_table_ptr+ + safe_request.to(tl.int64) * stride_table_req+ + tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1),+ mask=valid,+ other=-1,+ )+ valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)+ # physical_page * block stride can overflow int32 for large caches.+ safe_page = tl.maximum(physical_page, 0).to(tl.int64)+ keys = tl.load(+ k_cache_ptr+ + safe_page[None, :] * stride_k_block+ + page_offset[None, :] * stride_k_token+ + kv_head * stride_k_head+ + dim_offsets[:, None],+ mask=valid[None, :],+ other=0.0,+ )+ values = tl.load(+ v_cache_ptr+ + safe_page[:, None] * stride_v_block+ + page_offset[:, None] * stride_v_token+ + kv_head * stride_v_head+ + dim_offsets[None, :],+ mask=valid[:, None],+ other=0.0,+ )+ scores = tl.dot(query, keys)+ # Scaling scores avoids re-quantizing a scaled query to BF16.+ scores *= softmax_scale_log2+ scores = tl.where(valid[None, :], scores, -1.0e20)+ next_max = tl.maximum(max_value, tl.max(scores, axis=1))+ alpha = tl.math.exp2(max_value - next_max)+ probabilities = tl.where(+ valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0+ )+ accumulator = tl.dot(+ probabilities.to(values.dtype),+ values,+ acc=accumulator * alpha[:, None],+ )+ normalizer = normalizer * alpha + tl.sum(probabilities, axis=1)+ max_value = next_max++ has_values = normalizer > 0+ normalized_output = tl.where(+ has_values[:, None],+ accumulator / tl.maximum(normalizer[:, None], 1.0e-20),+ 0.0,+ )+ output_mask = head_offsets[:, None] < GROUP_SIZE+ if NUM_SPLITS == 1:+ tl.store(+ output_ptr+ + row * stride_output_row+ + (first_head + head_offsets[:, None]) * stride_output_head+ + dim_offsets[None, :],+ normalized_output,+ mask=output_mask,+ )+ else:+ partial_lse = tl.where(+ has_values,+ max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)),+ -float("inf"),+ )+ tl.store(+ partial_output_ptr+ + (+ (split_id.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS+ + first_head+ + head_offsets[:, None]+ )+ * HEAD_DIM+ + dim_offsets[None, :],+ normalized_output,+ mask=output_mask,+ )+ tl.store(+ partial_lse_ptr+ + (split_id.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS+ + first_head+ + head_offsets,+ partial_lse,+ mask=head_offsets < GROUP_SIZE,+ )+++def _qsa_merge_splitk_kernel(+ partial_output_ptr,+ partial_lse_ptr,+ output_ptr,+ stride_output_row,+ stride_output_head,+ num_rows,+ HEAD_DIM: tl.constexpr,+ NUM_QUERY_HEADS: tl.constexpr,+ NUM_SPLITS: tl.constexpr,+ BLOCK_SPLITS: tl.constexpr,+) -> None:+ row = tl.program_id(0).to(tl.int64)+ head = tl.program_id(1)+ split_offsets = tl.arange(0, BLOCK_SPLITS)+ dim_offsets = tl.arange(0, HEAD_DIM)+ split_mask = split_offsets < NUM_SPLITS+ lse = tl.load(+ partial_lse_ptr + (split_offsets.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + head,+ mask=split_mask,+ other=-float("inf"),+ )+ lse_max = tl.max(lse, axis=0)+ has_values = lse_max > -float("inf")+ shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf"))+ weights = tl.math.exp2(shifted)+ denominator = tl.sum(weights, axis=0)+ partial_output = tl.load(+ partial_output_ptr+ + ((split_offsets[:, None].to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + head)+ * HEAD_DIM+ + dim_offsets[None, :],+ mask=split_mask[:, None],+ other=0.0,+ )+ merged = tl.sum(partial_output * weights[:, None], axis=0)+ merged = tl.where(denominator > 0, merged / denominator, 0.0)+ tl.store(+ output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets,+ merged,+ )+++def qsa_sparse_paged_attention(+ q: torch.Tensor,+ k_cache: torch.Tensor,+ v_cache: torch.Tensor,+ logical_indices: torch.Tensor,+ block_table: torch.Tensor,+ token_to_req: torch.Tensor,+ out: torch.Tensor | None = None,+) -> torch.Tensor:+ """Run sparse GQA directly over paged BF16 K/V caches."""++ if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:+ raise ValueError("QSA sparse attention received invalid Q/K/V shapes")+ if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]:+ raise ValueError("QSA indices must have one row per query")+ if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2:+ raise ValueError("QSA sparse attention metadata has invalid shapes")+ if logical_indices.shape[1] <= 0:+ raise ValueError("QSA sparse attention requires a positive selection width")+ if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]:+ raise ValueError("QSA sparse attention requires valid grouped-query heads")+ head_dim = q.shape[2]+ assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0+ assert q.dtype == k_cache.dtype == v_cache.dtype+ assert logical_indices.dtype == block_table.dtype == torch.int32+ assert token_to_req.dtype == torch.int32+ assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1+ assert logical_indices.stride(1) == block_table.stride(1) == 1+ assert token_to_req.stride(0) == 1+ if out is None:+ out = torch.empty_like(q)+ assert out.shape == q.shape and out.dtype == q.dtype and out.stride(2) == 1+ if not q.shape[0]:+ return out++ group_size = q.shape[1] // k_cache.shape[2]+ block_m = triton.next_power_of_2(group_size)+ base_programs = q.shape[0] * k_cache.shape[2]+ small_profile_limit = 8 if block_m <= 8 else 4++ # Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes.+ # Narrow tiles favor decode; wide tiles improve throughput for prefill.+ if base_programs <= small_profile_limit:+ block_n, target_splits, partial_warps = 16, 64, 4+ elif base_programs < 32:+ block_n, target_splits, partial_warps = 16, 32, 4+ elif base_programs <= 256:+ block_n, target_splits, partial_warps = 64, 8, 2+ elif base_programs <= 512:+ block_n, target_splits, partial_warps = 64, 4, 2+ else:+ block_n, target_splits, partial_warps = 64, 1, 2++ num_tiles = triton.cdiv(logical_indices.shape[1], block_n)+ # Avoid empty splits when the selection width is smaller than the profile.+ max_useful_splits = 1 << (num_tiles.bit_length() - 1)+ num_splits = min(max_useful_splits, target_splits)++ # Split=1 writes output directly and compiles out all workspace accesses.+ if num_splits == 1:+ partial_output = out+ partial_lse = out+ else:+ # FP32 partials preserve accuracy when merging independently normalized+ # splits.+ partial_output = torch.empty(+ (num_splits, *q.shape), dtype=torch.float32, device=q.device+ )+ partial_lse = torch.empty(+ (num_splits, q.shape[0], q.shape[1]),+ dtype=torch.float32,+ device=q.device,+ )++ partial_grid = (q.shape[0], k_cache.shape[2], num_splits)+ _qsa_sparse_paged_gqa_splitk_kernel[partial_grid](+ q,+ k_cache,+ v_cache,+ logical_indices,+ block_table,+ token_to_req,+ partial_output,+ partial_lse,+ out,+ q.stride(0),+ q.stride(1),+ k_cache.stride(0),+ k_cache.stride(1),+ k_cache.stride(2),+ v_cache.stride(0),+ v_cache.stride(1),+ v_cache.stride(2),+ logical_indices.stride(0),+ block_table.stride(0),+ out.stride(0),+ out.stride(1),+ q.shape[0],+ k_cache.shape[0],+ block_table.shape[0],+ TOPK=logical_indices.shape[1],+ PAGE_SIZE=k_cache.shape[1],+ PAGE_TABLE_WIDTH=block_table.shape[1],+ GROUP_SIZE=group_size,+ HEAD_DIM=q.shape[2],+ NUM_QUERY_HEADS=q.shape[1],+ NUM_SPLITS=num_splits,+ NUM_TILES=num_tiles,+ BLOCK_M=block_m,+ BLOCK_N=block_n,+ num_warps=partial_warps,+ num_stages=2,+ )+ if num_splits == 1:+ return out++ _qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])](+ partial_output,+ partial_lse,+ out,+ out.stride(0),+ out.stride(1),+ q.shape[0],+ HEAD_DIM=q.shape[2],+ NUM_QUERY_HEADS=q.shape[1],+ NUM_SPLITS=num_splits,+ BLOCK_SPLITS=triton.next_power_of_2(num_splits),+ num_warps=2,+ num_stages=1,+ )+ return out+++__all__ = ["qsa_sparse_paged_attention"]diff --git a/python/freetoken/kernel/triton/qsa/compress.py b/python/freetoken/kernel/triton/qsa/compress.pynew file mode 100644index 00000000..1f19ed1e--- /dev/null+++ b/python/freetoken/kernel/triton/qsa/compress.py@@ -0,0 +1,331 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py and ops/qsa_pre_indexer.py)+"""QSA index-key compression, indexer norm+rope, and fixed-width row stores.++The pending ring is one row per (request slot, ring position) keyed by ``Req.table_idx``,+the caches are row-flat, and the fused (1+w) RMSNorm + partial NeoX rope takes the rotary+width as a parameter.+"""++from __future__ import annotations++import torch+import triton+import triton.language as tl+++def _compress_qsa_groups_kernel(+ raw_keys_ptr,+ ring_ptr,+ ring_slots_ptr,+ token_to_req_ptr,+ query_start_loc_ptr,+ logical_positions_ptr,+ pooled_ptr,+ first_positions_ptr,+ stride_raw_row,+ stride_ring_slot,+ stride_ring_row,+ stride_pooled_row,+ num_rows,+ num_ring_slots,+ num_requests,+ RING_CAPACITY: tl.constexpr,+ COMPRESS_RATIO: tl.constexpr,+ HEAD_DIM: tl.constexpr,+ BLOCK_D: tl.constexpr,+) -> None:+ row = tl.program_id(0)+ dims = tl.arange(0, BLOCK_D)+ request = tl.load(token_to_req_ptr + row)+ end_position = tl.load(logical_positions_ptr + row)+ valid_request = (request >= 0) & (request < num_requests)+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)+ query_row_start = tl.load(+ query_start_loc_ptr + safe_request, mask=valid_request, other=0+ )+ query_row_end = tl.load(+ query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0+ )+ chunk_start_position = end_position - (row - query_row_start)+ ring_slot = tl.load(ring_slots_ptr + safe_request, mask=valid_request, other=-1)+ valid_ring_slot = (ring_slot >= 0) & (ring_slot < num_ring_slots)+ valid_row = (+ (row < num_rows)+ & valid_request+ & (row >= query_row_start)+ & (row < query_row_end)+ & (end_position >= COMPRESS_RATIO - 1)+ )+ accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32)++ # A group can span the pending ring (older members) and this step's raw rows+ # (members at positions >= chunk_start_position).+ for group_offset in tl.range(0, COMPRESS_RATIO):+ position = end_position - (COMPRESS_RATIO - 1 - group_offset)+ use_raw = position >= chunk_start_position+ raw_row = query_row_start + position - chunk_start_position+ raw_values = tl.load(+ raw_keys_ptr + raw_row * stride_raw_row + dims,+ mask=valid_row+ & use_raw+ & (raw_row >= query_row_start)+ & (raw_row < query_row_end)+ & (raw_row < num_rows)+ & (dims < HEAD_DIM),+ other=0.0,+ ).to(tl.float32)+ ring_values = tl.load(+ ring_ptr+ + tl.maximum(ring_slot, 0).to(tl.int64) * stride_ring_slot+ + (position % RING_CAPACITY) * stride_ring_row+ + dims,+ mask=valid_row+ & ~use_raw+ & valid_ring_slot+ & (dims < HEAD_DIM),+ other=0.0,+ ).to(tl.float32)+ accumulator += tl.where(use_raw, raw_values, ring_values)++ tl.store(+ pooled_ptr + row * stride_pooled_row + dims,+ accumulator / COMPRESS_RATIO,+ mask=(row < num_rows) & (dims < HEAD_DIM),+ )+ first_position = end_position - COMPRESS_RATIO + 1+ tl.store(+ first_positions_ptr + row,+ tl.where(valid_row, first_position, 0),+ mask=row < num_rows,+ )+++def _index_norm_rope_kernel(+ x_ptr,+ positions_ptr,+ cos_sin_ptr,+ weight_ptr,+ out_ptr,+ dest_rows_ptr,+ stride_x_row,+ stride_out_row,+ stride_cos_sin_row,+ num_rows,+ eps,+ HEADS: tl.constexpr,+ HEAD_DIM: tl.constexpr,+ ROTARY_HALF: tl.constexpr,+ BLOCK_R: tl.constexpr,+ BLOCK_D: tl.constexpr,+ HAS_DEST_ROWS: tl.constexpr,+) -> None:+ rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)+ live = rows < num_rows+ dims = tl.arange(0, BLOCK_D)+ in_dim = dims < HEAD_DIM+ in_rotary = dims < 2 * ROTARY_HALF+ # NeoX pairs dim d with d + rotary_dim/2; both halves are read so the rotation needs+ # no cross-lane shuffle.+ pair = dims % ROTARY_HALF+ partner = tl.where(dims < ROTARY_HALF, dims + ROTARY_HALF, dims - ROTARY_HALF)+ partner = tl.where(in_rotary, partner, dims)++ base = x_ptr + rows[:, None].to(tl.int64) * stride_x_row+ mask = live[:, None] & in_dim[None, :]+ x = tl.load(base + dims[None, :], mask=mask, other=0.0).to(tl.float32)+ x_partner = tl.load(base + partner[None, :], mask=mask, other=0.0).to(tl.float32)+ weight = tl.load(weight_ptr + dims, mask=in_dim, other=0.0).to(tl.float32) + 1.0+ weight_partner = (+ tl.load(weight_ptr + partner, mask=in_dim, other=0.0).to(tl.float32) + 1.0+ )+ rrms = tl.rsqrt(tl.sum(x * x, axis=1) / HEAD_DIM + eps)+ y = x * rrms[:, None] * weight[None, :]+ y_partner = x_partner * rrms[:, None] * weight_partner[None, :]++ position = tl.load(positions_ptr + rows // HEADS, mask=live, other=0).to(tl.int64)+ cos_base = cos_sin_ptr + position[:, None] * stride_cos_sin_row+ rotary_mask = live[:, None] & in_rotary[None, :]+ cos = tl.load(cos_base + pair[None, :], mask=rotary_mask, other=1.0)+ sin = tl.load(cos_base + ROTARY_HALF + pair[None, :], mask=rotary_mask, other=0.0)+ sign = tl.where(dims < ROTARY_HALF, -1.0, 1.0)+ result = tl.where(in_rotary[None, :], y * cos + sign[None, :] * y_partner * sin, y)++ if HAS_DEST_ROWS:+ dest = tl.load(dest_rows_ptr + rows, mask=live, other=-1)+ live = live & (dest >= 0)+ dest_row = tl.maximum(dest, 0).to(tl.int64)+ else:+ dest_row = rows.to(tl.int64)+ tl.store(+ out_ptr + dest_row[:, None] * stride_out_row + dims[None, :],+ result.to(out_ptr.dtype.element_ty),+ mask=live[:, None] & in_dim[None, :],+ )+++def _store_qsa_rows_kernel(+ cache_ptr,+ slots_ptr,+ rows_ptr,+ stride_cache_block,+ stride_cache_token,+ stride_rows_row,+ num_rows,+ num_blocks,+ PAGE_SIZE: tl.constexpr,+ WIDTH: tl.constexpr,+ BLOCK_D: tl.constexpr,+) -> None:+ row = tl.program_id(0)+ dims = tl.arange(0, BLOCK_D)+ slot = tl.load(slots_ptr + row)+ valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE)+ block = tl.maximum(slot, 0) // PAGE_SIZE+ token = tl.maximum(slot, 0) % PAGE_SIZE+ values = tl.load(+ rows_ptr + row * stride_rows_row + dims,+ mask=valid & (dims < WIDTH),+ other=0,+ )+ tl.store(+ cache_ptr+ + block.to(tl.int64) * stride_cache_block+ + token * stride_cache_token+ + dims,+ values,+ mask=valid & (dims < WIDTH),+ )+++def qsa_compress_groups(+ raw_keys: torch.Tensor,+ ring: torch.Tensor,+ ring_slots: torch.Tensor,+ token_to_req: torch.Tensor,+ query_start_loc: torch.Tensor,+ logical_positions: torch.Tensor,+ compress_ratio: int,+ pooled: torch.Tensor,+ first_positions: torch.Tensor,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Pool each row's closing group from the pending ring and this step's raw rows."""++ rows = raw_keys.shape[0]+ head_dim = raw_keys.shape[1]+ if ring.ndim != 3 or ring.shape[2] != head_dim:+ raise ValueError("QSA pending ring must be [slots, capacity, head_dim]")+ if ring.shape[1] < compress_ratio:+ raise ValueError("QSA ring capacity must cover a whole group")+ if raw_keys.stride(1) != 1 or ring.stride(2) != 1 or pooled.stride(1) != 1:+ raise ValueError("QSA compression needs unit-stride key rows")+ if not rows:+ return pooled, first_positions+ _compress_qsa_groups_kernel[(rows,)](+ raw_keys,+ ring,+ ring_slots,+ token_to_req,+ query_start_loc,+ logical_positions,+ pooled,+ first_positions,+ raw_keys.stride(0),+ ring.stride(0),+ ring.stride(1),+ pooled.stride(0),+ rows,+ ring.shape[0],+ query_start_loc.shape[0] - 1,+ RING_CAPACITY=ring.shape[1],+ COMPRESS_RATIO=compress_ratio,+ HEAD_DIM=head_dim,+ BLOCK_D=triton.next_power_of_2(head_dim),+ num_warps=4,+ )+ return pooled, first_positions+++def qsa_index_norm_rope(+ x: torch.Tensor,+ positions: torch.Tensor,+ cos_sin_cache: torch.Tensor,+ norm_weight: torch.Tensor,+ eps: float,+ out: torch.Tensor,+ heads: int = 1,+ dest_rows: torch.Tensor | None = None,+) -> torch.Tensor:+ """Zero-centered RMSNorm then partial NeoX rope on [rows, head_dim] indexer rows."""++ rows, head_dim = x.shape+ rotary_dim = cos_sin_cache.shape[1]+ if rotary_dim % 2 or rotary_dim > head_dim:+ raise ValueError("QSA indexer rope needs an even rotary_dim <= head_dim")+ if x.stride(1) != 1 or out.stride(1) != 1 or not cos_sin_cache.is_contiguous():+ raise ValueError("QSA indexer norm+rope needs unit-stride rows")+ if rows % heads:+ raise ValueError("QSA indexer rows must be a whole number of head groups")+ if not rows:+ return out+ block_r = 8 if head_dim >= 128 else 16+ _index_norm_rope_kernel[(triton.cdiv(rows, block_r),)](+ x,+ positions,+ cos_sin_cache,+ norm_weight,+ out,+ dest_rows,+ x.stride(0),+ out.stride(0),+ cos_sin_cache.stride(0),+ rows,+ eps,+ HEADS=heads,+ HEAD_DIM=head_dim,+ ROTARY_HALF=rotary_dim // 2,+ BLOCK_R=block_r,+ BLOCK_D=triton.next_power_of_2(head_dim),+ HAS_DEST_ROWS=dest_rows is not None,+ num_warps=4,+ )+ return out+++def qsa_store_rows(+ cache: torch.Tensor,+ slot_mapping: torch.Tensor,+ rows: torch.Tensor,+) -> None:+ """Scatter rows into a ``[blocks, block_size, width]`` cache at ``block * block_size ++ offset``; negative slots are dropped. The cache may be a strided per-layer view."""++ if cache.ndim != 3 or rows.ndim != 2 or rows.shape[1] != cache.shape[2]:+ raise ValueError("QSA row store needs a [blocks, block_size, width] cache")+ if cache.stride(2) != 1 or rows.stride(1) != 1:+ raise ValueError("QSA row store needs unit-stride rows")+ if rows.shape[0] != slot_mapping.numel():+ raise ValueError("QSA row store slots and rows disagree")+ if not rows.shape[0]:+ return+ _store_qsa_rows_kernel[(rows.shape[0],)](+ cache,+ slot_mapping,+ rows,+ cache.stride(0),+ cache.stride(1),+ rows.stride(0),+ rows.shape[0],+ cache.shape[0],+ PAGE_SIZE=cache.shape[1],+ WIDTH=cache.shape[2],+ BLOCK_D=triton.next_power_of_2(cache.shape[2]),+ num_warps=4,+ )+++__all__ = ["qsa_compress_groups", "qsa_index_norm_rope", "qsa_store_rows"]diff --git a/python/freetoken/kernel/triton/qsa/expand.py b/python/freetoken/kernel/triton/qsa/expand.pynew file mode 100644index 00000000..877b5391--- /dev/null+++ b/python/freetoken/kernel/triton/qsa/expand.py@@ -0,0 +1,131 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py)+"""Expand QSA top-k blocks into token indices."""++from __future__ import annotations++import torch+import triton+import triton.language as tl+++def _expand_qsa_indices_kernel(+ block_indices_ptr,+ query_positions_ptr,+ sequence_lengths_ptr,+ token_to_req_ptr,+ output_ptr,+ stride_blocks_row,+ stride_blocks_column,+ stride_output_row,+ stride_output_column,+ rows,+ num_requests,+ BLOCK_TOPK: tl.constexpr,+ COMPRESS_RATIO: tl.constexpr,+ TOKEN_TOPK: tl.constexpr,+ OUTPUT_WIDTH: tl.constexpr,+ COLUMN_BLOCK: tl.constexpr,+) -> None:+ # row * stride can overflow int32 for large row counts.+ row = tl.program_id(0).to(tl.int64)+ columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK)+ query_position = tl.load(query_positions_ptr + row)+ request = tl.load(token_to_req_ptr + row)+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)+ sequence_length = tl.load(+ sequence_lengths_ptr + safe_request,+ mask=(request >= 0) & (request < num_requests),+ other=0,+ )+ complete_blocks = tl.minimum(+ tl.minimum(+ (query_position + 1) // COMPRESS_RATIO,+ sequence_length // COMPRESS_RATIO,+ ),+ BLOCK_TOPK,+ )+ expanded_count = complete_blocks * COMPRESS_RATIO+ tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO+ tail_count = (query_position + 1) - tail_start++ is_expanded = columns < expanded_count+ block_rank = columns // COMPRESS_RATIO+ offset = columns % COMPRESS_RATIO+ safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1)+ block = tl.load(+ block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column,+ mask=(row < rows) & is_expanded,+ other=-1,+ )+ expanded = block * COMPRESS_RATIO + offset+ tail_offset = columns - expanded_count+ is_tail = (+ (columns >= expanded_count)+ & (tail_offset < tail_count)+ & (tail_offset < COMPRESS_RATIO - 1)+ )+ token = tl.where(is_expanded, expanded, tail_start + tail_offset)+ valid = (+ (row < rows)+ & (columns < OUTPUT_WIDTH)+ & (is_expanded | is_tail)+ & (token >= 0)+ & (token < sequence_length)+ )+ tl.store(+ output_ptr + row * stride_output_row + columns * stride_output_column,+ tl.where(valid, token, -1),+ mask=(row < rows) & (columns < OUTPUT_WIDTH),+ )+++def expand_qsa_block_indices(+ block_indices: torch.Tensor,+ query_positions: torch.Tensor,+ sequence_lengths: torch.Tensor,+ token_to_req: torch.Tensor,+ compress_ratio: int,+ token_topk: int,+ out: torch.Tensor,+) -> torch.Tensor:+ """Expand compressed blocks and compact the causal tail of the open group."""++ if token_topk % compress_ratio:+ raise ValueError("QSA token top-k must be divisible by compression ratio")+ block_topk = token_topk // compress_ratio+ output_width = token_topk + compress_ratio - 1+ if block_indices.shape != (query_positions.numel(), block_topk):+ raise ValueError("QSA compressed top-k has an invalid shape")+ if out.shape != (block_indices.shape[0], output_width):+ raise ValueError("QSA expansion output has an invalid shape")+ if not block_indices.shape[0]:+ return out+ column_block = 256+ _expand_qsa_indices_kernel[+ (block_indices.shape[0], triton.cdiv(output_width, column_block))+ ](+ block_indices,+ query_positions,+ sequence_lengths,+ token_to_req,+ out,+ block_indices.stride(0),+ block_indices.stride(1),+ out.stride(0),+ out.stride(1),+ block_indices.shape[0],+ sequence_lengths.shape[0],+ BLOCK_TOPK=block_topk,+ COMPRESS_RATIO=compress_ratio,+ TOKEN_TOPK=token_topk,+ OUTPUT_WIDTH=output_width,+ COLUMN_BLOCK=column_block,+ num_warps=4,+ )+ return out+++__all__ = ["expand_qsa_block_indices"]diff --git a/python/freetoken/kernel/triton/qsa/score.py b/python/freetoken/kernel/triton/qsa/score.pynew file mode 100644index 00000000..49d70208--- /dev/null+++ b/python/freetoken/kernel/triton/qsa/score.py@@ -0,0 +1,191 @@+# SPDX-License-Identifier: Apache-2.0+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py)+"""QSA block scoring over the paged compressed-key slab."""++from __future__ import annotations++import math++import torch+import triton+import triton.language as tl+++def _qsa_mqa_paged_kernel(+ q_ptr,+ k_cache_ptr,+ page_table_ptr,+ token_to_req_ptr,+ query_positions_ptr,+ sequence_lengths_ptr,+ visible_blocks_ptr,+ logits_ptr,+ stride_q_row,+ stride_q_head,+ stride_q_dim,+ stride_cache_block,+ stride_cache_token,+ stride_cache_dim,+ stride_table_req,+ stride_table_page,+ stride_logits_row,+ num_rows,+ num_columns,+ num_pages,+ num_requests,+ score_divisor,+ PAGE_SIZE: tl.constexpr,+ PAGE_TABLE_WIDTH: tl.constexpr,+ NUM_HEADS: tl.constexpr,+ HEAD_DIM: tl.constexpr,+ BLOCK_N: tl.constexpr,+ BLOCK_D: tl.constexpr,+ TILES_PER_PROG: tl.constexpr,+ STAGES: tl.constexpr,+ MAX_N: tl.constexpr,+ COMPRESS_RATIO: tl.constexpr,+) -> None:+ row = tl.program_id(0)+ dims = tl.arange(0, BLOCK_D)+ heads = tl.arange(0, MAX_N)+ request = tl.load(token_to_req_ptr + row)+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)+ query_position = tl.load(query_positions_ptr + row)+ sequence_length = tl.load(+ sequence_lengths_ptr + safe_request,+ mask=(request >= 0) & (request < num_requests),+ other=0,+ )+ visible = tl.minimum(+ (query_position + 1) // COMPRESS_RATIO,+ sequence_length // COMPRESS_RATIO,+ )+ if tl.program_id(1) == 0:+ tl.store(visible_blocks_ptr + row, visible)+ tile_start = tl.program_id(1) * TILES_PER_PROG+ # Top-k is bounded by visible_blocks, so columns beyond it need no value.+ if tile_start * BLOCK_N >= visible:+ return+ tile_end = tl.minimum(tile_start + TILES_PER_PROG, tl.cdiv(visible, BLOCK_N))+ tile_end = tl.minimum(tile_end, tl.cdiv(num_columns, BLOCK_N))++ # Pad the small head axis to a tensor-core-compatible N dimension.+ query = tl.load(+ q_ptr+ + row * stride_q_row+ + heads[None, :] * stride_q_head+ + dims[:, None] * stride_q_dim,+ mask=(heads[None, :] < NUM_HEADS) & (dims[:, None] < HEAD_DIM),+ other=0.0,+ )+ column_offsets = tl.arange(0, BLOCK_N)+ for tile in tl.range(tile_start, tile_end, num_stages=STAGES):+ columns = tile * BLOCK_N + column_offsets+ live = columns < visible+ logical_page = tl.minimum(columns // PAGE_SIZE, PAGE_TABLE_WIDTH - 1)+ page_offset = columns % PAGE_SIZE+ physical_page = tl.load(+ page_table_ptr+ + safe_request * stride_table_req+ + logical_page * stride_table_page,+ mask=live,+ other=-1,+ )+ page_valid = live & (physical_page >= 0) & (physical_page < num_pages)+ # physical_page * block stride can overflow int32 for large caches.+ safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64)+ keys = tl.load(+ k_cache_ptr+ + safe_physical_page[:, None] * stride_cache_block+ + page_offset[:, None] * stride_cache_token+ + dims[None, :] * stride_cache_dim,+ mask=page_valid[:, None] & (dims[None, :] < HEAD_DIM),+ other=0.0,+ eviction_policy="evict_first",+ )+ scores = tl.dot(keys, query, out_dtype=tl.float32)+ scores = tl.where(heads[None, :] < NUM_HEADS, tl.maximum(scores, 0.0), 0.0)+ score = tl.sum(scores, axis=1) / score_divisor+ tl.store(+ logits_ptr + row * stride_logits_row + columns,+ tl.where(page_valid, score, -float("inf")),+ mask=live & (columns < num_columns),+ )+++def qsa_mqa_paged(+ q: torch.Tensor,+ k_cache: torch.Tensor,+ page_table: torch.Tensor,+ token_to_req: torch.Tensor,+ query_positions: torch.Tensor,+ sequence_lengths: torch.Tensor,+ compress_ratio: int,+ logits: torch.Tensor,+ visible_blocks: torch.Tensor,+ score_scale: float | None = None,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Compute QSA scores directly from a paged compressed-key cache."""++ if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0:+ raise ValueError("QSA query must be [rows, heads, head_dim]")+ if k_cache.ndim != 4 or k_cache.shape[2] != 1:+ raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]")+ if k_cache.shape[3] != q.shape[2]:+ raise ValueError("QSA query and cache dimensions must match")+ if token_to_req.shape != (q.shape[0],) or query_positions.shape != (q.shape[0],):+ raise ValueError("QSA request mapping and positions must match query rows")+ if sequence_lengths.shape != (page_table.shape[0],):+ raise ValueError("QSA sequence lengths must match page-table requests")+ score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale+ columns = logits.shape[1]+ if not q.shape[0] or not columns:+ return logits, visible_blocks+ BLOCK_N = 64+ BLOCK_D = max(16, triton.next_power_of_2(q.shape[2]))+ MAX_N = max(16, triton.next_power_of_2(q.shape[1]))+ # Tuned on GB300: larger row batches provide enough parallelism to reuse Q.+ tiles_per_program = 1 if q.shape[0] <= 32 else 8+ _qsa_mqa_paged_kernel[+ (q.shape[0], triton.cdiv(columns, BLOCK_N * tiles_per_program))+ ](+ q,+ k_cache,+ page_table,+ token_to_req,+ query_positions,+ sequence_lengths,+ visible_blocks,+ logits,+ q.stride(0),+ q.stride(1),+ q.stride(2),+ k_cache.stride(0),+ k_cache.stride(1),+ k_cache.stride(3),+ page_table.stride(0),+ page_table.stride(1),+ logits.stride(0),+ q.shape[0],+ columns,+ k_cache.shape[0],+ page_table.shape[0],+ float(score_divisor),+ PAGE_SIZE=k_cache.shape[1],+ PAGE_TABLE_WIDTH=page_table.shape[1],+ NUM_HEADS=q.shape[1],+ HEAD_DIM=q.shape[2],+ BLOCK_N=BLOCK_N,+ BLOCK_D=BLOCK_D,+ TILES_PER_PROG=tiles_per_program,+ STAGES=2,+ MAX_N=MAX_N,+ COMPRESS_RATIO=compress_ratio,+ num_warps=2,+ )+ return logits, visible_blocks+++__all__ = ["qsa_mqa_paged"]diff --git a/python/freetoken/kernel/triton/qsa/topk.py b/python/freetoken/kernel/triton/qsa/topk.pynew file mode 100644index 00000000..ae57566c--- /dev/null+++ b/python/freetoken/kernel/triton/qsa/topk.py@@ -0,0 +1,458 @@+"""Exact block top-k over the QSA indexer scores.++torch.topk also sorts its k winners, which the selection never uses: expand.py expands every+rank the same way and the sparse attend kernel softmaxes over the union, so only the SET+matters. One program per row runs an MSB-first radix select on the monotone uint32 image of+the fp32 scores -- ``PASSES`` ``RADIX``-bit passes, each a histogram of the columns that still+match the fixed prefix -- and one compaction pass then emits the winners in ascending column+order, -1 padded to the output width like the torch.topk path it replaces.++A row that fits one tile is held in registers across the passes (``SINGLE_TILE``); wider rows+re-read the tile per pass, which is the price of not spilling a 256 KB row.++One program per row stops scaling once a row runs to tens of thousands of columns: the whole+row goes through a single CTA. Wide buffers therefore split. Phase 1 gives each ``CHUNK``-wide+slice of a row its own program, which radix-selects the slice's own top-k into a candidate+workspace; phase 2 runs the same radix select over the union of those candidates. The global+top-k is a subset of that union -- a global winner beats at most k-1 columns, so it beats all+but at most k-1 of its own slice -- so the answer stays exact. ``_split_plan`` derives the+geometry from the buffer width alone, which is fixed when a graph is captured.+"""++from __future__ import annotations++import torch+import triton+import triton.language as tl++# 5-bit digits beat 8-bit ones here: tl.histogram cost grows faster with the bin count than the+# extra passes cost. 35 bits of digit cover the 32-bit key, the top pass just sees zero bits.+_RADIX = 5+_BINS = 1 << _RADIX+_PASSES = -(-32 // _RADIX)+_MAX_BLOCK_N = 4096+# A resident tile costs ~2 ns per column against ~3.2 ns for a re-read one, so both split+# phases stay resident; past 8192 the tile spills (255 registers) and that reverses.+_MAX_RESIDENT = 8192+_MIN_CHUNK = 4096+++def _monotone_key(value):+ """fp32 -> uint32 with the float order preserved; key 0 is reserved for a dead column."""+ bits = value.to(tl.uint32, bitcast=True)+ return tl.where((bits >> 31) == 0, bits | 0x80000000, ~bits)+++def _load_keys(logits_row, columns, limit):+ live = columns < limit+ value = tl.load(logits_row + columns, mask=live, other=-float("inf"))+ return tl.where(live & (value > -float("inf")), _monotone_key(value), 0)+++def _narrow(hist, prefix, keep, k_rem, shift, BINS: tl.constexpr):+ """One radix step: pin the digit at ``shift`` and report whether the range is settled."""+ bins = tl.arange(0, BINS)+ # Lowest bin whose strictly-greater bins no longer cover k_rem holds the k_rem-th key;+ # `above` falls with the bin id, so the predicate is an upper set.+ above = tl.sum(hist) - tl.cumsum(hist, axis=0)+ inside = above < k_rem+ bin_id = tl.min(tl.where(inside, bins, BINS - 1))+ hit = bins == bin_id+ prefix |= bin_id.to(tl.uint32) << shift+ keep |= tl.full((), BINS - 1, tl.uint32) << shift+ k_rem -= tl.sum(tl.where(hit, above, 0))+ # The whole bin fits: `prefix` is a lower bound, the ties below it all win.+ return prefix, keep, k_rem, k_rem == tl.sum(tl.where(hit, hist, 0))+++def _resident_prefix(key, k_eff, BINS: tl.constexpr, RADIX: tl.constexpr, PASSES: tl.constexpr):+ """Radix-select a register-resident tile; returns the winning key range and its tie budget."""+ # Invariant per pass: `k_rem` winners are still to be found inside the key range that+ # `prefix` pins on the `keep` bits, and every key above that range is already a winner.+ prefix = tl.zeros((), tl.uint32)+ keep = tl.zeros((), tl.uint32)+ k_rem = k_eff+ settled = False+ for step in tl.static_range(PASSES):+ shift = RADIX * (PASSES - 1 - step)+ if not settled:+ hist = tl.histogram(+ ((key >> shift) & (BINS - 1)).to(tl.int32),+ BINS,+ mask=(key != 0) & ((key & keep) == prefix),+ )+ prefix, keep, k_rem, settled = _narrow(hist, prefix, keep, k_rem, shift, BINS)+ return prefix, tl.where(settled, k_eff, k_rem)+++def _tile_ranks(key, prefix, ties, above_base, equal_base):+ """Rank every winner of one tile, and carry the running counts past it.++ One packed cumsum carries both: keys above the threshold in the low half, keys equal to it+ in the high half, so a winner's rank is ``above_before + min(equal_before, ties)``."""+ greater = ((key > prefix) & (key != 0)).to(tl.int32)+ equal = ((key == prefix) & (key != 0)).to(tl.int32)+ packed = greater | (equal << 16)+ before = tl.cumsum(packed, axis=0) - packed+ rank_equal = equal_base + (before >> 16)+ take = (greater == 1) | ((equal == 1) & (rank_equal < ties))+ rank = above_base + (before & 0xFFFF) + tl.minimum(rank_equal, ties)+ total = tl.sum(packed)+ return rank, take, above_base + (total & 0xFFFF), equal_base + (total >> 16)+++def _compact(+ out_row,+ key,+ columns,+ prefix,+ ties,+ above_base,+ equal_base,+ TOP_K: tl.constexpr,+ BLOCK_N: tl.constexpr,+):+ """Emit one tile's winners at their global ranks; returns the running counts after it."""+ tl.static_assert(BLOCK_N <= 0xFFFF, "packed cumsum keeps 16 bits per half")+ rank, take, above, equal = _tile_ranks(key, prefix, ties, above_base, equal_base)+ tl.store(out_row + rank, columns.to(tl.int32), mask=take & (rank < TOP_K))+ return above, equal+++def _qsa_block_topk_kernel(+ logits_ptr,+ visible_ptr,+ out_ptr,+ stride_logits_row,+ stride_out_row,+ num_columns,+ TOP_K: tl.constexpr,+ PAD_K: tl.constexpr,+ BLOCK_N: tl.constexpr,+ SINGLE_TILE: tl.constexpr,+ BINS: tl.constexpr,+ RADIX: tl.constexpr,+ PASSES: tl.constexpr,+) -> None:+ row = tl.program_id(0)+ limit = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0)+ k_eff = tl.minimum(limit, TOP_K)+ logits_row = logits_ptr + row.to(tl.int64) * stride_logits_row+ out_row = out_ptr + row.to(tl.int64) * stride_out_row+ offsets = tl.arange(0, BLOCK_N)+ tiles = tl.cdiv(limit, BLOCK_N)+ if SINGLE_TILE:+ resident = _load_keys(logits_row, offsets, limit)++ emitted = 0+ if k_eff > 0:+ if SINGLE_TILE:+ prefix, ties = _resident_prefix(resident, k_eff, BINS, RADIX, PASSES)+ else:+ prefix = tl.zeros((), tl.uint32)+ keep = tl.zeros((), tl.uint32)+ k_rem = k_eff+ settled = False+ for step in tl.static_range(PASSES):+ shift = RADIX * (PASSES - 1 - step)+ if not settled:+ hist = tl.zeros((BINS,), tl.int32)+ for tile in range(tiles):+ key = _load_keys(logits_row, tile * BLOCK_N + offsets, limit)+ hist += tl.histogram(+ ((key >> shift) & (BINS - 1)).to(tl.int32),+ BINS,+ mask=(key != 0) & ((key & keep) == prefix),+ )+ prefix, keep, k_rem, settled = _narrow(+ hist, prefix, keep, k_rem, shift, BINS+ )+ ties = tl.where(settled, k_eff, k_rem)++ above_base = 0+ equal_base = 0+ if SINGLE_TILE:+ above_base, equal_base = _compact(+ out_row, resident, offsets, prefix, ties, above_base, equal_base, TOP_K, BLOCK_N+ )+ else:+ for tile in range(tiles):+ columns = tile * BLOCK_N + offsets+ key = _load_keys(logits_row, columns, limit)+ above_base, equal_base = _compact(+ out_row, key, columns, prefix, ties, above_base, equal_base, TOP_K, BLOCK_N+ )+ emitted = above_base + tl.minimum(equal_base, ties)++ pad = tl.arange(0, PAD_K)+ tl.store(out_row + pad, -1, mask=(pad >= emitted) & (pad < TOP_K))+++def _qsa_topk_split_kernel(+ logits_ptr,+ visible_ptr,+ key_ptr,+ col_ptr,+ stride_logits_row,+ stride_scratch_row,+ num_columns,+ TOP_K: tl.constexpr,+ PAD_K: tl.constexpr,+ CHUNK: tl.constexpr,+ BINS: tl.constexpr,+ RADIX: tl.constexpr,+ PASSES: tl.constexpr,+) -> None:+ """Phase 1: one program per (row, chunk), writing the chunk's own top-k as candidates."""+ tl.static_assert(CHUNK <= 0xFFFF, "packed cumsum keeps 16 bits per half")+ row = tl.program_id(0)+ split = tl.program_id(1)+ base = split * CHUNK+ visible = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0)+ limit = tl.minimum(tl.maximum(visible - base, 0), CHUNK)+ k_eff = tl.minimum(limit, TOP_K)+ slot = row.to(tl.int64) * stride_scratch_row + split * TOP_K++ emitted = 0+ if k_eff > 0:+ offsets = tl.arange(0, CHUNK)+ key = _load_keys(logits_ptr + row.to(tl.int64) * stride_logits_row + base, offsets, limit)+ prefix, ties = _resident_prefix(key, k_eff, BINS, RADIX, PASSES)+ rank, take, above, equal = _tile_ranks(key, prefix, ties, 0, 0)+ write = take & (rank < TOP_K)+ tl.store(key_ptr + slot + rank, key.to(tl.int32, bitcast=True), mask=write)+ tl.store(col_ptr + slot + rank, (base + offsets).to(tl.int32), mask=write)+ emitted = above + tl.minimum(equal, ties)+ # A key of 0 is the dead-column sentinel, so the merge needs no separate count per slot.+ pad = tl.arange(0, PAD_K)+ tl.store(key_ptr + slot + pad, 0, mask=(pad >= emitted) & (pad < TOP_K))+++def _merge_tile(+ key_row,+ col_row,+ out_row,+ candidates,+ k_eff,+ TOP_K: tl.constexpr,+ BLOCK: tl.constexpr,+ BINS: tl.constexpr,+ RADIX: tl.constexpr,+ PASSES: tl.constexpr,+):+ """Top-k of one resident tile of candidates; returns how many winners it wrote."""+ tl.static_assert(BLOCK <= 0xFFFF, "packed cumsum keeps 16 bits per half")+ offsets = tl.arange(0, BLOCK)+ live = offsets < candidates+ key = tl.load(key_row + offsets, mask=live, other=0).to(tl.uint32, bitcast=True)+ prefix, ties = _resident_prefix(key, k_eff, BINS, RADIX, PASSES)+ rank, take, above, equal = _tile_ranks(key, prefix, ties, 0, 0)+ column = tl.load(col_row + offsets, mask=live, other=-1)+ tl.store(out_row + rank, column, mask=take & (rank < TOP_K))+ return above + tl.minimum(equal, ties)+++def _qsa_topk_merge_kernel(+ visible_ptr,+ key_ptr,+ col_ptr,+ out_ptr,+ stride_scratch_row,+ stride_out_row,+ num_columns,+ TOP_K: tl.constexpr,+ PAD_K: tl.constexpr,+ CHUNK: tl.constexpr,+ N_SPLITS: tl.constexpr,+ BLOCK_SMALL: tl.constexpr,+ BLOCK_MID: tl.constexpr,+ BLOCK_FULL: tl.constexpr,+ BINS: tl.constexpr,+ RADIX: tl.constexpr,+ PASSES: tl.constexpr,+) -> None:+ """Phase 2: one program per row over the candidates phase 1 left behind."""+ row = tl.program_id(0)+ limit = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0)+ k_eff = tl.minimum(limit, TOP_K)+ # Candidates sit chunk-major, so the splits past the visible tail are one skipped suffix+ # and the merge keeps costing what the live part of the row costs.+ candidates = tl.minimum(tl.cdiv(limit, CHUNK), N_SPLITS) * TOP_K+ key_row = key_ptr + row.to(tl.int64) * stride_scratch_row+ col_row = col_ptr + row.to(tl.int64) * stride_scratch_row+ out_row = out_ptr + row.to(tl.int64) * stride_out_row++ emitted = 0+ if k_eff > 0:+ if candidates <= TOP_K:+ # One live split: its own top-k is already the row's, ranked and packed.+ slot = tl.arange(0, PAD_K)+ inside = slot < TOP_K+ key = tl.load(key_row + slot, mask=inside, other=0)+ alive = inside & (key != 0)+ column = tl.load(col_row + slot, mask=alive, other=-1)+ tl.store(out_row + slot, column, mask=inside)+ emitted = tl.sum(alive.to(tl.int32))+ # Register residency costs the whole tile even when few candidates are live, so a+ # short row takes a narrower tile.+ elif candidates <= BLOCK_SMALL:+ emitted = _merge_tile(+ key_row, col_row, out_row, candidates, k_eff,+ TOP_K, BLOCK_SMALL, BINS, RADIX, PASSES,+ )+ elif candidates <= BLOCK_MID:+ emitted = _merge_tile(+ key_row, col_row, out_row, candidates, k_eff,+ TOP_K, BLOCK_MID, BINS, RADIX, PASSES,+ )+ else:+ emitted = _merge_tile(+ key_row, col_row, out_row, candidates, k_eff,+ TOP_K, BLOCK_FULL, BINS, RADIX, PASSES,+ )++ pad = tl.arange(0, PAD_K)+ tl.store(out_row + pad, -1, mask=(pad >= emitted) & (pad < TOP_K))+++def _split_plan(columns: int, top_k: int) -> tuple[int, int] | None:+ """``(chunk, n_splits)`` for the split+merge path, or None to keep the one-program path."""+ if top_k <= 0 or columns <= _MIN_CHUNK:+ return None+ max_splits = _MAX_RESIDENT // triton.next_power_of_2(top_k)+ if max_splits < 2:+ return None+ chunk = max(_MIN_CHUNK, triton.next_power_of_2(-(-columns // max_splits)))+ if chunk > _MAX_RESIDENT:+ return None+ n_splits = -(-columns // chunk)+ # Merging n_splits*top_k candidates has to be cheaper than scanning the row once.+ if n_splits < 2 or n_splits * top_k >= columns:+ return None+ return chunk, n_splits+++def qsa_block_topk_scratch_width(columns: int, top_k: int) -> int:+ """int32 columns of scratch ``qsa_block_topk`` wants per row; 0 when it needs none."""+ plan = _split_plan(columns, top_k)+ return 0 if plan is None else 2 * plan[1] * top_k+++def qsa_block_topk(+ logits: torch.Tensor,+ visible: torch.Tensor,+ out: torch.Tensor,+ scratch: torch.Tensor | None = None,+) -> torch.Tensor:+ """Top ``out.shape[1]`` columns of every ``logits`` row below ``visible``, -1 padded.++ Winners come out in ascending column order, packed at the front of the row; a row with+ fewer live columns than the width, and any column scored -inf, leaves -1 in the tail.+ ``scratch`` is the candidate workspace of the split path (see+ ``qsa_block_topk_scratch_width``); it is allocated per call when the caller passes none."""++ if logits.ndim != 2 or out.ndim != 2:+ raise ValueError("QSA block top-k takes 2-D logits and output")+ if out.shape[0] != logits.shape[0] or visible.shape != (logits.shape[0],):+ raise ValueError("QSA block top-k needs one visible count and one output row per row")+ if logits.dtype != torch.float32 or out.dtype != torch.int32:+ raise ValueError("QSA block top-k takes fp32 logits and an int32 output")+ if logits.stride(1) != 1 or out.stride(1) != 1 or visible.stride(0) != 1:+ raise ValueError("QSA block top-k needs row-contiguous logits, output and counts")+ rows, columns = logits.shape+ top_k = out.shape[1]+ if not rows or not top_k:+ return out+ pad_k = triton.next_power_of_2(top_k)+ plan = _split_plan(columns, top_k)+ if plan is None:+ block_n = min(_MAX_BLOCK_N, triton.next_power_of_2(max(columns, 1)))+ _qsa_block_topk_kernel[(rows,)](+ logits,+ visible,+ out,+ logits.stride(0),+ out.stride(0),+ columns,+ TOP_K=top_k,+ PAD_K=pad_k,+ BLOCK_N=block_n,+ SINGLE_TILE=columns <= block_n,+ BINS=_BINS,+ RADIX=_RADIX,+ PASSES=_PASSES,+ num_warps=8,+ num_stages=1,+ )+ return out++ chunk, n_splits = plan+ half = n_splits * top_k+ if scratch is None:+ scratch = torch.empty((rows, 2 * half), dtype=torch.int32, device=logits.device)+ elif (+ scratch.ndim != 2+ or scratch.shape[0] < rows+ or scratch.shape[1] < 2 * half+ or scratch.dtype != torch.int32+ or scratch.stride(1) != 1+ ):+ raise ValueError(+ f"QSA block top-k needs a row-contiguous int32 scratch of at least "+ f"[{rows}, {2 * half}], got {tuple(scratch.shape)} {scratch.dtype}"+ )+ keys = scratch[:rows, :half]+ cols = scratch[:rows, half : 2 * half]+ _qsa_topk_split_kernel[(rows, n_splits)](+ logits,+ visible,+ keys,+ cols,+ logits.stride(0),+ keys.stride(0),+ columns,+ TOP_K=top_k,+ PAD_K=pad_k,+ CHUNK=chunk,+ BINS=_BINS,+ RADIX=_RADIX,+ PASSES=_PASSES,+ num_warps=8,+ num_stages=1,+ )+ merge_block = triton.next_power_of_2(half)+ _qsa_topk_merge_kernel[(rows,)](+ visible,+ keys,+ cols,+ out,+ keys.stride(0),+ out.stride(0),+ columns,+ TOP_K=top_k,+ PAD_K=pad_k,+ CHUNK=chunk,+ N_SPLITS=n_splits,+ BLOCK_SMALL=min(1024, merge_block),+ BLOCK_MID=min(4096, merge_block),+ BLOCK_FULL=merge_block,+ BINS=_BINS,+ RADIX=_RADIX,+ PASSES=_PASSES,+ num_warps=8,+ num_stages=1,+ )+ return out+++__all__ = ["qsa_block_topk", "qsa_block_topk_scratch_width"]diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.pyindex 1bb352b4..c6c0f1bb 100644--- a/python/freetoken/kvcache/__init__.py+++ b/python/freetoken/kvcache/__init__.py@@ -57,6 +57,10 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:from .dsa_pool import MLAKVCachereturn MLAKVCache+ if AttnType.QSA in types:+ from .qsa_pool import QSAKVCache++ return QSAKVCacheif AttnType.BSA in types:from .bsa_pool import BSAKVCache@@ -106,6 +110,7 @@ def create_kv_pool(config, num_pages: int, device: torch.device, dtype: torch.dtnum_swa_tokens=num_swa_tokens,device=device,dtype=dtype,+ num_req_slots=config.max_running_req + 1, # + 1 for the dummy request row)@@ -116,6 +121,7 @@ def create_kvcache_pool(dtype: torch.dtype,device: torch.device,num_swa_tokens: int | None = None,+ num_req_slots: int | None = None,) -> BaseKVCachePool:if model_config.has_swa_attention:from .hybrid_swa_pool import HybridSWAKVCache@@ -174,6 +180,31 @@ def create_kvcache_pool(num_index_layers=spec.num_index_layers,)+ # QSA (Qwen3.8-Flash-Next): the same GQA group, but it stores one index key per+ # index_ratio tokens and adds per-request tiers sized by the concurrency, not the pages.+ # layer_ids is mandatory here -- the model is hybrid-linear, and letting MHAKVCache back+ # all num_layers would allocate K/V slabs for the GDN layers too.+ if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.QSA:+ from .qsa_pool import QSAKVCache++ spec = kv_specs[0]+ if num_req_slots is None:+ raise ValueError("QSA pools need num_req_slots (max_running_req + 1)")+ return QSAKVCache(+ num_kv_heads=spec.num_kv_heads,+ num_layers=model_config.num_layers,+ head_dim=spec.head_dim,+ num_pages=num_pages,+ page_size=page_size,+ dtype=dtype,+ device=device,+ index_head_dim=spec.index_head_dim,+ num_index_layers=spec.num_index_layers,+ index_ratio=spec.index_ratio,+ num_req_slots=num_req_slots,+ layer_ids=spec.layer_ids,+ )+if len(kv_specs) == 1 and kv_specs[0].mla:from .dsa_pool import DSAKVCache, MLAKVCachediff --git a/python/freetoken/kvcache/base.py b/python/freetoken/kvcache/base.pyindex ae8cf9ec..95669e8c 100644--- a/python/freetoken/kvcache/base.py+++ b/python/freetoken/kvcache/base.py@@ -21,7 +21,10 @@ def spec_kv_bytes_per_token(spec, config) -> int:x layers, plus the bf16 DSA index-key slab when the spec carries indexer dims. Pureper-spec arithmetic -- pool families compose it over THEIR OWN groups; no familybranching here. (2 bytes/elem == the torch.bfloat16 dsa_pool.DSAKVCache._alloc- hardcodes; keep the two in lockstep if the slab dtype ever changes.)"""+ hardcodes; keep the two in lockstep if the slab dtype ever changes.)++ ``index_ratio`` > 1 (QSA) stores one index key per token group, not per token; that slab's+ ring and scratch rows are fixed-size and priced in QSAKVCache.kv_cost instead."""per_token = ((1 if spec.mla else 2) # MLA latent groups store one slab (V aliases K)* spec.head_dim@@ -29,7 +32,7 @@ def spec_kv_bytes_per_token(spec, config) -> int:* config.dtype.itemsize* spec.num_layers)- return per_token + spec.index_head_dim * spec.num_index_layers * 2+ return per_token + spec.index_head_dim * spec.num_index_layers * 2 // spec.index_ratioclass BaseKVCachePool(ABC):diff --git a/python/freetoken/kvcache/linear_state_pool.py b/python/freetoken/kvcache/linear_state_pool.pyindex 2b3b0138..ff753941 100644--- a/python/freetoken/kvcache/linear_state_pool.py+++ b/python/freetoken/kvcache/linear_state_pool.py@@ -1,9 +1,11 @@from __future__ import annotations+import math+import torchfrom freetoken.distributed import get_tp_infofrom freetoken.env import ENV-from freetoken.models.config import LinearGatedDeltaGroupConfig+from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpecfrom freetoken.utils import div_even_SSM_DTYPES = {@@ -35,6 +37,11 @@ class LinearStatePool:Indexed by ``Req.table_idx`` (0..max_running_req), the same per-request slot thepage table uses, so the scheduler's existing admit/free of ``table_idx`` covers thestate's lifetime. One fixed slot per running request; no paging, no eviction.++ A model can declare extra per-request tensors on the same slots through+ ``ModelConfig.slot_states`` (see ``SlotStateSpec``); they advance, snapshot, COW and+ rebuild with the GDN state and are read back through ``slot_state(name, layer_id)``.+ Consumers must re-read them each forward: ``rebuild`` replaces the tensors."""def __init__(@@ -44,6 +51,7 @@ class LinearStatePool:dtype: torch.dtype,device: torch.device,tp_size: int | None = None,+ slot_states: tuple[SlotStateSpec, ...] = (),) -> None:if tp_size is None:tp_size = get_tp_info().size@@ -70,6 +78,16 @@ class LinearStatePool:)self._local_index = {layer_id: i for i, layer_id in enumerate(group.layer_ids)}+ self._slot_specs = tuple(slot_states)+ names = [spec.name for spec in self._slot_specs]+ if len(set(names)) != len(names):+ raise ValueError(f"duplicate slot_state names: {names}")+ self._state_layer_index = {+ spec.name: {lid: i for i, lid in enumerate(spec.layer_ids)}+ for spec in self._slot_specs+ }+ self.slot_states: dict[str, torch.Tensor] = self._alloc_slot_states(num_slots)+# Free-list allocator over slots 1..num_slots-1 (slot 0 reserved as a padding sink,# sglang MambaPool convention). Live working slots, ping-pong track slots, and# radix-tree-donated snapshots are all drawn from this single free-list, so memory@@ -77,6 +95,30 @@ class LinearStatePool:self.padding_slot = 0self._free_slots: list[int] = list(range(1, num_slots))+ def _alloc_slot_states(self, num_slots: int) -> dict[str, torch.Tensor]:+ return {+ spec.name: torch.full(+ (max(1, len(spec.layer_ids)), num_slots, *spec.shape),+ spec.fill_value,+ dtype=spec.dtype if spec.dtype is not None else self._conv_dtype,+ device=self._device,+ )+ for spec in self._slot_specs+ }++ def has_slot_state(self, name: str) -> bool:+ return name in self.slot_states++ def slot_state(self, name: str, layer_id: int | None = None) -> torch.Tensor:+ """One declared sibling state, ``[num_slots, *shape]``; ``layer_id`` picks the layer row."""+ t = self.slot_states[name]+ if layer_id is None:+ assert not self._state_layer_index[name], (+ f"slot_state {name!r} is per-layer, pass layer_id"+ )+ return t[0]+ return t[self._state_layer_index[name][layer_id]]+@propertydef num_free_slots(self) -> int:return len(self._free_slots)@@ -110,6 +152,7 @@ class LinearStatePool:device = self._deviceself.conv_states = Noneself.recurrent_states = None+ self.slot_states = {}if device.type == "cuda":torch.cuda.synchronize(device)torch.cuda.empty_cache()@@ -121,6 +164,7 @@ class LinearStatePool:dtype=rec_dtype,device=device,)+ self.slot_states = self._alloc_slot_states(num_slots)self._num_slots = num_slotsself._free_slots = list(range(1, num_slots))@@ -138,12 +182,16 @@ class LinearStatePool:slots = torch.as_tensor(slots, dtype=torch.long, device=self._device)self.conv_states[:, slots] = 0self.recurrent_states[:, slots] = 0+ for spec in self._slot_specs:+ self.slot_states[spec.name][:, slots] = spec.fill_valuedef copy_from(self, src: int, dst: int) -> None:"""Copy a whole-sequence snapshot (conv + recurrent, all layers) from slot ``src`` to``dst``. Used for COW-on-restore (donated snapshot -> fresh live slot)."""self.conv_states[:, dst].copy_(self.conv_states[:, src])self.recurrent_states[:, dst].copy_(self.recurrent_states[:, src])+ for t in self.slot_states.values():+ t[:, dst].copy_(t[:, src])def is_linear_layer(self, layer_id: int) -> bool:return layer_id in self._local_index@@ -161,6 +209,8 @@ class LinearStatePool:"""Zero a slot across all linear layers (new request takes this table_idx)."""self.conv_states[:, table_idx].zero_()self.recurrent_states[:, table_idx].zero_()+ for spec in self._slot_specs:+ self.slot_states[spec.name][:, table_idx] = spec.fill_value@propertydef num_linear_layers(self) -> int:@@ -180,6 +230,8 @@ class LinearStatePool:self.conv_states[:, 0].numel() * self.conv_states.element_size()+ self.recurrent_states[:, 0].numel() * self.recurrent_states.element_size())+ for t in self.slot_states.values():+ per += t[:, 0].numel() * t.element_size()return int(per)@@ -187,15 +239,22 @@ def linear_state_bytes_per_req(group: LinearGatedDeltaGroupConfig,tp_size: int,dtype: torch.dtype,+ slot_states: tuple[SlotStateSpec, ...] = (),) -> int:- """Linear-state bytes for one request across all linear layers (TP-local)."""+ """Linear-state bytes for one request across all linear layers (TP-local), plus any+ declared slot_states."""n_layers, local_conv_dim, local_v_heads = _linear_local_dims(group, tp_size)conv_elems = local_conv_dim * (group.conv_kernel_dim - 1)rec_elems = local_v_heads * group.key_head_dim * group.value_head_dimconv_bytes = conv_elems * dtype.itemsize # conv state in model dtyperec_bytes = rec_elems * ssm_state_dtype().itemsize # recurrent state (default fp32)- return int(n_layers * (conv_bytes + rec_bytes))+ total = n_layers * (conv_bytes + rec_bytes)++ for spec in slot_states:+ item = (spec.dtype if spec.dtype is not None else dtype).itemsize+ total += max(1, len(spec.layer_ids)) * math.prod(spec.shape) * item+ return int(total)__all__ = ["LinearStatePool", "linear_state_bytes_per_req"]@@ -206,10 +265,16 @@ def state_pool_bytes(config, num_slots: int | None = None) -> int:slot count). The engine adds this to the KV family's fixed cost when budgeting --the state pool is a sibling pool, not a KV tier."""linear_group = config.model_config.linear_attention_group()+ slot_states = getattr(config.model_config, "slot_states", ())if linear_group is None:+ if slot_states:+ raise ValueError("slot_states ride the linear-state slots; model has no linear group")return 0slots = num_slots if num_slots is not None else _linear_pool_num_slots(config)- return linear_state_bytes_per_req(linear_group, config.tp_info.size, config.dtype) * slots+ per_req = linear_state_bytes_per_req(+ linear_group, config.tp_info.size, config.dtype, slot_states+ )+ return per_req * slotsdef _linear_pool_num_slots(config) -> int:diff --git a/python/freetoken/kvcache/qsa_pool.py b/python/freetoken/kvcache/qsa_pool.pynew file mode 100644index 00000000..fddcdbd3--- /dev/null+++ b/python/freetoken/kvcache/qsa_pool.py@@ -0,0 +1,212 @@+"""QSA compressed-block sparse KV pool: paged GQA K/V + compressed index keys + pending ring.++Qwen3.8-Flash-Next scores whole ``index_ratio``-token groups instead of single tokens, so+its indexer slab holds ONE compressed key row per group, addressed by ``slot //+index_ratio``. Because ``page_size % index_ratio == 0``, a group's tokens always live in one+page at consecutive slots, which makes that division well-defined: the compressed rows are a+1/ratio shadow of the K/V pages and follow page sharing and eviction for free -- no+allocator, no free, no clear (SGLang qsa_kv_pool / vLLM compressed-region precedent).++Two tiers ride alongside the shadow slab and are NOT per-token:+- ``pending_ring``: the last ``ring_capacity`` pre-RoPE index keys of each running request (sized by ``ring_capacity_for``), indexed by ``Req.table_idx``. A group that straddles two forwards (chunked prefill, and+ every decode step) reads its already-consumed members from here. Never cleared: a new+ tenant of a table_idx starts at a group boundary (cached_len is 0 or a page multiple), so+ its first closing group takes every member from its own forward.+- scratch rows at ``cmp_scratch_base``: one row per request slot, the write target for rows+ whose group does not close in this forward, so the compress kernel scatters unconditionally+ with no negative index and no cross-row conflict (DSV4 precedent).++The slab is amortized into the per-token KV price (``unit_bytes``); the ring and scratch are+fixed and priced through ``kv_cost``'s ``fixed_cache_size``.+"""++from __future__ import annotations++import math+from typing import Sequence++import torch++from .mha_pool import MHAKVCache++# The index tiers are always 2-byte (compute dtype); spec_kv_bytes_per_token budgets the same.+_INDEX_DTYPE_BYTES = 2+++class QSAKVCache(MHAKVCache):+ """MHA paged pool + the compressed index-key slab + the per-request pending ring.++ ``cmp_k_cache(slot)`` is row-flat ``[num_pages * page_size // index_ratio + num_req_slots,+ index_head_dim]``: row ``r < cmp_scratch_base`` holds the compressed key of the token group+ whose K/V slots are ``[r * index_ratio, (r + 1) * index_ratio)``, and the rows from+ ``cmp_scratch_base`` on are the per-request-slot scratch sinks. ``slot`` is the sparse+ layer's order in the attention backend, same convention as BSAKVCache/DSAKVCache.+ """++ @classmethod+ def ring_capacity_for(cls, index_ratio: int, num_speculative_tokens: int = 0) -> int:+ """Ring depth: one row per pending position, keyed ``position % capacity``; spec decode widens by the draft depth (vLLM sizing)."""+ return index_ratio * math.ceil((index_ratio + num_speculative_tokens) / index_ratio)++ def __init__(+ self,+ num_kv_heads: int,+ num_layers: int,+ head_dim: int,+ num_pages: int,+ page_size: int,+ dtype: torch.dtype,+ device: torch.device,+ index_head_dim: int,+ num_index_layers: int,+ index_ratio: int,+ num_req_slots: int,+ ring_capacity: int | None = None,+ layer_ids: Sequence[int] | None = None,+ ) -> None:+ if index_ratio < 1 or page_size % index_ratio != 0:+ # slot // index_ratio only names one group when a group never straddles a page.+ raise ValueError(+ f"QSA needs page_size ({page_size}) divisible by index_ratio ({index_ratio})"+ )+ if ring_capacity is None:+ ring_capacity = self.ring_capacity_for(index_ratio)+ if ring_capacity < index_ratio:+ # A closing group reads up to index_ratio - 1 past members plus this forward's.+ raise ValueError(+ f"QSA needs ring_capacity ({ring_capacity}) >= index_ratio ({index_ratio})"+ )+ # Index keys ride the compute dtype (the model's index_k is engine-dtype). The KV cost+ # model budgets 2 bytes per token per index layer for the slab+ # (base.spec_kv_bytes_per_token); keep the two in lockstep.+ assert dtype.itemsize == _INDEX_DTYPE_BYTES, (+ f"QSA index slab budgets 2 bytes/token (spec_kv_bytes_per_token); got {dtype}"+ )+ self._index_head_dim = index_head_dim+ self._num_index_layers = num_index_layers+ self._index_ratio = index_ratio+ self._num_req_slots = num_req_slots+ self._ring_capacity = ring_capacity+ self._index_dtype = dtype+ self._page_size = page_size+ super().__init__(+ num_kv_heads=num_kv_heads,+ num_layers=num_layers,+ head_dim=head_dim,+ num_pages=num_pages,+ page_size=page_size,+ dtype=dtype,+ device=device,+ layer_ids=layer_ids,+ )+ self._zero_kv_slabs()+ self._alloc_index_tiers(num_pages)++ def _zero_kv_slabs(self) -> None:+ # Defense-in-depth: the attend kernels pos-mask every K/V load (the real fix for+ # torch.empty's recycled NaN/Inf bit patterns), but a zeroed slab keeps any future+ # unmasked read finite instead of model-poisoning. One memset per (re)allocation.+ self._kv_buffer.zero_()++ def _alloc_index_tiers(self, num_pages: int) -> None:+ # ZERO-initialized: the score kernel reads whole rows of blocks unmasked and relies on+ # never-written tail rows dotting to a finite 0. Written rows are never cleared again,+ # so the kernel must clamp visible blocks to kvlen // index_ratio.+ self._cmp_scratch_base = num_pages * self._page_size // self._index_ratio+ self._cmp_k_buffer = torch.zeros(+ self._num_index_layers,+ self._cmp_scratch_base + self._num_req_slots,+ self._index_head_dim,+ dtype=self._index_dtype,+ device=self._device,+ )+ self._pending_ring = torch.zeros(+ self._num_req_slots,+ self._num_index_layers,+ self._ring_capacity,+ self._index_head_dim,+ dtype=self._index_dtype,+ device=self._device,+ )++ def rebuild(self, num_pages: int) -> None:+ # Free the index tiers BEFORE the K/V realloc (super().rebuild frees + syncs ++ # empty_cache), then re-derive them at the new page count. If the index alloc itself+ # fails (OOM), null the K/V slab too and re-raise: a pool with a grown K/V slab and no+ # index slab would mis-serve silently. Rebuild is idle-only, so zeroing the ring here+ # cannot drop a live request's pending members.+ self._cmp_k_buffer = None+ self._pending_ring = None+ super().rebuild(num_pages)+ self._zero_kv_slabs()+ try:+ self._alloc_index_tiers(num_pages)+ except Exception:+ self._kv_buffer = None+ self._k_buffer = None+ self._v_buffer = None+ raise++ @classmethod+ def kv_cost(cls, config) -> tuple[int, int, int, int]:+ from .base import spec_kv_bytes_per_token+ from freetoken.attention import AttnType++ num_req_slots = config.max_running_req + 1+ per_token = 0+ fixed = 0+ for spec in config.model_config.kv_cache_group_specs():+ if spec.is_swa:+ continue+ per_token += spec_kv_bytes_per_token(spec, config)+ if spec.attn_type is AttnType.QSA:+ # One index-key row = all index layers at one position.+ row = spec.index_head_dim * spec.num_index_layers * _INDEX_DTYPE_BYTES+ fixed += num_req_slots * row * (cls.ring_capacity_for(spec.index_ratio) + 1)+ return per_token * config.page_size, fixed, config.page_size, 0++ def unit_bytes(self) -> tuple[int, int]:+ # Only the shadow slab scales with pages, and only its non-scratch rows; the ring and+ # the scratch rows are the fixed term kv_cost reports separately.+ kv, swa = super().unit_bytes()+ tokens = int(self._kv_buffer.shape[2]) * int(self._kv_buffer.shape[3])+ slab = (+ self._num_index_layers+ * self._cmp_scratch_base+ * self._index_head_dim+ * self._index_dtype.itemsize+ )+ return kv + slab // tokens, swa++ def cmp_k_cache(self, slot: int) -> torch.Tensor:+ """Compressed index keys of one sparse layer: ``[rows, index_head_dim]``."""+ return self._cmp_k_buffer[slot]++ def pending_ring(self, slot: int) -> torch.Tensor:+ """One sparse layer's pending ring: ``[num_req_slots, ring_capacity, index_head_dim]``."""+ return self._pending_ring[:, slot]++ @property+ def cmp_scratch_base(self) -> int:+ """First scratch row of ``cmp_k_cache``; row ``cmp_scratch_base + table_idx`` sinks a+ forward whose group does not close."""+ return self._cmp_scratch_base++ @property+ def index_ratio(self) -> int:+ return self._index_ratio++ @property+ def index_head_dim(self) -> int:+ return self._index_head_dim++ @property+ def ring_capacity(self) -> int:+ return self._ring_capacity++ @property+ def num_req_slots(self) -> int:+ return self._num_req_slots+++__all__ = ["QSAKVCache"]diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex f6105e1f..229cce81 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -97,6 +97,9 @@ class KVCacheGroupSpec:mla: bool = Falseindex_head_dim: int = 0num_index_layers: int = 0+ # QSA compression: one index-key row per index_ratio tokens (1 keeps the BSA/DSA+ # per-token slab). The pool factory and the cost model divide by the same value.+ index_ratio: int = 1# Attention-type taxonomy value for this group; drives the backend capability# matrix and (with the pool factory) selects the KV pool family.attn_type: AttnType = AttnType.FULL@@ -134,6 +137,9 @@ class FullAttentionGroupConfig(BaseAttentionGroupConfig):mla: bool = Falseindex_head_dim: int = 0num_index_layers: int = 0+ # QSA compression ratio (> 1 -> AttnType.QSA): index-key rows are per token group,+ # so the index slab costs index_head_dim * num_index_layers * 2 // index_ratio per token.+ index_ratio: int = 1@dataclass(frozen=True)@@ -157,7 +163,8 @@ class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig):key_head_dim: intvalue_head_dim: intconv_kernel_dim: int- output_gate: bool+ # Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated.+ output_gate: str@dataclass(frozen=True)@@ -185,16 +192,33 @@ AttentionGroupConfig: TypeAlias = (def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType:# Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache,- # GQA (non-mla) + index slab -> BSAKVCache (MiniMax-M3 block-sparse attention).+ # GQA (non-mla) + index slab -> QSAKVCache when the index keys are compressed+ # (index_ratio > 1, Qwen3.8-Flash-Next) else BSAKVCache (MiniMax-M3 block-sparse).if not group.mla:if group.index_head_dim > 0 and group.num_index_layers > 0:- return AttnType.BSA+ return AttnType.QSA if group.index_ratio > 1 else AttnType.BSAreturn AttnType.FULLif group.index_head_dim > 0 and group.num_index_layers > 0:return AttnType.DSAreturn AttnType.MLA+@dataclass(frozen=True)+class SlotStateSpec:+ """One extra per-request tensor riding the LinearStatePool slots.++ Allocated as ``[max(1, len(layer_ids)), num_slots, *shape]`` and advanced, snapshot,+ COW'd and rebuilt with the GDN state; the owner reads it back through+ ``pool.slot_state(name, layer_id)``. ``shape`` is per slot and TP-replicated.+ """++ name: str+ shape: Tuple[int, ...]+ layer_ids: Tuple[int, ...] = ()+ dtype: Any | None = None # a torch dtype; None -> the pool's compute dtype+ fill_value: float = 0.0++@dataclass(frozen=True)class ModelConfig:num_layers: int@@ -288,9 +312,16 @@ class ModelConfig:# swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine# code; None for every other model.m3_args: Any | None = None+ # Qwen3.8-Flash-Next (qwen4_exp) payload (Qwen4ExpArgs): hyper-connection widths, PLE+ # n-gram embedding geometry and the QSA indexer scoring geometry the model module+ # needs. Opaque to model-agnostic engine code; None for every other model.+ qwen4_args: Any | None = None# Generic execution-path capability flags (set by a model's parse_config) so the engine and# factories stay model-agnostic instead of branching on dsv4_args:single_stream_only: bool = False # model runs one sequence at a time -> force bs=1+ # Extra per-request tensors riding the LinearStatePool slots (see SlotStateSpec);+ # () for models without any. Requires a linear-attention group to ride on.+ slot_states: Tuple[SlotStateSpec, ...] = ()@propertydef is_moe(self) -> bool:@@ -409,6 +440,7 @@ class ModelConfig:mla=group.mla,index_head_dim=group.index_head_dim,num_index_layers=group.num_index_layers,+ index_ratio=group.index_ratio,attn_type=_full_group_attn_type(group),))diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.pyindex dc16cfff..2ac4b607 100644--- a/python/freetoken/models/qwen3_5_moe/config.py+++ b/python/freetoken/models/qwen3_5_moe/config.py@@ -152,7 +152,7 @@ def parse_config(hf_config: Any) -> ModelConfig:or getattr(text, "partial_rotary_factor", None)or 1.0)- rotary_dim = round(head_dim * partial)+ rotary_dim = int(head_dim * partial)# For text-only with the default rope type, partial NeoX rope needs no scaling dict# (the mRoPE params reduce to standard partial rope for text). Avoid carrying the@@ -218,7 +218,7 @@ def parse_config(hf_config: Any) -> ModelConfig:key_head_dim=text.linear_key_head_dim,value_head_dim=text.linear_value_head_dim,conv_kernel_dim=text.linear_conv_kernel_dim,- output_gate=True,+ output_gate="silu",)# Order groups by their first layer id for deterministic iteration.groups = tuple(@@ -244,7 +244,7 @@ def parse_config(hf_config: Any) -> ModelConfig:num_experts_per_tok=getattr(text, "num_experts_per_tok", 0),moe_intermediate_size=getattr(text, "moe_intermediate_size", 0),shared_expert_intermediate_size=getattr(text, "shared_expert_intermediate_size", 0),- norm_topk_prob=bool(getattr(text, "norm_topk_prob", False)),+ norm_topk_prob=True,moe_enabled=moe_enabled,use_qk_norm=True,model_type=getattr(hf_config, "model_type", "qwen3_5_moe"),diff --git a/python/freetoken/models/qwen3_5_moe/moe.py b/python/freetoken/models/qwen3_5_moe/moe.pyindex fc0bb7c2..b5ab1cf3 100644--- a/python/freetoken/models/qwen3_5_moe/moe.py+++ b/python/freetoken/models/qwen3_5_moe/moe.py@@ -69,7 +69,10 @@ class Qwen3_5MoE(BaseOP):"fp8_block" if getattr(config, "expert_quant", "none") == "fp8_block" else "bf16")self.experts = make_moe_layer(- config, layer_id=layer_id, renormalize=True, weight_format=weight_format+ config,+ layer_id=layer_id,+ renormalize=config.norm_topk_prob,+ weight_format=weight_format,)self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)self.shared_expert = _SharedExpert(diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex b3414089..d07f18cd 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -911,11 +911,17 @@ def _build_fp8_expert_banks(B = 128L, E, H, I, dense = _moe_dims(config)++ # 16B-align the per-expert scale rows (Qwen3.8: down_scale is 20x5 bf16 = 200 B) so the+ # fused multi-bank copy engages; the GEMMs read scales through explicit strides, so the+ # padding is inert. Unconditional: one layout per format, shared with the byte formulas.+ from freetoken.moe.offload_cache import fp8_block_scale_pad as _pad_cols+specs = {"gate_up": ((E, 2 * I, H), FP8),- "gate_up_scale": ((E, 2 * I // B, H // B), torch.bfloat16),+ "gate_up_scale": ((E, 2 * I // B, _pad_cols(2 * I // B, H // B)), torch.bfloat16),"down": ((E, H, I), FP8),- "down_scale": ((E, H // B, I // B), torch.bfloat16),+ "down_scale": ((E, H // B, _pad_cols(H // B, I // B)), torch.bfloat16),}hb = Noneif pin:@@ -950,8 +956,9 @@ def _build_fp8_expert_banks((gate_up[li][e, :I] if proj == "gate" elsegate_up[li][e, I:] if proj == "up" else down[li][e]).copy_(t)else: # weight_scale_inv- (gate_up_scale[li][e, : I // B] if proj == "gate" else- gate_up_scale[li][e, I // B:] if proj == "up" else down_scale[li][e]).copy_(t)+ (gate_up_scale[li][e, : I // B, : H // B] if proj == "gate" else+ gate_up_scale[li][e, I // B :, : H // B] if proj == "up" else+ down_scale[li][e, :, : I // B]).copy_(t)return liif parallel is None:diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.pynew file mode 100644index 00000000..04c29d77--- /dev/null+++ b/python/freetoken/models/qwen4_exp/__init__.py@@ -0,0 +1,35 @@+"""Qwen3.8-Flash-Next (model_type qwen4_exp), served text-only.++48 decoder layers on hc_count=4 hyper-connection residual streams R [T, 4*hidden]:+embed -> repeat(1, 4) -> [PLE at zero-based layer 1] -> per layer attn_hc.mix -> (GDN | QSA) -> attn_hc.combine -> mlp_hc.mix -> MoE -> mlp_hc.combine -> top-level mixer.mix -> lm_head.+Layer contract: forward(R [T, 4*hidden], batch) -> R' [T, 4*hidden].++Contracts shared across modules (do not rename):+- The PLE dilated-conv left context lives on the LinearStatePool slots as the declared slot state ``ple_conv`` (config.ple_slot_states -> ModelConfig.slot_states), read back with ``pool.slot_state("ple_conv", layer_id)``; same slot / COW / snapshot lifecycle as conv_states / recurrent_states.+- kvcache.qsa_pool.QSAKVCache(MHAKVCache): ``cmp_k_cache(slot) -> [rows, index_head_dim]`` (compressed index keys, row = kv slot // index_ratio), ``pending_ring(slot) -> [num_req_slots, ring_capacity, index_head_dim]`` (per-request pre-RoPE index-k tail indexed by table_idx, never cleared), ``cmp_scratch_base`` (int, first scratch row for non-closing decode writes). ``slot`` is the sparse layer's order in the attention backend.+"""++from .config import parse_config+from .model import Qwen4ExpForCausalLM+from .weight import (+ iter_weights,+ load_nvfp4_expert_sources,+ load_nvfp4_expert_sources_parallel,+ load_ple_table,+)++# Official FP8 checkpoints share qwen3_5_moe's block-fp8 expert layout (same+# model.language_model.layers.* keys), so reuse its bank hook; for every other+# expert_quant it defers to the per-quant providers, which resolve this module's+# load_nvfp4_expert_sources via the model spec.+from freetoken.models.qwen3_5_moe.weight import setup_offload_expert_banks++__all__ = [+ "Qwen4ExpForCausalLM",+ "iter_weights",+ "load_nvfp4_expert_sources",+ "load_nvfp4_expert_sources_parallel",+ "load_ple_table",+ "parse_config",+ "setup_offload_expert_banks",+]diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.pynew file mode 100644index 00000000..d6ab2867--- /dev/null+++ b/python/freetoken/models/qwen4_exp/attention.py@@ -0,0 +1,241 @@+"""QSA full-attention layer for Qwen3.8-Flash-Next (12 of 48 layers).++Gated GQA (24 q heads / 2 kv heads / head_dim 256, per-head zero-centered q/k norms, partial NeoX+rope over 64 dims, ``q_proj`` twice as wide for the output gate) plus the weights of the QSA+indexer (``index_qk_proj`` [640, 2560] = 4 index q heads x 128 then 1 index k head x 128, and the+two per-head index norms).++Model/backend split, same shape as MiniMax-M3's ``bsa_forward``: the layer owns the weights and+hands the backend the RAW index projections; the backend owns everything stateful (compressed key+slab, pending ring, scoring, top-k, expansion, sparse attend). The index k norm runs AFTER the+fp32 mean over each group of ``index_ratio`` raw keys, so it cannot be applied here -- both index+norm weights travel with the call (:class:`QSAIndexerInputs`).+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import TYPE_CHECKING, Protocol++import torch+from freetoken.core import get_global_ctx+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm, LinearColParallelMerged, LinearReplicated+from freetoken.layers.rotary import get_rope+from freetoken.utils import nvtx_annotate++if TYPE_CHECKING:+ from freetoken.core import Batch+ from freetoken.models.config import ModelConfig+++@dataclass(frozen=True)+class QSAIndexerInputs:+ """Everything the QSA backend needs from the indexer for one layer's forward.++ Frozen contract. ``q``/``k`` are the raw ``index_qk_proj`` slices: no norm, no rope. The+ backend applies, per HF ``Qwen4ExpTextQSAIndexer`` (modeling_qwen4_exp.py:611)::++ q_h = rope64(rmsnorm(q_h) * (1 + q_norm_weight), pos = query position)+ kbar_b = rope64(rmsnorm(mean_fp32(k[4b:4b+4])) * (1 + k_norm_weight), pos = 4b)+ s_b = sum_h relu(<q_h, kbar_b>) / sqrt(index_head_dim)++ and the pending ring stores ``k`` PRE-norm and PRE-rope, because a group's mean is only final+ once all ``index_ratio`` members exist. rope64 is ``get_rope(index_head_dim,+ config.rotary_config.rotary_dim, ...)`` -- the same frequencies as the main attention, a+ different ``head_size``, so the backend builds its own (cached) instance.+ """++ q: torch.Tensor # [T, index_n_heads, index_head_dim]+ k: torch.Tensor # [T, index_head_dim]+ q_norm_weight: torch.Tensor # [index_head_dim], zero-centered: scale is (1 + w), fp32+ k_norm_weight: torch.Tensor # [index_head_dim], zero-centered+ eps: float+++class QSAAttentionBackend(Protocol):+ """The hook ``Qwen4ExpAttention`` calls; ``attention/qsa_sparse.py`` implements it.++ ``q`` is [T, num_qo_heads, head_dim] and ``k``/``v`` are [T, num_kv_heads*head_dim], all post+ norm+rope and in the model dtype; the return is [T, num_qo_heads, head_dim] (the layer applies+ the output gate and ``o_proj``). ``layer_id`` is the decoder layer id; the backend maps it to+ its own sparse-layer slot. Everything else -- KV store, per-request lengths, page rows -- comes+ from ``batch`` exactly as for ``BaseAttnBackend.forward``.+ """++ def qsa_forward(+ self,+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ index: QSAIndexerInputs,+ layer_id: int,+ batch: Batch,+ ) -> torch.Tensor: ...+++class Qwen4ExpIndexer(BaseOP):+ """QSA indexer weights (checkpoint prefix ``self_attn.indexer``); the scoring lives in the backend."""++ def __init__(self, config: ModelConfig, layer_id: int) -> None:+ args = config.qwen4_args+ self.layer_id = layer_id+ self.num_heads = args.index_n_heads+ self.num_kv_heads = args.index_kv_heads+ self.head_dim = args.index_head_dim+ self.eps = config.rms_norm_eps+ self._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim]+ self.index_qk_proj = LinearReplicated(args.hidden_size, sum(self._split), has_bias=False)+ self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)+ self.k_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)++ def forward(self, x: torch.Tensor) -> QSAIndexerInputs:+ q, k = self.index_qk_proj.forward(x).split(self._split, dim=-1)+ return QSAIndexerInputs(+ q=q.reshape(-1, self.num_heads, self.head_dim).contiguous(),+ k=k.reshape(-1, self.head_dim).contiguous(),+ q_norm_weight=self.q_layernorm.weight,+ k_norm_weight=self.k_layernorm.weight,+ eps=self.eps,+ )+++class Qwen4ExpAttention(BaseOP):+ """Gated GQA with a QSA indexer::++ q, gate = chunk(q_proj(x).view(-1, num_q, 2*head_dim), 2, -1)+ q, k = rope(q_norm(q), k_norm(k_proj(x))) # first rotary_dim dims+ o = backend.qsa_forward(q, k, v_proj(x), indexer(x), layer_id, batch)+ out = o_proj(o * sigmoid(gate))++ q/k/v are one merged GEMM (``qkv_proj``, split ``[num_q*head_dim*2, kv, kv]``); the checkpoint+ ships ``q_proj``/``k_proj``/``v_proj`` separately, so the loader concatenates along dim 0.+ Other keys keep the checkpoint names: ``o_proj.weight``, ``q_norm.weight``, ``k_norm.weight``+ (both zero-centered, loaded RAW), ``indexer.*``.+ """++ def __init__(self, config: ModelConfig, layer_id: int) -> None:+ self.layer_id = layer_id+ self.num_q = config.num_qo_heads+ self.num_kv = config.num_kv_heads+ self.head_dim = config.head_dim+ self.qo_attn_dim = self.num_q * self.head_dim+ self.kv_attn_dim = self.num_kv * self.head_dim+ self._qkv_split = [self.qo_attn_dim * 2, self.kv_attn_dim, self.kv_attn_dim]+ self.qkv_proj = LinearColParallelMerged(+ config.hidden_size, self._qkv_split, has_bias=False+ )+ self.o_proj = LinearReplicated(self.qo_attn_dim, config.hidden_size, has_bias=False)+ self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)+ self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)+ rotary = config.rotary_config+ self.rotary = get_rope(+ head_dim=self.head_dim,+ rotary_dim=rotary.rotary_dim,+ max_position=rotary.max_position,+ base=rotary.base,+ rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,+ )+ self.indexer = Qwen4ExpIndexer(config, layer_id)++ @nvtx_annotate("QSA")+ def forward(self, x: torch.Tensor, batch: Batch) -> torch.Tensor:+ qg, k, v = self.qkv_proj.forward(x).split(self._qkv_split, dim=-1)+ qg = qg.view(-1, self.num_q, self.head_dim * 2)+ q = qg[..., : self.head_dim].contiguous()+ gate = qg[..., self.head_dim :].reshape(-1, self.qo_attn_dim)+ k = k.contiguous().view(-1, self.num_kv, self.head_dim)+ v = v.contiguous()+ self.q_norm.forward_inplace(q)+ self.k_norm.forward_inplace(k)+ q, k = self.rotary.forward(+ batch.positions, q.view(-1, self.qo_attn_dim), k.view(-1, self.kv_attn_dim)+ )+ index = self.indexer.forward(x)+ o = get_global_ctx().attn_backend.qsa_forward(+ q.view(-1, self.num_q, self.head_dim), k, v, index, self.layer_id, batch+ )+ gated = o.reshape(-1, self.qo_attn_dim) * torch.sigmoid(gate)+ return self.o_proj.forward(gated)+++class TorchDenseQSAReference:+ """Dense oracle for :class:`QSAAttentionBackend` (fp32 math): attend to every visible token.++ QSA is exactly dense while a request sees at most ``index_budget + index_ratio - 1`` tokens+ (every complete block is selected), so this doubles as the equivalence oracle for the sparse backend. It+ keeps its own ``[slot, position]`` KV instead of a paged pool, so it needs no engine wiring;+ it is a test/reference object and is never registered as an attention backend.+ """++ def __init__(+ self,+ config: ModelConfig,+ num_slots: int,+ max_len: int,+ device: torch.device,+ dtype: torch.dtype,+ ) -> None:+ self.num_kv = config.num_kv_heads+ self.head_dim = config.head_dim+ self.sm_scale = config.attn_sm_scale or self.head_dim**-0.5+ self._cache: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}+ self._shape = (num_slots, max_len, self.num_kv, self.head_dim)+ self._device = device+ self._dtype = dtype++ def _layer_cache(self, layer_id: int) -> tuple[torch.Tensor, torch.Tensor]:+ if layer_id not in self._cache:+ self._cache[layer_id] = tuple(+ torch.zeros(self._shape, device=self._device, dtype=self._dtype) for _ in range(2)+ )+ return self._cache[layer_id]++ def qsa_forward(+ self,+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ index: QSAIndexerInputs,+ layer_id: int,+ batch: Batch,+ ) -> torch.Tensor:+ del index+ k_cache, v_cache = self._layer_cache(layer_id)+ k = k.view(-1, self.num_kv, self.head_dim)+ v = v.view(-1, self.num_kv, self.head_dim)+ out = torch.empty_like(q)+ offset = 0+ for r in batch.padded_reqs:+ n, slot, prefix = r.extend_len, r.table_idx, r.cached_len+ rows = slice(offset, offset + n)+ k_cache[slot, prefix : prefix + n] = k[rows]+ v_cache[slot, prefix : prefix + n] = v[rows]+ out[rows] = self._attend(+ q[rows], k_cache[slot, : prefix + n], v_cache[slot, : prefix + n], prefix+ )+ offset += n+ return out++ def _attend(+ self, q: torch.Tensor, keys: torch.Tensor, values: torch.Tensor, prefix: int+ ) -> torch.Tensor:+ n, num_q, _ = q.shape+ total = keys.shape[0]+ rep = num_q // self.num_kv+ keys = keys.repeat_interleave(rep, dim=1).float()+ values = values.repeat_interleave(rep, dim=1).float()+ scores = torch.einsum("qhd,khd->hqk", q.float(), keys) * self.sm_scale+ visible = torch.arange(total, device=q.device) <= (+ prefix + torch.arange(n, device=q.device)+ ).unsqueeze(-1)+ scores = scores.masked_fill(~visible, float("-inf"))+ return torch.einsum("hqk,khd->qhd", scores.softmax(-1), values).to(q.dtype)+++__all__ = [+ "QSAAttentionBackend",+ "QSAIndexerInputs",+ "Qwen4ExpAttention",+ "Qwen4ExpIndexer",+ "TorchDenseQSAReference",+]diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.pynew file mode 100644index 00000000..bb5d1dff--- /dev/null+++ b/python/freetoken/models/qwen4_exp/config.py@@ -0,0 +1,293 @@+from __future__ import annotations++from dataclasses import dataclass+from fnmatch import fnmatch+from typing import Any, Tuple++import torch++from freetoken.models.config import (+ FullAttentionGroupConfig,+ LinearGatedDeltaGroupConfig,+ ModelConfig,+ RotaryConfig,+ SlotStateSpec,+)+++@dataclass(frozen=True)+class Qwen4ExpArgs:+ """Qwen3.8-Flash-Next geometry beyond the generic ModelConfig fields (ModelConfig.qwen4_args)."""++ hidden_size: int+ # Hyper-connections: every layer reads/writes hc_count residual streams [T, hc_count*hidden].+ hc_count: int+ hc_lowrank: int+ # PLE n-gram embedding; layer ids are zero-based decoder layers.+ ple_layer_ids: Tuple[int, ...]+ ple_embed_dim: int+ ple_conv_kernel_size: int+ ngram_size: int+ heads_per_ngram: int+ ngram_vocab_size_base: int+ make_ngram_vocab_size_divisible_by: int+ split_ngram_parts: int+ # n-gram hash windows never cross this token (the eos id); they restart after it.+ ngram_boundary_token_id: int+ # QSA indexer scoring geometry (the slab/ratio geometry lives on the attention group).+ index_n_heads: int+ index_kv_heads: int+ index_head_dim: int+ index_budget: int+ index_ratio: int++ @property+ def index_topk_blocks(self) -> int:+ return self.index_budget // self.index_ratio++ @property+ def num_ngram_heads(self) -> int:+ # one head group per n-gram order 2..ngram_size (Qwen3.8: 8 x 2-gram + 8 x 3-gram)+ return (self.ngram_size - 1) * self.heads_per_ngram++ @property+ def ngram_head_dim(self) -> int:+ return self.ple_embed_dim // self.num_ngram_heads++ @property+ def ple_conv_dilation(self) -> int:+ # HF Qwen4ExpTextPLELayer sets the depthwise conv dilation to ngram_size+ return self.ngram_size++ @property+ def ple_conv_state_len(self) -> int:+ return (self.ple_conv_kernel_size - 1) * self.ple_conv_dilation++ @property+ def ple_state_width(self) -> int:+ return self.hc_count * self.hidden_size+++PLE_CONV_STATE = "ple_conv"+PLE_NGRAM_STATE = "ple_ngram_ctx"+++def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]:+ """Per-request PLE state riding the linear-state slots (see LinearStatePool.slot_states)."""+ if not args.ple_layer_ids:+ return ()+ return (+ # dilated-conv left context; replicated (not TP-sharded), model dtype+ SlotStateSpec(+ name=PLE_CONV_STATE,+ shape=(args.ple_state_width, args.ple_conv_state_len),+ layer_ids=args.ple_layer_ids,+ ),+ # last ngram_size-1 token ids, shared by every PLE layer; eos = hash boundary+ SlotStateSpec(+ name=PLE_NGRAM_STATE,+ shape=(args.ngram_size - 1,),+ dtype=torch.int32,+ fill_value=float(args.ngram_boundary_token_id),+ ),+ )+++def _quant_get(hf_config: Any):+ quant = getattr(hf_config, "quantization_config", None)+ if quant is None:+ return None+ return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))+++def _ignored(patterns, module_name: str) -> bool:+ return any(fnmatch(module_name, pat) for pat in patterns)+++def _layer_types(text: Any) -> list[str]:+ layer_types = getattr(text, "layer_types", None)+ if layer_types is not None:+ # HF Qwen4ExpTextConfig rewrites full_attention to qwen_sparse_attention in __post_init__.+ return [+ "full_attention" if t == "qwen_sparse_attention" else t for t in layer_types+ ]+ # Fall back to full_attention_interval: every Nth layer (1-indexed) is full.+ interval = int(getattr(text, "full_attention_interval", 4))+ n = int(text.num_hidden_layers)+ return [+ "full_attention" if (i + 1) % interval == 0 else "linear_attention"+ for i in range(n)+ ]+++def parse_config(hf_config: Any) -> ModelConfig:+ text = getattr(hf_config, "text_config", hf_config)++ head_dim = (+ getattr(text, "head_dim", None)+ or text.hidden_size // text.num_attention_heads+ )+ num_kv_heads = getattr(text, "num_key_value_heads", text.num_attention_heads)++ rope_params = getattr(text, "rope_parameters", None) or {}+ rope_theta = rope_params.get("rope_theta", getattr(text, "rope_theta", None))+ partial = (+ rope_params.get("partial_rotary_factor")+ or getattr(text, "partial_rotary_factor", None)+ or 1.0+ )+ # int(), not round(): HF configuration_qwen4_exp truncates head_dim * partial.+ rotary_dim = int(head_dim * partial)++ # Text-only serving with the default rope type: the mRoPE sections reduce to standard+ # partial rope, and the unhashable ``mrope_section`` list must not reach get_rope's+ # cache key.+ rope_type = rope_params.get("rope_type", "default")+ rope_scaling = (+ None+ if rope_type in (None, "default")+ else {k: v for k, v in rope_params.items() if not isinstance(v, (list, dict))}+ )++ get = _quant_get(hf_config)+ if get is None:+ expert_quant = attn_quant = dense_quant = lm_head_quant = "none"+ else:+ algo = str(get("quant_algo") or get("quant_method") or "").lower()+ block = get("weight_block_size")+ if algo == "fp8" and block:+ # Official FP8 build (DeepSeek-V3-style block-fp8): only the routed experts+ # are quantized (fp8-e4m3 weights + per-block weight_scale_inv); attention,+ # GDN, the shared expert, HC, PLE and lm_head stay bf16.+ bs = tuple(int(x) for x in block)+ assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"+ expert_quant = "fp8_block"+ attn_quant = dense_quant = lm_head_quant = "none"+ else:+ is_fp4 = "fp4" in algo+ ignore = list(get("ignore") or [])++ # The RadixArk NVFP4 build quantizes only the routed experts; attention/GDN,+ # the shared expert, HC, PLE and lm_head all sit in the modelopt ignore list+ # and stay bf16. Derive every flag from that list instead of assuming the split.+ def _quant(probe: str) -> str:+ return "nvfp4" if is_fp4 and not _ignored(ignore, probe) else "none"++ prefix = "model.language_model.layers.0"+ expert_quant = _quant(f"{prefix}.mlp.experts.0.gate_proj")+ dense_quant = _quant(f"{prefix}.mlp.shared_expert.gate_proj")+ attn_quant = _quant(f"{prefix}.self_attn.q_proj")+ lm_head_quant = _quant("lm_head")++ layer_types = _layer_types(text)+ full_ids = tuple(i for i, t in enumerate(layer_types) if t == "full_attention")+ linear_ids = tuple(i for i, t in enumerate(layer_types) if t == "linear_attention")++ # HF stores ple_layer_ids one-indexed (validated upstream as [1, num_layers]).+ ple_layer_ids = tuple(int(i) - 1 for i in (getattr(text, "ple_layer_ids", None) or ()))+ for lid in ple_layer_ids:+ if layer_types[lid] != "linear_attention":+ raise ValueError(f"PLE must sit on a linear_attention layer, got layer {lid}")++ full_rotary = RotaryConfig(+ head_dim=head_dim,+ rotary_dim=rotary_dim,+ max_position=text.max_position_embeddings,+ base=rope_theta,+ scaling=rope_scaling,+ )+ full_group = FullAttentionGroupConfig(+ name="full",+ layer_ids=full_ids,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ rotary_config=full_rotary,+ index_head_dim=int(text.indexer_head_dim),+ num_index_layers=len(full_ids),+ index_ratio=int(text.indexer_compress_ratio),+ )+ linear_group = LinearGatedDeltaGroupConfig(+ name="linear",+ layer_ids=linear_ids,+ num_key_heads=text.linear_num_key_heads,+ num_value_heads=text.linear_num_value_heads,+ key_head_dim=text.linear_key_head_dim,+ value_head_dim=text.linear_value_head_dim,+ conv_kernel_dim=text.linear_conv_kernel_dim,+ # HF resolves a null output_gate_type to hidden_act; mirror that instead of+ # stringifying None.+ output_gate=str(getattr(text, "output_gate_type", None) or text.hidden_act),+ )+ # Order groups by their first layer id for deterministic iteration.+ groups = tuple(+ sorted(+ (full_group, linear_group),+ key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,+ )+ )++ num_experts = int(getattr(text, "num_experts", 0) or 0)++ # HF accepts int | list here and uses the first entry (modeling_qwen4_exp Qwen4ExpTextNGramEmbedding)+ eos_token_id = text.eos_token_id+ if isinstance(eos_token_id, (list, tuple)):+ eos_token_id = eos_token_id[0]++ qwen4_args = Qwen4ExpArgs(+ hidden_size=text.hidden_size,+ hc_count=int(text.hc_count),+ hc_lowrank=int(text.hc_lowrank),+ ple_layer_ids=ple_layer_ids,+ ple_embed_dim=int(text.ple_embed_dim),+ ple_conv_kernel_size=int(text.ple_conv_kernel_size),+ ngram_size=int(text.ngram_size),+ heads_per_ngram=int(text.heads_per_ngram),+ ngram_vocab_size_base=int(text.ngram_vocab_size_base),+ make_ngram_vocab_size_divisible_by=int(text.make_ngram_vocab_size_divisible_by),+ split_ngram_parts=int(text.split_ngram_parts),+ ngram_boundary_token_id=int(eos_token_id),+ index_n_heads=int(text.indexer_n_heads),+ index_kv_heads=int(text.indexer_kv_heads),+ index_head_dim=int(text.indexer_head_dim),+ index_budget=int(text.indexer_budget),+ index_ratio=int(text.indexer_compress_ratio),+ )++ return ModelConfig(+ num_layers=text.num_hidden_layers,+ num_qo_heads=text.num_attention_heads,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ hidden_size=text.hidden_size,+ vocab_size=text.vocab_size,+ intermediate_size=getattr(text, "intermediate_size", 0) or 0,+ hidden_act=text.hidden_act,+ rms_norm_eps=text.rms_norm_eps,+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),+ rotary_config=full_rotary,+ num_experts=num_experts,+ num_experts_per_tok=int(getattr(text, "num_experts_per_tok", 0) or 0),+ moe_intermediate_size=int(getattr(text, "moe_intermediate_size", 0) or 0),+ shared_expert_intermediate_size=int(+ getattr(text, "shared_expert_intermediate_size", 0) or 0+ ),+ # Absent from the shipped config; HF Qwen4ExpTextConfig defaults it True and the+ # Qwen3_5MoE block renormalizes unconditionally -- keep the two in agreement.+ norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),+ moe_enabled=num_experts > 0,+ use_qk_norm=True,+ model_type=getattr(hf_config, "model_type", "qwen4_exp"),+ architectures=getattr(hf_config, "architectures", ["Qwen4ExpForConditionalGeneration"]),+ vision_config=None, # served text-only+ image_token_id=getattr(hf_config, "image_token_id", None),+ attention_groups=groups,+ expert_quant=expert_quant,+ attn_quant=attn_quant,+ dense_quant=dense_quant,+ lm_head_quant=lm_head_quant,+ qwen4_args=qwen4_args,+ slot_states=ple_slot_states(qwen4_args),+ )+++__all__ = ["PLE_CONV_STATE", "PLE_NGRAM_STATE", "Qwen4ExpArgs", "parse_config", "ple_slot_states"]diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.pynew file mode 100644index 00000000..69838153--- /dev/null+++ b/python/freetoken/models/qwen4_exp/gdn.py@@ -0,0 +1,233 @@+from __future__ import annotations++import torch+import torch.nn.functional as F+from freetoken.core import get_global_ctx+from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen+from freetoken.layers import BaseOP, LinearColParallelMerged++from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged+from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged+from freetoken.models.qwen3_5_moe.gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla+from freetoken.models.quant_linear import make_replicated_quant+++_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid")+++class _DepthwiseConv1d(BaseOP):+ """Holds the depthwise conv weight ``[conv_dim, 1, K]`` (key ``conv1d.weight``)."""++ def __init__(self, conv_dim: int, kernel: int):+ self.weight = torch.empty(conv_dim, 1, kernel)+++class _GatedRMSNorm(BaseOP):+ """RMSNorm of x followed by an ``activation(z)`` gate (HF Qwen4ExpTextRMSNormGated).++ Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * act(z) in one+ kernel) instead of the unfused pow/mean/rsqrt/mul/act chain, matching sglang's+ ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one.+ Qwen3.8-Flash-Next gates with sigmoid where Qwen3.5 gates with silu."""++ def __init__(self, dim: int, eps: float, activation: str):+ # rms_norm_gated drops the gate entirely (no error) for a name it does not know.+ assert activation in _GATE_ACTIVATIONS, f"unsupported GDN output gate {activation!r}"+ self.weight = torch.empty(dim)+ self.eps = eps+ self.activation = activation++ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.fla import rms_norm_gated++ return rms_norm_gated(+ x=x, weight=self.weight, bias=None, z=z, eps=self.eps,+ is_rms_norm=True, norm_before_gate=True, activation=self.activation,+ )+++class Qwen4ExpGatedDeltaNet(BaseOP):+ """GatedDeltaNet op using the vendored flash-linear-attention triton kernels+ (``freetoken.kernel.fla``) for the recurrence and a per-request+ recurrent + conv state held in ``ctx.linear_state_pool`` (keyed by ``Req.table_idx``).++ Parameter names match HF (``in_proj_qkv``/``in_proj_z``/``in_proj_b``/``in_proj_a``/+ ``conv1d``/``A_log``/``dt_bias``/``norm``/``out_proj``). Handles prefill (incl. chunked+ continuation) and single-token decode; state is fresh when ``req.cached_len == 0``.++ ``output_gate`` is the gate activation name from ``LinearGatedDeltaGroupConfig``+ ("sigmoid" for Qwen3.8-Flash-Next).+ """++ def __init__(+ self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,+ conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid",+ expert_quant: str = "none", attn_quant: str = "none",+ ):+ self.layer_id = layer_id+ # The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as+ # [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the+ # hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head+ # dims are equal. Qwen3.5/3.6/3.8 satisfy this (128/128); guard any future config.+ assert head_k_dim == head_v_dim, (+ f"GatedDeltaNet requires head_k_dim == head_v_dim, got {head_k_dim} != {head_v_dim}"+ )+ self.num_k_heads = num_k_heads+ self.num_v_heads = num_v_heads+ self.head_k_dim = head_k_dim+ self.head_v_dim = head_v_dim+ self.key_dim = num_k_heads * head_k_dim+ self.value_dim = num_v_heads * head_v_dim+ self.conv_dim = 2 * self.key_dim + self.value_dim+ self.conv_kernel_size = conv_kernel_size+ # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8+ # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way+ # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM).+ self._block_fp8 = expert_quant == "fp8_block"+ self._pertensor_fp8 = attn_quant == "fp8_pertensor"+ self._fp8 = self._block_fp8 or self._pertensor_fp8++ self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads]+ if self._fp8:+ ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged+ self.in_proj_qkvz = ColMerged(+ hidden_size, [self.conv_dim, self.value_dim], has_bias=False+ )+ self.in_proj_ba = LinearColParallelMerged(+ hidden_size, [num_v_heads, num_v_heads], has_bias=False+ )+ else:+ # Fused input projection (one GEMM instead of four): qkv | z | b | a.+ self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False)+ self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size)+ # Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive,+ # and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a+ # per-call .float() upcast in the decode wrapper. The weight loader exempts+ # *.A_log / *.dt_bias from the model-dtype downcast.+ self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32)+ self.A_log = torch.empty(num_v_heads, dtype=torch.float32)+ self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate)+ # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors+ # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors+ # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4.+ self.out_proj = make_replicated_quant(+ expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False+ )++ def _gate_params(self, a: torch.Tensor, b: torch.Tensor):+ beta = b.sigmoid()+ g = -self.A_log.exp() * F.softplus(a.float() + self.dt_bias)+ return g, beta++ def _conv_weight(self) -> torch.Tensor:+ return self.conv1d.weight.squeeze(1) # [conv_dim, kernel] for the fused kernel++ def _conv_prefill(self, conv_in, pool, cu_seqlens, cache_indices, has_initial_state) -> torch.Tensor:+ """Varlen causal conv (fused sgl_kernel) with silu; reads/updates each request's+ conv state in place by ``cache_indices`` slot. ``conv_in`` [total, conv_dim].+ ``cu_seqlens`` / ``cache_indices`` / ``has_initial_state`` come from FLAMetadata."""+ li = pool.local_index(self.layer_id)+ x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total]+ out = causal_conv1d_varlen(x, self._conv_weight(), pool.conv_states[li],+ cu_seqlens, cache_indices, has_initial_state)+ return out.transpose(0, 1) # [total, conv_dim]++ def _conv_decode(self, conv_in: torch.Tensor, table_idx: torch.Tensor, pool) -> torch.Tensor:+ """Single-token causal conv update (fused sgl_kernel) by ``table_idx`` slot;+ updates conv state in place, no host loop -> CUDA-graph capturable.+ ``conv_in`` [B, conv_dim] -> silu(conv) [B, conv_dim]."""+ li = pool.local_index(self.layer_id)+ return causal_conv1d_decode(conv_in, pool.conv_states[li], self._conv_weight(), table_idx)++ def _write_track_snapshot(self, pool, li: int, conv_in: torch.Tensor,+ h: torch.Tensor, fla) -> None:+ """Snapshot this layer's recurrent + conv state at the chunk-aligned track boundary+ into a donatable pool slot, on the forward stream (hybrid-radix extra_buffer path).+ SSM: ``recurrent_states[li, dst] = h[0, h_row]`` -- a DIRECT copy (h is [V,K], the+ state pool is [K,V]; they coincide because GDN requires head_k_dim == head_v_dim).+ Conv: the last (kernel-1) raw conv-input timesteps ending at the boundary."""+ rec = pool.recurrent_states[li]+ rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype))+ cv = pool.conv_states[li]+ # conv_in [total, conv_dim]; gather the (kernel-1) window per tracked req.+ conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous() # [nt, conv_dim, K-1]+ cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype))++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ ctx = get_global_ctx()+ batch = ctx.batch+ pool = ctx.linear_state_pool+ total = hidden_states.shape[0]+ dtype = hidden_states.dtype++ # Per-forward GDN metadata (cu_seqlens / cache_indices / continuation flags),+ # built once and shared by all GDN layers. The scheduler/graph set it; build it+ # lazily here (cached on the batch) for direct-op callers (tests).+ fla = batch.fla_metadata+ if fla is None:+ from freetoken.attention.linear import build_fla_metadata++ fla = build_fla_metadata(batch, hidden_states.device)+ batch.fla_metadata = fla++ if self._fp8:+ qkvz = self.in_proj_qkvz.forward(hidden_states)+ conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1)+ ba = self.in_proj_ba.forward(hidden_states)+ b, a = torch.split(ba, [self.num_v_heads, self.num_v_heads], dim=-1)+ else:+ proj = self.in_proj.forward(hidden_states)+ conv_in, z, b, a = torch.split(proj, self._in_proj_split, dim=-1)+ z = z.reshape(total, self.num_v_heads, self.head_v_dim)+ li = pool.local_index(self.layer_id)++ if batch.is_decode:+ # Fused fla decode kernel: gating + in-kernel l2norm + recurrent update ++ # per-request state read/write-by-index, all in one kernel (no gather/scatter,+ # no clone, no external l2norm). q/k stay at num_k_heads (kernel handles GQA).+ mixed = self._conv_decode(conv_in, fla.cache_indices, pool) # [B, conv_dim]+ B = mixed.shape[0]+ qf, kf, vf = torch.split(mixed, [self.key_dim, self.key_dim, self.value_dim], dim=-1)+ q = qf.reshape(1, B, self.num_k_heads, self.head_k_dim).to(dtype)+ k = kf.reshape(1, B, self.num_k_heads, self.head_k_dim).to(dtype)+ v = vf.reshape(1, B, self.num_v_heads, self.head_v_dim).to(dtype)+ core_out = gdn_decode_fla(+ q, k, v, a, b, A_log=self.A_log, dt_bias=self.dt_bias,+ state_source=pool.recurrent_states[li], indices=fla.cache_indices,+ cu_seqlens=fla.cu_seqlens, scale=self.head_k_dim ** -0.5,+ )+ else:+ mixed = self._conv_prefill(+ conv_in, pool, fla.cu_seqlens, fla.cache_indices, fla.has_initial_state)+ # fla chunk handles GQA in-kernel: q/k stay at num_k_heads, v at num_v_heads.+ qf, kf, vf = torch.split(mixed, [self.key_dim, self.key_dim, self.value_dim], dim=-1)+ q = qf.reshape(1, total, self.num_k_heads, self.head_k_dim).to(dtype)+ k = kf.reshape(1, total, self.num_k_heads, self.head_k_dim).to(dtype)+ v = vf.reshape(1, total, self.num_v_heads, self.head_v_dim).to(dtype)+ g, beta = self._gate_params(a, b)+ g = g.reshape(1, total, self.num_v_heads)+ beta = beta.float().reshape(1, total, self.num_v_heads)+ # The chunk kernel reads + writes back initial_state[cache_indices] in place;+ # fresh sequences (cached_len==0) must start from a zeroed slot.+ if fla.fresh_state_indices is not None:+ pool.recurrent_states[li].index_fill_(0, fla.fresh_state_indices, 0.0)+ track = fla.track_dst is not None+ result = gdn_prefill_chunk_fla(+ q, k, v, g, beta,+ state_source=pool.recurrent_states[li], indices=fla.cache_indices,+ cu_seqlens=fla.cu_seqlens, scale=self.head_k_dim ** -0.5,+ return_h=track,+ )+ if track:+ core_out, h = result+ self._write_track_snapshot(pool, li, conv_in, h, fla)+ else:+ core_out = result++ core_out = core_out.reshape(-1, self.head_v_dim)+ z = z.reshape(-1, self.head_v_dim)+ out = self.norm.forward(core_out, z).reshape(total, -1)+ return self.out_proj.forward(out)+++__all__ = ["Qwen4ExpGatedDeltaNet"]diff --git a/python/freetoken/models/qwen4_exp/gdn_reference.py b/python/freetoken/models/qwen4_exp/gdn_reference.pynew file mode 100644index 00000000..fbd9647b--- /dev/null+++ b/python/freetoken/models/qwen4_exp/gdn_reference.py@@ -0,0 +1,279 @@+"""Pure-torch Gated DeltaNet reference (text-only, no cache).++Correctness oracle for the kernel-backed GDN op (``gdn.Qwen4ExpGatedDeltaNet``).+The two delta rules and the forward are transcribed from+``transformers.models.qwen4_exp.modeling_qwen4_exp`` (``torch_chunk_gated_delta_rule``,+``torch_recurrent_gated_delta_rule`` and ``Qwen4ExpTextGatedDeltaNet.forward`` no-cache path).+Qwen3.8-Flash-Next gates the output norm with ``config.output_gate_type`` (sigmoid) where+Qwen3.5 hardcodes silu; the conv keeps ``config.hidden_act`` (silu).+"""++from __future__ import annotations++import torch+import torch.nn as nn+import torch.nn.functional as F++_GATE_ACTS = {"silu": F.silu, "swish": F.silu, "sigmoid": torch.sigmoid}+++def _l2norm(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:+ return x * torch.rsqrt(x.pow(2).sum(dim=-1, keepdim=True) + eps)+++def recurrent_gated_delta_rule(+ query: torch.Tensor, # [B, T, Hv, Dk]+ key: torch.Tensor, # [B, T, Hv, Dk]+ value: torch.Tensor, # [B, T, Hv, Dv]+ g: torch.Tensor, # [B, T, Hv] (log-decay; per-step decay = exp(g))+ beta: torch.Tensor, # [B, T, Hv]+ *,+ initial_state: torch.Tensor | None = None,+ use_qk_l2norm: bool = True,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Verbatim port of HF ``torch_recurrent_gated_delta_rule`` (output_final_state=True)."""+ initial_dtype = query.dtype+ if use_qk_l2norm:+ query = _l2norm(query, eps=1e-6)+ key = _l2norm(key, eps=1e-6)+ query, key, value, beta, g = [+ t.transpose(1, 2).contiguous().to(torch.float32)+ for t in (query, key, value, beta, g)+ ]+ b, h, t_len, dk = key.shape+ dv = value.shape[-1]+ scale = 1.0 / (dk ** 0.5)+ query = query * scale++ out = torch.zeros(b, h, t_len, dv, dtype=value.dtype, device=value.device)+ state = (+ torch.zeros(b, h, dk, dv, dtype=value.dtype, device=value.device)+ if initial_state is None+ else initial_state.to(value)+ )+ for i in range(t_len):+ q_t = query[:, :, i]+ k_t = key[:, :, i]+ v_t = value[:, :, i]+ g_t = g[:, :, i].exp().unsqueeze(-1).unsqueeze(-1)+ beta_t = beta[:, :, i].unsqueeze(-1)+ state = state * g_t+ kv_mem = (state * k_t.unsqueeze(-1)).sum(dim=-2)+ delta = (v_t - kv_mem) * beta_t+ state = state + k_t.unsqueeze(-1) * delta.unsqueeze(-2)+ out[:, :, i] = (state * q_t.unsqueeze(-1)).sum(dim=-2)++ out = out.transpose(1, 2).contiguous().to(initial_dtype) # [B, T, Hv, Dv]+ return out, state+++def chunk_gated_delta_rule(+ query: torch.Tensor, # [B, T, Hv, Dk]+ key: torch.Tensor, # [B, T, Hv, Dk]+ value: torch.Tensor, # [B, T, Hv, Dv]+ g: torch.Tensor, # [B, T, Hv]+ beta: torch.Tensor, # [B, T, Hv]+ *,+ chunk_size: int = 64,+ initial_state: torch.Tensor | None = None,+ use_qk_l2norm: bool = True,+) -> tuple[torch.Tensor, torch.Tensor]:+ """Verbatim port of HF ``torch_chunk_gated_delta_rule`` (output_final_state=True).++ Same recurrence as ``recurrent_gated_delta_rule`` in exact arithmetic; it is the form the+ fla chunk kernel implements, so it is the closer oracle for the prefill path."""+ initial_dtype = query.dtype+ if use_qk_l2norm:+ query = _l2norm(query, eps=1e-6)+ key = _l2norm(key, eps=1e-6)+ query, key, value, beta, g = [+ x.transpose(1, 2).contiguous().to(torch.float32)+ for x in (query, key, value, beta, g)+ ]++ batch_size, num_heads, sequence_length, k_head_dim = key.shape+ v_head_dim = value.shape[-1]+ pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size+ query = F.pad(query, (0, 0, 0, pad_size))+ key = F.pad(key, (0, 0, 0, pad_size))+ value = F.pad(value, (0, 0, 0, pad_size))+ beta = F.pad(beta, (0, pad_size))+ g = F.pad(g, (0, pad_size))+ total_sequence_length = sequence_length + pad_size+ scale = 1 / (query.shape[-1] ** 0.5)+ query = query * scale++ v_beta = value * beta.unsqueeze(-1)+ k_beta = key * beta.unsqueeze(-1)+ # reshape to chunks+ query, key, value, k_beta, v_beta = [+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1])+ for x in (query, key, value, k_beta, v_beta)+ ]+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)+ mask = torch.triu(+ torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0+ )++ # chunk decay+ g = g.cumsum(dim=-1)+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)+ for i in range(1, chunk_size):+ row = attn[..., i, :i].clone()+ sub = attn[..., :i, :i].clone()+ attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)+ value = attn @ v_beta+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))+ last_recurrent_state = (+ torch.zeros(+ batch_size, num_heads, k_head_dim, v_head_dim,+ dtype=value.dtype, device=value.device,+ )+ if initial_state is None+ else initial_state.to(value)+ )+ core_attn_out = torch.zeros_like(value)++ # for each chunk; decay_mask is already lower-triangular so no extra causal mask is needed+ for i in range(0, total_sequence_length // chunk_size):+ q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]+ attn = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]+ v_prime = (k_cumdecay[:, :, i]) @ last_recurrent_state+ v_new = v_i - v_prime+ attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state+ core_attn_out[:, :, i] = attn_inter + attn @ v_new+ last_recurrent_state = (+ last_recurrent_state * g[:, :, i, -1, None, None].exp()+ + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2)+ @ v_new+ )++ core_attn_out = core_attn_out.reshape(+ core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1]+ )+ core_attn_out = core_attn_out[:, :, :sequence_length]+ core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype)+ return core_attn_out, last_recurrent_state+++class _RMSNormGated(nn.Module):+ """RMSNorm of x followed by an ``activation(z)`` gate (norm_before_gate=True).++ Mirrors ``Qwen4ExpTextRMSNormGated`` over head_v_dim groups.+ """++ def __init__(self, dim: int, eps: float, activation: str = "sigmoid"):+ super().__init__()+ self.weight = nn.Parameter(torch.ones(dim))+ self.eps = eps+ self.act = _GATE_ACTS[activation]++ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:+ in_dtype = x.dtype+ x = x.float()+ x = x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)+ x = x * self.weight.float()+ x = x * self.act(z.float())+ return x.to(in_dtype)+++class Qwen4ExpGatedDeltaNetReference(nn.Module):+ """Pure-torch Gated DeltaNet (text-only, no cache)."""++ def __init__(+ self,+ hidden_size: int,+ num_k_heads: int,+ num_v_heads: int,+ head_k_dim: int,+ head_v_dim: int,+ conv_kernel_size: int,+ rms_norm_eps: float,+ hidden_act: str = "silu",+ output_gate: str = "sigmoid",+ ):+ super().__init__()+ if hidden_act != "silu":+ raise ValueError(f"GDN reference only supports silu conv activation, got {hidden_act!r}")+ if output_gate not in _GATE_ACTS:+ raise ValueError(f"unsupported GDN output gate {output_gate!r}")+ self.num_k_heads = num_k_heads+ self.num_v_heads = num_v_heads+ self.head_k_dim = head_k_dim+ self.head_v_dim = head_v_dim+ self.key_dim = num_k_heads * head_k_dim+ self.value_dim = num_v_heads * head_v_dim+ self.conv_dim = self.key_dim * 2 + self.value_dim+ self.conv_kernel_size = conv_kernel_size++ self.in_proj_qkv = nn.Linear(hidden_size, self.conv_dim, bias=False)+ self.in_proj_z = nn.Linear(hidden_size, self.value_dim, bias=False)+ self.in_proj_b = nn.Linear(hidden_size, num_v_heads, bias=False)+ self.in_proj_a = nn.Linear(hidden_size, num_v_heads, bias=False)+ self.conv1d = nn.Conv1d(+ self.conv_dim, self.conv_dim, kernel_size=conv_kernel_size,+ groups=self.conv_dim, padding=conv_kernel_size - 1, bias=False,+ )+ self.dt_bias = nn.Parameter(torch.zeros(num_v_heads))+ self.A_log = nn.Parameter(torch.zeros(num_v_heads))+ self.norm = _RMSNormGated(head_v_dim, eps=rms_norm_eps, activation=output_gate)+ self.out_proj = nn.Linear(self.value_dim, hidden_size, bias=False)++ @torch.no_grad()+ def load_from_hf(self, hf_gdn) -> None:+ """Copy weights from a transformers ``Qwen4ExpTextGatedDeltaNet``."""+ self.in_proj_qkv.weight.copy_(hf_gdn.in_proj_qkv.weight)+ self.in_proj_z.weight.copy_(hf_gdn.in_proj_z.weight)+ self.in_proj_b.weight.copy_(hf_gdn.in_proj_b.weight)+ self.in_proj_a.weight.copy_(hf_gdn.in_proj_a.weight)+ # HF stores conv1d weight as [conv_dim, 1, K]; our depthwise Conv1d matches.+ self.conv1d.weight.copy_(hf_gdn.conv1d.weight.view_as(self.conv1d.weight))+ if hf_gdn.conv1d.bias is not None and self.conv1d.bias is not None:+ self.conv1d.bias.copy_(hf_gdn.conv1d.bias)+ self.dt_bias.copy_(hf_gdn.dt_bias)+ self.A_log.copy_(hf_gdn.A_log)+ self.norm.weight.copy_(hf_gdn.norm.weight)+ self.out_proj.weight.copy_(hf_gdn.out_proj.weight)++ def forward(self, hidden_states: torch.Tensor, *, use_chunk_rule: bool = False) -> torch.Tensor:+ b, t_len, _ = hidden_states.shape++ mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2) # [B, conv_dim, T]+ z = self.in_proj_z(hidden_states).reshape(b, t_len, -1, self.head_v_dim)+ a = self.in_proj_a(hidden_states)+ bb = self.in_proj_b(hidden_states)++ # causal depthwise conv + silu (drop the right padding back to T)+ mixed_qkv = F.silu(self.conv1d(mixed_qkv)[..., :t_len]).transpose(1, 2) # [B, T, conv_dim]+ query, key, value = torch.split(+ mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1+ )+ query = query.reshape(b, t_len, -1, self.head_k_dim)+ key = key.reshape(b, t_len, -1, self.head_k_dim)+ value = value.reshape(b, t_len, -1, self.head_v_dim)++ beta = bb.sigmoid()+ g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)++ # GQA expand: replicate q/k heads up to num_v_heads+ rep = self.num_v_heads // self.num_k_heads+ if rep > 1:+ query = query.repeat_interleave(rep, dim=2)+ key = key.repeat_interleave(rep, dim=2)++ rule = chunk_gated_delta_rule if use_chunk_rule else recurrent_gated_delta_rule+ core, _ = rule(query, key, value, g, beta, use_qk_l2norm=True)++ core = core.reshape(-1, self.head_v_dim)+ z = z.reshape(-1, self.head_v_dim)+ core = self.norm(core, z).reshape(b, t_len, -1)+ return self.out_proj(core)+++__all__ = [+ "Qwen4ExpGatedDeltaNetReference",+ "chunk_gated_delta_rule",+ "recurrent_gated_delta_rule",+]diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.pynew file mode 100644index 00000000..6fed76c0--- /dev/null+++ b/python/freetoken/models/qwen4_exp/hc.py@@ -0,0 +1,147 @@+"""Hyper-connection (gated residual) blocks for Qwen3.8-Flash-Next.++Every layer reads and writes ``hc_count`` residual streams packed as ``R [T, hc_count*hidden]``+(stream outer, hidden inner -- the checkpoint layout). On CUDA the mix/combine bodies are the+vendored vLLM Triton kernels (``kernel/triton/hc.py``: grouped_gemma_rmsnorm / hc_silu /+hc_gate_mix / hc_combine) around two ``F.linear`` GEMMs; the pure-torch chain stays as the CPU+path and as the reference the kernels are diffed against. Both keep fp32 intermediates and cast+back at the store, so they agree to fp32 rounding.+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Tuple++import torch+import torch.nn.functional as F+from freetoken.kernel.triton.hc import (+ grouped_gemma_rmsnorm,+ hc_combine,+ hc_gate_mix,+ hc_silu,+)+from freetoken.layers import BaseOP, LinearReplicated++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++def grouped_plus_one_rms_norm(+ x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int+) -> torch.Tensor:+ """RMSNorm each of ``num_groups`` equal slices of the last dim on its own fp32 statistic, then scale by (1+w)."""+ xf = x.float().unflatten(-1, (num_groups, -1))+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)+ return (xf.flatten(-2) * (1.0 + weight.float())).to(x.dtype)+++class GroupedPlusOneRMSNorm(BaseOP):+ """Per-stream RMSNorm of an ``[..., num_groups*group]`` tensor with one weight element per feature.++ HF ``Qwen4ExpTextRMSNorm(dim, group_size)``. The checkpoint weight is zero-centered and is+ loaded RAW: (1+w) is applied at runtime in fp32, never folded into the bf16 weight (the+ vendored Triton kernel does the same). ``ple.py`` reuses this class for norm_key /+ norm_query / norm_conv, so keep it exported.+ """++ def __init__(self, size: int, eps: float, num_groups: int) -> None:+ self.weight = torch.empty(size)+ self.eps = eps+ self.num_groups = num_groups++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ # the kernel is 2D-only, higher-rank callers keep the torch chain+ if x.is_cuda and x.dim() == 2:+ return grouped_gemma_rmsnorm(x, self.weight, self.eps, self.num_groups)+ return grouped_plus_one_rms_norm(x, self.weight, self.eps, self.num_groups)+++class GatedResidual(BaseOP):+ """One hyper-connection block: ``mix`` reads the residual streams, ``combine`` writes a block output back.++ Frozen API (HF ``Qwen4ExpTextGatedResidual``, formulas at modeling_qwen4_exp.py:959-969)::++ x, s = hc.mix(R) # R [T, hc_count*hidden] -> x [T, hidden], s [T, hc_count] or None+ y = block(x) # attention / GDN / MoE, plain [T, hidden] -> [T, hidden]+ R = hc.combine(R, y, s)++ Rn = groupRMSNorm(R) * (1 + hc_norm.weight) # per hidden-size stream, fp32 stats+ lora, s = input_mix_weight_down_block_inject(Rn) # merged GEMM: [lowrank | hc_count | pad]+ gate = input_mix_weight_up(silu(lora / hc_count))+ x = mean_i(sigmoid(gate_i) * Rn_i)+ R'_i = R_i + 2*sigmoid(s_i / hc_count) * y++ ``s`` is the RAW inject logit slice of the merged GEMM (pre 2*sigmoid), which is what the+ vendored ``hc_combine`` kernel expects; ``combine`` applies the activation. The merged weight+ is ``[lowrank + hc_count + pad, hc_count*hidden]`` (Qwen3.8: 320 + 4 + 12 = 336 rows), the pad+ rows are zero and their GEMM output is dropped. ``use_combine=False`` is the top-level mixer:+ it owns the unmerged ``input_mix_weight_down``, returns ``s = None`` and has no ``combine``.++ Weight keys (checkpoint names, prefix stripped): ``hc_norm.weight``,+ ``input_mix_weight_down_block_inject.weight`` (loader: concat of+ ``input_mix_weight_down`` [lowrank, hc*hidden], ``block_inject_weight`` [hc_count, hc*hidden]+ and ``pad`` zero rows), ``input_mix_weight_up.weight``.++ Launch budget on CUDA: ``mix`` is 3 kernels around 2 GEMMs, ``combine`` is 1.+ """++ def __init__(self, config: ModelConfig, use_combine: bool = True) -> None:+ args = config.qwen4_args+ self.hc_count = args.hc_count+ self.hidden_size = args.hidden_size+ self.lowrank = args.hc_lowrank+ self.use_combine = use_combine+ width = args.ple_state_width+ self.hc_norm = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)+ if use_combine:+ # 16-row alignment for the merged skinny GEMM (vLLM hyperconnection.py:98)+ self.pad_size = (-(self.lowrank + self.hc_count)) % 16+ self.input_mix_weight_down_block_inject = LinearReplicated(+ width, self.lowrank + self.hc_count + self.pad_size, has_bias=False+ )+ else:+ self.pad_size = 0+ self.input_mix_weight_down = LinearReplicated(width, self.lowrank, has_bias=False)+ self.input_mix_weight_up = LinearReplicated(self.lowrank, width, has_bias=False)++ def _down(self, rn: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:+ """Run the down GEMM and split off the raw inject logits; the pad columns are dropped."""+ if not self.use_combine:+ return self.input_mix_weight_down.forward(rn), None+ down = self.input_mix_weight_down_block_inject.forward(rn)+ # both slices keep unit inner stride, so the kernels read them without a copy+ return down[:, : self.lowrank], down[:, self.lowrank : self.lowrank + self.hc_count]++ def _mix_kernel(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:+ rn = grouped_gemma_rmsnorm(R, self.hc_norm.weight, self.hc_norm.eps, self.hc_count)+ lora, s = self._down(rn)+ gate = self.input_mix_weight_up.forward(hc_silu(lora, self.hc_count))+ return hc_gate_mix(rn, gate, self.hc_count), s++ def _mix_torch(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:+ rn = grouped_plus_one_rms_norm(R, self.hc_norm.weight, self.hc_norm.eps, self.hc_count)+ lora, s = self._down(rn)+ lora = F.silu(lora.float() / self.hc_count)+ gate = self.input_mix_weight_up.forward(lora.to(R.dtype))+ mixed = torch.sigmoid(gate.float()).unflatten(-1, (self.hc_count, self.hidden_size))+ mixed = mixed * rn.float().unflatten(-1, (self.hc_count, self.hidden_size))+ return mixed.mean(-2).to(R.dtype), s++ def _combine_torch(self, R: torch.Tensor, y: torch.Tensor, s: torch.Tensor) -> torch.Tensor:+ inject = 2.0 * torch.sigmoid(s.float() / self.hc_count)+ out = R.float().unflatten(-1, (self.hc_count, self.hidden_size))+ out = out + y.float().unsqueeze(-2) * inject.unsqueeze(-1)+ return out.flatten(-2).to(R.dtype)++ def mix(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:+ """Return the block input ``x [T, hidden]`` and the inject logits ``s [T, hc_count]`` (None if no combine)."""+ return self._mix_kernel(R) if R.is_cuda else self._mix_torch(R)++ def combine(self, R: torch.Tensor, y: torch.Tensor, s: torch.Tensor) -> torch.Tensor:+ """Inject the block output ``y [T, hidden]`` back into every stream of ``R``."""+ if R.is_cuda:+ return hc_combine(R, y, s, self.hc_count)+ return self._combine_torch(R, y, s)+++__all__ = ["GatedResidual", "GroupedPlusOneRMSNorm", "grouped_plus_one_rms_norm"]diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.pynew file mode 100644index 00000000..e5b365aa--- /dev/null+++ b/python/freetoken/models/qwen4_exp/model.py@@ -0,0 +1,187 @@+"""Qwen3.8-Flash-Next decoder stack (text-only).++The residual state is ``R [T, hc_count*hidden]`` end to end: the embedding is repeated over the+``hc_count`` streams, every layer mixes them down to one ``[T, hidden]`` block input and injects+its output back, and the top-level mixer collapses them once before ``lm_head``. There is no+input/post layernorm and no final ``model.norm`` -- the hyper-connection norms are the only ones.++Layer contract (frozen): ``forward(R [T, hc*hidden], batch) -> R' [T, hc*hidden]`` with an+immediate combine::++ R = R + ple(R, batch) # zero-based layer 1 only+ x, s = attn_hc.mix(R); y = (GDN | QSA)(x); R = attn_hc.combine(R, y, s)+ x, s = mlp_hc.mix(R); y = MoE(x); R = mlp_hc.combine(R, y, s)+"""++from __future__ import annotations++from typing import TYPE_CHECKING, List++import torch+from freetoken.core import get_global_ctx+from freetoken.layers import BaseOP, OPList, ParallelLMHead, VocabParallelEmbedding+from freetoken.models.blocks import BaseLLMModel+from freetoken.utils import nvtx_annotate++from .attention import Qwen4ExpAttention+from .hc import GatedResidual+from .moe import Qwen4ExpMoE+from .ple import PLELayer++if TYPE_CHECKING:+ from freetoken.core import Batch+ from freetoken.models.config import ModelConfig+++def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP:+ """GDN mixer of a linear_attention layer (Qwen3.5's GDN with a configurable output gate)."""+ from .gdn import Qwen4ExpGatedDeltaNet++ g = config.linear_attention_group()+ return Qwen4ExpGatedDeltaNet(+ hidden_size=config.hidden_size,+ num_k_heads=g.num_key_heads,+ num_v_heads=g.num_value_heads,+ head_k_dim=g.key_head_dim,+ head_v_dim=g.value_head_dim,+ conv_kernel_size=g.conv_kernel_dim,+ rms_norm_eps=config.rms_norm_eps,+ layer_id=layer_id,+ output_gate=g.output_gate,+ # Qwen3.8's block-fp8 checkpoint keeps the GDN projections bf16 (only the routed+ # experts are quantized), so do not let expert_quant flip them to Fp8Block.+ expert_quant="none" if config.expert_quant == "fp8_block" else config.expert_quant,+ attn_quant=config.attn_quant,+ )+++class Qwen4ExpDecoderLayer(BaseOP):+ """One decoder layer over the hyper-connection streams (see the module docstring for the flow)."""++ def __init__(self, config: ModelConfig, layer_id: int) -> None:+ self._layer_id = layer_id+ self._is_linear = config.is_linear_layer(layer_id)+ if self._is_linear:+ self.linear_attn = build_linear_mixer(config, layer_id)+ else:+ self.self_attn = Qwen4ExpAttention(config, layer_id)+ self.mlp = Qwen4ExpMoE(config, layer_id)+ self.attn_hyper_connection = GatedResidual(config)+ self.mlp_hyper_connection = GatedResidual(config)+ self.ple = (+ PLELayer(config, layer_id) if layer_id in config.qwen4_args.ple_layer_ids else None+ )++ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")+ def forward(self, hidden: torch.Tensor, batch: Batch) -> torch.Tensor:+ if self.ple is not None:+ hidden = hidden + self.ple.forward(hidden, batch)+ block_input, inject = self.attn_hyper_connection.mix(hidden)+ if self._is_linear:+ block_output = self.linear_attn.forward(block_input)+ else:+ block_output = self.self_attn.forward(block_input, batch)+ hidden = self.attn_hyper_connection.combine(hidden, block_output, inject)+ block_input, inject = self.mlp_hyper_connection.mix(hidden)+ return self.mlp_hyper_connection.combine(hidden, self.mlp.forward(block_input), inject)+++class Qwen4ExpModel(BaseOP):+ def __init__(self, config: ModelConfig) -> None:+ self.hc_count = config.qwen4_args.hc_count+ self.embed_tokens = VocabParallelEmbedding(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ )+ self.layers = OPList(+ [Qwen4ExpDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ )+ self.hyper_connection_mixer = GatedResidual(config, use_combine=False)+ # plain tuple (not an OP child), so it never shows up in the state dict+ self._ple = tuple(layer.ple for layer in self.layers.op_list if layer.ple is not None)++ @property+ def ple_layers(self) -> List[PLELayer]:+ """The PLE layers in decoder order -- the seam the loader attaches table backends to."""+ return list(self._ple)++ def forward(self, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor:+ hidden = self.embed_tokens.forward(input_ids).repeat(1, self.hc_count)+ meta = None+ if self._ple:+ from .ple import build_ple_metadata, commit_ngram_context++ meta = build_ple_metadata(batch, self._ple[0].args, input_ids.device)+ for ple in self._ple: # gather the pinned-host PLE rows while the early layers run+ ple.start_prefetch(batch, meta)+ for layer in self.layers.op_list:+ hidden = layer.forward(hidden, batch)+ if meta is not None:+ # single writer: the layers only read the context, so a second PLE layer's+ # prefetch sees the un-rolled window+ commit_ngram_context(meta, getattr(batch, "fla_metadata", None))+ return self.hyper_connection_mixer.mix(hidden)[0]+++class Qwen4ExpForCausalLM(BaseLLMModel):+ def __init__(self, config: ModelConfig) -> None:+ self._config = config+ self.model = Qwen4ExpModel(config)+ if getattr(config, "lm_head_quant", "none") == "nvfp4":+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead++ assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings"+ self.lm_head = Nvfp4LMHead(+ num_embeddings=config.vocab_size, embedding_dim=config.hidden_size+ )+ else:+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ )+ super().__init__()++ def load_host_tables(self, engine_config) -> int:+ """Attach the PLE n-gram table (pinned checkpoint bank, or zeros for dummy weights); returns the pinned host bytes the engine reserves from its pin budget."""+ ple_layers = self.model.ple_layers+ if not ple_layers:+ return 0+ from .ple import PinnedUVATable, ZeroTable, derive_ngram_hash_constants++ if getattr(engine_config, "use_dummy_weight", False):+ # Dummy fill leaves the int64 hash buffers garbage (a zero vocab size divides by+ # zero in the hash), so re-derive the real constants and read a zero table.+ for ple in ple_layers:+ args = ple.args+ mult, sizes, offsets = derive_ngram_hash_constants(+ vocab_size=self._config.vocab_size,+ ngram_size=args.ngram_size,+ num_ngram_heads=args.num_ngram_heads,+ ngram_vocab_size_base=args.ngram_vocab_size_base,+ ple_layer_index=ple.ple_index,+ )+ emb = ple.ple_embedding+ emb.layer_multipliers.copy_(torch.tensor(mult, dtype=torch.int64))+ emb.ngram_heads_vocab_sizes.copy_(torch.tensor(sizes, dtype=torch.int64))+ emb.ngram_heads_offsets.copy_(torch.tensor(offsets, dtype=torch.int64))+ emb.attach_table(ZeroTable(offsets[-1] + sizes[-1], args.ngram_head_dim))+ return 0++ from .weight import load_ple_table++ table = load_ple_table(engine_config.model_path, self._config.qwen4_args)+ self._ple_table = table # owns the pinned HostBank; keep it alive+ for ple in ple_layers:+ ple.ple_embedding.attach_table(+ PinnedUVATable(table.bank.tensor, float(table.weight_scale))+ )+ return table.bank.nbytes++ def forward(self) -> torch.Tensor:+ batch = get_global_ctx().batch+ return self.lm_head.forward(self.model.forward(batch.input_ids, batch))+++__all__ = ["Qwen4ExpDecoderLayer", "Qwen4ExpForCausalLM", "Qwen4ExpModel", "build_linear_mixer"]diff --git a/python/freetoken/models/qwen4_exp/moe.py b/python/freetoken/models/qwen4_exp/moe.pynew file mode 100644index 00000000..9ef65c0a--- /dev/null+++ b/python/freetoken/models/qwen4_exp/moe.py@@ -0,0 +1,46 @@+from __future__ import annotations++from dataclasses import replace+from typing import TYPE_CHECKING++import torch+from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid+from freetoken.layers.moe import make_moe_layer+from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class Qwen4ExpMoE(Qwen3_5MoE):+ """Qwen3_5MoE with the shared-expert gate on triton instead of gemv + sigmoid + mul + add.++ Same weights, same state dict. The gate reduction stays ahead of the routed experts, which may write into ``hidden_states`` in place.+ """++ def __init__(self, config: ModelConfig, layer_id: int | None = None) -> None:+ if getattr(config, "expert_quant", "none") != "fp8_block":+ super().__init__(config, layer_id=layer_id)+ return+ # Qwen3.8's block-fp8 checkpoint quantizes only the routed experts; the shared+ # expert stays bf16, so hide expert_quant from _SharedExpert's fp8 branch and+ # rebuild the routed experts with the fp8_block bank layout.+ super().__init__(replace(config, expert_quant="none"), layer_id=layer_id)+ self.experts = make_moe_layer(+ config,+ layer_id=layer_id,+ renormalize=config.norm_topk_prob,+ weight_format="fp8_block",+ )++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ num_tokens, hidden_dim = hidden_states.shape+ hidden_states = hidden_states.view(-1, hidden_dim)+ router_logits = self.gate.forward(hidden_states)+ shared = self.shared_expert.forward(hidden_states)+ gate = shared_gate_sigmoid(hidden_states, self.shared_expert_gate.weight.view(-1))+ routed = self.experts.forward(hidden_states=hidden_states, router_logits=router_logits)+ return shared_gate_mul_add(routed, shared, gate).view(num_tokens, hidden_dim)+++__all__ = ["Qwen4ExpMoE"]diff --git a/python/freetoken/models/qwen4_exp/ple.py b/python/freetoken/models/qwen4_exp/ple.pynew file mode 100644index 00000000..5100229a--- /dev/null+++ b/python/freetoken/models/qwen4_exp/ple.py@@ -0,0 +1,721 @@+"""Per-Layer Embedding (PLE) for Qwen3.8-Flash-Next: hashed n-gram features injected at layer 1.++HF reference: ``Qwen4ExpTextNGramEmbedding`` (modeling_qwen4_exp.py:1018) and+``Qwen4ExpTextPLELayer`` (:1117). Per token::++ E = table[hash(ngram)] # 16 heads (8 x 2-gram, 8 x 3-gram) x 160 -> 2560+ K = norm_key(key_proj(E)).view(hc, hidden) # V = value_proj(E) [hidden]+ Q = norm_query(R).view(hc, hidden)+ u = <K_i, Q_i> / sqrt(hidden) # per stream+ U = sigmoid(sign(u) * sqrt(max(|u|, 1e-6))) * V+ D = U + silu(conv1d(norm_conv(U))) # depthwise, kernel 4, dilation ngram_size+ R += D # before the attention hyper-connection mix++The table is the 47.7 GiB FP8 n-gram store: ``PinnedUVATable`` keeps it in pinned host memory and+gathers rows over UVA, optionally started early on a side stream (``PLELayer.start_prefetch``).+``GpuResidentTable`` is the small-table oracle the pinned backend is diffed against.+"""++from __future__ import annotations++import math+from dataclasses import dataclass+from typing import TYPE_CHECKING, List, Protocol, Sequence, Tuple++import torch+import torch.nn.functional as F+from freetoken.core import get_global_ctx+from freetoken.layers import BaseOP, LinearReplicated++from .config import PLE_CONV_STATE, PLE_NGRAM_STATE+from .hc import GroupedPlusOneRMSNorm++if TYPE_CHECKING:+ from freetoken.core import Batch+ from freetoken.models.config import ModelConfig++ from .config import Qwen4ExpArgs+++_MASK64 = (1 << 64) - 1+_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15+_SPLITMIX_M1 = 0xBF58476D1CE4E5B9+_SPLITMIX_M2 = 0x94D049BB133111EB+_PLE_LAYER_PRIME = 10007+++class PLETableBackend(Protocol):+ """Row store for one PLE layer's n-gram embedding table (Qwen3.8: 40M rows x 160, FP8 + one scalar scale).++ Frozen contract. ``GpuResidentTable`` (oracle, small tables) and ``PinnedUVATable`` (the real 47.7 GiB pinned-host table) implement it. Rows are addressed by the+ GLOBAL hashed id, i.e. the per-head vocab offset is already added by ``NGramEmbedding``.++ ``lookup`` gets ``row_ids [T, num_ngram_heads]`` (int64, device) and returns+ ``[T, num_ngram_heads * head_dim]`` in ``dtype``, already dequantized (fp8 -> dtype, times the+ scalar weight_scale). ``out``, when given, is the destination and is returned as-is (CUDA-graph+ decode reuses a fixed buffer).++ ``prefetch`` may start the gather early on a side stream (the model issues it before layer 0 and+ joins it in ``lookup``); a backend with no async path makes it a no-op.+ """++ num_rows: int+ head_dim: int+ dtype: torch.dtype++ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: ...++ def prefetch(self, row_ids: torch.Tensor) -> None: ...+++class GpuResidentTable:+ """PLE table held whole in GPU memory; ``index_select`` oracle for the pinned-host backend."""++ def __init__(+ self, weight: torch.Tensor, scale: float = 1.0, dtype: torch.dtype | None = None+ ) -> None:+ self.weight = weight+ self.scale = float(scale)+ self.num_rows, self.head_dim = weight.shape+ self.dtype = dtype if dtype is not None else (+ torch.bfloat16 if weight.dtype.itemsize < 2 else weight.dtype+ )++ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:+ rows = self.weight.index_select(0, row_ids.reshape(-1)).to(self.dtype)+ if self.scale != 1.0:+ rows = rows * self.scale+ rows = rows.view(*row_ids.shape[:-1], -1)+ if out is None:+ return rows+ out.copy_(rows)+ return out++ def prefetch(self, row_ids: torch.Tensor) -> None:+ return None+++class ZeroTable:+ """Dummy-weight stand-in: every lookup reads zeros (dummy checkpoints ship no table)."""++ def __init__(self, num_rows: int, head_dim: int, dtype: torch.dtype = torch.bfloat16) -> None:+ self.num_rows = int(num_rows)+ self.head_dim = head_dim+ self.dtype = dtype++ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:+ if out is not None:+ return out.zero_()+ return torch.zeros(+ (*row_ids.shape[:-1], row_ids.shape[-1] * self.head_dim),+ dtype=self.dtype,+ device=row_ids.device,+ )++ def prefetch(self, row_ids: torch.Tensor) -> None:+ return None+++class PinnedUVATable:+ """PLE table left in pinned host memory; rows are gathered over UVA by a Triton kernel.++ ``weight`` must be the filled and ``pin()``ed ``HostBank.tensor`` from+ ``weight.load_ple_table`` (``[num_rows, head_dim]``, fp8-e4m3 or bf16); an unregistered host+ buffer is not device-addressable and the kernel faults on it. ``scale`` is the checkpoint's+ scalar ``weight_scale``. Gathers emit bf16 into a staging buffer, one per captured decode size+ and one growable buffer for everything else.++ ``prefetch`` runs the gather on a private stream and the next ``lookup`` joins it. ``lookup``+ returns a view of that staging buffer, so the rows must be consumed before the next lookup.+ """++ def __init__(+ self,+ weight: torch.Tensor,+ scale: float = 1.0,+ *,+ device: torch.device | None = None,+ prefetch: bool = True,+ ) -> None:+ assert weight.device.type == "cpu" and weight.is_contiguous()+ assert weight.dtype in (torch.float8_e4m3fn, torch.bfloat16), weight.dtype+ from freetoken.kernel.pinned import device_ptr++ self.weight = weight+ self.scale = float(scale)+ self.num_rows, self.head_dim = weight.shape+ self.dtype = torch.bfloat16+ self._is_fp8 = weight.dtype == torch.float8_e4m3fn+ self._device = device or torch.device("cuda", torch.cuda.current_device())+ # WDDM maps registered host memory at a different device address; on Linux/UVA this is data_ptr+ self._table_ptr = device_ptr(weight)+ self._stream = torch.cuda.Stream(device=self._device) if prefetch else None+ self._staging: torch.Tensor | None = None+ self._graph_staging: dict[int, torch.Tensor] = {}+ self._pending: Tuple[torch.Tensor, torch.Tensor] | None = None++ def _stage(self, rows: int) -> torch.Tensor:+ # Captured graphs keep one buffer per size for good: growing the eager one would free the+ # block a replay still writes to.+ if torch.cuda.is_current_stream_capturing():+ buf = self._graph_staging.get(rows)+ if buf is None:+ buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device)+ self._graph_staging[rows] = buf+ return buf+ buf = self._staging+ if buf is None or buf.shape[0] < rows:+ buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device)+ self._staging = buf+ return buf[:rows]++ def _gather(self, row_ids: torch.Tensor, dst: torch.Tensor) -> torch.Tensor:+ from freetoken.kernel.triton.ple import ple_gather_rows++ return ple_gather_rows(+ self._table_ptr,+ self.num_rows,+ self.head_dim,+ row_ids.reshape(-1),+ dst,+ self.scale,+ self._is_fp8,+ )++ def prefetch(self, row_ids: torch.Tensor) -> None:+ if self._stream is None or row_ids.numel() == 0:+ return+ dst = self._stage(row_ids.numel())+ self._stream.wait_stream(torch.cuda.current_stream(self._device))+ if not torch.cuda.is_current_stream_capturing():+ row_ids.record_stream(self._stream)+ with torch.cuda.stream(self._stream):+ self._gather(row_ids, dst)+ self._pending = (row_ids, dst)++ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:+ pending, self._pending = self._pending, None+ if pending is not None:+ # join even on a miss: the stale prefetch owns the staging buffer about to be reused+ torch.cuda.current_stream(self._device).wait_stream(self._stream)+ if pending is not None and pending[0] is row_ids:+ rows = pending[1]+ else:+ rows = self._gather(row_ids, self._stage(row_ids.numel()))+ rows = rows.view(*row_ids.shape[:-1], -1)+ if out is None:+ return rows+ out.copy_(rows)+ return out+++def _splitmix64(value: int) -> int:+ value = (value + _SPLITMIX_GAMMA) & _MASK64+ value = ((value ^ (value >> 30)) * _SPLITMIX_M1) & _MASK64+ value = ((value ^ (value >> 27)) * _SPLITMIX_M2) & _MASK64+ return (value ^ (value >> 31)) & _MASK64+++def _is_prime(value: int) -> bool:+ if value < 2:+ return False+ if value % 2 == 0:+ return value == 2+ for divisor in range(3, math.isqrt(value) + 1, 2):+ if value % divisor == 0:+ return False+ return True+++def _nth_prime_after(start: int, count: int) -> int:+ prime = start+ for _ in range(count):+ prime += 1+ while not _is_prime(prime):+ prime += 1+ return prime+++def derive_ngram_hash_constants(+ *,+ vocab_size: int,+ ngram_size: int,+ num_ngram_heads: int,+ ngram_vocab_size_base: int,+ ple_layer_index: int,+ seed: int = 1234,+) -> Tuple[List[int], List[int], List[int]]:+ """Recompute (multipliers, per-head vocab sizes, per-head offsets) the way HF derives them at init.++ The checkpoint ships these as int64 tensors, so serving loads them; this is the dummy-weight+ path and the oracle a loader test can check the checkpoint values against.+ """+ half_bound = max(1, ((1 << 63) - 1) // max(vocab_size, 1) // 2)+ base_seed = seed + _PLE_LAYER_PRIME * ple_layer_index+ multipliers = [+ 2 * (_splitmix64((base_seed + _SPLITMIX_GAMMA * (i + 1)) & _MASK64) % half_bound) + 1+ for i in range(ngram_size)+ ]+ sizes: List[int] = []+ offsets: List[int] = []+ total = 0+ for head in range(num_ngram_heads):+ global_head = ple_layer_index * num_ngram_heads + head+ size = _nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)+ sizes.append(size)+ offsets.append(total)+ total += size+ return multipliers, sizes, offsets+++@dataclass+class PLEMetadata:+ """Per-forward PLE inputs, built once and shared by every PLE layer (sibling of ``FLAMetadata``).++ Frozen contract:+ input_ids [T] int device -- this forward's tokens, ragged, concatenated in request order+ cu_seqlens [B+1] int device -- query indptr; decode is ``arange(B+1)``+ seq_lens host per-request token counts; avoids a device sync in the ragged conv loop+ ngram_context [B, ngram_size-1] int64 device -- the tokens immediately BEFORE each request's+ first token of this forward, read from the ``ple_ngram_ctx`` slot state and+ forced to the boundary (eos) id for fresh rows. The hash never crosses eos,+ so a fresh sequence passes all-eos.+ state_slots [B] int64 device -- linear-state slot per request (``Req.linear_slot_idx`` or+ ``Req.table_idx``); keys every PLE slot state+ fresh_slots [B] bool device or None -- request starts a new sequence, so read a zero state+ is_decode one token per request (the batched 4-tap path)+ """++ input_ids: torch.Tensor+ cu_seqlens: torch.Tensor+ seq_lens: Sequence[int]+ ngram_context: torch.Tensor+ state_slots: torch.Tensor+ fresh_slots: torch.Tensor | None+ is_decode: bool+++def _state_slot(req) -> int:+ slot = getattr(req, "linear_slot_idx", None)+ return req.table_idx if slot is None else slot+++def _ngram_context_pool() -> torch.Tensor:+ pool = get_global_ctx().linear_state_pool+ assert pool is not None and pool.has_slot_state(PLE_NGRAM_STATE), (+ "PLE needs the ple_ngram_ctx slot state (or an explicit context_pool=)"+ )+ return pool.slot_state(PLE_NGRAM_STATE)+++def build_ple_metadata(+ batch: Batch,+ args: Qwen4ExpArgs,+ device: torch.device,+ context_pool: torch.Tensor | None = None,+) -> PLEMetadata:+ """Build ``PLEMetadata`` from a scheduler batch.++ The n-gram context is per-request device state (``ple_ngram_ctx`` [num_slots, ngram_size-1],+ rolled forward once per forward by ``commit_ngram_context``), so it never lags the sampled+ token under overlap scheduling and follows the slot on COW/snapshot. A decode batch reads it+ straight off the persistent ``linear_table_idx`` buffer, so the build is capture-safe and+ sync-free. Reuses ``batch.fla_metadata`` (slots / indptr / fresh mask) when the scheduler+ built it.+ """+ reqs = batch.padded_reqs+ ctx_len = args.ngram_size - 1+ eos = args.ngram_boundary_token_id+ if context_pool is None:+ context_pool = _ngram_context_pool()+ assert context_pool.shape[-1] == ctx_len, (+ f"ple_ngram_ctx holds {context_pool.shape[-1]} ids, config wants {ctx_len}"+ )+ fla = getattr(batch, "fla_metadata", None)+ slots_dev = getattr(batch, "linear_table_idx", None)++ if batch.is_decode and slots_dev is not None:+ slots = slots_dev.long()+ bs = slots.numel()+ return PLEMetadata(+ input_ids=batch.input_ids,+ cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device),+ seq_lens=(1,) * bs,+ ngram_context=context_pool.index_select(0, slots).long(),+ state_slots=slots,+ fresh_slots=None,+ is_decode=True,+ )++ lens = [r.extend_len for r in reqs]+ if fla is not None and fla.has_initial_state is not None:+ cu = fla.cu_seqlens+ slots = fla.cache_indices.long()+ fresh = ~fla.has_initial_state+ else: # direct-op callers (tests) with no scheduler metadata+ pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()}+ cu = torch.tensor([0, *lens], dtype=torch.int64, **pin).cumsum_(0).to(device, non_blocking=True)+ slots = torch.tensor([_state_slot(r) for r in reqs], dtype=torch.int64, **pin).to(device, non_blocking=True)+ fresh = torch.tensor([r.cached_len == 0 for r in reqs], dtype=torch.bool, **pin).to(device, non_blocking=True)+ context = context_pool.index_select(0, slots).long()+ context = torch.where(fresh.unsqueeze(1), context.new_full((), eos), context)+ return PLEMetadata(+ input_ids=batch.input_ids,+ cu_seqlens=cu,+ seq_lens=tuple(lens),+ ngram_context=context,+ state_slots=slots,+ fresh_slots=fresh,+ is_decode=batch.is_decode,+ )+++def commit_ngram_context(meta: PLEMetadata, fla, context_pool: torch.Tensor | None = None) -> None:+ """Roll each request's ``ple_ngram_ctx`` forward past this forward's tokens.++ Called ONCE per forward after every PLE layer ran (the layers only read the context);+ also writes the boundary-aligned window to the track slot so a donated snapshot restores+ the context together with the conv state. Pure device arithmetic, capture-safe.+ """+ if context_pool is None:+ context_pool = _ngram_context_pool()+ ids = meta.input_ids.long()+ ctx_len = meta.ngram_context.shape[1]+ steps = torch.arange(ctx_len, device=ids.device)+ if meta.is_decode:+ nxt = torch.cat([meta.ngram_context[:, 1:], ids.view(-1, 1)], dim=1)+ else:+ cu = meta.cu_seqlens.long()+ cand = cu[1:].unsqueeze(1) - ctx_len + steps+ # short extends fall back to the old context: token j of the new window sits at+ # old-context column extend_len + j when it predates this forward+ old = meta.ngram_context.gather(+ 1, ((cu[1:] - cu[:-1]).unsqueeze(1) + steps).clamp_(max=ctx_len - 1)+ )+ nxt = torch.where(cand >= cu[:-1].unsqueeze(1), ids[cand.clamp_min(0)], old)+ context_pool.index_copy_(0, meta.state_slots, nxt.to(context_pool.dtype))+ if fla is not None and fla.track_boundary_row is not None:+ win = ids[fla.track_boundary_row.unsqueeze(1) - ctx_len + steps]+ context_pool.index_copy_(0, fla.track_dst, win.to(context_pool.dtype))+++class NGramEmbedding(BaseOP):+ """Hashed n-gram lookup: splitmix64 mix of the last n token ids -> per-head prime vocab -> table rows.++ Weight keys (checkpoint names): ``layer_multipliers`` [ngram_size], ``ngram_heads_vocab_sizes``+ and ``ngram_heads_offsets`` [num_ngram_heads], all int64. The table itself is NOT a state-dict+ entry (128 checkpoint shards land in a ``PLETableBackend``); attach it with ``attach_table``.+ """++ def __init__(self, args: Qwen4ExpArgs, table: PLETableBackend | None = None) -> None:+ self.ngram_size = args.ngram_size+ self.heads_per_ngram = args.heads_per_ngram+ self.num_heads = args.num_ngram_heads+ self.eos_token_id = args.ngram_boundary_token_id+ self.layer_multipliers = torch.empty(args.ngram_size, dtype=torch.int64)+ self.ngram_heads_vocab_sizes = torch.empty(self.num_heads, dtype=torch.int64)+ self.ngram_heads_offsets = torch.empty(self.num_heads, dtype=torch.int64)+ self._table = table++ def attach_table(self, table: PLETableBackend) -> None:+ self._table = table++ @property+ def table(self) -> PLETableBackend:+ assert self._table is not None, "PLE table backend was never attached"+ return self._table++ def _window(self, meta: PLEMetadata):+ """The hash window as ``(packed [B, W], select)``, where ``select`` picks this forward's tokens."""+ ids = meta.input_ids.long()+ ctx_len = self.ngram_size - 1+ if meta.is_decode:+ # a window of exactly ngram_size columns holds every shift the hash can reach+ return torch.cat([meta.ngram_context, ids.view(-1, 1)], dim=1), lambda t: t[:, -1]++ num_reqs = len(meta.seq_lens)+ width = ctx_len + max(meta.seq_lens)+ cu = meta.cu_seqlens.long()+ # Pack the ragged tokens into [B, ctx+max_len] so the shift/boundary logic is one gather.+ flat_pos = torch.arange(ids.numel(), device=ids.device)+ req = (torch.searchsorted(cu, flat_pos, right=True) - 1).clamp_(max=num_reqs - 1)+ col = flat_pos - cu[req] + ctx_len+ packed = ids.new_full((num_reqs, width), self.eos_token_id)+ packed[:, :ctx_len] = meta.ngram_context+ packed[req, col] = ids+ return packed, lambda t: t[req, col]++ def _shift_ignore_eos(self, packed: torch.Tensor) -> List[torch.Tensor]:+ """``out[s][b, p]`` = the token ``s`` places left of ``p``, or eos when the window crosses a boundary."""+ num_reqs, width = packed.shape+ pos = torch.arange(width, device=packed.device)+ eos_pos = torch.where(packed == self.eos_token_id, pos, -1)+ prev_eos = torch.cummax(eos_pos, dim=1).values+ prev_eos = torch.cat([eos_pos.new_full((num_reqs, 1), -1), prev_eos[:, :-1]], dim=1)+ in_segment = pos.unsqueeze(0) - prev_eos - 1++ shifted = [packed]+ for shift in range(1, self.ngram_size):+ src = pos - shift+ gathered = packed.gather(1, src.clamp_min(0).unsqueeze(0).expand(num_reqs, -1))+ valid = (src.unsqueeze(0) >= 0) & (in_segment >= shift)+ shifted.append(torch.where(valid, gathered, packed.new_full((), self.eos_token_id)))+ return shifted++ def row_ids(self, meta: PLEMetadata) -> torch.Tensor:+ """Global table row per (token, hash head): ``[T, num_ngram_heads]`` int64."""+ packed, select = self._window(meta)+ tokens = [select(s) for s in self._shift_ignore_eos(packed)]+ blocks = []+ for ngram in range(2, self.ngram_size + 1):+ start = (ngram - 2) * self.heads_per_ngram+ end = start + self.heads_per_ngram+ mixed = tokens[0] * self.layer_multipliers[0]+ for position in range(1, ngram):+ mixed = torch.bitwise_xor(mixed, tokens[position] * self.layer_multipliers[position])+ head_ids = torch.remainder(mixed.unsqueeze(-1), self.ngram_heads_vocab_sizes[start:end])+ blocks.append(head_ids + self.ngram_heads_offsets[start:end])+ return torch.cat(blocks, dim=-1)++ def forward(self, meta: PLEMetadata, out: torch.Tensor | None = None) -> torch.Tensor:+ return self.table.lookup(self.row_ids(meta), out)+++class _DepthwiseConv1d(BaseOP):+ """Holds the depthwise conv weight ``[width, 1, kernel]`` (key ``conv1d.weight``)."""++ def __init__(self, width: int, kernel: int) -> None:+ self.weight = torch.empty(width, 1, kernel)+++def short_conv_reference(+ x: torch.Tensor,+ meta: PLEMetadata,+ states: torch.Tensor,+ weight: torch.Tensor,+ dilation: int,+) -> torch.Tensor:+ """Per-request ``F.conv1d`` over ``[state | chunk]``, advancing ``states`` in place.++ Transcription of the HF conv; the shipping paths (one packed conv for prefill, a tap read for+ decode) are diffed against it.+ """+ groups = weight.shape[0]+ state_len = states.shape[-1]+ slots = meta.state_slots+ state = states.index_select(0, slots).to(x.dtype)+ if meta.fresh_slots is not None:+ state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state)++ outs = []+ new_state = torch.empty_like(state)+ offset = 0+ for i, n in enumerate(meta.seq_lens):+ chunk = x[offset : offset + n].transpose(0, 1).unsqueeze(0)+ history = torch.cat([state[i : i + 1], chunk], dim=-1)+ out = F.conv1d(history, weight, groups=groups, dilation=dilation)+ outs.append(out.squeeze(0).transpose(0, 1))+ new_state[i] = history[0, :, -state_len:]+ offset += n+ states.index_copy_(0, slots, new_state.to(states.dtype))+ return F.silu(torch.cat(outs, dim=0))+++class PLELayer(BaseOP):+ """PLE block: hashed n-gram value gated by the residual streams, then a dilated depthwise conv.++ ``forward(R, batch) -> D [T, hc_count*hidden]``; the caller adds ``D`` to ``R`` before the+ attention hyper-connection mix. ``meta`` defaults to ``build_ple_metadata(batch, ...)``;+ ``conv_states`` defaults to ``ctx.linear_state_pool.slot_state("ple_conv", layer_id)`` and is+ ``[num_slots, hc_count*hidden, (ple_conv_kernel_size-1)*ngram_size]`` in the model dtype -- the+ last conv-input columns per request, oldest first. Both are arguments so the reference is+ testable before the pool and the scheduler carry them.++ ``start_prefetch(batch)`` builds the metadata and starts the table gather on the backend's side+ stream; call it at the top of the model forward so the rows land while layer 0 runs, and+ ``forward`` joins it.++ Weight keys (checkpoint names, prefix stripped): ``key_proj.weight`` [hc*hidden, ple_embed_dim],+ ``value_proj.weight`` [hidden, ple_embed_dim], ``norm_key/norm_query/norm_conv.weight``+ [hc*hidden] (zero-centered, loaded RAW), ``conv1d.weight`` [hc*hidden, 1, kernel], plus the+ three ``ple_embedding`` int64 hash buffers.+ """++ def __init__(+ self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None+ ) -> None:+ args = config.qwen4_args+ self.args = args+ self.layer_id = layer_id+ self.ple_index = args.ple_layer_ids.index(layer_id)+ self.hc_count = args.hc_count+ self.hidden_size = args.hidden_size+ self.dilation = args.ple_conv_dilation+ self.state_len = args.ple_conv_state_len+ width = args.ple_state_width+ self.ple_embedding = NGramEmbedding(args, table)+ self.key_proj = LinearReplicated(args.ple_embed_dim, width, has_bias=False)+ self.value_proj = LinearReplicated(args.ple_embed_dim, args.hidden_size, has_bias=False)+ self.norm_key = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)+ self.norm_query = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)+ self.norm_conv = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)+ self.conv1d = _DepthwiseConv1d(width, args.ple_conv_kernel_size)+ from freetoken.kernel.fla.chunk import CHUNK_SIZE++ # the track snapshot gathers the last state_len conv inputs before a xCHUNK boundary; a longer history would reach before the forward's first token+ assert self.state_len <= CHUNK_SIZE, (+ f"PLE conv history {self.state_len} exceeds CHUNK_SIZE {CHUNK_SIZE}"+ )+ self._pending: Tuple[PLEMetadata, torch.Tensor] | None = None++ def start_prefetch(self, batch: Batch, meta: PLEMetadata | None = None) -> None:+ """Hash this forward's n-grams and start the table gather on the side stream."""+ if meta is None:+ meta = build_ple_metadata(batch, self.args, batch.input_ids.device)+ row_ids = self.ple_embedding.row_ids(meta)+ self._pending = (meta, row_ids)+ self.ple_embedding.table.prefetch(row_ids)++ def forward(+ self,+ R: torch.Tensor,+ batch: Batch,+ meta: PLEMetadata | None = None,+ conv_states: torch.Tensor | None = None,+ ) -> torch.Tensor:+ pending, self._pending = self._pending, None+ row_ids = None+ if meta is None:+ if pending is not None:+ meta, row_ids = pending+ else:+ meta = build_ple_metadata(batch, self.args, R.device)+ elif pending is not None and pending[0] is meta:+ row_ids = pending[1]+ if row_ids is None:+ row_ids = self.ple_embedding.row_ids(meta)++ embeddings = self.ple_embedding.table.lookup(row_ids).to(R.dtype)+ key = self.norm_key.forward(self.key_proj.forward(embeddings))+ value = self.value_proj.forward(embeddings)+ query = self.norm_query.forward(R)+ shape = (-1, self.hc_count, self.hidden_size)+ gate = (key.view(shape) * query.view(shape)).sum(-1, keepdim=True) / math.sqrt(self.hidden_size)+ gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())+ gated = (gate * value.unsqueeze(-2)).flatten(-2)+ states = conv_states if conv_states is not None else self._conv_state_slab(R)+ x = self.norm_conv.forward(gated)+ fla = getattr(batch, "fla_metadata", None)+ if fla is not None and fla.track_boundary_row is not None:+ self._write_track_snapshot(states, x, fla)+ return gated + self._short_conv(x, meta, states)++ def _write_track_snapshot(self, states: torch.Tensor, x: torch.Tensor, fla) -> None:+ """Copy the conv history at the GDN track boundary into the same donatable slot, so a radix+ prefix hit restores PLE and GDN state together. Track slots never alias the live slots this+ forward advances, so the two writes are order-independent."""+ src = fla.track_boundary_row.unsqueeze(1) + torch.arange(+ -self.state_len, 0, device=x.device+ )+ window = x[src].transpose(-1, -2).contiguous()+ states.index_copy_(0, fla.track_dst, window.to(states.dtype))++ def _conv_state_slab(self, R: torch.Tensor) -> torch.Tensor:+ pool = get_global_ctx().linear_state_pool+ assert pool is not None, "PLE needs ctx.linear_state_pool or an explicit conv_states"+ assert pool.has_slot_state(PLE_CONV_STATE), (+ "ModelConfig.slot_states does not declare the PLE conv history"+ )+ return pool.slot_state(PLE_CONV_STATE, self.layer_id)++ def _read_state(+ self, meta: PLEMetadata, states: torch.Tensor, dtype: torch.dtype+ ) -> torch.Tensor:+ state = states.index_select(0, meta.state_slots).to(dtype)+ if meta.fresh_slots is not None:+ state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state)+ return state++ def _short_conv(+ self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor+ ) -> torch.Tensor:+ """silu of the dilated depthwise conv over [state | x], and roll the per-request state."""+ if meta.is_decode:+ return self._decode_conv(x, meta, states)+ return self._prefill_conv(x, meta, states)++ def _decode_conv(+ self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor+ ) -> torch.Tensor:+ """Batched tap read: taps t-9, t-6, t-3 come off the state slab, tap t from this token."""+ state = self._read_state(meta, states, x.dtype)+ column = x.unsqueeze(-1)+ # fp32 products, like the conv1d the prefill path runs+ window = torch.cat([state[..., :: self.dilation], column], dim=-1).float()+ out = (window * self.conv1d.weight.squeeze(1).float()).sum(-1)+ states.index_copy_(+ 0, meta.state_slots, torch.cat([state[..., 1:], column], dim=-1).to(states.dtype)+ )+ return F.silu(out.to(x.dtype))++ def _prefill_conv(+ self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor+ ) -> torch.Tensor:+ """One conv over every request packed as ``[state_0 | chunk_0 | state_1 | chunk_1 | ...]``.++ The blocks abut exactly, so each output window stays inside its own request: request i's+ first token reads history columns base_i .. base_i+state_len, which is its own state.+ """+ lens = list(meta.seq_lens)+ num_reqs, width = len(lens), x.shape[1]+ out_index, state_index, next_state_index = self._prefill_indices(lens, x.device)++ state = self._read_state(meta, states, x.dtype)+ history = x.new_empty(width, x.shape[0] + num_reqs * self.state_len)+ history.index_copy_(1, state_index, state.permute(1, 0, 2).reshape(width, -1))+ history.index_copy_(1, out_index + self.state_len, x.transpose(0, 1).contiguous())++ out = F.conv1d(+ history.unsqueeze(0), self.conv1d.weight, groups=width, dilation=self.dilation+ ).squeeze(0)+ new_state = history.index_select(1, next_state_index).view(width, num_reqs, self.state_len)+ states.index_copy_(+ 0, meta.state_slots, new_state.permute(1, 0, 2).to(states.dtype).contiguous()+ )+ return F.silu(out.index_select(1, out_index).transpose(0, 1))++ def _prefill_indices(self, lens: List[int], device: torch.device):+ """Columns of the packed history: this forward's outputs, the state block, the next state block."""+ state_len = self.state_len+ counts = torch.tensor(lens, dtype=torch.int64)+ cu = torch.cat([counts.new_zeros(1), counts.cumsum(0)])+ pad = torch.arange(len(lens), dtype=torch.int64) * state_len+ base = cu[:-1] + pad+ out_index = torch.arange(int(cu[-1])) + torch.repeat_interleave(pad, counts)+ span = torch.arange(state_len, dtype=torch.int64)+ packed = torch.cat(+ [+ out_index,+ (base.unsqueeze(1) + span).reshape(-1),+ ((base + counts).unsqueeze(1) + span).reshape(-1),+ ]+ )+ if torch.cuda.is_available():+ packed = packed.pin_memory()+ packed = packed.to(device, non_blocking=True)+ n_out, n_state = out_index.numel(), len(lens) * state_len+ return packed[:n_out], packed[n_out : n_out + n_state], packed[n_out + n_state :]+++__all__ = [+ "GpuResidentTable",+ "NGramEmbedding",+ "ZeroTable",+ "PLELayer",+ "PLEMetadata",+ "PLETableBackend",+ "PinnedUVATable",+ "build_ple_metadata",+ "derive_ngram_hash_constants",+ "short_conv_reference",+]diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.pynew file mode 100644index 00000000..f8d2a749--- /dev/null+++ b/python/freetoken/models/qwen4_exp/weight.py@@ -0,0 +1,328 @@+"""Qwen3.8-Flash-Next (RadixArk NVFP4) checkpoint reader.++Three separate paths, because the checkpoint's three weight classes live in different places:++* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_FUSIONS``.+* :func:`load_ple_table` -- the 47.7 GiB FP8 n-gram table, 128 checkpoint shards concatenated into one pinned :class:`HostBank`.+* :func:`load_nvfp4_expert_sources` -- the routed NVFP4 experts, into the offload cache's source banks.++Dropped: ``mtp.*`` (speculative head, including its stacked ``mtp.layers.0.mlp.experts.*``) and ``model.visual.*`` (served text-only).+"""++from __future__ import annotations++import json+import os+import re+import struct+from dataclasses import dataclass+from typing import Iterator++import safetensors+import torch+from freetoken.distributed import get_tp_info+from freetoken.models.loader import drop_page_cache, iter_weight_files+from freetoken.models.nvfp4_banks import (+ Nvfp4ExpertSourceSpec,+ load_nvfp4_expert_source_banks,+)+from freetoken.moe.host_banks import HostBank, read_range_into+from freetoken.utils import download_hf_weight+from freetoken.utils.progress import byte_bar+from tqdm import tqdm++# Routed NVFP4 experts (nvidia modelopt layout): per-expert, un-fused. Matched against the RAW+# weight_map key in nvfp4_banks. The ``model.language_model.`` anchor excludes the MTP head's+# stacked ``mtp.layers.N.mlp.experts.*`` tensors.+_EXPERT_KEY_RE = re.compile(+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."+ r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"+)+_EXPERT_RE = re.compile(r"\.mlp\.experts\.\d+\.")+_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(+ key_pattern=_EXPERT_KEY_RE,+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},+ layer_to_bank=lambda layer, config: layer, # every layer is MoE+ desc="Qwen3.8-Flash-Next NVFP4 experts",+)+# Per-tensor modelopt quant scales; consumed with their ``.weight`` (experts) or unused.+_SCALE_SUFFIXES = (".weight_scale", ".weight_scale_2", ".input_scale")++# The n-gram table itself: too big for the dense state dict, loaded by load_ple_table.+_PLE_TABLE_INFIX = ".ple.ple_embedding.ngram_embedding."+_PLE_SHARD_RE = re.compile(+ r"\.ple\.ple_embedding\.ngram_embedding\.shard_(?P<shard>\d+)\.weight$"+)+_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"++# Zero-centered Qwen4ExpTextRMSNorm weights, loaded RAW: GroupedPlusOneRMSNorm / GemmaPlusOneRMSNorm+# and the vendored grouped_gemma_rmsnorm all apply (1+w) at runtime in fp32, so folding the +1 into+# the bf16 weight here would double-apply it and round away small |w|. The GDN gated norm+# (linear_attn.norm) is a plain weight*x norm and is not in this set.+_ZERO_CENTERED_NORM_SUFFIXES = (+ ".hc_norm.weight",+ ".ple.norm_key.weight",+ ".ple.norm_query.weight",+ ".ple.norm_conv.weight",+ ".self_attn.q_norm.weight",+ ".self_attn.k_norm.weight",+ ".self_attn.indexer.q_layernorm.weight",+ ".self_attn.indexer.k_layernorm.weight",+)++# Fused projections: concat the checkpoint parts along dim 0 in this exact order. A nonzero pad+# rounds the merged row count up; the model splits the result back with the same sizes.+_FUSIONS: dict[str, tuple[tuple[str, ...], int]] = {+ # q carries the output gate, so its half is twice the attention width: [2*qo | kv | kv].+ ".self_attn.qkv_proj.weight": ((+ ".self_attn.q_proj.weight", ".self_attn.k_proj.weight", ".self_attn.v_proj.weight",+ ), 0),+ ".linear_attn.in_proj.weight": ((+ ".linear_attn.in_proj_qkv.weight", ".linear_attn.in_proj_z.weight",+ ".linear_attn.in_proj_b.weight", ".linear_attn.in_proj_a.weight",+ ), 0),+ ".mlp.shared_expert.gate_up_proj.weight": ((+ ".mlp.shared_expert.gate_proj.weight", ".mlp.shared_expert.up_proj.weight",+ ), 0),+ # HC mix reads the low-rank down projection and the injection logits from one GEMM; vLLM+ # pads the merged output to a multiple of 16 rows for cuBLAS (hyperconnection.py pad_size).+ # The top-level hyper_connection_mixer has no injection and so never fuses.+ ".attn_hyper_connection.input_mix_weight_down_block_inject.weight": ((+ ".attn_hyper_connection.input_mix_weight_down.weight",+ ".attn_hyper_connection.block_inject_weight.weight",+ ), 16),+ ".mlp_hyper_connection.input_mix_weight_down_block_inject.weight": ((+ ".mlp_hyper_connection.input_mix_weight_down.weight",+ ".mlp_hyper_connection.block_inject_weight.weight",+ ), 16),+}+++def _rename(raw_name: str) -> str | None:+ """Checkpoint key -> FreeToken state-dict key, or None to skip."""+ if raw_name.startswith(("mtp.", "model.visual.", "visual.")):+ return None+ if _PLE_TABLE_INFIX in raw_name:+ return None # n-gram table + its scale: load_ple_table+ if _EXPERT_RE.search(raw_name):+ return None # routed experts: offload source banks+ if raw_name.endswith(_SCALE_SUFFIXES):+ return None+ if raw_name.startswith("model.language_model."):+ return "model." + raw_name[len("model.language_model.") :]+ if raw_name.startswith("language_model."):+ return "model." + raw_name[len("language_model.") :]+ return raw_name+++def _try_fuse(+ name: str, tensor: torch.Tensor, buf: dict[str, dict[int, torch.Tensor]]+) -> tuple[str, torch.Tensor] | tuple[()] | None:+ """Buffer a fusion part; return the merged ``(name, tensor)`` once all parts arrive, ``()`` while incomplete, ``None`` if ``name`` is not a fusion part."""+ for fused_suffix, (parts, pad_to) in _FUSIONS.items():+ for idx, part in enumerate(parts):+ if not name.endswith(part):+ continue+ key = name[: -len(part)] + fused_suffix+ slots = buf.setdefault(key, {})+ slots[idx] = tensor+ if len(slots) < len(parts):+ return ()+ del buf[key]+ rows = [slots[i] for i in range(len(parts))]+ pad = (-sum(t.shape[0] for t in rows)) % pad_to if pad_to else 0+ if pad:+ rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=rows[0].dtype, device=rows[0].device))+ return key, torch.cat(rows, dim=0)+ return None+++def iter_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ """Yield the dense (non-expert) weights, prefix-stripped and fused to the model's buffers.++ Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the+ model's state dict minus the routed experts. Nothing here is quantized: the modelopt+ ``ignore`` list covers everything except those experts, so attention, GDN, HC, PLE, the shared+ expert and lm_head are all plain bf16 (the n-gram hash constants stay int64). Fusions:+ attention q|k|v -> ``qkv_proj``, GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, shared-expert+ gate|up -> ``gate_up_proj``, and each per-layer HC's ``input_mix_weight_down`` |+ ``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``.++ ``include_moe_experts`` is accepted for the loader contract but never yields anything: the+ routed experts are NVFP4 and always come from :func:`load_nvfp4_expert_sources`.+ """+ if get_tp_info().size > 1:+ raise NotImplementedError("qwen4_exp weight loading supports TP=1 only")+ if not include_non_moe:+ return++ fuse_buf: dict[str, dict[int, torch.Tensor]] = {}+ for file in tqdm(+ iter_weight_files(model_path),+ desc="Loading weights",+ disable=not get_tp_info().is_primary(),+ ):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ name = _rename(raw_name)+ if name is None:+ continue+ tensor = f.get_tensor(raw_name)+ fused = _try_fuse(name, tensor, fuse_buf)+ if fused is not None:+ if fused != (): # () means buffered, not yet complete+ yield fused+ continue+ yield name, tensor++ assert not fuse_buf, f"Incomplete projection fusions: {sorted(fuse_buf)}"+++# ======================================================================================+# PLE n-gram table+# ======================================================================================+++@dataclass(frozen=True)+class PleTable:+ """The filled n-gram table: one pinned host bank plus the checkpoint's per-tensor FP8 scale."""++ bank: HostBank+ weight_scale: torch.Tensor # scalar, checkpoint dtype (bf16)++ @property+ def tensor(self) -> torch.Tensor:+ """``[total_rows, ngram_head_dim]`` float8_e4m3fn view of the bank."""+ return self.bank.tensor+++_PLE_ST_DTYPE = "F8_E4M3"+++def _safetensors_header(path: str) -> tuple[dict, int]:+ with open(path, "rb") as fh:+ n = struct.unpack("<Q", fh.read(8))[0]+ return json.loads(fh.read(n)), 8 + n+++def _ple_table_files(folder: str) -> list[str]:+ """Shards holding a piece of the n-gram table, from the index when there is one."""+ index = os.path.join(folder, "model.safetensors.index.json")+ if not os.path.exists(index):+ return sorted(iter_weight_files(folder))+ with open(index, encoding="utf-8") as fh:+ weight_map = json.load(fh)["weight_map"]+ files = {shard for name, shard in weight_map.items() if _PLE_TABLE_INFIX in name}+ return sorted(os.path.join(folder, shard) for shard in files)+++def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True,+ workers: int = 8, chunk: int = 8 << 20) -> PleTable:+ """Concatenate the checkpoint's ``ngram_embedding.shard_<i>`` tensors into one pinned host bank.++ The checkpoint splits the table into ``split_ngram_parts`` equal row blocks named by shard+ index and scattered over the ``model-plefp8-*`` shards in header (lexicographic) order, so the+ bank is filled shard by shard at ``shard_index * rows_per_shard``. Each read is O_DIRECT: the+ table is ~47.7 GiB and must not also sit in the page cache while the bank holds the same bytes.+ """+ folder = download_hf_weight(model_path)+ parts: dict[int, tuple[str, int, int]] = {} # shard index -> (path, file offset, bytes)+ scale: torch.Tensor | None = None+ rows = cols = 0+ for path in _ple_table_files(folder):+ header, base = _safetensors_header(path)+ for key, meta in header.items():+ if key == "__metadata__":+ continue+ if key.endswith(_PLE_SCALE_SUFFIX):+ with safetensors.safe_open(path, framework="pt", device="cpu") as f:+ scale = f.get_tensor(key).reshape(())+ continue+ match = _PLE_SHARD_RE.search(key)+ if match is None:+ continue+ if meta["dtype"] != _PLE_ST_DTYPE:+ raise ValueError(f"PLE table shard {key} has unsupported dtype {meta['dtype']}")+ shape = meta["shape"]+ if rows and tuple(shape) != (rows, cols):+ raise ValueError(f"PLE table shard {key} is {shape}, expected {[rows, cols]}")+ rows, cols = shape+ begin, end = meta["data_offsets"]+ parts[int(match.group("shard"))] = (path, base + begin, end - begin)++ expected = int(qwen4_args.split_ngram_parts)+ if sorted(parts) != list(range(expected)):+ raise ValueError(+ f"PLE table needs shards 0..{expected - 1}, found {len(parts)}: {sorted(parts)[:8]}"+ )+ if cols != qwen4_args.ngram_head_dim:+ raise ValueError(f"PLE table row is {cols} wide, config says {qwen4_args.ngram_head_dim}")+ if scale is None:+ raise ValueError("PLE table has no weight_scale")++ bank = HostBank((expected * rows, cols), torch.float8_e4m3fn)+ shard_bytes = rows * cols+ bar = byte_bar(expected * shard_bytes, "Loading PLE table")+ try:+ buf = bank.memoryview()+ for shard in range(expected):+ path, offset, nbytes = parts[shard]+ assert nbytes == shard_bytes, f"PLE shard {shard} is {nbytes} B, expected {shard_bytes}"+ read_range_into(buf, path, file_offset=offset, nbytes=nbytes,+ dest_offset=shard * shard_bytes, workers=workers, chunk=chunk)+ bar.update(nbytes)+ finally:+ bar.close()+ if pin and torch.cuda.is_available():+ bank.pin()+ return PleTable(bank=bank, weight_scale=scale)+++# ======================================================================================+# Routed NVFP4 experts+# ======================================================================================+++def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict:+ """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the output-row axis, down separate; weight_scale_2 carried as the per-row global scale)."""+ return load_nvfp4_expert_source_banks(+ model_path,+ config,+ _NVFP4_SOURCE_SPEC,+ drop_page_cache=drop_page_cache,+ primary=get_tp_info().is_primary(),+ layer_sink=layer_sink,+ )+++def load_nvfp4_expert_sources_parallel(+ model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None+) -> dict:+ """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""+ from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel++ return load_nvfp4_expert_source_banks_parallel(+ model_path,+ config,+ _NVFP4_SOURCE_SPEC,+ drop_page_cache=drop_page_cache,+ primary=get_tp_info().is_primary(),+ workers=workers,+ chunk=chunk,+ layer_sink=layer_sink,+ )+++__all__ = [+ "PleTable",+ "iter_weights",+ "load_nvfp4_expert_sources",+ "load_nvfp4_expert_sources_parallel",+ "load_ple_table",+]diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 0c033ca0..b94d8291 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -58,6 +58,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"freetoken.models.qwen3_5_moe","Qwen3_5MoEForCausalLM",),+ # Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in+ # text_config, weights under model.language_model.); served text-only. 36 GDN + 12 QSA+ # compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE+ # n-gram embedding layer, 512 NVFP4 routed experts top-10 + a gated shared expert.+ "Qwen4ExpForConditionalGeneration": ModelSpec(+ "freetoken.models.qwen4_exp",+ "Qwen4ExpForCausalLM",+ ),# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the# loader handles the compressed-tensors NVFP4 layout.diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.pyindex fe7e417d..4b9a4875 100644--- a/python/freetoken/moe/fused.py+++ b/python/freetoken/moe/fused.py@@ -62,6 +62,13 @@ def fused_topk()return _torch_fused_topk(gating_output, topk, renormalize, num_token_non_padded)+ if topk & (topk - 1):+ # triton_kernels.topk builds tl.arange(0, k), which must be a power of 2; a+ # top-10 router (qwen4_exp) takes the equivalent vendored triton router instead.+ from freetoken.kernel.triton.moe_router import fused_topk_softmax++ return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)+from triton_kernels.topk import topk as triton_kernels_topklogits = gating_output.float()diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.pyindex 5ed8d766..29a561e6 100644--- a/python/freetoken/moe/fused_nvfp4.py+++ b/python/freetoken/moe/fused_nvfp4.py@@ -61,6 +61,12 @@ _DECODE_WARPS = 4_DECODE_MARLIN_BLOCK_N = 16_DECODE_MARLIN_BLOCK_KW = 16_DECODE_MARLIN_WARPS = 4+# Deep-K variant: at K > 2048 (qwen4_exp gate_up, K=2560) a narrower N tile with the whole+# K strip in one program iteration measures ~13% faster (18.6 vs 21.0us); short-K shapes+# regress under it, so the split is by K, not by gemm position.+_DECODE_MARLIN_DEEPK_BLOCK_N = 8+_DECODE_MARLIN_DEEPK_BLOCK_KW = 128+_DECODE_MARLIN_DEEPK_THRESHOLD = 2048def _tl_dtype(dt: torch.dtype):@@ -129,7 +135,10 @@ def _decode_gemm_marlin(packed_i32 = packed.view(torch.int32) # [S, N, K // 8]scale = e4m3_kernel_view(scale)total_routes = M * top_k- grid = (total_routes, triton.cdiv(N, _DECODE_MARLIN_BLOCK_N))+ deep_k = K > _DECODE_MARLIN_DEEPK_THRESHOLD+ block_n = _DECODE_MARLIN_DEEPK_BLOCK_N if deep_k else _DECODE_MARLIN_BLOCK_N+ block_kw = _DECODE_MARLIN_DEEPK_BLOCK_KW if deep_k else _DECODE_MARLIN_BLOCK_KW+ grid = (total_routes, triton.cdiv(N, block_n))_decode_nvfp4_marlin_kernel[grid](a, packed_i32, scale, glob, c, topk_weights, topk_ids,_e2m1_lut(a.device.index),@@ -141,8 +150,8 @@ def _decode_gemm_marlin(c.stride(0), c.stride(1), c.stride(2),topk_weights.stride(0), topk_weights.stride(1),topk_ids.stride(0), topk_ids.stride(1),- BLOCK_SIZE_N=_DECODE_MARLIN_BLOCK_N,- BLOCK_SIZE_KW=_DECODE_MARLIN_BLOCK_KW,+ BLOCK_SIZE_N=block_n,+ BLOCK_SIZE_KW=block_kw,TOP_K=top_k,A_ROW_IS_ROUTE=a_row_is_route,MUL_ROUTED_WEIGHT=mul_routed_weight,diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.pyindex d7af348a..436f52d2 100644--- a/python/freetoken/moe/host_banks.py+++ b/python/freetoken/moe/host_banks.py@@ -410,6 +410,67 @@ def read_file_into(buf: memoryview | mmap.mmap, path: str, *, workers: int = 8,return size+def _preadv_all(fd: int, dst: memoryview, offset: int, need: int) -> None:+ """preadv into ``dst`` until ``need`` bytes have landed; O_DIRECT may return a short count."""+ done = 0+ while done < need:+ if done % _BLK: # a continuation read has to stay block-aligned on both sides+ raise OSError(f"unaligned short O_DIRECT read: {done} of {need} bytes at {offset}")+ got = os.preadv(fd, [dst[done:]], offset + done)+ if got <= 0:+ raise OSError(f"short O_DIRECT read: {done} of {need} bytes at {offset}")+ done += got+++def read_range_into(buf: memoryview | mmap.mmap, path: str, *, file_offset: int, nbytes: int,+ dest_offset: int = 0, workers: int = 8, chunk: int = _DEFAULT_CHUNK,+ drop_cache: bool = True) -> int:+ """Chunked multi-threaded O_DIRECT read of ``path[file_offset : file_offset + nbytes]`` into ``buf`` at ``dest_offset``. Returns ``nbytes``.++ Byte-range counterpart of :func:`read_file_into`, for one tensor inside a shard. O_DIRECT needs the file offset AND the destination address block-aligned at the same time, which only holds when the two share their offset mod 4096 -- a safetensors data offset practically never lines up with the tensor's slot in the bank. Chunks that do line up DMA straight into ``buf``; the rest DMA into a page-aligned bounce (source window rounded out to whole blocks) and are copied into place, which also covers the unaligned head and tail.+ """+ mv = (buf if isinstance(buf, memoryview) else memoryview(buf)).cast("B")+ if dest_offset + nbytes > len(mv):+ raise ValueError(f"destination holds {len(mv)} bytes, need {dest_offset + nbytes}")+ base = ctypes.addressof(ctypes.c_char.from_buffer(mv))+ if drop_cache:+ try:+ fd0 = os.open(path, os.O_RDONLY)+ os.posix_fadvise(fd0, file_offset, nbytes, os.POSIX_FADV_DONTNEED)+ os.close(fd0)+ except OSError:+ pass+ fd = os.open(path, os.O_RDONLY | os.O_DIRECT)+ scratch = threading.local()++ def rd(i: int) -> None:+ n = min(chunk, nbytes - i)+ src, dst = file_offset + i, dest_offset + i+ if src % _BLK == 0 and (base + dst) % _BLK == 0 and n % _BLK == 0:+ _preadv_all(fd, mv[dst:dst + n], src, n)+ return+ head = src % _BLK+ span = ((head + n + _BLK - 1) // _BLK) * _BLK+ bounce = getattr(scratch, "buf", None)+ if bounce is None or len(bounce) < span:+ bounce = scratch.buf = mmap.mmap(-1, span) # anonymous mmaps are page-aligned+ bmv = memoryview(bounce)+ _preadv_all(fd, bmv[:span], src - head, head + n)+ mv[dst:dst + n] = bmv[head:head + n]++ try:+ offs = list(range(0, nbytes, chunk))+ if len(offs) <= 1:+ for o in offs:+ rd(o)+ else:+ with ThreadPoolExecutor(workers) as ex:+ list(ex.map(rd, offs))+ finally:+ os.close(fd)+ return nbytes++__all__ = ["HostBank","HostResidency",@@ -420,5 +481,6 @@ __all__ = ["born_pinned_default","pin_banks","read_file_into",+ "read_range_into","requested_residency",]diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.pyindex 6ee76406..33a47553 100644--- a/python/freetoken/moe/offload_cache.py+++ b/python/freetoken/moe/offload_cache.py@@ -77,11 +77,21 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {"ds_fp4": ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale"),}+def fp8_block_scale_pad(rows: int, cols: int) -> int:+ """Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy)."""+ while (rows * cols * 2) % 16:+ cols += 1+ return cols++# bytes per (expert, layer) as f(hidden, moe_intermediate), from the bank shapes above; keep in sync with _BANK_SCHEMAS# keyed by the config-time format tag (expert_quant / moe_weight_format), not quant_format: "mxfp4" sizes the mxfp4_triton banks, "nvfp4" also covers its repacked variants_BANK_BYTES_PER_EXPERT = {"bf16": lambda H, I: 3 * I * H * 2,- "fp8_block": lambda H, I: 3 * I * H + ((2 * I // 128) * (H // 128) + (H // 128) * (I // 128)) * 2,+ "fp8_block": lambda H, I: 3 * I * H + (+ (2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128)+ + (H // 128) * fp8_block_scale_pad(H // 128, I // 128)+ ) * 2,"q4_0": lambda H, I: 2 * I * (H // 32) * 18 + H * (I // 32) * 18,"nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2),"mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2),diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.pyindex 9be235b2..44adde42 100644--- a/python/freetoken/scheduler/cache.py+++ b/python/freetoken/scheduler/cache.py@@ -66,6 +66,13 @@ class CacheManager:supports_runtime_rebuild = Trueprefill_chunk_budget = None # generic shared page pool: no per-model prefill chunk cap+ @property+ def prefill_chunk_align(self) -> int:+ """Granularity a non-final prefill chunk should end on. A hybrid snapshot is donated only+ at a page-aligned boundary, so at page_size>1 one unaligned chunk end costs every reuse+ point for the rest of the prompt. 1 (no-op) everywhere else."""+ return self.page_size if self.is_hybrid else 1+def page_usage(self) -> tuple[int, int]:"""(used_pages, total_pages): allocated, non-evictable pages over the pool total(active requests + protected prefix; evictable prefix-cache pages are excluded)."""diff --git a/python/freetoken/scheduler/prefill.py b/python/freetoken/scheduler/prefill.pyindex be84874c..f5bc8f7a 100644--- a/python/freetoken/scheduler/prefill.py+++ b/python/freetoken/scheduler/prefill.py@@ -156,6 +156,13 @@ class PrefillAdder:self.reserved_swa += (div_ceil(cached_len + chunk_size, ps) - div_ceil(cached_len, ps)) * ps+ align = self.cache_manager.prefill_chunk_align+ if align > 1 and 0 < chunk_size < remain_len:+ # An unaligned chunk end is correct, it just loses this prompt's snapshot boundaries --+ # so keep it when the leftover budget cannot fill one whole unit instead of stalling+ # the request until it gets a bigger turn.+ aligned = align_down(cached_len + chunk_size, align) - cached_len+ chunk_size = aligned if aligned > 0 else chunk_sizeis_chunked = chunk_size < remain_lenCLS = ChunkedReq if is_chunked else Reqself.token_budget -= chunk_sizediff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex a71b6819..6696f65d 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -147,6 +147,8 @@ def parse_args(return "muse_glimmer"if "gemma4" in marker:return "gemma4"+ if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker:+ return "qwen3_coder"if ("qwen3_5" in markeror "qwen3.5" in marker@@ -188,6 +190,8 @@ def parse_args(tag in marker for tag in ("v4", "deepseek_v4", "v3.2", "v32")):return "deepseekv32"+ if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker:+ return "qwen3"if "qwen3" in marker or "qwen3.5" in marker or "qwen3_5" in marker:return "qwen3"if "glm" in marker:diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.pyindex f27640b9..fc9abfbe 100644--- a/tests/engine/test_attention_backend_matrix.py+++ b/tests/engine/test_attention_backend_matrix.py@@ -24,7 +24,7 @@ from freetoken.models.config import ()-def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0):+def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0, index_ratio=1):return KVCacheGroupSpec(name=name,layer_ids=(0, 1),@@ -34,6 +34,7 @@ def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0):mla=mla,index_head_dim=index_head_dim,num_index_layers=2 if index_head_dim else 0,+ index_ratio=index_ratio,attn_type=attn_type,)@@ -67,6 +68,11 @@ def _model_config(kind):elif kind == "bsa":# MiniMax-M3 shape: one FULL-family group, mla=False + index dims -> BSA.specs = (_spec("full", AttnType.BSA, index_head_dim=128),)+ elif kind == "qsa":+ # Qwen3.8-Flash-Next shape: hybrid-linear + one FULL-family group whose index keys+ # are compressed index_ratio:1 -> QSA.+ mc.has_linear_attention = True+ specs = (_spec("full", AttnType.QSA, index_head_dim=128, index_ratio=4),)elif kind == "linear_hybrid":mc.has_linear_attention = Truespecs = (_spec("full", AttnType.FULL),)@@ -109,6 +115,7 @@ def _patch_env(monkeypatch, *, major=9, flashinfer=True, sgl=True):("dsa", "dsa"), # MLA + DSA indexer (GLM-5.2 shape)("dsv4", "dsv4_sparse"),("bsa", "m3_sparse"), # MiniMax-M3 block-sparse GQA+ ("qsa", "qsa_sparse"), # Qwen3.8-Flash-Next compressed-block sparse],)def test_auto_resolves_per_type(monkeypatch, kind, expected):@@ -143,6 +150,39 @@ def test_bsa_rejects_float32_dtype(monkeypatch):_adjust_config(config)+def test_auto_qsa_sets_page_size_64(monkeypatch):+ # qsa_sparse registers page_sizes=(64,) (a 4-token compress group must never straddle+ # a page); the generic backend page-size coercion takes the default 1 to 64.+ from freetoken.engine.engine import _adjust_config++ _patch_env(monkeypatch)+ config = _config("qsa", attention_backend="auto")+ assert config.page_size == 1+ _adjust_config(config)+ assert config.page_size == 64+++def test_qsa_coerces_explicit_page_size(monkeypatch):+ # Same policy as m3_sparse (page_sizes=(128,)): an unsupported explicit value is+ # coerced to the backend's page size with a warning, not rejected.+ from freetoken.engine.engine import _adjust_config++ _patch_env(monkeypatch)+ config = _config("qsa", attention_backend="auto", page_size=16)+ _adjust_config(config)+ assert config.page_size == 64+++def test_qsa_rejects_float32_dtype(monkeypatch):+ from freetoken.engine.engine import _adjust_config++ _patch_env(monkeypatch)+ config = _config("qsa", attention_backend="auto")+ object.__setattr__(config, "dtype", torch.float32)+ with pytest.raises(ValueError, match="16-bit"):+ _adjust_config(config)++def test_auto_dsv4_sets_window_page_size(monkeypatch):from freetoken.engine.engine import _adjust_config@@ -160,9 +200,16 @@ def test_auto_dsv4_sets_window_page_size(monkeypatch):("full", "dsv4_sparse"),("full", "m3_sparse"),("swa", "dsa"),+ ("full", "qsa_sparse"),# forward gates: generic backends on the BSA-locked model("bsa", "fi"),("bsa", "triton"),+ # forward gates: generic and neighbouring sparse backends on the QSA-locked model+ ("qsa", "fi"),+ ("qsa", "fa"),+ ("qsa", "triton"),+ ("qsa", "m3_sparse"),+ ("bsa", "qsa_sparse"),# forward gates: generic backends on type-locked models("mla", "fi"),("mla", "triton"),@@ -196,6 +243,7 @@ def test_illegal_combinations_rejected_at_config_time(monkeypatch, kind, backend("mla", "dsa"),("dsa", "dsa"),("dsv4", "dsv4_sparse"),+ ("qsa", "qsa_sparse"),("swa", "triton"),("full", "triton"),("full", "fa,fi"),@@ -220,31 +268,6 @@ def test_mla_requires_page_size_one(monkeypatch, kind):_adjust_config(config)-def test_hybrid_linear_opt_out_rejects_backend(monkeypatch):- import dataclasses-- from freetoken.attention import attention_backend_info- from freetoken.engine import engine- from freetoken.engine.engine import _adjust_config-- _patch_env(monkeypatch)- real_info = attention_backend_info-- def _info(name):- info = real_info(name)- return dataclasses.replace(info, hybrid_linear_ok=False) if name == "fa" else info-- monkeypatch.setattr(engine, "attention_backend_info", _info)- # auto skips the opted-out backend ...- config = _config("linear_hybrid", attention_backend="auto")- _adjust_config(config)- assert config.attention_backend == "fi"- # ... and an explicit choice of it is rejected- config = _config("linear_hybrid", attention_backend="fa")- with pytest.raises(ValueError, match="hybrid-linear"):- _adjust_config(config)--def test_trtllm_page_size_coercion_is_part_aware(monkeypatch):from freetoken.engine.engine import _adjust_config@@ -372,3 +395,10 @@ def dataclasses_replace_groups(mc, groups):import dataclassesreturn dataclasses.replace(mc, attention_groups=groups)+++def test_linear_attention_defaults_to_hybrid_radix():+ from freetoken.engine.engine import _resolve_cache_type++ assert _resolve_cache_type(True, "radix") == "hybrid_radix"+ assert _resolve_cache_type(True, "naive") == "naive"diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.pyindex a164f0b4..9ac2a4f4 100644--- a/tests/engine/test_cache_budget.py+++ b/tests/engine/test_cache_budget.py@@ -5,7 +5,10 @@ from types import SimpleNamespaceimport pytestimport torch+import os+from freetoken.engine.cache_budget import expert_bytes_per_slot, plan_cache_budget, resolve_moe_cache_auto+from freetoken.engine.engine import _pin_budget_bytesdef test_moe_priority_fills_experts_up_to_total():@@ -480,3 +483,20 @@ def test_adjust_config_rope_gate_exempts_dsv4():cfg = _dsv4_adjust_cfg(max_seq_len_override=10_000_000)cfg.model_config.rotary_config = SimpleNamespace(max_position=1024)_adjust_config(cfg) # must not raise+++# ---- _pin_budget_bytes: host bytes already pinned outside the expert banks ----+++def test_reserved_subtracts_from_the_cap(monkeypatch):+ monkeypatch.setenv("FREETOKEN_PIN_BUDGET_GB", "2")+ assert _pin_budget_bytes() == 2 * 2**30+ assert _pin_budget_bytes(reserved=2**30) == 2**30+ assert _pin_budget_bytes(reserved=4 * 2**30) == 0+++def test_uncapped_platform_stays_uncapped(monkeypatch):+ monkeypatch.delenv("FREETOKEN_PIN_BUDGET_GB", raising=False)+ if hasattr(os, "uname") and "microsoft" in os.uname().release.lower():+ pytest.skip("WSL caps pinning")+ assert _pin_budget_bytes(reserved=2**30) is Nonediff --git a/tests/kvcache/test_kv_cache_rebuild.py b/tests/kvcache/test_kv_cache_rebuild.pyindex 6dff34ea..a5ec94a0 100644--- a/tests/kvcache/test_kv_cache_rebuild.py+++ b/tests/kvcache/test_kv_cache_rebuild.py@@ -150,7 +150,7 @@ def test_linear_state_pool_rebuild_resizes_preserves_identity_and_dtypes():_init_tp()group = LinearGatedDeltaGroupConfig(name="linear", layer_ids=(0, 1, 2), num_key_heads=4, num_value_heads=8,- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",)pool = LinearStatePool(group=group, num_slots=10, dtype=torch.bfloat16, device=torch.device("cpu"))pid = id(pool)diff --git a/tests/kvcache/test_linear_state_pool_alloc.py b/tests/kvcache/test_linear_state_pool_alloc.pyindex 16058efc..e9fe4054 100644--- a/tests/kvcache/test_linear_state_pool_alloc.py+++ b/tests/kvcache/test_linear_state_pool_alloc.py@@ -1,22 +1,31 @@-"""P1 unit: LinearStatePool free-list allocator (alloc/free/clear_slots/copy_from).+"""LinearStatePool unit: the free-list allocator and the declared slot-state siblings.CPU-only, fast — pure slot bookkeeping + state copy/zero, no kernels."""from __future__ import annotations+from types import SimpleNamespace+import pytestimport torch-from freetoken.kvcache.linear_state_pool import LinearStatePool-from freetoken.models.config import LinearGatedDeltaGroupConfig+from freetoken.kvcache.linear_state_pool import (+ LinearStatePool,+ linear_state_bytes_per_req,+ state_pool_bytes,+)+from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpec-def _pool(num_slots=8, device="cpu"):- group = LinearGatedDeltaGroupConfig(+def _group():+ return LinearGatedDeltaGroupConfig(name="linear", layer_ids=(0, 1),num_key_heads=2, num_value_heads=4,- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",)- return LinearStatePool(group=group, num_slots=num_slots, dtype=torch.bfloat16,- device=torch.device(device), tp_size=1)+++def _pool(num_slots=8, device="cpu", slot_states=()):+ return LinearStatePool(group=_group(), num_slots=num_slots, dtype=torch.bfloat16,+ device=torch.device(device), tp_size=1, slot_states=slot_states)def test_alloc_free_roundtrip():@@ -70,3 +79,113 @@ if __name__ == "__main__":test_clear_slots_zeros_all_layers()test_copy_from_snapshot()print("LinearStatePool allocator unit: PASS")+++# qwen4_exp PLE conv-history shape at toy size: one PLE layer, 32 channels, 9 taps+_SPECS = (SlotStateSpec(name="ple_conv", shape=(32, 9), layer_ids=(1,)),)+++def test_no_slot_states_by_default():+ pool = _pool()+ assert pool.slot_states == {} and not pool.has_slot_state("ple_conv")+ base = pool.bytes_per_slot()+ pool.clear_slots([1, 2])+ pool.copy_from(1, 2)+ pool.reset(3)+ pool.rebuild(4)+ assert pool.slot_states == {} and pool.bytes_per_slot() == base+++def test_slot_state_geometry_and_accessor():+ pool = _pool(num_slots=6, slot_states=_SPECS)+ slab = pool.slot_states["ple_conv"]+ assert slab.shape == (1, 6, 32, 9) and slab.dtype is torch.bfloat16+ assert pool.slot_state("ple_conv", 1).shape == (6, 32, 9)+ with pytest.raises(KeyError):+ pool.slot_state("ple_conv", 0) # not a declared layer+ with pytest.raises(KeyError):+ pool.slot_state("other")+ with pytest.raises(AssertionError):+ pool.slot_state("ple_conv") # per-layer state needs layer_id+++def test_layerless_spec_dtype_and_fill_value():+ spec = SlotStateSpec(name="ngram", shape=(2,), dtype=torch.int32, fill_value=7.0)+ pool = _pool(num_slots=6, slot_states=(spec,))+ t = pool.slot_state("ngram")+ assert t.shape == (6, 2) and t.dtype is torch.int32 and bool((t == 7).all())+ t[3] = 1+ pool.clear_slots([3])+ assert bool((pool.slot_state("ngram")[3] == 7).all())+ t[2] = 1+ pool.reset(2)+ assert bool((pool.slot_state("ngram")[2] == 7).all())+ pool.rebuild(5)+ assert bool((pool.slot_state("ngram") == 7).all())+++def test_duplicate_names_rejected():+ with pytest.raises(ValueError, match="duplicate"):+ _pool(slot_states=_SPECS + _SPECS)+++def test_slot_state_follows_every_slot_operation():+ pool = _pool(slot_states=_SPECS)+ pool.slot_states["ple_conv"].fill_(1.0)+ pool.conv_states.fill_(1.0)++ pool.clear_slots([2])+ slab = pool.slot_state("ple_conv", 1)+ assert slab[2].abs().sum().item() == 0.0+ assert slab[3].abs().sum().item() > 0.0++ pool.copy_from(3, 2)+ assert torch.equal(slab[2], slab[3])++ pool.reset(3)+ assert slab[3].abs().sum().item() == 0.0++ pool.rebuild(9)+ slab = pool.slot_state("ple_conv", 1) # rebuild replaces the tensor+ assert pool.slot_states["ple_conv"].shape == (1, 9, 32, 9)+ assert slab.abs().sum().item() == 0.0++ # the ops write the rebuilt tensor, not a stale alias+ pool.slot_states["ple_conv"].fill_(1.0)+ pool.clear_slots([5])+ assert pool.slot_state("ple_conv", 1)[5].abs().sum().item() == 0.0+ pool.copy_from(1, 5)+ assert torch.equal(pool.slot_state("ple_conv", 1)[5], pool.slot_state("ple_conv", 1)[1])+++def test_slot_state_in_the_byte_account():+ group = _group()+ gdn_only = linear_state_bytes_per_req(group, 1, torch.bfloat16)+ with_state = linear_state_bytes_per_req(group, 1, torch.bfloat16, _SPECS)+ assert with_state - gdn_only == 32 * 9 * 2+ assert _pool(slot_states=_SPECS).bytes_per_slot() == with_state++ mc = SimpleNamespace(slot_states=_SPECS)+ mc.linear_attention_group = lambda: group+ config = SimpleNamespace(+ model_config=mc, dtype=torch.bfloat16, tp_info=SimpleNamespace(size=1),+ cache_type="naive", max_running_req=3, linear_state_cache_ratio=0.5,+ )+ assert state_pool_bytes(config, num_slots=4) == with_state * 4++ mc.slot_states = ()+ assert state_pool_bytes(config, num_slots=4) == gdn_only * 4++ mc.slot_states = _SPECS+ mc.linear_attention_group = lambda: None+ with pytest.raises(ValueError, match="slot_states"):+ state_pool_bytes(config, num_slots=4)+++def test_slot_state_bytes_for_the_real_geometry():+ # qwen4_exp PLE conv history: 4 streams x 2560 channels x 9 taps bf16 = 180 KiB per slot+ spec = SlotStateSpec(name="ple_conv", shape=(4 * 2560, 9), layer_ids=(1,))+ group = _group()+ delta = linear_state_bytes_per_req(group, 1, torch.bfloat16, (spec,)) - \+ linear_state_bytes_per_req(group, 1, torch.bfloat16)+ assert delta == 4 * 2560 * 9 * 2 == 180 * 1024diff --git a/tests/kvcache/test_pool_sizing_surface.py b/tests/kvcache/test_pool_sizing_surface.pyindex d50a5be4..516090b0 100644--- a/tests/kvcache/test_pool_sizing_surface.py+++ b/tests/kvcache/test_pool_sizing_surface.py@@ -249,7 +249,7 @@ def test_linear_state_pool_prices_itself():group = LinearGatedDeltaGroupConfig(name="linear", layer_ids=(1, 3), num_key_heads=2, num_value_heads=4,- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",)config = _generic_config()config.linear_state_cache_ratio = 0.5diff --git a/tests/kvcache/test_qsa_pool.py b/tests/kvcache/test_qsa_pool.pynew file mode 100644index 00000000..9aec2bb6--- /dev/null+++ b/tests/kvcache/test_qsa_pool.py@@ -0,0 +1,218 @@+"""QSAKVCache tiers (paged K/V, compressed index slab, pending ring, scratch).++Pins the three things the QSA kernels and the startup budget both depend on: the compressed+slab is a 1/index_ratio shadow of the K/V pages with the scratch rows behind it, the ring and+scratch are fixed (concurrency-sized) and priced apart from the per-token slider, and the K/V+slabs cover the sparse layers only. The PLE conv history rides the GDN slots, so it must+follow every slot operation and show up in the state-pool byte account.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++from freetoken.attention import AttnType+from freetoken.kvcache.base import spec_kv_bytes_per_token+from freetoken.kvcache.qsa_pool import QSAKVCache+from freetoken.models.config import KVCacheGroupSpec++DEV = torch.device("cpu")++# Qwen3.8-Flash-Next: 48 layers, every 4th is QSA; 2 kv heads x 256, indexer 128 wide, ratio 4.+FULL_LAYER_IDS = tuple(range(3, 48, 4))+REAL_KV_BYTES = 2 * 256 * 2 * 2 * 12+REAL_INDEX_BYTES = 128 * 12 * 2 // 4++[email protected](autouse=True)+def _tp(monkeypatch):+ from freetoken.distributed.info import DistributedInfo++ monkeypatch.setattr(+ "freetoken.kvcache.mha_pool.get_tp_info",+ lambda: DistributedInfo(rank=0, size=1),+ )+++def _pool(num_pages=4, page_size=64, index_ratio=4, num_req_slots=4, ring_capacity=None):+ return QSAKVCache(+ num_kv_heads=2,+ num_layers=8,+ head_dim=64,+ num_pages=num_pages,+ page_size=page_size,+ dtype=torch.bfloat16,+ device=DEV,+ index_head_dim=32,+ num_index_layers=4,+ index_ratio=index_ratio,+ num_req_slots=num_req_slots,+ ring_capacity=ring_capacity,+ layer_ids=(1, 3, 5, 7),+ )+++def _spec(*, index_ratio=4, attn_type=AttnType.QSA, num_kv_heads=2, head_dim=64,+ index_head_dim=32, num_index_layers=4, layer_ids=(1, 3, 5, 7)):+ return KVCacheGroupSpec(+ name="full",+ layer_ids=layer_ids,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ sliding_window=None,+ index_head_dim=index_head_dim,+ num_index_layers=num_index_layers,+ index_ratio=index_ratio,+ attn_type=attn_type,+ )+++def _config(spec, *, page_size=64, max_running_req=3):+ mc = SimpleNamespace(num_layers=8, has_swa_attention=False, has_linear_attention=True)+ mc.kv_cache_group_specs = lambda: (spec,)+ return SimpleNamespace(+ model_config=mc,+ page_size=page_size,+ dtype=torch.bfloat16,+ tp_info=SimpleNamespace(size=1),+ max_running_req=max_running_req,+ )+++# --------------------------------------------------------------------- slab / ring geometry+++def test_slab_ring_and_scratch_shapes():+ pool = _pool(num_pages=4)+ # 4 pages x 64 tokens / ratio 4 = 64 shadow rows, then one scratch row per request slot+ assert pool.cmp_scratch_base == 64+ assert pool.cmp_k_cache(0).shape == (64 + 4, 32)+ assert pool.cmp_k_cache(3).shape == (64 + 4, 32)+ assert pool.pending_ring(0).shape == (4, QSAKVCache.ring_capacity_for(4), 32)+ assert pool.cmp_k_cache(0).abs().sum().item() == 0.0+ assert pool.k_cache(1).shape == (4, 64, 2, 64)+++def test_kv_slabs_cover_sparse_layers_only():+ # Copying the BSA branch (no layer_ids) would back all 8 model layers instead of 4.+ pool = _pool()+ assert pool._kv_buffer.shape[1] == 4+ pool.k_cache(7)+ with pytest.raises(KeyError):+ pool.k_cache(0)+++def test_ring_capacity_and_ratio_are_parameters():+ pool = _pool(num_pages=8, index_ratio=2, num_req_slots=3, ring_capacity=6)+ assert pool.index_ratio == 2 and pool.ring_capacity == 6+ assert pool.cmp_scratch_base == 8 * 64 // 2+ assert pool.pending_ring(0).shape == (3, 6, 32)+++def test_ring_capacity_formula_and_floor():+ assert QSAKVCache.ring_capacity_for(4) == 4+ assert QSAKVCache.ring_capacity_for(8) == 8+ assert QSAKVCache.ring_capacity_for(4, num_speculative_tokens=3) == 8+ with pytest.raises(ValueError, match="ring_capacity"):+ _pool(ring_capacity=2, index_ratio=4)+++def test_group_must_not_straddle_a_page():+ with pytest.raises(ValueError, match="divisible"):+ _pool(page_size=6, index_ratio=4)+++def test_index_slab_needs_a_two_byte_dtype():+ with pytest.raises(AssertionError, match="2 bytes"):+ QSAKVCache(+ num_kv_heads=2, num_layers=8, head_dim=64, num_pages=4, page_size=64,+ dtype=torch.float32, device=DEV, index_head_dim=32, num_index_layers=4,+ index_ratio=4, num_req_slots=4, layer_ids=(1, 3, 5, 7),+ )+++def test_shadow_row_is_shared_by_a_whole_group():+ pool = _pool()+ cmp = pool.cmp_k_cache(2)+ row = torch.randn(32, dtype=torch.bfloat16)+ for slot in (64, 65, 66, 67):+ assert slot // pool.index_ratio == 16+ cmp[16] = row+ assert torch.equal(pool.cmp_k_cache(2)[16], row)+ # the other sparse layers keep their own rows+ assert pool.cmp_k_cache(1)[16].abs().sum().item() == 0.0+++def test_rebuild_resizes_every_tier_and_keeps_identity():+ pool = _pool(num_pages=4)+ ident = id(pool)+ pool.rebuild(16)+ assert id(pool) == ident+ assert pool.k_cache(1).shape == (16, 64, 2, 64)+ assert pool.cmp_scratch_base == 16 * 64 // 4+ assert pool.cmp_k_cache(0).shape == (16 * 64 // 4 + 4, 32)+ assert pool.pending_ring(3).shape == (4, pool.ring_capacity, 32)+ assert pool._kv_buffer.shape[1] == 4 # sparse-layer slabs survive the resize+ pool.k_cache(7)+++# ------------------------------------------------------------------------------ budgeting+++def test_spec_bytes_per_token_divides_the_index_slab():+ spec = _spec(num_kv_heads=2, head_dim=256, index_head_dim=128, num_index_layers=12,+ layer_ids=FULL_LAYER_IDS)+ config = _config(spec)+ assert spec_kv_bytes_per_token(spec, config) == REAL_KV_BYTES + REAL_INDEX_BYTES+ assert spec_kv_bytes_per_token(spec, config) == 24576 + 768++ # BSA/DSA keep one index row per token (ratio 1)+ bsa = _spec(num_kv_heads=2, head_dim=256, index_head_dim=128, num_index_layers=12,+ layer_ids=FULL_LAYER_IDS, index_ratio=1, attn_type=AttnType.BSA)+ assert spec_kv_bytes_per_token(bsa, config) == REAL_KV_BYTES + 128 * 12 * 2+++def test_kv_cost_prices_ring_and_scratch_as_fixed():+ spec = _spec()+ config = _config(spec, max_running_req=3)+ per_page, fixed, page_tokens, min_reserve = QSAKVCache.kv_cost(config)+ assert per_page == spec_kv_bytes_per_token(spec, config) * 64+ assert page_tokens == 64 and min_reserve == 0+ row = 32 * 4 * 2+ assert fixed == 4 * row * (QSAKVCache.ring_capacity_for(4) + 1)+++def test_unit_bytes_matches_the_cost_model():+ spec = _spec()+ config = _config(spec)+ pool = _pool()+ kv_bytes, swa_bytes = pool.unit_bytes()+ assert swa_bytes == 0+ # the scratch rows and the ring must NOT inflate the per-token slider+ assert kv_bytes == spec_kv_bytes_per_token(spec, config)+ assert kv_bytes * 64 == QSAKVCache.kv_cost(config)[0]+++def test_resolve_pool_class_and_factory():+ from freetoken.kvcache import create_kvcache_pool, resolve_pool_class++ spec = _spec()+ mc = SimpleNamespace(+ num_layers=8, has_swa_attention=False, has_linear_attention=True,+ num_kv_heads=2, head_dim=64, dsv4_args=None,+ )+ mc.kv_cache_group_specs = lambda: (spec,)+ assert resolve_pool_class(mc) is QSAKVCache++ pool = create_kvcache_pool(+ mc, num_pages=4, page_size=64, dtype=torch.bfloat16, device=DEV, num_req_slots=4+ )+ assert isinstance(pool, QSAKVCache)+ assert pool._kv_buffer.shape[1] == 4 # not the model's 8 layers+ assert pool.cmp_k_cache(0).shape == (4 * 64 // 4 + 4, 32)++ with pytest.raises(ValueError, match="num_req_slots"):+ create_kvcache_pool(mc, num_pages=4, page_size=64, dtype=torch.bfloat16, device=DEV)diff --git a/tests/models/qwen4_exp/__init__.py b/tests/models/qwen4_exp/__init__.pynew file mode 100644index 00000000..e69de29bdiff --git a/tests/models/qwen4_exp/common.py b/tests/models/qwen4_exp/common.pynew file mode 100644index 00000000..1f9c117b--- /dev/null+++ b/tests/models/qwen4_exp/common.py@@ -0,0 +1,257 @@+"""Shared fixtures for the qwen4_exp tests: toy configs, hash constants, the QSA pool and backend.++The geometry is the shipping one everywhere it matters for QSA (head_dim 256, index head_dim+128 with a 64-wide partial rope, index_ratio 4, budget 2048 -> 512 blocks -> 2051 selected+tokens, page_size 64); only the head counts, the hidden size and the layer count are scaled+down so a test fits on a shared GPU. Holds no tests itself.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++from freetoken.distributed import set_tp_info, try_get_tp_info++EOS = 7+VOCAB = 512++requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")+++def hf_config(+ num_layers: int = 4,+ head_dim: int = 256,+ num_q: int = 4,+ num_kv: int = 2,+ index_head_dim: int = 128,+ index_heads: int = 4,+ budget: int = 2048,+ ratio: int = 4,+ hidden: int = 256,+ max_position: int = 1 << 16,+ rope_theta: float = 10000000.0,+ **text_overrides,+) -> SimpleNamespace:+ text = SimpleNamespace(+ num_hidden_layers=num_layers,+ hidden_size=hidden,+ vocab_size=VOCAB,+ head_dim=head_dim,+ num_attention_heads=num_q,+ num_key_value_heads=num_kv,+ layer_types=[+ "full_attention" if (i + 1) % 4 == 0 else "linear_attention"+ for i in range(num_layers)+ ],+ rope_parameters={+ "rope_type": "default",+ "rope_theta": rope_theta,+ "partial_rotary_factor": 0.25,+ },+ max_position_embeddings=max_position,+ rms_norm_eps=1e-6,+ hidden_act="silu",+ tie_word_embeddings=False,+ num_experts=8,+ num_experts_per_tok=2,+ moe_intermediate_size=64,+ shared_expert_intermediate_size=64,+ linear_num_key_heads=2,+ linear_num_value_heads=6,+ linear_key_head_dim=32,+ linear_value_head_dim=32,+ linear_conv_kernel_dim=4,+ output_gate_type="sigmoid",+ indexer_n_heads=index_heads,+ indexer_kv_heads=1,+ indexer_head_dim=index_head_dim,+ indexer_budget=budget,+ indexer_compress_ratio=ratio,+ hc_count=4,+ hc_lowrank=16,+ ple_layer_ids=[2],+ ple_embed_dim=64,+ ple_conv_kernel_size=4,+ ngram_size=3,+ heads_per_ngram=2,+ ngram_vocab_size_base=1000,+ make_ngram_vocab_size_divisible_by=8,+ split_ngram_parts=4,+ eos_token_id=EOS,+ )+ for name, value in text_overrides.items():+ setattr(text, name, value)+ return SimpleNamespace(+ model_type="qwen4_exp",+ architectures=["Qwen4ExpForConditionalGeneration"],+ text_config=text,+ quantization_config=None,+ )+++def toy_hf_config(num_layers: int = 4, **text_overrides) -> SimpleNamespace:+ """The small-geometry config the PLE/skeleton tests share (hidden 128, head_dim 64)."""+ return hf_config(+ num_layers=num_layers, head_dim=64, num_kv=1, index_head_dim=64, index_heads=2,+ budget=16, hidden=128, max_position=4096, rope_theta=10000.0, **text_overrides,+ )+++def hash_constants(args):+ """Checkpoint-shape int64 hash tensors, derived like the dummy-weight path."""+ from freetoken.models.qwen4_exp.ple import derive_ngram_hash_constants++ multipliers, sizes, offsets = derive_ngram_hash_constants(+ vocab_size=VOCAB,+ ngram_size=args.ngram_size,+ num_ngram_heads=args.num_ngram_heads,+ ngram_vocab_size_base=args.ngram_vocab_size_base,+ ple_layer_index=0,+ )+ return [torch.tensor(v, dtype=torch.int64) for v in (multipliers, sizes, offsets)]+++def parsed_config(**kwargs):+ from freetoken.models.qwen4_exp.config import parse_config++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ return parse_config(hf_config(**kwargs))+++def fresh_ctx(page_size: int = 64, **fields):+ import freetoken.core as core+ from freetoken.core import Context, set_global_ctx++ core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx+ ctx = Context(page_size=page_size)+ for name, value in fields.items():+ setattr(ctx, name, value)+ set_global_ctx(ctx)+ return ctx+++def fill_weights(op, seed: int, device: torch.device, scale: float = 0.05) -> None:+ gen = torch.Generator(device=device).manual_seed(seed)+ for tensor in op.state_dict().values():+ if tensor.is_floating_point():+ tensor.normal_(0.0, scale, generator=gen)+ else:+ tensor.zero_()+++class Fixture:+ """QSA pool + page table + the sparse backend, with a first-fit page allocator."""++ def __init__(+ self,+ config,+ num_pages: int,+ max_running_req: int = 8,+ device: str = "cuda",+ dtype: torch.dtype = torch.bfloat16,+ page_size: int = 64,+ ) -> None:+ from freetoken.attention.qsa_sparse import QSASparseAttnBackend+ from freetoken.kvcache import create_kvcache_pool++ self.config = config+ self.device = torch.device(device)+ self.dtype = dtype+ self.page_size = page_size+ self.num_req_slots = max_running_req + 1+ self.pool = create_kvcache_pool(+ model_config=config,+ num_pages=num_pages + 1, # + 1 for the dummy page, as create_kv_pool does+ page_size=page_size,+ dtype=dtype,+ device=self.device,+ num_req_slots=self.num_req_slots,+ )+ self.page_table = torch.zeros(+ (self.num_req_slots, num_pages * page_size), dtype=torch.int32, device=self.device+ )+ self.page_table[max_running_req].fill_(num_pages * page_size) # dummy page+ self.ctx = fresh_ctx(+ page_size=page_size, page_table=self.page_table, kv_cache=self.pool+ )+ self.backend = QSASparseAttnBackend(config)+ self.ctx.attn_backend = self.backend+ self._free = list(range(num_pages))++ def layer(self, layer_id: int, seed: int = 1):+ from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention+ from freetoken.utils.torch_utils import torch_dtype++ with torch.device(self.device), torch_dtype(self.dtype):+ attn = Qwen4ExpAttention(self.config, layer_id=layer_id)+ fill_weights(attn, seed, self.device)+ return attn++ def allocate(self, table_idx: int, cached_len: int, device_len: int) -> None:+ for page in range(-(-cached_len // self.page_size), -(-device_len // self.page_size)):+ base = self._free.pop(0) * self.page_size+ columns = slice(page * self.page_size, (page + 1) * self.page_size)+ self.page_table[table_idx, columns] = torch.arange(+ base, base + self.page_size, dtype=torch.int32, device=self.device+ )++ def req(self, table_idx: int, cached_len: int, device_len: int) -> SimpleNamespace:+ self.allocate(table_idx, cached_len, device_len)+ return SimpleNamespace(+ table_idx=table_idx,+ cached_len=cached_len,+ device_len=device_len,+ extend_len=device_len - cached_len,+ )++ def step(self, req: SimpleNamespace) -> None:+ self.allocate(req.table_idx, req.device_len, req.device_len + 1)+ req.cached_len, req.device_len, req.extend_len = req.device_len, req.device_len + 1, 1++ def batch(self, reqs, phase: str) -> SimpleNamespace:+ positions = torch.cat(+ [+ torch.arange(r.cached_len, r.device_len, dtype=torch.int32, device=self.device)+ for r in reqs+ ]+ )+ out_loc = torch.cat(+ [self.page_table[r.table_idx, r.cached_len : r.device_len] for r in reqs]+ ).contiguous()+ batch = SimpleNamespace(+ reqs=reqs,+ padded_reqs=reqs,+ phase=phase,+ size=len(reqs),+ padded_size=len(reqs),+ is_prefill=phase == "prefill",+ is_decode=phase == "decode",+ positions=positions,+ out_loc=out_loc,+ attn_metadata=None,+ active_table_idx=torch.tensor(+ [r.table_idx for r in reqs], dtype=torch.int32, device=self.device+ ),+ )+ self.backend.prepare_metadata(batch)+ return batch+++def selection_spy(monkeypatch, backend) -> dict:+ """Record the expanded token selection of every ``_select`` call."""+ from freetoken.attention.qsa_sparse import QSASparseAttnBackend++ seen: dict[str, torch.Tensor] = {}+ original = QSASparseAttnBackend._select++ def spy(self, index, md, slot):+ indices = original(self, index, md, slot)+ seen["indices"] = indices.clone()+ return indices++ monkeypatch.setattr(QSASparseAttnBackend, "_select", spy)+ return seendiff --git a/tests/models/qwen4_exp/conftest.py b/tests/models/qwen4_exp/conftest.pynew file mode 100644index 00000000..2b9e8021--- /dev/null+++ b/tests/models/qwen4_exp/conftest.py@@ -0,0 +1,15 @@+"""Package-wide runtime hygiene: TP info set once, the global ctx never leaks across tests."""++import pytest++[email protected](autouse=True)+def _runtime():+ import freetoken.core as core+ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ core._GLOBAL_CTX = None+ yield+ core._GLOBAL_CTX = Nonediff --git a/tests/models/qwen4_exp/ple_hf_ref.py b/tests/models/qwen4_exp/ple_hf_ref.pynew file mode 100644index 00000000..35c52955--- /dev/null+++ b/tests/models/qwen4_exp/ple_hf_ref.py@@ -0,0 +1,70 @@+"""HF ground truth for the PLE parity tests, run as a script under the transformers-main venv.++``transformers`` in the serving venv predates ``models/qwen4_exp``, so the reference classes cannot+be imported next to FreeToken. ``test_ple.py`` spawns this file with the reference interpreter:+``python test_ple_hf_ref.py spec.json inputs.npz out.npz``. It holds no tests; every import is+inside ``main`` so pytest can still collect the module.+"""++from __future__ import annotations+++def main() -> None:+ import json+ import sys++ import numpy as np+ import torch+ from torch import nn+ from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig+ from transformers.models.qwen4_exp.modeling_qwen4_exp import (+ Qwen4ExpTextNGramEmbedding,+ Qwen4ExpTextPLELayer,+ )++ class CaptureEmbedding(nn.Module):+ """Stands in for the table so the hashed ids can be read out of the HF module."""++ def __init__(self, dim: int) -> None:+ super().__init__()+ self.weight = nn.Parameter(torch.zeros(1, dim), requires_grad=False)+ self.ids = None++ def forward(self, ids: torch.Tensor) -> torch.Tensor:+ self.ids = ids.clone()+ return torch.zeros(*ids.shape, self.weight.shape[1])++ with open(sys.argv[1], encoding="utf-8") as fh:+ spec = json.load(fh)+ data = np.load(sys.argv[2])+ config = Qwen4ExpTextConfig(**spec["config"])+ layer_idx, ple_index = spec["layer_idx"], spec["ple_layer_index"]+ out = {}++ embed = Qwen4ExpTextNGramEmbedding(config, config.ple_embed_dim, layer_idx, ple_index)+ out["layer_multipliers"] = embed.layer_multipliers.numpy()+ out["ngram_heads_vocab_sizes"] = embed.ngram_heads_vocab_sizes.numpy()+ out["ngram_heads_offsets"] = embed.ngram_heads_offsets.numpy()+ out["padded_vocab_size"] = np.array(embed.ngram_embedding.weight.shape[0])++ capture = CaptureEmbedding(embed.ngram_embedding.embedding_dim)+ embed.ngram_embedding = capture+ embed(torch.as_tensor(data["hash_tokens"]).long(), None)+ out["hash_ids"] = capture.ids.numpy()++ layer = Qwen4ExpTextPLELayer(config, layer_idx, ple_index)+ with torch.no_grad():+ for name in ("key_proj", "value_proj", "norm_key", "norm_query", "norm_conv"):+ getattr(layer, name).weight.copy_(torch.as_tensor(data[name]))+ layer.conv1d.weight.copy_(torch.as_tensor(data["conv1d"]))+ layer.ple_embedding.ngram_embedding.weight.copy_(torch.as_tensor(data["table"]))+ out["layer_out"] = layer(+ torch.as_tensor(data["hidden"]).float(),+ torch.as_tensor(data["layer_tokens"]).long(),+ None,+ ).numpy()+ np.savez(sys.argv[3], **out)+++if __name__ == "__main__":+ main()diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.pynew file mode 100644index 00000000..8fc2e550--- /dev/null+++ b/tests/models/qwen4_exp/test_config.py@@ -0,0 +1,170 @@+"""qwen4_exp.parse_config against a synthetic config shaped like the RadixArk NVFP4 checkpoint."""++from types import SimpleNamespace++import pytest++from freetoken.attention import AttnType+from freetoken.models.config import FullAttentionGroupConfig, LinearGatedDeltaGroupConfig+from freetoken.models.qwen4_exp.config import parse_config+++def _text_config():+ return SimpleNamespace(+ num_hidden_layers=48,+ hidden_size=2560,+ vocab_size=248320,+ head_dim=256,+ num_attention_heads=24,+ num_key_value_heads=2,+ layer_types=[+ "full_attention" if (i + 1) % 4 == 0 else "linear_attention" for i in range(48)+ ],+ rope_parameters={+ "rope_type": "default",+ "rope_theta": 10000000,+ "partial_rotary_factor": 0.25,+ "mrope_interleaved": True,+ "mrope_section": [11, 11, 10],+ },+ max_position_embeddings=262144,+ rms_norm_eps=1e-6,+ hidden_act="silu",+ tie_word_embeddings=False,+ num_experts=512,+ num_experts_per_tok=10,+ moe_intermediate_size=640,+ shared_expert_intermediate_size=640,+ linear_num_key_heads=16,+ linear_num_value_heads=48,+ linear_key_head_dim=128,+ linear_value_head_dim=128,+ linear_conv_kernel_dim=4,+ output_gate_type="sigmoid",+ indexer_n_heads=4,+ indexer_kv_heads=1,+ indexer_head_dim=128,+ indexer_budget=2048,+ indexer_compress_ratio=4,+ hc_count=4,+ hc_lowrank=320,+ ple_layer_ids=[2],+ ple_embed_dim=2560,+ ple_conv_kernel_size=4,+ ngram_size=3,+ heads_per_ngram=8,+ ngram_vocab_size_base=20000000,+ make_ngram_vocab_size_divisible_by=128,+ split_ngram_parts=128,+ bos_token_id=248044,+ eos_token_id=248044,+ )+++def _hf_config():+ return SimpleNamespace(+ model_type="qwen4_exp",+ architectures=["Qwen4ExpForConditionalGeneration"],+ image_token_id=248056,+ text_config=_text_config(),+ quantization_config={+ "quant_algo": "NVFP4",+ "quant_method": "modelopt",+ "ignore": [+ "model.embed_tokens",+ "mtp.*",+ "model.mtp.*",+ "*.self_attn.*",+ "*.linear_attn.*",+ "*.mlp.gate*",+ "*.mlp.shared_expert.*",+ "*.mlp.shared_expert_gate*",+ "*hyper_connection*",+ "*.ple.*",+ "model.visual.*",+ "model.language_model.embed_tokens",+ "lm_head",+ ],+ },+ )+++def test_groups_and_layer_split():+ cfg = parse_config(_hf_config())+ full = [g for g in cfg.attention_groups if isinstance(g, FullAttentionGroupConfig)]+ linear = [g for g in cfg.attention_groups if isinstance(g, LinearGatedDeltaGroupConfig)]+ assert len(full) == 1 and len(linear) == 1+ assert full[0].layer_ids == tuple(range(3, 48, 4))+ assert len(linear[0].layer_ids) == 36+ assert full[0].index_head_dim == 128+ assert full[0].num_index_layers == 12+ assert full[0].index_ratio == 4+ assert full[0].rotary_config.rotary_dim == 64+ assert full[0].rotary_config.scaling is None+ assert linear[0].output_gate == "sigmoid"+ assert linear[0].num_key_heads == 16 and linear[0].num_value_heads == 48+++def test_kv_specs_resolve_qsa():+ cfg = parse_config(_hf_config())+ specs = {s.name: s for s in cfg.kv_cache_group_specs()}+ assert specs["full"].attn_type is AttnType.QSA+ assert specs["full"].index_ratio == 4+ assert cfg.attn_type_for_layer(3) is AttnType.QSA+ assert cfg.attn_type_for_layer(0) is AttnType.LINEAR+ assert cfg.has_linear_attention+++def test_moe_and_quant_flags():+ cfg = parse_config(_hf_config())+ assert cfg.num_experts == 512+ assert cfg.num_experts_per_tok == 10+ assert cfg.norm_topk_prob is True+ assert cfg.moe_enabled+ assert cfg.expert_quant == "nvfp4"+ assert cfg.dense_quant == "none"+ assert cfg.attn_quant == "none"+ assert cfg.lm_head_quant == "none"+++def test_unquantized_config_parses():+ hf = _hf_config()+ hf.quantization_config = None+ assert parse_config(hf).expert_quant == "none"+++def test_qwen4_args_payload():+ args = parse_config(_hf_config()).qwen4_args+ assert args.ple_layer_ids == (1,)+ assert args.hc_count == 4 and args.hc_lowrank == 320+ assert args.index_topk_blocks == 512+ assert args.num_ngram_heads == 16+ assert args.ngram_head_dim == 160+ assert args.ple_conv_state_len == 9+ assert args.ple_state_width == 10240+ assert args.ngram_boundary_token_id == 248044+++def test_ple_on_full_attention_layer_rejected():+ hf = _hf_config()+ hf.text_config.ple_layer_ids = [4] # one-indexed 4 == zero-based 3, a full_attention layer+ with pytest.raises(ValueError, match="linear_attention"):+ parse_config(hf)+++def test_output_gate_null_falls_back_to_hidden_act():+ hf = _hf_config()+ hf.text_config.output_gate_type = None+ linear = [+ g+ for g in parse_config(hf).attention_groups+ if isinstance(g, LinearGatedDeltaGroupConfig)+ ]+ assert linear[0].output_gate == "silu"+++def test_eos_token_id_list_uses_the_first_entry():+ base = parse_config(_hf_config()).qwen4_args.ngram_boundary_token_id+ hf = _hf_config()+ hf.text_config.eos_token_id = [base, base + 1]+ assert parse_config(hf).qwen4_args.ngram_boundary_token_id == basediff --git a/tests/models/qwen4_exp/test_gdn.py b/tests/models/qwen4_exp/test_gdn.pynew file mode 100644index 00000000..81dd0e76--- /dev/null+++ b/tests/models/qwen4_exp/test_gdn.py@@ -0,0 +1,175 @@+"""qwen4_exp GatedDeltaNet op vs the pure-torch HF reference math.++The oracle is ``models/qwen4_exp/gdn_reference.py``, whose two delta rules and forward are+transcribed from the ``modeling_qwen4_exp.py`` snapshot, so no transformers build carrying+qwen4_exp is needed here. Covered: prefill at 128 and 1000 tokens, a decode step continuing+from the prefill state, ragged bs=3, both GQA head ratios, and the sigmoid output gate.+"""++from __future__ import annotations++import pytest+import torch++from freetoken.core import Batch, Context, Req, SamplingParams+from freetoken.models.config import LinearGatedDeltaGroupConfig+from freetoken.models.qwen4_exp.gdn import Qwen4ExpGatedDeltaNet+from freetoken.models.qwen4_exp.gdn_reference import Qwen4ExpGatedDeltaNetReference+from freetoken.utils import torch_dtype++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")++DEV = torch.device("cuda")+HIDDEN, HEAD_DIM, CONV_K, EPS = 256, 128, 4, 1e-6+RTOL = ATOL = 2e-2+# (num_k_heads, num_v_heads) per value:key head ratio; 3:1 is the Qwen3.8-Flash-Next shape.+HEADS = {2: (8, 16), 3: (16, 48)}+++def _bf(t: torch.Tensor) -> torch.Tensor:+ return t.detach().to(DEV, torch.bfloat16)+++def _state_dict(ref) -> dict[str, torch.Tensor]:+ """HF's four in_proj matrices fused into the op's single qkv|z|b|a GEMM. A_log / dt_bias+ stay fp32, as the weight loader keeps them."""+ return {+ "in_proj.weight": _bf(torch.cat([ref.in_proj_qkv.weight, ref.in_proj_z.weight,+ ref.in_proj_b.weight, ref.in_proj_a.weight], dim=0)),+ "conv1d.weight": _bf(ref.conv1d.weight),+ "dt_bias": ref.dt_bias.detach().to(DEV, torch.float32),+ "A_log": ref.A_log.detach().to(DEV, torch.float32),+ "norm.weight": _bf(ref.norm.weight),+ "out_proj.weight": _bf(ref.out_proj.weight),+ }+++def _make_layer(ratio: int, output_gate: str = "sigmoid", seed: int = 0):+ """fp32 reference + bf16 kernel op over one set of weights. The op is built on meta under+ the serving dtype, the way the engine builds a model, so load_state_dict's dtype check bites."""+ num_k, num_v = HEADS[ratio]+ torch.manual_seed(seed)+ ref = Qwen4ExpGatedDeltaNetReference(+ hidden_size=HIDDEN, num_k_heads=num_k, num_v_heads=num_v, head_k_dim=HEAD_DIM,+ head_v_dim=HEAD_DIM, conv_kernel_size=CONV_K, rms_norm_eps=EPS, output_gate=output_gate,+ ).to(DEV).float().eval()+ with torch.no_grad():+ # HF inits A_log = log(U(0.01, 16)); a zero dt_bias or a unit gate norm would hide sign errors.+ ref.A_log.uniform_(0.01, 16.0).log_()+ ref.dt_bias.uniform_(-1.0, 1.0)+ ref.norm.weight.normal_(1.0, 0.1)+ with torch.device("meta"), torch_dtype(torch.bfloat16):+ op = Qwen4ExpGatedDeltaNet(+ hidden_size=HIDDEN, num_k_heads=num_k, num_v_heads=num_v, head_k_dim=HEAD_DIM,+ head_v_dim=HEAD_DIM, conv_kernel_size=CONV_K, rms_norm_eps=EPS, layer_id=0,+ output_gate=output_gate,+ )+ op.load_state_dict(_state_dict(ref))+ return op, ref+++def _ctx(ratio: int, num_slots: int = 8) -> Context:+ import freetoken.core as core+ from freetoken.kvcache.linear_state_pool import LinearStatePool++ num_k, num_v = HEADS[ratio]+ group = LinearGatedDeltaGroupConfig(+ name="linear", layer_ids=(0,), num_key_heads=num_k, num_value_heads=num_v,+ key_head_dim=HEAD_DIM, value_head_dim=HEAD_DIM, conv_kernel_dim=CONV_K,+ output_gate="sigmoid",+ )+ core._GLOBAL_CTX = None+ ctx = Context(page_size=64)+ ctx.linear_state_pool = LinearStatePool(group, num_slots, torch.bfloat16, DEV, tp_size=1)+ core.set_global_ctx(ctx)+ return ctx+++def _prefill(op, ctx: Context, lengths: list[int], seed: int):+ """One ragged prefill batch, one state slot per request. Returns the per-request hidden+ states, the reqs (for a follow-up decode) and the packed output."""+ torch.manual_seed(seed)+ hidden = [torch.randn(n, HIDDEN, device=DEV, dtype=torch.bfloat16) for n in lengths]+ reqs = [+ Req(input_ids=torch.zeros(n, dtype=torch.int32), table_idx=i + 1, cached_len=0,+ output_len=1, uid=i, sampling_params=SamplingParams(), cache_handle=None)+ for i, n in enumerate(lengths)+ ]+ batch = Batch(reqs=reqs, phase="prefill")+ batch.padded_reqs = reqs+ with ctx.forward_batch(batch):+ out = op.forward(torch.cat(hidden, dim=0))+ return hidden, reqs, out+++def _decode(op, ctx: Context, reqs, hidden: torch.Tensor) -> torch.Tensor:+ batch = Batch(reqs=reqs, phase="decode")+ batch.padded_reqs = reqs+ batch.linear_table_idx = torch.tensor(+ [r.table_idx for r in reqs], dtype=torch.int32, device=DEV+ )+ with ctx.forward_batch(batch):+ return op.forward(hidden)++[email protected]_grad()+def _ref_out(ref, hidden: torch.Tensor, use_chunk_rule: bool = False) -> torch.Tensor:+ return ref(hidden.float().unsqueeze(0), use_chunk_rule=use_chunk_rule)[0]++[email protected]("length", (1000,))[email protected]("ratio", (2, 3))+def test_prefill_matches_reference(ratio, length):+ op, ref = _make_layer(ratio, seed=ratio)+ hidden, _, out = _prefill(op, _ctx(ratio), [length], seed=11)+ torch.testing.assert_close(out.float(), _ref_out(ref, hidden[0]), rtol=RTOL, atol=ATOL)++[email protected]("ratio", (2, 3))+def test_ragged_prefill_then_decode(ratio):+ """bs=3 ragged prefill, then one decode step per request off the carried conv + recurrent+ state. The decode oracle is the whole (prefill + 1) sequence in one reference pass, so a+ state that did not survive the prefill shows up immediately."""+ op, ref = _make_layer(ratio, seed=ratio)+ ctx = _ctx(ratio)+ lengths = [128, 1000, 37]+ hidden, reqs, out = _prefill(op, ctx, lengths, seed=13)++ off = 0+ for h, n in zip(hidden, lengths):+ torch.testing.assert_close(+ out[off:off + n].float(), _ref_out(ref, h), rtol=RTOL, atol=ATOL+ )+ off += n++ nxt = torch.randn(len(lengths), HIDDEN, device=DEV, dtype=torch.bfloat16)+ dec = _decode(op, ctx, reqs, nxt)+ for i, h in enumerate(hidden):+ full = _ref_out(ref, torch.cat([h, nxt[i:i + 1]], dim=0))+ torch.testing.assert_close(dec[i].float(), full[-1], rtol=RTOL, atol=ATOL)+++def test_chunk_and_recurrent_rules_agree():+ """The chunked form (what the fla prefill kernel implements) against the sequential+ definition, both fp32: the chunk oracle is only worth anything if it reproduces the+ recurrence to fp32 precision."""+ _, ref = _make_layer(3, seed=1)+ torch.manual_seed(17)+ hidden = torch.randn(1000, HIDDEN, device=DEV, dtype=torch.bfloat16)+ torch.testing.assert_close(+ _ref_out(ref, hidden, use_chunk_rule=True), _ref_out(ref, hidden), rtol=1e-4, atol=1e-4+ )+++def test_output_gate_comes_from_the_config():+ """The gate activation is the group config's string, not a hardcoded silu. Both gates track+ their own reference, and the two are far apart -- so a stuck activation cannot pass."""+ op_silu, ref_silu = _make_layer(3, output_gate="silu", seed=2)+ hidden, _, out_silu = _prefill(op_silu, _ctx(3), [128], seed=19)+ torch.testing.assert_close(out_silu.float(), _ref_out(ref_silu, hidden[0]), rtol=RTOL, atol=ATOL)++ op_sig, ref_sig = _make_layer(3, output_gate="sigmoid", seed=2)+ _, _, out_sig = _prefill(op_sig, _ctx(3), [128], seed=19)+ torch.testing.assert_close(out_sig.float(), _ref_out(ref_sig, hidden[0]), rtol=RTOL, atol=ATOL)++ assert (out_sig.float() - out_silu.float()).abs().max().item() > 10 * ATOLdiff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.pynew file mode 100644index 00000000..0d246142--- /dev/null+++ b/tests/models/qwen4_exp/test_ple.py@@ -0,0 +1,697 @@+"""PLE layer acceptance: hash vs HF, pinned-host table vs the GPU oracle, conv state+advancement (prefill / chunked / stepwise decode), CUDA-graph decode, and prefetch overlap.++The HF ground truth comes from ``ple_hf_ref.py`` run under a transformers build that ships+qwen4_exp (``FREETOKEN_QWEN4_HF_PYTHON``); those tests skip when it is unset.+"""++from __future__ import annotations++import json+import os+import subprocess+from pathlib import Path+from types import SimpleNamespace++import numpy as np+import pytest+import torch++from freetoken.models.config import ModelConfig+from freetoken.models.qwen4_exp.config import parse_config+from freetoken.models.qwen4_exp.ple import (+ GpuResidentTable,+ PinnedUVATable,+ PLELayer,+ PLEMetadata,+ build_ple_metadata,+ commit_ngram_context,+ short_conv_reference,+)++from .common import EOS, VOCAB, hash_constants, requires_cuda, toy_hf_config++_HF_REF_PYTHON = os.environ.get("FREETOKEN_QWEN4_HF_PYTHON", "")+_HF_REF_SCRIPT = Path(__file__).with_name("ple_hf_ref.py")++requires_hf_ref = pytest.mark.skipif(+ not (_HF_REF_PYTHON and Path(_HF_REF_PYTHON).exists()),+ reason="set FREETOKEN_QWEN4_HF_PYTHON to a transformers build that ships qwen4_exp",+)+++# --------------------------------------------------------------------------------------+# fixtures+# --------------------------------------------------------------------------------------++def _config() -> ModelConfig:+ return parse_config(toy_hf_config())+++def _padded_vocab(args) -> int:+ """HF pads the concatenated per-head vocabs up to make_ngram_vocab_size_divisible_by."""+ _, sizes, _ = hash_constants(args)+ div = args.make_ngram_vocab_size_divisible_by+ return -(-int(sizes.sum()) // div) * div+++def _make_layer(config, *, device="cpu", dtype=torch.float32, rows=None, seed=3, table=None):+ from freetoken.utils.torch_utils import torch_dtype++ args = config.qwen4_args+ rows = _padded_vocab(args) if rows is None else rows+ device = torch.device(device)+ gen = torch.Generator(device=device).manual_seed(seed)+ with torch.device(device), torch_dtype(dtype):+ layer = PLELayer(config, args.ple_layer_ids[0])+ for tensor in layer.state_dict().values():+ if tensor.is_floating_point():+ tensor.normal_(0.0, 0.05, generator=gen)+ multipliers, sizes, offsets = hash_constants(args)+ layer.ple_embedding.layer_multipliers.copy_(multipliers)+ layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)+ layer.ple_embedding.ngram_heads_offsets.copy_(offsets)+ if table is None:+ weight = torch.randn(rows, args.ngram_head_dim, generator=gen, device=device, dtype=dtype)+ table = GpuResidentTable(weight * 0.05, dtype=dtype)+ layer.ple_embedding.attach_table(table)+ return layer+++def _meta(sequences, contexts, *, device="cpu", slots=None, fresh=None, decode=False):+ lens = [len(s) for s in sequences]+ to = lambda xs, dtype: torch.tensor(xs, dtype=dtype, device=device)+ cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0).to(device)+ return PLEMetadata(+ input_ids=to([t for s in sequences for t in s], torch.int64),+ cu_seqlens=cu,+ seq_lens=tuple(lens),+ ngram_context=to(contexts, torch.int64),+ state_slots=(+ torch.arange(len(sequences), dtype=torch.int64, device=device)+ if slots is None+ else to(slots, torch.int64)+ ),+ fresh_slots=None if fresh is None else to(fresh, torch.bool),+ is_decode=decode,+ )+++def _run_hf_reference(tmp_path, data: dict, layer_idx=2, ple_layer_index=0) -> dict:+ spec = {"config": vars(toy_hf_config().text_config), "layer_idx": layer_idx, "ple_layer_index": ple_layer_index}+ (tmp_path / "spec.json").write_text(json.dumps(spec), encoding="utf-8")+ np.savez(tmp_path / "in.npz", **data)+ subprocess.run(+ [_HF_REF_PYTHON, str(_HF_REF_SCRIPT), str(tmp_path / "spec.json"),+ str(tmp_path / "in.npz"), str(tmp_path / "out.npz")],+ check=True,+ capture_output=True,+ )+ return dict(np.load(tmp_path / "out.npz"))+++# --------------------------------------------------------------------------------------+# hash+# --------------------------------------------------------------------------------------++# sequence start (all-eos context), eos inside the chunk, eos as the newest context token+_HASH_CASES = [+ ([EOS, EOS], [3, 4, EOS, 5, 6, 8]),+ ([21, 22], [2, EOS, 11, 12, 13, 14]),+ ([EOS, 31], [9, 10, 11, 12, 13, 14]),+ ([31, EOS], [9, 10, 11, 12, 13, 14]),+]+++@requires_hf_ref+def test_hash_ids_match_hf(tmp_path):+ """row_ids equals HF Qwen4ExpTextNGramEmbedding per id, over eos boundaries and at sequence start."""+ config = _config()+ layer = _make_layer(config)+ # HF pads its own all-eos context, so feeding [context | tokens] reproduces a resumed request+ tokens = np.array([c + s for c, s in _HASH_CASES], dtype=np.int64)+ ref = _run_hf_reference(tmp_path, {"hash_tokens": tokens, **_layer_ref_inputs(config, layer)})+ hf_ids = torch.as_tensor(ref["hash_ids"])[:, len(_HASH_CASES[0][0]) :]++ contexts = [c for c, _ in _HASH_CASES]+ sequences = [s for _, s in _HASH_CASES]+ got = layer.ple_embedding.row_ids(_meta(sequences, contexts))+ offset = 0+ for i, seq in enumerate(sequences):+ assert torch.equal(got[offset : offset + len(seq)], hf_ids[i]), f"request {i}"+ offset += len(seq)+++@requires_hf_ref+def test_hash_constants_match_hf(tmp_path):+ """derive_ngram_hash_constants reproduces the multipliers/vocab sizes/offsets HF builds at init."""+ config = _config()+ layer = _make_layer(config)+ ref = _run_hf_reference(+ tmp_path,+ {"hash_tokens": np.array([[EOS, EOS, 3, 4]], dtype=np.int64), **_layer_ref_inputs(config, layer)},+ )+ multipliers, sizes, offsets = hash_constants(config.qwen4_args)+ assert torch.equal(multipliers, torch.as_tensor(ref["layer_multipliers"]))+ assert torch.equal(sizes, torch.as_tensor(ref["ngram_heads_vocab_sizes"]))+ assert torch.equal(offsets, torch.as_tensor(ref["ngram_heads_offsets"]))+++def test_decode_hash_matches_prefill_hash():+ """The decode window (context + one token) hashes to the same ids as the same token in a prefill."""+ config = _config()+ layer = _make_layer(config)+ sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]]+ contexts = [[EOS, EOS], [21, 22]]+ prefill = layer.ple_embedding.row_ids(_meta(sequences, contexts))+ for step in range(len(sequences[0])):+ window = [(contexts[i] + s)[step : step + 2] for i, s in enumerate(sequences)]+ got = layer.ple_embedding.row_ids(+ _meta([[s[step]] for s in sequences], window, decode=True)+ )+ for i in range(len(sequences)):+ assert torch.equal(got[i], prefill[i * len(sequences[0]) + step])+++# --------------------------------------------------------------------------------------+# table backends+# --------------------------------------------------------------------------------------+++def _pinned_bank(rows: int, dim: int, dtype: torch.dtype, seed: int = 11):+ from freetoken.moe.host_banks import HostBank++ gen = torch.Generator().manual_seed(seed)+ bank = HostBank((rows, dim), dtype)+ bank.tensor.copy_((torch.randn(rows, dim, generator=gen) * 0.4).to(dtype))+ bank.pin()+ return bank+++@requires_cuda[email protected]("dtype", [torch.float8_e4m3fn, torch.bfloat16])+def test_pinned_uva_matches_gpu_resident(dtype):+ """PinnedUVATable is bitwise equal to the GPU-resident oracle, through lookup and prefetch."""+ rows, dim, scale = 8192, 160, 0.0234375+ bank = _pinned_bank(rows, dim, dtype)+ oracle = GpuResidentTable(bank.tensor.cuda(), scale, dtype=torch.bfloat16)+ pinned = PinnedUVATable(bank.tensor, scale)++ ids = torch.randint(0, rows, (37, 16), device="cuda")+ want = oracle.lookup(ids)+ assert torch.equal(pinned.lookup(ids), want)++ pinned.prefetch(ids)+ assert torch.equal(pinned.lookup(ids), want)++ # a stale prefetch must still be joined before its staging buffer is reused+ pinned.prefetch(ids)+ other = torch.randint(0, rows, (37, 16), device="cuda")+ assert torch.equal(pinned.lookup(other), oracle.lookup(other))++ out = torch.empty(37, 16 * dim, dtype=torch.bfloat16, device="cuda")+ assert pinned.lookup(ids, out) is out+ assert torch.equal(out, want)+++@requires_cuda+def test_pinned_uva_zeroes_out_of_range_ids():+ bank = _pinned_bank(64, 160, torch.float8_e4m3fn)+ pinned = PinnedUVATable(bank.tensor, 1.0)+ ids = torch.tensor([[0, 64, 1, -1]], device="cuda")+ rows = pinned.lookup(ids).view(4, 160)+ assert rows[1].abs().sum() == 0 and rows[3].abs().sum() == 0+ assert torch.equal(rows[0], bank.tensor[0].cuda().to(torch.bfloat16))+++ not os.environ.get("FREETOKEN_QWEN4EXP_MODEL"), reason="needs FREETOKEN_QWEN4EXP_MODEL"+)+@requires_cuda+def test_pinned_uva_real_table():+ """The real 47.7 GiB FP8 table: sampled rows equal the checkpoint bytes dequantized on CPU."""+ import safetensors+ from freetoken.models.qwen4_exp.weight import _PLE_SHARD_RE, _ple_table_files, load_ple_table++ path = os.environ["FREETOKEN_QWEN4EXP_MODEL"]+ with open(os.path.join(path, "config.json"), encoding="utf-8") as fh:+ text = json.load(fh)["text_config"]+ heads = (text["ngram_size"] - 1) * text["heads_per_ngram"]+ args = SimpleNamespace(+ split_ngram_parts=text["split_ngram_parts"],+ ngram_head_dim=text["ple_embed_dim"] // heads,+ )+ table = load_ple_table(path, args)+ scale = float(table.weight_scale)+ rows_per_shard = table.tensor.shape[0] // args.split_ngram_parts+ backend = PinnedUVATable(table.tensor, scale)++ gen = torch.Generator().manual_seed(5)+ sample = torch.randint(0, table.tensor.shape[0], (1000,), generator=gen)+ got = backend.lookup(sample.view(-1, 1).cuda()).cpu()++ shard_key = {}+ for file in _ple_table_files(path):+ with safetensors.safe_open(file, framework="pt", device="cpu") as fh:+ for key in fh.keys():+ match = _PLE_SHARD_RE.search(key)+ if match is not None:+ shard_key[int(match.group("shard"))] = (file, key)++ by_file = {}+ for i, row in enumerate(sample.tolist()):+ file, key = shard_key[row // rows_per_shard]+ by_file.setdefault(file, []).append((i, key, row % rows_per_shard))+ for file, items in by_file.items():+ with safetensors.safe_open(file, framework="pt", device="cpu") as fh:+ for i, key, offset in items:+ raw = fh.get_slice(key)[offset : offset + 1]+ want = (raw.float() * scale).to(torch.bfloat16).reshape(-1)+ assert torch.equal(got[i], want), f"sample {i}"+++# --------------------------------------------------------------------------------------+# conv state+# --------------------------------------------------------------------------------------+++def _forward(layer, R, meta, states):+ return layer.forward(R, batch=None, meta=meta, conv_states=states)+++def test_prefill_conv_matches_reference():+ """The packed single-conv prefill equals the per-request reference conv, chunks shorter than the state included."""+ torch.manual_seed(12)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6], [2, EOS, 11], [9]]+ contexts = [[EOS, EOS], [21, 22], [EOS, 31]]+ meta = _meta(sequences, contexts)+ total = sum(len(s) for s in sequences)+ x = torch.randn(total, args.ple_state_width)+ states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1++ got_states = states.clone()+ got = layer._short_conv(x, meta, got_states)+ ref_states = states.clone()+ ref = short_conv_reference(x, meta, ref_states, layer.conv1d.weight, args.ple_conv_dilation)+ assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6)+ assert torch.allclose(got_states, ref_states, rtol=1e-5, atol=1e-6)+++def test_fresh_slots_read_a_zero_state():+ """A request marked fresh ignores whatever the pool slot still holds."""+ torch.manual_seed(13)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ meta = _meta([[3, 4, 5], [6, 7, 8]], [[EOS, EOS]] * 2, fresh=[True, False])+ x = torch.randn(6, args.ple_state_width)+ dirty = torch.randn(2, args.ple_state_width, args.ple_conv_state_len)+ clean = dirty.clone()+ clean[0] = 0+ got = layer._short_conv(x, meta, dirty.clone())+ want = layer._short_conv(x, _meta([[3, 4, 5], [6, 7, 8]], [[EOS, EOS]] * 2), clean.clone())+ assert torch.equal(got, want)++[email protected]("cuts", [[1], [2, 3, 4], [9]], ids=["first-token", "uneven-mix", "penultimate"])+def test_chunked_prefill_matches_one_shot(cuts):+ """Chunked prefill at arbitrary cut points (including chunks shorter than the conv state) matches one shot."""+ torch.manual_seed(14)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 7], [2, EOS, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]]+ length = len(sequences[0])+ contexts = [[EOS, EOS], [21, 22]]+ x = torch.randn(len(sequences) * length, args.ple_state_width)+ per_req = [x[i * length : (i + 1) * length] for i in range(len(sequences))]+ zeros = torch.zeros(len(sequences), args.ple_state_width, args.ple_conv_state_len)++ full_states = zeros.clone()+ full = _forward(layer, x, _meta(sequences, contexts), full_states)++ chunk_states = zeros.clone()+ pieces, start = [], 0+ for size in [*cuts, length]:+ end = min(start + size, length)+ if end == start:+ continue+ window = [(c + s)[start : start + 2] for c, s in zip(contexts, sequences)]+ pieces.append(+ _forward(+ layer,+ torch.cat([r[start:end] for r in per_req]),+ _meta([s[start:end] for s in sequences], window),+ chunk_states,+ )+ )+ start = end++ for i, seq in enumerate(sequences):+ rebuilt = torch.cat(+ [p.chunk(len(sequences))[i] for p in pieces]+ )+ assert torch.allclose(rebuilt, full[i * length : (i + 1) * length], rtol=1e-4, atol=1e-5)+ assert torch.allclose(chunk_states, full_states, rtol=1e-4, atol=1e-5)+++def _state_pool(config, num_slots=8):+ from freetoken.kvcache.linear_state_pool import LinearStatePool++ return LinearStatePool(+ config.linear_attention_group(), num_slots, torch.float32,+ torch.device("cpu"), tp_size=1, slot_states=config.slot_states,+ )+++def _track_batch(req, tokens, pool):+ """Prefill batch whose FLAMetadata carries the hybrid-radix track indices for ``req``."""+ import freetoken.core as core+ from freetoken.attention.linear import build_fla_metadata+ from freetoken.core import Context, set_global_ctx++ core._GLOBAL_CTX = None # test-only: build_fla_metadata reads the state pool off the ctx+ set_global_ctx(Context(page_size=64, linear_state_pool=pool))+ batch = _fake_batch([req], decode=False, input_ids=tokens)+ batch.fla_metadata = build_fla_metadata(batch, torch.device("cpu"))+ return batch+++def _tracked_req(table_idx, cached_len, tokens, *, live, ping_pong):+ req = _req(table_idx, cached_len, tokens, extend_len=len(tokens))+ req.linear_slot_idx = live+ req.mamba_ping_pong = ping_pong+ req.mamba_next_track_idx = 0+ return req+++def _no_eos_tokens(n, start=0):+ return [(t + start) * 13 % (VOCAB - 8) + 8 for t in range(n)]+++def test_track_snapshot_equals_a_prefill_stopped_at_the_boundary():+ """The snapshot in the donated slot equals the state a prefill truncated at the boundary leaves."""+ from freetoken.kernel.fla.chunk import CHUNK_SIZE++ torch.manual_seed(17)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ pool = _state_pool(config)+ live, dst = 1, 5+ tokens = _no_eos_tokens(CHUNK_SIZE + 6)+ req = _tracked_req(0, 0, tokens, live=live, ping_pong=(dst, 6))+ batch = _track_batch(req, tokens, pool)++ fla = batch.fla_metadata+ assert fla.track_dst.tolist() == [dst]+ assert req.mamba_last_track_seqlen == CHUNK_SIZE+ assert fla.track_boundary_row.tolist() == [CHUNK_SIZE]++ R = torch.randn(len(tokens), args.ple_state_width)+ slab = pool.slot_state("ple_conv", args.ple_layer_ids[0])+ layer.forward(R, batch, meta=_meta([tokens], [[EOS, EOS]], slots=[live]), conv_states=slab)+ got = pool.slot_state("ple_conv", args.ple_layer_ids[0])[dst].clone()++ stopped = torch.zeros_like(slab)+ _forward(layer, R[:CHUNK_SIZE], _meta([tokens[:CHUNK_SIZE]], [[EOS, EOS]], slots=[live]), stopped)+ assert torch.equal(got, stopped[live])+++def test_prefix_hit_matches_the_uncached_run():+ """A prefix hit that COW-restores the donated snapshot reproduces the tail of an uncached prefill."""+ from freetoken.kernel.fla.chunk import CHUNK_SIZE++ torch.manual_seed(18)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ pool = _state_pool(config)+ tokens = _no_eos_tokens(CHUNK_SIZE + 6)+ context = [[EOS, EOS]]+ R = torch.randn(len(tokens), args.ple_state_width)++ uncached = _forward(+ layer, R, _meta([tokens], context, slots=[1]), torch.zeros_like(pool.slot_state("ple_conv", args.ple_layer_ids[0]))+ )++ live, dst = 1, 5+ req = _tracked_req(0, 0, tokens, live=live, ping_pong=(dst, 6))+ batch = _track_batch(req, tokens, pool)+ layer.forward(R, batch, meta=_meta([tokens], context, slots=[live]), conv_states=pool.slot_state("ple_conv", args.ple_layer_ids[0]))++ resumed_slot = 3+ pool.copy_from(dst, resumed_slot)+ tail = tokens[CHUNK_SIZE:]+ got = _forward(+ layer,+ R[CHUNK_SIZE:],+ _meta([tail], [tokens[CHUNK_SIZE - 2 : CHUNK_SIZE]], slots=[resumed_slot]),+ pool.slot_state("ple_conv", args.ple_layer_ids[0]),+ )+ assert torch.allclose(got, uncached[CHUNK_SIZE:], rtol=1e-5, atol=1e-6)+++def test_prefill_matches_stepwise_decode():+ """bs=3 ragged prefill equals feeding the same tokens one decode step at a time."""+ torch.manual_seed(15)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 12], [2, EOS, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],+ [9, 10, 11, 12, 13, 14, EOS, 16, 17, 18, 19, 20]]+ length = len(sequences[0])+ contexts = [[EOS, EOS], [21, 22], [EOS, 31]]+ x = torch.randn(len(sequences) * length, args.ple_state_width)+ per_req = [x[i * length : (i + 1) * length] for i in range(len(sequences))]+ zeros = torch.zeros(len(sequences), args.ple_state_width, args.ple_conv_state_len)++ full_states = zeros.clone()+ full = _forward(layer, x, _meta(sequences, contexts), full_states)++ step_states = zeros.clone()+ steps = []+ for t in range(length):+ window = [(c + s)[t : t + 2] for c, s in zip(contexts, sequences)]+ steps.append(+ _forward(+ layer,+ torch.stack([r[t] for r in per_req]),+ _meta([[s[t]] for s in sequences], window, decode=True),+ step_states,+ )+ )+ for i in range(len(sequences)):+ got = torch.stack([step[i] for step in steps])+ assert torch.allclose(got, full[i * length : (i + 1) * length], rtol=1e-4, atol=1e-5)+ assert torch.allclose(step_states, full_states, rtol=1e-4, atol=1e-5)+++# --------------------------------------------------------------------------------------+# full layer vs HF+# --------------------------------------------------------------------------------------+++def _layer_ref_inputs(config, layer, tokens=None, hidden=None):+ args = config.qwen4_args+ data = {+ "key_proj": layer.key_proj.weight.float().cpu().numpy(),+ "value_proj": layer.value_proj.weight.float().cpu().numpy(),+ "norm_key": layer.norm_key.weight.float().cpu().numpy(),+ "norm_query": layer.norm_query.weight.float().cpu().numpy(),+ "norm_conv": layer.norm_conv.weight.float().cpu().numpy(),+ "conv1d": layer.conv1d.weight.float().cpu().numpy(),+ "table": layer.ple_embedding.table.weight.float().cpu().numpy(),+ }+ if tokens is None:+ tokens = np.array([[3, 4]], dtype=np.int64)+ if hidden is None:+ hidden = np.zeros((1, tokens.shape[1], args.ple_state_width), dtype=np.float32)+ data["layer_tokens"] = tokens+ data["hidden"] = hidden+ return data+++@requires_cuda+@requires_hf_ref+def test_layer_matches_hf(tmp_path):+ """bf16 PLELayer output matches the fp32 HF Qwen4ExpTextPLELayer within 2e-2."""+ torch.manual_seed(16)+ config = _config()+ args = config.qwen4_args+ layer = _make_layer(config)+ tokens = [3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 12, 13, 14]+ hidden = (torch.randn(1, len(tokens), args.ple_state_width) * 0.5).numpy()+ ref = _run_hf_reference(+ tmp_path,+ {+ "hash_tokens": np.array([[EOS, EOS, 3]], dtype=np.int64),+ **_layer_ref_inputs(config, layer, np.array([tokens], dtype=np.int64), hidden),+ },+ )+ want = torch.as_tensor(ref["layer_out"])[0]+ assert int(ref["padded_vocab_size"]) == layer.ple_embedding.table.num_rows++ gpu = _make_layer(config, device="cuda", dtype=torch.bfloat16)+ for name in ("key_proj", "value_proj", "norm_key", "norm_query", "norm_conv"):+ getattr(gpu, name).weight.copy_(getattr(layer, name).weight)+ gpu.conv1d.weight.copy_(layer.conv1d.weight)+ gpu.ple_embedding.attach_table(+ GpuResidentTable(layer.ple_embedding.table.weight.to("cuda", torch.bfloat16), dtype=torch.bfloat16)+ )+ R = torch.as_tensor(hidden)[0].to("cuda", torch.bfloat16)+ states = torch.zeros(1, args.ple_state_width, args.ple_conv_state_len, device="cuda", dtype=torch.bfloat16)+ got = _forward(gpu, R, _meta([tokens], [[EOS, EOS]], device="cuda"), states)+ assert torch.allclose(got.float().cpu(), want, rtol=2e-2, atol=2e-2)+++# --------------------------------------------------------------------------------------+# metadata+# --------------------------------------------------------------------------------------+++def _fake_batch(reqs, *, decode, input_ids, positions=None, table_idx=None, device="cpu"):+ return SimpleNamespace(+ padded_reqs=reqs,+ reqs=reqs,+ is_decode=decode,+ is_prefill=not decode,+ input_ids=torch.tensor(input_ids, dtype=torch.int64, device=device),+ positions=None if positions is None else torch.tensor(positions, dtype=torch.int32, device=device),+ linear_table_idx=(+ None if table_idx is None else torch.tensor(table_idx, dtype=torch.int32, device=device)+ ),+ )+++def _req(table_idx, cached_len, host_ids, extend_len=1):+ return SimpleNamespace(+ table_idx=table_idx,+ cached_len=cached_len,+ extend_len=extend_len,+ linear_slot_idx=None,+ input_ids=torch.tensor(host_ids, dtype=torch.int64),+ )+++def test_commit_writes_the_track_slot_at_the_boundary():+ """The donated snapshot must carry the context AT the xCHUNK boundary, not the chunk end."""+ from freetoken.kernel.fla.chunk import CHUNK_SIZE++ args = _config().qwen4_args+ eos = args.ngram_boundary_token_id+ ctxp = torch.full((8, 2), eos, dtype=torch.int32)+ tokens = _no_eos_tokens(CHUNK_SIZE + 6)+ batch = _fake_batch([_req(1, 0, tokens, extend_len=len(tokens))], decode=False, input_ids=tokens)+ meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp)+ fla = SimpleNamespace(+ track_boundary_row=torch.tensor([CHUNK_SIZE]), track_dst=torch.tensor([5])+ )+ commit_ngram_context(meta, fla, ctxp)+ assert ctxp[1].tolist() == tokens[-2:]+ assert ctxp[5].tolist() == tokens[CHUNK_SIZE - 2 : CHUNK_SIZE]+++def test_context_matches_the_token_history_across_chunks_and_decode():+ """Rolling the slot state chunk by chunk reproduces the last-2-tokens oracle exactly."""+ args = _config().qwen4_args+ eos = args.ngram_boundary_token_id+ ctxp = torch.full((3, 2), 99, dtype=torch.int32) # stale tenant garbage; fresh rows must mask to eos+ history = _no_eos_tokens(11, start=3)+ cached = 0+ for chunk in (3, 1, 2, 5):+ ids = history[cached : cached + chunk]+ batch = _fake_batch(+ [_req(1, cached, history[: cached + chunk], extend_len=chunk)],+ decode=False, input_ids=ids,+ )+ meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp)+ assert meta.ngram_context.tolist() == [([eos, eos] + history[:cached])[-2:]]+ commit_ngram_context(meta, None, ctxp)+ cached += chunk+ for step in range(3):+ tok = 200 + step+ batch = _fake_batch(+ [_req(1, cached, history + [tok], extend_len=1)],+ decode=True, input_ids=[tok], positions=[cached], table_idx=[1],+ )+ meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp)+ assert meta.ngram_context.tolist() == [history[-2:]]+ commit_ngram_context(meta, None, ctxp)+ history.append(tok)+ cached += 1+++# --------------------------------------------------------------------------------------+# CUDA graph + prefetch overlap+# --------------------------------------------------------------------------------------+++@requires_cuda+def test_decode_graph_replay_matches_eager():+ """A captured decode PLE forward replays to the eager result, table gather included."""+ torch.manual_seed(17)+ config = _config()+ args = config.qwen4_args+ rows, bs = 4096, 4+ layer = _make_layer(config, device="cuda", dtype=torch.bfloat16, rows=rows)+ bank = _pinned_bank(rows, args.ngram_head_dim, torch.float8_e4m3fn)+ layer.ple_embedding.attach_table(PinnedUVATable(bank.tensor, 0.05))++ ctxp = torch.full((bs + 1, 2), EOS, dtype=torch.int32, device="cuda")+ ctxp[1:] = torch.randint(0, VOCAB, (bs, 2), device="cuda", dtype=torch.int32)+ positions = torch.full((bs,), 8, dtype=torch.int32, device="cuda")+ slots = torch.arange(1, bs + 1, dtype=torch.int32, device="cuda")+ batch = SimpleNamespace(+ padded_reqs=[None] * bs, is_decode=True, is_prefill=False,+ input_ids=torch.randint(0, VOCAB, (bs,), device="cuda", dtype=torch.int32),+ positions=positions, linear_table_idx=slots,+ )+ R = torch.randn(bs, args.ple_state_width, device="cuda", dtype=torch.bfloat16)+ states0 = torch.randn(bs + 1, args.ple_state_width, args.ple_conv_state_len,+ device="cuda", dtype=torch.bfloat16) * 0.1+ states = states0.clone()++ def step():+ layer.start_prefetch(batch, build_ple_metadata(batch, args, R.device, context_pool=ctxp))+ return layer.forward(R, batch, conv_states=states)++ eager = step().clone()+ eager_states = states.clone()++ warmup = torch.cuda.Stream()+ warmup.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(warmup):+ for _ in range(3):+ states.copy_(states0)+ step()+ torch.cuda.current_stream().wait_stream(warmup)++ graph = torch.cuda.CUDAGraph()+ states.copy_(states0)+ with torch.cuda.graph(graph):+ static_out = step()+ states.copy_(states0)+ graph.replay()+ torch.cuda.synchronize()+ assert torch.equal(static_out, eager)+ assert torch.equal(states, eager_states)++ # new inputs in the same buffers must flow through the replay+ ctxp[1:] = torch.randint(0, VOCAB, (bs, 2), device="cuda", dtype=torch.int32)+ batch.input_ids.copy_(torch.randint(0, VOCAB, (bs,), device="cuda", dtype=torch.int32))+ states.copy_(states0)+ graph.replay()+ replayed = static_out.clone()+ states.copy_(states0)+ assert torch.equal(step(), replayed)++ # a bigger eager gather (prefill) must not move the buffer the graph writes into+ layer.ple_embedding.table.lookup(torch.randint(0, rows, (4096, 16), device="cuda"))+ states.copy_(states0)+ graph.replay()+ torch.cuda.synchronize()+ assert torch.equal(static_out, replayed)diff --git a/tests/models/qwen4_exp/test_qsa_backend.py b/tests/models/qwen4_exp/test_qsa_backend.pynew file mode 100644index 00000000..1d3b944c--- /dev/null+++ b/tests/models/qwen4_exp/test_qsa_backend.py@@ -0,0 +1,250 @@+"""The QSA backend behind the real Qwen4ExpAttention layer.++(a) dense-oracle equivalence -- while a request sees at most ``index_budget + index_ratio - 1``+ tokens every complete block is selected, so QSA IS dense attention: the selection must be+ exactly the causal prefix and the layer output must match ``TorchDenseQSAReference`` (fp32)+ and a flashinfer dense run over the same pool;+(b) chunked prefill at unaligned cut points equals one-shot prefill (the dual-source compress);+(c) a captured decode replay equals the eager decode step.+"""++from __future__ import annotations++from types import SimpleNamespace++import pytest+import torch++from .common import Fixture, requires_cuda, parsed_config, selection_spy++QSA_LAYER = 3+++def _inputs(fixture: Fixture, lengths, extra: int = 0, seed: int = 11):+ generator = torch.Generator(device=fixture.device).manual_seed(seed)+ return [+ torch.randn(+ n + extra, fixture.config.hidden_size, device=fixture.device,+ dtype=fixture.dtype, generator=generator,+ )+ * 0.5+ for n in lengths+ ]+++def _assert_selection_is_causal_prefix(indices: torch.Tensor, positions: torch.Tensor) -> None:+ for row, position in enumerate(positions.tolist()):+ selected = indices[row][indices[row] >= 0]+ assert torch.equal(+ selected.sort().values,+ torch.arange(position + 1, dtype=selected.dtype, device=selected.device),+ ), f"row {row} (position {position}) did not select its whole causal prefix"+++@requires_cuda+def test_prefill_is_dense_below_the_budget(monkeypatch):+ """bs=3 ragged prefill, longest request exactly at budget + ratio - 1."""+ config = parsed_config()+ fixture = Fixture(config, num_pages=128)+ attn = fixture.layer(QSA_LAYER)+ lengths = [2051, 1000, 137]+ inputs = _inputs(fixture, lengths)+ x = torch.cat([row[:n] for row, n in zip(inputs, lengths)])+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]++ seen = selection_spy(monkeypatch, fixture.backend)+ batch = fixture.batch(reqs, "prefill")+ got = attn.forward(x, batch)+ _assert_selection_is_causal_prefix(seen["indices"], batch.positions)++ fixture.ctx.attn_backend = _dense_oracle(fixture)+ reference = attn.forward(x, batch)+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)+++def _dense_oracle(fixture: Fixture):+ from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference++ return TorchDenseQSAReference(+ fixture.config,+ num_slots=fixture.num_req_slots,+ max_len=4096,+ device=fixture.device,+ dtype=fixture.dtype,+ )+++@requires_cuda+def test_decode_is_dense_below_the_budget(monkeypatch):+ """Prefill then five decode steps, sparse path vs the fp32 dense oracle."""+ config = parsed_config()+ fixture = Fixture(config, num_pages=128)+ attn = fixture.layer(QSA_LAYER)+ lengths, steps = [300, 411, 64], 5+ inputs = _inputs(fixture, lengths, extra=steps)+ oracle = _dense_oracle(fixture)++ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]+ seen = selection_spy(monkeypatch, fixture.backend)++ steps_x = [torch.cat([row[:n] for row, n in zip(inputs, lengths)])]+ steps_x += [+ torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) for step in range(steps)+ ]+ for step, x in enumerate(steps_x):+ if step:+ for req in reqs:+ fixture.step(req)+ batch = fixture.batch(reqs, "prefill" if step == 0 else "decode")+ fixture.ctx.attn_backend = fixture.backend+ got = attn.forward(x, batch)+ _assert_selection_is_causal_prefix(seen["indices"], batch.positions)+ fixture.ctx.attn_backend = oracle+ reference = attn.forward(x, batch)+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)+++@requires_cuda+def test_flashinfer_dense_matches_the_sparse_path():+ """The engine's dense FULL backend over the same pool, as an independent oracle."""+ pytest.importorskip("flashinfer")+ from freetoken.attention.fi import FlashInferBackend++ config = parsed_config()+ fixture = Fixture(config, num_pages=64)+ attn = fixture.layer(QSA_LAYER)+ length = 500+ x = _inputs(fixture, [length])[0]+ req = fixture.req(0, 0, length)+ got = attn.forward(x, fixture.batch([req], "prefill"))++ dense = FlashInferBackend(config)+ fixture.ctx.attn_backend = SimpleNamespace(+ qsa_forward=lambda q, k, v, index, layer_id, batch: dense.forward(+ q, k, v, layer_id, batch+ )+ )+ batch = fixture.batch([req], "prefill")+ dense.prepare_metadata(batch)+ reference = attn.forward(x, batch)+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)+++@requires_cuda[email protected]("cut", [1001, 4096, 4097], ids=["unaligned", "page-boundary", "boundary+1"])+def test_chunked_prefill_matches_one_shot(cut: int):+ """Cut points that are not multiples of index_ratio exercise the dual-source compress."""+ config = parsed_config()+ fixture = Fixture(config, num_pages=512)+ attn = fixture.layer(QSA_LAYER)+ length = 5000+ x = _inputs(fixture, [length])[0]++ one_shot = attn.forward(x, fixture.batch([fixture.req(0, 0, length)], "prefill"))+ head = fixture.req(1, 0, cut)+ attn.forward(x[:cut], fixture.batch([head], "prefill"))+ tail = fixture.req(1, cut, length)+ got = attn.forward(x[cut:], fixture.batch([tail], "prefill"))+ assert torch.equal(got, one_shot[cut:])+++@requires_cuda+def test_decode_graph_replay_matches_eager():+ config = parsed_config()+ fixture = Fixture(config, num_pages=256)+ attn = fixture.layer(QSA_LAYER)+ lengths, steps = [300, 411], 4+ bs = len(lengths)+ inputs = _inputs(fixture, lengths, extra=steps)+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]+ attn.forward(+ torch.cat([row[:n] for row, n in zip(inputs, lengths)]),+ fixture.batch(reqs, "prefill"),+ )++ fixture.backend.init_capture_graph(max_seq_len=fixture.page_table.shape[1], bs_list=[bs])+ dummy = SimpleNamespace(+ table_idx=fixture.num_req_slots - 1, cached_len=1, device_len=2, extend_len=1+ )+ static = {+ "x": torch.zeros(bs, config.hidden_size, device=fixture.device, dtype=fixture.dtype),+ "positions": torch.zeros(bs, dtype=torch.int32, device=fixture.device),+ "out_loc": torch.zeros(bs, dtype=torch.int32, device=fixture.device),+ }+ capture_batch = SimpleNamespace(+ padded_reqs=[dummy] * bs, reqs=[dummy] * bs, phase="decode", size=bs, padded_size=bs,+ is_prefill=False, is_decode=True, positions=static["positions"],+ out_loc=static["out_loc"], attn_metadata=None, active_table_idx=None,+ )+ fixture.backend.prepare_for_capture(capture_batch)+ attn.forward(static["x"], capture_batch) # warmup, same metadata object as the capture+ torch.cuda.synchronize()+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ captured_out = attn.forward(static["x"], capture_batch)+ torch.cuda.synchronize()++ for step in range(steps):+ for req in reqs:+ fixture.step(req)+ x = torch.stack([row[n + step] for row, n in zip(inputs, lengths)])+ batch = fixture.batch(reqs, "decode")+ static["x"].copy_(x)+ static["positions"].copy_(batch.positions)+ static["out_loc"].copy_(batch.out_loc)+ fixture.backend.prepare_for_replay(batch)+ # replay must stage into the captured buffers, never reallocate them+ md = batch.attn_metadata+ assert md.block_table.data_ptr() == fixture.backend._graph["block_table"].data_ptr()+ graph.replay()+ replayed = captured_out.clone()+ eager = attn.forward(x, fixture.batch(reqs, "decode"))+ assert torch.equal(replayed, eager), f"graph replay diverged at decode step {step}"+++@requires_cuda+def test_row_chunked_scoring_matches_one_chunk(monkeypatch):+ """The scoring workspace bound splits long prefills into row chunks."""+ import freetoken.attention.qsa_sparse as qsa_sparse++ config = parsed_config()+ fixture = Fixture(config, num_pages=64)+ attn = fixture.layer(QSA_LAYER)+ length = 600+ x = _inputs(fixture, [length])[0]+ whole = attn.forward(x, fixture.batch([fixture.req(0, 0, length)], "prefill"))++ columns = fixture.page_table.shape[1] // config.qwen4_args.index_ratio+ monkeypatch.setattr(qsa_sparse, "_LOGITS_WORKSPACE_BYTES", 64 * columns * 4)+ chunked = attn.forward(x, fixture.batch([fixture.req(1, 0, length)], "prefill"))+ assert torch.equal(chunked, whole)+++@requires_cuda+def test_two_qsa_layers_keep_separate_slab_slots(monkeypatch):+ """Both QSA layers of one forward must hit their own slab slot and ring slice."""+ config = parsed_config(num_layers=8)+ assert config.attention_groups[1].layer_ids == (3, 7)+ fixture = Fixture(config, num_pages=64)+ layers = [fixture.layer(layer_id, seed=layer_id) for layer_id in (3, 7)]+ oracle = _dense_oracle(fixture)+ lengths, steps = [200, 71], 3+ inputs = _inputs(fixture, lengths, extra=steps)+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]++ xs = [torch.cat([row[:n] for row, n in zip(inputs, lengths)])]+ xs += [torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) for step in range(steps)]+ for step, x in enumerate(xs):+ if step:+ for req in reqs:+ fixture.step(req)+ batch = fixture.batch(reqs, "prefill" if step == 0 else "decode")+ for attn in layers:+ fixture.ctx.attn_backend = fixture.backend+ got = attn.forward(x, batch)+ fixture.ctx.attn_backend = oracle+ reference = attn.forward(x, batch)+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)++ slab = fixture.pool.cmp_k_cache+ assert not torch.equal(slab(0), slab(1))diff --git a/tests/models/qwen4_exp/test_qsa_hf.py b/tests/models/qwen4_exp/test_qsa_hf.pynew file mode 100644index 00000000..3d3a8382--- /dev/null+++ b/tests/models/qwen4_exp/test_qsa_hf.py@@ -0,0 +1,253 @@+"""One QSA layer against the HF reference math at L = 3000.++The fp32 reference here is transcribed from ``modeling_qwen4_exp.py``+(``Qwen4ExpTextQSAIndexer``:611, ``Qwen4ExpTextAttention``:757): pool a group of raw index+keys in fp32, ``(1 + w)`` rmsnorm it, rope it at the group's FIRST position, score+``sum_h relu(<q_h, k_bar_b>) / sqrt(index_head_dim)`` over complete blocks, keep the top+``budget // ratio``, expand, then attend to that set only.++Two claims: the selected sets agree (ties near the top-k boundary may differ, so the bar is a+Jaccard floor) and, GIVEN the backend's own selection, the attention output matches. Set+``FREETOKEN_QWEN4_HF_PYTHON`` to an interpreter whose transformers ships ``qwen4_exp`` to run+the same comparison against the real HF module in a subprocess.+"""++from __future__ import annotations++import math+import os+import subprocess+import sys++import pytest+import torch+import torch.nn.functional as F++from .common import Fixture, requires_cuda, parsed_config, selection_spy++QSA_LAYER = 3+LENGTH = 3000+++def _plus_one_rmsnorm(x, weight, eps):+ xf = x.float()+ return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + weight.float())+++def _hf_rope(x, positions, rotary_dim, base):+ """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims."""+ inv = 1.0 / (+ base+ ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim)+ )+ freqs = positions.float().unsqueeze(-1) * inv+ cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1)+ sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1)+ rotated = x[..., :rotary_dim].float()+ half = rotary_dim // 2+ swapped = torch.cat([-rotated[..., half:], rotated[..., :half]], dim=-1)+ return torch.cat([rotated * cos + swapped * sin, x[..., rotary_dim:].float()], dim=-1)+++def _hf_block_scores(x, indexer, config, positions):+ """[T, blocks] indexer scores; the pooled block keys do not depend on the query row."""+ args = config.qwen4_args+ rotary = config.rotary_config+ heads, dim, ratio = args.index_n_heads, args.index_head_dim, args.index_ratio+ qk = F.linear(x.float(), indexer.index_qk_proj.weight.float())+ q = _plus_one_rmsnorm(+ qk[:, : heads * dim].view(-1, heads, dim), indexer.q_layernorm.weight, config.rms_norm_eps+ )+ q = _hf_rope(q, positions, rotary.rotary_dim, rotary.base)+ raw = qk[:, heads * dim :]+ blocks = raw.shape[0] // ratio+ pooled = raw[: blocks * ratio].view(blocks, ratio, dim).mean(1)+ pooled = _plus_one_rmsnorm(pooled, indexer.k_layernorm.weight, config.rms_norm_eps)+ kbar = _hf_rope(+ pooled.unsqueeze(1), positions[: blocks * ratio : ratio], rotary.rotary_dim, rotary.base+ ).squeeze(1)+ return torch.relu(torch.einsum("thd,bd->tbh", q, kbar)).sum(-1) / math.sqrt(dim)+++def _hf_selection(scores, positions, ratio, budget):+ """Per-query token ids: the top-(budget // ratio) complete blocks plus the open tail."""+ offsets = torch.arange(ratio, device=scores.device)+ selected = []+ for row, position in enumerate(positions.tolist()):+ visible = (position + 1) // ratio+ chosen = torch.empty(0, dtype=torch.int64, device=scores.device)+ if visible:+ top = scores[row, :visible].topk(min(budget // ratio, visible)).indices+ chosen = (top.unsqueeze(-1) * ratio + offsets).flatten()+ tail = torch.arange(visible * ratio, position + 1, device=scores.device)+ selected.append(torch.cat([chosen, tail]).sort().values)+ return selected+++def _hf_layer_output(x, attn, config, positions, selection):+ """The HF gated-GQA layer restricted to a given per-query token selection."""+ rotary = config.rotary_config+ length, dim = x.shape[0], attn.head_dim+ qkv = F.linear(x.float(), attn.qkv_proj.weight.float())+ qg, k, v = qkv.split(attn._qkv_split, dim=-1)+ qg = qg.view(length, attn.num_q, dim * 2)+ q = _plus_one_rmsnorm(qg[..., :dim], attn.q_norm.weight, config.rms_norm_eps)+ q = _hf_rope(q, positions, rotary.rotary_dim, rotary.base)+ k = _plus_one_rmsnorm(k.view(length, attn.num_kv, dim), attn.k_norm.weight, config.rms_norm_eps)+ k = _hf_rope(k, positions, rotary.rotary_dim, rotary.base)+ repeat = attn.num_q // attn.num_kv+ k = k.repeat_interleave(repeat, dim=1)+ v = v.view(length, attn.num_kv, dim).repeat_interleave(repeat, dim=1).float()+ out = torch.zeros(length, attn.num_q, dim, device=x.device, dtype=torch.float32)+ for row, tokens in enumerate(selection):+ scores = torch.einsum("hd,khd->hk", q[row], k[tokens]) * dim**-0.5+ out[row] = torch.einsum("hk,khd->hd", scores.softmax(-1), v[tokens])+ gate = torch.sigmoid(qg[..., dim:].reshape(length, -1).float())+ return F.linear(out.reshape(length, -1) * gate, attn.o_proj.weight.float())+++def _jaccard(indices, selection):+ scores = []+ for row, tokens in enumerate(selection):+ mine = set(indices[row][indices[row] >= 0].tolist())+ theirs = set(tokens.tolist())+ scores.append(len(mine & theirs) / max(len(mine | theirs), 1))+ return torch.tensor(scores)+++@requires_cuda+def test_single_layer_matches_hf_reference(monkeypatch):+ config = parsed_config()+ fixture = Fixture(config, num_pages=128, max_running_req=4)+ attn = fixture.layer(QSA_LAYER)+ generator = torch.Generator(device=fixture.device).manual_seed(13)+ x = (+ torch.randn(+ LENGTH, config.hidden_size, device=fixture.device, dtype=fixture.dtype,+ generator=generator,+ )+ * 0.5+ )+ seen = selection_spy(monkeypatch, fixture.backend)+ batch = fixture.batch([fixture.req(0, 0, LENGTH)], "prefill")+ got = attn.forward(x, batch)+ indices = seen["indices"]++ args = config.qwen4_args+ scores = _hf_block_scores(x, attn.indexer, config, batch.positions)+ reference_selection = _hf_selection(+ scores, batch.positions, args.index_ratio, args.index_budget+ )+ jaccard = _jaccard(indices, reference_selection)+ assert jaccard.min() >= 0.97, f"worst-row Jaccard {jaccard.min():.4f}"++ own_selection = [row[row >= 0].long().sort().values for row in indices]+ reference = _hf_layer_output(x, attn, config, batch.positions, own_selection)+ torch.testing.assert_close(got.float(), reference, rtol=2e-2, atol=2e-2)+++_HF_DRIVER = '''+import sys, torch+from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig+from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextAttention++payload = torch.load(sys.argv[1], map_location="cuda", weights_only=False)+meta = payload["meta"]+config = Qwen4ExpTextConfig(+ hidden_size=meta["hidden_size"], num_attention_heads=meta["num_q"],+ num_key_value_heads=meta["num_kv"], head_dim=meta["head_dim"], rms_norm_eps=meta["eps"],+ max_position_embeddings=meta["max_position"],+ rope_parameters={"rope_type": "default", "rope_theta": meta["base"],+ "partial_rotary_factor": meta["rotary_dim"] / meta["head_dim"],+ "mrope_section": [11, 11, 10]},+ indexer_n_heads=meta["index_heads"], indexer_kv_heads=1, indexer_head_dim=meta["index_dim"],+ indexer_budget=meta["budget"], indexer_compress_ratio=meta["ratio"],+)+config._attn_implementation = "eager"+torch.set_grad_enabled(False)+attn = Qwen4ExpTextAttention(config, layer_idx=0).to("cuda", torch.float32)+attn.load_state_dict({k: v.to("cuda", torch.float32) for k, v in payload["weights"].items()})++x = payload["x"].to(torch.float32).unsqueeze(0)+positions = payload["positions"].to("cuda").to(torch.long)+rotary_dim = meta["rotary_dim"]+pairs = torch.arange(0, rotary_dim, 2, device="cuda", dtype=torch.float32)+inv = 1.0 / (meta["base"] ** (pairs / rotary_dim))+freqs = positions.float().unsqueeze(-1) * inv+cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(0)+sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(0)+length = x.shape[1]+causal = torch.arange(length, device="cuda")+mask = torch.zeros(1, 1, length, length, device="cuda", dtype=torch.float32)+mask.masked_fill_(causal[None, :] > causal[:, None], torch.finfo(torch.float32).min)++out, _ = attn(x, (cos, sin), mask)+selected = attn.indexer(x, (cos, sin), mask, None)[0, 0] == 0+torch.save({"out": out[0].cpu(), "selected": selected.cpu()}, sys.argv[2])+'''+++@requires_cuda+ not os.environ.get("FREETOKEN_QWEN4_HF_PYTHON"),+ reason="set FREETOKEN_QWEN4_HF_PYTHON to a transformers build that ships qwen4_exp",+)+def test_single_layer_matches_upstream_hf(tmp_path, monkeypatch):+ config = parsed_config()+ fixture = Fixture(config, num_pages=128, max_running_req=4)+ attn = fixture.layer(QSA_LAYER)+ generator = torch.Generator(device=fixture.device).manual_seed(13)+ x = (+ torch.randn(+ LENGTH, config.hidden_size, device=fixture.device, dtype=fixture.dtype,+ generator=generator,+ )+ * 0.5+ )+ seen = selection_spy(monkeypatch, fixture.backend)+ batch = fixture.batch([fixture.req(0, 0, LENGTH)], "prefill")+ got = attn.forward(x, batch)++ args = config.qwen4_args+ rotary = config.rotary_config+ q_rows, kv_rows = attn.qo_attn_dim * 2, attn.kv_attn_dim+ fused = attn.qkv_proj.weight+ payload = tmp_path / "payload.pt"+ result = tmp_path / "hf.pt"+ driver = tmp_path / "driver.py"+ driver.write_text(_HF_DRIVER)+ torch.save(+ {+ "weights": {+ "q_proj.weight": fused[:q_rows].cpu(),+ "k_proj.weight": fused[q_rows : q_rows + kv_rows].cpu(),+ "v_proj.weight": fused[q_rows + kv_rows :].cpu(),+ "o_proj.weight": attn.o_proj.weight.cpu(),+ "q_norm.weight": attn.q_norm.weight.cpu(),+ "k_norm.weight": attn.k_norm.weight.cpu(),+ "indexer.index_qk_proj.weight": attn.indexer.index_qk_proj.weight.cpu(),+ "indexer.q_layernorm.weight": attn.indexer.q_layernorm.weight.cpu(),+ "indexer.k_layernorm.weight": attn.indexer.k_layernorm.weight.cpu(),+ },+ "x": x.cpu(),+ "positions": batch.positions.cpu(),+ "meta": {+ "hidden_size": config.hidden_size, "num_q": attn.num_q, "num_kv": attn.num_kv,+ "head_dim": attn.head_dim, "eps": config.rms_norm_eps,+ "max_position": rotary.max_position, "base": rotary.base,+ "rotary_dim": rotary.rotary_dim, "index_heads": args.index_n_heads,+ "index_dim": args.index_head_dim, "budget": args.index_budget,+ "ratio": args.index_ratio,+ },+ },+ payload,+ )+ subprocess.run(+ [os.environ["FREETOKEN_QWEN4_HF_PYTHON"], str(driver), str(payload), str(result)],+ check=True, stdout=sys.stderr, timeout=1800,+ )+ upstream = torch.load(result, map_location=fixture.device, weights_only=False)+ selection = [row.nonzero().flatten() for row in upstream["selected"].to(fixture.device)]+ jaccard = _jaccard(seen["indices"], selection)+ assert jaccard.min() >= 0.97, f"worst-row Jaccard {jaccard.min():.4f}"+ torch.testing.assert_close(got.float(), upstream["out"].float(), rtol=2e-2, atol=2e-2)diff --git a/tests/models/qwen4_exp/test_qsa_kernels.py b/tests/models/qwen4_exp/test_qsa_kernels.pynew file mode 100644index 00000000..4b678b3b--- /dev/null+++ b/tests/models/qwen4_exp/test_qsa_kernels.py@@ -0,0 +1,486 @@+"""The modified and original QSA Triton kernels against pure-torch references.++Only kernels FreeToken changed or wrote get unit tests: the compression kernel (re-addressed+pending ring, its own torch check) and the block top-k (original radix select, checked against+torch.topk and, through the expansion chain, against the vLLM reference semantics). score.py+and attend.py are vendored from vLLM and are covered by the backend and e2e tests.+``_qsa_mqa_paged_reference`` / ``_qsa_relative_topk_reference`` / ``_expand_qsa_indices_reference``+are transcribed from ``vllm/tests/test_qsa_reference.py`` (Apache-2.0).+"""++from __future__ import annotations++import math++import pytest+import torch++from .common import Fixture, requires_cuda, parsed_config++PAGE_SIZE = 64+RATIO = 4+BUDGET = 2048+INDEX_DIM = 128+CMP_PAGE = PAGE_SIZE // RATIO+++# --------------------------------------------------------------------------------------+# vLLM pure-torch references (tests/test_qsa_reference.py:87-227)+# --------------------------------------------------------------------------------------+++def _qsa_mqa_paged_reference(q, k_cache, page_table, token_to_req, visible_lengths):+ pages = page_table.index_select(0, token_to_req.long()).long()+ keys = k_cache[pages, :, 0, :].flatten(1, 2)+ scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())+ logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])+ positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)+ return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)+++def _qsa_relative_topk_reference(logits, row_starts, row_ends, topk):+ output = torch.full((logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device)+ for row in range(logits.shape[0]):+ start = int(row_starts[row].item())+ length = int((row_ends[row] - row_starts[row]).item())+ width = min(length, topk)+ if width:+ output[row, :width] = torch.topk(+ logits[row, start : start + length], width+ ).indices.to(torch.int32)+ return output+++def _expand_qsa_indices_reference(+ block_indices, query_positions, sequence_lengths, compress_ratio, token_topk+):+ rows = block_indices.shape[0]+ block_topk = token_topk // compress_ratio+ output_width = token_topk + compress_ratio - 1+ offsets = torch.arange(compress_ratio, device=block_indices.device)+ blocks = block_indices.long()+ expanded = blocks.unsqueeze(-1) * compress_ratio + offsets+ expanded = torch.where(+ blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)+ ).reshape(rows, block_topk * compress_ratio)+ expanded = expanded[:, :token_topk]+ expanded = torch.where(+ (expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),+ expanded,+ torch.full_like(expanded, -1),+ )++ tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)+ visible_tokens = query_positions + 1+ tail_start = visible_tokens // compress_ratio * compress_ratio+ tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)+ tail_count = (visible_tokens - tail_start).unsqueeze(1)+ tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (+ tail < sequence_lengths.unsqueeze(1)+ )+ tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))++ result = torch.cat((expanded, tail), dim=1)+ order = torch.arange(output_width, device=result.device).expand(rows, -1)+ sort_key = torch.where(result >= 0, order, order + output_width)+ return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)+++class _Case:+ def __init__(self, **fields):+ self.__dict__.update(fields)+++def _paged_case(length: int, bs: int, rows_per_req: int, seed: int, index_heads: int = 4):+ """Synthetic paged geometry: shuffled pages, the last ``rows_per_req`` queries per request."""+ device = torch.device("cuda")+ torch.manual_seed(seed)+ generator = torch.Generator(device=device).manual_seed(seed)+ pages_per_req = -(-length // PAGE_SIZE)+ total_pages = bs * pages_per_req+ block_table = (+ torch.randperm(total_pages, device=device).reshape(bs, pages_per_req).to(torch.int32)+ )+ q = torch.randn(+ bs * rows_per_req, index_heads, INDEX_DIM, device=device, dtype=torch.bfloat16,+ generator=generator,+ )+ token_to_req = torch.repeat_interleave(+ torch.arange(bs, device=device, dtype=torch.int32), rows_per_req+ )+ query_positions = torch.cat(+ [+ torch.arange(length - rows_per_req, length, device=device, dtype=torch.int32)+ for _ in range(bs)+ ]+ )+ seq_lens = torch.full((bs,), length, device=device, dtype=torch.int32)+ return _Case(+ device=device,+ generator=generator,+ length=length,+ bs=bs,+ pages_per_req=pages_per_req,+ total_pages=total_pages,+ block_table=block_table,+ q=q,+ token_to_req=token_to_req,+ query_positions=query_positions,+ seq_lens=seq_lens,+ )+++@requires_cuda[email protected]("length", [20000])[email protected]("bs", [1])[email protected]("torch_topk", [False, True])+def test_top_blocks_and_expansion_match_vllm_reference(length: int, bs: int, torch_topk: bool):+ from freetoken.kernel.triton.qsa import expand_qsa_block_indices, qsa_mqa_paged++ config = parsed_config()+ fixture = Fixture(config, num_pages=4, max_running_req=2)+ case = _paged_case(length, bs, rows_per_req=4, seed=7 * length + bs)+ cache = torch.randn(+ case.total_pages, CMP_PAGE, 1, INDEX_DIM, device=case.device,+ dtype=torch.bfloat16, generator=case.generator,+ )+ rows, columns = case.q.shape[0], case.pages_per_req * CMP_PAGE+ logits = torch.empty(rows, columns, dtype=torch.float32, device=case.device)+ visible = torch.empty(rows, dtype=torch.int32, device=case.device)+ qsa_mqa_paged(+ case.q, cache, case.block_table, case.token_to_req, case.query_positions,+ case.seq_lens, RATIO, logits, visible,+ )+ blocks = torch.empty(rows, BUDGET // RATIO, dtype=torch.int32, device=case.device)+ reference_logits = _qsa_mqa_paged_reference(+ case.q, cache, case.block_table, case.token_to_req, visible+ )+ if torch_topk:+ fixture.backend._block_topk_kernel = None+ fixture.backend._top_blocks(logits, visible, blocks)++ expected_blocks = _qsa_relative_topk_reference(+ reference_logits, torch.zeros_like(visible), visible, BUDGET // RATIO+ )+ # Ties between equal scores may land on either index; the SET is what selection means.+ torch.testing.assert_close(blocks.sort(-1).values, expected_blocks.sort(-1).values)++ row_seq_lens = case.seq_lens.index_select(0, case.token_to_req.long())+ indices = torch.empty(rows, BUDGET + RATIO - 1, dtype=torch.int32, device=case.device)+ expand_qsa_block_indices(+ expected_blocks, case.query_positions, case.seq_lens, case.token_to_req,+ RATIO, BUDGET, indices,+ )+ expected = _expand_qsa_indices_reference(+ expected_blocks, case.query_positions, row_seq_lens, RATIO, BUDGET+ )+ torch.testing.assert_close(indices, expected)+++@requires_cuda[email protected]("ring_capacity", [4, 8])+def test_compression_reads_both_sources(ring_capacity: int):+ """Members already consumed come from the ring, the rest from this forward's raw rows."""+ from freetoken.kernel.triton.qsa import qsa_compress_groups, qsa_store_rows++ device = torch.device("cuda")+ dim, slots = 8, 3+ pairs = [(0, p) for p in range(2, 9)] + [(1, p) for p in range(5, 11)]++ def key(request: int, position: int) -> torch.Tensor:+ return (torch.arange(dim, dtype=torch.float32) + request * 1000 + position * 10).to(+ torch.bfloat16+ )++ raw = torch.stack([key(*pair) for pair in pairs]).to(device)+ token_to_req = torch.tensor([r for r, _ in pairs], dtype=torch.int32, device=device)+ positions = torch.tensor([p for _, p in pairs], dtype=torch.int32, device=device)+ cu_seqlens = torch.tensor([0, 7, 13], dtype=torch.int32, device=device)+ ring_slots = torch.tensor([2, 0], dtype=torch.int32, device=device)+ ring = torch.zeros(slots, ring_capacity, dim, device=device, dtype=torch.bfloat16)+ for request, position, slot in ((0, 0, 2), (0, 1, 2), (1, 4, 0)):+ ring[slot, position % ring_capacity] = key(request, position).to(device)++ pooled = torch.empty(len(pairs), dim, device=device, dtype=torch.bfloat16)+ first = torch.empty(len(pairs), dtype=torch.int32, device=device)+ qsa_compress_groups(+ raw, ring, ring_slots, token_to_req, cu_seqlens, positions, RATIO, pooled, first+ )++ for row, (request, position) in enumerate(pairs):+ if (position + 1) % RATIO:+ continue+ group = torch.stack([key(request, position - RATIO + 1 + k).float() for k in range(RATIO)])+ expected = group.mean(0).to(torch.bfloat16).to(device)+ assert torch.equal(pooled[row], expected), (request, position)+ assert int(first[row]) == position - RATIO + 1++ # The ring keeps only each request's last ring_capacity rows.+ rows = torch.arange(len(pairs), device=device)+ ends = cu_seqlens.long().index_select(0, token_to_req.long() + 1)+ slot = torch.where(+ rows >= ends - ring_capacity,+ ring_slots.long().index_select(0, token_to_req.long()) * ring_capacity+ + positions.long() % ring_capacity,+ torch.full_like(rows, -1),+ )+ qsa_store_rows(ring, slot.to(torch.int32), raw)+ for request, ring_slot in ((0, 2), (1, 0)):+ for position in [p for r, p in pairs if r == request][-ring_capacity:]:+ assert torch.equal(+ ring[ring_slot, position % ring_capacity], key(request, position).to(device)+ )+++# --------------------------------------------------------------------------------------+# Block top-k (kernel/triton/qsa/topk.py)+# --------------------------------------------------------------------------------------++TOPK_SHAPES = [(512, 512), (4096, 512)]+++def _torch_topk_blocks(logits, visible, width):+ """The torch.topk fallback of ``qsa_sparse._top_blocks``, kept here as the reference."""+ columns = logits.shape[1]+ out = torch.full((logits.shape[0], width), -1, dtype=torch.int32, device=logits.device)+ column = torch.arange(columns, device=logits.device)+ masked = logits.masked_fill(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf"))+ take = min(width, columns)+ values, chosen = torch.topk(masked, take, dim=-1)+ out[:, :take] = torch.where(values > -float("inf"), chosen.to(torch.int32), -1)+ return out+++def _topk_case(n_blocks: int, bs: int, mode: str, seed: int):+ device = torch.device("cuda")+ generator = torch.Generator(device=device).manual_seed(seed)+ logits = torch.randn(bs, n_blocks, device=device, generator=generator)+ visible = torch.full((bs,), n_blocks, dtype=torch.int32, device=device)+ if mode == "ties":+ # Three distinct scores over every column: almost every selection sits on a tie.+ logits = torch.randint(0, 3, (bs, n_blocks), device=device, generator=generator).float()+ if mode == "ragged":+ visible = torch.randint(+ 0, n_blocks + 1, (bs,), dtype=torch.int32, device=device, generator=generator+ )+ if mode == "dead":+ logits[:, ::5] = -float("inf")+ return logits, visible+++@requires_cuda[email protected]("n_blocks,width", TOPK_SHAPES)[email protected]("bs", [4])[email protected]("mode", ["random", "ties", "ragged", "dead"])+def test_block_topk_matches_torch_topk(n_blocks: int, width: int, bs: int, mode: str):+ from freetoken.kernel.triton.qsa import qsa_block_topk++ logits, visible = _topk_case(n_blocks, bs, mode, seed=31 * n_blocks + 7 * width + bs)+ blocks = torch.empty(bs, width, dtype=torch.int32, device=logits.device)+ qsa_block_topk(logits, visible, blocks)+ expected = _torch_topk_blocks(logits, visible, width)++ # Selection is a set: torch.topk orders by descending score, the kernel by column id.+ torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values)+ live = (blocks >= 0).sum(-1)+ torch.testing.assert_close(live, (expected >= 0).sum(-1))+ # expand.py reads ranks [0, complete_blocks), so a -1 may only sit in the tail.+ ranks = torch.arange(width, device=blocks.device)+ assert torch.equal(blocks >= 0, ranks.unsqueeze(0) < live.unsqueeze(1))+ assert bool(((blocks[:, 1:] > blocks[:, :-1]) | (blocks[:, 1:] < 0)).all())+++@requires_cuda+def test_block_topk_replays_in_a_cuda_graph():+ """Fixed grid, no host read: one capture serves every later sequence length."""+ from freetoken.kernel.triton.qsa import qsa_block_topk++ rows, columns, width = 4, 4096, 512+ device = torch.device("cuda")+ logits = torch.randn(rows, columns, device=device)+ visible = torch.full((rows,), columns, dtype=torch.int32, device=device)+ blocks = torch.empty(rows, width, dtype=torch.int32, device=device)++ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ qsa_block_topk(logits, visible, blocks)+ torch.cuda.current_stream().wait_stream(side)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ qsa_block_topk(logits, visible, blocks)++ for lengths in ([columns] * rows, [columns // 2, 900, 37, 0], [4095, 512, 511, 4096]):+ logits.normal_()+ visible.copy_(torch.tensor(lengths, dtype=torch.int32, device=device))+ blocks.fill_(0)+ graph.replay()+ expected = _torch_topk_blocks(logits, visible, width)+ torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values)+++def test_torch_topk_env_picks_the_fallback(monkeypatch):+ from freetoken.attention.qsa_sparse import TORCH_TOPK_ENV, _resolve_block_topk++ assert _resolve_block_topk() is not None+ monkeypatch.setenv(TORCH_TOPK_ENV, "1")+ assert _resolve_block_topk() is None+++# --------------------------------------------------------------------------------------+# Block top-k, split + merge path (wide buffers)+# --------------------------------------------------------------------------------------++SPLIT_CHUNK = 4096 # _split_plan's chunk for every buffer these tests use+++def _policy_topk_blocks(logits, visible, width):+ """The kernel's documented order: highest score wins, lowest column breaks a tie."""+ columns = logits.shape[1]+ column = torch.arange(columns, device=logits.device)+ masked = logits.masked_fill(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf"))+ take = min(width, columns)+ order = masked.argsort(dim=-1, descending=True, stable=True)[:, :take]+ out = torch.full((logits.shape[0], width), -1, dtype=torch.int32, device=logits.device)+ out[:, :take] = torch.where(+ masked.gather(1, order) > -float("inf"), order.to(torch.int32), -1+ )+ return out+++def _split_topk_case(n_blocks: int, width: int, bs: int, mode: str, seed: int):+ if mode != "boundary":+ return _topk_case(n_blocks, bs, mode, seed)+ # width - 212 columns beat the tie, so the 212 remaining winners start 100 columns below+ # a chunk boundary and run past it into a chunk that is all tie.+ logits = torch.zeros(bs, n_blocks, device="cuda")+ for row in range(bs):+ cut = SPLIT_CHUNK * (1 + row % (n_blocks // SPLIT_CHUNK - 1))+ logits[row, : width - 212] = 2.0+ logits[row, cut - 100 :] = 1.0+ return logits, torch.full((bs,), n_blocks, dtype=torch.int32, device="cuda")+++@requires_cuda[email protected]("n_blocks", [65536])[email protected]("bs", [4])[email protected]("mode", ["random", "boundary", "ragged", "dead"])+def test_block_topk_split_path_matches_torch_topk(n_blocks: int, bs: int, mode: str):+ from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width++ width = 512+ assert qsa_block_topk_scratch_width(n_blocks, width) > 0, "case must take the split path"+ logits, visible = _split_topk_case(n_blocks, width, bs, mode, seed=n_blocks + bs + len(mode))+ blocks = torch.empty(bs, width, dtype=torch.int32, device=logits.device)+ qsa_block_topk(logits, visible, blocks)++ torch.testing.assert_close(+ blocks.sort(-1).values, _torch_topk_blocks(logits, visible, width).sort(-1).values+ )+ # Tie determinism: the winners are the exact set the lowest-column-first policy names,+ # including the ties that straddle a chunk boundary.+ torch.testing.assert_close(+ blocks.sort(-1).values, _policy_topk_blocks(logits, visible, width).sort(-1).values+ )+ live = (blocks >= 0).sum(-1)+ ranks = torch.arange(width, device=blocks.device)+ assert torch.equal(blocks >= 0, ranks.unsqueeze(0) < live.unsqueeze(1))+ assert bool(((blocks[:, 1:] > blocks[:, :-1]) | (blocks[:, 1:] < 0)).all())+++@requires_cuda[email protected]("preallocated", [False, True])+def test_block_topk_split_path_replays_in_a_cuda_graph(preallocated: bool):+ """The split geometry comes from the buffer width, so one capture serves every length."""+ from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width++ rows, columns, width = 4, 65536, 512+ device = torch.device("cuda")+ scratch_width = qsa_block_topk_scratch_width(columns, width)+ assert scratch_width > 0+ logits = torch.randn(rows, columns, device=device)+ visible = torch.full((rows,), columns, dtype=torch.int32, device=device)+ blocks = torch.empty(rows, width, dtype=torch.int32, device=device)+ # The scratch never needs clearing: every split rewrites its own slots on every replay.+ scratch = (+ torch.empty(rows, scratch_width, dtype=torch.int32, device=device)+ if preallocated+ else None+ )++ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ qsa_block_topk(logits, visible, blocks, scratch)+ torch.cuda.current_stream().wait_stream(side)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ qsa_block_topk(logits, visible, blocks, scratch)++ for lengths in ([columns] * rows, [4096, 40000, 0, 65535], [1, 4097, 8192, columns]):+ logits.normal_()+ visible.copy_(torch.tensor(lengths, dtype=torch.int32, device=device))+ blocks.fill_(0)+ graph.replay()+ expected = _torch_topk_blocks(logits, visible, width)+ torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values)+++@requires_cuda+def test_block_topk_split_path_cost_tracks_live_blocks():+ """A wide buffer with a short row must not pay for the splits past its visible tail."""+ from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width++ # wide enough that the split work dwarfs the 20-launch floor even at boosted clocks+ rows, columns, width = 1, 262144, 512+ device = torch.device("cuda")+ logits = torch.randn(rows, columns, device=device)+ visible = torch.full((rows,), columns, dtype=torch.int32, device=device)+ blocks = torch.empty(rows, width, dtype=torch.int32, device=device)+ scratch = torch.empty(+ rows, qsa_block_topk_scratch_width(columns, width), dtype=torch.int32, device=device+ )++ side = torch.cuda.Stream()+ side.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(side):+ qsa_block_topk(logits, visible, blocks, scratch)+ torch.cuda.current_stream().wait_stream(side)+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ for _ in range(20):+ qsa_block_topk(logits, visible, blocks, scratch)++ def replay_us(live: int) -> float:+ visible.fill_(live)+ best = float("inf")+ for _ in range(5):+ start, stop = torch.cuda.Event(True), torch.cuda.Event(True)+ start.record()+ graph.replay()+ stop.record()+ torch.cuda.synchronize()+ best = min(best, start.elapsed_time(stop) * 1000.0 / 20)+ return best++ full, short = replay_us(columns), replay_us(SPLIT_CHUNK)+ assert full > 2.0 * short, f"{columns} live {full:.1f}us vs {SPLIT_CHUNK} live {short:.1f}us"+++@requires_cuda+def test_capture_graph_provisions_the_block_topk_scratch():+ from freetoken.kernel.triton.qsa import qsa_block_topk_scratch_width++ fixture = Fixture(parsed_config(), num_pages=320, max_running_req=2)+ backend = fixture.backend+ table_width = fixture.page_table.shape[1]+ columns = table_width // PAGE_SIZE * CMP_PAGE+ width = qsa_block_topk_scratch_width(columns, backend.block_topk)+ assert width > 0, "the fixture must be wide enough to reach the split path"++ backend.init_capture_graph(table_width, [2])+ static = backend._graph["topk_scratch"]+ assert static.shape[1] == width+ assert backend._scratch("topk_scratch", 2, width, dtype=torch.int32).data_ptr() == (+ static.data_ptr()+ )diff --git a/tests/models/qwen4_exp/test_skeleton.py b/tests/models/qwen4_exp/test_skeleton.pynew file mode 100644index 00000000..00f7c43f--- /dev/null+++ b/tests/models/qwen4_exp/test_skeleton.py@@ -0,0 +1,503 @@+"""Skeleton tests: the frozen qwen4_exp interfaces and their torch references.++Everything runs on a scaled-down copy of the real geometry (4 layers, full attention on layer 3,+PLE on layer 1, hc_count 4, 3-gram hash) so the shapes and the layer split are the shipping ones.+The hyper-connection and PLE references transcribed here are HF ``modeling_qwen4_exp.py``+(``Qwen4ExpTextGatedResidual``:941, ``Qwen4ExpTextNGramEmbedding``:1018, ``Qwen4ExpTextPLELayer``+:1117), so the torch implementations are checked against the math, not against themselves.+"""++from __future__ import annotations++import math+from types import SimpleNamespace++import pytest+import torch+import torch.nn.functional as F++from freetoken.layers import BaseOP, LinearReplicated+from freetoken.models.config import ModelConfig+from freetoken.models.qwen4_exp.config import parse_config+from freetoken.models.qwen4_exp.hc import GatedResidual+from freetoken.models.qwen4_exp.ple import GpuResidentTable, PLELayer, PLEMetadata++from .common import EOS, hash_constants, requires_cuda, toy_hf_config+++def _config(num_layers: int = 4) -> ModelConfig:+ return parse_config(toy_hf_config(num_layers))+++def _fill(op, gen: torch.Generator, scale: float = 0.05) -> None:+ """Random floats / zeroed ints for every state-dict tensor of an op tree."""+ for tensor in op.state_dict().values():+ if tensor.is_floating_point():+ tensor.normal_(0.0, scale, generator=gen)+ else:+ tensor.zero_()+++def _group_norm(x, weight, eps, groups):+ xf = x.float().reshape(*x.shape[:-1], groups, -1)+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)+ return (xf.flatten(-2) * (1.0 + weight.float())).type_as(x)+++# --------------------------------------------------------------------------------------+# hyper-connections+# --------------------------------------------------------------------------------------+++def _hf_gated_residual(hc, R, w_norm, w_down, w_up, w_inject, hidden, eps):+ xn = _group_norm(R, w_norm, eps, hc)+ mix = F.silu(F.linear(xn, w_down) / hc)+ mix = torch.sigmoid(F.linear(mix, w_up)).unflatten(-1, (hc, hidden))+ mixed = (mix * xn.unflatten(-1, (hc, hidden))).mean(-2)+ if w_inject is None:+ return mixed, None+ return mixed, 2 * torch.sigmoid(F.linear(xn, w_inject) / hc)++[email protected]("tokens", [1, 7])+def test_hc_mix_and_combine_match_hf(tokens: int):+ torch.manual_seed(0)+ config = _config()+ args = config.qwen4_args+ hc = GatedResidual(config)+ _fill(hc, torch.Generator().manual_seed(1))++ R = torch.randn(tokens, args.ple_state_width)+ y = torch.randn(tokens, args.hidden_size)+ x, s = hc.mix(R)+ got = hc.combine(R, y, s)++ merged = hc.input_mix_weight_down_block_inject.weight+ ref_x, ref_inject = _hf_gated_residual(+ args.hc_count,+ R,+ hc.hc_norm.weight,+ merged[: args.hc_lowrank],+ hc.input_mix_weight_up.weight,+ merged[args.hc_lowrank : args.hc_lowrank + args.hc_count],+ args.hidden_size,+ config.rms_norm_eps,+ )+ ref = R.unflatten(-1, (args.hc_count, args.hidden_size))+ ref = (ref + y.unsqueeze(-2) * ref_inject.unsqueeze(-1)).flatten(-2)++ assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6)+ assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6)+++def test_hc_merged_gemm_layout_and_top_level_mixer():+ config = _config()+ args = config.qwen4_args+ hc = GatedResidual(config)+ # 320 + 4 lowrank/inject rows padded to a multiple of 16 in the real config+ assert hc.pad_size == (-(args.hc_lowrank + args.hc_count)) % 16+ merged = hc.input_mix_weight_down_block_inject.weight+ assert merged.shape == (+ args.hc_lowrank + args.hc_count + hc.pad_size,+ args.ple_state_width,+ )+ assert set(hc.state_dict()) == {+ "hc_norm.weight",+ "input_mix_weight_down_block_inject.weight",+ "input_mix_weight_up.weight",+ }++ mixer = GatedResidual(config, use_combine=False)+ _fill(mixer, torch.Generator().manual_seed(2))+ assert set(mixer.state_dict()) == {+ "hc_norm.weight",+ "input_mix_weight_down.weight",+ "input_mix_weight_up.weight",+ }+ R = torch.randn(5, args.ple_state_width)+ x, s = mixer.mix(R)+ assert s is None+ ref_x, ref_inject = _hf_gated_residual(+ args.hc_count,+ R,+ mixer.hc_norm.weight,+ mixer.input_mix_weight_down.weight,+ mixer.input_mix_weight_up.weight,+ None,+ args.hidden_size,+ config.rms_norm_eps,+ )+ assert ref_inject is None+ assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6)+++# --------------------------------------------------------------------------------------+# PLE+# --------------------------------------------------------------------------------------+++def _hf_shift_right(tokens, shift, eos):+ if shift == 0:+ return tokens+ batch, seq_len = tokens.shape+ positions = torch.arange(seq_len)+ eos_positions = torch.where(tokens == eos, positions, torch.full_like(positions, -1))+ previous = torch.cummax(eos_positions, dim=1).values+ previous = torch.cat([eos_positions.new_full((batch, 1), -1), previous[:, :-1]], dim=1)+ in_segment = positions.unsqueeze(0) - (previous + 1)+ source = positions - shift+ shifted = tokens.gather(1, source.clamp_min(0).unsqueeze(0).expand(batch, -1))+ valid = (in_segment >= shift) & (source.unsqueeze(0) >= 0)+ return torch.where(valid, shifted, tokens.new_full((), eos))+++def _hf_ngram_ids(tokens, context, args, multipliers, sizes, offsets):+ """HF Qwen4ExpTextNGramEmbedding id computation over a dense [B, L] batch."""+ history = torch.cat([context, tokens], dim=-1)+ shifted = [_hf_shift_right(history, s, args.ngram_boundary_token_id) for s in range(args.ngram_size)]+ blocks = []+ for ngram in range(2, args.ngram_size + 1):+ start = (ngram - 2) * args.heads_per_ngram+ end = start + args.heads_per_ngram+ mixed = shifted[0] * multipliers[0]+ for position in range(1, ngram):+ mixed = torch.bitwise_xor(mixed, shifted[position] * multipliers[position])+ ids = torch.remainder(mixed.unsqueeze(-1), sizes[start:end].view(1, 1, -1))+ blocks.append(ids + offsets[start:end].view(1, 1, -1))+ return torch.cat(blocks, dim=-1)[:, -tokens.shape[1] :]+++def _ragged(sequences, contexts, args, device="cpu"):+ """Ragged PLEMetadata (prefill) for a list of per-request token lists."""+ lens = [len(s) for s in sequences]+ cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0)+ return PLEMetadata(+ input_ids=torch.tensor([t for s in sequences for t in s], dtype=torch.int64, device=device),+ cu_seqlens=cu.to(device),+ seq_lens=tuple(lens),+ ngram_context=torch.tensor(contexts, dtype=torch.int64, device=device),+ state_slots=torch.arange(len(sequences), dtype=torch.int64, device=device),+ fresh_slots=None,+ is_decode=False,+ )+++def _make_ple(config, seed: int = 3, rows: int = 4096):+ args = config.qwen4_args+ gen = torch.Generator().manual_seed(seed)+ layer = PLELayer(config, args.ple_layer_ids[0])+ _fill(layer, gen)+ multipliers, sizes, offsets = hash_constants(args)+ layer.ple_embedding.layer_multipliers.copy_(multipliers)+ layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)+ layer.ple_embedding.ngram_heads_offsets.copy_(offsets)+ table = torch.randn(rows, args.ngram_head_dim, generator=gen) * 0.05+ layer.ple_embedding.attach_table(GpuResidentTable(table, dtype=torch.float32))+ return layer, (multipliers, sizes, offsets)+++def test_ple_hash_matches_hf():+ """Ragged hash ids vs the HF dense reference, including eos inside and at the start of a request."""+ config = _config()+ args = config.qwen4_args+ layer, (multipliers, sizes, offsets) = _make_ple(config)++ sequences = [[3, 4, EOS, 5, 6], [EOS, 11, 12], [9]]+ contexts = [[EOS, EOS], [21, 22], [EOS, 31]]+ meta = _ragged(sequences, contexts, args)+ got = layer.ple_embedding.row_ids(meta)++ offset = 0+ for tokens, context in zip(sequences, contexts):+ ref = _hf_ngram_ids(+ torch.tensor([tokens]),+ torch.tensor([context]),+ args,+ multipliers,+ sizes,+ offsets,+ )[0]+ assert torch.equal(got[offset : offset + len(tokens)], ref)+ offset += len(tokens)+ # sequences[0][3] sits right after the boundary token, so its window is cut to eos padding+ after_eos = layer.ple_embedding.row_ids(_ragged([[5]], [[EOS, EOS]], args))[0]+ with_history = layer.ple_embedding.row_ids(_ragged([[5]], [[3, 4]], args))[0]+ assert torch.equal(got[3], after_eos)+ assert not torch.equal(got[3], with_history)+++def test_ple_forward_matches_hf():+ """Full PLE forward (fp32) against the HF gate/norm chain and an explicit conv tap sum."""+ torch.manual_seed(4)+ config = _config()+ args = config.qwen4_args+ layer, (multipliers, sizes, offsets) = _make_ple(config)+ hidden, hc = args.hidden_size, args.hc_count++ sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]]+ contexts = [[EOS, EOS], [21, 22]]+ meta = _ragged(sequences, contexts, args)+ total = sum(len(s) for s in sequences)+ R = torch.randn(total, args.ple_state_width)+ states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1+ got = layer.forward(R, batch=None, meta=meta, conv_states=states.clone())++ offset = 0+ for i, (tokens, context) in enumerate(zip(sequences, contexts)):+ ids = _hf_ngram_ids(+ torch.tensor([tokens]), torch.tensor([context]), args, multipliers, sizes, offsets+ )[0]+ embed = layer.ple_embedding.table.weight[ids.reshape(-1)].view(len(tokens), -1)+ key = _group_norm(+ F.linear(embed, layer.key_proj.weight), layer.norm_key.weight, config.rms_norm_eps, hc+ ).unflatten(-1, (hc, hidden))+ value = F.linear(embed, layer.value_proj.weight)+ rows = R[offset : offset + len(tokens)]+ query = _group_norm(+ rows, layer.norm_query.weight, config.rms_norm_eps, hc+ ).unflatten(-1, (hc, hidden))+ gate = (key * query).sum(-1, keepdim=True) / math.sqrt(hidden)+ gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())+ gated = (gate * value.unsqueeze(-2)).flatten(-2)+ normed = _group_norm(gated, layer.norm_conv.weight, config.rms_norm_eps, hc)+ history = torch.cat([states[i], normed.transpose(0, 1)], dim=-1)+ taps = sum(+ layer.conv1d.weight[:, 0, k].unsqueeze(0)+ * history.transpose(0, 1)[k * args.ple_conv_dilation :][: len(tokens)]+ for k in range(args.ple_conv_kernel_size)+ )+ ref = gated + F.silu(taps)+ assert torch.allclose(got[offset : offset + len(tokens)], ref, rtol=1e-4, atol=1e-5)+ offset += len(tokens)+++def _fresh_ctx(**fields):+ import freetoken.core as core+ from freetoken.core import Context, set_global_ctx++ core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx+ ctx = Context(page_size=64)+ for name, value in fields.items():+ setattr(ctx, name, value)+ set_global_ctx(ctx)+ return ctx+++def _plus_one_rmsnorm(x, weight, eps):+ xf = x.float()+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)+ return (xf * (1.0 + weight.float())).type_as(x)+++def _hf_rope(x, positions, rotary_dim, base):+ """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims."""+ inv = 1.0 / (+ base ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim)+ )+ freqs = positions.float().unsqueeze(-1) * inv+ cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1)+ sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1)+ rot = x[..., :rotary_dim].float()+ half = rotary_dim // 2+ rotated = torch.cat([-rot[..., half:], rot[..., :half]], dim=-1)+ out = (rot * cos + rotated * sin).type_as(x)+ return torch.cat([out, x[..., rotary_dim:]], dim=-1)+++def _hf_attention(x, attn, config, positions):+ """HF Qwen4ExpTextAttention with a dense causal mask (QSA selects every block at this length)."""+ num_q, num_kv, dim = attn.num_q, attn.num_kv, attn.head_dim+ qkv = F.linear(x, attn.qkv_proj.weight)+ qg, k, v = qkv.split(attn._qkv_split, dim=-1)+ qg = qg.view(-1, num_q, dim * 2)+ q, gate = qg[..., :dim], qg[..., dim:].reshape(-1, num_q * dim)+ q = _hf_rope(_plus_one_rmsnorm(q, attn.q_norm.weight, config.rms_norm_eps), positions,+ config.rotary_config.rotary_dim, config.rotary_config.base)+ k = _hf_rope(+ _plus_one_rmsnorm(k.view(-1, num_kv, dim), attn.k_norm.weight, config.rms_norm_eps),+ positions, config.rotary_config.rotary_dim, config.rotary_config.base,+ )+ v = v.view(-1, num_kv, dim)+ rep = num_q // num_kv+ scores = torch.einsum("qhd,khd->hqk", q.float(), k.repeat_interleave(rep, 1).float())+ scores = scores * dim**-0.5+ mask = torch.arange(x.shape[0], device=x.device) > positions.unsqueeze(-1)+ out = torch.einsum(+ "hqk,khd->qhd", scores.masked_fill(mask, float("-inf")).softmax(-1),+ v.repeat_interleave(rep, 1).float(),+ ).to(x.dtype)+ return F.linear(out.reshape(-1, num_q * dim) * torch.sigmoid(gate), attn.o_proj.weight)+++@requires_cuda+def test_qsa_layer_matches_hf_dense():+ """The QSA layer under the dense oracle backend equals HF attention, and freezes what the indexer hands the backend."""+ from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention, TorchDenseQSAReference+ from freetoken.utils.torch_utils import torch_dtype++ torch.manual_seed(6)+ config = _config()+ device, dtype = torch.device("cuda"), torch.bfloat16+ with torch.device(device), torch_dtype(dtype):+ attn = Qwen4ExpAttention(config, layer_id=3)+ _fill(attn, torch.Generator(device=device).manual_seed(7))++ seq_len = 24+ x = (torch.randn(seq_len, config.hidden_size, device=device, dtype=dtype) * 0.5)+ positions = torch.arange(seq_len, device=device, dtype=torch.int64)+ req = SimpleNamespace(extend_len=seq_len, cached_len=0, table_idx=1)+ batch = SimpleNamespace(padded_reqs=[req], reqs=[req], positions=positions)++ backend = TorchDenseQSAReference(config, num_slots=4, max_len=64, device=device, dtype=dtype)+ _fresh_ctx(attn_backend=backend)+ ref = _hf_attention(x, attn, config, positions)+ got = attn.forward(x, batch)+ assert torch.allclose(got.float(), ref.float(), rtol=2e-2, atol=2e-2)++ index = attn.indexer.forward(x)+ args = config.qwen4_args+ raw = F.linear(x, attn.indexer.index_qk_proj.weight)+ assert index.q.shape == (seq_len, args.index_n_heads, args.index_head_dim)+ assert index.k.shape == (seq_len, args.index_head_dim)+ assert torch.equal(index.q.reshape(seq_len, -1), raw[:, : args.index_n_heads * args.index_head_dim])+ assert torch.equal(index.k, raw[:, args.index_n_heads * args.index_head_dim :])+ assert index.q_norm_weight.data_ptr() == attn.indexer.q_layernorm.weight.data_ptr()+++class _StubLinearMixer(BaseOP):+ """Stands in for the GDN layer; same [T, hidden] -> [T, hidden] shape."""++ def __init__(self, config, layer_id):+ self.out_proj = LinearReplicated(config.hidden_size, config.hidden_size, has_bias=False)++ def forward(self, x):+ return self.out_proj.forward(x)+++@requires_cuda+def test_shared_expert_gate_fusion_matches_eager():+ """Qwen4ExpMoE only swaps qwen3_5's gemv+sigmoid+mul+add gate chain for two triton kernels."""+ from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE+ from freetoken.models.qwen4_exp.moe import Qwen4ExpMoE+ from freetoken.moe.fused import FusedMoe+ from freetoken.utils.torch_utils import torch_dtype++ config = _config()+ device, dtype = torch.device("cuda"), torch.bfloat16+ with torch.device(device), torch_dtype(dtype):+ moe = Qwen4ExpMoE(config, 0)+ _fill(moe, torch.Generator(device=device).manual_seed(21), scale=0.2)+ _fresh_ctx(moe_backend=FusedMoe())++ x = torch.randn(6, config.hidden_size, device=device, dtype=dtype) * 0.5+ fused = moe.forward(x.clone())+ eager = Qwen3_5MoE.forward(moe, x.clone())++ routed = moe.experts.forward(hidden_states=x.clone(), router_logits=moe.gate.forward(x))+ gate = torch.sigmoid(x.float() @ moe.shared_expert_gate.weight.float().view(-1))+ ref = routed.float() + gate.unsqueeze(1) * moe.shared_expert.forward(x).float()++ assert fused.shape == x.shape and fused.dtype == dtype+ torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2)+ # The fused gate stays in fp32 where the eager chain rounds the scalar to bf16.+ assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max()+++@requires_cuda[email protected]("dtype", [torch.bfloat16])[email protected]("num_tokens,hidden", [(1, 2560), (7, 640)])+def test_shared_gate_kernels_match_torch(num_tokens, hidden, dtype):+ """Shipping hidden size for the two shared-gate kernels, against the torch chain they replace."""+ from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid++ gen = torch.Generator(device="cuda").manual_seed(hidden + num_tokens)+ kw = {"generator": gen, "device": "cuda", "dtype": dtype}+ x = torch.randn(num_tokens, hidden, **kw)+ weight = torch.randn(1, hidden, **kw) * 0.05+ shared = torch.randn(num_tokens, hidden, **kw)+ routed = torch.randn(num_tokens, hidden, **kw)++ fused = shared_gate_mul_add(routed, shared, shared_gate_sigmoid(x, weight.view(-1)))+ eager = routed + shared * torch.sigmoid(F.linear(x, weight))+ ref = routed.float() + shared.float() * torch.sigmoid(x.float() @ weight.float().view(-1))[:, None]++ assert fused.dtype == dtype and fused.shape == routed.shape+ torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2)+ assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max() + 1e-6+++@requires_cuda+def test_decoder_stack_prefill_and_decode(monkeypatch):+ """Ragged bs=3 prefill then a bs=3 decode step through the whole model with dummy weights."""+ from freetoken.kvcache.linear_state_pool import LinearStatePool+ from freetoken.models.qwen4_exp import model as model_module+ from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference+ from freetoken.models.qwen4_exp.ple import GpuResidentTable+ from freetoken.moe.fused import FusedMoe+ from freetoken.utils.torch_utils import torch_dtype++ torch.manual_seed(8)+ config = _config()+ args = config.qwen4_args+ device, dtype = torch.device("cuda"), torch.bfloat16+ monkeypatch.setattr(model_module, "build_linear_mixer", _StubLinearMixer)++ with torch.device(device), torch_dtype(dtype):+ model = model_module.Qwen4ExpForCausalLM(config)+ gen = torch.Generator(device=device).manual_seed(9)+ _fill(model, gen)+ multipliers, sizes, offsets = hash_constants(args)+ table = torch.randn(4096, args.ngram_head_dim, generator=gen, device=device, dtype=dtype) * 0.05+ for ple in model.model.ple_layers:+ ple.ple_embedding.layer_multipliers.copy_(multipliers)+ ple.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)+ ple.ple_embedding.ngram_heads_offsets.copy_(offsets)+ ple.ple_embedding.attach_table(GpuResidentTable(table, dtype=dtype))++ num_slots, max_len = 4, 64+ pool = LinearStatePool(+ config.linear_attention_group(), num_slots, dtype, device,+ slot_states=config.slot_states,+ )+ prompts = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12], [9, 10, 11, 12, 13]]+ ctx = _fresh_ctx(+ attn_backend=TorchDenseQSAReference(config, num_slots, max_len, device, dtype),+ moe_backend=FusedMoe(),+ linear_state_pool=pool,+ )+ reqs = [+ SimpleNamespace(+ extend_len=len(p), cached_len=0, table_idx=i + 1, linear_slot_idx=None,+ input_ids=torch.tensor(p, dtype=torch.int64),+ )+ for i, p in enumerate(prompts)+ ]+ flat = [t for p in prompts for t in p]+ last = torch.tensor(+ [sum(len(p) for p in prompts[: i + 1]) - 1 for i in range(len(prompts))], device=device+ )+ batch = SimpleNamespace(+ padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=True, is_decode=False,+ input_ids=torch.tensor(flat, dtype=torch.int64, device=device),+ positions=torch.cat([torch.arange(len(p)) for p in prompts]).to(device),+ attn_metadata=SimpleNamespace(get_last_indices=lambda bs: last[:bs]),+ )+ with ctx.forward_batch(batch):+ logits = model.forward()+ assert logits.shape == (len(prompts), config.vocab_size)+ assert torch.isfinite(logits.float()).all()++ for r, p in zip(reqs, prompts):+ r.cached_len = len(p)+ r.extend_len = 1+ r.input_ids = torch.cat([r.input_ids, torch.tensor([14], dtype=torch.int64)])+ decode = SimpleNamespace(+ padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=False, is_decode=True,+ input_ids=torch.tensor([14] * len(reqs), dtype=torch.int64, device=device),+ positions=torch.tensor([len(p) for p in prompts], dtype=torch.int64, device=device),+ attn_metadata=None,+ )+ with ctx.forward_batch(decode):+ decode_logits = model.forward()+ assert decode_logits.shape == (len(prompts), config.vocab_size)+ assert torch.isfinite(decode_logits.float()).all()diff --git a/tests/models/qwen4_exp/test_weight.py b/tests/models/qwen4_exp/test_weight.pynew file mode 100644index 00000000..b3f1f851--- /dev/null+++ b/tests/models/qwen4_exp/test_weight.py@@ -0,0 +1,410 @@+"""qwen4_exp weight loading against a synthetic checkpoint shaped like the RadixArk NVFP4 one.++The tensors are tiny but the key names, dtypes and the fusion geometry that matters+(hc_lowrank=320 + hc_count=4 -> a 12-row zero pad) are the real ones.+"""++from __future__ import annotations++import random+from types import SimpleNamespace++import pytest+import torch+from safetensors.torch import save_file++from freetoken.distributed import set_tp_info, try_get_tp_info+from freetoken.kernel.aot_models import SUPPORTED_MODELS, expert_bank_row_bytes+from freetoken.models.qwen4_exp.weight import (+ _ZERO_CENTERED_NORM_SUFFIXES,+ iter_weights,+ load_ple_table,+)+from freetoken.moe.host_banks import HostBank, read_range_into++H = 32 # hidden_size+HC = 4 # hc_count+LR = 320 # hc_lowrank; kept real so the merged HC pad is the real (-(320+4)) % 16 = 12+HCH = HC * H # hyper-connection stream width+KH, VH, HD = 2, 6, 8 # GDN key / value heads, head dim+QH, KVH, AHD = 4, 2, 16 # QSA q / kv heads, head dim+IHD = 8 # indexer head dim+E, I = 3, 6 # routed experts, moe_intermediate_size+NGRAM_DIM, NGRAM_ROWS, NGRAM_SHARDS = 4, 7, 4++[email protected](scope="session", autouse=True)+def _tp_info():+ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+++def _bf16(*shape: int) -> torch.Tensor:+ return torch.randn(*shape).to(torch.bfloat16)+++def _hc_weights(prefix: str, inject: bool) -> dict[str, torch.Tensor]:+ w = {+ f"{prefix}.hc_norm.weight": _bf16(HCH),+ f"{prefix}.input_mix_weight_down.weight": _bf16(LR, HCH),+ f"{prefix}.input_mix_weight_up.weight": _bf16(HCH, LR),+ }+ if inject:+ w[f"{prefix}.block_inject_weight.weight"] = _bf16(HC, HCH)+ return w+++def _raw_checkpoint() -> dict[str, torch.Tensor]:+ """Layer 0 = GDN + PLE, layer 1 = QSA; plus the mtp / visual / routed-expert noise."""+ lm = "model.language_model"+ raw: dict[str, torch.Tensor] = {+ f"{lm}.embed_tokens.weight": _bf16(11, H),+ "lm_head.weight": _bf16(11, H),+ }+ raw.update(_hc_weights(f"{lm}.hyper_connection_mixer", inject=False))+ for layer in (0, 1):+ raw.update(_hc_weights(f"{lm}.layers.{layer}.attn_hyper_connection", inject=True))+ raw.update(_hc_weights(f"{lm}.layers.{layer}.mlp_hyper_connection", inject=True))+ raw.update({+ f"{lm}.layers.{layer}.mlp.gate.weight": _bf16(E, H),+ f"{lm}.layers.{layer}.mlp.shared_expert.gate_proj.weight": _bf16(I, H),+ f"{lm}.layers.{layer}.mlp.shared_expert.up_proj.weight": _bf16(I, H),+ f"{lm}.layers.{layer}.mlp.shared_expert.down_proj.weight": _bf16(H, I),+ f"{lm}.layers.{layer}.mlp.shared_expert_gate.weight": _bf16(1, H),+ })+ for expert in range(E):+ base = f"{lm}.layers.{layer}.mlp.experts.{expert}"+ for proj, out, inn in (("gate_proj", I, H), ("up_proj", I, H), ("down_proj", H, I)):+ raw[f"{base}.{proj}.weight"] = torch.randint(+ 0, 256, (out, inn // 2), dtype=torch.uint8+ )+ raw[f"{base}.{proj}.weight_scale"] = torch.ones(+ out, inn // 16 or 1, dtype=torch.float8_e4m3fn+ )+ raw[f"{base}.{proj}.weight_scale_2"] = torch.tensor(0.5)+ raw[f"{base}.{proj}.input_scale"] = torch.tensor(0.25)+ gdn = f"{lm}.layers.0.linear_attn"+ raw.update({+ f"{gdn}.in_proj_qkv.weight": _bf16(2 * KH * HD + VH * HD, H),+ f"{gdn}.in_proj_z.weight": _bf16(VH * HD, H),+ f"{gdn}.in_proj_b.weight": _bf16(VH, H),+ f"{gdn}.in_proj_a.weight": _bf16(VH, H),+ f"{gdn}.conv1d.weight": _bf16(2 * KH * HD + VH * HD, 1, 4),+ f"{gdn}.A_log": _bf16(VH),+ f"{gdn}.dt_bias": _bf16(VH),+ f"{gdn}.norm.weight": _bf16(HD),+ f"{gdn}.out_proj.weight": _bf16(H, VH * HD),+ })+ ple = f"{lm}.layers.0.ple"+ raw.update({+ f"{ple}.key_proj.weight": _bf16(HCH, H),+ f"{ple}.value_proj.weight": _bf16(H, H),+ f"{ple}.norm_key.weight": _bf16(HCH),+ f"{ple}.norm_query.weight": _bf16(HCH),+ f"{ple}.norm_conv.weight": _bf16(HCH),+ f"{ple}.conv1d.weight": _bf16(HCH, 1, 4),+ f"{ple}.ple_embedding.layer_multipliers": torch.randint(1, 1 << 40, (3,)),+ f"{ple}.ple_embedding.ngram_heads_offsets": torch.arange(4),+ f"{ple}.ple_embedding.ngram_heads_vocab_sizes": torch.full((4,), 5),+ })+ attn = f"{lm}.layers.1.self_attn"+ raw.update({+ f"{attn}.q_proj.weight": _bf16(2 * QH * AHD, H),+ f"{attn}.k_proj.weight": _bf16(KVH * AHD, H),+ f"{attn}.v_proj.weight": _bf16(KVH * AHD, H),+ f"{attn}.o_proj.weight": _bf16(H, QH * AHD),+ f"{attn}.q_norm.weight": _bf16(AHD),+ f"{attn}.k_norm.weight": _bf16(AHD),+ f"{attn}.indexer.index_qk_proj.weight": _bf16(5 * IHD, H),+ f"{attn}.indexer.q_layernorm.weight": _bf16(IHD),+ f"{attn}.indexer.k_layernorm.weight": _bf16(IHD),+ })+ raw.update({+ "mtp.hyper_connection_mixer.hc_norm.weight": _bf16(HCH),+ "mtp.layers.0.self_attn.q_proj.weight": _bf16(2 * QH * AHD, H),+ "mtp.layers.0.mlp.experts.gate_up_proj": _bf16(E, 2 * I, H),+ "mtp.layers.0.mlp.experts.down_proj": _bf16(E, H, I),+ "model.visual.blocks.0.attn.qkv.weight": _bf16(3 * H, H),+ "model.visual.merger.norm.weight": _bf16(H),+ })+ return raw+++def _ngram_table() -> tuple[dict[str, torch.Tensor], torch.Tensor]:+ prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding"+ shards = {+ f"{prefix}.shard_{i}.weight": (+ torch.arange(i * NGRAM_ROWS * NGRAM_DIM, (i + 1) * NGRAM_ROWS * NGRAM_DIM)+ .remainder(200).to(torch.uint8).view(NGRAM_ROWS, NGRAM_DIM).view(torch.float8_e4m3fn)+ )+ for i in range(NGRAM_SHARDS)+ }+ scale = torch.tensor([0.125], dtype=torch.bfloat16)+ shards[f"{prefix}.weight_scale"] = scale+ return shards, scale++[email protected](scope="module")+def checkpoint(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:+ torch.manual_seed(0)+ folder = tmp_path_factory.mktemp("qwen4_exp_ckpt")+ raw = _raw_checkpoint()+ table, _scale = _ngram_table()+ # Spread the dense tensors over two shards so the fusion buffer has to survive a file+ # boundary, and put the n-gram table in its own shards like the real checkpoint does.+ names = sorted(raw)+ save_file({n: raw[n] for n in names[::2]}, str(folder / "model-bf16-00001.safetensors"))+ save_file({n: raw[n] for n in names[1::2]}, str(folder / "model-bf16-00002.safetensors"))+ shard_names = sorted(table)+ save_file({n: table[n] for n in shard_names[:2]}, str(folder / "model-plefp8-00000.safetensors"))+ save_file({n: table[n] for n in shard_names[2:]}, str(folder / "model-plefp8-00001.safetensors"))+ return str(folder), {**raw, **table}++[email protected](scope="module")+def loaded(checkpoint) -> dict[str, torch.Tensor]:+ folder, _raw = checkpoint+ return {+ name: tensor.clone()+ for name, tensor in iter_weights(+ folder, torch.device("cpu"), include_moe_experts=True, include_non_moe=True+ )+ }+++def _expected_names() -> set[str]:+ names = {"model.embed_tokens.weight", "lm_head.weight"}+ names |= {f"model.hyper_connection_mixer.{leaf}" for leaf in+ ("hc_norm.weight", "input_mix_weight_down.weight", "input_mix_weight_up.weight")}+ for layer in (0, 1):+ for hc in ("attn_hyper_connection", "mlp_hyper_connection"):+ names |= {f"model.layers.{layer}.{hc}.{leaf}" for leaf in (+ "hc_norm.weight", "input_mix_weight_down_block_inject.weight",+ "input_mix_weight_up.weight")}+ names |= {f"model.layers.{layer}.mlp.{leaf}" for leaf in (+ "gate.weight", "shared_expert.gate_up_proj.weight",+ "shared_expert.down_proj.weight", "shared_expert_gate.weight")}+ names |= {f"model.layers.0.linear_attn.{leaf}" for leaf in (+ "in_proj.weight", "conv1d.weight", "A_log", "dt_bias", "norm.weight", "out_proj.weight")}+ names |= {f"model.layers.0.ple.{leaf}" for leaf in (+ "key_proj.weight", "value_proj.weight", "norm_key.weight", "norm_query.weight",+ "norm_conv.weight", "conv1d.weight", "ple_embedding.layer_multipliers",+ "ple_embedding.ngram_heads_offsets", "ple_embedding.ngram_heads_vocab_sizes")}+ names |= {f"model.layers.1.self_attn.{leaf}" for leaf in (+ "qkv_proj.weight", "o_proj.weight", "q_norm.weight", "k_norm.weight",+ "indexer.index_qk_proj.weight", "indexer.q_layernorm.weight",+ "indexer.k_layernorm.weight")}+ return names+++def test_key_map_is_exactly_the_model_state_dict(loaded):+ assert set(loaded) == _expected_names()+++def test_mtp_visual_experts_and_table_never_loaded(loaded):+ for name in loaded:+ assert not name.startswith(("mtp.", "model.visual."))+ assert ".mlp.experts." not in name+ assert "ngram_embedding" not in name+ assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale"))+++def test_hc_merge_is_down_then_inject_then_zero_pad(loaded, checkpoint):+ _folder, raw = checkpoint+ key = "model.layers.0.attn_hyper_connection.input_mix_weight_down_block_inject.weight"+ merged = loaded[key]+ assert merged.shape == (LR + HC + 12, HCH) # pad = (-(320 + 4)) % 16+ down = raw["model.language_model.layers.0.attn_hyper_connection.input_mix_weight_down.weight"]+ inject = raw["model.language_model.layers.0.attn_hyper_connection.block_inject_weight.weight"]+ assert torch.equal(merged[:LR], down)+ assert torch.equal(merged[LR:LR + HC], inject)+ assert torch.equal(merged[LR + HC:], torch.zeros(12, HCH, dtype=merged.dtype))+++def test_top_level_mixer_keeps_the_unmerged_down(loaded, checkpoint):+ _folder, raw = checkpoint+ got = loaded["model.hyper_connection_mixer.input_mix_weight_down.weight"]+ assert got.shape == (LR, HCH)+ assert torch.equal(+ got, raw["model.language_model.hyper_connection_mixer.input_mix_weight_down.weight"]+ )+ assert torch.equal(+ loaded["model.hyper_connection_mixer.input_mix_weight_up.weight"],+ raw["model.language_model.hyper_connection_mixer.input_mix_weight_up.weight"],+ )+++def test_qkv_fusion_slices_back_to_q_k_v(loaded, checkpoint):+ _folder, raw = checkpoint+ attn = "model.language_model.layers.1.self_attn"+ parts = [raw[f"{attn}.{p}_proj.weight"] for p in ("q", "k", "v")]+ fused = loaded["model.layers.1.self_attn.qkv_proj.weight"]+ assert fused.shape == (2 * QH * AHD + 2 * KVH * AHD, H) # q carries the output gate+ for part, back in zip(parts, torch.split(fused, [p.shape[0] for p in parts], dim=0)):+ assert torch.equal(part, back)+++def test_gdn_in_proj_slices_round_trip(loaded, checkpoint):+ _folder, raw = checkpoint+ gdn = "model.language_model.layers.0.linear_attn"+ parts = [raw[f"{gdn}.in_proj_{p}.weight"] for p in ("qkv", "z", "b", "a")]+ fused = loaded["model.layers.0.linear_attn.in_proj.weight"]+ assert fused.shape == (sum(p.shape[0] for p in parts), H)+ splits = torch.split(fused, [p.shape[0] for p in parts], dim=0)+ for part, back in zip(parts, splits):+ assert torch.equal(part, back)+++def test_shared_expert_gate_up_merge(loaded, checkpoint):+ _folder, raw = checkpoint+ base = "model.language_model.layers.1.mlp.shared_expert"+ merged = loaded["model.layers.1.mlp.shared_expert.gate_up_proj.weight"]+ assert torch.equal(merged[:I], raw[f"{base}.gate_proj.weight"])+ assert torch.equal(merged[I:], raw[f"{base}.up_proj.weight"])+++ZERO_CENTERED = (+ "model.layers.0.attn_hyper_connection.hc_norm.weight",+ "model.layers.0.mlp_hyper_connection.hc_norm.weight",+ "model.hyper_connection_mixer.hc_norm.weight",+ "model.layers.0.ple.norm_key.weight",+ "model.layers.0.ple.norm_query.weight",+ "model.layers.0.ple.norm_conv.weight",+ "model.layers.1.self_attn.q_norm.weight",+ "model.layers.1.self_attn.k_norm.weight",+ "model.layers.1.self_attn.indexer.q_layernorm.weight",+ "model.layers.1.self_attn.indexer.k_layernorm.weight",+)+++def test_zero_centered_norms_are_loaded_raw(loaded, checkpoint):+ """(1+w) is applied at runtime in fp32, so the loader must not fold it into the bf16 weight."""+ _folder, raw = checkpoint+ for name in ZERO_CENTERED:+ raw_name = name.replace("model.", "model.language_model.", 1)+ assert torch.equal(loaded[name], raw[raw_name]), name+++def test_the_zero_centered_suffix_list_covers_every_such_norm():+ assert {n for n in ZERO_CENTERED if n.endswith(_ZERO_CENTERED_NORM_SUFFIXES)} == set(ZERO_CENTERED)+ assert not "model.layers.0.linear_attn.norm.weight".endswith(_ZERO_CENTERED_NORM_SUFFIXES)+++def test_gdn_gated_norm_passes_through(loaded, checkpoint):+ _folder, raw = checkpoint+ assert torch.equal(+ loaded["model.layers.0.linear_attn.norm.weight"],+ raw["model.language_model.layers.0.linear_attn.norm.weight"],+ )+++def test_hash_constants_stay_int64(loaded):+ for leaf in ("layer_multipliers", "ngram_heads_offsets", "ngram_heads_vocab_sizes"):+ assert loaded[f"model.layers.0.ple.ple_embedding.{leaf}"].dtype is torch.int64+++def test_load_ple_table_concatenates_shards_in_index_order(checkpoint):+ folder, raw = checkpoint+ args = SimpleNamespace(split_ngram_parts=NGRAM_SHARDS, ngram_head_dim=NGRAM_DIM)+ table = load_ple_table(folder, args, pin=False)+ assert table.tensor.shape == (NGRAM_SHARDS * NGRAM_ROWS, NGRAM_DIM)+ assert table.tensor.dtype is torch.float8_e4m3fn+ prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding"+ for shard in range(NGRAM_SHARDS):+ rows = table.tensor[shard * NGRAM_ROWS: (shard + 1) * NGRAM_ROWS]+ assert torch.equal(rows.view(torch.uint8),+ raw[f"{prefix}.shard_{shard}.weight"].view(torch.uint8))+ assert table.weight_scale.dtype is torch.bfloat16+ assert float(table.weight_scale) == 0.125+++def test_load_ple_table_rejects_a_shard_count_mismatch(checkpoint):+ folder, _raw = checkpoint+ args = SimpleNamespace(split_ngram_parts=NGRAM_SHARDS + 1, ngram_head_dim=NGRAM_DIM)+ with pytest.raises(ValueError, match="shards 0"):+ load_ple_table(folder, args, pin=False)+++# ======================================================================================+# read_range_into: the O_DIRECT byte-range read the PLE table load is built on+# ======================================================================================++[email protected](scope="module")+def blob(tmp_path_factory) -> tuple[str, bytes]:+ data = random.Random(7).randbytes(5_000_003)+ path = tmp_path_factory.mktemp("blob") / "data.bin"+ path.write_bytes(data)+ return str(path), data++[email protected]("file_offset, nbytes, dest_offset", [+ (1, 4095, 0), # sub-block, unaligned source+ (2239, 1_000_000, 0), # the real checkpoint's header-end phase+ (4095, 4097, 1), # straddles two block boundaries+ (4_999_000, 1003, 123_456), # runs to EOF+])+def test_read_range_into_matches_the_file(blob, file_offset, nbytes, dest_offset):+ path, data = blob+ bank = HostBank((6_000_000,), torch.uint8)+ view = bank.memoryview()+ got = read_range_into(view, path, file_offset=file_offset, nbytes=nbytes,+ dest_offset=dest_offset, chunk=1 << 20)+ assert got == nbytes+ assert bytes(view[dest_offset:dest_offset + nbytes]) == data[file_offset:file_offset + nbytes]+++def test_read_range_into_is_chunk_and_thread_safe(blob):+ path, data = blob+ bank = HostBank((6_000_000,), torch.uint8)+ view = bank.memoryview()+ read_range_into(view, path, file_offset=2239, nbytes=4_000_000, dest_offset=1024,+ workers=8, chunk=64 << 10)+ assert bytes(view[1024:1024 + 4_000_000]) == data[2239:2239 + 4_000_000]+++def test_read_range_into_rejects_a_short_destination(blob):+ path, _data = blob+ bank = HostBank((1024,), torch.uint8)+ with pytest.raises(ValueError, match="destination holds"):+ read_range_into(bank.memoryview(), path, file_offset=0, nbytes=1 << 20)+++# ======================================================================================+# AOT shape table+# ======================================================================================+++def test_aot_entry_carries_the_checkpoint_geometry():+ entry = next(m for m in SUPPORTED_MODELS+ if m.architecture == "Qwen4ExpForConditionalGeneration")+ assert (entry.hidden_size, entry.moe_intermediate_size, entry.top_k) == (2560, 640, 10)+ assert entry.kv_groups == ((2, 256),)+ rows = expert_bank_row_bytes("nvfp4", entry.hidden_size, entry.moe_intermediate_size)+ assert set(rows) == {"gate_up_packed", "gate_up_scale", "gate_up_global",+ "down_packed", "down_scale", "down_global"}+ for name, nbytes in rows.items():+ assert nbytes % 16 == 0, name # fused multi-bank copy only engages on 16B multiples+++def test_every_registry_architecture_is_claimed_by_an_aot_entry():+ from freetoken.models.register import _MODEL_REGISTRY++ claimed = {m.architecture for m in SUPPORTED_MODELS}+ claimed |= {a for m in SUPPORTED_MODELS for a in m.arch_aliases}+ assert "Qwen4ExpForConditionalGeneration" in claimed+ assert set(_MODEL_REGISTRY) - claimed == set()++[email protected](not torch.cuda.is_available(), reason="needs cuda")+def test_fusion_pad_rides_the_tensor_device():+ """safetensors loads straight to cuda; a cpu-allocated pad row would break torch.cat."""+ from freetoken.models.qwen4_exp.weight import _try_fuse++ buf = {}+ down = torch.randn(320, 64, device="cuda", dtype=torch.bfloat16)+ inject = torch.randn(4, 64, device="cuda", dtype=torch.bfloat16)+ assert _try_fuse("model.layers.0.attn_hyper_connection.input_mix_weight_down.weight", down, buf) == ()+ key, fused = _try_fuse("model.layers.0.attn_hyper_connection.block_inject_weight.weight", inject, buf)+ assert fused.device.type == "cuda" and fused.shape[0] == 336+ assert torch.equal(fused[324:], torch.zeros(12, 64, device="cuda", dtype=torch.bfloat16))diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.pynew file mode 100644index 00000000..747d22f2--- /dev/null+++ b/tests/models/qwen4_exp/test_weight_ckpt.py@@ -0,0 +1,338 @@+"""qwen4_exp weight loading against the real RadixArk/Qwen3.8-Flash-Next-NVFP4 checkpoint.++Set ``FREETOKEN_QWEN4EXP_MODEL`` to the local checkpoint directory to run these. Everything is+sampled except the PLE table, which is loaded and pinned in full once (~47.7 GiB) because that+is the only way to check the shard concatenation and the pin budget.+"""++from __future__ import annotations++import dataclasses+import json+import os+import random+from types import SimpleNamespace++import pytest+import safetensors+import torch++from freetoken.distributed import set_tp_info, try_get_tp_info+from freetoken.kernel.aot_models import expert_bank_row_bytes+from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks+from freetoken.models.qwen4_exp.config import parse_config+from freetoken.models.qwen4_exp.weight import (+ _NVFP4_SOURCE_SPEC,+ _ZERO_CENTERED_NORM_SUFFIXES,+ iter_weights,+ load_ple_table,+)+from freetoken.moe.host_banks import HostResidency+from freetoken.utils import cached_load_hf_config++MODEL_PATH = os.environ.get("FREETOKEN_QWEN4EXP_MODEL")+pytestmark = [+ pytest.mark.needs_weights,+ pytest.mark.skipif(not MODEL_PATH, reason="FREETOKEN_QWEN4EXP_MODEL is not set"),+]++E, H, I = 512, 2560, 640+NUM_LAYERS = 48+PLE_LAYER = 1+PLE_SHARDS, PLE_ROWS_PER_SHARD, PLE_DIM = 128, 2_500_012, 160+PLE_BYTES = PLE_SHARDS * PLE_ROWS_PER_SHARD * PLE_DIM+EXPERT_LAYER_BYTES = E * sum(expert_bank_row_bytes("nvfp4", H, I).values())+LM = "model.language_model"++[email protected](scope="session", autouse=True)+def _tp_info():+ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+++class _Reader:+ """Serves checkpoint tensors by their raw key, through the index shard map."""++ def __init__(self, folder: str):+ with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as fh:+ self._map = json.load(fh)["weight_map"]+ self._folder = folder+ self._handles: dict = {}++ def get(self, name: str) -> torch.Tensor:+ shard = self._map[name]+ handle = self._handles.get(shard)+ if handle is None:+ handle = safetensors.safe_open(+ os.path.join(self._folder, shard), framework="pt", device="cpu"+ ).__enter__()+ self._handles[shard] = handle+ return handle.get_tensor(name)++ def close(self) -> None:+ for handle in self._handles.values():+ handle.__exit__(None, None, None)+ self._handles.clear()+++def _gdn_parts(layer: int) -> list[str]:+ return [f"{LM}.layers.{layer}.linear_attn.in_proj_{p}.weight" for p in ("qkv", "z", "b", "a")]+++def _hc_parts(layer: int, hc: str) -> list[str]:+ return [f"{LM}.layers.{layer}.{hc}.input_mix_weight_down.weight",+ f"{LM}.layers.{layer}.{hc}.block_inject_weight.weight"]+++def _qkv_parts(layer: int) -> list[str]:+ return [f"{LM}.layers.{layer}.self_attn.{p}_proj.weight" for p in ("q", "k", "v")]+++# (model key, checkpoint keys, mode). "cat16" additionally requires the merged rows to be+# zero-padded up to a multiple of 16. Zero-centered norms are "same": (1+w) is a runtime op.+SAMPLES: tuple[tuple[str, list[str], str], ...] = (+ ("model.embed_tokens.weight", [f"{LM}.embed_tokens.weight"], "same"),+ ("lm_head.weight", ["lm_head.weight"], "same"),+ ("model.layers.0.linear_attn.in_proj.weight", _gdn_parts(0), "cat"),+ ("model.layers.46.linear_attn.in_proj.weight", _gdn_parts(46), "cat"),+ ("model.layers.0.linear_attn.conv1d.weight", [f"{LM}.layers.0.linear_attn.conv1d.weight"], "same"),+ ("model.layers.0.linear_attn.A_log", [f"{LM}.layers.0.linear_attn.A_log"], "same"),+ ("model.layers.0.linear_attn.dt_bias", [f"{LM}.layers.0.linear_attn.dt_bias"], "same"),+ ("model.layers.0.linear_attn.norm.weight", [f"{LM}.layers.0.linear_attn.norm.weight"], "same"),+ ("model.layers.0.linear_attn.out_proj.weight", [f"{LM}.layers.0.linear_attn.out_proj.weight"], "same"),+ ("model.layers.3.self_attn.qkv_proj.weight", _qkv_parts(3), "cat"),+ ("model.layers.47.self_attn.qkv_proj.weight", _qkv_parts(47), "cat"),+ ("model.layers.3.self_attn.o_proj.weight", [f"{LM}.layers.3.self_attn.o_proj.weight"], "same"),+ ("model.layers.3.self_attn.q_norm.weight", [f"{LM}.layers.3.self_attn.q_norm.weight"], "same"),+ ("model.layers.3.self_attn.k_norm.weight", [f"{LM}.layers.3.self_attn.k_norm.weight"], "same"),+ ("model.layers.3.self_attn.indexer.index_qk_proj.weight",+ [f"{LM}.layers.3.self_attn.indexer.index_qk_proj.weight"], "same"),+ ("model.layers.3.self_attn.indexer.q_layernorm.weight",+ [f"{LM}.layers.3.self_attn.indexer.q_layernorm.weight"], "same"),+ ("model.layers.47.self_attn.indexer.k_layernorm.weight",+ [f"{LM}.layers.47.self_attn.indexer.k_layernorm.weight"], "same"),+ ("model.layers.7.attn_hyper_connection.hc_norm.weight",+ [f"{LM}.layers.7.attn_hyper_connection.hc_norm.weight"], "same"),+ ("model.layers.7.attn_hyper_connection.input_mix_weight_down_block_inject.weight",+ _hc_parts(7, "attn_hyper_connection"), "cat16"),+ ("model.layers.7.attn_hyper_connection.input_mix_weight_up.weight",+ [f"{LM}.layers.7.attn_hyper_connection.input_mix_weight_up.weight"], "same"),+ ("model.layers.47.mlp_hyper_connection.input_mix_weight_down_block_inject.weight",+ _hc_parts(47, "mlp_hyper_connection"), "cat16"),+ ("model.hyper_connection_mixer.hc_norm.weight",+ [f"{LM}.hyper_connection_mixer.hc_norm.weight"], "same"),+ ("model.hyper_connection_mixer.input_mix_weight_down.weight",+ [f"{LM}.hyper_connection_mixer.input_mix_weight_down.weight"], "same"),+ ("model.hyper_connection_mixer.input_mix_weight_up.weight",+ [f"{LM}.hyper_connection_mixer.input_mix_weight_up.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.key_proj.weight", [f"{LM}.layers.{PLE_LAYER}.ple.key_proj.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.value_proj.weight", [f"{LM}.layers.{PLE_LAYER}.ple.value_proj.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.norm_key.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_key.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.norm_query.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_query.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.norm_conv.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_conv.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.conv1d.weight", [f"{LM}.layers.{PLE_LAYER}.ple.conv1d.weight"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.ple_embedding.layer_multipliers",+ [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.layer_multipliers"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_offsets",+ [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_offsets"], "same"),+ (f"model.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_vocab_sizes",+ [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_vocab_sizes"], "same"),+ ("model.layers.5.mlp.gate.weight", [f"{LM}.layers.5.mlp.gate.weight"], "same"),+ ("model.layers.5.mlp.shared_expert.gate_up_proj.weight",+ [f"{LM}.layers.5.mlp.shared_expert.gate_proj.weight",+ f"{LM}.layers.5.mlp.shared_expert.up_proj.weight"], "cat"),+ ("model.layers.5.mlp.shared_expert.down_proj.weight",+ [f"{LM}.layers.5.mlp.shared_expert.down_proj.weight"], "same"),+ ("model.layers.5.mlp.shared_expert_gate.weight",+ [f"{LM}.layers.5.mlp.shared_expert_gate.weight"], "same"),+)++[email protected](scope="module")+def reader() -> _Reader:+ r = _Reader(MODEL_PATH)+ yield r+ r.close()++[email protected](scope="module")+def dense_pass() -> tuple[list[str], dict[str, torch.Tensor]]:+ """One full iter_weights sweep: every emitted name, plus a clone of each sampled tensor.++ The zero-centered norms are kept too -- they are tiny, and checking all of them is the+ cheapest guard against the +1 creeping back into the load path."""+ wanted = {name for name, _raw, _mode in SAMPLES}+ names: list[str] = []+ sampled: dict[str, torch.Tensor] = {}+ for name, tensor in iter_weights(+ MODEL_PATH, torch.device("cpu"), include_moe_experts=True, include_non_moe=True+ ):+ names.append(name)+ if name in wanted or name.endswith(_ZERO_CENTERED_NORM_SUFFIXES):+ sampled[name] = tensor.clone()+ return names, sampled+++def test_emitted_names_are_unique_and_complete(dense_pass):+ names, _sampled = dense_pass+ assert len(names) == len(set(names))+ assert len([n for n in names if n.endswith(".linear_attn.in_proj.weight")]) == 36+ assert len([n for n in names if n.endswith(".self_attn.qkv_proj.weight")]) == 12+ assert len([n for n in names+ if n.endswith(".input_mix_weight_down_block_inject.weight")]) == 2 * NUM_LAYERS+ assert len([n for n in names if ".ple." in n]) == 9+ assert {"model.embed_tokens.weight", "lm_head.weight",+ "model.hyper_connection_mixer.input_mix_weight_down.weight"} <= set(names)++[email protected](scope="module")+def model_state_dict_keys() -> set[str]:+ """Keys ``Qwen4ExpForCausalLM`` declares -- the authoritative target the loader must fill."""+ from freetoken.layers import rotary+ from freetoken.models.qwen4_exp.model import Qwen4ExpForCausalLM++ config = parse_config(cached_load_hf_config(MODEL_PATH))+ saved = rotary._ROPE_DEVICE+ rotary.set_rope_device(torch.device("cpu")) # get_rope refuses to build on meta+ rotary.get_rope.cache_clear()+ try:+ with torch.device("meta"):+ return set(Qwen4ExpForCausalLM(config).state_dict())+ finally:+ rotary.set_rope_device(saved)+ rotary.get_rope.cache_clear()+++def test_emitted_names_are_the_model_state_dict(dense_pass, model_state_dict_keys):+ names, _sampled = dense_pass+ # The routed NVFP4 experts come from the offload source banks, never from the dense pass.+ expected = {k for k in model_state_dict_keys+ if not k.endswith((".mlp.experts.gate_up_proj", ".mlp.experts.down_proj"))}+ assert set(names) == expected+++def test_every_zero_centered_norm_is_present_and_raw(dense_pass, reader):+ names, sampled = dense_pass+ zero_centered = [n for n in names if n.endswith(_ZERO_CENTERED_NORM_SUFFIXES)]+ # 2 HC per layer + the top-level mixer + 3 PLE norms + q/k_norm and indexer q/k per QSA layer+ assert len(zero_centered) == 2 * NUM_LAYERS + 1 + 3 + 4 * 12+ for name in zero_centered:+ assert torch.equal(sampled[name], reader.get(name.replace("model.", f"{LM}.", 1))), name+++def test_no_mtp_visual_expert_or_table_tensor_is_loaded(dense_pass):+ names, _sampled = dense_pass+ for name in names:+ assert not name.startswith(("mtp.", "model.visual.", "visual."))+ assert ".mlp.experts." not in name+ assert "ngram_embedding" not in name+ assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale"))++[email protected]("name, raw_names, mode", SAMPLES, ids=[s[0] for s in SAMPLES])+def test_sampled_tensor_matches_the_checkpoint(dense_pass, reader, name, raw_names, mode):+ _names, sampled = dense_pass+ parts = [reader.get(raw) for raw in raw_names]+ got = sampled[name]+ if mode == "same":+ assert torch.equal(got, parts[0])+ assert got.dtype is parts[0].dtype+ else:+ rows = sum(p.shape[0] for p in parts)+ assert torch.equal(got[:rows], torch.cat(parts, dim=0))+ if mode == "cat16":+ assert got.shape[0] == rows + (-rows) % 16+ assert not got[rows:].any()+ else:+ assert got.shape[0] == rows++[email protected](not torch.cuda.is_available(), reason="pinning needs CUDA")+def test_ple_table_loads_pinned_and_matches_the_checkpoint(reader):+ args = parse_config(cached_load_hf_config(MODEL_PATH)).qwen4_args+ table = load_ple_table(MODEL_PATH, args)+ assert table.bank.residency is HostResidency.PINNED+ assert table.bank.nbytes == PLE_BYTES+ assert abs(PLE_BYTES / 2**30 - 47.68) < 0.05+ assert table.tensor.shape == (PLE_SHARDS * PLE_ROWS_PER_SHARD, PLE_DIM)+ assert table.tensor.dtype is torch.float8_e4m3fn++ prefix = f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_embedding"+ assert torch.equal(table.weight_scale.reshape(1),+ reader.get(f"{prefix}.weight_scale").reshape(1))+ rows = random.Random(0).sample(range(PLE_SHARDS * PLE_ROWS_PER_SHARD), 1000)+ by_shard: dict[int, list[int]] = {}+ for row in rows:+ by_shard.setdefault(row // PLE_ROWS_PER_SHARD, []).append(row)+ got = table.tensor.view(torch.uint8)+ for shard, shard_rows in by_shard.items():+ ref = reader.get(f"{prefix}.shard_{shard}.weight").view(torch.uint8)+ local = torch.tensor([r - shard * PLE_ROWS_PER_SHARD for r in shard_rows])+ assert torch.equal(got[torch.tensor(shard_rows)], ref[local])++ # The two pinned host allocations the engine must budget for.+ total = PLE_BYTES + NUM_LAYERS * EXPERT_LAYER_BYTES+ assert abs(total / 2**30 - 111.14) < 0.05++[email protected](scope="module")+def layer0_expert_banks():+ """The real NVFP4 source-bank loader, restricted to layer 0 (1.32 GiB instead of 63.5)."""+ if not torch.cuda.is_available():+ pytest.skip("expert bank pinning needs CUDA")+ spec = dataclasses.replace(+ _NVFP4_SOURCE_SPEC, layer_to_bank=lambda layer, config: 0 if layer == 0 else None+ )+ config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,+ num_moe_layers=1)+ return load_nvfp4_expert_source_banks(+ MODEL_PATH, config, spec, drop_page_cache=lambda path: None, primary=False+ )+++def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader):+ banks = layer0_expert_banks+ for expert in random.Random(1).sample(range(E), 8):+ base = f"{LM}.layers.0.mlp.experts.{expert}"+ assert torch.equal(banks["gate_up_packed"][0][expert, :I],+ reader.get(f"{base}.gate_proj.weight"))+ assert torch.equal(banks["gate_up_packed"][0][expert, I:],+ reader.get(f"{base}.up_proj.weight"))+ assert torch.equal(banks["down_packed"][0][expert],+ reader.get(f"{base}.down_proj.weight"))+ for proj, bank, rows in (("gate_proj", "gate_up_scale", slice(0, I)),+ ("up_proj", "gate_up_scale", slice(I, 2 * I)),+ ("down_proj", "down_scale", slice(None))):+ scale = reader.get(f"{base}.{proj}.weight_scale")+ assert torch.equal(banks[bank][0][expert][rows].reshape(-1).view(torch.uint8),+ scale.reshape(-1).view(torch.uint8))+ gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float16)+ assert torch.equal(banks["gate_up_global"][0][expert, :I], gate_g.reshape(1).expand(I))+++def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks):+ measured = sum(t[0].numel() * t[0].element_size() for t in layer0_expert_banks.values())+ assert measured == EXPERT_LAYER_BYTES+ assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.46) < 0.05++[email protected](not torch.cuda.is_available(), reason="dummy banks are pinned")+def test_dummy_expert_sources_have_the_real_bank_shapes(layer0_expert_banks):+ from freetoken.models.weight import _model_override, dummy_nvfp4_expert_sources+ from freetoken.models.register import get_model_spec++ spec = get_model_spec("Qwen4ExpForConditionalGeneration")+ # No dummy_* override, so --use-dummy-weight goes through the generic builders.+ for hook in ("dummy_nvfp4_expert_sources", "dummy_moe_expert_sources", "dummy_q4_0_expert_sources"):+ assert _model_override(spec, hook) is None++ config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,+ num_moe_layers=1)+ dummy = dummy_nvfp4_expert_sources(config)+ assert set(dummy) == set(layer0_expert_banks)+ for name, banks in dummy.items():+ assert banks[0].shape == layer0_expert_banks[name][0].shape+ assert banks[0].dtype is layer0_expert_banks[name][0].dtypediff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.pyindex 787661dc..9cd8c7fb 100644--- a/tests/models/test_minimax_m3.py+++ b/tests/models/test_minimax_m3.py@@ -234,7 +234,7 @@ def test_auto_backend_resolution():cfg = parse_config(_hf_config())required = _required_attn_types(cfg)assert required == frozenset({AttnType.BSA})- assert _resolve_auto_attention_backend(required, False) == "m3_sparse"+ assert _resolve_auto_attention_backend(required) == "m3_sparse"assert attention_backend_info("m3_sparse").page_sizes == (128,)diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.pyindex 4a4457e9..cbd579e4 100644--- a/tests/models/test_muse_glimmer.py+++ b/tests/models/test_muse_glimmer.py@@ -153,7 +153,7 @@ def test_pool_family_and_backend_resolution():assert required == frozenset({AttnType.FULL, AttnType.SWA})# SWA restricts serving to the triton backend (the only one in the capability# matrix that consumes per-call sliding windows), same as gemma4.- assert _resolve_auto_attention_backend(required, False) == "triton"+ assert _resolve_auto_attention_backend(required) == "triton"def test_registry_resolves_architecture():diff --git a/tests/moe/test_fused_moe.py b/tests/moe/test_fused_moe.pyindex 1fd0f2e5..41a75c20 100644--- a/tests/moe/test_fused_moe.py+++ b/tests/moe/test_fused_moe.py@@ -274,3 +274,98 @@ def test_fused_experts_decode_activation_and_router_weight_modes(torch.cuda.synchronize()torch.testing.assert_close(output, expected, rtol=5e-2, atol=5e-2)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+def test_fused_topk_non_power_of_2_k_routes_vendored_router():+ """triton_kernels.topk builds tl.arange(0, k) (power-of-2 only); k=10 must not reach it."""+ from freetoken.moe.fused import _torch_fused_topk, fused_topk++ gating = torch.randn(5, 64, device="cuda")+ hidden = torch.randn(5, 8, device="cuda")+ weights, ids = fused_topk(hidden, gating, 10, renormalize=True)+ ref_w, ref_i = _torch_fused_topk(gating, 10, True, None)+ assert torch.equal(ids, ref_i)+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)+++# The vendored triton router behind that k=10 branch; fp32 logits keep the reference top-k tie-free.+# Ties get their own case below, because torch.topk does not break them by expert id.[email protected](not torch.cuda.is_available(), reason="CUDA is required")[email protected]("renormalize", [True, False])+ "num_tokens,num_experts,topk",+ [(1, 512, 10), (7, 512, 10), (129, 512, 10), (33, 512, 6), (4, 64, 3)],+)+def test_fused_topk_softmax_matches_torch_reference(num_tokens, num_experts, topk, renormalize):+ from freetoken.kernel.triton.moe_router import fused_topk_softmax+ from freetoken.moe.fused import _torch_fused_topk++ gen = torch.Generator(device="cuda").manual_seed(num_tokens * 31 + topk)+ gating = torch.randn(num_tokens, num_experts, generator=gen, device="cuda")++ weights, ids = fused_topk_softmax(gating, topk, renormalize)+ ref_w, ref_i = _torch_fused_topk(gating, topk, renormalize, None)++ assert weights.dtype == torch.float32 and ids.dtype == torch.int32+ assert torch.equal(ids, ref_i)+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+def test_fused_topk_softmax_ties_pick_the_lowest_expert_id():+ from freetoken.kernel.triton.moe_router import fused_topk_softmax++ gating = torch.full((2, 8), -10.0, device="cuda")+ gating[0, [6, 2, 5]] = 1.0 # three-way tie for two slots+ gating[1] = 0.0 # whole row tied++ weights, ids = fused_topk_softmax(gating, 3, renormalize=True)++ assert ids[0].tolist() == [2, 5, 6]+ assert ids[1].tolist() == [0, 1, 2]+ torch.testing.assert_close(weights, torch.full((2, 3), 1.0 / 3.0, device="cuda"))++[email protected](not torch.cuda.is_available(), reason="CUDA is required")[email protected]("limit_dtype", [torch.int32, torch.int64])+def test_fused_topk_softmax_masks_padded_rows(limit_dtype):+ from freetoken.kernel.triton.moe_router import fused_topk_softmax+ from freetoken.moe.fused import _torch_fused_topk++ gen = torch.Generator(device="cuda").manual_seed(5)+ gating = torch.randn(16, 512, generator=gen, device="cuda")+ limit = torch.tensor(5, dtype=limit_dtype, device="cuda")++ weights, ids = fused_topk_softmax(gating, 10, True, limit)+ ref_w, ref_i = _torch_fused_topk(gating, 10, True, limit)++ assert (ids[5:] == -1).all()+ assert torch.equal(ids, ref_i)+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)++[email protected](not torch.cuda.is_available(), reason="CUDA is required")+def test_fused_topk_softmax_is_cuda_graph_capturable():+ """The padded-row limit must come off the device tensor, not a host read baked into the graph."""+ from freetoken.kernel.triton.moe_router import fused_topk_softmax++ gen = torch.Generator(device="cuda").manual_seed(11)+ gating = torch.randn(8, 512, generator=gen, device="cuda")+ limit = torch.tensor(8, dtype=torch.int32, device="cuda")++ stream = torch.cuda.Stream()+ stream.wait_stream(torch.cuda.current_stream())+ with torch.cuda.stream(stream):+ fused_topk_softmax(gating, 10, True, limit)+ torch.cuda.current_stream().wait_stream(stream)++ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ _, ids = fused_topk_softmax(gating, 10, True, limit)++ limit.fill_(3)+ graph.replay()+ torch.cuda.synchronize()+ assert (ids[:3] != -1).all()+ assert (ids[3:] == -1).all()diff --git a/tests/scheduler/test_abort_inflight_prefill.py b/tests/scheduler/test_abort_inflight_prefill.pyindex 1c9a9fba..a50d0d32 100644--- a/tests/scheduler/test_abort_inflight_prefill.py+++ b/tests/scheduler/test_abort_inflight_prefill.py@@ -37,7 +37,7 @@ UID = 2def _pool(num_slots=16):g = LinearGatedDeltaGroupConfig(name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4,- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",)return LinearStatePool(group=g, num_slots=num_slots, dtype=torch.bfloat16,device=torch.device("cpu"), tp_size=1)diff --git a/tests/scheduler/test_hybrid_cache_manager.py b/tests/scheduler/test_hybrid_cache_manager.pyindex 19be56f3..080999ad 100644--- a/tests/scheduler/test_hybrid_cache_manager.py+++ b/tests/scheduler/test_hybrid_cache_manager.py@@ -16,7 +16,7 @@ from freetoken.scheduler.cache import CacheManagerdef _pool(num_slots=16):g = LinearGatedDeltaGroupConfig(name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4,- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",)return LinearStatePool(group=g, num_slots=num_slots, dtype=torch.bfloat16,device=torch.device("cpu"), tp_size=1)@@ -116,6 +116,43 @@ def test_rebuild_reclaims_donated_gdn_slots():assert pool.num_free_slots == pool.num_slots - 1 # all GDN slots reclaimed (no leak)+def test_prefill_chunk_ends_on_a_page_boundary():+ """A hybrid chunk must end page-aligned: the snapshot commit skips any other boundary."""+ from freetoken.scheduler.prefill import ChunkedReq, PrefillAdder+ from freetoken.scheduler.table import TableManager+ from freetoken.scheduler.utils import PendingReq++ pool = _pool()+ pt = torch.zeros(4, 512, dtype=torch.int32)+ cm = CacheManager(64, 64, pt, "hybrid_radix", linear_state_pool=pool)+ assert cm.prefill_chunk_align == 64+ tm = TableManager(max_running_reqs=4, page_table=pt)+ pending = PendingReq(0, torch.arange(300, dtype=torch.int32), SamplingParams(max_tokens=1))++ adder = PrefillAdder(token_budget=100, reserved_size=0, cache_manager=cm, table_manager=tm)+ req = adder.try_add_one(pending)+ assert isinstance(req, ChunkedReq) and req.extend_len == 64++ # a budget below one page keeps the unaligned chunk rather than stalling the request+ adder = PrefillAdder(token_budget=40, reserved_size=0, cache_manager=cm, table_manager=tm)+ assert adder.try_add_one(pending).extend_len == 40+++def test_naive_cache_does_not_align_prefill_chunks():+ """The alignment hook is hybrid-only; every other cache keeps the raw budget chunk."""+ from freetoken.scheduler.prefill import PrefillAdder+ from freetoken.scheduler.table import TableManager+ from freetoken.scheduler.utils import PendingReq++ pt = torch.zeros(4, 512, dtype=torch.int32)+ cm = CacheManager(64, 64, pt, "radix")+ assert cm.prefill_chunk_align == 1+ tm = TableManager(max_running_reqs=4, page_table=pt)+ adder = PrefillAdder(token_budget=100, reserved_size=0, cache_manager=cm, table_manager=tm)+ pending = PendingReq(0, torch.arange(300, dtype=torch.int32), SamplingParams(max_tokens=1))+ assert adder.try_add_one(pending).extend_len == 100++def test_pool_sizing_covers_4mr_floor():"""C6: pool must reserve the 4-slot-per-request non-evictable floor even at a tiny ratio."""from types import SimpleNamespace