Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(qwen4_exp): support Qwen3.8-Flash-Next (#257)

Xiaoze Fanbd8f3d519a48
diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.py
index 746c04c4..72b01c04 100644
--- a/python/freetoken/attention/__init__.py
+++ b/python/freetoken/attention/__init__.py
@@ -33,10 +33,6 @@ class BackendInfo:
# Whether forward() honors a per-call AttentionSpec (window/sm_scale/sinks).
# Non-consumers raise on a non-None spec instead of silently dropping it.
consumes_attn_spec: bool = False
- # Whether this backend coexists with hybrid-linear (GDN/mamba) models. The
- # linear layers bypass the backend entirely, but a backend whose metadata or
- # graph machinery assumes layer 0 is an attention layer can opt out here.
- hybrid_linear_ok: bool = True
SUPPORTED_ATTENTION_BACKENDS = Registry[BackendCreator]("Attention Backend")
@@ -132,6 +128,21 @@ def create_m3_sparse_backend(config: ModelConfig):
return M3SparseAttnBackend(config)
+@SUPPORTED_ATTENTION_BACKENDS.register(
+ "qsa_sparse",
+ BackendInfo(
+ supported_types=frozenset({AttnType.QSA}),
+ # 64-token pages: a 4-token compress group never straddles a page, so the
+ # compressed row of a group is page_base // 4 + block-in-page.
+ page_sizes=(64,),
+ ),
+)
+def create_qsa_sparse_backend(config: ModelConfig):
+ from .qsa_sparse import QSASparseAttnBackend
+
+ return QSASparseAttnBackend(config)
+
+
def attention_backend_info(name: str) -> BackendInfo:
return SUPPORTED_ATTENTION_BACKENDS.info(name)
diff --git a/python/freetoken/attention/base.py b/python/freetoken/attention/base.py
index eb39d472..ca36fb88 100644
--- a/python/freetoken/attention/base.py
+++ b/python/freetoken/attention/base.py
@@ -24,6 +24,10 @@ class AttnType(str, Enum):
# GQA block-sparse (MiniMax-M3): paged GQA K/V + a per-sparse-layer index-key
# slab; the indexer picks top-k 128-token blocks per query -> BSAKVCache
BSA = "bsa"
+ # QSA compressed-block sparse (Qwen3.8-Flash-Next): paged GQA K/V + a compressed
+ # index-key slab (one row per index_ratio tokens, row = slot // index_ratio) +
+ # a per-request pending ring for unclosed groups -> QSAKVCache
+ QSA = "qsa"
@property
def backend_driven(self) -> bool:
diff --git a/python/freetoken/attention/linear.py b/python/freetoken/attention/linear.py
index 1283ba02..f3717e58 100644
--- a/python/freetoken/attention/linear.py
+++ b/python/freetoken/attention/linear.py
@@ -41,6 +41,7 @@ class FLAMetadata:
track_dst: torch.Tensor | None = None # [nt] int64 dst pool slot per tracked req
track_h_row: torch.Tensor | None = None # [nt] int64 row into h (boh_i + aligned//CHUNK)
track_conv_src: torch.Tensor | None = None # [nt, kernel-1] int64 conv-input token positions
+ track_boundary_row: torch.Tensor | None = None # [nt] int64 forward-local row of the track boundary; states with their own left context (qwen4_exp PLE) derive their windows from it
def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:
@@ -55,7 +56,7 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:
builder serves the eager scheduler path and direct-op test callers.
"""
reqs = batch.padded_reqs
- pin = {"device": "cpu", "pin_memory": True}
+ pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()}
# GDN state slot per request: the hybrid-radix live slot (decoupled from table_idx) when
# allocated, else table_idx (naive / force-naive GDN models keep the old keying).
@@ -77,7 +78,7 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:
fresh = [gdn_slot(r) for r in reqs if r.cached_len == 0]
fresh_host = torch.tensor(fresh, dtype=torch.int64, **pin) if fresh else None
- track_dst, track_h_row, track_conv_src = _build_track_metadata(reqs, cu_host, device, pin)
+ track = _build_track_metadata(reqs, cu_host, device, pin)
return FLAMetadata(
cu_seqlens=cu_host.to(device, non_blocking=True),
@@ -86,24 +87,29 @@ def build_fla_metadata(batch: "Batch", device: torch.device) -> FLAMetadata:
fresh_state_indices=(
fresh_host.to(device, non_blocking=True) if fresh_host is not None else None
),
- track_dst=track_dst, track_h_row=track_h_row, track_conv_src=track_conv_src,
+ **track,
)
def _build_track_metadata(reqs, cu_host, device, pin):
"""Hybrid-radix (extra_buffer): for each request that crosses a ×CHUNK boundary this
prefill forward, snapshot its GDN state at the deepest mid-chunk boundary into its current
- ping-pong slot. Returns (track_dst, track_h_row, track_conv_src) device int64 tensors, or
- (None, None, None) when no request tracks (non-hybrid, or all extends < CHUNK+1)."""
+ ping-pong slot. Returns the ``FLAMetadata`` track kwargs, all None when no request
+ tracks (non-hybrid, or all extends < CHUNK+1)."""
+ empty = dict(track_dst=None, track_h_row=None, track_conv_src=None, track_boundary_row=None)
if not any(r.mamba_ping_pong is not None for r in reqs):
- return None, None, None
+ return empty
from freetoken.core import get_global_ctx
from freetoken.kernel.fla.chunk import CHUNK_SIZE
from freetoken.kernel.fla.index import prepare_chunk_offsets
km1 = get_global_ctx().linear_state_pool.conv_states.shape[-1] # conv_kernel_dim - 1
+ assert km1 <= CHUNK_SIZE, (
+ f"conv history {km1} exceeds CHUNK_SIZE {CHUNK_SIZE}: the snapshot window "
+ "would reach before this forward's first token"
+ )
boh = prepare_chunk_offsets(cu_host, CHUNK_SIZE).tolist()
- dst, h_row, conv_src = [], [], []
+ dst, h_row, conv_src, boundary_rows = [], [], [], []
for i, r in enumerate(reqs):
if r.mamba_ping_pong is None:
continue
@@ -117,13 +123,18 @@ def _build_track_metadata(reqs, cu_host, device, pin):
dst.append(r.mamba_ping_pong[r.mamba_next_track_idx])
h_row.append(boh[i] + c)
conv_src.append([off + c * CHUNK_SIZE - km1 + j for j in range(km1)])
+ boundary_rows.append(off + c * CHUNK_SIZE)
r.mamba_last_track_seqlen = boundary
r.mamba_next_track_idx = 1 - r.mamba_next_track_idx
if not dst:
- return None, None, None
+ return empty
to = lambda xs, **kw: torch.tensor(xs, **pin, **kw).to(device, non_blocking=True)
- return (to(dst, dtype=torch.int64), to(h_row, dtype=torch.int64),
- to(conv_src, dtype=torch.int64))
+ return dict(
+ track_dst=to(dst, dtype=torch.int64),
+ track_h_row=to(h_row, dtype=torch.int64),
+ track_conv_src=to(conv_src, dtype=torch.int64),
+ track_boundary_row=to(boundary_rows, dtype=torch.int64),
+ )
__all__ = ["FLAMetadata", "build_fla_metadata"]
diff --git a/python/freetoken/attention/qsa_sparse.py b/python/freetoken/attention/qsa_sparse.py
new file mode 100644
index 00000000..4a28dc85
--- /dev/null
+++ b/python/freetoken/attention/qsa_sparse.py
@@ -0,0 +1,507 @@
+"""Qwen3.8-Flash-Next QSA compressed-block sparse attention backend.
+
+Serves ``AttnType.QSA`` over ``kvcache/qsa_pool.py``: paged GQA K/V for the 12 full-attention
+layers, a compressed index-key slab holding one key per ``index_ratio`` tokens, and a
+per-request pending ring for the group a forward leaves open. The 36 GDN layers never reach
+this backend, and the model has no dense attention layer, so :meth:`forward` is not served --
+the only entry point is :meth:`qsa_forward` (``models/qwen4_exp/attention.py``).
+
+One QSA layer's forward, all ragged over ``[T, ...]`` metadata:
+
+1. store K/V at ``batch.out_loc``;
+2. pool each row's closing group (members at positions >= ``cached_len`` come from this
+ forward's raw index keys, the older ones from the pending ring), zero-centered rmsnorm it
+ and rope it at the group's first position, then scatter it into the slab row
+ ``out_loc // index_ratio`` (rows whose group does not close land on the request's scratch
+ row and are never read);
+3. store this forward's last ``ring_capacity`` raw index keys per request into the ring;
+4. norm+rope the indexer queries at their own positions;
+5. score every COMPLETE visible block (``sum_h relu(<q_h, k_bar_b>) / sqrt(index_head_dim)``,
+ clamped to ``kvlen // index_ratio`` -- slab rows are never cleared, so stale rows must stay
+ unreachable), take the top ``index_budget // index_ratio`` blocks, expand them to token
+ indices plus the causal tail of the open group;
+6. attend to exactly those tokens.
+
+Addressing: the engine pins ``page_size == 64`` (this backend's ``page_sizes``), so a group of
+``index_ratio`` tokens never straddles a page and ``block_table[req, p] = page_table[req, p *
+64] // 64`` names both the K/V page and, viewed as ``page_size // index_ratio`` compressed
+rows, the block's slab page. Decode stages that table plus the live lengths and table_idx into
+static buffers (``prepare_for_replay``) so the whole path is CUDA-graph capturable.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Callable, List
+
+import torch
+from freetoken.core import Batch, get_global_ctx
+from freetoken.utils import init_logger
+
+from .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadata
+
+logger = init_logger(__name__)
+
+if TYPE_CHECKING:
+ from freetoken.models import ModelConfig
+
+_CPU_PINNED = {"device": "cpu", "dtype": torch.int32, "pin_memory": True}
+# Block-score transient budget (vLLM's number): the fp32 [rows, n_blocks] logits tile is
+# 256 KB per row at a 1M-token context, so a long prefill must be scored in row chunks.
+_LOGITS_WORKSPACE_BYTES = 128 << 20
+
+
+TORCH_TOPK_ENV = "FREETOKEN_QSA_TORCH_TOPK"
+
+
+def _resolve_block_topk() -> Callable | None:
+ """The in-repo Triton block top-k, or None to fall back on torch.topk."""
+ if os.getenv(TORCH_TOPK_ENV, "0") == "1":
+ logger.info(f"qsa_sparse block top-k: torch.topk ({TORCH_TOPK_ENV}=1)")
+ return None
+ try:
+ from freetoken.kernel.triton.qsa import qsa_block_topk
+ except Exception as exc:
+ logger.info(f"qsa_sparse block top-k: torch.topk (triton unavailable: {exc})")
+ return None
+ logger.info("qsa_sparse block top-k: triton qsa_block_topk")
+ return qsa_block_topk
+
+
+@dataclass
+class QSASparseMetadata(BaseAttnMetadata):
+ # fmt: off
+ is_decode: bool
+ last_indices: torch.Tensor # gpu
+ qo_indptr_cpu: torch.Tensor # cpu pinned int32 [bs+1]
+ kv_len_cpu: torch.Tensor # cpu pinned int32 [bs]
+ # Ragged per-token / per-request addressing. Decode defers these to the static graph
+ # buffers (prepare_for_replay) or to a lazy eager snapshot at the first QSA layer.
+ token_to_req: torch.Tensor | None = None # [T] int32
+ cu_seqlens: torch.Tensor | None = None # [bs+1] int32
+ seq_lens: torch.Tensor | None = None # [bs] int32, device_len
+ ring_slots: torch.Tensor | None = None # [bs] int32, Req.table_idx
+ block_table: torch.Tensor | None = None # [bs, W//page_size] int32, physical page ids
+ # Per-forward scatter plans, built once by the first QSA layer and reused by the rest.
+ # positions is bound here (not in prepare_metadata) because a capture batch has none yet.
+ cmp_rows: torch.Tensor | None = None # [T] int32, compressed slab destination
+ ring_rows: torch.Tensor | None = None # [T] int32, flat ring row or -1
+ positions: torch.Tensor | None = None # [T] int32, logical query positions
+ # fmt: on
+
+ def get_last_indices(self, bs: int) -> torch.Tensor:
+ return self.last_indices[:bs]
+
+
+class QSASparseAttnBackend(BaseAttnBackend):
+ def __init__(self, config: ModelConfig) -> None:
+ from freetoken.kvcache.qsa_pool import QSAKVCache
+
+ args = config.qwen4_args
+ assert args is not None, "qsa_sparse backend needs ModelConfig.qwen4_args"
+ self.head_dim = config.head_dim
+ self.index_heads = args.index_n_heads
+ self.token_topk = args.index_budget
+ self.kvcache = get_global_ctx().kv_cache
+ assert isinstance(self.kvcache, QSAKVCache), (
+ f"qsa_sparse backend needs a QSA pool, got {type(self.kvcache).__name__}"
+ )
+ self.device = self.kvcache.device
+ self.dtype = self.kvcache.dtype
+ self.index_head_dim = self.kvcache.index_head_dim
+ self.ratio = self.kvcache.index_ratio
+ self.ring_capacity = self.kvcache.ring_capacity
+ self.page_size = get_global_ctx().page_size
+ assert self.page_size % self.ratio == 0, (
+ f"QSA needs page_size ({self.page_size}) divisible by index_ratio ({self.ratio})"
+ )
+ self.cmp_page_size = self.page_size // self.ratio
+ self.block_topk = self.token_topk // self.ratio
+ self.select_width = self.token_topk + self.ratio - 1
+ assert self.token_topk % self.ratio == 0, "QSA budget must be a whole number of blocks"
+ # The sparse attend kernel bakes 1/sqrt(head_dim) into its exp2 scale.
+ assert config.attn_sm_scale in (None, self.head_dim**-0.5), (
+ "qsa_sparse serves the default 1/sqrt(head_dim) attention scale only"
+ )
+ # QSA layer -> index slab slot, in sparse-layer order (the pool's own convention).
+ group = self._qsa_group(config)
+ self._idx_slot = {lid: i for i, lid in enumerate(group.layer_ids)}
+ self.rotary_config = group.rotary_config
+ self._index_cos_sin: torch.Tensor | None = None
+
+ self._block_topk_kernel = _resolve_block_topk()
+ # decode staging (static buffers under CUDA graphs; eager decode snapshots per step)
+ self._graph: dict[str, torch.Tensor] = {}
+ self.capture_bs: List[int] = []
+
+ @staticmethod
+ def _qsa_group(config: ModelConfig):
+ from freetoken.models.config import FullAttentionGroupConfig
+
+ groups = [
+ g
+ for g in config.attention_groups
+ if isinstance(g, FullAttentionGroupConfig) and g.index_ratio > 1
+ ]
+ assert len(groups) == 1, f"expected one QSA attention group, got {len(groups)}"
+ return groups[0]
+
+ # ----- slab views ---------------------------------------------------------------------
+ def _cmp_pages(self, slot: int) -> torch.Tensor:
+ """The compressed slab as ``[pages, page_size // ratio, 1, dim]``, the score kernel's
+ paged layout. The scratch rows past ``cmp_scratch_base`` stay out of the view."""
+ rows = self.kvcache.cmp_k_cache(slot)[: self.kvcache.cmp_scratch_base]
+ return rows.view(-1, self.cmp_page_size, 1, self.index_head_dim)
+
+ def _index_rope_cache(self) -> torch.Tensor:
+ """cos/sin table of the indexer rope: same rotary_dim and frequencies as the main
+ attention, ``head_size`` 128 instead of 256, so it is a separate get_rope instance.
+
+ The table itself (not RotaryEmbedding.forward) because the indexer's norm+rope is one
+ fused kernel and the compressed keys rope at their group's position, not the query's."""
+ if self._index_cos_sin is None:
+ from freetoken.layers.rotary import get_rope
+
+ rotary = self.rotary_config
+ with torch.device(self.device):
+ rope = get_rope(
+ head_dim=self.index_head_dim,
+ rotary_dim=rotary.rotary_dim,
+ max_position=rotary.max_position,
+ base=rotary.base,
+ rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,
+ )
+ self._index_cos_sin = rope._cos_sin_cache.to(self.device)
+ return self._index_cos_sin
+
+ # ----- metadata -----------------------------------------------------------------------
+ def prepare_metadata(self, batch: Batch) -> None:
+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
+ seqlens_q = [r.extend_len for r in reqs]
+ seqlens_k = [r.device_len for r in reqs]
+ is_decode = getattr(batch, "phase", None) == "decode"
+ qo_indptr = torch.tensor([0] + seqlens_q, **_CPU_PINNED).cumsum_(0).to(torch.int32)
+ kv_len = torch.tensor(seqlens_k, **_CPU_PINNED)
+ last = (qo_indptr[1:].to(torch.int32) - 1).to(self.device, non_blocking=True)
+ md = QSASparseMetadata(
+ is_decode=is_decode,
+ last_indices=last,
+ qo_indptr_cpu=qo_indptr,
+ kv_len_cpu=kv_len,
+ )
+ batch.attn_metadata = md
+ if not is_decode:
+ table_idx = torch.tensor([r.table_idx for r in reqs], **_CPU_PINNED)
+ token_to_req = torch.repeat_interleave(
+ torch.arange(len(reqs), dtype=torch.int32),
+ torch.tensor(seqlens_q, dtype=torch.int32),
+ ).pin_memory()
+ md.cu_seqlens = qo_indptr.to(self.device, non_blocking=True)
+ md.token_to_req = token_to_req.to(self.device, non_blocking=True)
+ md.seq_lens = kv_len.to(self.device, non_blocking=True)
+ md.ring_slots = table_idx.to(self.device, non_blocking=True)
+ md.block_table = self._block_table(md.ring_slots.to(torch.int64))
+ # Decode addressing is DEFERRED: a graph-bound step stages it into the static
+ # buffers (prepare_for_replay), an eager step snapshots at the first QSA layer.
+
+ def _block_base_view(self) -> torch.Tensor:
+ """Every-``page_size``-th column of the page table: the per-page base slots. A strided
+ VIEW, so gathering rows through it materializes only [bs, W/page_size]."""
+ return get_global_ctx().page_table[:, :: self.page_size]
+
+ def _block_table(self, table_idx: torch.Tensor) -> torch.Tensor:
+ return (self._block_base_view().index_select(0, table_idx) // self.page_size).to(
+ torch.int32
+ )
+
+ def _stage_decode(self, md: QSASparseMetadata, bs: int, table_idx: torch.Tensor) -> None:
+ """Copy this step's addressing into the static graph buffers and point the metadata
+ at them (restage-per-replay, m3/dsa precedent)."""
+ self._graph["block_table"][:bs].copy_(
+ self._block_base_view().index_select(0, table_idx) // self.page_size
+ )
+ self._graph["kvlen"][:bs].copy_(md.kv_len_cpu.to(self.device, non_blocking=True))
+ self._graph["table_idx"][:bs].copy_(table_idx)
+ md.block_table = self._graph["block_table"][:bs]
+ md.seq_lens = self._graph["kvlen"][:bs]
+ md.ring_slots = self._graph["table_idx"][:bs]
+ md.token_to_req = self._graph["token_to_req"][:bs]
+ md.cu_seqlens = self._graph["cu_seqlens"][: bs + 1]
+
+ def _snapshot_decode(self, md: QSASparseMetadata, batch: Batch) -> None:
+ """Eager decode (not graph-staged): this step's rows, once per forward. The live
+ page-table row may mutate for the next batch while this one runs, so gather now."""
+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
+ bs = len(reqs)
+ table_idx = torch.tensor([r.table_idx for r in reqs], **_CPU_PINNED)
+ md.ring_slots = table_idx.to(self.device, non_blocking=True)
+ md.block_table = self._block_table(md.ring_slots.to(torch.int64))
+ md.seq_lens = md.kv_len_cpu.to(self.device, non_blocking=True)
+ md.token_to_req = torch.arange(bs, dtype=torch.int32, device=self.device)
+ md.cu_seqlens = torch.arange(bs + 1, dtype=torch.int32, device=self.device)
+
+ # ----- dense layers -------------------------------------------------------------------
+ def forward(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer_id: int,
+ batch: Batch,
+ attn_spec: AttentionSpec | None = None,
+ ) -> torch.Tensor:
+ raise NotImplementedError(
+ "qsa_sparse serves QSA layers only (Qwen3.8-Flash-Next has no dense attention "
+ "layer); the QSA layer calls qsa_forward"
+ )
+
+ # ----- QSA layers ---------------------------------------------------------------------
+ def qsa_forward(
+ self,
+ q: torch.Tensor, # [T, HQ, D]
+ k: torch.Tensor, # [T, KVH * D]
+ v: torch.Tensor, # [T, KVH * D]
+ index, # models.qwen4_exp.attention.QSAIndexerInputs
+ layer_id: int,
+ batch: Batch,
+ ) -> torch.Tensor:
+ from freetoken.kernel.triton.qsa import qsa_sparse_paged_attention
+
+ md = batch.attn_metadata
+ assert isinstance(md, QSASparseMetadata)
+ slot = self._idx_slot[layer_id]
+ self.kvcache.store_kv(k, v, batch.out_loc, layer_id)
+ if md.block_table is None:
+ self._snapshot_decode(md, batch)
+ if slot == 0 or md.cmp_rows is None:
+ # Rebuilt at the first QSA layer of every forward, not cached on the metadata: a
+ # capture batch runs its warmup and its capture through ONE metadata object, and a
+ # cached plan would bake the warmup's addresses into the graph.
+ self._plan_index_writes(md, batch)
+
+ self._update_index_cache(index, md, slot)
+ indices = self._select(index, md, slot)
+ return qsa_sparse_paged_attention(
+ q,
+ self.kvcache.k_cache(layer_id),
+ self.kvcache.v_cache(layer_id),
+ indices,
+ md.block_table,
+ md.token_to_req,
+ torch.empty_like(q),
+ )
+
+ def _plan_index_writes(self, md: QSASparseMetadata, batch: Batch) -> None:
+ """Per-token slab row and ring row for this forward; the other QSA layers reuse it
+ (it is layer-invariant). Pure device arithmetic: no host sync, graph-capturable."""
+ md.positions = batch.positions
+ out_loc = batch.out_loc.to(torch.int64)
+ positions = batch.positions.to(torch.int64)
+ rows = torch.arange(out_loc.numel(), device=self.device)
+ req = md.token_to_req.to(torch.int64)
+ slots = md.ring_slots.to(torch.int64).index_select(0, req)
+ # out_loc % page_size == position % page_size and index_ratio divides page_size, so a
+ # group closes exactly on out_loc % index_ratio == index_ratio - 1.
+ closing = out_loc % self.ratio == self.ratio - 1
+ scratch = self.kvcache.cmp_scratch_base + slots
+ md.cmp_rows = torch.where(closing, out_loc // self.ratio, scratch).to(torch.int32)
+ # Only the last ring_capacity rows of a request survive to the next forward; the rest
+ # are masked off instead of dumped somewhere (vLLM rule).
+ ends = md.cu_seqlens.to(torch.int64).index_select(0, req + 1)
+ keep = rows >= ends - self.ring_capacity
+ ring_row = slots * self.ring_capacity + positions % self.ring_capacity
+ md.ring_rows = torch.where(keep, ring_row, torch.full_like(ring_row, -1)).to(
+ torch.int32
+ )
+
+ def _update_index_cache(self, index, md: QSASparseMetadata, slot: int) -> None:
+ """Compress each closing group into the slab, then refresh the pending ring."""
+ from freetoken.kernel.triton.qsa import (
+ qsa_compress_groups,
+ qsa_index_norm_rope,
+ qsa_store_rows,
+ )
+
+ rows = index.k.shape[0]
+ ring = self.kvcache.pending_ring(slot)
+ pooled = self._scratch("pooled", rows, self.index_head_dim, dtype=self.dtype)
+ first = self._scratch("first_pos", rows, dtype=torch.int32)
+ qsa_compress_groups(
+ index.k,
+ ring,
+ md.ring_slots,
+ md.token_to_req,
+ md.cu_seqlens,
+ md.positions,
+ self.ratio,
+ pooled,
+ first,
+ )
+ qsa_index_norm_rope(
+ pooled,
+ first,
+ self._index_rope_cache(),
+ index.k_norm_weight,
+ index.eps,
+ self.kvcache.cmp_k_cache(slot),
+ dest_rows=md.cmp_rows,
+ )
+ # After the compression read: the ring rows this forward overwrites are exactly the
+ # ones a straddling group just consumed.
+ qsa_store_rows(ring, md.ring_rows, index.k)
+
+ def _select(self, index, md: QSASparseMetadata, slot: int) -> torch.Tensor:
+ """Score complete visible blocks, take the top-k, expand them to token indices."""
+ from freetoken.kernel.triton.qsa import (
+ expand_qsa_block_indices,
+ qsa_index_norm_rope,
+ qsa_mqa_paged,
+ )
+
+ rows = index.q.shape[0]
+ positions = md.positions
+ q_index = self._scratch(
+ "q_index", rows, self.index_heads, self.index_head_dim, dtype=self.dtype
+ )
+ qsa_index_norm_rope(
+ index.q.view(-1, self.index_head_dim),
+ positions,
+ self._index_rope_cache(),
+ index.q_norm_weight,
+ index.eps,
+ q_index.view(-1, self.index_head_dim),
+ heads=self.index_heads,
+ )
+ cmp_pages = self._cmp_pages(slot)
+ columns = md.block_table.shape[1] * self.cmp_page_size
+ indices = self._scratch("indices", rows, self.select_width, dtype=torch.int32)
+ rows_per_chunk = max(1, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1))
+ for start in range(0, rows, rows_per_chunk):
+ end = min(start + rows_per_chunk, rows)
+ chunk = slice(start, end)
+ logits = self._scratch("logits", end - start, columns, dtype=torch.float32)
+ visible = self._scratch("visible", end - start, dtype=torch.int32)
+ qsa_mqa_paged(
+ q_index[chunk],
+ cmp_pages,
+ md.block_table,
+ md.token_to_req[chunk],
+ positions[chunk],
+ md.seq_lens,
+ self.ratio,
+ logits,
+ visible,
+ )
+ blocks = self._scratch("blocks", end - start, self.block_topk, dtype=torch.int32)
+ self._top_blocks(logits, visible, blocks)
+ expand_qsa_block_indices(
+ blocks,
+ positions[chunk],
+ md.seq_lens,
+ md.token_to_req[chunk],
+ self.ratio,
+ self.token_topk,
+ indices[chunk],
+ )
+ return indices
+
+ def _top_blocks(
+ self,
+ logits: torch.Tensor,
+ visible: torch.Tensor,
+ blocks: torch.Tensor,
+ ) -> None:
+ """Top ``block_topk`` complete blocks per row, row-relative, -1 padded."""
+ assert blocks.shape == (logits.shape[0], self.block_topk), (
+ f"qsa block top-k output must be [rows, {self.block_topk}], got {tuple(blocks.shape)}"
+ )
+ if self._block_topk_kernel is not None:
+ scratch_width = self._topk_scratch_width(logits.shape[1])
+ scratch = (
+ self._scratch("topk_scratch", logits.shape[0], scratch_width, dtype=torch.int32)
+ if scratch_width
+ else None
+ )
+ self._block_topk_kernel(logits, visible, blocks, scratch)
+ return
+ # The score kernel only writes columns below visible_blocks; mask the rest so a
+ # stale row cannot win a slot. Real block scores are relu sums, never -inf.
+ columns = logits.shape[1]
+ column = torch.arange(columns, dtype=torch.int32, device=logits.device)
+ logits.masked_fill_(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf"))
+ width = min(self.block_topk, columns)
+ values, chosen = torch.topk(logits, width, dim=-1)
+ blocks[:, :width] = torch.where(values > -float("inf"), chosen.to(torch.int32), -1)
+ if width < self.block_topk:
+ blocks[:, width:] = -1
+
+ def _topk_scratch_width(self, columns: int) -> int:
+ """int32 columns per row the block top-k wants as scratch, 0 when it wants none."""
+ if self._block_topk_kernel is None:
+ return 0
+ from freetoken.kernel.triton.qsa import qsa_block_topk_scratch_width
+
+ return qsa_block_topk_scratch_width(columns, self.block_topk)
+
+ # ----- scratch ------------------------------------------------------------------------
+ def _scratch(self, name: str, rows: int, *shape: int, dtype: torch.dtype) -> torch.Tensor:
+ """A per-forward transient: the static decode buffer when it is wide enough (so a
+ captured graph keeps one address), otherwise a fresh allocation."""
+ buffer = self._graph.get(name)
+ if buffer is not None and rows <= buffer.shape[0] and buffer.shape[1:] == shape:
+ return buffer[:rows]
+ return torch.empty((rows, *shape), dtype=dtype, device=self.device)
+
+ # ----- CUDA graph (decode) --------------------------------------------------------------
+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:
+ self.capture_bs = sorted(bs_list)
+ max_bs = max(bs_list)
+ width = get_global_ctx().page_table.shape[1]
+ pages = -(-width // self.page_size)
+ columns = pages * self.cmp_page_size
+ chunk = max(1, min(max_bs, _LOGITS_WORKSPACE_BYTES // max(columns * 4, 1)))
+ topk_scratch = self._topk_scratch_width(columns)
+
+ def empty(*shape: int, dtype: torch.dtype) -> torch.Tensor:
+ return torch.empty(shape, dtype=dtype, device=self.device)
+
+ self._graph = {
+ "block_table": torch.zeros((max_bs, pages), dtype=torch.int32, device=self.device),
+ "kvlen": torch.zeros(max_bs, dtype=torch.int32, device=self.device),
+ "table_idx": torch.zeros(max_bs, dtype=torch.int32, device=self.device),
+ "token_to_req": torch.arange(max_bs, dtype=torch.int32, device=self.device),
+ "cu_seqlens": torch.arange(max_bs + 1, dtype=torch.int32, device=self.device),
+ "logits": empty(chunk, columns, dtype=torch.float32),
+ "visible": empty(max_bs, dtype=torch.int32),
+ "blocks": empty(max_bs, self.block_topk, dtype=torch.int32),
+ "indices": empty(max_bs, self.select_width, dtype=torch.int32),
+ "pooled": empty(max_bs, self.index_head_dim, dtype=self.dtype),
+ "first_pos": empty(max_bs, dtype=torch.int32),
+ "q_index": empty(max_bs, self.index_heads, self.index_head_dim, dtype=self.dtype),
+ }
+ if topk_scratch:
+ self._graph["topk_scratch"] = empty(chunk, topk_scratch, dtype=torch.int32)
+
+ def prepare_for_capture(self, batch: Batch) -> None:
+ self.prepare_metadata(batch)
+ md = batch.attn_metadata
+ assert isinstance(md, QSASparseMetadata)
+ bs = batch.size
+ dummy = torch.full(
+ (bs,), batch.padded_reqs[0].table_idx, dtype=torch.int64, device=self.device
+ )
+ self._stage_decode(md, bs, dummy)
+
+ def prepare_for_replay(self, batch: Batch) -> None:
+ md = batch.attn_metadata
+ assert isinstance(md, QSASparseMetadata)
+ assert batch.active_table_idx is not None, "decode batch is missing its page-table rows"
+ self._stage_decode(md, batch.padded_size, batch.active_table_idx.to(torch.int64))
+
+ def reset_capture(self) -> None:
+ super().reset_capture()
+ self._graph = {}
+
+
+__all__ = ["QSASparseAttnBackend", "QSASparseMetadata"]
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index cd6505d2..73dc7688 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -114,9 +114,7 @@ def _backend_requirements_met(name: str) -> bool:
return True
-def _resolve_auto_attention_backend(
- required: frozenset[AttnType], hybrid_linear: bool
-) -> str:
+def _resolve_auto_attention_backend(required: frozenset[AttnType]) -> str:
"""First candidate (in per-type priority order) whose arch condition holds,
whose packages are installed, and whose every comma part serves ALL required
types. Reproduces the historical hardware tree for FULL-only models:
@@ -128,6 +126,8 @@ def _resolve_auto_attention_backend(
candidates.append(("dsa", True))
if AttnType.BSA in required:
candidates.append(("m3_sparse", True))
+ if AttnType.QSA in required:
+ candidates.append(("qsa_sparse", True))
if AttnType.SWA in required:
candidates.append(("triton", True))
if AttnType.FULL in required:
@@ -142,10 +142,6 @@ def _resolve_auto_attention_backend(
continue
if not _backend_parts_serve(name, required):
continue
- if hybrid_linear and not all(
- attention_backend_info(p).hybrid_linear_ok for p in name.split(",")
- ):
- continue
if not _backend_requirements_met(name):
continue
return name
@@ -176,7 +172,10 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att
if missing:
valid = [
name
- for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse")
+ for name in (
+ "fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse",
+ "qsa_sparse",
+ )
if required <= attention_backend_info(name).supported_types
]
missing_names = "/".join(sorted(t.value for t in missing))
@@ -185,11 +184,6 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att
f"attention, which backend {part!r} does not support; valid backends: "
f"{', '.join(valid)} (or auto), got {config.attention_backend!r}."
)
- if getattr(model_config, "has_linear_attention", False) and not info.hybrid_linear_ok:
- raise ValueError(
- f"backend {part!r} does not support hybrid-linear (GDN/mamba) models, "
- f"got {config.attention_backend!r}."
- )
if AttnType.SWA in required and not info.consumes_attn_spec:
# SWA models drive window/sinks/sm_scale through the per-call AttentionSpec;
# a backend that drops it would attend with the wrong window silently.
@@ -333,6 +327,12 @@ class Engine:
self._post_weights_free = post_weights_free
self.moe_offload_cache = None
self.cpu_moe_executor = None
+ # Host-side auxiliary stores (qwen4_exp's pinned PLE table): after the weights so a
+ # load failure is not masked, before the MoE offload cache so the bank residency
+ # planning sees the pin quota the table already spent.
+ self._host_tables_bytes = 0
+ if hasattr(self.model, "load_host_tables"):
+ self._host_tables_bytes = int(self.model.load_host_tables(config) or 0)
if is_offload_moe_backend(config.moe_backend):
self._init_offload_moe_cache(config)
if hasattr(self.model, "prepare_for_runtime"):
@@ -361,6 +361,7 @@ class Engine:
dtype=self.dtype,
device=self.device,
tp_size=config.tp_info.size,
+ slot_states=config.model_config.slot_states,
)
self.ctx.linear_state_pool = self.linear_state_pool
else:
@@ -518,9 +519,11 @@ class Engine:
not cpu_layer_ids
and config.moe_cpu_layers is None
and config.moe_backend in ("offload", "hybrid")
- and _pin_budget_bytes() is not None
+ and _pin_budget_bytes(self._host_tables_bytes) is not None
):
- cpu_layer_ids = _auto_cpu_layers(config, config.model_config.num_moe_layers)
+ cpu_layer_ids = _auto_cpu_layers(
+ config, config.model_config.num_moe_layers, reserved=self._host_tables_bytes
+ )
if config.moe_backend == "hybrid":
decode_target = "hybrid"
elif cpu_layer_ids:
@@ -533,13 +536,13 @@ class Engine:
split_residency = (
bool(cpu_layer_ids)
and config.moe_backend in ("offload", "hybrid")
- and _pin_budget_bytes() is not None
+ and _pin_budget_bytes(self._host_tables_bytes) is not None
)
if config.moe_backend == "cpu" and not split_residency:
# cpu mode pins every bank for the prefill double buffer; over the pin cap that dies in cudaHostRegister, so lock everything instead
from freetoken.moe.expert_banks import bank_bytes_estimate, ftw_bank_bytes
- budget = _pin_budget_bytes()
+ budget = _pin_budget_bytes(self._host_tables_bytes)
bank_bytes = None
if budget is not None:
bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)
@@ -1158,18 +1161,20 @@ def _cpu_moe_executor_viable(model_config) -> bool:
return fmt == "mxfp4" or fmt in _WFMT_IDS
-def _pin_budget_bytes() -> int | None:
- """Bytes this process can safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).
+def _pin_budget_bytes(reserved: int = 0) -> int | None:
+ """Bytes this process can still safely cudaHostRegister, or None when the platform does not cap pinning (plain Linux).
- WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere."""
+ WSL's WDDM-backed CUDA caps pinning near half of RAM, shared across processes -- budget 40%. FREETOKEN_PIN_BUDGET_GB overrides anywhere. ``reserved`` subtracts host bytes already pinned outside the expert banks (qwen4_exp's PLE table)."""
if env := os.environ.get("FREETOKEN_PIN_BUDGET_GB"):
- return int(float(env) * 2**30)
- if not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag
+ cap = int(float(env) * 2**30)
+ elif not hasattr(os, "uname") or "microsoft" not in os.uname().release.lower(): # WSL kernel tag
return None
- return int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)
+ else:
+ cap = int(os.sysconf("SC_PHYS_PAGES") * os.sysconf("SC_PAGE_SIZE") * 0.4)
+ return max(0, cap - reserved)
-def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int]:
+def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int, reserved: int = 0) -> frozenset[int]:
"""Pick CPU (locked) MoE layers automatically when the banks exceed the pin budget.
Locks just enough head+tail layers: per-layer decode miss rates are U-shaped, so the ends are the cheapest to move off the slot cache."""
@@ -1178,7 +1183,7 @@ def _auto_cpu_layers(config: EngineConfig, num_moe_layers: int) -> frozenset[int
bank_bytes = ftw_bank_bytes(config.model_path) or bank_bytes_estimate(config.model_config)
if not bank_bytes:
return frozenset()
- budget = _pin_budget_bytes()
+ budget = _pin_budget_bytes(reserved)
if budget is None or bank_bytes <= budget:
return frozenset()
if not _cpu_moe_executor_viable(config.model_config):
@@ -1292,8 +1297,12 @@ def _adjust_config(config: EngineConfig):
# comma part must serve every required type, with packages/arch available.
required_attn_types = _required_attn_types(model_config)
_dtype = getattr(config, "dtype", None) # duck-typed test configs omit it
- if AttnType.BSA in required_attn_types and _dtype is not None and _dtype.itemsize != 2:
- # Reject at config time: the BSA pool's own assert only fires after the
+ if (
+ required_attn_types & {AttnType.BSA, AttnType.QSA}
+ and _dtype is not None
+ and _dtype.itemsize != 2
+ ):
+ # Reject at config time: the BSA/QSA pool's own assert only fires after the
# model is resident (and not at all under `python -O`).
raise ValueError(
f"--dtype {config.dtype}: block-sparse attention serves 16-bit "
@@ -1314,7 +1323,7 @@ def _adjust_config(config: EngineConfig):
if config.attention_backend == "auto":
override(
"attention_backend",
- _resolve_auto_attention_backend(required_attn_types, has_linear_attention),
+ _resolve_auto_attention_backend(required_attn_types),
)
logger.info_rank0(f"Auto-selected attention backend: {config.attention_backend}")
_validate_attention_backend_choice(config, override, required_attn_types)
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index a00154d0..c9c2fb98 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -75,13 +75,16 @@ def expert_bank_row_bytes(fmt: str, hidden_size: int, moe_intermediate_size: int
# models/loader.py stream_moe_expert_sources: gate_up [E, 2I, H], down [E, H, I], bf16
return {"gate_up": 2 * I * H * 2, "down": H * I * 2}
if fmt == "fp8_block":
- # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block scales
+ # qwen3_5_moe/weight.py _build_fp8_expert_banks: fp8 weights + bf16 128x128 block
+ # scales, trailing scale dim 16B-padded (same helper as the loader)
+ from freetoken.moe.offload_cache import fp8_block_scale_pad
+
B = 128
return {
"gate_up": 2 * I * H,
- "gate_up_scale": (2 * I // B) * (H // B) * 2,
+ "gate_up_scale": (2 * I // B) * fp8_block_scale_pad(2 * I // B, H // B) * 2,
"down": H * I,
- "down_scale": (H // B) * (I // B) * 2,
+ "down_scale": (H // B) * fp8_block_scale_pad(H // B, I // B) * 2,
}
if fmt == "q4_0":
# gemma4/gguf.py _q4_0_expert_specs: GGML Q4_0 rows, 32 elems -> 18 bytes
@@ -183,6 +186,19 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
moe_intermediate_size=512,
expert_formats=_NVFP4_FORMATS,
),
+ AotModel(
+ # QSA compressed-sparse attention (12 of 48 layers): the QSAKVCache stores K/V
+ # through store_cache (2 kv heads x 256 head_dim), the compressed index-key slab
+ # and the pending ring write via the vendored qsa triton kernels. Hyper-connections
+ # carry the residual, so the embedding row indexing() sees is still hidden_size.
+ name="RadixArk/Qwen3.8-Flash-Next-NVFP4",
+ architecture="Qwen4ExpForConditionalGeneration",
+ hidden_size=2560,
+ kv_groups=((2, 256),),
+ top_k=10,
+ moe_intermediate_size=640,
+ expert_formats=(*_NVFP4_FORMATS, "fp8_block"),
+ ),
AotModel(
name="google/gemma-4-26B-A4B-it",
architecture="Gemma4ForConditionalGeneration",
diff --git a/python/freetoken/kernel/triton/hc.py b/python/freetoken/kernel/triton/hc.py
new file mode 100644
index 00000000..45d71bd7
--- /dev/null
+++ b/python/freetoken/kernel/triton/hc.py
@@ -0,0 +1,395 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/hc.py)
+"""NVIDIA HyperConnection kernels for Qwen4Exp."""
+
+from __future__ import annotations
+
+import functools
+
+import torch
+import triton
+import triton.language as tl
+
+from freetoken.utils.arch import is_sm90_supported
+
+
+def _pdl_supported() -> bool:
+ return is_sm90_supported()
+
+
+def _grouped_gemma_rmsnorm_kernel(
+ x_ptr,
+ w_ptr,
+ y_ptr,
+ stride_x,
+ stride_y,
+ DIM: tl.constexpr,
+ NUM_GROUPS: tl.constexpr,
+ W_SHARED: tl.constexpr,
+ EPS: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ GROUP_DIM: tl.constexpr = DIM // NUM_GROUPS
+ BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(GROUP_DIM)
+
+ pid = tl.program_id(0)
+ group_id = pid % NUM_GROUPS
+ # row * stride can overflow int32 for large token counts.
+ row = (pid // NUM_GROUPS).to(tl.int64)
+
+ offs_g = tl.arange(0, BLOCK_SIZE)
+ offsets = group_id * GROUP_DIM + offs_g
+ mask = offs_g < GROUP_DIM
+ # A [GROUP_DIM] affine is shared; a [DIM] affine follows the grouped
+ # checkpoint layout.
+ w_offs = offs_g if W_SHARED else offsets
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0).to(tl.float32)
+ w = tl.load(w_ptr + w_offs, mask, other=0.0)
+
+ rrms = tl.rsqrt(tl.sum(x * x) / GROUP_DIM + EPS)
+ # Gemma's (1 + w) affine is written this way to lower to an FMA.
+ y = x * rrms
+ y += y * w.to(tl.float32)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+ tl.store(y_ptr + row * stride_y + offsets, y, mask)
+
+
+def grouped_gemma_rmsnorm(
+ x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
+) -> torch.Tensor:
+ N, DIM = x.shape
+ assert x.stride(1) == 1, "grouped Gemma RMSNorm requires unit inner stride"
+ assert weight.is_contiguous(), "grouped Gemma RMSNorm weight must be contiguous"
+ assert DIM % num_groups == 0
+ group_dim = DIM // num_groups
+ assert weight.numel() in (group_dim, DIM)
+
+ y = x.new_empty(x.shape)
+ _grouped_gemma_rmsnorm_kernel[(N * num_groups,)](
+ x,
+ weight,
+ y,
+ x.stride(0),
+ y.stride(0),
+ DIM,
+ num_groups,
+ W_SHARED=weight.numel() == group_dim,
+ EPS=eps,
+ launch_pdl=_pdl_supported(),
+ )
+ return y
+
+
+def _hc_silu_kernel(
+ x_ptr,
+ y_ptr,
+ stride_x,
+ stride_y,
+ DIM: tl.constexpr,
+ HC: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ BLOCK_SIZE: tl.constexpr = triton.next_power_of_2(DIM)
+
+ row = tl.program_id(0).to(tl.int64)
+ offs = tl.arange(0, BLOCK_SIZE)
+ mask = offs < DIM
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ x = tl.load(x_ptr + row * stride_x + offs, mask).to(tl.float32) / HC
+ y = x * tl.sigmoid(x)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+ tl.store(y_ptr + row * stride_y + offs, y, mask)
+
+
+def hc_silu(x: torch.Tensor, hc_count: int) -> torch.Tensor:
+ num_tokens, DIM = x.shape
+ assert x.stride(1) == 1
+
+ output = x.new_empty(x.shape)
+ _hc_silu_kernel[(num_tokens,)](
+ x,
+ output,
+ x.stride(0),
+ output.stride(0),
+ DIM=DIM,
+ HC=hc_count,
+ launch_pdl=_pdl_supported(),
+ )
+ return output
+
+
+def _hc_gate_mix_kernel(
+ x_ptr,
+ g_ptr,
+ y_ptr,
+ stride_x,
+ stride_g,
+ stride_y,
+ DIM: tl.constexpr,
+ HC: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ HC_DIM: tl.constexpr = DIM // HC
+
+ row = tl.program_id(0).to(tl.int64)
+ tile_id = tl.program_id(1)
+ offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+ mask = offs_inner < HC_DIM
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ # The constexpr loop is unrolled and keeps one stream live at a time.
+ # Materializing [HC, BLOCK_SIZE] more than doubles latency at large M.
+ acc = tl.zeros([BLOCK_SIZE], dtype=tl.float32)
+ for stream in tl.static_range(HC):
+ offsets = stream * HC_DIM + offs_inner
+ g = tl.load(g_ptr + row * stride_g + offsets, mask, other=0.0)
+ x = tl.load(x_ptr + row * stride_x + offsets, mask, other=0.0)
+ acc += tl.sigmoid(g.to(tl.float32)) * x.to(tl.float32)
+ acc /= HC
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+ tl.store(y_ptr + row * stride_y + offs_inner, acc, mask)
+
+
+def hc_gate_mix(x: torch.Tensor, gate: torch.Tensor, hc_count: int) -> torch.Tensor:
+ N, DIM = gate.shape
+ assert x.shape == gate.shape
+ assert DIM % hc_count == 0
+ assert x.stride(1) == 1
+ assert gate.stride(1) == 1
+
+ HC_DIM = DIM // hc_count
+ out = x.new_empty(N, HC_DIM)
+ BLOCK_SIZE = 512
+ _hc_gate_mix_kernel[(N, triton.cdiv(HC_DIM, BLOCK_SIZE))](
+ x,
+ gate,
+ out,
+ x.stride(0),
+ gate.stride(0),
+ out.stride(0),
+ DIM,
+ hc_count,
+ BLOCK_SIZE,
+ launch_pdl=_pdl_supported(),
+ )
+ return out
+
+
+def _hc_combine_kernel(
+ block_ptr,
+ res_ptr,
+ inj_ptr,
+ out_ptr,
+ stride_block,
+ stride_res,
+ stride_inj,
+ stride_out,
+ HC_DIM: tl.constexpr,
+ HC: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
+
+ row = tl.program_id(0).to(tl.int64)
+ tile_id = tl.program_id(1)
+
+ offs_inner = tile_id * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+ mask_inner = offs_inner < HC_DIM
+ offs_hc = tl.arange(0, HC_PAD)
+ mask_hc = offs_hc < HC
+ offs = offs_hc[:, None] * HC_DIM + offs_inner[None, :]
+ mask = mask_hc[:, None] & mask_inner[None, :]
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
+ block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
+ res = tl.load(res_ptr + row * stride_res + offs, mask, other=0.0)
+
+ # Keeping HC as a broadcast dimension is faster here than four separate
+ # residual load/store sequences.
+ inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
+ out = res.to(tl.float32) + block.to(tl.float32)[None, :] * inj[:, None]
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+ tl.store(out_ptr + row * stride_out + offs, out, mask=mask)
+
+
+def hc_combine(
+ residual: torch.Tensor,
+ block_output: torch.Tensor,
+ injection_logits: torch.Tensor,
+ hc_count: int,
+) -> torch.Tensor:
+ N, DIM = residual.shape
+ assert DIM % hc_count == 0
+ hc_dim = DIM // hc_count
+ assert block_output.shape == (N, hc_dim)
+ assert injection_logits.shape == (N, hc_count)
+ assert residual.stride(1) == 1
+ assert block_output.stride(1) == 1
+ assert injection_logits.stride(1) == 1
+
+ out = residual.new_empty(residual.shape)
+ BLOCK_SIZE = 512
+ _hc_combine_kernel[(N, triton.cdiv(hc_dim, BLOCK_SIZE))](
+ block_output,
+ residual,
+ injection_logits,
+ out,
+ block_output.stride(0),
+ residual.stride(0),
+ injection_logits.stride(0),
+ out.stride(0),
+ hc_dim,
+ hc_count,
+ BLOCK_SIZE,
+ launch_pdl=_pdl_supported(),
+ )
+ return out
+
+
+def _hc_combine_norm_kernel(
+ block_ptr,
+ res_ptr,
+ inj_ptr,
+ w_ptr,
+ out_ptr,
+ y_ptr,
+ stride_block,
+ stride_res,
+ stride_inj,
+ stride_out,
+ stride_y,
+ HC_DIM: tl.constexpr,
+ HC: tl.constexpr,
+ W_SHARED: tl.constexpr,
+ EPS: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ HC_PAD: tl.constexpr = triton.next_power_of_2(HC)
+ NUM_TILES: tl.constexpr = triton.cdiv(HC_DIM, BLOCK_SIZE)
+ NUM_TILES_PAD: tl.constexpr = triton.next_power_of_2(NUM_TILES)
+
+ row = tl.program_id(0).to(tl.int64)
+ stream = tl.program_id(1)
+ offs_hc = tl.arange(0, HC_PAD)
+ mask_hc = offs_hc < HC
+ tile_ids = tl.arange(0, NUM_TILES_PAD)
+ offs_inner = tile_ids[:, None] * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)[None, :]
+ mask_inner = offs_inner < HC_DIM
+ offs = stream * HC_DIM + offs_inner
+ # Shared norm weights repeat across streams; per-branch weights use the
+ # same flattened HC layout as the residual.
+ w_offs = offs_inner if W_SHARED else offs
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ # Start the uncached residual load first, then issue the other combine
+ # loads before consuming any of them.
+ res = tl.load(res_ptr + row * stride_res + offs, mask_inner, other=0.0)
+ inj = tl.load(inj_ptr + row * stride_inj + offs_hc, mask_hc, other=0.0)
+ block = tl.load(block_ptr + row * stride_block + offs_inner, mask_inner, other=0.0)
+ inj = 2.0 * tl.sigmoid(inj.to(tl.float32) / HC)
+ inj = tl.sum(tl.where(offs_hc == stream, inj, 0.0))
+ # Round the materialized combine result before normalization. This matches
+ # the unfused combine -> RMSNorm boundary.
+ out = (res.to(tl.float32) + block.to(tl.float32) * inj).to(out_ptr.dtype.element_ty)
+ tl.store(out_ptr + row * stride_out + offs, out, mask=mask_inner)
+
+ out = out.to(tl.float32)
+ # Keep the two-axis reduction: flattening the padded tile is ~40% slower
+ # at decode sizes.
+ sum_sq = tl.sum(tl.sum(out * out, axis=1), axis=0)
+ rrms = tl.rsqrt(sum_sq / HC_DIM + EPS)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+
+ # Loading the weight earlier helps decode but keeps the tile live across
+ # the reduction and regresses larger batches, so defer it to the norm.
+ w = tl.load(w_ptr + w_offs, mask_inner, other=0.0)
+ y = out * rrms
+ y += y * w.to(tl.float32)
+ tl.store(y_ptr + row * stride_y + offs, y, mask_inner)
+
+
+def hc_combine_norm(
+ residual: torch.Tensor,
+ block_output: torch.Tensor,
+ injection_logits: torch.Tensor,
+ norm_weight: torch.Tensor,
+ eps: float,
+ hc_count: int,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ N, DIM = residual.shape
+ assert DIM % hc_count == 0
+ hc_dim = DIM // hc_count
+ assert block_output.shape == (N, hc_dim)
+ assert injection_logits.shape == (N, hc_count)
+ assert residual.stride(1) == 1
+ assert block_output.stride(1) == 1
+ assert injection_logits.stride(1) == 1
+ assert norm_weight.is_contiguous()
+ assert norm_weight.numel() in (hc_dim, DIM)
+
+ out = residual.new_empty(residual.shape)
+ y = residual.new_empty(residual.shape)
+ BLOCK_SIZE = 512
+ _hc_combine_norm_kernel[(N, hc_count)](
+ block_output,
+ residual,
+ injection_logits,
+ norm_weight,
+ out,
+ y,
+ block_output.stride(0),
+ residual.stride(0),
+ injection_logits.stride(0),
+ out.stride(0),
+ y.stride(0),
+ hc_dim,
+ hc_count,
+ W_SHARED=norm_weight.numel() == hc_dim,
+ EPS=eps,
+ BLOCK_SIZE=BLOCK_SIZE,
+ launch_pdl=_pdl_supported(),
+ )
+ return out, y
+
+
+__all__ = [
+ "grouped_gemma_rmsnorm",
+ "hc_combine",
+ "hc_combine_norm",
+ "hc_gate_mix",
+ "hc_silu",
+]
diff --git a/python/freetoken/kernel/triton/moe_router.py b/python/freetoken/kernel/triton/moe_router.py
new file mode 100644
index 00000000..54ecfe32
--- /dev/null
+++ b/python/freetoken/kernel/triton/moe_router.py
@@ -0,0 +1,160 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
+# Adapted from SGLang (kernels/ops/moe/moe_fused_gate.py)
+"""Fused softmax top-k MoE router (bias-free, ungrouped experts).
+
+The ``num_token_non_padded`` row mask reads the device tensor, so it survives CUDA-graph capture.
+"""
+
+from __future__ import annotations
+
+import functools
+from typing import Tuple
+
+import torch
+import triton
+import triton.language as tl
+
+from freetoken.utils.arch import is_sm90_supported
+
+
+def _pdl_supported() -> bool:
+ return is_sm90_supported()
+
+
+def _router_triton_kernel(
+ scores_ptr,
+ out_weights_ptr,
+ out_indices_ptr,
+ num_token_non_padded_ptr,
+ M,
+ stride_sm,
+ stride_sn,
+ stride_wm,
+ stride_wk,
+ stride_im,
+ stride_ik,
+ N: tl.constexpr,
+ K: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_K: tl.constexpr,
+ RENORMALIZE: tl.constexpr,
+ HAS_TOKEN_LIMIT: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ # Row-tiled: each program handles BLOCK_M rows; all reductions run along the
+ # expert (N) axis. Tiling rows keeps CTAs large enough to stay occupancy-bound
+ # rather than launch-bound at small N (many tiny 1-warp CTAs otherwise).
+ pid = tl.program_id(0)
+ offs_m = pid * BLOCK_M + tl.arange(0, BLOCK_M)
+ offs_n = tl.arange(0, BLOCK_N)
+ mask_m = offs_m < M
+ mask_n = offs_n < N
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ # offs_m * stride can overflow int32 for large token counts.
+ row_ptr = scores_ptr + offs_m[:, None].to(tl.int64) * stride_sm + offs_n[None, :] * stride_sn
+ mask2d = mask_m[:, None] & mask_n[None, :]
+ logits = tl.load(row_ptr, mask=mask2d, other=0.0).to(tl.float32)
+
+ ranked = tl.where(mask_n[None, :], logits, -float("inf"))
+ row_max = tl.max(ranked, axis=1)[:, None]
+ exp_row = tl.where(mask_n[None, :], tl.exp(ranked - row_max), 0.0)
+ activated = exp_row / tl.sum(exp_row, axis=1)[:, None]
+
+ # Map NaN -> a finite floor
+ ranked = tl.where(ranked == ranked, ranked, -1e30)
+
+ offs_k = tl.arange(0, BLOCK_K)
+ mask_k = offs_k < K
+ selected_vals = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.float32)
+ selected_idx = tl.zeros([BLOCK_M, BLOCK_K], dtype=tl.int32)
+
+ cur = ranked
+ for k in tl.static_range(K):
+ max_val = tl.max(cur, axis=1)[:, None]
+ lane_id = tl.where(cur == max_val, offs_n[None, :], N + 1) # lowest expert id wins ties
+ win_lane = tl.min(lane_id, axis=1)[:, None].to(tl.int32)
+ win_activated = tl.sum(
+ tl.where(offs_n[None, :] == win_lane, activated, 0.0), axis=1
+ )[:, None]
+ slot = offs_k[None, :] == k
+ selected_vals = tl.where(slot, win_activated, selected_vals)
+ selected_idx = tl.where(slot, win_lane, selected_idx)
+ cur = tl.where(offs_n[None, :] == win_lane, -float("inf"), cur)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+
+ if RENORMALIZE:
+ routed_sum = tl.sum(tl.where(mask_k[None, :], selected_vals, 0.0), axis=1)[:, None]
+ selected_vals = selected_vals / tl.where(routed_sum > 0.0, routed_sum, 1.0)
+
+ if HAS_TOKEN_LIMIT:
+ limit = tl.load(num_token_non_padded_ptr)
+ selected_idx = tl.where(offs_m[:, None] < limit, selected_idx, -1)
+
+ out_w_ptr = out_weights_ptr + offs_m[:, None].to(tl.int64) * stride_wm + offs_k[None, :] * stride_wk
+ out_i_ptr = out_indices_ptr + offs_m[:, None].to(tl.int64) * stride_im + offs_k[None, :] * stride_ik
+ store_mask = mask_m[:, None] & mask_k[None, :]
+ tl.store(out_w_ptr, selected_vals, mask=store_mask)
+ tl.store(out_i_ptr, selected_idx, mask=store_mask)
+
+
+def fused_topk_softmax(
+ gating_output: torch.Tensor,
+ topk: int,
+ renormalize: bool,
+ num_token_non_padded: torch.Tensor | None = None,
+) -> Tuple[torch.Tensor, torch.Tensor]:
+ """Softmax over all experts, top-k, then renormalize; ties keep the lowest expert id.
+
+ ``num_token_non_padded`` is a device scalar; rows at or past it get expert id -1.
+ """
+ assert gating_output.ndim == 2, "gating_output must be 2D"
+ M, N = gating_output.shape
+ weights = torch.empty((M, topk), dtype=torch.float32, device=gating_output.device)
+ indices = torch.empty((M, topk), dtype=torch.int32, device=gating_output.device)
+
+ BLOCK_N = triton.next_power_of_2(N)
+ BLOCK_K = triton.next_power_of_2(topk)
+ # Single warp per program keeps the per-row top-k reductions on cheap warp
+ # shuffles; pack a few rows per program only when N is small so tiny launches
+ # stay occupancy-bound. Swept on H100/B200; larger tiles / more warps regress
+ # (register pressure).
+ BLOCK_M = max(1, min(4, 256 // BLOCK_N))
+ # For wide rows the K sequential argmax passes dominate and benefit from more
+ # warps despite the cross-warp reduction cost.
+ num_warps = 1 if BLOCK_N <= 512 else 4
+
+ _router_triton_kernel[(triton.cdiv(M, BLOCK_M),)](
+ gating_output,
+ weights,
+ indices,
+ num_token_non_padded,
+ M,
+ gating_output.stride(0),
+ gating_output.stride(1),
+ weights.stride(0),
+ weights.stride(1),
+ indices.stride(0),
+ indices.stride(1),
+ N=N,
+ K=topk,
+ BLOCK_M=BLOCK_M,
+ BLOCK_N=BLOCK_N,
+ BLOCK_K=BLOCK_K,
+ RENORMALIZE=renormalize,
+ HAS_TOKEN_LIMIT=num_token_non_padded is not None,
+ launch_pdl=_pdl_supported(),
+ num_warps=num_warps,
+ )
+ return weights, indices
+
+
+__all__ = ["fused_topk_softmax"]
diff --git a/python/freetoken/kernel/triton/moe_shared_gate.py b/python/freetoken/kernel/triton/moe_shared_gate.py
new file mode 100644
index 00000000..0615c2fa
--- /dev/null
+++ b/python/freetoken/kernel/triton/moe_shared_gate.py
@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
+# Adapted from SGLang (kernels/ops/elementwise.py, ``_fused_gate_sigmoid_mul_add``)
+"""Gated shared-expert epilogue: the gate reduction and the sigmoid-mul-add.
+
+The routed experts may write into ``hidden_states`` in place, so the gate reduction runs
+before them and the mul-add after.
+"""
+
+from __future__ import annotations
+
+import functools
+
+import torch
+import triton
+import triton.language as tl
+
+from freetoken.utils.arch import is_sm90_supported
+
+
+def _pdl_supported() -> bool:
+ return is_sm90_supported()
+
+
+def _reduction_warps(hidden_dim: int, num_tokens: int) -> int:
+ warps = max(min(triton.next_power_of_2(triton.cdiv(hidden_dim, 256)), 32), 4)
+ return min(warps, 8) if num_tokens >= 1024 else warps
+
+
+def _gate_sigmoid_kernel(
+ hidden_ptr,
+ weight_ptr,
+ gate_ptr,
+ stride_h,
+ HIDDEN: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ # row * stride can overflow int32 for large token counts.
+ row = tl.program_id(0).to(tl.int64)
+ offs = tl.arange(0, BLOCK_SIZE)
+ mask = offs < HIDDEN
+
+ w = tl.load(weight_ptr + offs, mask=mask, other=0.0).to(tl.float32)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ h = tl.load(hidden_ptr + row * stride_h + offs, mask=mask, other=0.0).to(tl.float32)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+
+ tl.store(gate_ptr + row, tl.sigmoid(tl.sum(h * w, axis=0)))
+
+
+def shared_gate_sigmoid(hidden_states: torch.Tensor, gate_weight: torch.Tensor) -> torch.Tensor:
+ """Per-token ``sigmoid(hidden_states @ gate_weight)`` as fp32 [num_tokens]."""
+ num_tokens, hidden_dim = hidden_states.shape
+ assert hidden_states.stride(1) == 1, "shared gate requires unit inner stride"
+ assert gate_weight.shape == (hidden_dim,) and gate_weight.is_contiguous()
+
+ gate = torch.empty(num_tokens, dtype=torch.float32, device=hidden_states.device)
+ _gate_sigmoid_kernel[(num_tokens,)](
+ hidden_states,
+ gate_weight,
+ gate,
+ hidden_states.stride(0),
+ HIDDEN=hidden_dim,
+ BLOCK_SIZE=triton.next_power_of_2(hidden_dim),
+ launch_pdl=_pdl_supported(),
+ num_warps=_reduction_warps(hidden_dim, num_tokens),
+ )
+ return gate
+
+
+def _gate_mul_add_kernel(
+ routed_ptr,
+ shared_ptr,
+ gate_ptr,
+ out_ptr,
+ stride_r,
+ stride_s,
+ stride_o,
+ HIDDEN: tl.constexpr,
+ BLOCK_SIZE: tl.constexpr,
+ launch_pdl: tl.constexpr,
+) -> None:
+ row = tl.program_id(0).to(tl.int64)
+ block = tl.program_id(1)
+ offs = block * BLOCK_SIZE + tl.arange(0, BLOCK_SIZE)
+ mask = offs < HIDDEN
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_wait()
+
+ gate = tl.load(gate_ptr + row)
+ routed = tl.load(routed_ptr + row * stride_r + offs, mask=mask, other=0.0).to(tl.float32)
+ shared = tl.load(shared_ptr + row * stride_s + offs, mask=mask, other=0.0).to(tl.float32)
+
+ if launch_pdl:
+ tl.extra.cuda.gdc_launch_dependents()
+
+ tl.store(out_ptr + row * stride_o + offs, routed + gate * shared, mask=mask)
+
+
+def shared_gate_mul_add(
+ routed: torch.Tensor, shared: torch.Tensor, gate: torch.Tensor
+) -> torch.Tensor:
+ """``routed + gate[:, None] * shared`` into a fresh tensor."""
+ num_tokens, hidden_dim = routed.shape
+ assert shared.shape == routed.shape
+ assert routed.stride(1) == 1 and shared.stride(1) == 1
+ assert gate.shape == (num_tokens,)
+
+ out = torch.empty_like(routed)
+ block_size = min(triton.next_power_of_2(hidden_dim), 2048)
+ _gate_mul_add_kernel[(num_tokens, triton.cdiv(hidden_dim, block_size))](
+ routed,
+ shared,
+ gate,
+ out,
+ routed.stride(0),
+ shared.stride(0),
+ out.stride(0),
+ HIDDEN=hidden_dim,
+ BLOCK_SIZE=block_size,
+ launch_pdl=_pdl_supported(),
+ num_warps=4,
+ )
+ return out
+
+
+__all__ = ["shared_gate_mul_add", "shared_gate_sigmoid"]
diff --git a/python/freetoken/kernel/triton/ple.py b/python/freetoken/kernel/triton/ple.py
new file mode 100644
index 00000000..2314e94e
--- /dev/null
+++ b/python/freetoken/kernel/triton/ple.py
@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the SGLang project
+# Adapted from SGLang (python/sglang/srt/models/qwen4_exp.py)
+"""UVA row gather for the Qwen3.8-Flash-Next PLE n-gram table.
+
+The table (320,001,536 rows x 160, FP8-e4m3 + one scalar scale = 47.7 GiB) stays in pinned
+host memory and the GPU dereferences it in place over PCIe -- at its host VA on Linux/UVA, at
+the mapped device address on WDDM (``kernel/pinned.device_ptr``). One program per requested
+row: read the row, widen to fp32, apply the per-tensor scale, store bf16.
+
+Ids outside the table store zeros.
+"""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+from freetoken.kernel.triton.e4m3_compat import e4m3_native_cx, e4m3_u8_to_f32
+
+# Latency-bound over PCIe, so keep the block small and let many of them be in flight.
+_NUM_WARPS = 1
+
+
+def _ple_gather_kernel(
+ table_ptr,
+ ids_ptr,
+ out_ptr,
+ scale,
+ num_rows,
+ EMB_DIM: tl.constexpr,
+ IS_FP8: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+):
+ row = tl.program_id(0)
+ idx = tl.load(ids_ptr + row).to(tl.int64)
+ in_range = (idx >= 0) & (idx < num_rows)
+ idx = tl.where(in_range, idx, 0)
+ offsets = tl.arange(0, BLOCK_D)
+ mask = offsets < EMB_DIM
+ # the table is a host allocation: rebuild the typed pointer from the raw address
+ if IS_FP8:
+ if e4m3_native_cx():
+ base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.float8e4nv))
+ values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)
+ else:
+ # pre-sm_89 has no fp8e4nv type: load raw bytes and decode in software
+ base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.uint8))
+ values = e4m3_u8_to_f32(tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0))
+ else:
+ base = table_ptr.to(tl.int64).to(tl.pointer_type(tl.bfloat16))
+ values = tl.load(base + idx * EMB_DIM + offsets, mask=mask, other=0.0).to(tl.float32)
+ values = tl.where(in_range, values * scale, 0.0)
+ tl.store(
+ out_ptr + row * EMB_DIM + offsets,
+ values.to(out_ptr.dtype.element_ty),
+ mask=mask,
+ )
+
+
+def ple_gather_rows(
+ table_ptr: int,
+ num_rows: int,
+ embed_dim: int,
+ row_ids: torch.Tensor,
+ out: torch.Tensor,
+ scale: float = 1.0,
+ is_fp8: bool = True,
+) -> torch.Tensor:
+ """Gather ``row_ids`` from the host-resident table at ``table_ptr`` into ``out``.
+
+ ``row_ids`` is a flat device int tensor; ``out`` is ``[row_ids.numel(), embed_dim]``
+ bf16 on the same device. ``table_ptr`` is the address the GPU must dereference
+ (``kernel/pinned.device_ptr``), not necessarily the host ``data_ptr``.
+ """
+ n = row_ids.numel()
+ assert out.shape == (n, embed_dim) and out.is_contiguous(), out.shape
+ if n:
+ _ple_gather_kernel[(n,)](
+ table_ptr,
+ row_ids,
+ out,
+ float(scale),
+ num_rows,
+ EMB_DIM=embed_dim,
+ IS_FP8=is_fp8,
+ BLOCK_D=triton.next_power_of_2(embed_dim),
+ num_warps=_NUM_WARPS,
+ )
+ return out
+
+
+__all__ = ["ple_gather_rows"]
diff --git a/python/freetoken/kernel/triton/qsa/__init__.py b/python/freetoken/kernel/triton/qsa/__init__.py
new file mode 100644
index 00000000..91753b79
--- /dev/null
+++ b/python/freetoken/kernel/triton/qsa/__init__.py
@@ -0,0 +1,18 @@
+"""Triton kernels for Qwen3.8-Flash-Next QSA sparse attention."""
+
+from .attend import qsa_sparse_paged_attention
+from .compress import qsa_compress_groups, qsa_index_norm_rope, qsa_store_rows
+from .expand import expand_qsa_block_indices
+from .score import qsa_mqa_paged
+from .topk import qsa_block_topk, qsa_block_topk_scratch_width
+
+__all__ = [
+ "expand_qsa_block_indices",
+ "qsa_block_topk",
+ "qsa_block_topk_scratch_width",
+ "qsa_compress_groups",
+ "qsa_index_norm_rope",
+ "qsa_mqa_paged",
+ "qsa_sparse_paged_attention",
+ "qsa_store_rows",
+]
diff --git a/python/freetoken/kernel/triton/qsa/attend.py b/python/freetoken/kernel/triton/qsa/attend.py
new file mode 100644
index 00000000..541e27c6
--- /dev/null
+++ b/python/freetoken/kernel/triton/qsa/attend.py
@@ -0,0 +1,360 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py)
+"""Sparse paged GQA over the QSA selection."""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+
+def _qsa_sparse_paged_gqa_splitk_kernel(
+ q_ptr,
+ k_cache_ptr,
+ v_cache_ptr,
+ indices_ptr,
+ block_table_ptr,
+ token_to_req_ptr,
+ partial_output_ptr,
+ partial_lse_ptr,
+ output_ptr,
+ stride_q_row,
+ stride_q_head,
+ stride_k_block,
+ stride_k_token,
+ stride_k_head,
+ stride_v_block,
+ stride_v_token,
+ stride_v_head,
+ stride_indices_row,
+ stride_table_req,
+ stride_output_row,
+ stride_output_head,
+ num_rows,
+ num_cache_blocks,
+ num_requests,
+ TOPK: tl.constexpr,
+ PAGE_SIZE: tl.constexpr,
+ PAGE_TABLE_WIDTH: tl.constexpr,
+ GROUP_SIZE: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ NUM_QUERY_HEADS: tl.constexpr,
+ NUM_SPLITS: tl.constexpr,
+ NUM_TILES: tl.constexpr,
+ BLOCK_M: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+) -> None:
+ # row * stride can overflow int32 for large row counts.
+ row = tl.program_id(0).to(tl.int64)
+ kv_head = tl.program_id(1)
+ split_id = tl.program_id(2)
+ request = tl.load(token_to_req_ptr + row)
+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
+
+ head_offsets = tl.arange(0, BLOCK_M)
+ dim_offsets = tl.arange(0, HEAD_DIM)
+ column_offsets = tl.arange(0, BLOCK_N)
+ first_head = kv_head * GROUP_SIZE
+ query = tl.load(
+ q_ptr
+ + row * stride_q_row
+ + (first_head + head_offsets[:, None]) * stride_q_head
+ + dim_offsets[None, :],
+ mask=head_offsets[:, None] < GROUP_SIZE,
+ other=0.0,
+ )
+
+ max_value = tl.full((BLOCK_M,), -1.0e20, dtype=tl.float32)
+ normalizer = tl.zeros((BLOCK_M,), dtype=tl.float32)
+ accumulator = tl.zeros((BLOCK_M, HEAD_DIM), dtype=tl.float32)
+ softmax_scale_log2: tl.constexpr = (HEAD_DIM**-0.5) * 1.4426950408889634
+
+ # Dynamic bounds avoid padded main-loop iterations for uneven splits.
+ split_tile_start = split_id * NUM_TILES // NUM_SPLITS
+ split_tile_end = (split_id + 1) * NUM_TILES // NUM_SPLITS
+ for tile in range(split_tile_start, split_tile_end):
+ columns = tile * BLOCK_N + column_offsets
+ logical_token = tl.load(
+ indices_ptr + row * stride_indices_row + columns,
+ mask=columns < TOPK,
+ other=-1,
+ )
+ safe_token = tl.maximum(logical_token, 0)
+ logical_page = safe_token // PAGE_SIZE
+ page_offset = safe_token % PAGE_SIZE
+ valid = (
+ (request >= 0)
+ & (request < num_requests)
+ & (logical_token >= 0)
+ & (logical_page < PAGE_TABLE_WIDTH)
+ )
+ physical_page = tl.load(
+ block_table_ptr
+ + safe_request.to(tl.int64) * stride_table_req
+ + tl.minimum(logical_page, PAGE_TABLE_WIDTH - 1),
+ mask=valid,
+ other=-1,
+ )
+ valid &= (physical_page >= 0) & (physical_page < num_cache_blocks)
+ # physical_page * block stride can overflow int32 for large caches.
+ safe_page = tl.maximum(physical_page, 0).to(tl.int64)
+ keys = tl.load(
+ k_cache_ptr
+ + safe_page[None, :] * stride_k_block
+ + page_offset[None, :] * stride_k_token
+ + kv_head * stride_k_head
+ + dim_offsets[:, None],
+ mask=valid[None, :],
+ other=0.0,
+ )
+ values = tl.load(
+ v_cache_ptr
+ + safe_page[:, None] * stride_v_block
+ + page_offset[:, None] * stride_v_token
+ + kv_head * stride_v_head
+ + dim_offsets[None, :],
+ mask=valid[:, None],
+ other=0.0,
+ )
+ scores = tl.dot(query, keys)
+ # Scaling scores avoids re-quantizing a scaled query to BF16.
+ scores *= softmax_scale_log2
+ scores = tl.where(valid[None, :], scores, -1.0e20)
+ next_max = tl.maximum(max_value, tl.max(scores, axis=1))
+ alpha = tl.math.exp2(max_value - next_max)
+ probabilities = tl.where(
+ valid[None, :], tl.math.exp2(scores - next_max[:, None]), 0.0
+ )
+ accumulator = tl.dot(
+ probabilities.to(values.dtype),
+ values,
+ acc=accumulator * alpha[:, None],
+ )
+ normalizer = normalizer * alpha + tl.sum(probabilities, axis=1)
+ max_value = next_max
+
+ has_values = normalizer > 0
+ normalized_output = tl.where(
+ has_values[:, None],
+ accumulator / tl.maximum(normalizer[:, None], 1.0e-20),
+ 0.0,
+ )
+ output_mask = head_offsets[:, None] < GROUP_SIZE
+ if NUM_SPLITS == 1:
+ tl.store(
+ output_ptr
+ + row * stride_output_row
+ + (first_head + head_offsets[:, None]) * stride_output_head
+ + dim_offsets[None, :],
+ normalized_output,
+ mask=output_mask,
+ )
+ else:
+ partial_lse = tl.where(
+ has_values,
+ max_value + tl.math.log2(tl.maximum(normalizer, 1.0e-20)),
+ -float("inf"),
+ )
+ tl.store(
+ partial_output_ptr
+ + (
+ (split_id.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS
+ + first_head
+ + head_offsets[:, None]
+ )
+ * HEAD_DIM
+ + dim_offsets[None, :],
+ normalized_output,
+ mask=output_mask,
+ )
+ tl.store(
+ partial_lse_ptr
+ + (split_id.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS
+ + first_head
+ + head_offsets,
+ partial_lse,
+ mask=head_offsets < GROUP_SIZE,
+ )
+
+
+def _qsa_merge_splitk_kernel(
+ partial_output_ptr,
+ partial_lse_ptr,
+ output_ptr,
+ stride_output_row,
+ stride_output_head,
+ num_rows,
+ HEAD_DIM: tl.constexpr,
+ NUM_QUERY_HEADS: tl.constexpr,
+ NUM_SPLITS: tl.constexpr,
+ BLOCK_SPLITS: tl.constexpr,
+) -> None:
+ row = tl.program_id(0).to(tl.int64)
+ head = tl.program_id(1)
+ split_offsets = tl.arange(0, BLOCK_SPLITS)
+ dim_offsets = tl.arange(0, HEAD_DIM)
+ split_mask = split_offsets < NUM_SPLITS
+ lse = tl.load(
+ partial_lse_ptr + (split_offsets.to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + head,
+ mask=split_mask,
+ other=-float("inf"),
+ )
+ lse_max = tl.max(lse, axis=0)
+ has_values = lse_max > -float("inf")
+ shifted = tl.where(split_mask & has_values, lse - lse_max, -float("inf"))
+ weights = tl.math.exp2(shifted)
+ denominator = tl.sum(weights, axis=0)
+ partial_output = tl.load(
+ partial_output_ptr
+ + ((split_offsets[:, None].to(tl.int64) * num_rows + row) * NUM_QUERY_HEADS + head)
+ * HEAD_DIM
+ + dim_offsets[None, :],
+ mask=split_mask[:, None],
+ other=0.0,
+ )
+ merged = tl.sum(partial_output * weights[:, None], axis=0)
+ merged = tl.where(denominator > 0, merged / denominator, 0.0)
+ tl.store(
+ output_ptr + row * stride_output_row + head * stride_output_head + dim_offsets,
+ merged,
+ )
+
+
+def qsa_sparse_paged_attention(
+ q: torch.Tensor,
+ k_cache: torch.Tensor,
+ v_cache: torch.Tensor,
+ logical_indices: torch.Tensor,
+ block_table: torch.Tensor,
+ token_to_req: torch.Tensor,
+ out: torch.Tensor | None = None,
+) -> torch.Tensor:
+ """Run sparse GQA directly over paged BF16 K/V caches."""
+
+ if q.ndim != 3 or k_cache.ndim != 4 or v_cache.shape != k_cache.shape:
+ raise ValueError("QSA sparse attention received invalid Q/K/V shapes")
+ if logical_indices.ndim != 2 or logical_indices.shape[0] != q.shape[0]:
+ raise ValueError("QSA indices must have one row per query")
+ if token_to_req.shape != (q.shape[0],) or block_table.ndim != 2:
+ raise ValueError("QSA sparse attention metadata has invalid shapes")
+ if logical_indices.shape[1] <= 0:
+ raise ValueError("QSA sparse attention requires a positive selection width")
+ if q.shape[2] != k_cache.shape[3] or q.shape[1] % k_cache.shape[2]:
+ raise ValueError("QSA sparse attention requires valid grouped-query heads")
+ head_dim = q.shape[2]
+ assert head_dim >= 16 and (head_dim & (head_dim - 1)) == 0
+ assert q.dtype == k_cache.dtype == v_cache.dtype
+ assert logical_indices.dtype == block_table.dtype == torch.int32
+ assert token_to_req.dtype == torch.int32
+ assert q.stride(2) == k_cache.stride(3) == v_cache.stride(3) == 1
+ assert logical_indices.stride(1) == block_table.stride(1) == 1
+ assert token_to_req.stride(0) == 1
+ if out is None:
+ out = torch.empty_like(q)
+ assert out.shape == q.shape and out.dtype == q.dtype and out.stride(2) == 1
+ if not q.shape[0]:
+ return out
+
+ group_size = q.shape[1] // k_cache.shape[2]
+ block_m = triton.next_power_of_2(group_size)
+ base_programs = q.shape[0] * k_cache.shape[2]
+ small_profile_limit = 8 if block_m <= 8 else 4
+
+ # Tuned on GB300 for the Qwen-Air TP1, TP2, and TP4 attention shapes.
+ # Narrow tiles favor decode; wide tiles improve throughput for prefill.
+ if base_programs <= small_profile_limit:
+ block_n, target_splits, partial_warps = 16, 64, 4
+ elif base_programs < 32:
+ block_n, target_splits, partial_warps = 16, 32, 4
+ elif base_programs <= 256:
+ block_n, target_splits, partial_warps = 64, 8, 2
+ elif base_programs <= 512:
+ block_n, target_splits, partial_warps = 64, 4, 2
+ else:
+ block_n, target_splits, partial_warps = 64, 1, 2
+
+ num_tiles = triton.cdiv(logical_indices.shape[1], block_n)
+ # Avoid empty splits when the selection width is smaller than the profile.
+ max_useful_splits = 1 << (num_tiles.bit_length() - 1)
+ num_splits = min(max_useful_splits, target_splits)
+
+ # Split=1 writes output directly and compiles out all workspace accesses.
+ if num_splits == 1:
+ partial_output = out
+ partial_lse = out
+ else:
+ # FP32 partials preserve accuracy when merging independently normalized
+ # splits.
+ partial_output = torch.empty(
+ (num_splits, *q.shape), dtype=torch.float32, device=q.device
+ )
+ partial_lse = torch.empty(
+ (num_splits, q.shape[0], q.shape[1]),
+ dtype=torch.float32,
+ device=q.device,
+ )
+
+ partial_grid = (q.shape[0], k_cache.shape[2], num_splits)
+ _qsa_sparse_paged_gqa_splitk_kernel[partial_grid](
+ q,
+ k_cache,
+ v_cache,
+ logical_indices,
+ block_table,
+ token_to_req,
+ partial_output,
+ partial_lse,
+ out,
+ q.stride(0),
+ q.stride(1),
+ k_cache.stride(0),
+ k_cache.stride(1),
+ k_cache.stride(2),
+ v_cache.stride(0),
+ v_cache.stride(1),
+ v_cache.stride(2),
+ logical_indices.stride(0),
+ block_table.stride(0),
+ out.stride(0),
+ out.stride(1),
+ q.shape[0],
+ k_cache.shape[0],
+ block_table.shape[0],
+ TOPK=logical_indices.shape[1],
+ PAGE_SIZE=k_cache.shape[1],
+ PAGE_TABLE_WIDTH=block_table.shape[1],
+ GROUP_SIZE=group_size,
+ HEAD_DIM=q.shape[2],
+ NUM_QUERY_HEADS=q.shape[1],
+ NUM_SPLITS=num_splits,
+ NUM_TILES=num_tiles,
+ BLOCK_M=block_m,
+ BLOCK_N=block_n,
+ num_warps=partial_warps,
+ num_stages=2,
+ )
+ if num_splits == 1:
+ return out
+
+ _qsa_merge_splitk_kernel[(q.shape[0], q.shape[1])](
+ partial_output,
+ partial_lse,
+ out,
+ out.stride(0),
+ out.stride(1),
+ q.shape[0],
+ HEAD_DIM=q.shape[2],
+ NUM_QUERY_HEADS=q.shape[1],
+ NUM_SPLITS=num_splits,
+ BLOCK_SPLITS=triton.next_power_of_2(num_splits),
+ num_warps=2,
+ num_stages=1,
+ )
+ return out
+
+
+__all__ = ["qsa_sparse_paged_attention"]
diff --git a/python/freetoken/kernel/triton/qsa/compress.py b/python/freetoken/kernel/triton/qsa/compress.py
new file mode 100644
index 00000000..1f19ed1e
--- /dev/null
+++ b/python/freetoken/kernel/triton/qsa/compress.py
@@ -0,0 +1,331 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py and ops/qsa_pre_indexer.py)
+"""QSA index-key compression, indexer norm+rope, and fixed-width row stores.
+
+The pending ring is one row per (request slot, ring position) keyed by ``Req.table_idx``,
+the caches are row-flat, and the fused (1+w) RMSNorm + partial NeoX rope takes the rotary
+width as a parameter.
+"""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+
+def _compress_qsa_groups_kernel(
+ raw_keys_ptr,
+ ring_ptr,
+ ring_slots_ptr,
+ token_to_req_ptr,
+ query_start_loc_ptr,
+ logical_positions_ptr,
+ pooled_ptr,
+ first_positions_ptr,
+ stride_raw_row,
+ stride_ring_slot,
+ stride_ring_row,
+ stride_pooled_row,
+ num_rows,
+ num_ring_slots,
+ num_requests,
+ RING_CAPACITY: tl.constexpr,
+ COMPRESS_RATIO: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+) -> None:
+ row = tl.program_id(0)
+ dims = tl.arange(0, BLOCK_D)
+ request = tl.load(token_to_req_ptr + row)
+ end_position = tl.load(logical_positions_ptr + row)
+ valid_request = (request >= 0) & (request < num_requests)
+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
+ query_row_start = tl.load(
+ query_start_loc_ptr + safe_request, mask=valid_request, other=0
+ )
+ query_row_end = tl.load(
+ query_start_loc_ptr + safe_request + 1, mask=valid_request, other=0
+ )
+ chunk_start_position = end_position - (row - query_row_start)
+ ring_slot = tl.load(ring_slots_ptr + safe_request, mask=valid_request, other=-1)
+ valid_ring_slot = (ring_slot >= 0) & (ring_slot < num_ring_slots)
+ valid_row = (
+ (row < num_rows)
+ & valid_request
+ & (row >= query_row_start)
+ & (row < query_row_end)
+ & (end_position >= COMPRESS_RATIO - 1)
+ )
+ accumulator = tl.zeros((BLOCK_D,), dtype=tl.float32)
+
+ # A group can span the pending ring (older members) and this step's raw rows
+ # (members at positions >= chunk_start_position).
+ for group_offset in tl.range(0, COMPRESS_RATIO):
+ position = end_position - (COMPRESS_RATIO - 1 - group_offset)
+ use_raw = position >= chunk_start_position
+ raw_row = query_row_start + position - chunk_start_position
+ raw_values = tl.load(
+ raw_keys_ptr + raw_row * stride_raw_row + dims,
+ mask=valid_row
+ & use_raw
+ & (raw_row >= query_row_start)
+ & (raw_row < query_row_end)
+ & (raw_row < num_rows)
+ & (dims < HEAD_DIM),
+ other=0.0,
+ ).to(tl.float32)
+ ring_values = tl.load(
+ ring_ptr
+ + tl.maximum(ring_slot, 0).to(tl.int64) * stride_ring_slot
+ + (position % RING_CAPACITY) * stride_ring_row
+ + dims,
+ mask=valid_row
+ & ~use_raw
+ & valid_ring_slot
+ & (dims < HEAD_DIM),
+ other=0.0,
+ ).to(tl.float32)
+ accumulator += tl.where(use_raw, raw_values, ring_values)
+
+ tl.store(
+ pooled_ptr + row * stride_pooled_row + dims,
+ accumulator / COMPRESS_RATIO,
+ mask=(row < num_rows) & (dims < HEAD_DIM),
+ )
+ first_position = end_position - COMPRESS_RATIO + 1
+ tl.store(
+ first_positions_ptr + row,
+ tl.where(valid_row, first_position, 0),
+ mask=row < num_rows,
+ )
+
+
+def _index_norm_rope_kernel(
+ x_ptr,
+ positions_ptr,
+ cos_sin_ptr,
+ weight_ptr,
+ out_ptr,
+ dest_rows_ptr,
+ stride_x_row,
+ stride_out_row,
+ stride_cos_sin_row,
+ num_rows,
+ eps,
+ HEADS: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ ROTARY_HALF: tl.constexpr,
+ BLOCK_R: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+ HAS_DEST_ROWS: tl.constexpr,
+) -> None:
+ rows = tl.program_id(0) * BLOCK_R + tl.arange(0, BLOCK_R)
+ live = rows < num_rows
+ dims = tl.arange(0, BLOCK_D)
+ in_dim = dims < HEAD_DIM
+ in_rotary = dims < 2 * ROTARY_HALF
+ # NeoX pairs dim d with d + rotary_dim/2; both halves are read so the rotation needs
+ # no cross-lane shuffle.
+ pair = dims % ROTARY_HALF
+ partner = tl.where(dims < ROTARY_HALF, dims + ROTARY_HALF, dims - ROTARY_HALF)
+ partner = tl.where(in_rotary, partner, dims)
+
+ base = x_ptr + rows[:, None].to(tl.int64) * stride_x_row
+ mask = live[:, None] & in_dim[None, :]
+ x = tl.load(base + dims[None, :], mask=mask, other=0.0).to(tl.float32)
+ x_partner = tl.load(base + partner[None, :], mask=mask, other=0.0).to(tl.float32)
+ weight = tl.load(weight_ptr + dims, mask=in_dim, other=0.0).to(tl.float32) + 1.0
+ weight_partner = (
+ tl.load(weight_ptr + partner, mask=in_dim, other=0.0).to(tl.float32) + 1.0
+ )
+ rrms = tl.rsqrt(tl.sum(x * x, axis=1) / HEAD_DIM + eps)
+ y = x * rrms[:, None] * weight[None, :]
+ y_partner = x_partner * rrms[:, None] * weight_partner[None, :]
+
+ position = tl.load(positions_ptr + rows // HEADS, mask=live, other=0).to(tl.int64)
+ cos_base = cos_sin_ptr + position[:, None] * stride_cos_sin_row
+ rotary_mask = live[:, None] & in_rotary[None, :]
+ cos = tl.load(cos_base + pair[None, :], mask=rotary_mask, other=1.0)
+ sin = tl.load(cos_base + ROTARY_HALF + pair[None, :], mask=rotary_mask, other=0.0)
+ sign = tl.where(dims < ROTARY_HALF, -1.0, 1.0)
+ result = tl.where(in_rotary[None, :], y * cos + sign[None, :] * y_partner * sin, y)
+
+ if HAS_DEST_ROWS:
+ dest = tl.load(dest_rows_ptr + rows, mask=live, other=-1)
+ live = live & (dest >= 0)
+ dest_row = tl.maximum(dest, 0).to(tl.int64)
+ else:
+ dest_row = rows.to(tl.int64)
+ tl.store(
+ out_ptr + dest_row[:, None] * stride_out_row + dims[None, :],
+ result.to(out_ptr.dtype.element_ty),
+ mask=live[:, None] & in_dim[None, :],
+ )
+
+
+def _store_qsa_rows_kernel(
+ cache_ptr,
+ slots_ptr,
+ rows_ptr,
+ stride_cache_block,
+ stride_cache_token,
+ stride_rows_row,
+ num_rows,
+ num_blocks,
+ PAGE_SIZE: tl.constexpr,
+ WIDTH: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+) -> None:
+ row = tl.program_id(0)
+ dims = tl.arange(0, BLOCK_D)
+ slot = tl.load(slots_ptr + row)
+ valid = (row < num_rows) & (slot >= 0) & (slot < num_blocks * PAGE_SIZE)
+ block = tl.maximum(slot, 0) // PAGE_SIZE
+ token = tl.maximum(slot, 0) % PAGE_SIZE
+ values = tl.load(
+ rows_ptr + row * stride_rows_row + dims,
+ mask=valid & (dims < WIDTH),
+ other=0,
+ )
+ tl.store(
+ cache_ptr
+ + block.to(tl.int64) * stride_cache_block
+ + token * stride_cache_token
+ + dims,
+ values,
+ mask=valid & (dims < WIDTH),
+ )
+
+
+def qsa_compress_groups(
+ raw_keys: torch.Tensor,
+ ring: torch.Tensor,
+ ring_slots: torch.Tensor,
+ token_to_req: torch.Tensor,
+ query_start_loc: torch.Tensor,
+ logical_positions: torch.Tensor,
+ compress_ratio: int,
+ pooled: torch.Tensor,
+ first_positions: torch.Tensor,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Pool each row's closing group from the pending ring and this step's raw rows."""
+
+ rows = raw_keys.shape[0]
+ head_dim = raw_keys.shape[1]
+ if ring.ndim != 3 or ring.shape[2] != head_dim:
+ raise ValueError("QSA pending ring must be [slots, capacity, head_dim]")
+ if ring.shape[1] < compress_ratio:
+ raise ValueError("QSA ring capacity must cover a whole group")
+ if raw_keys.stride(1) != 1 or ring.stride(2) != 1 or pooled.stride(1) != 1:
+ raise ValueError("QSA compression needs unit-stride key rows")
+ if not rows:
+ return pooled, first_positions
+ _compress_qsa_groups_kernel[(rows,)](
+ raw_keys,
+ ring,
+ ring_slots,
+ token_to_req,
+ query_start_loc,
+ logical_positions,
+ pooled,
+ first_positions,
+ raw_keys.stride(0),
+ ring.stride(0),
+ ring.stride(1),
+ pooled.stride(0),
+ rows,
+ ring.shape[0],
+ query_start_loc.shape[0] - 1,
+ RING_CAPACITY=ring.shape[1],
+ COMPRESS_RATIO=compress_ratio,
+ HEAD_DIM=head_dim,
+ BLOCK_D=triton.next_power_of_2(head_dim),
+ num_warps=4,
+ )
+ return pooled, first_positions
+
+
+def qsa_index_norm_rope(
+ x: torch.Tensor,
+ positions: torch.Tensor,
+ cos_sin_cache: torch.Tensor,
+ norm_weight: torch.Tensor,
+ eps: float,
+ out: torch.Tensor,
+ heads: int = 1,
+ dest_rows: torch.Tensor | None = None,
+) -> torch.Tensor:
+ """Zero-centered RMSNorm then partial NeoX rope on [rows, head_dim] indexer rows."""
+
+ rows, head_dim = x.shape
+ rotary_dim = cos_sin_cache.shape[1]
+ if rotary_dim % 2 or rotary_dim > head_dim:
+ raise ValueError("QSA indexer rope needs an even rotary_dim <= head_dim")
+ if x.stride(1) != 1 or out.stride(1) != 1 or not cos_sin_cache.is_contiguous():
+ raise ValueError("QSA indexer norm+rope needs unit-stride rows")
+ if rows % heads:
+ raise ValueError("QSA indexer rows must be a whole number of head groups")
+ if not rows:
+ return out
+ block_r = 8 if head_dim >= 128 else 16
+ _index_norm_rope_kernel[(triton.cdiv(rows, block_r),)](
+ x,
+ positions,
+ cos_sin_cache,
+ norm_weight,
+ out,
+ dest_rows,
+ x.stride(0),
+ out.stride(0),
+ cos_sin_cache.stride(0),
+ rows,
+ eps,
+ HEADS=heads,
+ HEAD_DIM=head_dim,
+ ROTARY_HALF=rotary_dim // 2,
+ BLOCK_R=block_r,
+ BLOCK_D=triton.next_power_of_2(head_dim),
+ HAS_DEST_ROWS=dest_rows is not None,
+ num_warps=4,
+ )
+ return out
+
+
+def qsa_store_rows(
+ cache: torch.Tensor,
+ slot_mapping: torch.Tensor,
+ rows: torch.Tensor,
+) -> None:
+ """Scatter rows into a ``[blocks, block_size, width]`` cache at ``block * block_size +
+ offset``; negative slots are dropped. The cache may be a strided per-layer view."""
+
+ if cache.ndim != 3 or rows.ndim != 2 or rows.shape[1] != cache.shape[2]:
+ raise ValueError("QSA row store needs a [blocks, block_size, width] cache")
+ if cache.stride(2) != 1 or rows.stride(1) != 1:
+ raise ValueError("QSA row store needs unit-stride rows")
+ if rows.shape[0] != slot_mapping.numel():
+ raise ValueError("QSA row store slots and rows disagree")
+ if not rows.shape[0]:
+ return
+ _store_qsa_rows_kernel[(rows.shape[0],)](
+ cache,
+ slot_mapping,
+ rows,
+ cache.stride(0),
+ cache.stride(1),
+ rows.stride(0),
+ rows.shape[0],
+ cache.shape[0],
+ PAGE_SIZE=cache.shape[1],
+ WIDTH=cache.shape[2],
+ BLOCK_D=triton.next_power_of_2(cache.shape[2]),
+ num_warps=4,
+ )
+
+
+__all__ = ["qsa_compress_groups", "qsa_index_norm_rope", "qsa_store_rows"]
diff --git a/python/freetoken/kernel/triton/qsa/expand.py b/python/freetoken/kernel/triton/qsa/expand.py
new file mode 100644
index 00000000..877b5391
--- /dev/null
+++ b/python/freetoken/kernel/triton/qsa/expand.py
@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py)
+"""Expand QSA top-k blocks into token indices."""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+
+def _expand_qsa_indices_kernel(
+ block_indices_ptr,
+ query_positions_ptr,
+ sequence_lengths_ptr,
+ token_to_req_ptr,
+ output_ptr,
+ stride_blocks_row,
+ stride_blocks_column,
+ stride_output_row,
+ stride_output_column,
+ rows,
+ num_requests,
+ BLOCK_TOPK: tl.constexpr,
+ COMPRESS_RATIO: tl.constexpr,
+ TOKEN_TOPK: tl.constexpr,
+ OUTPUT_WIDTH: tl.constexpr,
+ COLUMN_BLOCK: tl.constexpr,
+) -> None:
+ # row * stride can overflow int32 for large row counts.
+ row = tl.program_id(0).to(tl.int64)
+ columns = tl.program_id(1) * COLUMN_BLOCK + tl.arange(0, COLUMN_BLOCK)
+ query_position = tl.load(query_positions_ptr + row)
+ request = tl.load(token_to_req_ptr + row)
+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
+ sequence_length = tl.load(
+ sequence_lengths_ptr + safe_request,
+ mask=(request >= 0) & (request < num_requests),
+ other=0,
+ )
+ complete_blocks = tl.minimum(
+ tl.minimum(
+ (query_position + 1) // COMPRESS_RATIO,
+ sequence_length // COMPRESS_RATIO,
+ ),
+ BLOCK_TOPK,
+ )
+ expanded_count = complete_blocks * COMPRESS_RATIO
+ tail_start = ((query_position + 1) // COMPRESS_RATIO) * COMPRESS_RATIO
+ tail_count = (query_position + 1) - tail_start
+
+ is_expanded = columns < expanded_count
+ block_rank = columns // COMPRESS_RATIO
+ offset = columns % COMPRESS_RATIO
+ safe_rank = tl.minimum(block_rank, BLOCK_TOPK - 1)
+ block = tl.load(
+ block_indices_ptr + row * stride_blocks_row + safe_rank * stride_blocks_column,
+ mask=(row < rows) & is_expanded,
+ other=-1,
+ )
+ expanded = block * COMPRESS_RATIO + offset
+ tail_offset = columns - expanded_count
+ is_tail = (
+ (columns >= expanded_count)
+ & (tail_offset < tail_count)
+ & (tail_offset < COMPRESS_RATIO - 1)
+ )
+ token = tl.where(is_expanded, expanded, tail_start + tail_offset)
+ valid = (
+ (row < rows)
+ & (columns < OUTPUT_WIDTH)
+ & (is_expanded | is_tail)
+ & (token >= 0)
+ & (token < sequence_length)
+ )
+ tl.store(
+ output_ptr + row * stride_output_row + columns * stride_output_column,
+ tl.where(valid, token, -1),
+ mask=(row < rows) & (columns < OUTPUT_WIDTH),
+ )
+
+
+def expand_qsa_block_indices(
+ block_indices: torch.Tensor,
+ query_positions: torch.Tensor,
+ sequence_lengths: torch.Tensor,
+ token_to_req: torch.Tensor,
+ compress_ratio: int,
+ token_topk: int,
+ out: torch.Tensor,
+) -> torch.Tensor:
+ """Expand compressed blocks and compact the causal tail of the open group."""
+
+ if token_topk % compress_ratio:
+ raise ValueError("QSA token top-k must be divisible by compression ratio")
+ block_topk = token_topk // compress_ratio
+ output_width = token_topk + compress_ratio - 1
+ if block_indices.shape != (query_positions.numel(), block_topk):
+ raise ValueError("QSA compressed top-k has an invalid shape")
+ if out.shape != (block_indices.shape[0], output_width):
+ raise ValueError("QSA expansion output has an invalid shape")
+ if not block_indices.shape[0]:
+ return out
+ column_block = 256
+ _expand_qsa_indices_kernel[
+ (block_indices.shape[0], triton.cdiv(output_width, column_block))
+ ](
+ block_indices,
+ query_positions,
+ sequence_lengths,
+ token_to_req,
+ out,
+ block_indices.stride(0),
+ block_indices.stride(1),
+ out.stride(0),
+ out.stride(1),
+ block_indices.shape[0],
+ sequence_lengths.shape[0],
+ BLOCK_TOPK=block_topk,
+ COMPRESS_RATIO=compress_ratio,
+ TOKEN_TOPK=token_topk,
+ OUTPUT_WIDTH=output_width,
+ COLUMN_BLOCK=column_block,
+ num_warps=4,
+ )
+ return out
+
+
+__all__ = ["expand_qsa_block_indices"]
diff --git a/python/freetoken/kernel/triton/qsa/score.py b/python/freetoken/kernel/triton/qsa/score.py
new file mode 100644
index 00000000..49d70208
--- /dev/null
+++ b/python/freetoken/kernel/triton/qsa/score.py
@@ -0,0 +1,191 @@
+# SPDX-License-Identifier: Apache-2.0
+# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
+# Adapted from vLLM (vllm/models/qwen4_exp/nvidia/ops/qsa.py)
+"""QSA block scoring over the paged compressed-key slab."""
+
+from __future__ import annotations
+
+import math
+
+import torch
+import triton
+import triton.language as tl
+
+
+def _qsa_mqa_paged_kernel(
+ q_ptr,
+ k_cache_ptr,
+ page_table_ptr,
+ token_to_req_ptr,
+ query_positions_ptr,
+ sequence_lengths_ptr,
+ visible_blocks_ptr,
+ logits_ptr,
+ stride_q_row,
+ stride_q_head,
+ stride_q_dim,
+ stride_cache_block,
+ stride_cache_token,
+ stride_cache_dim,
+ stride_table_req,
+ stride_table_page,
+ stride_logits_row,
+ num_rows,
+ num_columns,
+ num_pages,
+ num_requests,
+ score_divisor,
+ PAGE_SIZE: tl.constexpr,
+ PAGE_TABLE_WIDTH: tl.constexpr,
+ NUM_HEADS: tl.constexpr,
+ HEAD_DIM: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ BLOCK_D: tl.constexpr,
+ TILES_PER_PROG: tl.constexpr,
+ STAGES: tl.constexpr,
+ MAX_N: tl.constexpr,
+ COMPRESS_RATIO: tl.constexpr,
+) -> None:
+ row = tl.program_id(0)
+ dims = tl.arange(0, BLOCK_D)
+ heads = tl.arange(0, MAX_N)
+ request = tl.load(token_to_req_ptr + row)
+ safe_request = tl.minimum(tl.maximum(request, 0), num_requests - 1)
+ query_position = tl.load(query_positions_ptr + row)
+ sequence_length = tl.load(
+ sequence_lengths_ptr + safe_request,
+ mask=(request >= 0) & (request < num_requests),
+ other=0,
+ )
+ visible = tl.minimum(
+ (query_position + 1) // COMPRESS_RATIO,
+ sequence_length // COMPRESS_RATIO,
+ )
+ if tl.program_id(1) == 0:
+ tl.store(visible_blocks_ptr + row, visible)
+ tile_start = tl.program_id(1) * TILES_PER_PROG
+ # Top-k is bounded by visible_blocks, so columns beyond it need no value.
+ if tile_start * BLOCK_N >= visible:
+ return
+ tile_end = tl.minimum(tile_start + TILES_PER_PROG, tl.cdiv(visible, BLOCK_N))
+ tile_end = tl.minimum(tile_end, tl.cdiv(num_columns, BLOCK_N))
+
+ # Pad the small head axis to a tensor-core-compatible N dimension.
+ query = tl.load(
+ q_ptr
+ + row * stride_q_row
+ + heads[None, :] * stride_q_head
+ + dims[:, None] * stride_q_dim,
+ mask=(heads[None, :] < NUM_HEADS) & (dims[:, None] < HEAD_DIM),
+ other=0.0,
+ )
+ column_offsets = tl.arange(0, BLOCK_N)
+ for tile in tl.range(tile_start, tile_end, num_stages=STAGES):
+ columns = tile * BLOCK_N + column_offsets
+ live = columns < visible
+ logical_page = tl.minimum(columns // PAGE_SIZE, PAGE_TABLE_WIDTH - 1)
+ page_offset = columns % PAGE_SIZE
+ physical_page = tl.load(
+ page_table_ptr
+ + safe_request * stride_table_req
+ + logical_page * stride_table_page,
+ mask=live,
+ other=-1,
+ )
+ page_valid = live & (physical_page >= 0) & (physical_page < num_pages)
+ # physical_page * block stride can overflow int32 for large caches.
+ safe_physical_page = tl.maximum(physical_page, 0).to(tl.int64)
+ keys = tl.load(
+ k_cache_ptr
+ + safe_physical_page[:, None] * stride_cache_block
+ + page_offset[:, None] * stride_cache_token
+ + dims[None, :] * stride_cache_dim,
+ mask=page_valid[:, None] & (dims[None, :] < HEAD_DIM),
+ other=0.0,
+ eviction_policy="evict_first",
+ )
+ scores = tl.dot(keys, query, out_dtype=tl.float32)
+ scores = tl.where(heads[None, :] < NUM_HEADS, tl.maximum(scores, 0.0), 0.0)
+ score = tl.sum(scores, axis=1) / score_divisor
+ tl.store(
+ logits_ptr + row * stride_logits_row + columns,
+ tl.where(page_valid, score, -float("inf")),
+ mask=live & (columns < num_columns),
+ )
+
+
+def qsa_mqa_paged(
+ q: torch.Tensor,
+ k_cache: torch.Tensor,
+ page_table: torch.Tensor,
+ token_to_req: torch.Tensor,
+ query_positions: torch.Tensor,
+ sequence_lengths: torch.Tensor,
+ compress_ratio: int,
+ logits: torch.Tensor,
+ visible_blocks: torch.Tensor,
+ score_scale: float | None = None,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Compute QSA scores directly from a paged compressed-key cache."""
+
+ if q.ndim != 3 or q.shape[1] <= 0 or q.shape[2] <= 0:
+ raise ValueError("QSA query must be [rows, heads, head_dim]")
+ if k_cache.ndim != 4 or k_cache.shape[2] != 1:
+ raise ValueError("QSA cache must be [pages, page_size, 1, head_dim]")
+ if k_cache.shape[3] != q.shape[2]:
+ raise ValueError("QSA query and cache dimensions must match")
+ if token_to_req.shape != (q.shape[0],) or query_positions.shape != (q.shape[0],):
+ raise ValueError("QSA request mapping and positions must match query rows")
+ if sequence_lengths.shape != (page_table.shape[0],):
+ raise ValueError("QSA sequence lengths must match page-table requests")
+ score_divisor = math.sqrt(q.shape[2]) if score_scale is None else score_scale
+ columns = logits.shape[1]
+ if not q.shape[0] or not columns:
+ return logits, visible_blocks
+ BLOCK_N = 64
+ BLOCK_D = max(16, triton.next_power_of_2(q.shape[2]))
+ MAX_N = max(16, triton.next_power_of_2(q.shape[1]))
+ # Tuned on GB300: larger row batches provide enough parallelism to reuse Q.
+ tiles_per_program = 1 if q.shape[0] <= 32 else 8
+ _qsa_mqa_paged_kernel[
+ (q.shape[0], triton.cdiv(columns, BLOCK_N * tiles_per_program))
+ ](
+ q,
+ k_cache,
+ page_table,
+ token_to_req,
+ query_positions,
+ sequence_lengths,
+ visible_blocks,
+ logits,
+ q.stride(0),
+ q.stride(1),
+ q.stride(2),
+ k_cache.stride(0),
+ k_cache.stride(1),
+ k_cache.stride(3),
+ page_table.stride(0),
+ page_table.stride(1),
+ logits.stride(0),
+ q.shape[0],
+ columns,
+ k_cache.shape[0],
+ page_table.shape[0],
+ float(score_divisor),
+ PAGE_SIZE=k_cache.shape[1],
+ PAGE_TABLE_WIDTH=page_table.shape[1],
+ NUM_HEADS=q.shape[1],
+ HEAD_DIM=q.shape[2],
+ BLOCK_N=BLOCK_N,
+ BLOCK_D=BLOCK_D,
+ TILES_PER_PROG=tiles_per_program,
+ STAGES=2,
+ MAX_N=MAX_N,
+ COMPRESS_RATIO=compress_ratio,
+ num_warps=2,
+ )
+ return logits, visible_blocks
+
+
+__all__ = ["qsa_mqa_paged"]
diff --git a/python/freetoken/kernel/triton/qsa/topk.py b/python/freetoken/kernel/triton/qsa/topk.py
new file mode 100644
index 00000000..ae57566c
--- /dev/null
+++ b/python/freetoken/kernel/triton/qsa/topk.py
@@ -0,0 +1,458 @@
+"""Exact block top-k over the QSA indexer scores.
+
+torch.topk also sorts its k winners, which the selection never uses: expand.py expands every
+rank the same way and the sparse attend kernel softmaxes over the union, so only the SET
+matters. One program per row runs an MSB-first radix select on the monotone uint32 image of
+the fp32 scores -- ``PASSES`` ``RADIX``-bit passes, each a histogram of the columns that still
+match the fixed prefix -- and one compaction pass then emits the winners in ascending column
+order, -1 padded to the output width like the torch.topk path it replaces.
+
+A row that fits one tile is held in registers across the passes (``SINGLE_TILE``); wider rows
+re-read the tile per pass, which is the price of not spilling a 256 KB row.
+
+One program per row stops scaling once a row runs to tens of thousands of columns: the whole
+row goes through a single CTA. Wide buffers therefore split. Phase 1 gives each ``CHUNK``-wide
+slice of a row its own program, which radix-selects the slice's own top-k into a candidate
+workspace; phase 2 runs the same radix select over the union of those candidates. The global
+top-k is a subset of that union -- a global winner beats at most k-1 columns, so it beats all
+but at most k-1 of its own slice -- so the answer stays exact. ``_split_plan`` derives the
+geometry from the buffer width alone, which is fixed when a graph is captured.
+"""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+# 5-bit digits beat 8-bit ones here: tl.histogram cost grows faster with the bin count than the
+# extra passes cost. 35 bits of digit cover the 32-bit key, the top pass just sees zero bits.
+_RADIX = 5
+_BINS = 1 << _RADIX
+_PASSES = -(-32 // _RADIX)
+_MAX_BLOCK_N = 4096
+# A resident tile costs ~2 ns per column against ~3.2 ns for a re-read one, so both split
+# phases stay resident; past 8192 the tile spills (255 registers) and that reverses.
+_MAX_RESIDENT = 8192
+_MIN_CHUNK = 4096
+
+
+def _monotone_key(value):
+ """fp32 -> uint32 with the float order preserved; key 0 is reserved for a dead column."""
+ bits = value.to(tl.uint32, bitcast=True)
+ return tl.where((bits >> 31) == 0, bits | 0x80000000, ~bits)
+
+
+def _load_keys(logits_row, columns, limit):
+ live = columns < limit
+ value = tl.load(logits_row + columns, mask=live, other=-float("inf"))
+ return tl.where(live & (value > -float("inf")), _monotone_key(value), 0)
+
+
+def _narrow(hist, prefix, keep, k_rem, shift, BINS: tl.constexpr):
+ """One radix step: pin the digit at ``shift`` and report whether the range is settled."""
+ bins = tl.arange(0, BINS)
+ # Lowest bin whose strictly-greater bins no longer cover k_rem holds the k_rem-th key;
+ # `above` falls with the bin id, so the predicate is an upper set.
+ above = tl.sum(hist) - tl.cumsum(hist, axis=0)
+ inside = above < k_rem
+ bin_id = tl.min(tl.where(inside, bins, BINS - 1))
+ hit = bins == bin_id
+ prefix |= bin_id.to(tl.uint32) << shift
+ keep |= tl.full((), BINS - 1, tl.uint32) << shift
+ k_rem -= tl.sum(tl.where(hit, above, 0))
+ # The whole bin fits: `prefix` is a lower bound, the ties below it all win.
+ return prefix, keep, k_rem, k_rem == tl.sum(tl.where(hit, hist, 0))
+
+
+def _resident_prefix(key, k_eff, BINS: tl.constexpr, RADIX: tl.constexpr, PASSES: tl.constexpr):
+ """Radix-select a register-resident tile; returns the winning key range and its tie budget."""
+ # Invariant per pass: `k_rem` winners are still to be found inside the key range that
+ # `prefix` pins on the `keep` bits, and every key above that range is already a winner.
+ prefix = tl.zeros((), tl.uint32)
+ keep = tl.zeros((), tl.uint32)
+ k_rem = k_eff
+ settled = False
+ for step in tl.static_range(PASSES):
+ shift = RADIX * (PASSES - 1 - step)
+ if not settled:
+ hist = tl.histogram(
+ ((key >> shift) & (BINS - 1)).to(tl.int32),
+ BINS,
+ mask=(key != 0) & ((key & keep) == prefix),
+ )
+ prefix, keep, k_rem, settled = _narrow(hist, prefix, keep, k_rem, shift, BINS)
+ return prefix, tl.where(settled, k_eff, k_rem)
+
+
+def _tile_ranks(key, prefix, ties, above_base, equal_base):
+ """Rank every winner of one tile, and carry the running counts past it.
+
+ One packed cumsum carries both: keys above the threshold in the low half, keys equal to it
+ in the high half, so a winner's rank is ``above_before + min(equal_before, ties)``."""
+ greater = ((key > prefix) & (key != 0)).to(tl.int32)
+ equal = ((key == prefix) & (key != 0)).to(tl.int32)
+ packed = greater | (equal << 16)
+ before = tl.cumsum(packed, axis=0) - packed
+ rank_equal = equal_base + (before >> 16)
+ take = (greater == 1) | ((equal == 1) & (rank_equal < ties))
+ rank = above_base + (before & 0xFFFF) + tl.minimum(rank_equal, ties)
+ total = tl.sum(packed)
+ return rank, take, above_base + (total & 0xFFFF), equal_base + (total >> 16)
+
+
+def _compact(
+ out_row,
+ key,
+ columns,
+ prefix,
+ ties,
+ above_base,
+ equal_base,
+ TOP_K: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+):
+ """Emit one tile's winners at their global ranks; returns the running counts after it."""
+ tl.static_assert(BLOCK_N <= 0xFFFF, "packed cumsum keeps 16 bits per half")
+ rank, take, above, equal = _tile_ranks(key, prefix, ties, above_base, equal_base)
+ tl.store(out_row + rank, columns.to(tl.int32), mask=take & (rank < TOP_K))
+ return above, equal
+
+
+def _qsa_block_topk_kernel(
+ logits_ptr,
+ visible_ptr,
+ out_ptr,
+ stride_logits_row,
+ stride_out_row,
+ num_columns,
+ TOP_K: tl.constexpr,
+ PAD_K: tl.constexpr,
+ BLOCK_N: tl.constexpr,
+ SINGLE_TILE: tl.constexpr,
+ BINS: tl.constexpr,
+ RADIX: tl.constexpr,
+ PASSES: tl.constexpr,
+) -> None:
+ row = tl.program_id(0)
+ limit = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0)
+ k_eff = tl.minimum(limit, TOP_K)
+ logits_row = logits_ptr + row.to(tl.int64) * stride_logits_row
+ out_row = out_ptr + row.to(tl.int64) * stride_out_row
+ offsets = tl.arange(0, BLOCK_N)
+ tiles = tl.cdiv(limit, BLOCK_N)
+ if SINGLE_TILE:
+ resident = _load_keys(logits_row, offsets, limit)
+
+ emitted = 0
+ if k_eff > 0:
+ if SINGLE_TILE:
+ prefix, ties = _resident_prefix(resident, k_eff, BINS, RADIX, PASSES)
+ else:
+ prefix = tl.zeros((), tl.uint32)
+ keep = tl.zeros((), tl.uint32)
+ k_rem = k_eff
+ settled = False
+ for step in tl.static_range(PASSES):
+ shift = RADIX * (PASSES - 1 - step)
+ if not settled:
+ hist = tl.zeros((BINS,), tl.int32)
+ for tile in range(tiles):
+ key = _load_keys(logits_row, tile * BLOCK_N + offsets, limit)
+ hist += tl.histogram(
+ ((key >> shift) & (BINS - 1)).to(tl.int32),
+ BINS,
+ mask=(key != 0) & ((key & keep) == prefix),
+ )
+ prefix, keep, k_rem, settled = _narrow(
+ hist, prefix, keep, k_rem, shift, BINS
+ )
+ ties = tl.where(settled, k_eff, k_rem)
+
+ above_base = 0
+ equal_base = 0
+ if SINGLE_TILE:
+ above_base, equal_base = _compact(
+ out_row, resident, offsets, prefix, ties, above_base, equal_base, TOP_K, BLOCK_N
+ )
+ else:
+ for tile in range(tiles):
+ columns = tile * BLOCK_N + offsets
+ key = _load_keys(logits_row, columns, limit)
+ above_base, equal_base = _compact(
+ out_row, key, columns, prefix, ties, above_base, equal_base, TOP_K, BLOCK_N
+ )
+ emitted = above_base + tl.minimum(equal_base, ties)
+
+ pad = tl.arange(0, PAD_K)
+ tl.store(out_row + pad, -1, mask=(pad >= emitted) & (pad < TOP_K))
+
+
+def _qsa_topk_split_kernel(
+ logits_ptr,
+ visible_ptr,
+ key_ptr,
+ col_ptr,
+ stride_logits_row,
+ stride_scratch_row,
+ num_columns,
+ TOP_K: tl.constexpr,
+ PAD_K: tl.constexpr,
+ CHUNK: tl.constexpr,
+ BINS: tl.constexpr,
+ RADIX: tl.constexpr,
+ PASSES: tl.constexpr,
+) -> None:
+ """Phase 1: one program per (row, chunk), writing the chunk's own top-k as candidates."""
+ tl.static_assert(CHUNK <= 0xFFFF, "packed cumsum keeps 16 bits per half")
+ row = tl.program_id(0)
+ split = tl.program_id(1)
+ base = split * CHUNK
+ visible = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0)
+ limit = tl.minimum(tl.maximum(visible - base, 0), CHUNK)
+ k_eff = tl.minimum(limit, TOP_K)
+ slot = row.to(tl.int64) * stride_scratch_row + split * TOP_K
+
+ emitted = 0
+ if k_eff > 0:
+ offsets = tl.arange(0, CHUNK)
+ key = _load_keys(logits_ptr + row.to(tl.int64) * stride_logits_row + base, offsets, limit)
+ prefix, ties = _resident_prefix(key, k_eff, BINS, RADIX, PASSES)
+ rank, take, above, equal = _tile_ranks(key, prefix, ties, 0, 0)
+ write = take & (rank < TOP_K)
+ tl.store(key_ptr + slot + rank, key.to(tl.int32, bitcast=True), mask=write)
+ tl.store(col_ptr + slot + rank, (base + offsets).to(tl.int32), mask=write)
+ emitted = above + tl.minimum(equal, ties)
+ # A key of 0 is the dead-column sentinel, so the merge needs no separate count per slot.
+ pad = tl.arange(0, PAD_K)
+ tl.store(key_ptr + slot + pad, 0, mask=(pad >= emitted) & (pad < TOP_K))
+
+
+def _merge_tile(
+ key_row,
+ col_row,
+ out_row,
+ candidates,
+ k_eff,
+ TOP_K: tl.constexpr,
+ BLOCK: tl.constexpr,
+ BINS: tl.constexpr,
+ RADIX: tl.constexpr,
+ PASSES: tl.constexpr,
+):
+ """Top-k of one resident tile of candidates; returns how many winners it wrote."""
+ tl.static_assert(BLOCK <= 0xFFFF, "packed cumsum keeps 16 bits per half")
+ offsets = tl.arange(0, BLOCK)
+ live = offsets < candidates
+ key = tl.load(key_row + offsets, mask=live, other=0).to(tl.uint32, bitcast=True)
+ prefix, ties = _resident_prefix(key, k_eff, BINS, RADIX, PASSES)
+ rank, take, above, equal = _tile_ranks(key, prefix, ties, 0, 0)
+ column = tl.load(col_row + offsets, mask=live, other=-1)
+ tl.store(out_row + rank, column, mask=take & (rank < TOP_K))
+ return above + tl.minimum(equal, ties)
+
+
+def _qsa_topk_merge_kernel(
+ visible_ptr,
+ key_ptr,
+ col_ptr,
+ out_ptr,
+ stride_scratch_row,
+ stride_out_row,
+ num_columns,
+ TOP_K: tl.constexpr,
+ PAD_K: tl.constexpr,
+ CHUNK: tl.constexpr,
+ N_SPLITS: tl.constexpr,
+ BLOCK_SMALL: tl.constexpr,
+ BLOCK_MID: tl.constexpr,
+ BLOCK_FULL: tl.constexpr,
+ BINS: tl.constexpr,
+ RADIX: tl.constexpr,
+ PASSES: tl.constexpr,
+) -> None:
+ """Phase 2: one program per row over the candidates phase 1 left behind."""
+ row = tl.program_id(0)
+ limit = tl.maximum(tl.minimum(tl.load(visible_ptr + row), num_columns), 0)
+ k_eff = tl.minimum(limit, TOP_K)
+ # Candidates sit chunk-major, so the splits past the visible tail are one skipped suffix
+ # and the merge keeps costing what the live part of the row costs.
+ candidates = tl.minimum(tl.cdiv(limit, CHUNK), N_SPLITS) * TOP_K
+ key_row = key_ptr + row.to(tl.int64) * stride_scratch_row
+ col_row = col_ptr + row.to(tl.int64) * stride_scratch_row
+ out_row = out_ptr + row.to(tl.int64) * stride_out_row
+
+ emitted = 0
+ if k_eff > 0:
+ if candidates <= TOP_K:
+ # One live split: its own top-k is already the row's, ranked and packed.
+ slot = tl.arange(0, PAD_K)
+ inside = slot < TOP_K
+ key = tl.load(key_row + slot, mask=inside, other=0)
+ alive = inside & (key != 0)
+ column = tl.load(col_row + slot, mask=alive, other=-1)
+ tl.store(out_row + slot, column, mask=inside)
+ emitted = tl.sum(alive.to(tl.int32))
+ # Register residency costs the whole tile even when few candidates are live, so a
+ # short row takes a narrower tile.
+ elif candidates <= BLOCK_SMALL:
+ emitted = _merge_tile(
+ key_row, col_row, out_row, candidates, k_eff,
+ TOP_K, BLOCK_SMALL, BINS, RADIX, PASSES,
+ )
+ elif candidates <= BLOCK_MID:
+ emitted = _merge_tile(
+ key_row, col_row, out_row, candidates, k_eff,
+ TOP_K, BLOCK_MID, BINS, RADIX, PASSES,
+ )
+ else:
+ emitted = _merge_tile(
+ key_row, col_row, out_row, candidates, k_eff,
+ TOP_K, BLOCK_FULL, BINS, RADIX, PASSES,
+ )
+
+ pad = tl.arange(0, PAD_K)
+ tl.store(out_row + pad, -1, mask=(pad >= emitted) & (pad < TOP_K))
+
+
+def _split_plan(columns: int, top_k: int) -> tuple[int, int] | None:
+ """``(chunk, n_splits)`` for the split+merge path, or None to keep the one-program path."""
+ if top_k <= 0 or columns <= _MIN_CHUNK:
+ return None
+ max_splits = _MAX_RESIDENT // triton.next_power_of_2(top_k)
+ if max_splits < 2:
+ return None
+ chunk = max(_MIN_CHUNK, triton.next_power_of_2(-(-columns // max_splits)))
+ if chunk > _MAX_RESIDENT:
+ return None
+ n_splits = -(-columns // chunk)
+ # Merging n_splits*top_k candidates has to be cheaper than scanning the row once.
+ if n_splits < 2 or n_splits * top_k >= columns:
+ return None
+ return chunk, n_splits
+
+
+def qsa_block_topk_scratch_width(columns: int, top_k: int) -> int:
+ """int32 columns of scratch ``qsa_block_topk`` wants per row; 0 when it needs none."""
+ plan = _split_plan(columns, top_k)
+ return 0 if plan is None else 2 * plan[1] * top_k
+
+
+def qsa_block_topk(
+ logits: torch.Tensor,
+ visible: torch.Tensor,
+ out: torch.Tensor,
+ scratch: torch.Tensor | None = None,
+) -> torch.Tensor:
+ """Top ``out.shape[1]`` columns of every ``logits`` row below ``visible``, -1 padded.
+
+ Winners come out in ascending column order, packed at the front of the row; a row with
+ fewer live columns than the width, and any column scored -inf, leaves -1 in the tail.
+ ``scratch`` is the candidate workspace of the split path (see
+ ``qsa_block_topk_scratch_width``); it is allocated per call when the caller passes none."""
+
+ if logits.ndim != 2 or out.ndim != 2:
+ raise ValueError("QSA block top-k takes 2-D logits and output")
+ if out.shape[0] != logits.shape[0] or visible.shape != (logits.shape[0],):
+ raise ValueError("QSA block top-k needs one visible count and one output row per row")
+ if logits.dtype != torch.float32 or out.dtype != torch.int32:
+ raise ValueError("QSA block top-k takes fp32 logits and an int32 output")
+ if logits.stride(1) != 1 or out.stride(1) != 1 or visible.stride(0) != 1:
+ raise ValueError("QSA block top-k needs row-contiguous logits, output and counts")
+ rows, columns = logits.shape
+ top_k = out.shape[1]
+ if not rows or not top_k:
+ return out
+ pad_k = triton.next_power_of_2(top_k)
+ plan = _split_plan(columns, top_k)
+ if plan is None:
+ block_n = min(_MAX_BLOCK_N, triton.next_power_of_2(max(columns, 1)))
+ _qsa_block_topk_kernel[(rows,)](
+ logits,
+ visible,
+ out,
+ logits.stride(0),
+ out.stride(0),
+ columns,
+ TOP_K=top_k,
+ PAD_K=pad_k,
+ BLOCK_N=block_n,
+ SINGLE_TILE=columns <= block_n,
+ BINS=_BINS,
+ RADIX=_RADIX,
+ PASSES=_PASSES,
+ num_warps=8,
+ num_stages=1,
+ )
+ return out
+
+ chunk, n_splits = plan
+ half = n_splits * top_k
+ if scratch is None:
+ scratch = torch.empty((rows, 2 * half), dtype=torch.int32, device=logits.device)
+ elif (
+ scratch.ndim != 2
+ or scratch.shape[0] < rows
+ or scratch.shape[1] < 2 * half
+ or scratch.dtype != torch.int32
+ or scratch.stride(1) != 1
+ ):
+ raise ValueError(
+ f"QSA block top-k needs a row-contiguous int32 scratch of at least "
+ f"[{rows}, {2 * half}], got {tuple(scratch.shape)} {scratch.dtype}"
+ )
+ keys = scratch[:rows, :half]
+ cols = scratch[:rows, half : 2 * half]
+ _qsa_topk_split_kernel[(rows, n_splits)](
+ logits,
+ visible,
+ keys,
+ cols,
+ logits.stride(0),
+ keys.stride(0),
+ columns,
+ TOP_K=top_k,
+ PAD_K=pad_k,
+ CHUNK=chunk,
+ BINS=_BINS,
+ RADIX=_RADIX,
+ PASSES=_PASSES,
+ num_warps=8,
+ num_stages=1,
+ )
+ merge_block = triton.next_power_of_2(half)
+ _qsa_topk_merge_kernel[(rows,)](
+ visible,
+ keys,
+ cols,
+ out,
+ keys.stride(0),
+ out.stride(0),
+ columns,
+ TOP_K=top_k,
+ PAD_K=pad_k,
+ CHUNK=chunk,
+ N_SPLITS=n_splits,
+ BLOCK_SMALL=min(1024, merge_block),
+ BLOCK_MID=min(4096, merge_block),
+ BLOCK_FULL=merge_block,
+ BINS=_BINS,
+ RADIX=_RADIX,
+ PASSES=_PASSES,
+ num_warps=8,
+ num_stages=1,
+ )
+ return out
+
+
+__all__ = ["qsa_block_topk", "qsa_block_topk_scratch_width"]
diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.py
index 1bb352b4..c6c0f1bb 100644
--- a/python/freetoken/kvcache/__init__.py
+++ b/python/freetoken/kvcache/__init__.py
@@ -57,6 +57,10 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:
from .dsa_pool import MLAKVCache
return MLAKVCache
+ if AttnType.QSA in types:
+ from .qsa_pool import QSAKVCache
+
+ return QSAKVCache
if AttnType.BSA in types:
from .bsa_pool import BSAKVCache
@@ -106,6 +110,7 @@ def create_kv_pool(config, num_pages: int, device: torch.device, dtype: torch.dt
num_swa_tokens=num_swa_tokens,
device=device,
dtype=dtype,
+ num_req_slots=config.max_running_req + 1, # + 1 for the dummy request row
)
@@ -116,6 +121,7 @@ def create_kvcache_pool(
dtype: torch.dtype,
device: torch.device,
num_swa_tokens: int | None = None,
+ num_req_slots: int | None = None,
) -> BaseKVCachePool:
if model_config.has_swa_attention:
from .hybrid_swa_pool import HybridSWAKVCache
@@ -174,6 +180,31 @@ def create_kvcache_pool(
num_index_layers=spec.num_index_layers,
)
+ # QSA (Qwen3.8-Flash-Next): the same GQA group, but it stores one index key per
+ # index_ratio tokens and adds per-request tiers sized by the concurrency, not the pages.
+ # layer_ids is mandatory here -- the model is hybrid-linear, and letting MHAKVCache back
+ # all num_layers would allocate K/V slabs for the GDN layers too.
+ if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.QSA:
+ from .qsa_pool import QSAKVCache
+
+ spec = kv_specs[0]
+ if num_req_slots is None:
+ raise ValueError("QSA pools need num_req_slots (max_running_req + 1)")
+ return QSAKVCache(
+ num_kv_heads=spec.num_kv_heads,
+ num_layers=model_config.num_layers,
+ head_dim=spec.head_dim,
+ num_pages=num_pages,
+ page_size=page_size,
+ dtype=dtype,
+ device=device,
+ index_head_dim=spec.index_head_dim,
+ num_index_layers=spec.num_index_layers,
+ index_ratio=spec.index_ratio,
+ num_req_slots=num_req_slots,
+ layer_ids=spec.layer_ids,
+ )
+
if len(kv_specs) == 1 and kv_specs[0].mla:
from .dsa_pool import DSAKVCache, MLAKVCache
diff --git a/python/freetoken/kvcache/base.py b/python/freetoken/kvcache/base.py
index ae8cf9ec..95669e8c 100644
--- a/python/freetoken/kvcache/base.py
+++ b/python/freetoken/kvcache/base.py
@@ -21,7 +21,10 @@ def spec_kv_bytes_per_token(spec, config) -> int:
x layers, plus the bf16 DSA index-key slab when the spec carries indexer dims. Pure
per-spec arithmetic -- pool families compose it over THEIR OWN groups; no family
branching here. (2 bytes/elem == the torch.bfloat16 dsa_pool.DSAKVCache._alloc
- hardcodes; keep the two in lockstep if the slab dtype ever changes.)"""
+ hardcodes; keep the two in lockstep if the slab dtype ever changes.)
+
+ ``index_ratio`` > 1 (QSA) stores one index key per token group, not per token; that slab's
+ ring and scratch rows are fixed-size and priced in QSAKVCache.kv_cost instead."""
per_token = (
(1 if spec.mla else 2) # MLA latent groups store one slab (V aliases K)
* spec.head_dim
@@ -29,7 +32,7 @@ def spec_kv_bytes_per_token(spec, config) -> int:
* config.dtype.itemsize
* spec.num_layers
)
- return per_token + spec.index_head_dim * spec.num_index_layers * 2
+ return per_token + spec.index_head_dim * spec.num_index_layers * 2 // spec.index_ratio
class BaseKVCachePool(ABC):
diff --git a/python/freetoken/kvcache/linear_state_pool.py b/python/freetoken/kvcache/linear_state_pool.py
index 2b3b0138..ff753941 100644
--- a/python/freetoken/kvcache/linear_state_pool.py
+++ b/python/freetoken/kvcache/linear_state_pool.py
@@ -1,9 +1,11 @@
from __future__ import annotations
+import math
+
import torch
from freetoken.distributed import get_tp_info
from freetoken.env import ENV
-from freetoken.models.config import LinearGatedDeltaGroupConfig
+from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpec
from freetoken.utils import div_even
_SSM_DTYPES = {
@@ -35,6 +37,11 @@ class LinearStatePool:
Indexed by ``Req.table_idx`` (0..max_running_req), the same per-request slot the
page table uses, so the scheduler's existing admit/free of ``table_idx`` covers the
state's lifetime. One fixed slot per running request; no paging, no eviction.
+
+ A model can declare extra per-request tensors on the same slots through
+ ``ModelConfig.slot_states`` (see ``SlotStateSpec``); they advance, snapshot, COW and
+ rebuild with the GDN state and are read back through ``slot_state(name, layer_id)``.
+ Consumers must re-read them each forward: ``rebuild`` replaces the tensors.
"""
def __init__(
@@ -44,6 +51,7 @@ class LinearStatePool:
dtype: torch.dtype,
device: torch.device,
tp_size: int | None = None,
+ slot_states: tuple[SlotStateSpec, ...] = (),
) -> None:
if tp_size is None:
tp_size = get_tp_info().size
@@ -70,6 +78,16 @@ class LinearStatePool:
)
self._local_index = {layer_id: i for i, layer_id in enumerate(group.layer_ids)}
+ self._slot_specs = tuple(slot_states)
+ names = [spec.name for spec in self._slot_specs]
+ if len(set(names)) != len(names):
+ raise ValueError(f"duplicate slot_state names: {names}")
+ self._state_layer_index = {
+ spec.name: {lid: i for i, lid in enumerate(spec.layer_ids)}
+ for spec in self._slot_specs
+ }
+ self.slot_states: dict[str, torch.Tensor] = self._alloc_slot_states(num_slots)
+
# Free-list allocator over slots 1..num_slots-1 (slot 0 reserved as a padding sink,
# sglang MambaPool convention). Live working slots, ping-pong track slots, and
# radix-tree-donated snapshots are all drawn from this single free-list, so memory
@@ -77,6 +95,30 @@ class LinearStatePool:
self.padding_slot = 0
self._free_slots: list[int] = list(range(1, num_slots))
+ def _alloc_slot_states(self, num_slots: int) -> dict[str, torch.Tensor]:
+ return {
+ spec.name: torch.full(
+ (max(1, len(spec.layer_ids)), num_slots, *spec.shape),
+ spec.fill_value,
+ dtype=spec.dtype if spec.dtype is not None else self._conv_dtype,
+ device=self._device,
+ )
+ for spec in self._slot_specs
+ }
+
+ def has_slot_state(self, name: str) -> bool:
+ return name in self.slot_states
+
+ def slot_state(self, name: str, layer_id: int | None = None) -> torch.Tensor:
+ """One declared sibling state, ``[num_slots, *shape]``; ``layer_id`` picks the layer row."""
+ t = self.slot_states[name]
+ if layer_id is None:
+ assert not self._state_layer_index[name], (
+ f"slot_state {name!r} is per-layer, pass layer_id"
+ )
+ return t[0]
+ return t[self._state_layer_index[name][layer_id]]
+
@property
def num_free_slots(self) -> int:
return len(self._free_slots)
@@ -110,6 +152,7 @@ class LinearStatePool:
device = self._device
self.conv_states = None
self.recurrent_states = None
+ self.slot_states = {}
if device.type == "cuda":
torch.cuda.synchronize(device)
torch.cuda.empty_cache()
@@ -121,6 +164,7 @@ class LinearStatePool:
dtype=rec_dtype,
device=device,
)
+ self.slot_states = self._alloc_slot_states(num_slots)
self._num_slots = num_slots
self._free_slots = list(range(1, num_slots))
@@ -138,12 +182,16 @@ class LinearStatePool:
slots = torch.as_tensor(slots, dtype=torch.long, device=self._device)
self.conv_states[:, slots] = 0
self.recurrent_states[:, slots] = 0
+ for spec in self._slot_specs:
+ self.slot_states[spec.name][:, slots] = spec.fill_value
def copy_from(self, src: int, dst: int) -> None:
"""Copy a whole-sequence snapshot (conv + recurrent, all layers) from slot ``src`` to
``dst``. Used for COW-on-restore (donated snapshot -> fresh live slot)."""
self.conv_states[:, dst].copy_(self.conv_states[:, src])
self.recurrent_states[:, dst].copy_(self.recurrent_states[:, src])
+ for t in self.slot_states.values():
+ t[:, dst].copy_(t[:, src])
def is_linear_layer(self, layer_id: int) -> bool:
return layer_id in self._local_index
@@ -161,6 +209,8 @@ class LinearStatePool:
"""Zero a slot across all linear layers (new request takes this table_idx)."""
self.conv_states[:, table_idx].zero_()
self.recurrent_states[:, table_idx].zero_()
+ for spec in self._slot_specs:
+ self.slot_states[spec.name][:, table_idx] = spec.fill_value
@property
def num_linear_layers(self) -> int:
@@ -180,6 +230,8 @@ class LinearStatePool:
self.conv_states[:, 0].numel() * self.conv_states.element_size()
+ self.recurrent_states[:, 0].numel() * self.recurrent_states.element_size()
)
+ for t in self.slot_states.values():
+ per += t[:, 0].numel() * t.element_size()
return int(per)
@@ -187,15 +239,22 @@ def linear_state_bytes_per_req(
group: LinearGatedDeltaGroupConfig,
tp_size: int,
dtype: torch.dtype,
+ slot_states: tuple[SlotStateSpec, ...] = (),
) -> int:
- """Linear-state bytes for one request across all linear layers (TP-local)."""
+ """Linear-state bytes for one request across all linear layers (TP-local), plus any
+ declared slot_states."""
n_layers, local_conv_dim, local_v_heads = _linear_local_dims(group, tp_size)
conv_elems = local_conv_dim * (group.conv_kernel_dim - 1)
rec_elems = local_v_heads * group.key_head_dim * group.value_head_dim
conv_bytes = conv_elems * dtype.itemsize # conv state in model dtype
rec_bytes = rec_elems * ssm_state_dtype().itemsize # recurrent state (default fp32)
- return int(n_layers * (conv_bytes + rec_bytes))
+ total = n_layers * (conv_bytes + rec_bytes)
+
+ for spec in slot_states:
+ item = (spec.dtype if spec.dtype is not None else dtype).itemsize
+ total += max(1, len(spec.layer_ids)) * math.prod(spec.shape) * item
+ return int(total)
__all__ = ["LinearStatePool", "linear_state_bytes_per_req"]
@@ -206,10 +265,16 @@ def state_pool_bytes(config, num_slots: int | None = None) -> int:
slot count). The engine adds this to the KV family's fixed cost when budgeting --
the state pool is a sibling pool, not a KV tier."""
linear_group = config.model_config.linear_attention_group()
+ slot_states = getattr(config.model_config, "slot_states", ())
if linear_group is None:
+ if slot_states:
+ raise ValueError("slot_states ride the linear-state slots; model has no linear group")
return 0
slots = num_slots if num_slots is not None else _linear_pool_num_slots(config)
- return linear_state_bytes_per_req(linear_group, config.tp_info.size, config.dtype) * slots
+ per_req = linear_state_bytes_per_req(
+ linear_group, config.tp_info.size, config.dtype, slot_states
+ )
+ return per_req * slots
def _linear_pool_num_slots(config) -> int:
diff --git a/python/freetoken/kvcache/qsa_pool.py b/python/freetoken/kvcache/qsa_pool.py
new file mode 100644
index 00000000..fddcdbd3
--- /dev/null
+++ b/python/freetoken/kvcache/qsa_pool.py
@@ -0,0 +1,212 @@
+"""QSA compressed-block sparse KV pool: paged GQA K/V + compressed index keys + pending ring.
+
+Qwen3.8-Flash-Next scores whole ``index_ratio``-token groups instead of single tokens, so
+its indexer slab holds ONE compressed key row per group, addressed by ``slot //
+index_ratio``. Because ``page_size % index_ratio == 0``, a group's tokens always live in one
+page at consecutive slots, which makes that division well-defined: the compressed rows are a
+1/ratio shadow of the K/V pages and follow page sharing and eviction for free -- no
+allocator, no free, no clear (SGLang qsa_kv_pool / vLLM compressed-region precedent).
+
+Two tiers ride alongside the shadow slab and are NOT per-token:
+- ``pending_ring``: the last ``ring_capacity`` pre-RoPE index keys of each running request (sized by ``ring_capacity_for``), indexed by ``Req.table_idx``. A group that straddles two forwards (chunked prefill, and
+ every decode step) reads its already-consumed members from here. Never cleared: a new
+ tenant of a table_idx starts at a group boundary (cached_len is 0 or a page multiple), so
+ its first closing group takes every member from its own forward.
+- scratch rows at ``cmp_scratch_base``: one row per request slot, the write target for rows
+ whose group does not close in this forward, so the compress kernel scatters unconditionally
+ with no negative index and no cross-row conflict (DSV4 precedent).
+
+The slab is amortized into the per-token KV price (``unit_bytes``); the ring and scratch are
+fixed and priced through ``kv_cost``'s ``fixed_cache_size``.
+"""
+
+from __future__ import annotations
+
+import math
+from typing import Sequence
+
+import torch
+
+from .mha_pool import MHAKVCache
+
+# The index tiers are always 2-byte (compute dtype); spec_kv_bytes_per_token budgets the same.
+_INDEX_DTYPE_BYTES = 2
+
+
+class QSAKVCache(MHAKVCache):
+ """MHA paged pool + the compressed index-key slab + the per-request pending ring.
+
+ ``cmp_k_cache(slot)`` is row-flat ``[num_pages * page_size // index_ratio + num_req_slots,
+ index_head_dim]``: row ``r < cmp_scratch_base`` holds the compressed key of the token group
+ whose K/V slots are ``[r * index_ratio, (r + 1) * index_ratio)``, and the rows from
+ ``cmp_scratch_base`` on are the per-request-slot scratch sinks. ``slot`` is the sparse
+ layer's order in the attention backend, same convention as BSAKVCache/DSAKVCache.
+ """
+
+ @classmethod
+ def ring_capacity_for(cls, index_ratio: int, num_speculative_tokens: int = 0) -> int:
+ """Ring depth: one row per pending position, keyed ``position % capacity``; spec decode widens by the draft depth (vLLM sizing)."""
+ return index_ratio * math.ceil((index_ratio + num_speculative_tokens) / index_ratio)
+
+ def __init__(
+ self,
+ num_kv_heads: int,
+ num_layers: int,
+ head_dim: int,
+ num_pages: int,
+ page_size: int,
+ dtype: torch.dtype,
+ device: torch.device,
+ index_head_dim: int,
+ num_index_layers: int,
+ index_ratio: int,
+ num_req_slots: int,
+ ring_capacity: int | None = None,
+ layer_ids: Sequence[int] | None = None,
+ ) -> None:
+ if index_ratio < 1 or page_size % index_ratio != 0:
+ # slot // index_ratio only names one group when a group never straddles a page.
+ raise ValueError(
+ f"QSA needs page_size ({page_size}) divisible by index_ratio ({index_ratio})"
+ )
+ if ring_capacity is None:
+ ring_capacity = self.ring_capacity_for(index_ratio)
+ if ring_capacity < index_ratio:
+ # A closing group reads up to index_ratio - 1 past members plus this forward's.
+ raise ValueError(
+ f"QSA needs ring_capacity ({ring_capacity}) >= index_ratio ({index_ratio})"
+ )
+ # Index keys ride the compute dtype (the model's index_k is engine-dtype). The KV cost
+ # model budgets 2 bytes per token per index layer for the slab
+ # (base.spec_kv_bytes_per_token); keep the two in lockstep.
+ assert dtype.itemsize == _INDEX_DTYPE_BYTES, (
+ f"QSA index slab budgets 2 bytes/token (spec_kv_bytes_per_token); got {dtype}"
+ )
+ self._index_head_dim = index_head_dim
+ self._num_index_layers = num_index_layers
+ self._index_ratio = index_ratio
+ self._num_req_slots = num_req_slots
+ self._ring_capacity = ring_capacity
+ self._index_dtype = dtype
+ self._page_size = page_size
+ super().__init__(
+ num_kv_heads=num_kv_heads,
+ num_layers=num_layers,
+ head_dim=head_dim,
+ num_pages=num_pages,
+ page_size=page_size,
+ dtype=dtype,
+ device=device,
+ layer_ids=layer_ids,
+ )
+ self._zero_kv_slabs()
+ self._alloc_index_tiers(num_pages)
+
+ def _zero_kv_slabs(self) -> None:
+ # Defense-in-depth: the attend kernels pos-mask every K/V load (the real fix for
+ # torch.empty's recycled NaN/Inf bit patterns), but a zeroed slab keeps any future
+ # unmasked read finite instead of model-poisoning. One memset per (re)allocation.
+ self._kv_buffer.zero_()
+
+ def _alloc_index_tiers(self, num_pages: int) -> None:
+ # ZERO-initialized: the score kernel reads whole rows of blocks unmasked and relies on
+ # never-written tail rows dotting to a finite 0. Written rows are never cleared again,
+ # so the kernel must clamp visible blocks to kvlen // index_ratio.
+ self._cmp_scratch_base = num_pages * self._page_size // self._index_ratio
+ self._cmp_k_buffer = torch.zeros(
+ self._num_index_layers,
+ self._cmp_scratch_base + self._num_req_slots,
+ self._index_head_dim,
+ dtype=self._index_dtype,
+ device=self._device,
+ )
+ self._pending_ring = torch.zeros(
+ self._num_req_slots,
+ self._num_index_layers,
+ self._ring_capacity,
+ self._index_head_dim,
+ dtype=self._index_dtype,
+ device=self._device,
+ )
+
+ def rebuild(self, num_pages: int) -> None:
+ # Free the index tiers BEFORE the K/V realloc (super().rebuild frees + syncs +
+ # empty_cache), then re-derive them at the new page count. If the index alloc itself
+ # fails (OOM), null the K/V slab too and re-raise: a pool with a grown K/V slab and no
+ # index slab would mis-serve silently. Rebuild is idle-only, so zeroing the ring here
+ # cannot drop a live request's pending members.
+ self._cmp_k_buffer = None
+ self._pending_ring = None
+ super().rebuild(num_pages)
+ self._zero_kv_slabs()
+ try:
+ self._alloc_index_tiers(num_pages)
+ except Exception:
+ self._kv_buffer = None
+ self._k_buffer = None
+ self._v_buffer = None
+ raise
+
+ @classmethod
+ def kv_cost(cls, config) -> tuple[int, int, int, int]:
+ from .base import spec_kv_bytes_per_token
+ from freetoken.attention import AttnType
+
+ num_req_slots = config.max_running_req + 1
+ per_token = 0
+ fixed = 0
+ for spec in config.model_config.kv_cache_group_specs():
+ if spec.is_swa:
+ continue
+ per_token += spec_kv_bytes_per_token(spec, config)
+ if spec.attn_type is AttnType.QSA:
+ # One index-key row = all index layers at one position.
+ row = spec.index_head_dim * spec.num_index_layers * _INDEX_DTYPE_BYTES
+ fixed += num_req_slots * row * (cls.ring_capacity_for(spec.index_ratio) + 1)
+ return per_token * config.page_size, fixed, config.page_size, 0
+
+ def unit_bytes(self) -> tuple[int, int]:
+ # Only the shadow slab scales with pages, and only its non-scratch rows; the ring and
+ # the scratch rows are the fixed term kv_cost reports separately.
+ kv, swa = super().unit_bytes()
+ tokens = int(self._kv_buffer.shape[2]) * int(self._kv_buffer.shape[3])
+ slab = (
+ self._num_index_layers
+ * self._cmp_scratch_base
+ * self._index_head_dim
+ * self._index_dtype.itemsize
+ )
+ return kv + slab // tokens, swa
+
+ def cmp_k_cache(self, slot: int) -> torch.Tensor:
+ """Compressed index keys of one sparse layer: ``[rows, index_head_dim]``."""
+ return self._cmp_k_buffer[slot]
+
+ def pending_ring(self, slot: int) -> torch.Tensor:
+ """One sparse layer's pending ring: ``[num_req_slots, ring_capacity, index_head_dim]``."""
+ return self._pending_ring[:, slot]
+
+ @property
+ def cmp_scratch_base(self) -> int:
+ """First scratch row of ``cmp_k_cache``; row ``cmp_scratch_base + table_idx`` sinks a
+ forward whose group does not close."""
+ return self._cmp_scratch_base
+
+ @property
+ def index_ratio(self) -> int:
+ return self._index_ratio
+
+ @property
+ def index_head_dim(self) -> int:
+ return self._index_head_dim
+
+ @property
+ def ring_capacity(self) -> int:
+ return self._ring_capacity
+
+ @property
+ def num_req_slots(self) -> int:
+ return self._num_req_slots
+
+
+__all__ = ["QSAKVCache"]
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index f6105e1f..229cce81 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -97,6 +97,9 @@ class KVCacheGroupSpec:
mla: bool = False
index_head_dim: int = 0
num_index_layers: int = 0
+ # QSA compression: one index-key row per index_ratio tokens (1 keeps the BSA/DSA
+ # per-token slab). The pool factory and the cost model divide by the same value.
+ index_ratio: int = 1
# Attention-type taxonomy value for this group; drives the backend capability
# matrix and (with the pool factory) selects the KV pool family.
attn_type: AttnType = AttnType.FULL
@@ -134,6 +137,9 @@ class FullAttentionGroupConfig(BaseAttentionGroupConfig):
mla: bool = False
index_head_dim: int = 0
num_index_layers: int = 0
+ # QSA compression ratio (> 1 -> AttnType.QSA): index-key rows are per token group,
+ # so the index slab costs index_head_dim * num_index_layers * 2 // index_ratio per token.
+ index_ratio: int = 1
@dataclass(frozen=True)
@@ -157,7 +163,8 @@ class LinearGatedDeltaGroupConfig(BaseAttentionGroupConfig):
key_head_dim: int
value_head_dim: int
conv_kernel_dim: int
- output_gate: bool
+ # Output-gate activation name ("silu", "sigmoid"), forwarded to rms_norm_gated.
+ output_gate: str
@dataclass(frozen=True)
@@ -185,16 +192,33 @@ AttentionGroupConfig: TypeAlias = (
def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType:
# Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache,
- # GQA (non-mla) + index slab -> BSAKVCache (MiniMax-M3 block-sparse attention).
+ # GQA (non-mla) + index slab -> QSAKVCache when the index keys are compressed
+ # (index_ratio > 1, Qwen3.8-Flash-Next) else BSAKVCache (MiniMax-M3 block-sparse).
if not group.mla:
if group.index_head_dim > 0 and group.num_index_layers > 0:
- return AttnType.BSA
+ return AttnType.QSA if group.index_ratio > 1 else AttnType.BSA
return AttnType.FULL
if group.index_head_dim > 0 and group.num_index_layers > 0:
return AttnType.DSA
return AttnType.MLA
+@dataclass(frozen=True)
+class SlotStateSpec:
+ """One extra per-request tensor riding the LinearStatePool slots.
+
+ Allocated as ``[max(1, len(layer_ids)), num_slots, *shape]`` and advanced, snapshot,
+ COW'd and rebuilt with the GDN state; the owner reads it back through
+ ``pool.slot_state(name, layer_id)``. ``shape`` is per slot and TP-replicated.
+ """
+
+ name: str
+ shape: Tuple[int, ...]
+ layer_ids: Tuple[int, ...] = ()
+ dtype: Any | None = None # a torch dtype; None -> the pool's compute dtype
+ fill_value: float = 0.0
+
+
@dataclass(frozen=True)
class ModelConfig:
num_layers: int
@@ -288,9 +312,16 @@ class ModelConfig:
# swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine
# code; None for every other model.
m3_args: Any | None = None
+ # Qwen3.8-Flash-Next (qwen4_exp) payload (Qwen4ExpArgs): hyper-connection widths, PLE
+ # n-gram embedding geometry and the QSA indexer scoring geometry the model module
+ # needs. Opaque to model-agnostic engine code; None for every other model.
+ qwen4_args: Any | None = None
# Generic execution-path capability flags (set by a model's parse_config) so the engine and
# factories stay model-agnostic instead of branching on dsv4_args:
single_stream_only: bool = False # model runs one sequence at a time -> force bs=1
+ # Extra per-request tensors riding the LinearStatePool slots (see SlotStateSpec);
+ # () for models without any. Requires a linear-attention group to ride on.
+ slot_states: Tuple[SlotStateSpec, ...] = ()
@property
def is_moe(self) -> bool:
@@ -409,6 +440,7 @@ class ModelConfig:
mla=group.mla,
index_head_dim=group.index_head_dim,
num_index_layers=group.num_index_layers,
+ index_ratio=group.index_ratio,
attn_type=_full_group_attn_type(group),
)
)
diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.py
index dc16cfff..2ac4b607 100644
--- a/python/freetoken/models/qwen3_5_moe/config.py
+++ b/python/freetoken/models/qwen3_5_moe/config.py
@@ -152,7 +152,7 @@ def parse_config(hf_config: Any) -> ModelConfig:
or getattr(text, "partial_rotary_factor", None)
or 1.0
)
- rotary_dim = round(head_dim * partial)
+ rotary_dim = int(head_dim * partial)
# For text-only with the default rope type, partial NeoX rope needs no scaling dict
# (the mRoPE params reduce to standard partial rope for text). Avoid carrying the
@@ -218,7 +218,7 @@ def parse_config(hf_config: Any) -> ModelConfig:
key_head_dim=text.linear_key_head_dim,
value_head_dim=text.linear_value_head_dim,
conv_kernel_dim=text.linear_conv_kernel_dim,
- output_gate=True,
+ output_gate="silu",
)
# Order groups by their first layer id for deterministic iteration.
groups = tuple(
@@ -244,7 +244,7 @@ def parse_config(hf_config: Any) -> ModelConfig:
num_experts_per_tok=getattr(text, "num_experts_per_tok", 0),
moe_intermediate_size=getattr(text, "moe_intermediate_size", 0),
shared_expert_intermediate_size=getattr(text, "shared_expert_intermediate_size", 0),
- norm_topk_prob=bool(getattr(text, "norm_topk_prob", False)),
+ norm_topk_prob=True,
moe_enabled=moe_enabled,
use_qk_norm=True,
model_type=getattr(hf_config, "model_type", "qwen3_5_moe"),
diff --git a/python/freetoken/models/qwen3_5_moe/moe.py b/python/freetoken/models/qwen3_5_moe/moe.py
index fc0bb7c2..b5ab1cf3 100644
--- a/python/freetoken/models/qwen3_5_moe/moe.py
+++ b/python/freetoken/models/qwen3_5_moe/moe.py
@@ -69,7 +69,10 @@ class Qwen3_5MoE(BaseOP):
"fp8_block" if getattr(config, "expert_quant", "none") == "fp8_block" else "bf16"
)
self.experts = make_moe_layer(
- config, layer_id=layer_id, renormalize=True, weight_format=weight_format
+ config,
+ layer_id=layer_id,
+ renormalize=config.norm_topk_prob,
+ weight_format=weight_format,
)
self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
self.shared_expert = _SharedExpert(
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index b3414089..d07f18cd 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -911,11 +911,17 @@ def _build_fp8_expert_banks(
B = 128
L, E, H, I, dense = _moe_dims(config)
+
+ # 16B-align the per-expert scale rows (Qwen3.8: down_scale is 20x5 bf16 = 200 B) so the
+ # fused multi-bank copy engages; the GEMMs read scales through explicit strides, so the
+ # padding is inert. Unconditional: one layout per format, shared with the byte formulas.
+ from freetoken.moe.offload_cache import fp8_block_scale_pad as _pad_cols
+
specs = {
"gate_up": ((E, 2 * I, H), FP8),
- "gate_up_scale": ((E, 2 * I // B, H // B), torch.bfloat16),
+ "gate_up_scale": ((E, 2 * I // B, _pad_cols(2 * I // B, H // B)), torch.bfloat16),
"down": ((E, H, I), FP8),
- "down_scale": ((E, H // B, I // B), torch.bfloat16),
+ "down_scale": ((E, H // B, _pad_cols(H // B, I // B)), torch.bfloat16),
}
hb = None
if pin:
@@ -950,8 +956,9 @@ def _build_fp8_expert_banks(
(gate_up[li][e, :I] if proj == "gate" else
gate_up[li][e, I:] if proj == "up" else down[li][e]).copy_(t)
else: # weight_scale_inv
- (gate_up_scale[li][e, : I // B] if proj == "gate" else
- gate_up_scale[li][e, I // B:] if proj == "up" else down_scale[li][e]).copy_(t)
+ (gate_up_scale[li][e, : I // B, : H // B] if proj == "gate" else
+ gate_up_scale[li][e, I // B :, : H // B] if proj == "up" else
+ down_scale[li][e, :, : I // B]).copy_(t)
return li
if parallel is None:
diff --git a/python/freetoken/models/qwen4_exp/__init__.py b/python/freetoken/models/qwen4_exp/__init__.py
new file mode 100644
index 00000000..04c29d77
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/__init__.py
@@ -0,0 +1,35 @@
+"""Qwen3.8-Flash-Next (model_type qwen4_exp), served text-only.
+
+48 decoder layers on hc_count=4 hyper-connection residual streams R [T, 4*hidden]:
+embed -> repeat(1, 4) -> [PLE at zero-based layer 1] -> per layer attn_hc.mix -> (GDN | QSA) -> attn_hc.combine -> mlp_hc.mix -> MoE -> mlp_hc.combine -> top-level mixer.mix -> lm_head.
+Layer contract: forward(R [T, 4*hidden], batch) -> R' [T, 4*hidden].
+
+Contracts shared across modules (do not rename):
+- The PLE dilated-conv left context lives on the LinearStatePool slots as the declared slot state ``ple_conv`` (config.ple_slot_states -> ModelConfig.slot_states), read back with ``pool.slot_state("ple_conv", layer_id)``; same slot / COW / snapshot lifecycle as conv_states / recurrent_states.
+- kvcache.qsa_pool.QSAKVCache(MHAKVCache): ``cmp_k_cache(slot) -> [rows, index_head_dim]`` (compressed index keys, row = kv slot // index_ratio), ``pending_ring(slot) -> [num_req_slots, ring_capacity, index_head_dim]`` (per-request pre-RoPE index-k tail indexed by table_idx, never cleared), ``cmp_scratch_base`` (int, first scratch row for non-closing decode writes). ``slot`` is the sparse layer's order in the attention backend.
+"""
+
+from .config import parse_config
+from .model import Qwen4ExpForCausalLM
+from .weight import (
+ iter_weights,
+ load_nvfp4_expert_sources,
+ load_nvfp4_expert_sources_parallel,
+ load_ple_table,
+)
+
+# Official FP8 checkpoints share qwen3_5_moe's block-fp8 expert layout (same
+# model.language_model.layers.* keys), so reuse its bank hook; for every other
+# expert_quant it defers to the per-quant providers, which resolve this module's
+# load_nvfp4_expert_sources via the model spec.
+from freetoken.models.qwen3_5_moe.weight import setup_offload_expert_banks
+
+__all__ = [
+ "Qwen4ExpForCausalLM",
+ "iter_weights",
+ "load_nvfp4_expert_sources",
+ "load_nvfp4_expert_sources_parallel",
+ "load_ple_table",
+ "parse_config",
+ "setup_offload_expert_banks",
+]
diff --git a/python/freetoken/models/qwen4_exp/attention.py b/python/freetoken/models/qwen4_exp/attention.py
new file mode 100644
index 00000000..d6ab2867
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/attention.py
@@ -0,0 +1,241 @@
+"""QSA full-attention layer for Qwen3.8-Flash-Next (12 of 48 layers).
+
+Gated GQA (24 q heads / 2 kv heads / head_dim 256, per-head zero-centered q/k norms, partial NeoX
+rope over 64 dims, ``q_proj`` twice as wide for the output gate) plus the weights of the QSA
+indexer (``index_qk_proj`` [640, 2560] = 4 index q heads x 128 then 1 index k head x 128, and the
+two per-head index norms).
+
+Model/backend split, same shape as MiniMax-M3's ``bsa_forward``: the layer owns the weights and
+hands the backend the RAW index projections; the backend owns everything stateful (compressed key
+slab, pending ring, scoring, top-k, expansion, sparse attend). The index k norm runs AFTER the
+fp32 mean over each group of ``index_ratio`` raw keys, so it cannot be applied here -- both index
+norm weights travel with the call (:class:`QSAIndexerInputs`).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, Protocol
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm, LinearColParallelMerged, LinearReplicated
+from freetoken.layers.rotary import get_rope
+from freetoken.utils import nvtx_annotate
+
+if TYPE_CHECKING:
+ from freetoken.core import Batch
+ from freetoken.models.config import ModelConfig
+
+
+@dataclass(frozen=True)
+class QSAIndexerInputs:
+ """Everything the QSA backend needs from the indexer for one layer's forward.
+
+ Frozen contract. ``q``/``k`` are the raw ``index_qk_proj`` slices: no norm, no rope. The
+ backend applies, per HF ``Qwen4ExpTextQSAIndexer`` (modeling_qwen4_exp.py:611)::
+
+ q_h = rope64(rmsnorm(q_h) * (1 + q_norm_weight), pos = query position)
+ kbar_b = rope64(rmsnorm(mean_fp32(k[4b:4b+4])) * (1 + k_norm_weight), pos = 4b)
+ s_b = sum_h relu(<q_h, kbar_b>) / sqrt(index_head_dim)
+
+ and the pending ring stores ``k`` PRE-norm and PRE-rope, because a group's mean is only final
+ once all ``index_ratio`` members exist. rope64 is ``get_rope(index_head_dim,
+ config.rotary_config.rotary_dim, ...)`` -- the same frequencies as the main attention, a
+ different ``head_size``, so the backend builds its own (cached) instance.
+ """
+
+ q: torch.Tensor # [T, index_n_heads, index_head_dim]
+ k: torch.Tensor # [T, index_head_dim]
+ q_norm_weight: torch.Tensor # [index_head_dim], zero-centered: scale is (1 + w), fp32
+ k_norm_weight: torch.Tensor # [index_head_dim], zero-centered
+ eps: float
+
+
+class QSAAttentionBackend(Protocol):
+ """The hook ``Qwen4ExpAttention`` calls; ``attention/qsa_sparse.py`` implements it.
+
+ ``q`` is [T, num_qo_heads, head_dim] and ``k``/``v`` are [T, num_kv_heads*head_dim], all post
+ norm+rope and in the model dtype; the return is [T, num_qo_heads, head_dim] (the layer applies
+ the output gate and ``o_proj``). ``layer_id`` is the decoder layer id; the backend maps it to
+ its own sparse-layer slot. Everything else -- KV store, per-request lengths, page rows -- comes
+ from ``batch`` exactly as for ``BaseAttnBackend.forward``.
+ """
+
+ def qsa_forward(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ index: QSAIndexerInputs,
+ layer_id: int,
+ batch: Batch,
+ ) -> torch.Tensor: ...
+
+
+class Qwen4ExpIndexer(BaseOP):
+ """QSA indexer weights (checkpoint prefix ``self_attn.indexer``); the scoring lives in the backend."""
+
+ def __init__(self, config: ModelConfig, layer_id: int) -> None:
+ args = config.qwen4_args
+ self.layer_id = layer_id
+ self.num_heads = args.index_n_heads
+ self.num_kv_heads = args.index_kv_heads
+ self.head_dim = args.index_head_dim
+ self.eps = config.rms_norm_eps
+ self._split = [self.num_heads * self.head_dim, self.num_kv_heads * self.head_dim]
+ self.index_qk_proj = LinearReplicated(args.hidden_size, sum(self._split), has_bias=False)
+ self.q_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)
+ self.k_layernorm = GemmaPlusOneRMSNorm(self.head_dim, eps=self.eps)
+
+ def forward(self, x: torch.Tensor) -> QSAIndexerInputs:
+ q, k = self.index_qk_proj.forward(x).split(self._split, dim=-1)
+ return QSAIndexerInputs(
+ q=q.reshape(-1, self.num_heads, self.head_dim).contiguous(),
+ k=k.reshape(-1, self.head_dim).contiguous(),
+ q_norm_weight=self.q_layernorm.weight,
+ k_norm_weight=self.k_layernorm.weight,
+ eps=self.eps,
+ )
+
+
+class Qwen4ExpAttention(BaseOP):
+ """Gated GQA with a QSA indexer::
+
+ q, gate = chunk(q_proj(x).view(-1, num_q, 2*head_dim), 2, -1)
+ q, k = rope(q_norm(q), k_norm(k_proj(x))) # first rotary_dim dims
+ o = backend.qsa_forward(q, k, v_proj(x), indexer(x), layer_id, batch)
+ out = o_proj(o * sigmoid(gate))
+
+ q/k/v are one merged GEMM (``qkv_proj``, split ``[num_q*head_dim*2, kv, kv]``); the checkpoint
+ ships ``q_proj``/``k_proj``/``v_proj`` separately, so the loader concatenates along dim 0.
+ Other keys keep the checkpoint names: ``o_proj.weight``, ``q_norm.weight``, ``k_norm.weight``
+ (both zero-centered, loaded RAW), ``indexer.*``.
+ """
+
+ def __init__(self, config: ModelConfig, layer_id: int) -> None:
+ self.layer_id = layer_id
+ self.num_q = config.num_qo_heads
+ self.num_kv = config.num_kv_heads
+ self.head_dim = config.head_dim
+ self.qo_attn_dim = self.num_q * self.head_dim
+ self.kv_attn_dim = self.num_kv * self.head_dim
+ self._qkv_split = [self.qo_attn_dim * 2, self.kv_attn_dim, self.kv_attn_dim]
+ self.qkv_proj = LinearColParallelMerged(
+ config.hidden_size, self._qkv_split, has_bias=False
+ )
+ self.o_proj = LinearReplicated(self.qo_attn_dim, config.hidden_size, has_bias=False)
+ self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+ self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=config.rms_norm_eps)
+ rotary = config.rotary_config
+ self.rotary = get_rope(
+ head_dim=self.head_dim,
+ rotary_dim=rotary.rotary_dim,
+ max_position=rotary.max_position,
+ base=rotary.base,
+ rope_scaling=tuple(rotary.scaling.items()) if rotary.scaling else None,
+ )
+ self.indexer = Qwen4ExpIndexer(config, layer_id)
+
+ @nvtx_annotate("QSA")
+ def forward(self, x: torch.Tensor, batch: Batch) -> torch.Tensor:
+ qg, k, v = self.qkv_proj.forward(x).split(self._qkv_split, dim=-1)
+ qg = qg.view(-1, self.num_q, self.head_dim * 2)
+ q = qg[..., : self.head_dim].contiguous()
+ gate = qg[..., self.head_dim :].reshape(-1, self.qo_attn_dim)
+ k = k.contiguous().view(-1, self.num_kv, self.head_dim)
+ v = v.contiguous()
+ self.q_norm.forward_inplace(q)
+ self.k_norm.forward_inplace(k)
+ q, k = self.rotary.forward(
+ batch.positions, q.view(-1, self.qo_attn_dim), k.view(-1, self.kv_attn_dim)
+ )
+ index = self.indexer.forward(x)
+ o = get_global_ctx().attn_backend.qsa_forward(
+ q.view(-1, self.num_q, self.head_dim), k, v, index, self.layer_id, batch
+ )
+ gated = o.reshape(-1, self.qo_attn_dim) * torch.sigmoid(gate)
+ return self.o_proj.forward(gated)
+
+
+class TorchDenseQSAReference:
+ """Dense oracle for :class:`QSAAttentionBackend` (fp32 math): attend to every visible token.
+
+ QSA is exactly dense while a request sees at most ``index_budget + index_ratio - 1`` tokens
+ (every complete block is selected), so this doubles as the equivalence oracle for the sparse backend. It
+ keeps its own ``[slot, position]`` KV instead of a paged pool, so it needs no engine wiring;
+ it is a test/reference object and is never registered as an attention backend.
+ """
+
+ def __init__(
+ self,
+ config: ModelConfig,
+ num_slots: int,
+ max_len: int,
+ device: torch.device,
+ dtype: torch.dtype,
+ ) -> None:
+ self.num_kv = config.num_kv_heads
+ self.head_dim = config.head_dim
+ self.sm_scale = config.attn_sm_scale or self.head_dim**-0.5
+ self._cache: dict[int, tuple[torch.Tensor, torch.Tensor]] = {}
+ self._shape = (num_slots, max_len, self.num_kv, self.head_dim)
+ self._device = device
+ self._dtype = dtype
+
+ def _layer_cache(self, layer_id: int) -> tuple[torch.Tensor, torch.Tensor]:
+ if layer_id not in self._cache:
+ self._cache[layer_id] = tuple(
+ torch.zeros(self._shape, device=self._device, dtype=self._dtype) for _ in range(2)
+ )
+ return self._cache[layer_id]
+
+ def qsa_forward(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ index: QSAIndexerInputs,
+ layer_id: int,
+ batch: Batch,
+ ) -> torch.Tensor:
+ del index
+ k_cache, v_cache = self._layer_cache(layer_id)
+ k = k.view(-1, self.num_kv, self.head_dim)
+ v = v.view(-1, self.num_kv, self.head_dim)
+ out = torch.empty_like(q)
+ offset = 0
+ for r in batch.padded_reqs:
+ n, slot, prefix = r.extend_len, r.table_idx, r.cached_len
+ rows = slice(offset, offset + n)
+ k_cache[slot, prefix : prefix + n] = k[rows]
+ v_cache[slot, prefix : prefix + n] = v[rows]
+ out[rows] = self._attend(
+ q[rows], k_cache[slot, : prefix + n], v_cache[slot, : prefix + n], prefix
+ )
+ offset += n
+ return out
+
+ def _attend(
+ self, q: torch.Tensor, keys: torch.Tensor, values: torch.Tensor, prefix: int
+ ) -> torch.Tensor:
+ n, num_q, _ = q.shape
+ total = keys.shape[0]
+ rep = num_q // self.num_kv
+ keys = keys.repeat_interleave(rep, dim=1).float()
+ values = values.repeat_interleave(rep, dim=1).float()
+ scores = torch.einsum("qhd,khd->hqk", q.float(), keys) * self.sm_scale
+ visible = torch.arange(total, device=q.device) <= (
+ prefix + torch.arange(n, device=q.device)
+ ).unsqueeze(-1)
+ scores = scores.masked_fill(~visible, float("-inf"))
+ return torch.einsum("hqk,khd->qhd", scores.softmax(-1), values).to(q.dtype)
+
+
+__all__ = [
+ "QSAAttentionBackend",
+ "QSAIndexerInputs",
+ "Qwen4ExpAttention",
+ "Qwen4ExpIndexer",
+ "TorchDenseQSAReference",
+]
diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.py
new file mode 100644
index 00000000..bb5d1dff
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/config.py
@@ -0,0 +1,293 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from fnmatch import fnmatch
+from typing import Any, Tuple
+
+import torch
+
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ LinearGatedDeltaGroupConfig,
+ ModelConfig,
+ RotaryConfig,
+ SlotStateSpec,
+)
+
+
+@dataclass(frozen=True)
+class Qwen4ExpArgs:
+ """Qwen3.8-Flash-Next geometry beyond the generic ModelConfig fields (ModelConfig.qwen4_args)."""
+
+ hidden_size: int
+ # Hyper-connections: every layer reads/writes hc_count residual streams [T, hc_count*hidden].
+ hc_count: int
+ hc_lowrank: int
+ # PLE n-gram embedding; layer ids are zero-based decoder layers.
+ ple_layer_ids: Tuple[int, ...]
+ ple_embed_dim: int
+ ple_conv_kernel_size: int
+ ngram_size: int
+ heads_per_ngram: int
+ ngram_vocab_size_base: int
+ make_ngram_vocab_size_divisible_by: int
+ split_ngram_parts: int
+ # n-gram hash windows never cross this token (the eos id); they restart after it.
+ ngram_boundary_token_id: int
+ # QSA indexer scoring geometry (the slab/ratio geometry lives on the attention group).
+ index_n_heads: int
+ index_kv_heads: int
+ index_head_dim: int
+ index_budget: int
+ index_ratio: int
+
+ @property
+ def index_topk_blocks(self) -> int:
+ return self.index_budget // self.index_ratio
+
+ @property
+ def num_ngram_heads(self) -> int:
+ # one head group per n-gram order 2..ngram_size (Qwen3.8: 8 x 2-gram + 8 x 3-gram)
+ return (self.ngram_size - 1) * self.heads_per_ngram
+
+ @property
+ def ngram_head_dim(self) -> int:
+ return self.ple_embed_dim // self.num_ngram_heads
+
+ @property
+ def ple_conv_dilation(self) -> int:
+ # HF Qwen4ExpTextPLELayer sets the depthwise conv dilation to ngram_size
+ return self.ngram_size
+
+ @property
+ def ple_conv_state_len(self) -> int:
+ return (self.ple_conv_kernel_size - 1) * self.ple_conv_dilation
+
+ @property
+ def ple_state_width(self) -> int:
+ return self.hc_count * self.hidden_size
+
+
+PLE_CONV_STATE = "ple_conv"
+PLE_NGRAM_STATE = "ple_ngram_ctx"
+
+
+def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]:
+ """Per-request PLE state riding the linear-state slots (see LinearStatePool.slot_states)."""
+ if not args.ple_layer_ids:
+ return ()
+ return (
+ # dilated-conv left context; replicated (not TP-sharded), model dtype
+ SlotStateSpec(
+ name=PLE_CONV_STATE,
+ shape=(args.ple_state_width, args.ple_conv_state_len),
+ layer_ids=args.ple_layer_ids,
+ ),
+ # last ngram_size-1 token ids, shared by every PLE layer; eos = hash boundary
+ SlotStateSpec(
+ name=PLE_NGRAM_STATE,
+ shape=(args.ngram_size - 1,),
+ dtype=torch.int32,
+ fill_value=float(args.ngram_boundary_token_id),
+ ),
+ )
+
+
+def _quant_get(hf_config: Any):
+ quant = getattr(hf_config, "quantization_config", None)
+ if quant is None:
+ return None
+ return quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
+
+
+def _ignored(patterns, module_name: str) -> bool:
+ return any(fnmatch(module_name, pat) for pat in patterns)
+
+
+def _layer_types(text: Any) -> list[str]:
+ layer_types = getattr(text, "layer_types", None)
+ if layer_types is not None:
+ # HF Qwen4ExpTextConfig rewrites full_attention to qwen_sparse_attention in __post_init__.
+ return [
+ "full_attention" if t == "qwen_sparse_attention" else t for t in layer_types
+ ]
+ # Fall back to full_attention_interval: every Nth layer (1-indexed) is full.
+ interval = int(getattr(text, "full_attention_interval", 4))
+ n = int(text.num_hidden_layers)
+ return [
+ "full_attention" if (i + 1) % interval == 0 else "linear_attention"
+ for i in range(n)
+ ]
+
+
+def parse_config(hf_config: Any) -> ModelConfig:
+ text = getattr(hf_config, "text_config", hf_config)
+
+ head_dim = (
+ getattr(text, "head_dim", None)
+ or text.hidden_size // text.num_attention_heads
+ )
+ num_kv_heads = getattr(text, "num_key_value_heads", text.num_attention_heads)
+
+ rope_params = getattr(text, "rope_parameters", None) or {}
+ rope_theta = rope_params.get("rope_theta", getattr(text, "rope_theta", None))
+ partial = (
+ rope_params.get("partial_rotary_factor")
+ or getattr(text, "partial_rotary_factor", None)
+ or 1.0
+ )
+ # int(), not round(): HF configuration_qwen4_exp truncates head_dim * partial.
+ rotary_dim = int(head_dim * partial)
+
+ # Text-only serving with the default rope type: the mRoPE sections reduce to standard
+ # partial rope, and the unhashable ``mrope_section`` list must not reach get_rope's
+ # cache key.
+ rope_type = rope_params.get("rope_type", "default")
+ rope_scaling = (
+ None
+ if rope_type in (None, "default")
+ else {k: v for k, v in rope_params.items() if not isinstance(v, (list, dict))}
+ )
+
+ get = _quant_get(hf_config)
+ if get is None:
+ expert_quant = attn_quant = dense_quant = lm_head_quant = "none"
+ else:
+ algo = str(get("quant_algo") or get("quant_method") or "").lower()
+ block = get("weight_block_size")
+ if algo == "fp8" and block:
+ # Official FP8 build (DeepSeek-V3-style block-fp8): only the routed experts
+ # are quantized (fp8-e4m3 weights + per-block weight_scale_inv); attention,
+ # GDN, the shared expert, HC, PLE and lm_head stay bf16.
+ bs = tuple(int(x) for x in block)
+ assert bs == (128, 128), f"only 128x128 block-fp8 is supported, got {bs}"
+ expert_quant = "fp8_block"
+ attn_quant = dense_quant = lm_head_quant = "none"
+ else:
+ is_fp4 = "fp4" in algo
+ ignore = list(get("ignore") or [])
+
+ # The RadixArk NVFP4 build quantizes only the routed experts; attention/GDN,
+ # the shared expert, HC, PLE and lm_head all sit in the modelopt ignore list
+ # and stay bf16. Derive every flag from that list instead of assuming the split.
+ def _quant(probe: str) -> str:
+ return "nvfp4" if is_fp4 and not _ignored(ignore, probe) else "none"
+
+ prefix = "model.language_model.layers.0"
+ expert_quant = _quant(f"{prefix}.mlp.experts.0.gate_proj")
+ dense_quant = _quant(f"{prefix}.mlp.shared_expert.gate_proj")
+ attn_quant = _quant(f"{prefix}.self_attn.q_proj")
+ lm_head_quant = _quant("lm_head")
+
+ layer_types = _layer_types(text)
+ full_ids = tuple(i for i, t in enumerate(layer_types) if t == "full_attention")
+ linear_ids = tuple(i for i, t in enumerate(layer_types) if t == "linear_attention")
+
+ # HF stores ple_layer_ids one-indexed (validated upstream as [1, num_layers]).
+ ple_layer_ids = tuple(int(i) - 1 for i in (getattr(text, "ple_layer_ids", None) or ()))
+ for lid in ple_layer_ids:
+ if layer_types[lid] != "linear_attention":
+ raise ValueError(f"PLE must sit on a linear_attention layer, got layer {lid}")
+
+ full_rotary = RotaryConfig(
+ head_dim=head_dim,
+ rotary_dim=rotary_dim,
+ max_position=text.max_position_embeddings,
+ base=rope_theta,
+ scaling=rope_scaling,
+ )
+ full_group = FullAttentionGroupConfig(
+ name="full",
+ layer_ids=full_ids,
+ num_kv_heads=num_kv_heads,
+ head_dim=head_dim,
+ rotary_config=full_rotary,
+ index_head_dim=int(text.indexer_head_dim),
+ num_index_layers=len(full_ids),
+ index_ratio=int(text.indexer_compress_ratio),
+ )
+ linear_group = LinearGatedDeltaGroupConfig(
+ name="linear",
+ layer_ids=linear_ids,
+ num_key_heads=text.linear_num_key_heads,
+ num_value_heads=text.linear_num_value_heads,
+ key_head_dim=text.linear_key_head_dim,
+ value_head_dim=text.linear_value_head_dim,
+ conv_kernel_dim=text.linear_conv_kernel_dim,
+ # HF resolves a null output_gate_type to hidden_act; mirror that instead of
+ # stringifying None.
+ output_gate=str(getattr(text, "output_gate_type", None) or text.hidden_act),
+ )
+ # Order groups by their first layer id for deterministic iteration.
+ groups = tuple(
+ sorted(
+ (full_group, linear_group),
+ key=lambda g: g.layer_ids[0] if g.layer_ids else 1 << 30,
+ )
+ )
+
+ num_experts = int(getattr(text, "num_experts", 0) or 0)
+
+ # HF accepts int | list here and uses the first entry (modeling_qwen4_exp Qwen4ExpTextNGramEmbedding)
+ eos_token_id = text.eos_token_id
+ if isinstance(eos_token_id, (list, tuple)):
+ eos_token_id = eos_token_id[0]
+
+ qwen4_args = Qwen4ExpArgs(
+ hidden_size=text.hidden_size,
+ hc_count=int(text.hc_count),
+ hc_lowrank=int(text.hc_lowrank),
+ ple_layer_ids=ple_layer_ids,
+ ple_embed_dim=int(text.ple_embed_dim),
+ ple_conv_kernel_size=int(text.ple_conv_kernel_size),
+ ngram_size=int(text.ngram_size),
+ heads_per_ngram=int(text.heads_per_ngram),
+ ngram_vocab_size_base=int(text.ngram_vocab_size_base),
+ make_ngram_vocab_size_divisible_by=int(text.make_ngram_vocab_size_divisible_by),
+ split_ngram_parts=int(text.split_ngram_parts),
+ ngram_boundary_token_id=int(eos_token_id),
+ index_n_heads=int(text.indexer_n_heads),
+ index_kv_heads=int(text.indexer_kv_heads),
+ index_head_dim=int(text.indexer_head_dim),
+ index_budget=int(text.indexer_budget),
+ index_ratio=int(text.indexer_compress_ratio),
+ )
+
+ return ModelConfig(
+ num_layers=text.num_hidden_layers,
+ num_qo_heads=text.num_attention_heads,
+ num_kv_heads=num_kv_heads,
+ head_dim=head_dim,
+ hidden_size=text.hidden_size,
+ vocab_size=text.vocab_size,
+ intermediate_size=getattr(text, "intermediate_size", 0) or 0,
+ hidden_act=text.hidden_act,
+ rms_norm_eps=text.rms_norm_eps,
+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
+ rotary_config=full_rotary,
+ num_experts=num_experts,
+ num_experts_per_tok=int(getattr(text, "num_experts_per_tok", 0) or 0),
+ moe_intermediate_size=int(getattr(text, "moe_intermediate_size", 0) or 0),
+ shared_expert_intermediate_size=int(
+ getattr(text, "shared_expert_intermediate_size", 0) or 0
+ ),
+ # Absent from the shipped config; HF Qwen4ExpTextConfig defaults it True and the
+ # Qwen3_5MoE block renormalizes unconditionally -- keep the two in agreement.
+ norm_topk_prob=bool(getattr(text, "norm_topk_prob", True)),
+ moe_enabled=num_experts > 0,
+ use_qk_norm=True,
+ model_type=getattr(hf_config, "model_type", "qwen4_exp"),
+ architectures=getattr(hf_config, "architectures", ["Qwen4ExpForConditionalGeneration"]),
+ vision_config=None, # served text-only
+ image_token_id=getattr(hf_config, "image_token_id", None),
+ attention_groups=groups,
+ expert_quant=expert_quant,
+ attn_quant=attn_quant,
+ dense_quant=dense_quant,
+ lm_head_quant=lm_head_quant,
+ qwen4_args=qwen4_args,
+ slot_states=ple_slot_states(qwen4_args),
+ )
+
+
+__all__ = ["PLE_CONV_STATE", "PLE_NGRAM_STATE", "Qwen4ExpArgs", "parse_config", "ple_slot_states"]
diff --git a/python/freetoken/models/qwen4_exp/gdn.py b/python/freetoken/models/qwen4_exp/gdn.py
new file mode 100644
index 00000000..69838153
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/gdn.py
@@ -0,0 +1,233 @@
+from __future__ import annotations
+
+import torch
+import torch.nn.functional as F
+from freetoken.core import get_global_ctx
+from freetoken.kernel.causal_conv1d import causal_conv1d_decode, causal_conv1d_varlen
+from freetoken.layers import BaseOP, LinearColParallelMerged
+
+from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged
+from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
+from freetoken.models.qwen3_5_moe.gdn_kernels import gdn_decode_fla, gdn_prefill_chunk_fla
+from freetoken.models.quant_linear import make_replicated_quant
+
+
+_GATE_ACTIVATIONS = ("silu", "swish", "sigmoid")
+
+
+class _DepthwiseConv1d(BaseOP):
+ """Holds the depthwise conv weight ``[conv_dim, 1, K]`` (key ``conv1d.weight``)."""
+
+ def __init__(self, conv_dim: int, kernel: int):
+ self.weight = torch.empty(conv_dim, 1, kernel)
+
+
+class _GatedRMSNorm(BaseOP):
+ """RMSNorm of x followed by an ``activation(z)`` gate (HF Qwen4ExpTextRMSNormGated).
+
+ Uses the fused fla ``rms_norm_gated`` triton kernel (norm(x) * act(z) in one
+ kernel) instead of the unfused pow/mean/rsqrt/mul/act chain, matching sglang's
+ ``RMSNormGated`` -- collapses ~8 elementwise kernels per GDN layer into one.
+ Qwen3.8-Flash-Next gates with sigmoid where Qwen3.5 gates with silu."""
+
+ def __init__(self, dim: int, eps: float, activation: str):
+ # rms_norm_gated drops the gate entirely (no error) for a name it does not know.
+ assert activation in _GATE_ACTIVATIONS, f"unsupported GDN output gate {activation!r}"
+ self.weight = torch.empty(dim)
+ self.eps = eps
+ self.activation = activation
+
+ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.fla import rms_norm_gated
+
+ return rms_norm_gated(
+ x=x, weight=self.weight, bias=None, z=z, eps=self.eps,
+ is_rms_norm=True, norm_before_gate=True, activation=self.activation,
+ )
+
+
+class Qwen4ExpGatedDeltaNet(BaseOP):
+ """GatedDeltaNet op using the vendored flash-linear-attention triton kernels
+ (``freetoken.kernel.fla``) for the recurrence and a per-request
+ recurrent + conv state held in ``ctx.linear_state_pool`` (keyed by ``Req.table_idx``).
+
+ Parameter names match HF (``in_proj_qkv``/``in_proj_z``/``in_proj_b``/``in_proj_a``/
+ ``conv1d``/``A_log``/``dt_bias``/``norm``/``out_proj``). Handles prefill (incl. chunked
+ continuation) and single-token decode; state is fresh when ``req.cached_len == 0``.
+
+ ``output_gate`` is the gate activation name from ``LinearGatedDeltaGroupConfig``
+ ("sigmoid" for Qwen3.8-Flash-Next).
+ """
+
+ def __init__(
+ self, hidden_size, num_k_heads, num_v_heads, head_k_dim, head_v_dim,
+ conv_kernel_size, rms_norm_eps, layer_id, output_gate: str = "sigmoid",
+ expert_quant: str = "none", attn_quant: str = "none",
+ ):
+ self.layer_id = layer_id
+ # The fla chunk/decode kernels read+write the recurrent state and the per-chunk h as
+ # [V, K] while the LinearStatePool declares it [K, V]; these coincide (and the
+ # hybrid-radix snapshot scatter h[h_row]->slot is a plain copy) only when the two head
+ # dims are equal. Qwen3.5/3.6/3.8 satisfy this (128/128); guard any future config.
+ assert head_k_dim == head_v_dim, (
+ f"GatedDeltaNet requires head_k_dim == head_v_dim, got {head_k_dim} != {head_v_dim}"
+ )
+ self.num_k_heads = num_k_heads
+ self.num_v_heads = num_v_heads
+ self.head_k_dim = head_k_dim
+ self.head_v_dim = head_v_dim
+ self.key_dim = num_k_heads * head_k_dim
+ self.value_dim = num_v_heads * head_v_dim
+ self.conv_dim = 2 * self.key_dim + self.value_dim
+ self.conv_kernel_size = conv_kernel_size
+ # qkv|z carry a weight scale (block-fp8 weight_scale_inv, or per-tensor FP8
+ # weight_scale); b|a stay bf16. Both quant modes therefore split the four-way
+ # fusion into an fp8 qkvz GEMM + a bf16 ba GEMM (matches sglang/vLLM).
+ self._block_fp8 = expert_quant == "fp8_block"
+ self._pertensor_fp8 = attn_quant == "fp8_pertensor"
+ self._fp8 = self._block_fp8 or self._pertensor_fp8
+
+ self._in_proj_split = [self.conv_dim, self.value_dim, num_v_heads, num_v_heads]
+ if self._fp8:
+ ColMerged = Fp8BlockColMerged if self._block_fp8 else Fp8PerTensorColMerged
+ self.in_proj_qkvz = ColMerged(
+ hidden_size, [self.conv_dim, self.value_dim], has_bias=False
+ )
+ self.in_proj_ba = LinearColParallelMerged(
+ hidden_size, [num_v_heads, num_v_heads], has_bias=False
+ )
+ else:
+ # Fused input projection (one GEMM instead of four): qkv | z | b | a.
+ self.in_proj = LinearColParallelMerged(hidden_size, self._in_proj_split, has_bias=False)
+ self.conv1d = _DepthwiseConv1d(self.conv_dim, conv_kernel_size)
+ # Recurrence-gating params kept in fp32 (exp/softplus is precision-sensitive,
+ # and the fla kernel reads them as fp32) -- matches HF/sglang, and avoids a
+ # per-call .float() upcast in the decode wrapper. The weight loader exempts
+ # *.A_log / *.dt_bias from the model-dtype downcast.
+ self.dt_bias = torch.empty(num_v_heads, dtype=torch.float32)
+ self.A_log = torch.empty(num_v_heads, dtype=torch.float32)
+ self.norm = _GatedRMSNorm(head_v_dim, eps=rms_norm_eps, activation=output_gate)
+ # out_proj follows the checkpoint quant: block-fp8 / per-tensor-fp8 / compressed-tensors
+ # NVFP4 (W4A16) / bf16. in_proj_* stay bf16 in every mode (above), so a compressed-tensors
+ # NVFP4 checkpoint (attn_quant=="nvfp4") only makes out_proj native FP4.
+ self.out_proj = make_replicated_quant(
+ expert_quant, attn_quant, self.value_dim, hidden_size, has_bias=False
+ )
+
+ def _gate_params(self, a: torch.Tensor, b: torch.Tensor):
+ beta = b.sigmoid()
+ g = -self.A_log.exp() * F.softplus(a.float() + self.dt_bias)
+ return g, beta
+
+ def _conv_weight(self) -> torch.Tensor:
+ return self.conv1d.weight.squeeze(1) # [conv_dim, kernel] for the fused kernel
+
+ def _conv_prefill(self, conv_in, pool, cu_seqlens, cache_indices, has_initial_state) -> torch.Tensor:
+ """Varlen causal conv (fused sgl_kernel) with silu; reads/updates each request's
+ conv state in place by ``cache_indices`` slot. ``conv_in`` [total, conv_dim].
+ ``cu_seqlens`` / ``cache_indices`` / ``has_initial_state`` come from FLAMetadata."""
+ li = pool.local_index(self.layer_id)
+ x = conv_in.transpose(0, 1).contiguous() # [conv_dim, total]
+ out = causal_conv1d_varlen(x, self._conv_weight(), pool.conv_states[li],
+ cu_seqlens, cache_indices, has_initial_state)
+ return out.transpose(0, 1) # [total, conv_dim]
+
+ def _conv_decode(self, conv_in: torch.Tensor, table_idx: torch.Tensor, pool) -> torch.Tensor:
+ """Single-token causal conv update (fused sgl_kernel) by ``table_idx`` slot;
+ updates conv state in place, no host loop -> CUDA-graph capturable.
+ ``conv_in`` [B, conv_dim] -> silu(conv) [B, conv_dim]."""
+ li = pool.local_index(self.layer_id)
+ return causal_conv1d_decode(conv_in, pool.conv_states[li], self._conv_weight(), table_idx)
+
+ def _write_track_snapshot(self, pool, li: int, conv_in: torch.Tensor,
+ h: torch.Tensor, fla) -> None:
+ """Snapshot this layer's recurrent + conv state at the chunk-aligned track boundary
+ into a donatable pool slot, on the forward stream (hybrid-radix extra_buffer path).
+ SSM: ``recurrent_states[li, dst] = h[0, h_row]`` -- a DIRECT copy (h is [V,K], the
+ state pool is [K,V]; they coincide because GDN requires head_k_dim == head_v_dim).
+ Conv: the last (kernel-1) raw conv-input timesteps ending at the boundary."""
+ rec = pool.recurrent_states[li]
+ rec.index_copy_(0, fla.track_dst, h[0, fla.track_h_row].to(rec.dtype))
+ cv = pool.conv_states[li]
+ # conv_in [total, conv_dim]; gather the (kernel-1) window per tracked req.
+ conv_win = conv_in[fla.track_conv_src].transpose(-1, -2).contiguous() # [nt, conv_dim, K-1]
+ cv.index_copy_(0, fla.track_dst, conv_win.to(cv.dtype))
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ ctx = get_global_ctx()
+ batch = ctx.batch
+ pool = ctx.linear_state_pool
+ total = hidden_states.shape[0]
+ dtype = hidden_states.dtype
+
+ # Per-forward GDN metadata (cu_seqlens / cache_indices / continuation flags),
+ # built once and shared by all GDN layers. The scheduler/graph set it; build it
+ # lazily here (cached on the batch) for direct-op callers (tests).
+ fla = batch.fla_metadata
+ if fla is None:
+ from freetoken.attention.linear import build_fla_metadata
+
+ fla = build_fla_metadata(batch, hidden_states.device)
+ batch.fla_metadata = fla
+
+ if self._fp8:
+ qkvz = self.in_proj_qkvz.forward(hidden_states)
+ conv_in, z = torch.split(qkvz, [self.conv_dim, self.value_dim], dim=-1)
+ ba = self.in_proj_ba.forward(hidden_states)
+ b, a = torch.split(ba, [self.num_v_heads, self.num_v_heads], dim=-1)
+ else:
+ proj = self.in_proj.forward(hidden_states)
+ conv_in, z, b, a = torch.split(proj, self._in_proj_split, dim=-1)
+ z = z.reshape(total, self.num_v_heads, self.head_v_dim)
+ li = pool.local_index(self.layer_id)
+
+ if batch.is_decode:
+ # Fused fla decode kernel: gating + in-kernel l2norm + recurrent update +
+ # per-request state read/write-by-index, all in one kernel (no gather/scatter,
+ # no clone, no external l2norm). q/k stay at num_k_heads (kernel handles GQA).
+ mixed = self._conv_decode(conv_in, fla.cache_indices, pool) # [B, conv_dim]
+ B = mixed.shape[0]
+ qf, kf, vf = torch.split(mixed, [self.key_dim, self.key_dim, self.value_dim], dim=-1)
+ q = qf.reshape(1, B, self.num_k_heads, self.head_k_dim).to(dtype)
+ k = kf.reshape(1, B, self.num_k_heads, self.head_k_dim).to(dtype)
+ v = vf.reshape(1, B, self.num_v_heads, self.head_v_dim).to(dtype)
+ core_out = gdn_decode_fla(
+ q, k, v, a, b, A_log=self.A_log, dt_bias=self.dt_bias,
+ state_source=pool.recurrent_states[li], indices=fla.cache_indices,
+ cu_seqlens=fla.cu_seqlens, scale=self.head_k_dim ** -0.5,
+ )
+ else:
+ mixed = self._conv_prefill(
+ conv_in, pool, fla.cu_seqlens, fla.cache_indices, fla.has_initial_state)
+ # fla chunk handles GQA in-kernel: q/k stay at num_k_heads, v at num_v_heads.
+ qf, kf, vf = torch.split(mixed, [self.key_dim, self.key_dim, self.value_dim], dim=-1)
+ q = qf.reshape(1, total, self.num_k_heads, self.head_k_dim).to(dtype)
+ k = kf.reshape(1, total, self.num_k_heads, self.head_k_dim).to(dtype)
+ v = vf.reshape(1, total, self.num_v_heads, self.head_v_dim).to(dtype)
+ g, beta = self._gate_params(a, b)
+ g = g.reshape(1, total, self.num_v_heads)
+ beta = beta.float().reshape(1, total, self.num_v_heads)
+ # The chunk kernel reads + writes back initial_state[cache_indices] in place;
+ # fresh sequences (cached_len==0) must start from a zeroed slot.
+ if fla.fresh_state_indices is not None:
+ pool.recurrent_states[li].index_fill_(0, fla.fresh_state_indices, 0.0)
+ track = fla.track_dst is not None
+ result = gdn_prefill_chunk_fla(
+ q, k, v, g, beta,
+ state_source=pool.recurrent_states[li], indices=fla.cache_indices,
+ cu_seqlens=fla.cu_seqlens, scale=self.head_k_dim ** -0.5,
+ return_h=track,
+ )
+ if track:
+ core_out, h = result
+ self._write_track_snapshot(pool, li, conv_in, h, fla)
+ else:
+ core_out = result
+
+ core_out = core_out.reshape(-1, self.head_v_dim)
+ z = z.reshape(-1, self.head_v_dim)
+ out = self.norm.forward(core_out, z).reshape(total, -1)
+ return self.out_proj.forward(out)
+
+
+__all__ = ["Qwen4ExpGatedDeltaNet"]
diff --git a/python/freetoken/models/qwen4_exp/gdn_reference.py b/python/freetoken/models/qwen4_exp/gdn_reference.py
new file mode 100644
index 00000000..fbd9647b
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/gdn_reference.py
@@ -0,0 +1,279 @@
+"""Pure-torch Gated DeltaNet reference (text-only, no cache).
+
+Correctness oracle for the kernel-backed GDN op (``gdn.Qwen4ExpGatedDeltaNet``).
+The two delta rules and the forward are transcribed from
+``transformers.models.qwen4_exp.modeling_qwen4_exp`` (``torch_chunk_gated_delta_rule``,
+``torch_recurrent_gated_delta_rule`` and ``Qwen4ExpTextGatedDeltaNet.forward`` no-cache path).
+Qwen3.8-Flash-Next gates the output norm with ``config.output_gate_type`` (sigmoid) where
+Qwen3.5 hardcodes silu; the conv keeps ``config.hidden_act`` (silu).
+"""
+
+from __future__ import annotations
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+_GATE_ACTS = {"silu": F.silu, "swish": F.silu, "sigmoid": torch.sigmoid}
+
+
+def _l2norm(x: torch.Tensor, eps: float = 1e-6) -> torch.Tensor:
+ return x * torch.rsqrt(x.pow(2).sum(dim=-1, keepdim=True) + eps)
+
+
+def recurrent_gated_delta_rule(
+ query: torch.Tensor, # [B, T, Hv, Dk]
+ key: torch.Tensor, # [B, T, Hv, Dk]
+ value: torch.Tensor, # [B, T, Hv, Dv]
+ g: torch.Tensor, # [B, T, Hv] (log-decay; per-step decay = exp(g))
+ beta: torch.Tensor, # [B, T, Hv]
+ *,
+ initial_state: torch.Tensor | None = None,
+ use_qk_l2norm: bool = True,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Verbatim port of HF ``torch_recurrent_gated_delta_rule`` (output_final_state=True)."""
+ initial_dtype = query.dtype
+ if use_qk_l2norm:
+ query = _l2norm(query, eps=1e-6)
+ key = _l2norm(key, eps=1e-6)
+ query, key, value, beta, g = [
+ t.transpose(1, 2).contiguous().to(torch.float32)
+ for t in (query, key, value, beta, g)
+ ]
+ b, h, t_len, dk = key.shape
+ dv = value.shape[-1]
+ scale = 1.0 / (dk ** 0.5)
+ query = query * scale
+
+ out = torch.zeros(b, h, t_len, dv, dtype=value.dtype, device=value.device)
+ state = (
+ torch.zeros(b, h, dk, dv, dtype=value.dtype, device=value.device)
+ if initial_state is None
+ else initial_state.to(value)
+ )
+ for i in range(t_len):
+ q_t = query[:, :, i]
+ k_t = key[:, :, i]
+ v_t = value[:, :, i]
+ g_t = g[:, :, i].exp().unsqueeze(-1).unsqueeze(-1)
+ beta_t = beta[:, :, i].unsqueeze(-1)
+ state = state * g_t
+ kv_mem = (state * k_t.unsqueeze(-1)).sum(dim=-2)
+ delta = (v_t - kv_mem) * beta_t
+ state = state + k_t.unsqueeze(-1) * delta.unsqueeze(-2)
+ out[:, :, i] = (state * q_t.unsqueeze(-1)).sum(dim=-2)
+
+ out = out.transpose(1, 2).contiguous().to(initial_dtype) # [B, T, Hv, Dv]
+ return out, state
+
+
+def chunk_gated_delta_rule(
+ query: torch.Tensor, # [B, T, Hv, Dk]
+ key: torch.Tensor, # [B, T, Hv, Dk]
+ value: torch.Tensor, # [B, T, Hv, Dv]
+ g: torch.Tensor, # [B, T, Hv]
+ beta: torch.Tensor, # [B, T, Hv]
+ *,
+ chunk_size: int = 64,
+ initial_state: torch.Tensor | None = None,
+ use_qk_l2norm: bool = True,
+) -> tuple[torch.Tensor, torch.Tensor]:
+ """Verbatim port of HF ``torch_chunk_gated_delta_rule`` (output_final_state=True).
+
+ Same recurrence as ``recurrent_gated_delta_rule`` in exact arithmetic; it is the form the
+ fla chunk kernel implements, so it is the closer oracle for the prefill path."""
+ initial_dtype = query.dtype
+ if use_qk_l2norm:
+ query = _l2norm(query, eps=1e-6)
+ key = _l2norm(key, eps=1e-6)
+ query, key, value, beta, g = [
+ x.transpose(1, 2).contiguous().to(torch.float32)
+ for x in (query, key, value, beta, g)
+ ]
+
+ batch_size, num_heads, sequence_length, k_head_dim = key.shape
+ v_head_dim = value.shape[-1]
+ pad_size = (chunk_size - sequence_length % chunk_size) % chunk_size
+ query = F.pad(query, (0, 0, 0, pad_size))
+ key = F.pad(key, (0, 0, 0, pad_size))
+ value = F.pad(value, (0, 0, 0, pad_size))
+ beta = F.pad(beta, (0, pad_size))
+ g = F.pad(g, (0, pad_size))
+ total_sequence_length = sequence_length + pad_size
+ scale = 1 / (query.shape[-1] ** 0.5)
+ query = query * scale
+
+ v_beta = value * beta.unsqueeze(-1)
+ k_beta = key * beta.unsqueeze(-1)
+ # reshape to chunks
+ query, key, value, k_beta, v_beta = [
+ x.reshape(x.shape[0], x.shape[1], -1, chunk_size, x.shape[-1])
+ for x in (query, key, value, k_beta, v_beta)
+ ]
+ g = g.reshape(g.shape[0], g.shape[1], -1, chunk_size)
+ mask = torch.triu(
+ torch.ones(chunk_size, chunk_size, dtype=torch.bool, device=query.device), diagonal=0
+ )
+
+ # chunk decay
+ g = g.cumsum(dim=-1)
+ decay_mask = ((g.unsqueeze(-1) - g.unsqueeze(-2)).tril().exp().float()).tril()
+ attn = -((k_beta @ key.transpose(-1, -2)) * decay_mask).masked_fill(mask, 0)
+ for i in range(1, chunk_size):
+ row = attn[..., i, :i].clone()
+ sub = attn[..., :i, :i].clone()
+ attn[..., i, :i] = row + (row.unsqueeze(-1) * sub).sum(-2)
+ attn = attn + torch.eye(chunk_size, dtype=attn.dtype, device=attn.device)
+ value = attn @ v_beta
+ k_cumdecay = attn @ (k_beta * g.exp().unsqueeze(-1))
+ last_recurrent_state = (
+ torch.zeros(
+ batch_size, num_heads, k_head_dim, v_head_dim,
+ dtype=value.dtype, device=value.device,
+ )
+ if initial_state is None
+ else initial_state.to(value)
+ )
+ core_attn_out = torch.zeros_like(value)
+
+ # for each chunk; decay_mask is already lower-triangular so no extra causal mask is needed
+ for i in range(0, total_sequence_length // chunk_size):
+ q_i, k_i, v_i = query[:, :, i], key[:, :, i], value[:, :, i]
+ attn = q_i @ k_i.transpose(-1, -2) * decay_mask[:, :, i]
+ v_prime = (k_cumdecay[:, :, i]) @ last_recurrent_state
+ v_new = v_i - v_prime
+ attn_inter = (q_i * g[:, :, i, :, None].exp()) @ last_recurrent_state
+ core_attn_out[:, :, i] = attn_inter + attn @ v_new
+ last_recurrent_state = (
+ last_recurrent_state * g[:, :, i, -1, None, None].exp()
+ + (k_i * (g[:, :, i, -1, None] - g[:, :, i]).exp()[..., None]).transpose(-1, -2)
+ @ v_new
+ )
+
+ core_attn_out = core_attn_out.reshape(
+ core_attn_out.shape[0], core_attn_out.shape[1], -1, core_attn_out.shape[-1]
+ )
+ core_attn_out = core_attn_out[:, :, :sequence_length]
+ core_attn_out = core_attn_out.transpose(1, 2).contiguous().to(initial_dtype)
+ return core_attn_out, last_recurrent_state
+
+
+class _RMSNormGated(nn.Module):
+ """RMSNorm of x followed by an ``activation(z)`` gate (norm_before_gate=True).
+
+ Mirrors ``Qwen4ExpTextRMSNormGated`` over head_v_dim groups.
+ """
+
+ def __init__(self, dim: int, eps: float, activation: str = "sigmoid"):
+ super().__init__()
+ self.weight = nn.Parameter(torch.ones(dim))
+ self.eps = eps
+ self.act = _GATE_ACTS[activation]
+
+ def forward(self, x: torch.Tensor, z: torch.Tensor) -> torch.Tensor:
+ in_dtype = x.dtype
+ x = x.float()
+ x = x * torch.rsqrt(x.pow(2).mean(dim=-1, keepdim=True) + self.eps)
+ x = x * self.weight.float()
+ x = x * self.act(z.float())
+ return x.to(in_dtype)
+
+
+class Qwen4ExpGatedDeltaNetReference(nn.Module):
+ """Pure-torch Gated DeltaNet (text-only, no cache)."""
+
+ def __init__(
+ self,
+ hidden_size: int,
+ num_k_heads: int,
+ num_v_heads: int,
+ head_k_dim: int,
+ head_v_dim: int,
+ conv_kernel_size: int,
+ rms_norm_eps: float,
+ hidden_act: str = "silu",
+ output_gate: str = "sigmoid",
+ ):
+ super().__init__()
+ if hidden_act != "silu":
+ raise ValueError(f"GDN reference only supports silu conv activation, got {hidden_act!r}")
+ if output_gate not in _GATE_ACTS:
+ raise ValueError(f"unsupported GDN output gate {output_gate!r}")
+ self.num_k_heads = num_k_heads
+ self.num_v_heads = num_v_heads
+ self.head_k_dim = head_k_dim
+ self.head_v_dim = head_v_dim
+ self.key_dim = num_k_heads * head_k_dim
+ self.value_dim = num_v_heads * head_v_dim
+ self.conv_dim = self.key_dim * 2 + self.value_dim
+ self.conv_kernel_size = conv_kernel_size
+
+ self.in_proj_qkv = nn.Linear(hidden_size, self.conv_dim, bias=False)
+ self.in_proj_z = nn.Linear(hidden_size, self.value_dim, bias=False)
+ self.in_proj_b = nn.Linear(hidden_size, num_v_heads, bias=False)
+ self.in_proj_a = nn.Linear(hidden_size, num_v_heads, bias=False)
+ self.conv1d = nn.Conv1d(
+ self.conv_dim, self.conv_dim, kernel_size=conv_kernel_size,
+ groups=self.conv_dim, padding=conv_kernel_size - 1, bias=False,
+ )
+ self.dt_bias = nn.Parameter(torch.zeros(num_v_heads))
+ self.A_log = nn.Parameter(torch.zeros(num_v_heads))
+ self.norm = _RMSNormGated(head_v_dim, eps=rms_norm_eps, activation=output_gate)
+ self.out_proj = nn.Linear(self.value_dim, hidden_size, bias=False)
+
+ @torch.no_grad()
+ def load_from_hf(self, hf_gdn) -> None:
+ """Copy weights from a transformers ``Qwen4ExpTextGatedDeltaNet``."""
+ self.in_proj_qkv.weight.copy_(hf_gdn.in_proj_qkv.weight)
+ self.in_proj_z.weight.copy_(hf_gdn.in_proj_z.weight)
+ self.in_proj_b.weight.copy_(hf_gdn.in_proj_b.weight)
+ self.in_proj_a.weight.copy_(hf_gdn.in_proj_a.weight)
+ # HF stores conv1d weight as [conv_dim, 1, K]; our depthwise Conv1d matches.
+ self.conv1d.weight.copy_(hf_gdn.conv1d.weight.view_as(self.conv1d.weight))
+ if hf_gdn.conv1d.bias is not None and self.conv1d.bias is not None:
+ self.conv1d.bias.copy_(hf_gdn.conv1d.bias)
+ self.dt_bias.copy_(hf_gdn.dt_bias)
+ self.A_log.copy_(hf_gdn.A_log)
+ self.norm.weight.copy_(hf_gdn.norm.weight)
+ self.out_proj.weight.copy_(hf_gdn.out_proj.weight)
+
+ def forward(self, hidden_states: torch.Tensor, *, use_chunk_rule: bool = False) -> torch.Tensor:
+ b, t_len, _ = hidden_states.shape
+
+ mixed_qkv = self.in_proj_qkv(hidden_states).transpose(1, 2) # [B, conv_dim, T]
+ z = self.in_proj_z(hidden_states).reshape(b, t_len, -1, self.head_v_dim)
+ a = self.in_proj_a(hidden_states)
+ bb = self.in_proj_b(hidden_states)
+
+ # causal depthwise conv + silu (drop the right padding back to T)
+ mixed_qkv = F.silu(self.conv1d(mixed_qkv)[..., :t_len]).transpose(1, 2) # [B, T, conv_dim]
+ query, key, value = torch.split(
+ mixed_qkv, [self.key_dim, self.key_dim, self.value_dim], dim=-1
+ )
+ query = query.reshape(b, t_len, -1, self.head_k_dim)
+ key = key.reshape(b, t_len, -1, self.head_k_dim)
+ value = value.reshape(b, t_len, -1, self.head_v_dim)
+
+ beta = bb.sigmoid()
+ g = -self.A_log.float().exp() * F.softplus(a.float() + self.dt_bias)
+
+ # GQA expand: replicate q/k heads up to num_v_heads
+ rep = self.num_v_heads // self.num_k_heads
+ if rep > 1:
+ query = query.repeat_interleave(rep, dim=2)
+ key = key.repeat_interleave(rep, dim=2)
+
+ rule = chunk_gated_delta_rule if use_chunk_rule else recurrent_gated_delta_rule
+ core, _ = rule(query, key, value, g, beta, use_qk_l2norm=True)
+
+ core = core.reshape(-1, self.head_v_dim)
+ z = z.reshape(-1, self.head_v_dim)
+ core = self.norm(core, z).reshape(b, t_len, -1)
+ return self.out_proj(core)
+
+
+__all__ = [
+ "Qwen4ExpGatedDeltaNetReference",
+ "chunk_gated_delta_rule",
+ "recurrent_gated_delta_rule",
+]
diff --git a/python/freetoken/models/qwen4_exp/hc.py b/python/freetoken/models/qwen4_exp/hc.py
new file mode 100644
index 00000000..6fed76c0
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/hc.py
@@ -0,0 +1,147 @@
+"""Hyper-connection (gated residual) blocks for Qwen3.8-Flash-Next.
+
+Every layer reads and writes ``hc_count`` residual streams packed as ``R [T, hc_count*hidden]``
+(stream outer, hidden inner -- the checkpoint layout). On CUDA the mix/combine bodies are the
+vendored vLLM Triton kernels (``kernel/triton/hc.py``: grouped_gemma_rmsnorm / hc_silu /
+hc_gate_mix / hc_combine) around two ``F.linear`` GEMMs; the pure-torch chain stays as the CPU
+path and as the reference the kernels are diffed against. Both keep fp32 intermediates and cast
+back at the store, so they agree to fp32 rounding.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+import torch.nn.functional as F
+from freetoken.kernel.triton.hc import (
+ grouped_gemma_rmsnorm,
+ hc_combine,
+ hc_gate_mix,
+ hc_silu,
+)
+from freetoken.layers import BaseOP, LinearReplicated
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+def grouped_plus_one_rms_norm(
+ x: torch.Tensor, weight: torch.Tensor, eps: float, num_groups: int
+) -> torch.Tensor:
+ """RMSNorm each of ``num_groups`` equal slices of the last dim on its own fp32 statistic, then scale by (1+w)."""
+ xf = x.float().unflatten(-1, (num_groups, -1))
+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+ return (xf.flatten(-2) * (1.0 + weight.float())).to(x.dtype)
+
+
+class GroupedPlusOneRMSNorm(BaseOP):
+ """Per-stream RMSNorm of an ``[..., num_groups*group]`` tensor with one weight element per feature.
+
+ HF ``Qwen4ExpTextRMSNorm(dim, group_size)``. The checkpoint weight is zero-centered and is
+ loaded RAW: (1+w) is applied at runtime in fp32, never folded into the bf16 weight (the
+ vendored Triton kernel does the same). ``ple.py`` reuses this class for norm_key /
+ norm_query / norm_conv, so keep it exported.
+ """
+
+ def __init__(self, size: int, eps: float, num_groups: int) -> None:
+ self.weight = torch.empty(size)
+ self.eps = eps
+ self.num_groups = num_groups
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ # the kernel is 2D-only, higher-rank callers keep the torch chain
+ if x.is_cuda and x.dim() == 2:
+ return grouped_gemma_rmsnorm(x, self.weight, self.eps, self.num_groups)
+ return grouped_plus_one_rms_norm(x, self.weight, self.eps, self.num_groups)
+
+
+class GatedResidual(BaseOP):
+ """One hyper-connection block: ``mix`` reads the residual streams, ``combine`` writes a block output back.
+
+ Frozen API (HF ``Qwen4ExpTextGatedResidual``, formulas at modeling_qwen4_exp.py:959-969)::
+
+ x, s = hc.mix(R) # R [T, hc_count*hidden] -> x [T, hidden], s [T, hc_count] or None
+ y = block(x) # attention / GDN / MoE, plain [T, hidden] -> [T, hidden]
+ R = hc.combine(R, y, s)
+
+ Rn = groupRMSNorm(R) * (1 + hc_norm.weight) # per hidden-size stream, fp32 stats
+ lora, s = input_mix_weight_down_block_inject(Rn) # merged GEMM: [lowrank | hc_count | pad]
+ gate = input_mix_weight_up(silu(lora / hc_count))
+ x = mean_i(sigmoid(gate_i) * Rn_i)
+ R'_i = R_i + 2*sigmoid(s_i / hc_count) * y
+
+ ``s`` is the RAW inject logit slice of the merged GEMM (pre 2*sigmoid), which is what the
+ vendored ``hc_combine`` kernel expects; ``combine`` applies the activation. The merged weight
+ is ``[lowrank + hc_count + pad, hc_count*hidden]`` (Qwen3.8: 320 + 4 + 12 = 336 rows), the pad
+ rows are zero and their GEMM output is dropped. ``use_combine=False`` is the top-level mixer:
+ it owns the unmerged ``input_mix_weight_down``, returns ``s = None`` and has no ``combine``.
+
+ Weight keys (checkpoint names, prefix stripped): ``hc_norm.weight``,
+ ``input_mix_weight_down_block_inject.weight`` (loader: concat of
+ ``input_mix_weight_down`` [lowrank, hc*hidden], ``block_inject_weight`` [hc_count, hc*hidden]
+ and ``pad`` zero rows), ``input_mix_weight_up.weight``.
+
+ Launch budget on CUDA: ``mix`` is 3 kernels around 2 GEMMs, ``combine`` is 1.
+ """
+
+ def __init__(self, config: ModelConfig, use_combine: bool = True) -> None:
+ args = config.qwen4_args
+ self.hc_count = args.hc_count
+ self.hidden_size = args.hidden_size
+ self.lowrank = args.hc_lowrank
+ self.use_combine = use_combine
+ width = args.ple_state_width
+ self.hc_norm = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
+ if use_combine:
+ # 16-row alignment for the merged skinny GEMM (vLLM hyperconnection.py:98)
+ self.pad_size = (-(self.lowrank + self.hc_count)) % 16
+ self.input_mix_weight_down_block_inject = LinearReplicated(
+ width, self.lowrank + self.hc_count + self.pad_size, has_bias=False
+ )
+ else:
+ self.pad_size = 0
+ self.input_mix_weight_down = LinearReplicated(width, self.lowrank, has_bias=False)
+ self.input_mix_weight_up = LinearReplicated(self.lowrank, width, has_bias=False)
+
+ def _down(self, rn: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:
+ """Run the down GEMM and split off the raw inject logits; the pad columns are dropped."""
+ if not self.use_combine:
+ return self.input_mix_weight_down.forward(rn), None
+ down = self.input_mix_weight_down_block_inject.forward(rn)
+ # both slices keep unit inner stride, so the kernels read them without a copy
+ return down[:, : self.lowrank], down[:, self.lowrank : self.lowrank + self.hc_count]
+
+ def _mix_kernel(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:
+ rn = grouped_gemma_rmsnorm(R, self.hc_norm.weight, self.hc_norm.eps, self.hc_count)
+ lora, s = self._down(rn)
+ gate = self.input_mix_weight_up.forward(hc_silu(lora, self.hc_count))
+ return hc_gate_mix(rn, gate, self.hc_count), s
+
+ def _mix_torch(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:
+ rn = grouped_plus_one_rms_norm(R, self.hc_norm.weight, self.hc_norm.eps, self.hc_count)
+ lora, s = self._down(rn)
+ lora = F.silu(lora.float() / self.hc_count)
+ gate = self.input_mix_weight_up.forward(lora.to(R.dtype))
+ mixed = torch.sigmoid(gate.float()).unflatten(-1, (self.hc_count, self.hidden_size))
+ mixed = mixed * rn.float().unflatten(-1, (self.hc_count, self.hidden_size))
+ return mixed.mean(-2).to(R.dtype), s
+
+ def _combine_torch(self, R: torch.Tensor, y: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
+ inject = 2.0 * torch.sigmoid(s.float() / self.hc_count)
+ out = R.float().unflatten(-1, (self.hc_count, self.hidden_size))
+ out = out + y.float().unsqueeze(-2) * inject.unsqueeze(-1)
+ return out.flatten(-2).to(R.dtype)
+
+ def mix(self, R: torch.Tensor) -> Tuple[torch.Tensor, torch.Tensor | None]:
+ """Return the block input ``x [T, hidden]`` and the inject logits ``s [T, hc_count]`` (None if no combine)."""
+ return self._mix_kernel(R) if R.is_cuda else self._mix_torch(R)
+
+ def combine(self, R: torch.Tensor, y: torch.Tensor, s: torch.Tensor) -> torch.Tensor:
+ """Inject the block output ``y [T, hidden]`` back into every stream of ``R``."""
+ if R.is_cuda:
+ return hc_combine(R, y, s, self.hc_count)
+ return self._combine_torch(R, y, s)
+
+
+__all__ = ["GatedResidual", "GroupedPlusOneRMSNorm", "grouped_plus_one_rms_norm"]
diff --git a/python/freetoken/models/qwen4_exp/model.py b/python/freetoken/models/qwen4_exp/model.py
new file mode 100644
index 00000000..e5b365aa
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/model.py
@@ -0,0 +1,187 @@
+"""Qwen3.8-Flash-Next decoder stack (text-only).
+
+The residual state is ``R [T, hc_count*hidden]`` end to end: the embedding is repeated over the
+``hc_count`` streams, every layer mixes them down to one ``[T, hidden]`` block input and injects
+its output back, and the top-level mixer collapses them once before ``lm_head``. There is no
+input/post layernorm and no final ``model.norm`` -- the hyper-connection norms are the only ones.
+
+Layer contract (frozen): ``forward(R [T, hc*hidden], batch) -> R' [T, hc*hidden]`` with an
+immediate combine::
+
+ R = R + ple(R, batch) # zero-based layer 1 only
+ x, s = attn_hc.mix(R); y = (GDN | QSA)(x); R = attn_hc.combine(R, y, s)
+ x, s = mlp_hc.mix(R); y = MoE(x); R = mlp_hc.combine(R, y, s)
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.layers import BaseOP, OPList, ParallelLMHead, VocabParallelEmbedding
+from freetoken.models.blocks import BaseLLMModel
+from freetoken.utils import nvtx_annotate
+
+from .attention import Qwen4ExpAttention
+from .hc import GatedResidual
+from .moe import Qwen4ExpMoE
+from .ple import PLELayer
+
+if TYPE_CHECKING:
+ from freetoken.core import Batch
+ from freetoken.models.config import ModelConfig
+
+
+def build_linear_mixer(config: ModelConfig, layer_id: int) -> BaseOP:
+ """GDN mixer of a linear_attention layer (Qwen3.5's GDN with a configurable output gate)."""
+ from .gdn import Qwen4ExpGatedDeltaNet
+
+ g = config.linear_attention_group()
+ return Qwen4ExpGatedDeltaNet(
+ hidden_size=config.hidden_size,
+ num_k_heads=g.num_key_heads,
+ num_v_heads=g.num_value_heads,
+ head_k_dim=g.key_head_dim,
+ head_v_dim=g.value_head_dim,
+ conv_kernel_size=g.conv_kernel_dim,
+ rms_norm_eps=config.rms_norm_eps,
+ layer_id=layer_id,
+ output_gate=g.output_gate,
+ # Qwen3.8's block-fp8 checkpoint keeps the GDN projections bf16 (only the routed
+ # experts are quantized), so do not let expert_quant flip them to Fp8Block.
+ expert_quant="none" if config.expert_quant == "fp8_block" else config.expert_quant,
+ attn_quant=config.attn_quant,
+ )
+
+
+class Qwen4ExpDecoderLayer(BaseOP):
+ """One decoder layer over the hyper-connection streams (see the module docstring for the flow)."""
+
+ def __init__(self, config: ModelConfig, layer_id: int) -> None:
+ self._layer_id = layer_id
+ self._is_linear = config.is_linear_layer(layer_id)
+ if self._is_linear:
+ self.linear_attn = build_linear_mixer(config, layer_id)
+ else:
+ self.self_attn = Qwen4ExpAttention(config, layer_id)
+ self.mlp = Qwen4ExpMoE(config, layer_id)
+ self.attn_hyper_connection = GatedResidual(config)
+ self.mlp_hyper_connection = GatedResidual(config)
+ self.ple = (
+ PLELayer(config, layer_id) if layer_id in config.qwen4_args.ple_layer_ids else None
+ )
+
+ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
+ def forward(self, hidden: torch.Tensor, batch: Batch) -> torch.Tensor:
+ if self.ple is not None:
+ hidden = hidden + self.ple.forward(hidden, batch)
+ block_input, inject = self.attn_hyper_connection.mix(hidden)
+ if self._is_linear:
+ block_output = self.linear_attn.forward(block_input)
+ else:
+ block_output = self.self_attn.forward(block_input, batch)
+ hidden = self.attn_hyper_connection.combine(hidden, block_output, inject)
+ block_input, inject = self.mlp_hyper_connection.mix(hidden)
+ return self.mlp_hyper_connection.combine(hidden, self.mlp.forward(block_input), inject)
+
+
+class Qwen4ExpModel(BaseOP):
+ def __init__(self, config: ModelConfig) -> None:
+ self.hc_count = config.qwen4_args.hc_count
+ self.embed_tokens = VocabParallelEmbedding(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ )
+ self.layers = OPList(
+ [Qwen4ExpDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ )
+ self.hyper_connection_mixer = GatedResidual(config, use_combine=False)
+ # plain tuple (not an OP child), so it never shows up in the state dict
+ self._ple = tuple(layer.ple for layer in self.layers.op_list if layer.ple is not None)
+
+ @property
+ def ple_layers(self) -> List[PLELayer]:
+ """The PLE layers in decoder order -- the seam the loader attaches table backends to."""
+ return list(self._ple)
+
+ def forward(self, input_ids: torch.Tensor, batch: Batch) -> torch.Tensor:
+ hidden = self.embed_tokens.forward(input_ids).repeat(1, self.hc_count)
+ meta = None
+ if self._ple:
+ from .ple import build_ple_metadata, commit_ngram_context
+
+ meta = build_ple_metadata(batch, self._ple[0].args, input_ids.device)
+ for ple in self._ple: # gather the pinned-host PLE rows while the early layers run
+ ple.start_prefetch(batch, meta)
+ for layer in self.layers.op_list:
+ hidden = layer.forward(hidden, batch)
+ if meta is not None:
+ # single writer: the layers only read the context, so a second PLE layer's
+ # prefetch sees the un-rolled window
+ commit_ngram_context(meta, getattr(batch, "fla_metadata", None))
+ return self.hyper_connection_mixer.mix(hidden)[0]
+
+
+class Qwen4ExpForCausalLM(BaseLLMModel):
+ def __init__(self, config: ModelConfig) -> None:
+ self._config = config
+ self.model = Qwen4ExpModel(config)
+ if getattr(config, "lm_head_quant", "none") == "nvfp4":
+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4LMHead
+
+ assert not config.tie_word_embeddings, "NVFP4 lm_head assumes untied embeddings"
+ self.lm_head = Nvfp4LMHead(
+ num_embeddings=config.vocab_size, embedding_dim=config.hidden_size
+ )
+ else:
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ )
+ super().__init__()
+
+ def load_host_tables(self, engine_config) -> int:
+ """Attach the PLE n-gram table (pinned checkpoint bank, or zeros for dummy weights); returns the pinned host bytes the engine reserves from its pin budget."""
+ ple_layers = self.model.ple_layers
+ if not ple_layers:
+ return 0
+ from .ple import PinnedUVATable, ZeroTable, derive_ngram_hash_constants
+
+ if getattr(engine_config, "use_dummy_weight", False):
+ # Dummy fill leaves the int64 hash buffers garbage (a zero vocab size divides by
+ # zero in the hash), so re-derive the real constants and read a zero table.
+ for ple in ple_layers:
+ args = ple.args
+ mult, sizes, offsets = derive_ngram_hash_constants(
+ vocab_size=self._config.vocab_size,
+ ngram_size=args.ngram_size,
+ num_ngram_heads=args.num_ngram_heads,
+ ngram_vocab_size_base=args.ngram_vocab_size_base,
+ ple_layer_index=ple.ple_index,
+ )
+ emb = ple.ple_embedding
+ emb.layer_multipliers.copy_(torch.tensor(mult, dtype=torch.int64))
+ emb.ngram_heads_vocab_sizes.copy_(torch.tensor(sizes, dtype=torch.int64))
+ emb.ngram_heads_offsets.copy_(torch.tensor(offsets, dtype=torch.int64))
+ emb.attach_table(ZeroTable(offsets[-1] + sizes[-1], args.ngram_head_dim))
+ return 0
+
+ from .weight import load_ple_table
+
+ table = load_ple_table(engine_config.model_path, self._config.qwen4_args)
+ self._ple_table = table # owns the pinned HostBank; keep it alive
+ for ple in ple_layers:
+ ple.ple_embedding.attach_table(
+ PinnedUVATable(table.bank.tensor, float(table.weight_scale))
+ )
+ return table.bank.nbytes
+
+ def forward(self) -> torch.Tensor:
+ batch = get_global_ctx().batch
+ return self.lm_head.forward(self.model.forward(batch.input_ids, batch))
+
+
+__all__ = ["Qwen4ExpDecoderLayer", "Qwen4ExpForCausalLM", "Qwen4ExpModel", "build_linear_mixer"]
diff --git a/python/freetoken/models/qwen4_exp/moe.py b/python/freetoken/models/qwen4_exp/moe.py
new file mode 100644
index 00000000..9ef65c0a
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/moe.py
@@ -0,0 +1,46 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import TYPE_CHECKING
+
+import torch
+from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid
+from freetoken.layers.moe import make_moe_layer
+from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class Qwen4ExpMoE(Qwen3_5MoE):
+ """Qwen3_5MoE with the shared-expert gate on triton instead of gemv + sigmoid + mul + add.
+
+ Same weights, same state dict. The gate reduction stays ahead of the routed experts, which may write into ``hidden_states`` in place.
+ """
+
+ def __init__(self, config: ModelConfig, layer_id: int | None = None) -> None:
+ if getattr(config, "expert_quant", "none") != "fp8_block":
+ super().__init__(config, layer_id=layer_id)
+ return
+ # Qwen3.8's block-fp8 checkpoint quantizes only the routed experts; the shared
+ # expert stays bf16, so hide expert_quant from _SharedExpert's fp8 branch and
+ # rebuild the routed experts with the fp8_block bank layout.
+ super().__init__(replace(config, expert_quant="none"), layer_id=layer_id)
+ self.experts = make_moe_layer(
+ config,
+ layer_id=layer_id,
+ renormalize=config.norm_topk_prob,
+ weight_format="fp8_block",
+ )
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ num_tokens, hidden_dim = hidden_states.shape
+ hidden_states = hidden_states.view(-1, hidden_dim)
+ router_logits = self.gate.forward(hidden_states)
+ shared = self.shared_expert.forward(hidden_states)
+ gate = shared_gate_sigmoid(hidden_states, self.shared_expert_gate.weight.view(-1))
+ routed = self.experts.forward(hidden_states=hidden_states, router_logits=router_logits)
+ return shared_gate_mul_add(routed, shared, gate).view(num_tokens, hidden_dim)
+
+
+__all__ = ["Qwen4ExpMoE"]
diff --git a/python/freetoken/models/qwen4_exp/ple.py b/python/freetoken/models/qwen4_exp/ple.py
new file mode 100644
index 00000000..5100229a
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/ple.py
@@ -0,0 +1,721 @@
+"""Per-Layer Embedding (PLE) for Qwen3.8-Flash-Next: hashed n-gram features injected at layer 1.
+
+HF reference: ``Qwen4ExpTextNGramEmbedding`` (modeling_qwen4_exp.py:1018) and
+``Qwen4ExpTextPLELayer`` (:1117). Per token::
+
+ E = table[hash(ngram)] # 16 heads (8 x 2-gram, 8 x 3-gram) x 160 -> 2560
+ K = norm_key(key_proj(E)).view(hc, hidden) # V = value_proj(E) [hidden]
+ Q = norm_query(R).view(hc, hidden)
+ u = <K_i, Q_i> / sqrt(hidden) # per stream
+ U = sigmoid(sign(u) * sqrt(max(|u|, 1e-6))) * V
+ D = U + silu(conv1d(norm_conv(U))) # depthwise, kernel 4, dilation ngram_size
+ R += D # before the attention hyper-connection mix
+
+The table is the 47.7 GiB FP8 n-gram store: ``PinnedUVATable`` keeps it in pinned host memory and
+gathers rows over UVA, optionally started early on a side stream (``PLELayer.start_prefetch``).
+``GpuResidentTable`` is the small-table oracle the pinned backend is diffed against.
+"""
+
+from __future__ import annotations
+
+import math
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List, Protocol, Sequence, Tuple
+
+import torch
+import torch.nn.functional as F
+from freetoken.core import get_global_ctx
+from freetoken.layers import BaseOP, LinearReplicated
+
+from .config import PLE_CONV_STATE, PLE_NGRAM_STATE
+from .hc import GroupedPlusOneRMSNorm
+
+if TYPE_CHECKING:
+ from freetoken.core import Batch
+ from freetoken.models.config import ModelConfig
+
+ from .config import Qwen4ExpArgs
+
+
+_MASK64 = (1 << 64) - 1
+_SPLITMIX_GAMMA = 0x9E3779B97F4A7C15
+_SPLITMIX_M1 = 0xBF58476D1CE4E5B9
+_SPLITMIX_M2 = 0x94D049BB133111EB
+_PLE_LAYER_PRIME = 10007
+
+
+class PLETableBackend(Protocol):
+ """Row store for one PLE layer's n-gram embedding table (Qwen3.8: 40M rows x 160, FP8 + one scalar scale).
+
+ Frozen contract. ``GpuResidentTable`` (oracle, small tables) and ``PinnedUVATable`` (the real 47.7 GiB pinned-host table) implement it. Rows are addressed by the
+ GLOBAL hashed id, i.e. the per-head vocab offset is already added by ``NGramEmbedding``.
+
+ ``lookup`` gets ``row_ids [T, num_ngram_heads]`` (int64, device) and returns
+ ``[T, num_ngram_heads * head_dim]`` in ``dtype``, already dequantized (fp8 -> dtype, times the
+ scalar weight_scale). ``out``, when given, is the destination and is returned as-is (CUDA-graph
+ decode reuses a fixed buffer).
+
+ ``prefetch`` may start the gather early on a side stream (the model issues it before layer 0 and
+ joins it in ``lookup``); a backend with no async path makes it a no-op.
+ """
+
+ num_rows: int
+ head_dim: int
+ dtype: torch.dtype
+
+ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor: ...
+
+ def prefetch(self, row_ids: torch.Tensor) -> None: ...
+
+
+class GpuResidentTable:
+ """PLE table held whole in GPU memory; ``index_select`` oracle for the pinned-host backend."""
+
+ def __init__(
+ self, weight: torch.Tensor, scale: float = 1.0, dtype: torch.dtype | None = None
+ ) -> None:
+ self.weight = weight
+ self.scale = float(scale)
+ self.num_rows, self.head_dim = weight.shape
+ self.dtype = dtype if dtype is not None else (
+ torch.bfloat16 if weight.dtype.itemsize < 2 else weight.dtype
+ )
+
+ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
+ rows = self.weight.index_select(0, row_ids.reshape(-1)).to(self.dtype)
+ if self.scale != 1.0:
+ rows = rows * self.scale
+ rows = rows.view(*row_ids.shape[:-1], -1)
+ if out is None:
+ return rows
+ out.copy_(rows)
+ return out
+
+ def prefetch(self, row_ids: torch.Tensor) -> None:
+ return None
+
+
+class ZeroTable:
+ """Dummy-weight stand-in: every lookup reads zeros (dummy checkpoints ship no table)."""
+
+ def __init__(self, num_rows: int, head_dim: int, dtype: torch.dtype = torch.bfloat16) -> None:
+ self.num_rows = int(num_rows)
+ self.head_dim = head_dim
+ self.dtype = dtype
+
+ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
+ if out is not None:
+ return out.zero_()
+ return torch.zeros(
+ (*row_ids.shape[:-1], row_ids.shape[-1] * self.head_dim),
+ dtype=self.dtype,
+ device=row_ids.device,
+ )
+
+ def prefetch(self, row_ids: torch.Tensor) -> None:
+ return None
+
+
+class PinnedUVATable:
+ """PLE table left in pinned host memory; rows are gathered over UVA by a Triton kernel.
+
+ ``weight`` must be the filled and ``pin()``ed ``HostBank.tensor`` from
+ ``weight.load_ple_table`` (``[num_rows, head_dim]``, fp8-e4m3 or bf16); an unregistered host
+ buffer is not device-addressable and the kernel faults on it. ``scale`` is the checkpoint's
+ scalar ``weight_scale``. Gathers emit bf16 into a staging buffer, one per captured decode size
+ and one growable buffer for everything else.
+
+ ``prefetch`` runs the gather on a private stream and the next ``lookup`` joins it. ``lookup``
+ returns a view of that staging buffer, so the rows must be consumed before the next lookup.
+ """
+
+ def __init__(
+ self,
+ weight: torch.Tensor,
+ scale: float = 1.0,
+ *,
+ device: torch.device | None = None,
+ prefetch: bool = True,
+ ) -> None:
+ assert weight.device.type == "cpu" and weight.is_contiguous()
+ assert weight.dtype in (torch.float8_e4m3fn, torch.bfloat16), weight.dtype
+ from freetoken.kernel.pinned import device_ptr
+
+ self.weight = weight
+ self.scale = float(scale)
+ self.num_rows, self.head_dim = weight.shape
+ self.dtype = torch.bfloat16
+ self._is_fp8 = weight.dtype == torch.float8_e4m3fn
+ self._device = device or torch.device("cuda", torch.cuda.current_device())
+ # WDDM maps registered host memory at a different device address; on Linux/UVA this is data_ptr
+ self._table_ptr = device_ptr(weight)
+ self._stream = torch.cuda.Stream(device=self._device) if prefetch else None
+ self._staging: torch.Tensor | None = None
+ self._graph_staging: dict[int, torch.Tensor] = {}
+ self._pending: Tuple[torch.Tensor, torch.Tensor] | None = None
+
+ def _stage(self, rows: int) -> torch.Tensor:
+ # Captured graphs keep one buffer per size for good: growing the eager one would free the
+ # block a replay still writes to.
+ if torch.cuda.is_current_stream_capturing():
+ buf = self._graph_staging.get(rows)
+ if buf is None:
+ buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device)
+ self._graph_staging[rows] = buf
+ return buf
+ buf = self._staging
+ if buf is None or buf.shape[0] < rows:
+ buf = torch.empty((rows, self.head_dim), dtype=self.dtype, device=self._device)
+ self._staging = buf
+ return buf[:rows]
+
+ def _gather(self, row_ids: torch.Tensor, dst: torch.Tensor) -> torch.Tensor:
+ from freetoken.kernel.triton.ple import ple_gather_rows
+
+ return ple_gather_rows(
+ self._table_ptr,
+ self.num_rows,
+ self.head_dim,
+ row_ids.reshape(-1),
+ dst,
+ self.scale,
+ self._is_fp8,
+ )
+
+ def prefetch(self, row_ids: torch.Tensor) -> None:
+ if self._stream is None or row_ids.numel() == 0:
+ return
+ dst = self._stage(row_ids.numel())
+ self._stream.wait_stream(torch.cuda.current_stream(self._device))
+ if not torch.cuda.is_current_stream_capturing():
+ row_ids.record_stream(self._stream)
+ with torch.cuda.stream(self._stream):
+ self._gather(row_ids, dst)
+ self._pending = (row_ids, dst)
+
+ def lookup(self, row_ids: torch.Tensor, out: torch.Tensor | None = None) -> torch.Tensor:
+ pending, self._pending = self._pending, None
+ if pending is not None:
+ # join even on a miss: the stale prefetch owns the staging buffer about to be reused
+ torch.cuda.current_stream(self._device).wait_stream(self._stream)
+ if pending is not None and pending[0] is row_ids:
+ rows = pending[1]
+ else:
+ rows = self._gather(row_ids, self._stage(row_ids.numel()))
+ rows = rows.view(*row_ids.shape[:-1], -1)
+ if out is None:
+ return rows
+ out.copy_(rows)
+ return out
+
+
+def _splitmix64(value: int) -> int:
+ value = (value + _SPLITMIX_GAMMA) & _MASK64
+ value = ((value ^ (value >> 30)) * _SPLITMIX_M1) & _MASK64
+ value = ((value ^ (value >> 27)) * _SPLITMIX_M2) & _MASK64
+ return (value ^ (value >> 31)) & _MASK64
+
+
+def _is_prime(value: int) -> bool:
+ if value < 2:
+ return False
+ if value % 2 == 0:
+ return value == 2
+ for divisor in range(3, math.isqrt(value) + 1, 2):
+ if value % divisor == 0:
+ return False
+ return True
+
+
+def _nth_prime_after(start: int, count: int) -> int:
+ prime = start
+ for _ in range(count):
+ prime += 1
+ while not _is_prime(prime):
+ prime += 1
+ return prime
+
+
+def derive_ngram_hash_constants(
+ *,
+ vocab_size: int,
+ ngram_size: int,
+ num_ngram_heads: int,
+ ngram_vocab_size_base: int,
+ ple_layer_index: int,
+ seed: int = 1234,
+) -> Tuple[List[int], List[int], List[int]]:
+ """Recompute (multipliers, per-head vocab sizes, per-head offsets) the way HF derives them at init.
+
+ The checkpoint ships these as int64 tensors, so serving loads them; this is the dummy-weight
+ path and the oracle a loader test can check the checkpoint values against.
+ """
+ half_bound = max(1, ((1 << 63) - 1) // max(vocab_size, 1) // 2)
+ base_seed = seed + _PLE_LAYER_PRIME * ple_layer_index
+ multipliers = [
+ 2 * (_splitmix64((base_seed + _SPLITMIX_GAMMA * (i + 1)) & _MASK64) % half_bound) + 1
+ for i in range(ngram_size)
+ ]
+ sizes: List[int] = []
+ offsets: List[int] = []
+ total = 0
+ for head in range(num_ngram_heads):
+ global_head = ple_layer_index * num_ngram_heads + head
+ size = _nth_prime_after(ngram_vocab_size_base - 1, global_head + 1)
+ sizes.append(size)
+ offsets.append(total)
+ total += size
+ return multipliers, sizes, offsets
+
+
+@dataclass
+class PLEMetadata:
+ """Per-forward PLE inputs, built once and shared by every PLE layer (sibling of ``FLAMetadata``).
+
+ Frozen contract:
+ input_ids [T] int device -- this forward's tokens, ragged, concatenated in request order
+ cu_seqlens [B+1] int device -- query indptr; decode is ``arange(B+1)``
+ seq_lens host per-request token counts; avoids a device sync in the ragged conv loop
+ ngram_context [B, ngram_size-1] int64 device -- the tokens immediately BEFORE each request's
+ first token of this forward, read from the ``ple_ngram_ctx`` slot state and
+ forced to the boundary (eos) id for fresh rows. The hash never crosses eos,
+ so a fresh sequence passes all-eos.
+ state_slots [B] int64 device -- linear-state slot per request (``Req.linear_slot_idx`` or
+ ``Req.table_idx``); keys every PLE slot state
+ fresh_slots [B] bool device or None -- request starts a new sequence, so read a zero state
+ is_decode one token per request (the batched 4-tap path)
+ """
+
+ input_ids: torch.Tensor
+ cu_seqlens: torch.Tensor
+ seq_lens: Sequence[int]
+ ngram_context: torch.Tensor
+ state_slots: torch.Tensor
+ fresh_slots: torch.Tensor | None
+ is_decode: bool
+
+
+def _state_slot(req) -> int:
+ slot = getattr(req, "linear_slot_idx", None)
+ return req.table_idx if slot is None else slot
+
+
+def _ngram_context_pool() -> torch.Tensor:
+ pool = get_global_ctx().linear_state_pool
+ assert pool is not None and pool.has_slot_state(PLE_NGRAM_STATE), (
+ "PLE needs the ple_ngram_ctx slot state (or an explicit context_pool=)"
+ )
+ return pool.slot_state(PLE_NGRAM_STATE)
+
+
+def build_ple_metadata(
+ batch: Batch,
+ args: Qwen4ExpArgs,
+ device: torch.device,
+ context_pool: torch.Tensor | None = None,
+) -> PLEMetadata:
+ """Build ``PLEMetadata`` from a scheduler batch.
+
+ The n-gram context is per-request device state (``ple_ngram_ctx`` [num_slots, ngram_size-1],
+ rolled forward once per forward by ``commit_ngram_context``), so it never lags the sampled
+ token under overlap scheduling and follows the slot on COW/snapshot. A decode batch reads it
+ straight off the persistent ``linear_table_idx`` buffer, so the build is capture-safe and
+ sync-free. Reuses ``batch.fla_metadata`` (slots / indptr / fresh mask) when the scheduler
+ built it.
+ """
+ reqs = batch.padded_reqs
+ ctx_len = args.ngram_size - 1
+ eos = args.ngram_boundary_token_id
+ if context_pool is None:
+ context_pool = _ngram_context_pool()
+ assert context_pool.shape[-1] == ctx_len, (
+ f"ple_ngram_ctx holds {context_pool.shape[-1]} ids, config wants {ctx_len}"
+ )
+ fla = getattr(batch, "fla_metadata", None)
+ slots_dev = getattr(batch, "linear_table_idx", None)
+
+ if batch.is_decode and slots_dev is not None:
+ slots = slots_dev.long()
+ bs = slots.numel()
+ return PLEMetadata(
+ input_ids=batch.input_ids,
+ cu_seqlens=torch.arange(bs + 1, dtype=torch.int32, device=device),
+ seq_lens=(1,) * bs,
+ ngram_context=context_pool.index_select(0, slots).long(),
+ state_slots=slots,
+ fresh_slots=None,
+ is_decode=True,
+ )
+
+ lens = [r.extend_len for r in reqs]
+ if fla is not None and fla.has_initial_state is not None:
+ cu = fla.cu_seqlens
+ slots = fla.cache_indices.long()
+ fresh = ~fla.has_initial_state
+ else: # direct-op callers (tests) with no scheduler metadata
+ pin = {"device": "cpu", "pin_memory": torch.cuda.is_available()}
+ cu = torch.tensor([0, *lens], dtype=torch.int64, **pin).cumsum_(0).to(device, non_blocking=True)
+ slots = torch.tensor([_state_slot(r) for r in reqs], dtype=torch.int64, **pin).to(device, non_blocking=True)
+ fresh = torch.tensor([r.cached_len == 0 for r in reqs], dtype=torch.bool, **pin).to(device, non_blocking=True)
+ context = context_pool.index_select(0, slots).long()
+ context = torch.where(fresh.unsqueeze(1), context.new_full((), eos), context)
+ return PLEMetadata(
+ input_ids=batch.input_ids,
+ cu_seqlens=cu,
+ seq_lens=tuple(lens),
+ ngram_context=context,
+ state_slots=slots,
+ fresh_slots=fresh,
+ is_decode=batch.is_decode,
+ )
+
+
+def commit_ngram_context(meta: PLEMetadata, fla, context_pool: torch.Tensor | None = None) -> None:
+ """Roll each request's ``ple_ngram_ctx`` forward past this forward's tokens.
+
+ Called ONCE per forward after every PLE layer ran (the layers only read the context);
+ also writes the boundary-aligned window to the track slot so a donated snapshot restores
+ the context together with the conv state. Pure device arithmetic, capture-safe.
+ """
+ if context_pool is None:
+ context_pool = _ngram_context_pool()
+ ids = meta.input_ids.long()
+ ctx_len = meta.ngram_context.shape[1]
+ steps = torch.arange(ctx_len, device=ids.device)
+ if meta.is_decode:
+ nxt = torch.cat([meta.ngram_context[:, 1:], ids.view(-1, 1)], dim=1)
+ else:
+ cu = meta.cu_seqlens.long()
+ cand = cu[1:].unsqueeze(1) - ctx_len + steps
+ # short extends fall back to the old context: token j of the new window sits at
+ # old-context column extend_len + j when it predates this forward
+ old = meta.ngram_context.gather(
+ 1, ((cu[1:] - cu[:-1]).unsqueeze(1) + steps).clamp_(max=ctx_len - 1)
+ )
+ nxt = torch.where(cand >= cu[:-1].unsqueeze(1), ids[cand.clamp_min(0)], old)
+ context_pool.index_copy_(0, meta.state_slots, nxt.to(context_pool.dtype))
+ if fla is not None and fla.track_boundary_row is not None:
+ win = ids[fla.track_boundary_row.unsqueeze(1) - ctx_len + steps]
+ context_pool.index_copy_(0, fla.track_dst, win.to(context_pool.dtype))
+
+
+class NGramEmbedding(BaseOP):
+ """Hashed n-gram lookup: splitmix64 mix of the last n token ids -> per-head prime vocab -> table rows.
+
+ Weight keys (checkpoint names): ``layer_multipliers`` [ngram_size], ``ngram_heads_vocab_sizes``
+ and ``ngram_heads_offsets`` [num_ngram_heads], all int64. The table itself is NOT a state-dict
+ entry (128 checkpoint shards land in a ``PLETableBackend``); attach it with ``attach_table``.
+ """
+
+ def __init__(self, args: Qwen4ExpArgs, table: PLETableBackend | None = None) -> None:
+ self.ngram_size = args.ngram_size
+ self.heads_per_ngram = args.heads_per_ngram
+ self.num_heads = args.num_ngram_heads
+ self.eos_token_id = args.ngram_boundary_token_id
+ self.layer_multipliers = torch.empty(args.ngram_size, dtype=torch.int64)
+ self.ngram_heads_vocab_sizes = torch.empty(self.num_heads, dtype=torch.int64)
+ self.ngram_heads_offsets = torch.empty(self.num_heads, dtype=torch.int64)
+ self._table = table
+
+ def attach_table(self, table: PLETableBackend) -> None:
+ self._table = table
+
+ @property
+ def table(self) -> PLETableBackend:
+ assert self._table is not None, "PLE table backend was never attached"
+ return self._table
+
+ def _window(self, meta: PLEMetadata):
+ """The hash window as ``(packed [B, W], select)``, where ``select`` picks this forward's tokens."""
+ ids = meta.input_ids.long()
+ ctx_len = self.ngram_size - 1
+ if meta.is_decode:
+ # a window of exactly ngram_size columns holds every shift the hash can reach
+ return torch.cat([meta.ngram_context, ids.view(-1, 1)], dim=1), lambda t: t[:, -1]
+
+ num_reqs = len(meta.seq_lens)
+ width = ctx_len + max(meta.seq_lens)
+ cu = meta.cu_seqlens.long()
+ # Pack the ragged tokens into [B, ctx+max_len] so the shift/boundary logic is one gather.
+ flat_pos = torch.arange(ids.numel(), device=ids.device)
+ req = (torch.searchsorted(cu, flat_pos, right=True) - 1).clamp_(max=num_reqs - 1)
+ col = flat_pos - cu[req] + ctx_len
+ packed = ids.new_full((num_reqs, width), self.eos_token_id)
+ packed[:, :ctx_len] = meta.ngram_context
+ packed[req, col] = ids
+ return packed, lambda t: t[req, col]
+
+ def _shift_ignore_eos(self, packed: torch.Tensor) -> List[torch.Tensor]:
+ """``out[s][b, p]`` = the token ``s`` places left of ``p``, or eos when the window crosses a boundary."""
+ num_reqs, width = packed.shape
+ pos = torch.arange(width, device=packed.device)
+ eos_pos = torch.where(packed == self.eos_token_id, pos, -1)
+ prev_eos = torch.cummax(eos_pos, dim=1).values
+ prev_eos = torch.cat([eos_pos.new_full((num_reqs, 1), -1), prev_eos[:, :-1]], dim=1)
+ in_segment = pos.unsqueeze(0) - prev_eos - 1
+
+ shifted = [packed]
+ for shift in range(1, self.ngram_size):
+ src = pos - shift
+ gathered = packed.gather(1, src.clamp_min(0).unsqueeze(0).expand(num_reqs, -1))
+ valid = (src.unsqueeze(0) >= 0) & (in_segment >= shift)
+ shifted.append(torch.where(valid, gathered, packed.new_full((), self.eos_token_id)))
+ return shifted
+
+ def row_ids(self, meta: PLEMetadata) -> torch.Tensor:
+ """Global table row per (token, hash head): ``[T, num_ngram_heads]`` int64."""
+ packed, select = self._window(meta)
+ tokens = [select(s) for s in self._shift_ignore_eos(packed)]
+ blocks = []
+ for ngram in range(2, self.ngram_size + 1):
+ start = (ngram - 2) * self.heads_per_ngram
+ end = start + self.heads_per_ngram
+ mixed = tokens[0] * self.layer_multipliers[0]
+ for position in range(1, ngram):
+ mixed = torch.bitwise_xor(mixed, tokens[position] * self.layer_multipliers[position])
+ head_ids = torch.remainder(mixed.unsqueeze(-1), self.ngram_heads_vocab_sizes[start:end])
+ blocks.append(head_ids + self.ngram_heads_offsets[start:end])
+ return torch.cat(blocks, dim=-1)
+
+ def forward(self, meta: PLEMetadata, out: torch.Tensor | None = None) -> torch.Tensor:
+ return self.table.lookup(self.row_ids(meta), out)
+
+
+class _DepthwiseConv1d(BaseOP):
+ """Holds the depthwise conv weight ``[width, 1, kernel]`` (key ``conv1d.weight``)."""
+
+ def __init__(self, width: int, kernel: int) -> None:
+ self.weight = torch.empty(width, 1, kernel)
+
+
+def short_conv_reference(
+ x: torch.Tensor,
+ meta: PLEMetadata,
+ states: torch.Tensor,
+ weight: torch.Tensor,
+ dilation: int,
+) -> torch.Tensor:
+ """Per-request ``F.conv1d`` over ``[state | chunk]``, advancing ``states`` in place.
+
+ Transcription of the HF conv; the shipping paths (one packed conv for prefill, a tap read for
+ decode) are diffed against it.
+ """
+ groups = weight.shape[0]
+ state_len = states.shape[-1]
+ slots = meta.state_slots
+ state = states.index_select(0, slots).to(x.dtype)
+ if meta.fresh_slots is not None:
+ state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state)
+
+ outs = []
+ new_state = torch.empty_like(state)
+ offset = 0
+ for i, n in enumerate(meta.seq_lens):
+ chunk = x[offset : offset + n].transpose(0, 1).unsqueeze(0)
+ history = torch.cat([state[i : i + 1], chunk], dim=-1)
+ out = F.conv1d(history, weight, groups=groups, dilation=dilation)
+ outs.append(out.squeeze(0).transpose(0, 1))
+ new_state[i] = history[0, :, -state_len:]
+ offset += n
+ states.index_copy_(0, slots, new_state.to(states.dtype))
+ return F.silu(torch.cat(outs, dim=0))
+
+
+class PLELayer(BaseOP):
+ """PLE block: hashed n-gram value gated by the residual streams, then a dilated depthwise conv.
+
+ ``forward(R, batch) -> D [T, hc_count*hidden]``; the caller adds ``D`` to ``R`` before the
+ attention hyper-connection mix. ``meta`` defaults to ``build_ple_metadata(batch, ...)``;
+ ``conv_states`` defaults to ``ctx.linear_state_pool.slot_state("ple_conv", layer_id)`` and is
+ ``[num_slots, hc_count*hidden, (ple_conv_kernel_size-1)*ngram_size]`` in the model dtype -- the
+ last conv-input columns per request, oldest first. Both are arguments so the reference is
+ testable before the pool and the scheduler carry them.
+
+ ``start_prefetch(batch)`` builds the metadata and starts the table gather on the backend's side
+ stream; call it at the top of the model forward so the rows land while layer 0 runs, and
+ ``forward`` joins it.
+
+ Weight keys (checkpoint names, prefix stripped): ``key_proj.weight`` [hc*hidden, ple_embed_dim],
+ ``value_proj.weight`` [hidden, ple_embed_dim], ``norm_key/norm_query/norm_conv.weight``
+ [hc*hidden] (zero-centered, loaded RAW), ``conv1d.weight`` [hc*hidden, 1, kernel], plus the
+ three ``ple_embedding`` int64 hash buffers.
+ """
+
+ def __init__(
+ self, config: ModelConfig, layer_id: int, table: PLETableBackend | None = None
+ ) -> None:
+ args = config.qwen4_args
+ self.args = args
+ self.layer_id = layer_id
+ self.ple_index = args.ple_layer_ids.index(layer_id)
+ self.hc_count = args.hc_count
+ self.hidden_size = args.hidden_size
+ self.dilation = args.ple_conv_dilation
+ self.state_len = args.ple_conv_state_len
+ width = args.ple_state_width
+ self.ple_embedding = NGramEmbedding(args, table)
+ self.key_proj = LinearReplicated(args.ple_embed_dim, width, has_bias=False)
+ self.value_proj = LinearReplicated(args.ple_embed_dim, args.hidden_size, has_bias=False)
+ self.norm_key = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
+ self.norm_query = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
+ self.norm_conv = GroupedPlusOneRMSNorm(width, config.rms_norm_eps, self.hc_count)
+ self.conv1d = _DepthwiseConv1d(width, args.ple_conv_kernel_size)
+ from freetoken.kernel.fla.chunk import CHUNK_SIZE
+
+ # the track snapshot gathers the last state_len conv inputs before a xCHUNK boundary; a longer history would reach before the forward's first token
+ assert self.state_len <= CHUNK_SIZE, (
+ f"PLE conv history {self.state_len} exceeds CHUNK_SIZE {CHUNK_SIZE}"
+ )
+ self._pending: Tuple[PLEMetadata, torch.Tensor] | None = None
+
+ def start_prefetch(self, batch: Batch, meta: PLEMetadata | None = None) -> None:
+ """Hash this forward's n-grams and start the table gather on the side stream."""
+ if meta is None:
+ meta = build_ple_metadata(batch, self.args, batch.input_ids.device)
+ row_ids = self.ple_embedding.row_ids(meta)
+ self._pending = (meta, row_ids)
+ self.ple_embedding.table.prefetch(row_ids)
+
+ def forward(
+ self,
+ R: torch.Tensor,
+ batch: Batch,
+ meta: PLEMetadata | None = None,
+ conv_states: torch.Tensor | None = None,
+ ) -> torch.Tensor:
+ pending, self._pending = self._pending, None
+ row_ids = None
+ if meta is None:
+ if pending is not None:
+ meta, row_ids = pending
+ else:
+ meta = build_ple_metadata(batch, self.args, R.device)
+ elif pending is not None and pending[0] is meta:
+ row_ids = pending[1]
+ if row_ids is None:
+ row_ids = self.ple_embedding.row_ids(meta)
+
+ embeddings = self.ple_embedding.table.lookup(row_ids).to(R.dtype)
+ key = self.norm_key.forward(self.key_proj.forward(embeddings))
+ value = self.value_proj.forward(embeddings)
+ query = self.norm_query.forward(R)
+ shape = (-1, self.hc_count, self.hidden_size)
+ gate = (key.view(shape) * query.view(shape)).sum(-1, keepdim=True) / math.sqrt(self.hidden_size)
+ gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())
+ gated = (gate * value.unsqueeze(-2)).flatten(-2)
+ states = conv_states if conv_states is not None else self._conv_state_slab(R)
+ x = self.norm_conv.forward(gated)
+ fla = getattr(batch, "fla_metadata", None)
+ if fla is not None and fla.track_boundary_row is not None:
+ self._write_track_snapshot(states, x, fla)
+ return gated + self._short_conv(x, meta, states)
+
+ def _write_track_snapshot(self, states: torch.Tensor, x: torch.Tensor, fla) -> None:
+ """Copy the conv history at the GDN track boundary into the same donatable slot, so a radix
+ prefix hit restores PLE and GDN state together. Track slots never alias the live slots this
+ forward advances, so the two writes are order-independent."""
+ src = fla.track_boundary_row.unsqueeze(1) + torch.arange(
+ -self.state_len, 0, device=x.device
+ )
+ window = x[src].transpose(-1, -2).contiguous()
+ states.index_copy_(0, fla.track_dst, window.to(states.dtype))
+
+ def _conv_state_slab(self, R: torch.Tensor) -> torch.Tensor:
+ pool = get_global_ctx().linear_state_pool
+ assert pool is not None, "PLE needs ctx.linear_state_pool or an explicit conv_states"
+ assert pool.has_slot_state(PLE_CONV_STATE), (
+ "ModelConfig.slot_states does not declare the PLE conv history"
+ )
+ return pool.slot_state(PLE_CONV_STATE, self.layer_id)
+
+ def _read_state(
+ self, meta: PLEMetadata, states: torch.Tensor, dtype: torch.dtype
+ ) -> torch.Tensor:
+ state = states.index_select(0, meta.state_slots).to(dtype)
+ if meta.fresh_slots is not None:
+ state = torch.where(meta.fresh_slots.view(-1, 1, 1), torch.zeros_like(state), state)
+ return state
+
+ def _short_conv(
+ self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor
+ ) -> torch.Tensor:
+ """silu of the dilated depthwise conv over [state | x], and roll the per-request state."""
+ if meta.is_decode:
+ return self._decode_conv(x, meta, states)
+ return self._prefill_conv(x, meta, states)
+
+ def _decode_conv(
+ self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor
+ ) -> torch.Tensor:
+ """Batched tap read: taps t-9, t-6, t-3 come off the state slab, tap t from this token."""
+ state = self._read_state(meta, states, x.dtype)
+ column = x.unsqueeze(-1)
+ # fp32 products, like the conv1d the prefill path runs
+ window = torch.cat([state[..., :: self.dilation], column], dim=-1).float()
+ out = (window * self.conv1d.weight.squeeze(1).float()).sum(-1)
+ states.index_copy_(
+ 0, meta.state_slots, torch.cat([state[..., 1:], column], dim=-1).to(states.dtype)
+ )
+ return F.silu(out.to(x.dtype))
+
+ def _prefill_conv(
+ self, x: torch.Tensor, meta: PLEMetadata, states: torch.Tensor
+ ) -> torch.Tensor:
+ """One conv over every request packed as ``[state_0 | chunk_0 | state_1 | chunk_1 | ...]``.
+
+ The blocks abut exactly, so each output window stays inside its own request: request i's
+ first token reads history columns base_i .. base_i+state_len, which is its own state.
+ """
+ lens = list(meta.seq_lens)
+ num_reqs, width = len(lens), x.shape[1]
+ out_index, state_index, next_state_index = self._prefill_indices(lens, x.device)
+
+ state = self._read_state(meta, states, x.dtype)
+ history = x.new_empty(width, x.shape[0] + num_reqs * self.state_len)
+ history.index_copy_(1, state_index, state.permute(1, 0, 2).reshape(width, -1))
+ history.index_copy_(1, out_index + self.state_len, x.transpose(0, 1).contiguous())
+
+ out = F.conv1d(
+ history.unsqueeze(0), self.conv1d.weight, groups=width, dilation=self.dilation
+ ).squeeze(0)
+ new_state = history.index_select(1, next_state_index).view(width, num_reqs, self.state_len)
+ states.index_copy_(
+ 0, meta.state_slots, new_state.permute(1, 0, 2).to(states.dtype).contiguous()
+ )
+ return F.silu(out.index_select(1, out_index).transpose(0, 1))
+
+ def _prefill_indices(self, lens: List[int], device: torch.device):
+ """Columns of the packed history: this forward's outputs, the state block, the next state block."""
+ state_len = self.state_len
+ counts = torch.tensor(lens, dtype=torch.int64)
+ cu = torch.cat([counts.new_zeros(1), counts.cumsum(0)])
+ pad = torch.arange(len(lens), dtype=torch.int64) * state_len
+ base = cu[:-1] + pad
+ out_index = torch.arange(int(cu[-1])) + torch.repeat_interleave(pad, counts)
+ span = torch.arange(state_len, dtype=torch.int64)
+ packed = torch.cat(
+ [
+ out_index,
+ (base.unsqueeze(1) + span).reshape(-1),
+ ((base + counts).unsqueeze(1) + span).reshape(-1),
+ ]
+ )
+ if torch.cuda.is_available():
+ packed = packed.pin_memory()
+ packed = packed.to(device, non_blocking=True)
+ n_out, n_state = out_index.numel(), len(lens) * state_len
+ return packed[:n_out], packed[n_out : n_out + n_state], packed[n_out + n_state :]
+
+
+__all__ = [
+ "GpuResidentTable",
+ "NGramEmbedding",
+ "ZeroTable",
+ "PLELayer",
+ "PLEMetadata",
+ "PLETableBackend",
+ "PinnedUVATable",
+ "build_ple_metadata",
+ "derive_ngram_hash_constants",
+ "short_conv_reference",
+]
diff --git a/python/freetoken/models/qwen4_exp/weight.py b/python/freetoken/models/qwen4_exp/weight.py
new file mode 100644
index 00000000..f8d2a749
--- /dev/null
+++ b/python/freetoken/models/qwen4_exp/weight.py
@@ -0,0 +1,328 @@
+"""Qwen3.8-Flash-Next (RadixArk NVFP4) checkpoint reader.
+
+Three separate paths, because the checkpoint's three weight classes live in different places:
+
+* :func:`iter_weights` -- every dense (non-expert) tensor, with the ``model.language_model.`` prefix stripped and fused where the model expects one buffer. See ``_FUSIONS``.
+* :func:`load_ple_table` -- the 47.7 GiB FP8 n-gram table, 128 checkpoint shards concatenated into one pinned :class:`HostBank`.
+* :func:`load_nvfp4_expert_sources` -- the routed NVFP4 experts, into the offload cache's source banks.
+
+Dropped: ``mtp.*`` (speculative head, including its stacked ``mtp.layers.0.mlp.experts.*``) and ``model.visual.*`` (served text-only).
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import struct
+from dataclasses import dataclass
+from typing import Iterator
+
+import safetensors
+import torch
+from freetoken.distributed import get_tp_info
+from freetoken.models.loader import drop_page_cache, iter_weight_files
+from freetoken.models.nvfp4_banks import (
+ Nvfp4ExpertSourceSpec,
+ load_nvfp4_expert_source_banks,
+)
+from freetoken.moe.host_banks import HostBank, read_range_into
+from freetoken.utils import download_hf_weight
+from freetoken.utils.progress import byte_bar
+from tqdm import tqdm
+
+# Routed NVFP4 experts (nvidia modelopt layout): per-expert, un-fused. Matched against the RAW
+# weight_map key in nvfp4_banks. The ``model.language_model.`` anchor excludes the MTP head's
+# stacked ``mtp.layers.N.mlp.experts.*`` tensors.
+_EXPERT_KEY_RE = re.compile(
+ r"^model\.language_model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."
+ r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"
+)
+_EXPERT_RE = re.compile(r"\.mlp\.experts\.\d+\.")
+_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(
+ key_pattern=_EXPERT_KEY_RE,
+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},
+ layer_to_bank=lambda layer, config: layer, # every layer is MoE
+ desc="Qwen3.8-Flash-Next NVFP4 experts",
+)
+# Per-tensor modelopt quant scales; consumed with their ``.weight`` (experts) or unused.
+_SCALE_SUFFIXES = (".weight_scale", ".weight_scale_2", ".input_scale")
+
+# The n-gram table itself: too big for the dense state dict, loaded by load_ple_table.
+_PLE_TABLE_INFIX = ".ple.ple_embedding.ngram_embedding."
+_PLE_SHARD_RE = re.compile(
+ r"\.ple\.ple_embedding\.ngram_embedding\.shard_(?P<shard>\d+)\.weight$"
+)
+_PLE_SCALE_SUFFIX = ".ple.ple_embedding.ngram_embedding.weight_scale"
+
+# Zero-centered Qwen4ExpTextRMSNorm weights, loaded RAW: GroupedPlusOneRMSNorm / GemmaPlusOneRMSNorm
+# and the vendored grouped_gemma_rmsnorm all apply (1+w) at runtime in fp32, so folding the +1 into
+# the bf16 weight here would double-apply it and round away small |w|. The GDN gated norm
+# (linear_attn.norm) is a plain weight*x norm and is not in this set.
+_ZERO_CENTERED_NORM_SUFFIXES = (
+ ".hc_norm.weight",
+ ".ple.norm_key.weight",
+ ".ple.norm_query.weight",
+ ".ple.norm_conv.weight",
+ ".self_attn.q_norm.weight",
+ ".self_attn.k_norm.weight",
+ ".self_attn.indexer.q_layernorm.weight",
+ ".self_attn.indexer.k_layernorm.weight",
+)
+
+# Fused projections: concat the checkpoint parts along dim 0 in this exact order. A nonzero pad
+# rounds the merged row count up; the model splits the result back with the same sizes.
+_FUSIONS: dict[str, tuple[tuple[str, ...], int]] = {
+ # q carries the output gate, so its half is twice the attention width: [2*qo | kv | kv].
+ ".self_attn.qkv_proj.weight": ((
+ ".self_attn.q_proj.weight", ".self_attn.k_proj.weight", ".self_attn.v_proj.weight",
+ ), 0),
+ ".linear_attn.in_proj.weight": ((
+ ".linear_attn.in_proj_qkv.weight", ".linear_attn.in_proj_z.weight",
+ ".linear_attn.in_proj_b.weight", ".linear_attn.in_proj_a.weight",
+ ), 0),
+ ".mlp.shared_expert.gate_up_proj.weight": ((
+ ".mlp.shared_expert.gate_proj.weight", ".mlp.shared_expert.up_proj.weight",
+ ), 0),
+ # HC mix reads the low-rank down projection and the injection logits from one GEMM; vLLM
+ # pads the merged output to a multiple of 16 rows for cuBLAS (hyperconnection.py pad_size).
+ # The top-level hyper_connection_mixer has no injection and so never fuses.
+ ".attn_hyper_connection.input_mix_weight_down_block_inject.weight": ((
+ ".attn_hyper_connection.input_mix_weight_down.weight",
+ ".attn_hyper_connection.block_inject_weight.weight",
+ ), 16),
+ ".mlp_hyper_connection.input_mix_weight_down_block_inject.weight": ((
+ ".mlp_hyper_connection.input_mix_weight_down.weight",
+ ".mlp_hyper_connection.block_inject_weight.weight",
+ ), 16),
+}
+
+
+def _rename(raw_name: str) -> str | None:
+ """Checkpoint key -> FreeToken state-dict key, or None to skip."""
+ if raw_name.startswith(("mtp.", "model.visual.", "visual.")):
+ return None
+ if _PLE_TABLE_INFIX in raw_name:
+ return None # n-gram table + its scale: load_ple_table
+ if _EXPERT_RE.search(raw_name):
+ return None # routed experts: offload source banks
+ if raw_name.endswith(_SCALE_SUFFIXES):
+ return None
+ if raw_name.startswith("model.language_model."):
+ return "model." + raw_name[len("model.language_model.") :]
+ if raw_name.startswith("language_model."):
+ return "model." + raw_name[len("language_model.") :]
+ return raw_name
+
+
+def _try_fuse(
+ name: str, tensor: torch.Tensor, buf: dict[str, dict[int, torch.Tensor]]
+) -> tuple[str, torch.Tensor] | tuple[()] | None:
+ """Buffer a fusion part; return the merged ``(name, tensor)`` once all parts arrive, ``()`` while incomplete, ``None`` if ``name`` is not a fusion part."""
+ for fused_suffix, (parts, pad_to) in _FUSIONS.items():
+ for idx, part in enumerate(parts):
+ if not name.endswith(part):
+ continue
+ key = name[: -len(part)] + fused_suffix
+ slots = buf.setdefault(key, {})
+ slots[idx] = tensor
+ if len(slots) < len(parts):
+ return ()
+ del buf[key]
+ rows = [slots[i] for i in range(len(parts))]
+ pad = (-sum(t.shape[0] for t in rows)) % pad_to if pad_to else 0
+ if pad:
+ rows.append(torch.zeros(pad, *rows[0].shape[1:], dtype=rows[0].dtype, device=rows[0].device))
+ return key, torch.cat(rows, dim=0)
+ return None
+
+
+def iter_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """Yield the dense (non-expert) weights, prefix-stripped and fused to the model's buffers.
+
+ Keys keep the checkpoint's module names below the stripped prefix, so the emitted set is the
+ model's state dict minus the routed experts. Nothing here is quantized: the modelopt
+ ``ignore`` list covers everything except those experts, so attention, GDN, HC, PLE, the shared
+ expert and lm_head are all plain bf16 (the n-gram hash constants stay int64). Fusions:
+ attention q|k|v -> ``qkv_proj``, GDN ``in_proj_{qkv,z,b,a}`` -> ``in_proj``, shared-expert
+ gate|up -> ``gate_up_proj``, and each per-layer HC's ``input_mix_weight_down`` |
+ ``block_inject_weight`` -> a zero-padded ``input_mix_weight_down_block_inject``.
+
+ ``include_moe_experts`` is accepted for the loader contract but never yields anything: the
+ routed experts are NVFP4 and always come from :func:`load_nvfp4_expert_sources`.
+ """
+ if get_tp_info().size > 1:
+ raise NotImplementedError("qwen4_exp weight loading supports TP=1 only")
+ if not include_non_moe:
+ return
+
+ fuse_buf: dict[str, dict[int, torch.Tensor]] = {}
+ for file in tqdm(
+ iter_weight_files(model_path),
+ desc="Loading weights",
+ disable=not get_tp_info().is_primary(),
+ ):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ name = _rename(raw_name)
+ if name is None:
+ continue
+ tensor = f.get_tensor(raw_name)
+ fused = _try_fuse(name, tensor, fuse_buf)
+ if fused is not None:
+ if fused != (): # () means buffered, not yet complete
+ yield fused
+ continue
+ yield name, tensor
+
+ assert not fuse_buf, f"Incomplete projection fusions: {sorted(fuse_buf)}"
+
+
+# ======================================================================================
+# PLE n-gram table
+# ======================================================================================
+
+
+@dataclass(frozen=True)
+class PleTable:
+ """The filled n-gram table: one pinned host bank plus the checkpoint's per-tensor FP8 scale."""
+
+ bank: HostBank
+ weight_scale: torch.Tensor # scalar, checkpoint dtype (bf16)
+
+ @property
+ def tensor(self) -> torch.Tensor:
+ """``[total_rows, ngram_head_dim]`` float8_e4m3fn view of the bank."""
+ return self.bank.tensor
+
+
+_PLE_ST_DTYPE = "F8_E4M3"
+
+
+def _safetensors_header(path: str) -> tuple[dict, int]:
+ with open(path, "rb") as fh:
+ n = struct.unpack("<Q", fh.read(8))[0]
+ return json.loads(fh.read(n)), 8 + n
+
+
+def _ple_table_files(folder: str) -> list[str]:
+ """Shards holding a piece of the n-gram table, from the index when there is one."""
+ index = os.path.join(folder, "model.safetensors.index.json")
+ if not os.path.exists(index):
+ return sorted(iter_weight_files(folder))
+ with open(index, encoding="utf-8") as fh:
+ weight_map = json.load(fh)["weight_map"]
+ files = {shard for name, shard in weight_map.items() if _PLE_TABLE_INFIX in name}
+ return sorted(os.path.join(folder, shard) for shard in files)
+
+
+def load_ple_table(model_path: str, qwen4_args, *, pin: bool = True,
+ workers: int = 8, chunk: int = 8 << 20) -> PleTable:
+ """Concatenate the checkpoint's ``ngram_embedding.shard_<i>`` tensors into one pinned host bank.
+
+ The checkpoint splits the table into ``split_ngram_parts`` equal row blocks named by shard
+ index and scattered over the ``model-plefp8-*`` shards in header (lexicographic) order, so the
+ bank is filled shard by shard at ``shard_index * rows_per_shard``. Each read is O_DIRECT: the
+ table is ~47.7 GiB and must not also sit in the page cache while the bank holds the same bytes.
+ """
+ folder = download_hf_weight(model_path)
+ parts: dict[int, tuple[str, int, int]] = {} # shard index -> (path, file offset, bytes)
+ scale: torch.Tensor | None = None
+ rows = cols = 0
+ for path in _ple_table_files(folder):
+ header, base = _safetensors_header(path)
+ for key, meta in header.items():
+ if key == "__metadata__":
+ continue
+ if key.endswith(_PLE_SCALE_SUFFIX):
+ with safetensors.safe_open(path, framework="pt", device="cpu") as f:
+ scale = f.get_tensor(key).reshape(())
+ continue
+ match = _PLE_SHARD_RE.search(key)
+ if match is None:
+ continue
+ if meta["dtype"] != _PLE_ST_DTYPE:
+ raise ValueError(f"PLE table shard {key} has unsupported dtype {meta['dtype']}")
+ shape = meta["shape"]
+ if rows and tuple(shape) != (rows, cols):
+ raise ValueError(f"PLE table shard {key} is {shape}, expected {[rows, cols]}")
+ rows, cols = shape
+ begin, end = meta["data_offsets"]
+ parts[int(match.group("shard"))] = (path, base + begin, end - begin)
+
+ expected = int(qwen4_args.split_ngram_parts)
+ if sorted(parts) != list(range(expected)):
+ raise ValueError(
+ f"PLE table needs shards 0..{expected - 1}, found {len(parts)}: {sorted(parts)[:8]}"
+ )
+ if cols != qwen4_args.ngram_head_dim:
+ raise ValueError(f"PLE table row is {cols} wide, config says {qwen4_args.ngram_head_dim}")
+ if scale is None:
+ raise ValueError("PLE table has no weight_scale")
+
+ bank = HostBank((expected * rows, cols), torch.float8_e4m3fn)
+ shard_bytes = rows * cols
+ bar = byte_bar(expected * shard_bytes, "Loading PLE table")
+ try:
+ buf = bank.memoryview()
+ for shard in range(expected):
+ path, offset, nbytes = parts[shard]
+ assert nbytes == shard_bytes, f"PLE shard {shard} is {nbytes} B, expected {shard_bytes}"
+ read_range_into(buf, path, file_offset=offset, nbytes=nbytes,
+ dest_offset=shard * shard_bytes, workers=workers, chunk=chunk)
+ bar.update(nbytes)
+ finally:
+ bar.close()
+ if pin and torch.cuda.is_available():
+ bank.pin()
+ return PleTable(bank=bank, weight_scale=scale)
+
+
+# ======================================================================================
+# Routed NVFP4 experts
+# ======================================================================================
+
+
+def load_nvfp4_expert_sources(model_path: str, config, *, layer_sink=None) -> dict:
+ """Build the CPU NVFP4 expert source banks for the offload cache (gate/up fused on the output-row axis, down separate; weight_scale_2 carried as the per-row global scale)."""
+ return load_nvfp4_expert_source_banks(
+ model_path,
+ config,
+ _NVFP4_SOURCE_SPEC,
+ drop_page_cache=drop_page_cache,
+ primary=get_tp_info().is_primary(),
+ layer_sink=layer_sink,
+ )
+
+
+def load_nvfp4_expert_sources_parallel(
+ model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
+) -> dict:
+ """parallel: same NVFP4 source banks via the common chunked multi-threaded reader."""
+ from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
+
+ return load_nvfp4_expert_source_banks_parallel(
+ model_path,
+ config,
+ _NVFP4_SOURCE_SPEC,
+ drop_page_cache=drop_page_cache,
+ primary=get_tp_info().is_primary(),
+ workers=workers,
+ chunk=chunk,
+ layer_sink=layer_sink,
+ )
+
+
+__all__ = [
+ "PleTable",
+ "iter_weights",
+ "load_nvfp4_expert_sources",
+ "load_nvfp4_expert_sources_parallel",
+ "load_ple_table",
+]
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 0c033ca0..b94d8291 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -58,6 +58,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"freetoken.models.qwen3_5_moe",
"Qwen3_5MoEForCausalLM",
),
+ # Qwen3.8-Flash-Next (model_type qwen4_exp): multimodal wrapper config (text tower in
+ # text_config, weights under model.language_model.); served text-only. 36 GDN + 12 QSA
+ # compressed-sparse attention layers on 4 hyper-connection residual streams, a PLE
+ # n-gram embedding layer, 512 NVFP4 routed experts top-10 + a gated shared expert.
+ "Qwen4ExpForConditionalGeneration": ModelSpec(
+ "freetoken.models.qwen4_exp",
+ "Qwen4ExpForCausalLM",
+ ),
# Dense Qwen3.x (no "Moe" in the arch name, num_experts==0, e.g. Qwen3.6-27B). Shares the
# qwen3_5_moe package: the decoder routes its MLP through the dense Qwen3_5DenseMLP and the
# loader handles the compressed-tensors NVFP4 layout.
diff --git a/python/freetoken/moe/fused.py b/python/freetoken/moe/fused.py
index fe7e417d..4b9a4875 100644
--- a/python/freetoken/moe/fused.py
+++ b/python/freetoken/moe/fused.py
@@ -62,6 +62,13 @@ def fused_topk(
)
return _torch_fused_topk(gating_output, topk, renormalize, num_token_non_padded)
+ if topk & (topk - 1):
+ # triton_kernels.topk builds tl.arange(0, k), which must be a power of 2; a
+ # top-10 router (qwen4_exp) takes the equivalent vendored triton router instead.
+ from freetoken.kernel.triton.moe_router import fused_topk_softmax
+
+ return fused_topk_softmax(gating_output, topk, renormalize, num_token_non_padded)
+
from triton_kernels.topk import topk as triton_kernels_topk
logits = gating_output.float()
diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.py
index 5ed8d766..29a561e6 100644
--- a/python/freetoken/moe/fused_nvfp4.py
+++ b/python/freetoken/moe/fused_nvfp4.py
@@ -61,6 +61,12 @@ _DECODE_WARPS = 4
_DECODE_MARLIN_BLOCK_N = 16
_DECODE_MARLIN_BLOCK_KW = 16
_DECODE_MARLIN_WARPS = 4
+# Deep-K variant: at K > 2048 (qwen4_exp gate_up, K=2560) a narrower N tile with the whole
+# K strip in one program iteration measures ~13% faster (18.6 vs 21.0us); short-K shapes
+# regress under it, so the split is by K, not by gemm position.
+_DECODE_MARLIN_DEEPK_BLOCK_N = 8
+_DECODE_MARLIN_DEEPK_BLOCK_KW = 128
+_DECODE_MARLIN_DEEPK_THRESHOLD = 2048
def _tl_dtype(dt: torch.dtype):
@@ -129,7 +135,10 @@ def _decode_gemm_marlin(
packed_i32 = packed.view(torch.int32) # [S, N, K // 8]
scale = e4m3_kernel_view(scale)
total_routes = M * top_k
- grid = (total_routes, triton.cdiv(N, _DECODE_MARLIN_BLOCK_N))
+ deep_k = K > _DECODE_MARLIN_DEEPK_THRESHOLD
+ block_n = _DECODE_MARLIN_DEEPK_BLOCK_N if deep_k else _DECODE_MARLIN_BLOCK_N
+ block_kw = _DECODE_MARLIN_DEEPK_BLOCK_KW if deep_k else _DECODE_MARLIN_BLOCK_KW
+ grid = (total_routes, triton.cdiv(N, block_n))
_decode_nvfp4_marlin_kernel[grid](
a, packed_i32, scale, glob, c, topk_weights, topk_ids,
_e2m1_lut(a.device.index),
@@ -141,8 +150,8 @@ def _decode_gemm_marlin(
c.stride(0), c.stride(1), c.stride(2),
topk_weights.stride(0), topk_weights.stride(1),
topk_ids.stride(0), topk_ids.stride(1),
- BLOCK_SIZE_N=_DECODE_MARLIN_BLOCK_N,
- BLOCK_SIZE_KW=_DECODE_MARLIN_BLOCK_KW,
+ BLOCK_SIZE_N=block_n,
+ BLOCK_SIZE_KW=block_kw,
TOP_K=top_k,
A_ROW_IS_ROUTE=a_row_is_route,
MUL_ROUTED_WEIGHT=mul_routed_weight,
diff --git a/python/freetoken/moe/host_banks.py b/python/freetoken/moe/host_banks.py
index d7af348a..436f52d2 100644
--- a/python/freetoken/moe/host_banks.py
+++ b/python/freetoken/moe/host_banks.py
@@ -410,6 +410,67 @@ def read_file_into(buf: memoryview | mmap.mmap, path: str, *, workers: int = 8,
return size
+def _preadv_all(fd: int, dst: memoryview, offset: int, need: int) -> None:
+ """preadv into ``dst`` until ``need`` bytes have landed; O_DIRECT may return a short count."""
+ done = 0
+ while done < need:
+ if done % _BLK: # a continuation read has to stay block-aligned on both sides
+ raise OSError(f"unaligned short O_DIRECT read: {done} of {need} bytes at {offset}")
+ got = os.preadv(fd, [dst[done:]], offset + done)
+ if got <= 0:
+ raise OSError(f"short O_DIRECT read: {done} of {need} bytes at {offset}")
+ done += got
+
+
+def read_range_into(buf: memoryview | mmap.mmap, path: str, *, file_offset: int, nbytes: int,
+ dest_offset: int = 0, workers: int = 8, chunk: int = _DEFAULT_CHUNK,
+ drop_cache: bool = True) -> int:
+ """Chunked multi-threaded O_DIRECT read of ``path[file_offset : file_offset + nbytes]`` into ``buf`` at ``dest_offset``. Returns ``nbytes``.
+
+ Byte-range counterpart of :func:`read_file_into`, for one tensor inside a shard. O_DIRECT needs the file offset AND the destination address block-aligned at the same time, which only holds when the two share their offset mod 4096 -- a safetensors data offset practically never lines up with the tensor's slot in the bank. Chunks that do line up DMA straight into ``buf``; the rest DMA into a page-aligned bounce (source window rounded out to whole blocks) and are copied into place, which also covers the unaligned head and tail.
+ """
+ mv = (buf if isinstance(buf, memoryview) else memoryview(buf)).cast("B")
+ if dest_offset + nbytes > len(mv):
+ raise ValueError(f"destination holds {len(mv)} bytes, need {dest_offset + nbytes}")
+ base = ctypes.addressof(ctypes.c_char.from_buffer(mv))
+ if drop_cache:
+ try:
+ fd0 = os.open(path, os.O_RDONLY)
+ os.posix_fadvise(fd0, file_offset, nbytes, os.POSIX_FADV_DONTNEED)
+ os.close(fd0)
+ except OSError:
+ pass
+ fd = os.open(path, os.O_RDONLY | os.O_DIRECT)
+ scratch = threading.local()
+
+ def rd(i: int) -> None:
+ n = min(chunk, nbytes - i)
+ src, dst = file_offset + i, dest_offset + i
+ if src % _BLK == 0 and (base + dst) % _BLK == 0 and n % _BLK == 0:
+ _preadv_all(fd, mv[dst:dst + n], src, n)
+ return
+ head = src % _BLK
+ span = ((head + n + _BLK - 1) // _BLK) * _BLK
+ bounce = getattr(scratch, "buf", None)
+ if bounce is None or len(bounce) < span:
+ bounce = scratch.buf = mmap.mmap(-1, span) # anonymous mmaps are page-aligned
+ bmv = memoryview(bounce)
+ _preadv_all(fd, bmv[:span], src - head, head + n)
+ mv[dst:dst + n] = bmv[head:head + n]
+
+ try:
+ offs = list(range(0, nbytes, chunk))
+ if len(offs) <= 1:
+ for o in offs:
+ rd(o)
+ else:
+ with ThreadPoolExecutor(workers) as ex:
+ list(ex.map(rd, offs))
+ finally:
+ os.close(fd)
+ return nbytes
+
+
__all__ = [
"HostBank",
"HostResidency",
@@ -420,5 +481,6 @@ __all__ = [
"born_pinned_default",
"pin_banks",
"read_file_into",
+ "read_range_into",
"requested_residency",
]
diff --git a/python/freetoken/moe/offload_cache.py b/python/freetoken/moe/offload_cache.py
index 6ee76406..33a47553 100644
--- a/python/freetoken/moe/offload_cache.py
+++ b/python/freetoken/moe/offload_cache.py
@@ -77,11 +77,21 @@ _BANK_SCHEMAS: dict[str, tuple[str, ...]] = {
"ds_fp4": ("gate_up_packed", "gate_up_scale", "down_packed", "down_scale"),
}
+def fp8_block_scale_pad(rows: int, cols: int) -> int:
+ """Trailing scale-bank dim padded so per-expert row bytes are 16B-aligned (fused copy)."""
+ while (rows * cols * 2) % 16:
+ cols += 1
+ return cols
+
+
# bytes per (expert, layer) as f(hidden, moe_intermediate), from the bank shapes above; keep in sync with _BANK_SCHEMAS
# keyed by the config-time format tag (expert_quant / moe_weight_format), not quant_format: "mxfp4" sizes the mxfp4_triton banks, "nvfp4" also covers its repacked variants
_BANK_BYTES_PER_EXPERT = {
"bf16": lambda H, I: 3 * I * H * 2,
- "fp8_block": lambda H, I: 3 * I * H + ((2 * I // 128) * (H // 128) + (H // 128) * (I // 128)) * 2,
+ "fp8_block": lambda H, I: 3 * I * H + (
+ (2 * I // 128) * fp8_block_scale_pad(2 * I // 128, H // 128)
+ + (H // 128) * fp8_block_scale_pad(H // 128, I // 128)
+ ) * 2,
"q4_0": lambda H, I: 2 * I * (H // 32) * 18 + H * (I // 32) * 18,
"nvfp4": lambda H, I: 2 * I * (H // 2 + H // 16 + 2) + H * (I // 2 + I // 16 + 2),
"mxfp4": lambda H, I: 2 * I * (H // 2 + H // 32 + 2) + H * (I // 2 + I // 32 + 2),
diff --git a/python/freetoken/scheduler/cache.py b/python/freetoken/scheduler/cache.py
index 9be235b2..44adde42 100644
--- a/python/freetoken/scheduler/cache.py
+++ b/python/freetoken/scheduler/cache.py
@@ -66,6 +66,13 @@ class CacheManager:
supports_runtime_rebuild = True
prefill_chunk_budget = None # generic shared page pool: no per-model prefill chunk cap
+ @property
+ def prefill_chunk_align(self) -> int:
+ """Granularity a non-final prefill chunk should end on. A hybrid snapshot is donated only
+ at a page-aligned boundary, so at page_size>1 one unaligned chunk end costs every reuse
+ point for the rest of the prompt. 1 (no-op) everywhere else."""
+ return self.page_size if self.is_hybrid else 1
+
def page_usage(self) -> tuple[int, int]:
"""(used_pages, total_pages): allocated, non-evictable pages over the pool total
(active requests + protected prefix; evictable prefix-cache pages are excluded)."""
diff --git a/python/freetoken/scheduler/prefill.py b/python/freetoken/scheduler/prefill.py
index be84874c..f5bc8f7a 100644
--- a/python/freetoken/scheduler/prefill.py
+++ b/python/freetoken/scheduler/prefill.py
@@ -156,6 +156,13 @@ class PrefillAdder:
self.reserved_swa += (
div_ceil(cached_len + chunk_size, ps) - div_ceil(cached_len, ps)
) * ps
+ align = self.cache_manager.prefill_chunk_align
+ if align > 1 and 0 < chunk_size < remain_len:
+ # An unaligned chunk end is correct, it just loses this prompt's snapshot boundaries --
+ # so keep it when the leftover budget cannot fill one whole unit instead of stalling
+ # the request until it gets a bigger turn.
+ aligned = align_down(cached_len + chunk_size, align) - cached_len
+ chunk_size = aligned if aligned > 0 else chunk_size
is_chunked = chunk_size < remain_len
CLS = ChunkedReq if is_chunked else Req
self.token_budget -= chunk_size
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index a71b6819..6696f65d 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -147,6 +147,8 @@ def parse_args(
return "muse_glimmer"
if "gemma4" in marker:
return "gemma4"
+ if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker:
+ return "qwen3_coder"
if (
"qwen3_5" in marker
or "qwen3.5" in marker
@@ -188,6 +190,8 @@ def parse_args(
tag in marker for tag in ("v4", "deepseek_v4", "v3.2", "v32")
):
return "deepseekv32"
+ if "qwen4_exp" in marker or "qwen4exp" in marker or "qwen3.8-flash" in marker:
+ return "qwen3"
if "qwen3" in marker or "qwen3.5" in marker or "qwen3_5" in marker:
return "qwen3"
if "glm" in marker:
diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.py
index f27640b9..fc9abfbe 100644
--- a/tests/engine/test_attention_backend_matrix.py
+++ b/tests/engine/test_attention_backend_matrix.py
@@ -24,7 +24,7 @@ from freetoken.models.config import (
)
-def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0):
+def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0, index_ratio=1):
return KVCacheGroupSpec(
name=name,
layer_ids=(0, 1),
@@ -34,6 +34,7 @@ def _spec(name, attn_type, *, mla=False, sliding_window=None, index_head_dim=0):
mla=mla,
index_head_dim=index_head_dim,
num_index_layers=2 if index_head_dim else 0,
+ index_ratio=index_ratio,
attn_type=attn_type,
)
@@ -67,6 +68,11 @@ def _model_config(kind):
elif kind == "bsa":
# MiniMax-M3 shape: one FULL-family group, mla=False + index dims -> BSA.
specs = (_spec("full", AttnType.BSA, index_head_dim=128),)
+ elif kind == "qsa":
+ # Qwen3.8-Flash-Next shape: hybrid-linear + one FULL-family group whose index keys
+ # are compressed index_ratio:1 -> QSA.
+ mc.has_linear_attention = True
+ specs = (_spec("full", AttnType.QSA, index_head_dim=128, index_ratio=4),)
elif kind == "linear_hybrid":
mc.has_linear_attention = True
specs = (_spec("full", AttnType.FULL),)
@@ -109,6 +115,7 @@ def _patch_env(monkeypatch, *, major=9, flashinfer=True, sgl=True):
("dsa", "dsa"), # MLA + DSA indexer (GLM-5.2 shape)
("dsv4", "dsv4_sparse"),
("bsa", "m3_sparse"), # MiniMax-M3 block-sparse GQA
+ ("qsa", "qsa_sparse"), # Qwen3.8-Flash-Next compressed-block sparse
],
)
def test_auto_resolves_per_type(monkeypatch, kind, expected):
@@ -143,6 +150,39 @@ def test_bsa_rejects_float32_dtype(monkeypatch):
_adjust_config(config)
+def test_auto_qsa_sets_page_size_64(monkeypatch):
+ # qsa_sparse registers page_sizes=(64,) (a 4-token compress group must never straddle
+ # a page); the generic backend page-size coercion takes the default 1 to 64.
+ from freetoken.engine.engine import _adjust_config
+
+ _patch_env(monkeypatch)
+ config = _config("qsa", attention_backend="auto")
+ assert config.page_size == 1
+ _adjust_config(config)
+ assert config.page_size == 64
+
+
+def test_qsa_coerces_explicit_page_size(monkeypatch):
+ # Same policy as m3_sparse (page_sizes=(128,)): an unsupported explicit value is
+ # coerced to the backend's page size with a warning, not rejected.
+ from freetoken.engine.engine import _adjust_config
+
+ _patch_env(monkeypatch)
+ config = _config("qsa", attention_backend="auto", page_size=16)
+ _adjust_config(config)
+ assert config.page_size == 64
+
+
+def test_qsa_rejects_float32_dtype(monkeypatch):
+ from freetoken.engine.engine import _adjust_config
+
+ _patch_env(monkeypatch)
+ config = _config("qsa", attention_backend="auto")
+ object.__setattr__(config, "dtype", torch.float32)
+ with pytest.raises(ValueError, match="16-bit"):
+ _adjust_config(config)
+
+
def test_auto_dsv4_sets_window_page_size(monkeypatch):
from freetoken.engine.engine import _adjust_config
@@ -160,9 +200,16 @@ def test_auto_dsv4_sets_window_page_size(monkeypatch):
("full", "dsv4_sparse"),
("full", "m3_sparse"),
("swa", "dsa"),
+ ("full", "qsa_sparse"),
# forward gates: generic backends on the BSA-locked model
("bsa", "fi"),
("bsa", "triton"),
+ # forward gates: generic and neighbouring sparse backends on the QSA-locked model
+ ("qsa", "fi"),
+ ("qsa", "fa"),
+ ("qsa", "triton"),
+ ("qsa", "m3_sparse"),
+ ("bsa", "qsa_sparse"),
# forward gates: generic backends on type-locked models
("mla", "fi"),
("mla", "triton"),
@@ -196,6 +243,7 @@ def test_illegal_combinations_rejected_at_config_time(monkeypatch, kind, backend
("mla", "dsa"),
("dsa", "dsa"),
("dsv4", "dsv4_sparse"),
+ ("qsa", "qsa_sparse"),
("swa", "triton"),
("full", "triton"),
("full", "fa,fi"),
@@ -220,31 +268,6 @@ def test_mla_requires_page_size_one(monkeypatch, kind):
_adjust_config(config)
-def test_hybrid_linear_opt_out_rejects_backend(monkeypatch):
- import dataclasses
-
- from freetoken.attention import attention_backend_info
- from freetoken.engine import engine
- from freetoken.engine.engine import _adjust_config
-
- _patch_env(monkeypatch)
- real_info = attention_backend_info
-
- def _info(name):
- info = real_info(name)
- return dataclasses.replace(info, hybrid_linear_ok=False) if name == "fa" else info
-
- monkeypatch.setattr(engine, "attention_backend_info", _info)
- # auto skips the opted-out backend ...
- config = _config("linear_hybrid", attention_backend="auto")
- _adjust_config(config)
- assert config.attention_backend == "fi"
- # ... and an explicit choice of it is rejected
- config = _config("linear_hybrid", attention_backend="fa")
- with pytest.raises(ValueError, match="hybrid-linear"):
- _adjust_config(config)
-
-
def test_trtllm_page_size_coercion_is_part_aware(monkeypatch):
from freetoken.engine.engine import _adjust_config
@@ -372,3 +395,10 @@ def dataclasses_replace_groups(mc, groups):
import dataclasses
return dataclasses.replace(mc, attention_groups=groups)
+
+
+def test_linear_attention_defaults_to_hybrid_radix():
+ from freetoken.engine.engine import _resolve_cache_type
+
+ assert _resolve_cache_type(True, "radix") == "hybrid_radix"
+ assert _resolve_cache_type(True, "naive") == "naive"
diff --git a/tests/engine/test_cache_budget.py b/tests/engine/test_cache_budget.py
index a164f0b4..9ac2a4f4 100644
--- a/tests/engine/test_cache_budget.py
+++ b/tests/engine/test_cache_budget.py
@@ -5,7 +5,10 @@ from types import SimpleNamespace
import pytest
import torch
+import os
+
from freetoken.engine.cache_budget import expert_bytes_per_slot, plan_cache_budget, resolve_moe_cache_auto
+from freetoken.engine.engine import _pin_budget_bytes
def test_moe_priority_fills_experts_up_to_total():
@@ -480,3 +483,20 @@ def test_adjust_config_rope_gate_exempts_dsv4():
cfg = _dsv4_adjust_cfg(max_seq_len_override=10_000_000)
cfg.model_config.rotary_config = SimpleNamespace(max_position=1024)
_adjust_config(cfg) # must not raise
+
+
+# ---- _pin_budget_bytes: host bytes already pinned outside the expert banks ----
+
+
+def test_reserved_subtracts_from_the_cap(monkeypatch):
+ monkeypatch.setenv("FREETOKEN_PIN_BUDGET_GB", "2")
+ assert _pin_budget_bytes() == 2 * 2**30
+ assert _pin_budget_bytes(reserved=2**30) == 2**30
+ assert _pin_budget_bytes(reserved=4 * 2**30) == 0
+
+
+def test_uncapped_platform_stays_uncapped(monkeypatch):
+ monkeypatch.delenv("FREETOKEN_PIN_BUDGET_GB", raising=False)
+ if hasattr(os, "uname") and "microsoft" in os.uname().release.lower():
+ pytest.skip("WSL caps pinning")
+ assert _pin_budget_bytes(reserved=2**30) is None
diff --git a/tests/kvcache/test_kv_cache_rebuild.py b/tests/kvcache/test_kv_cache_rebuild.py
index 6dff34ea..a5ec94a0 100644
--- a/tests/kvcache/test_kv_cache_rebuild.py
+++ b/tests/kvcache/test_kv_cache_rebuild.py
@@ -150,7 +150,7 @@ def test_linear_state_pool_rebuild_resizes_preserves_identity_and_dtypes():
_init_tp()
group = LinearGatedDeltaGroupConfig(
name="linear", layer_ids=(0, 1, 2), num_key_heads=4, num_value_heads=8,
- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,
+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",
)
pool = LinearStatePool(group=group, num_slots=10, dtype=torch.bfloat16, device=torch.device("cpu"))
pid = id(pool)
diff --git a/tests/kvcache/test_linear_state_pool_alloc.py b/tests/kvcache/test_linear_state_pool_alloc.py
index 16058efc..e9fe4054 100644
--- a/tests/kvcache/test_linear_state_pool_alloc.py
+++ b/tests/kvcache/test_linear_state_pool_alloc.py
@@ -1,22 +1,31 @@
-"""P1 unit: LinearStatePool free-list allocator (alloc/free/clear_slots/copy_from).
+"""LinearStatePool unit: the free-list allocator and the declared slot-state siblings.
CPU-only, fast — pure slot bookkeeping + state copy/zero, no kernels."""
from __future__ import annotations
+from types import SimpleNamespace
+
import pytest
import torch
-from freetoken.kvcache.linear_state_pool import LinearStatePool
-from freetoken.models.config import LinearGatedDeltaGroupConfig
+from freetoken.kvcache.linear_state_pool import (
+ LinearStatePool,
+ linear_state_bytes_per_req,
+ state_pool_bytes,
+)
+from freetoken.models.config import LinearGatedDeltaGroupConfig, SlotStateSpec
-def _pool(num_slots=8, device="cpu"):
- group = LinearGatedDeltaGroupConfig(
+def _group():
+ return LinearGatedDeltaGroupConfig(
name="linear", layer_ids=(0, 1),
num_key_heads=2, num_value_heads=4,
- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,
+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",
)
- return LinearStatePool(group=group, num_slots=num_slots, dtype=torch.bfloat16,
- device=torch.device(device), tp_size=1)
+
+
+def _pool(num_slots=8, device="cpu", slot_states=()):
+ return LinearStatePool(group=_group(), num_slots=num_slots, dtype=torch.bfloat16,
+ device=torch.device(device), tp_size=1, slot_states=slot_states)
def test_alloc_free_roundtrip():
@@ -70,3 +79,113 @@ if __name__ == "__main__":
test_clear_slots_zeros_all_layers()
test_copy_from_snapshot()
print("LinearStatePool allocator unit: PASS")
+
+
+# qwen4_exp PLE conv-history shape at toy size: one PLE layer, 32 channels, 9 taps
+_SPECS = (SlotStateSpec(name="ple_conv", shape=(32, 9), layer_ids=(1,)),)
+
+
+def test_no_slot_states_by_default():
+ pool = _pool()
+ assert pool.slot_states == {} and not pool.has_slot_state("ple_conv")
+ base = pool.bytes_per_slot()
+ pool.clear_slots([1, 2])
+ pool.copy_from(1, 2)
+ pool.reset(3)
+ pool.rebuild(4)
+ assert pool.slot_states == {} and pool.bytes_per_slot() == base
+
+
+def test_slot_state_geometry_and_accessor():
+ pool = _pool(num_slots=6, slot_states=_SPECS)
+ slab = pool.slot_states["ple_conv"]
+ assert slab.shape == (1, 6, 32, 9) and slab.dtype is torch.bfloat16
+ assert pool.slot_state("ple_conv", 1).shape == (6, 32, 9)
+ with pytest.raises(KeyError):
+ pool.slot_state("ple_conv", 0) # not a declared layer
+ with pytest.raises(KeyError):
+ pool.slot_state("other")
+ with pytest.raises(AssertionError):
+ pool.slot_state("ple_conv") # per-layer state needs layer_id
+
+
+def test_layerless_spec_dtype_and_fill_value():
+ spec = SlotStateSpec(name="ngram", shape=(2,), dtype=torch.int32, fill_value=7.0)
+ pool = _pool(num_slots=6, slot_states=(spec,))
+ t = pool.slot_state("ngram")
+ assert t.shape == (6, 2) and t.dtype is torch.int32 and bool((t == 7).all())
+ t[3] = 1
+ pool.clear_slots([3])
+ assert bool((pool.slot_state("ngram")[3] == 7).all())
+ t[2] = 1
+ pool.reset(2)
+ assert bool((pool.slot_state("ngram")[2] == 7).all())
+ pool.rebuild(5)
+ assert bool((pool.slot_state("ngram") == 7).all())
+
+
+def test_duplicate_names_rejected():
+ with pytest.raises(ValueError, match="duplicate"):
+ _pool(slot_states=_SPECS + _SPECS)
+
+
+def test_slot_state_follows_every_slot_operation():
+ pool = _pool(slot_states=_SPECS)
+ pool.slot_states["ple_conv"].fill_(1.0)
+ pool.conv_states.fill_(1.0)
+
+ pool.clear_slots([2])
+ slab = pool.slot_state("ple_conv", 1)
+ assert slab[2].abs().sum().item() == 0.0
+ assert slab[3].abs().sum().item() > 0.0
+
+ pool.copy_from(3, 2)
+ assert torch.equal(slab[2], slab[3])
+
+ pool.reset(3)
+ assert slab[3].abs().sum().item() == 0.0
+
+ pool.rebuild(9)
+ slab = pool.slot_state("ple_conv", 1) # rebuild replaces the tensor
+ assert pool.slot_states["ple_conv"].shape == (1, 9, 32, 9)
+ assert slab.abs().sum().item() == 0.0
+
+ # the ops write the rebuilt tensor, not a stale alias
+ pool.slot_states["ple_conv"].fill_(1.0)
+ pool.clear_slots([5])
+ assert pool.slot_state("ple_conv", 1)[5].abs().sum().item() == 0.0
+ pool.copy_from(1, 5)
+ assert torch.equal(pool.slot_state("ple_conv", 1)[5], pool.slot_state("ple_conv", 1)[1])
+
+
+def test_slot_state_in_the_byte_account():
+ group = _group()
+ gdn_only = linear_state_bytes_per_req(group, 1, torch.bfloat16)
+ with_state = linear_state_bytes_per_req(group, 1, torch.bfloat16, _SPECS)
+ assert with_state - gdn_only == 32 * 9 * 2
+ assert _pool(slot_states=_SPECS).bytes_per_slot() == with_state
+
+ mc = SimpleNamespace(slot_states=_SPECS)
+ mc.linear_attention_group = lambda: group
+ config = SimpleNamespace(
+ model_config=mc, dtype=torch.bfloat16, tp_info=SimpleNamespace(size=1),
+ cache_type="naive", max_running_req=3, linear_state_cache_ratio=0.5,
+ )
+ assert state_pool_bytes(config, num_slots=4) == with_state * 4
+
+ mc.slot_states = ()
+ assert state_pool_bytes(config, num_slots=4) == gdn_only * 4
+
+ mc.slot_states = _SPECS
+ mc.linear_attention_group = lambda: None
+ with pytest.raises(ValueError, match="slot_states"):
+ state_pool_bytes(config, num_slots=4)
+
+
+def test_slot_state_bytes_for_the_real_geometry():
+ # qwen4_exp PLE conv history: 4 streams x 2560 channels x 9 taps bf16 = 180 KiB per slot
+ spec = SlotStateSpec(name="ple_conv", shape=(4 * 2560, 9), layer_ids=(1,))
+ group = _group()
+ delta = linear_state_bytes_per_req(group, 1, torch.bfloat16, (spec,)) - \
+ linear_state_bytes_per_req(group, 1, torch.bfloat16)
+ assert delta == 4 * 2560 * 9 * 2 == 180 * 1024
diff --git a/tests/kvcache/test_pool_sizing_surface.py b/tests/kvcache/test_pool_sizing_surface.py
index d50a5be4..516090b0 100644
--- a/tests/kvcache/test_pool_sizing_surface.py
+++ b/tests/kvcache/test_pool_sizing_surface.py
@@ -249,7 +249,7 @@ def test_linear_state_pool_prices_itself():
group = LinearGatedDeltaGroupConfig(
name="linear", layer_ids=(1, 3), num_key_heads=2, num_value_heads=4,
- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,
+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",
)
config = _generic_config()
config.linear_state_cache_ratio = 0.5
diff --git a/tests/kvcache/test_qsa_pool.py b/tests/kvcache/test_qsa_pool.py
new file mode 100644
index 00000000..9aec2bb6
--- /dev/null
+++ b/tests/kvcache/test_qsa_pool.py
@@ -0,0 +1,218 @@
+"""QSAKVCache tiers (paged K/V, compressed index slab, pending ring, scratch).
+
+Pins the three things the QSA kernels and the startup budget both depend on: the compressed
+slab is a 1/index_ratio shadow of the K/V pages with the scratch rows behind it, the ring and
+scratch are fixed (concurrency-sized) and priced apart from the per-token slider, and the K/V
+slabs cover the sparse layers only. The PLE conv history rides the GDN slots, so it must
+follow every slot operation and show up in the state-pool byte account.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from freetoken.attention import AttnType
+from freetoken.kvcache.base import spec_kv_bytes_per_token
+from freetoken.kvcache.qsa_pool import QSAKVCache
+from freetoken.models.config import KVCacheGroupSpec
+
+DEV = torch.device("cpu")
+
+# Qwen3.8-Flash-Next: 48 layers, every 4th is QSA; 2 kv heads x 256, indexer 128 wide, ratio 4.
+FULL_LAYER_IDS = tuple(range(3, 48, 4))
+REAL_KV_BYTES = 2 * 256 * 2 * 2 * 12
+REAL_INDEX_BYTES = 128 * 12 * 2 // 4
+
+
[email protected](autouse=True)
+def _tp(monkeypatch):
+ from freetoken.distributed.info import DistributedInfo
+
+ monkeypatch.setattr(
+ "freetoken.kvcache.mha_pool.get_tp_info",
+ lambda: DistributedInfo(rank=0, size=1),
+ )
+
+
+def _pool(num_pages=4, page_size=64, index_ratio=4, num_req_slots=4, ring_capacity=None):
+ return QSAKVCache(
+ num_kv_heads=2,
+ num_layers=8,
+ head_dim=64,
+ num_pages=num_pages,
+ page_size=page_size,
+ dtype=torch.bfloat16,
+ device=DEV,
+ index_head_dim=32,
+ num_index_layers=4,
+ index_ratio=index_ratio,
+ num_req_slots=num_req_slots,
+ ring_capacity=ring_capacity,
+ layer_ids=(1, 3, 5, 7),
+ )
+
+
+def _spec(*, index_ratio=4, attn_type=AttnType.QSA, num_kv_heads=2, head_dim=64,
+ index_head_dim=32, num_index_layers=4, layer_ids=(1, 3, 5, 7)):
+ return KVCacheGroupSpec(
+ name="full",
+ layer_ids=layer_ids,
+ num_kv_heads=num_kv_heads,
+ head_dim=head_dim,
+ sliding_window=None,
+ index_head_dim=index_head_dim,
+ num_index_layers=num_index_layers,
+ index_ratio=index_ratio,
+ attn_type=attn_type,
+ )
+
+
+def _config(spec, *, page_size=64, max_running_req=3):
+ mc = SimpleNamespace(num_layers=8, has_swa_attention=False, has_linear_attention=True)
+ mc.kv_cache_group_specs = lambda: (spec,)
+ return SimpleNamespace(
+ model_config=mc,
+ page_size=page_size,
+ dtype=torch.bfloat16,
+ tp_info=SimpleNamespace(size=1),
+ max_running_req=max_running_req,
+ )
+
+
+# --------------------------------------------------------------------- slab / ring geometry
+
+
+def test_slab_ring_and_scratch_shapes():
+ pool = _pool(num_pages=4)
+ # 4 pages x 64 tokens / ratio 4 = 64 shadow rows, then one scratch row per request slot
+ assert pool.cmp_scratch_base == 64
+ assert pool.cmp_k_cache(0).shape == (64 + 4, 32)
+ assert pool.cmp_k_cache(3).shape == (64 + 4, 32)
+ assert pool.pending_ring(0).shape == (4, QSAKVCache.ring_capacity_for(4), 32)
+ assert pool.cmp_k_cache(0).abs().sum().item() == 0.0
+ assert pool.k_cache(1).shape == (4, 64, 2, 64)
+
+
+def test_kv_slabs_cover_sparse_layers_only():
+ # Copying the BSA branch (no layer_ids) would back all 8 model layers instead of 4.
+ pool = _pool()
+ assert pool._kv_buffer.shape[1] == 4
+ pool.k_cache(7)
+ with pytest.raises(KeyError):
+ pool.k_cache(0)
+
+
+def test_ring_capacity_and_ratio_are_parameters():
+ pool = _pool(num_pages=8, index_ratio=2, num_req_slots=3, ring_capacity=6)
+ assert pool.index_ratio == 2 and pool.ring_capacity == 6
+ assert pool.cmp_scratch_base == 8 * 64 // 2
+ assert pool.pending_ring(0).shape == (3, 6, 32)
+
+
+def test_ring_capacity_formula_and_floor():
+ assert QSAKVCache.ring_capacity_for(4) == 4
+ assert QSAKVCache.ring_capacity_for(8) == 8
+ assert QSAKVCache.ring_capacity_for(4, num_speculative_tokens=3) == 8
+ with pytest.raises(ValueError, match="ring_capacity"):
+ _pool(ring_capacity=2, index_ratio=4)
+
+
+def test_group_must_not_straddle_a_page():
+ with pytest.raises(ValueError, match="divisible"):
+ _pool(page_size=6, index_ratio=4)
+
+
+def test_index_slab_needs_a_two_byte_dtype():
+ with pytest.raises(AssertionError, match="2 bytes"):
+ QSAKVCache(
+ num_kv_heads=2, num_layers=8, head_dim=64, num_pages=4, page_size=64,
+ dtype=torch.float32, device=DEV, index_head_dim=32, num_index_layers=4,
+ index_ratio=4, num_req_slots=4, layer_ids=(1, 3, 5, 7),
+ )
+
+
+def test_shadow_row_is_shared_by_a_whole_group():
+ pool = _pool()
+ cmp = pool.cmp_k_cache(2)
+ row = torch.randn(32, dtype=torch.bfloat16)
+ for slot in (64, 65, 66, 67):
+ assert slot // pool.index_ratio == 16
+ cmp[16] = row
+ assert torch.equal(pool.cmp_k_cache(2)[16], row)
+ # the other sparse layers keep their own rows
+ assert pool.cmp_k_cache(1)[16].abs().sum().item() == 0.0
+
+
+def test_rebuild_resizes_every_tier_and_keeps_identity():
+ pool = _pool(num_pages=4)
+ ident = id(pool)
+ pool.rebuild(16)
+ assert id(pool) == ident
+ assert pool.k_cache(1).shape == (16, 64, 2, 64)
+ assert pool.cmp_scratch_base == 16 * 64 // 4
+ assert pool.cmp_k_cache(0).shape == (16 * 64 // 4 + 4, 32)
+ assert pool.pending_ring(3).shape == (4, pool.ring_capacity, 32)
+ assert pool._kv_buffer.shape[1] == 4 # sparse-layer slabs survive the resize
+ pool.k_cache(7)
+
+
+# ------------------------------------------------------------------------------ budgeting
+
+
+def test_spec_bytes_per_token_divides_the_index_slab():
+ spec = _spec(num_kv_heads=2, head_dim=256, index_head_dim=128, num_index_layers=12,
+ layer_ids=FULL_LAYER_IDS)
+ config = _config(spec)
+ assert spec_kv_bytes_per_token(spec, config) == REAL_KV_BYTES + REAL_INDEX_BYTES
+ assert spec_kv_bytes_per_token(spec, config) == 24576 + 768
+
+ # BSA/DSA keep one index row per token (ratio 1)
+ bsa = _spec(num_kv_heads=2, head_dim=256, index_head_dim=128, num_index_layers=12,
+ layer_ids=FULL_LAYER_IDS, index_ratio=1, attn_type=AttnType.BSA)
+ assert spec_kv_bytes_per_token(bsa, config) == REAL_KV_BYTES + 128 * 12 * 2
+
+
+def test_kv_cost_prices_ring_and_scratch_as_fixed():
+ spec = _spec()
+ config = _config(spec, max_running_req=3)
+ per_page, fixed, page_tokens, min_reserve = QSAKVCache.kv_cost(config)
+ assert per_page == spec_kv_bytes_per_token(spec, config) * 64
+ assert page_tokens == 64 and min_reserve == 0
+ row = 32 * 4 * 2
+ assert fixed == 4 * row * (QSAKVCache.ring_capacity_for(4) + 1)
+
+
+def test_unit_bytes_matches_the_cost_model():
+ spec = _spec()
+ config = _config(spec)
+ pool = _pool()
+ kv_bytes, swa_bytes = pool.unit_bytes()
+ assert swa_bytes == 0
+ # the scratch rows and the ring must NOT inflate the per-token slider
+ assert kv_bytes == spec_kv_bytes_per_token(spec, config)
+ assert kv_bytes * 64 == QSAKVCache.kv_cost(config)[0]
+
+
+def test_resolve_pool_class_and_factory():
+ from freetoken.kvcache import create_kvcache_pool, resolve_pool_class
+
+ spec = _spec()
+ mc = SimpleNamespace(
+ num_layers=8, has_swa_attention=False, has_linear_attention=True,
+ num_kv_heads=2, head_dim=64, dsv4_args=None,
+ )
+ mc.kv_cache_group_specs = lambda: (spec,)
+ assert resolve_pool_class(mc) is QSAKVCache
+
+ pool = create_kvcache_pool(
+ mc, num_pages=4, page_size=64, dtype=torch.bfloat16, device=DEV, num_req_slots=4
+ )
+ assert isinstance(pool, QSAKVCache)
+ assert pool._kv_buffer.shape[1] == 4 # not the model's 8 layers
+ assert pool.cmp_k_cache(0).shape == (4 * 64 // 4 + 4, 32)
+
+ with pytest.raises(ValueError, match="num_req_slots"):
+ create_kvcache_pool(mc, num_pages=4, page_size=64, dtype=torch.bfloat16, device=DEV)
diff --git a/tests/models/qwen4_exp/__init__.py b/tests/models/qwen4_exp/__init__.py
new file mode 100644
index 00000000..e69de29b
diff --git a/tests/models/qwen4_exp/common.py b/tests/models/qwen4_exp/common.py
new file mode 100644
index 00000000..1f9c117b
--- /dev/null
+++ b/tests/models/qwen4_exp/common.py
@@ -0,0 +1,257 @@
+"""Shared fixtures for the qwen4_exp tests: toy configs, hash constants, the QSA pool and backend.
+
+The geometry is the shipping one everywhere it matters for QSA (head_dim 256, index head_dim
+128 with a 64-wide partial rope, index_ratio 4, budget 2048 -> 512 blocks -> 2051 selected
+tokens, page_size 64); only the head counts, the hidden size and the layer count are scaled
+down so a test fits on a shared GPU. Holds no tests itself.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from freetoken.distributed import set_tp_info, try_get_tp_info
+
+EOS = 7
+VOCAB = 512
+
+requires_cuda = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs a GPU")
+
+
+def hf_config(
+ num_layers: int = 4,
+ head_dim: int = 256,
+ num_q: int = 4,
+ num_kv: int = 2,
+ index_head_dim: int = 128,
+ index_heads: int = 4,
+ budget: int = 2048,
+ ratio: int = 4,
+ hidden: int = 256,
+ max_position: int = 1 << 16,
+ rope_theta: float = 10000000.0,
+ **text_overrides,
+) -> SimpleNamespace:
+ text = SimpleNamespace(
+ num_hidden_layers=num_layers,
+ hidden_size=hidden,
+ vocab_size=VOCAB,
+ head_dim=head_dim,
+ num_attention_heads=num_q,
+ num_key_value_heads=num_kv,
+ layer_types=[
+ "full_attention" if (i + 1) % 4 == 0 else "linear_attention"
+ for i in range(num_layers)
+ ],
+ rope_parameters={
+ "rope_type": "default",
+ "rope_theta": rope_theta,
+ "partial_rotary_factor": 0.25,
+ },
+ max_position_embeddings=max_position,
+ rms_norm_eps=1e-6,
+ hidden_act="silu",
+ tie_word_embeddings=False,
+ num_experts=8,
+ num_experts_per_tok=2,
+ moe_intermediate_size=64,
+ shared_expert_intermediate_size=64,
+ linear_num_key_heads=2,
+ linear_num_value_heads=6,
+ linear_key_head_dim=32,
+ linear_value_head_dim=32,
+ linear_conv_kernel_dim=4,
+ output_gate_type="sigmoid",
+ indexer_n_heads=index_heads,
+ indexer_kv_heads=1,
+ indexer_head_dim=index_head_dim,
+ indexer_budget=budget,
+ indexer_compress_ratio=ratio,
+ hc_count=4,
+ hc_lowrank=16,
+ ple_layer_ids=[2],
+ ple_embed_dim=64,
+ ple_conv_kernel_size=4,
+ ngram_size=3,
+ heads_per_ngram=2,
+ ngram_vocab_size_base=1000,
+ make_ngram_vocab_size_divisible_by=8,
+ split_ngram_parts=4,
+ eos_token_id=EOS,
+ )
+ for name, value in text_overrides.items():
+ setattr(text, name, value)
+ return SimpleNamespace(
+ model_type="qwen4_exp",
+ architectures=["Qwen4ExpForConditionalGeneration"],
+ text_config=text,
+ quantization_config=None,
+ )
+
+
+def toy_hf_config(num_layers: int = 4, **text_overrides) -> SimpleNamespace:
+ """The small-geometry config the PLE/skeleton tests share (hidden 128, head_dim 64)."""
+ return hf_config(
+ num_layers=num_layers, head_dim=64, num_kv=1, index_head_dim=64, index_heads=2,
+ budget=16, hidden=128, max_position=4096, rope_theta=10000.0, **text_overrides,
+ )
+
+
+def hash_constants(args):
+ """Checkpoint-shape int64 hash tensors, derived like the dummy-weight path."""
+ from freetoken.models.qwen4_exp.ple import derive_ngram_hash_constants
+
+ multipliers, sizes, offsets = derive_ngram_hash_constants(
+ vocab_size=VOCAB,
+ ngram_size=args.ngram_size,
+ num_ngram_heads=args.num_ngram_heads,
+ ngram_vocab_size_base=args.ngram_vocab_size_base,
+ ple_layer_index=0,
+ )
+ return [torch.tensor(v, dtype=torch.int64) for v in (multipliers, sizes, offsets)]
+
+
+def parsed_config(**kwargs):
+ from freetoken.models.qwen4_exp.config import parse_config
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+ return parse_config(hf_config(**kwargs))
+
+
+def fresh_ctx(page_size: int = 64, **fields):
+ import freetoken.core as core
+ from freetoken.core import Context, set_global_ctx
+
+ core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx
+ ctx = Context(page_size=page_size)
+ for name, value in fields.items():
+ setattr(ctx, name, value)
+ set_global_ctx(ctx)
+ return ctx
+
+
+def fill_weights(op, seed: int, device: torch.device, scale: float = 0.05) -> None:
+ gen = torch.Generator(device=device).manual_seed(seed)
+ for tensor in op.state_dict().values():
+ if tensor.is_floating_point():
+ tensor.normal_(0.0, scale, generator=gen)
+ else:
+ tensor.zero_()
+
+
+class Fixture:
+ """QSA pool + page table + the sparse backend, with a first-fit page allocator."""
+
+ def __init__(
+ self,
+ config,
+ num_pages: int,
+ max_running_req: int = 8,
+ device: str = "cuda",
+ dtype: torch.dtype = torch.bfloat16,
+ page_size: int = 64,
+ ) -> None:
+ from freetoken.attention.qsa_sparse import QSASparseAttnBackend
+ from freetoken.kvcache import create_kvcache_pool
+
+ self.config = config
+ self.device = torch.device(device)
+ self.dtype = dtype
+ self.page_size = page_size
+ self.num_req_slots = max_running_req + 1
+ self.pool = create_kvcache_pool(
+ model_config=config,
+ num_pages=num_pages + 1, # + 1 for the dummy page, as create_kv_pool does
+ page_size=page_size,
+ dtype=dtype,
+ device=self.device,
+ num_req_slots=self.num_req_slots,
+ )
+ self.page_table = torch.zeros(
+ (self.num_req_slots, num_pages * page_size), dtype=torch.int32, device=self.device
+ )
+ self.page_table[max_running_req].fill_(num_pages * page_size) # dummy page
+ self.ctx = fresh_ctx(
+ page_size=page_size, page_table=self.page_table, kv_cache=self.pool
+ )
+ self.backend = QSASparseAttnBackend(config)
+ self.ctx.attn_backend = self.backend
+ self._free = list(range(num_pages))
+
+ def layer(self, layer_id: int, seed: int = 1):
+ from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention
+ from freetoken.utils.torch_utils import torch_dtype
+
+ with torch.device(self.device), torch_dtype(self.dtype):
+ attn = Qwen4ExpAttention(self.config, layer_id=layer_id)
+ fill_weights(attn, seed, self.device)
+ return attn
+
+ def allocate(self, table_idx: int, cached_len: int, device_len: int) -> None:
+ for page in range(-(-cached_len // self.page_size), -(-device_len // self.page_size)):
+ base = self._free.pop(0) * self.page_size
+ columns = slice(page * self.page_size, (page + 1) * self.page_size)
+ self.page_table[table_idx, columns] = torch.arange(
+ base, base + self.page_size, dtype=torch.int32, device=self.device
+ )
+
+ def req(self, table_idx: int, cached_len: int, device_len: int) -> SimpleNamespace:
+ self.allocate(table_idx, cached_len, device_len)
+ return SimpleNamespace(
+ table_idx=table_idx,
+ cached_len=cached_len,
+ device_len=device_len,
+ extend_len=device_len - cached_len,
+ )
+
+ def step(self, req: SimpleNamespace) -> None:
+ self.allocate(req.table_idx, req.device_len, req.device_len + 1)
+ req.cached_len, req.device_len, req.extend_len = req.device_len, req.device_len + 1, 1
+
+ def batch(self, reqs, phase: str) -> SimpleNamespace:
+ positions = torch.cat(
+ [
+ torch.arange(r.cached_len, r.device_len, dtype=torch.int32, device=self.device)
+ for r in reqs
+ ]
+ )
+ out_loc = torch.cat(
+ [self.page_table[r.table_idx, r.cached_len : r.device_len] for r in reqs]
+ ).contiguous()
+ batch = SimpleNamespace(
+ reqs=reqs,
+ padded_reqs=reqs,
+ phase=phase,
+ size=len(reqs),
+ padded_size=len(reqs),
+ is_prefill=phase == "prefill",
+ is_decode=phase == "decode",
+ positions=positions,
+ out_loc=out_loc,
+ attn_metadata=None,
+ active_table_idx=torch.tensor(
+ [r.table_idx for r in reqs], dtype=torch.int32, device=self.device
+ ),
+ )
+ self.backend.prepare_metadata(batch)
+ return batch
+
+
+def selection_spy(monkeypatch, backend) -> dict:
+ """Record the expanded token selection of every ``_select`` call."""
+ from freetoken.attention.qsa_sparse import QSASparseAttnBackend
+
+ seen: dict[str, torch.Tensor] = {}
+ original = QSASparseAttnBackend._select
+
+ def spy(self, index, md, slot):
+ indices = original(self, index, md, slot)
+ seen["indices"] = indices.clone()
+ return indices
+
+ monkeypatch.setattr(QSASparseAttnBackend, "_select", spy)
+ return seen
diff --git a/tests/models/qwen4_exp/conftest.py b/tests/models/qwen4_exp/conftest.py
new file mode 100644
index 00000000..2b9e8021
--- /dev/null
+++ b/tests/models/qwen4_exp/conftest.py
@@ -0,0 +1,15 @@
+"""Package-wide runtime hygiene: TP info set once, the global ctx never leaks across tests."""
+
+import pytest
+
+
[email protected](autouse=True)
+def _runtime():
+ import freetoken.core as core
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+ core._GLOBAL_CTX = None
+ yield
+ core._GLOBAL_CTX = None
diff --git a/tests/models/qwen4_exp/ple_hf_ref.py b/tests/models/qwen4_exp/ple_hf_ref.py
new file mode 100644
index 00000000..35c52955
--- /dev/null
+++ b/tests/models/qwen4_exp/ple_hf_ref.py
@@ -0,0 +1,70 @@
+"""HF ground truth for the PLE parity tests, run as a script under the transformers-main venv.
+
+``transformers`` in the serving venv predates ``models/qwen4_exp``, so the reference classes cannot
+be imported next to FreeToken. ``test_ple.py`` spawns this file with the reference interpreter:
+``python test_ple_hf_ref.py spec.json inputs.npz out.npz``. It holds no tests; every import is
+inside ``main`` so pytest can still collect the module.
+"""
+
+from __future__ import annotations
+
+
+def main() -> None:
+ import json
+ import sys
+
+ import numpy as np
+ import torch
+ from torch import nn
+ from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig
+ from transformers.models.qwen4_exp.modeling_qwen4_exp import (
+ Qwen4ExpTextNGramEmbedding,
+ Qwen4ExpTextPLELayer,
+ )
+
+ class CaptureEmbedding(nn.Module):
+ """Stands in for the table so the hashed ids can be read out of the HF module."""
+
+ def __init__(self, dim: int) -> None:
+ super().__init__()
+ self.weight = nn.Parameter(torch.zeros(1, dim), requires_grad=False)
+ self.ids = None
+
+ def forward(self, ids: torch.Tensor) -> torch.Tensor:
+ self.ids = ids.clone()
+ return torch.zeros(*ids.shape, self.weight.shape[1])
+
+ with open(sys.argv[1], encoding="utf-8") as fh:
+ spec = json.load(fh)
+ data = np.load(sys.argv[2])
+ config = Qwen4ExpTextConfig(**spec["config"])
+ layer_idx, ple_index = spec["layer_idx"], spec["ple_layer_index"]
+ out = {}
+
+ embed = Qwen4ExpTextNGramEmbedding(config, config.ple_embed_dim, layer_idx, ple_index)
+ out["layer_multipliers"] = embed.layer_multipliers.numpy()
+ out["ngram_heads_vocab_sizes"] = embed.ngram_heads_vocab_sizes.numpy()
+ out["ngram_heads_offsets"] = embed.ngram_heads_offsets.numpy()
+ out["padded_vocab_size"] = np.array(embed.ngram_embedding.weight.shape[0])
+
+ capture = CaptureEmbedding(embed.ngram_embedding.embedding_dim)
+ embed.ngram_embedding = capture
+ embed(torch.as_tensor(data["hash_tokens"]).long(), None)
+ out["hash_ids"] = capture.ids.numpy()
+
+ layer = Qwen4ExpTextPLELayer(config, layer_idx, ple_index)
+ with torch.no_grad():
+ for name in ("key_proj", "value_proj", "norm_key", "norm_query", "norm_conv"):
+ getattr(layer, name).weight.copy_(torch.as_tensor(data[name]))
+ layer.conv1d.weight.copy_(torch.as_tensor(data["conv1d"]))
+ layer.ple_embedding.ngram_embedding.weight.copy_(torch.as_tensor(data["table"]))
+ out["layer_out"] = layer(
+ torch.as_tensor(data["hidden"]).float(),
+ torch.as_tensor(data["layer_tokens"]).long(),
+ None,
+ ).numpy()
+ np.savez(sys.argv[3], **out)
+
+
+if __name__ == "__main__":
+ main()
diff --git a/tests/models/qwen4_exp/test_config.py b/tests/models/qwen4_exp/test_config.py
new file mode 100644
index 00000000..8fc2e550
--- /dev/null
+++ b/tests/models/qwen4_exp/test_config.py
@@ -0,0 +1,170 @@
+"""qwen4_exp.parse_config against a synthetic config shaped like the RadixArk NVFP4 checkpoint."""
+
+from types import SimpleNamespace
+
+import pytest
+
+from freetoken.attention import AttnType
+from freetoken.models.config import FullAttentionGroupConfig, LinearGatedDeltaGroupConfig
+from freetoken.models.qwen4_exp.config import parse_config
+
+
+def _text_config():
+ return SimpleNamespace(
+ num_hidden_layers=48,
+ hidden_size=2560,
+ vocab_size=248320,
+ head_dim=256,
+ num_attention_heads=24,
+ num_key_value_heads=2,
+ layer_types=[
+ "full_attention" if (i + 1) % 4 == 0 else "linear_attention" for i in range(48)
+ ],
+ rope_parameters={
+ "rope_type": "default",
+ "rope_theta": 10000000,
+ "partial_rotary_factor": 0.25,
+ "mrope_interleaved": True,
+ "mrope_section": [11, 11, 10],
+ },
+ max_position_embeddings=262144,
+ rms_norm_eps=1e-6,
+ hidden_act="silu",
+ tie_word_embeddings=False,
+ num_experts=512,
+ num_experts_per_tok=10,
+ moe_intermediate_size=640,
+ shared_expert_intermediate_size=640,
+ linear_num_key_heads=16,
+ linear_num_value_heads=48,
+ linear_key_head_dim=128,
+ linear_value_head_dim=128,
+ linear_conv_kernel_dim=4,
+ output_gate_type="sigmoid",
+ indexer_n_heads=4,
+ indexer_kv_heads=1,
+ indexer_head_dim=128,
+ indexer_budget=2048,
+ indexer_compress_ratio=4,
+ hc_count=4,
+ hc_lowrank=320,
+ ple_layer_ids=[2],
+ ple_embed_dim=2560,
+ ple_conv_kernel_size=4,
+ ngram_size=3,
+ heads_per_ngram=8,
+ ngram_vocab_size_base=20000000,
+ make_ngram_vocab_size_divisible_by=128,
+ split_ngram_parts=128,
+ bos_token_id=248044,
+ eos_token_id=248044,
+ )
+
+
+def _hf_config():
+ return SimpleNamespace(
+ model_type="qwen4_exp",
+ architectures=["Qwen4ExpForConditionalGeneration"],
+ image_token_id=248056,
+ text_config=_text_config(),
+ quantization_config={
+ "quant_algo": "NVFP4",
+ "quant_method": "modelopt",
+ "ignore": [
+ "model.embed_tokens",
+ "mtp.*",
+ "model.mtp.*",
+ "*.self_attn.*",
+ "*.linear_attn.*",
+ "*.mlp.gate*",
+ "*.mlp.shared_expert.*",
+ "*.mlp.shared_expert_gate*",
+ "*hyper_connection*",
+ "*.ple.*",
+ "model.visual.*",
+ "model.language_model.embed_tokens",
+ "lm_head",
+ ],
+ },
+ )
+
+
+def test_groups_and_layer_split():
+ cfg = parse_config(_hf_config())
+ full = [g for g in cfg.attention_groups if isinstance(g, FullAttentionGroupConfig)]
+ linear = [g for g in cfg.attention_groups if isinstance(g, LinearGatedDeltaGroupConfig)]
+ assert len(full) == 1 and len(linear) == 1
+ assert full[0].layer_ids == tuple(range(3, 48, 4))
+ assert len(linear[0].layer_ids) == 36
+ assert full[0].index_head_dim == 128
+ assert full[0].num_index_layers == 12
+ assert full[0].index_ratio == 4
+ assert full[0].rotary_config.rotary_dim == 64
+ assert full[0].rotary_config.scaling is None
+ assert linear[0].output_gate == "sigmoid"
+ assert linear[0].num_key_heads == 16 and linear[0].num_value_heads == 48
+
+
+def test_kv_specs_resolve_qsa():
+ cfg = parse_config(_hf_config())
+ specs = {s.name: s for s in cfg.kv_cache_group_specs()}
+ assert specs["full"].attn_type is AttnType.QSA
+ assert specs["full"].index_ratio == 4
+ assert cfg.attn_type_for_layer(3) is AttnType.QSA
+ assert cfg.attn_type_for_layer(0) is AttnType.LINEAR
+ assert cfg.has_linear_attention
+
+
+def test_moe_and_quant_flags():
+ cfg = parse_config(_hf_config())
+ assert cfg.num_experts == 512
+ assert cfg.num_experts_per_tok == 10
+ assert cfg.norm_topk_prob is True
+ assert cfg.moe_enabled
+ assert cfg.expert_quant == "nvfp4"
+ assert cfg.dense_quant == "none"
+ assert cfg.attn_quant == "none"
+ assert cfg.lm_head_quant == "none"
+
+
+def test_unquantized_config_parses():
+ hf = _hf_config()
+ hf.quantization_config = None
+ assert parse_config(hf).expert_quant == "none"
+
+
+def test_qwen4_args_payload():
+ args = parse_config(_hf_config()).qwen4_args
+ assert args.ple_layer_ids == (1,)
+ assert args.hc_count == 4 and args.hc_lowrank == 320
+ assert args.index_topk_blocks == 512
+ assert args.num_ngram_heads == 16
+ assert args.ngram_head_dim == 160
+ assert args.ple_conv_state_len == 9
+ assert args.ple_state_width == 10240
+ assert args.ngram_boundary_token_id == 248044
+
+
+def test_ple_on_full_attention_layer_rejected():
+ hf = _hf_config()
+ hf.text_config.ple_layer_ids = [4] # one-indexed 4 == zero-based 3, a full_attention layer
+ with pytest.raises(ValueError, match="linear_attention"):
+ parse_config(hf)
+
+
+def test_output_gate_null_falls_back_to_hidden_act():
+ hf = _hf_config()
+ hf.text_config.output_gate_type = None
+ linear = [
+ g
+ for g in parse_config(hf).attention_groups
+ if isinstance(g, LinearGatedDeltaGroupConfig)
+ ]
+ assert linear[0].output_gate == "silu"
+
+
+def test_eos_token_id_list_uses_the_first_entry():
+ base = parse_config(_hf_config()).qwen4_args.ngram_boundary_token_id
+ hf = _hf_config()
+ hf.text_config.eos_token_id = [base, base + 1]
+ assert parse_config(hf).qwen4_args.ngram_boundary_token_id == base
diff --git a/tests/models/qwen4_exp/test_gdn.py b/tests/models/qwen4_exp/test_gdn.py
new file mode 100644
index 00000000..81dd0e76
--- /dev/null
+++ b/tests/models/qwen4_exp/test_gdn.py
@@ -0,0 +1,175 @@
+"""qwen4_exp GatedDeltaNet op vs the pure-torch HF reference math.
+
+The oracle is ``models/qwen4_exp/gdn_reference.py``, whose two delta rules and forward are
+transcribed from the ``modeling_qwen4_exp.py`` snapshot, so no transformers build carrying
+qwen4_exp is needed here. Covered: prefill at 128 and 1000 tokens, a decode step continuing
+from the prefill state, ragged bs=3, both GQA head ratios, and the sigmoid output gate.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from freetoken.core import Batch, Context, Req, SamplingParams
+from freetoken.models.config import LinearGatedDeltaGroupConfig
+from freetoken.models.qwen4_exp.gdn import Qwen4ExpGatedDeltaNet
+from freetoken.models.qwen4_exp.gdn_reference import Qwen4ExpGatedDeltaNetReference
+from freetoken.utils import torch_dtype
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+DEV = torch.device("cuda")
+HIDDEN, HEAD_DIM, CONV_K, EPS = 256, 128, 4, 1e-6
+RTOL = ATOL = 2e-2
+# (num_k_heads, num_v_heads) per value:key head ratio; 3:1 is the Qwen3.8-Flash-Next shape.
+HEADS = {2: (8, 16), 3: (16, 48)}
+
+
+def _bf(t: torch.Tensor) -> torch.Tensor:
+ return t.detach().to(DEV, torch.bfloat16)
+
+
+def _state_dict(ref) -> dict[str, torch.Tensor]:
+ """HF's four in_proj matrices fused into the op's single qkv|z|b|a GEMM. A_log / dt_bias
+ stay fp32, as the weight loader keeps them."""
+ return {
+ "in_proj.weight": _bf(torch.cat([ref.in_proj_qkv.weight, ref.in_proj_z.weight,
+ ref.in_proj_b.weight, ref.in_proj_a.weight], dim=0)),
+ "conv1d.weight": _bf(ref.conv1d.weight),
+ "dt_bias": ref.dt_bias.detach().to(DEV, torch.float32),
+ "A_log": ref.A_log.detach().to(DEV, torch.float32),
+ "norm.weight": _bf(ref.norm.weight),
+ "out_proj.weight": _bf(ref.out_proj.weight),
+ }
+
+
+def _make_layer(ratio: int, output_gate: str = "sigmoid", seed: int = 0):
+ """fp32 reference + bf16 kernel op over one set of weights. The op is built on meta under
+ the serving dtype, the way the engine builds a model, so load_state_dict's dtype check bites."""
+ num_k, num_v = HEADS[ratio]
+ torch.manual_seed(seed)
+ ref = Qwen4ExpGatedDeltaNetReference(
+ hidden_size=HIDDEN, num_k_heads=num_k, num_v_heads=num_v, head_k_dim=HEAD_DIM,
+ head_v_dim=HEAD_DIM, conv_kernel_size=CONV_K, rms_norm_eps=EPS, output_gate=output_gate,
+ ).to(DEV).float().eval()
+ with torch.no_grad():
+ # HF inits A_log = log(U(0.01, 16)); a zero dt_bias or a unit gate norm would hide sign errors.
+ ref.A_log.uniform_(0.01, 16.0).log_()
+ ref.dt_bias.uniform_(-1.0, 1.0)
+ ref.norm.weight.normal_(1.0, 0.1)
+ with torch.device("meta"), torch_dtype(torch.bfloat16):
+ op = Qwen4ExpGatedDeltaNet(
+ hidden_size=HIDDEN, num_k_heads=num_k, num_v_heads=num_v, head_k_dim=HEAD_DIM,
+ head_v_dim=HEAD_DIM, conv_kernel_size=CONV_K, rms_norm_eps=EPS, layer_id=0,
+ output_gate=output_gate,
+ )
+ op.load_state_dict(_state_dict(ref))
+ return op, ref
+
+
+def _ctx(ratio: int, num_slots: int = 8) -> Context:
+ import freetoken.core as core
+ from freetoken.kvcache.linear_state_pool import LinearStatePool
+
+ num_k, num_v = HEADS[ratio]
+ group = LinearGatedDeltaGroupConfig(
+ name="linear", layer_ids=(0,), num_key_heads=num_k, num_value_heads=num_v,
+ key_head_dim=HEAD_DIM, value_head_dim=HEAD_DIM, conv_kernel_dim=CONV_K,
+ output_gate="sigmoid",
+ )
+ core._GLOBAL_CTX = None
+ ctx = Context(page_size=64)
+ ctx.linear_state_pool = LinearStatePool(group, num_slots, torch.bfloat16, DEV, tp_size=1)
+ core.set_global_ctx(ctx)
+ return ctx
+
+
+def _prefill(op, ctx: Context, lengths: list[int], seed: int):
+ """One ragged prefill batch, one state slot per request. Returns the per-request hidden
+ states, the reqs (for a follow-up decode) and the packed output."""
+ torch.manual_seed(seed)
+ hidden = [torch.randn(n, HIDDEN, device=DEV, dtype=torch.bfloat16) for n in lengths]
+ reqs = [
+ Req(input_ids=torch.zeros(n, dtype=torch.int32), table_idx=i + 1, cached_len=0,
+ output_len=1, uid=i, sampling_params=SamplingParams(), cache_handle=None)
+ for i, n in enumerate(lengths)
+ ]
+ batch = Batch(reqs=reqs, phase="prefill")
+ batch.padded_reqs = reqs
+ with ctx.forward_batch(batch):
+ out = op.forward(torch.cat(hidden, dim=0))
+ return hidden, reqs, out
+
+
+def _decode(op, ctx: Context, reqs, hidden: torch.Tensor) -> torch.Tensor:
+ batch = Batch(reqs=reqs, phase="decode")
+ batch.padded_reqs = reqs
+ batch.linear_table_idx = torch.tensor(
+ [r.table_idx for r in reqs], dtype=torch.int32, device=DEV
+ )
+ with ctx.forward_batch(batch):
+ return op.forward(hidden)
+
+
+def _ref_out(ref, hidden: torch.Tensor, use_chunk_rule: bool = False) -> torch.Tensor:
+ return ref(hidden.float().unsqueeze(0), use_chunk_rule=use_chunk_rule)[0]
+
+
[email protected]("length", (1000,))
[email protected]("ratio", (2, 3))
+def test_prefill_matches_reference(ratio, length):
+ op, ref = _make_layer(ratio, seed=ratio)
+ hidden, _, out = _prefill(op, _ctx(ratio), [length], seed=11)
+ torch.testing.assert_close(out.float(), _ref_out(ref, hidden[0]), rtol=RTOL, atol=ATOL)
+
+
[email protected]("ratio", (2, 3))
+def test_ragged_prefill_then_decode(ratio):
+ """bs=3 ragged prefill, then one decode step per request off the carried conv + recurrent
+ state. The decode oracle is the whole (prefill + 1) sequence in one reference pass, so a
+ state that did not survive the prefill shows up immediately."""
+ op, ref = _make_layer(ratio, seed=ratio)
+ ctx = _ctx(ratio)
+ lengths = [128, 1000, 37]
+ hidden, reqs, out = _prefill(op, ctx, lengths, seed=13)
+
+ off = 0
+ for h, n in zip(hidden, lengths):
+ torch.testing.assert_close(
+ out[off:off + n].float(), _ref_out(ref, h), rtol=RTOL, atol=ATOL
+ )
+ off += n
+
+ nxt = torch.randn(len(lengths), HIDDEN, device=DEV, dtype=torch.bfloat16)
+ dec = _decode(op, ctx, reqs, nxt)
+ for i, h in enumerate(hidden):
+ full = _ref_out(ref, torch.cat([h, nxt[i:i + 1]], dim=0))
+ torch.testing.assert_close(dec[i].float(), full[-1], rtol=RTOL, atol=ATOL)
+
+
+def test_chunk_and_recurrent_rules_agree():
+ """The chunked form (what the fla prefill kernel implements) against the sequential
+ definition, both fp32: the chunk oracle is only worth anything if it reproduces the
+ recurrence to fp32 precision."""
+ _, ref = _make_layer(3, seed=1)
+ torch.manual_seed(17)
+ hidden = torch.randn(1000, HIDDEN, device=DEV, dtype=torch.bfloat16)
+ torch.testing.assert_close(
+ _ref_out(ref, hidden, use_chunk_rule=True), _ref_out(ref, hidden), rtol=1e-4, atol=1e-4
+ )
+
+
+def test_output_gate_comes_from_the_config():
+ """The gate activation is the group config's string, not a hardcoded silu. Both gates track
+ their own reference, and the two are far apart -- so a stuck activation cannot pass."""
+ op_silu, ref_silu = _make_layer(3, output_gate="silu", seed=2)
+ hidden, _, out_silu = _prefill(op_silu, _ctx(3), [128], seed=19)
+ torch.testing.assert_close(out_silu.float(), _ref_out(ref_silu, hidden[0]), rtol=RTOL, atol=ATOL)
+
+ op_sig, ref_sig = _make_layer(3, output_gate="sigmoid", seed=2)
+ _, _, out_sig = _prefill(op_sig, _ctx(3), [128], seed=19)
+ torch.testing.assert_close(out_sig.float(), _ref_out(ref_sig, hidden[0]), rtol=RTOL, atol=ATOL)
+
+ assert (out_sig.float() - out_silu.float()).abs().max().item() > 10 * ATOL
diff --git a/tests/models/qwen4_exp/test_ple.py b/tests/models/qwen4_exp/test_ple.py
new file mode 100644
index 00000000..0d246142
--- /dev/null
+++ b/tests/models/qwen4_exp/test_ple.py
@@ -0,0 +1,697 @@
+"""PLE layer acceptance: hash vs HF, pinned-host table vs the GPU oracle, conv state
+advancement (prefill / chunked / stepwise decode), CUDA-graph decode, and prefetch overlap.
+
+The HF ground truth comes from ``ple_hf_ref.py`` run under a transformers build that ships
+qwen4_exp (``FREETOKEN_QWEN4_HF_PYTHON``); those tests skip when it is unset.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+from pathlib import Path
+from types import SimpleNamespace
+
+import numpy as np
+import pytest
+import torch
+
+from freetoken.models.config import ModelConfig
+from freetoken.models.qwen4_exp.config import parse_config
+from freetoken.models.qwen4_exp.ple import (
+ GpuResidentTable,
+ PinnedUVATable,
+ PLELayer,
+ PLEMetadata,
+ build_ple_metadata,
+ commit_ngram_context,
+ short_conv_reference,
+)
+
+from .common import EOS, VOCAB, hash_constants, requires_cuda, toy_hf_config
+
+_HF_REF_PYTHON = os.environ.get("FREETOKEN_QWEN4_HF_PYTHON", "")
+_HF_REF_SCRIPT = Path(__file__).with_name("ple_hf_ref.py")
+
+requires_hf_ref = pytest.mark.skipif(
+ not (_HF_REF_PYTHON and Path(_HF_REF_PYTHON).exists()),
+ reason="set FREETOKEN_QWEN4_HF_PYTHON to a transformers build that ships qwen4_exp",
+)
+
+
+# --------------------------------------------------------------------------------------
+# fixtures
+# --------------------------------------------------------------------------------------
+
+def _config() -> ModelConfig:
+ return parse_config(toy_hf_config())
+
+
+def _padded_vocab(args) -> int:
+ """HF pads the concatenated per-head vocabs up to make_ngram_vocab_size_divisible_by."""
+ _, sizes, _ = hash_constants(args)
+ div = args.make_ngram_vocab_size_divisible_by
+ return -(-int(sizes.sum()) // div) * div
+
+
+def _make_layer(config, *, device="cpu", dtype=torch.float32, rows=None, seed=3, table=None):
+ from freetoken.utils.torch_utils import torch_dtype
+
+ args = config.qwen4_args
+ rows = _padded_vocab(args) if rows is None else rows
+ device = torch.device(device)
+ gen = torch.Generator(device=device).manual_seed(seed)
+ with torch.device(device), torch_dtype(dtype):
+ layer = PLELayer(config, args.ple_layer_ids[0])
+ for tensor in layer.state_dict().values():
+ if tensor.is_floating_point():
+ tensor.normal_(0.0, 0.05, generator=gen)
+ multipliers, sizes, offsets = hash_constants(args)
+ layer.ple_embedding.layer_multipliers.copy_(multipliers)
+ layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)
+ layer.ple_embedding.ngram_heads_offsets.copy_(offsets)
+ if table is None:
+ weight = torch.randn(rows, args.ngram_head_dim, generator=gen, device=device, dtype=dtype)
+ table = GpuResidentTable(weight * 0.05, dtype=dtype)
+ layer.ple_embedding.attach_table(table)
+ return layer
+
+
+def _meta(sequences, contexts, *, device="cpu", slots=None, fresh=None, decode=False):
+ lens = [len(s) for s in sequences]
+ to = lambda xs, dtype: torch.tensor(xs, dtype=dtype, device=device)
+ cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0).to(device)
+ return PLEMetadata(
+ input_ids=to([t for s in sequences for t in s], torch.int64),
+ cu_seqlens=cu,
+ seq_lens=tuple(lens),
+ ngram_context=to(contexts, torch.int64),
+ state_slots=(
+ torch.arange(len(sequences), dtype=torch.int64, device=device)
+ if slots is None
+ else to(slots, torch.int64)
+ ),
+ fresh_slots=None if fresh is None else to(fresh, torch.bool),
+ is_decode=decode,
+ )
+
+
+def _run_hf_reference(tmp_path, data: dict, layer_idx=2, ple_layer_index=0) -> dict:
+ spec = {"config": vars(toy_hf_config().text_config), "layer_idx": layer_idx, "ple_layer_index": ple_layer_index}
+ (tmp_path / "spec.json").write_text(json.dumps(spec), encoding="utf-8")
+ np.savez(tmp_path / "in.npz", **data)
+ subprocess.run(
+ [_HF_REF_PYTHON, str(_HF_REF_SCRIPT), str(tmp_path / "spec.json"),
+ str(tmp_path / "in.npz"), str(tmp_path / "out.npz")],
+ check=True,
+ capture_output=True,
+ )
+ return dict(np.load(tmp_path / "out.npz"))
+
+
+# --------------------------------------------------------------------------------------
+# hash
+# --------------------------------------------------------------------------------------
+
+# sequence start (all-eos context), eos inside the chunk, eos as the newest context token
+_HASH_CASES = [
+ ([EOS, EOS], [3, 4, EOS, 5, 6, 8]),
+ ([21, 22], [2, EOS, 11, 12, 13, 14]),
+ ([EOS, 31], [9, 10, 11, 12, 13, 14]),
+ ([31, EOS], [9, 10, 11, 12, 13, 14]),
+]
+
+
+@requires_hf_ref
+def test_hash_ids_match_hf(tmp_path):
+ """row_ids equals HF Qwen4ExpTextNGramEmbedding per id, over eos boundaries and at sequence start."""
+ config = _config()
+ layer = _make_layer(config)
+ # HF pads its own all-eos context, so feeding [context | tokens] reproduces a resumed request
+ tokens = np.array([c + s for c, s in _HASH_CASES], dtype=np.int64)
+ ref = _run_hf_reference(tmp_path, {"hash_tokens": tokens, **_layer_ref_inputs(config, layer)})
+ hf_ids = torch.as_tensor(ref["hash_ids"])[:, len(_HASH_CASES[0][0]) :]
+
+ contexts = [c for c, _ in _HASH_CASES]
+ sequences = [s for _, s in _HASH_CASES]
+ got = layer.ple_embedding.row_ids(_meta(sequences, contexts))
+ offset = 0
+ for i, seq in enumerate(sequences):
+ assert torch.equal(got[offset : offset + len(seq)], hf_ids[i]), f"request {i}"
+ offset += len(seq)
+
+
+@requires_hf_ref
+def test_hash_constants_match_hf(tmp_path):
+ """derive_ngram_hash_constants reproduces the multipliers/vocab sizes/offsets HF builds at init."""
+ config = _config()
+ layer = _make_layer(config)
+ ref = _run_hf_reference(
+ tmp_path,
+ {"hash_tokens": np.array([[EOS, EOS, 3, 4]], dtype=np.int64), **_layer_ref_inputs(config, layer)},
+ )
+ multipliers, sizes, offsets = hash_constants(config.qwen4_args)
+ assert torch.equal(multipliers, torch.as_tensor(ref["layer_multipliers"]))
+ assert torch.equal(sizes, torch.as_tensor(ref["ngram_heads_vocab_sizes"]))
+ assert torch.equal(offsets, torch.as_tensor(ref["ngram_heads_offsets"]))
+
+
+def test_decode_hash_matches_prefill_hash():
+ """The decode window (context + one token) hashes to the same ids as the same token in a prefill."""
+ config = _config()
+ layer = _make_layer(config)
+ sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]]
+ contexts = [[EOS, EOS], [21, 22]]
+ prefill = layer.ple_embedding.row_ids(_meta(sequences, contexts))
+ for step in range(len(sequences[0])):
+ window = [(contexts[i] + s)[step : step + 2] for i, s in enumerate(sequences)]
+ got = layer.ple_embedding.row_ids(
+ _meta([[s[step]] for s in sequences], window, decode=True)
+ )
+ for i in range(len(sequences)):
+ assert torch.equal(got[i], prefill[i * len(sequences[0]) + step])
+
+
+# --------------------------------------------------------------------------------------
+# table backends
+# --------------------------------------------------------------------------------------
+
+
+def _pinned_bank(rows: int, dim: int, dtype: torch.dtype, seed: int = 11):
+ from freetoken.moe.host_banks import HostBank
+
+ gen = torch.Generator().manual_seed(seed)
+ bank = HostBank((rows, dim), dtype)
+ bank.tensor.copy_((torch.randn(rows, dim, generator=gen) * 0.4).to(dtype))
+ bank.pin()
+ return bank
+
+
+@requires_cuda
[email protected]("dtype", [torch.float8_e4m3fn, torch.bfloat16])
+def test_pinned_uva_matches_gpu_resident(dtype):
+ """PinnedUVATable is bitwise equal to the GPU-resident oracle, through lookup and prefetch."""
+ rows, dim, scale = 8192, 160, 0.0234375
+ bank = _pinned_bank(rows, dim, dtype)
+ oracle = GpuResidentTable(bank.tensor.cuda(), scale, dtype=torch.bfloat16)
+ pinned = PinnedUVATable(bank.tensor, scale)
+
+ ids = torch.randint(0, rows, (37, 16), device="cuda")
+ want = oracle.lookup(ids)
+ assert torch.equal(pinned.lookup(ids), want)
+
+ pinned.prefetch(ids)
+ assert torch.equal(pinned.lookup(ids), want)
+
+ # a stale prefetch must still be joined before its staging buffer is reused
+ pinned.prefetch(ids)
+ other = torch.randint(0, rows, (37, 16), device="cuda")
+ assert torch.equal(pinned.lookup(other), oracle.lookup(other))
+
+ out = torch.empty(37, 16 * dim, dtype=torch.bfloat16, device="cuda")
+ assert pinned.lookup(ids, out) is out
+ assert torch.equal(out, want)
+
+
+@requires_cuda
+def test_pinned_uva_zeroes_out_of_range_ids():
+ bank = _pinned_bank(64, 160, torch.float8_e4m3fn)
+ pinned = PinnedUVATable(bank.tensor, 1.0)
+ ids = torch.tensor([[0, 64, 1, -1]], device="cuda")
+ rows = pinned.lookup(ids).view(4, 160)
+ assert rows[1].abs().sum() == 0 and rows[3].abs().sum() == 0
+ assert torch.equal(rows[0], bank.tensor[0].cuda().to(torch.bfloat16))
+
+
+ not os.environ.get("FREETOKEN_QWEN4EXP_MODEL"), reason="needs FREETOKEN_QWEN4EXP_MODEL"
+)
+@requires_cuda
+def test_pinned_uva_real_table():
+ """The real 47.7 GiB FP8 table: sampled rows equal the checkpoint bytes dequantized on CPU."""
+ import safetensors
+ from freetoken.models.qwen4_exp.weight import _PLE_SHARD_RE, _ple_table_files, load_ple_table
+
+ path = os.environ["FREETOKEN_QWEN4EXP_MODEL"]
+ with open(os.path.join(path, "config.json"), encoding="utf-8") as fh:
+ text = json.load(fh)["text_config"]
+ heads = (text["ngram_size"] - 1) * text["heads_per_ngram"]
+ args = SimpleNamespace(
+ split_ngram_parts=text["split_ngram_parts"],
+ ngram_head_dim=text["ple_embed_dim"] // heads,
+ )
+ table = load_ple_table(path, args)
+ scale = float(table.weight_scale)
+ rows_per_shard = table.tensor.shape[0] // args.split_ngram_parts
+ backend = PinnedUVATable(table.tensor, scale)
+
+ gen = torch.Generator().manual_seed(5)
+ sample = torch.randint(0, table.tensor.shape[0], (1000,), generator=gen)
+ got = backend.lookup(sample.view(-1, 1).cuda()).cpu()
+
+ shard_key = {}
+ for file in _ple_table_files(path):
+ with safetensors.safe_open(file, framework="pt", device="cpu") as fh:
+ for key in fh.keys():
+ match = _PLE_SHARD_RE.search(key)
+ if match is not None:
+ shard_key[int(match.group("shard"))] = (file, key)
+
+ by_file = {}
+ for i, row in enumerate(sample.tolist()):
+ file, key = shard_key[row // rows_per_shard]
+ by_file.setdefault(file, []).append((i, key, row % rows_per_shard))
+ for file, items in by_file.items():
+ with safetensors.safe_open(file, framework="pt", device="cpu") as fh:
+ for i, key, offset in items:
+ raw = fh.get_slice(key)[offset : offset + 1]
+ want = (raw.float() * scale).to(torch.bfloat16).reshape(-1)
+ assert torch.equal(got[i], want), f"sample {i}"
+
+
+# --------------------------------------------------------------------------------------
+# conv state
+# --------------------------------------------------------------------------------------
+
+
+def _forward(layer, R, meta, states):
+ return layer.forward(R, batch=None, meta=meta, conv_states=states)
+
+
+def test_prefill_conv_matches_reference():
+ """The packed single-conv prefill equals the per-request reference conv, chunks shorter than the state included."""
+ torch.manual_seed(12)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6], [2, EOS, 11], [9]]
+ contexts = [[EOS, EOS], [21, 22], [EOS, 31]]
+ meta = _meta(sequences, contexts)
+ total = sum(len(s) for s in sequences)
+ x = torch.randn(total, args.ple_state_width)
+ states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1
+
+ got_states = states.clone()
+ got = layer._short_conv(x, meta, got_states)
+ ref_states = states.clone()
+ ref = short_conv_reference(x, meta, ref_states, layer.conv1d.weight, args.ple_conv_dilation)
+ assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6)
+ assert torch.allclose(got_states, ref_states, rtol=1e-5, atol=1e-6)
+
+
+def test_fresh_slots_read_a_zero_state():
+ """A request marked fresh ignores whatever the pool slot still holds."""
+ torch.manual_seed(13)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ meta = _meta([[3, 4, 5], [6, 7, 8]], [[EOS, EOS]] * 2, fresh=[True, False])
+ x = torch.randn(6, args.ple_state_width)
+ dirty = torch.randn(2, args.ple_state_width, args.ple_conv_state_len)
+ clean = dirty.clone()
+ clean[0] = 0
+ got = layer._short_conv(x, meta, dirty.clone())
+ want = layer._short_conv(x, _meta([[3, 4, 5], [6, 7, 8]], [[EOS, EOS]] * 2), clean.clone())
+ assert torch.equal(got, want)
+
+
[email protected]("cuts", [[1], [2, 3, 4], [9]], ids=["first-token", "uneven-mix", "penultimate"])
+def test_chunked_prefill_matches_one_shot(cuts):
+ """Chunked prefill at arbitrary cut points (including chunks shorter than the conv state) matches one shot."""
+ torch.manual_seed(14)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 7], [2, EOS, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20]]
+ length = len(sequences[0])
+ contexts = [[EOS, EOS], [21, 22]]
+ x = torch.randn(len(sequences) * length, args.ple_state_width)
+ per_req = [x[i * length : (i + 1) * length] for i in range(len(sequences))]
+ zeros = torch.zeros(len(sequences), args.ple_state_width, args.ple_conv_state_len)
+
+ full_states = zeros.clone()
+ full = _forward(layer, x, _meta(sequences, contexts), full_states)
+
+ chunk_states = zeros.clone()
+ pieces, start = [], 0
+ for size in [*cuts, length]:
+ end = min(start + size, length)
+ if end == start:
+ continue
+ window = [(c + s)[start : start + 2] for c, s in zip(contexts, sequences)]
+ pieces.append(
+ _forward(
+ layer,
+ torch.cat([r[start:end] for r in per_req]),
+ _meta([s[start:end] for s in sequences], window),
+ chunk_states,
+ )
+ )
+ start = end
+
+ for i, seq in enumerate(sequences):
+ rebuilt = torch.cat(
+ [p.chunk(len(sequences))[i] for p in pieces]
+ )
+ assert torch.allclose(rebuilt, full[i * length : (i + 1) * length], rtol=1e-4, atol=1e-5)
+ assert torch.allclose(chunk_states, full_states, rtol=1e-4, atol=1e-5)
+
+
+def _state_pool(config, num_slots=8):
+ from freetoken.kvcache.linear_state_pool import LinearStatePool
+
+ return LinearStatePool(
+ config.linear_attention_group(), num_slots, torch.float32,
+ torch.device("cpu"), tp_size=1, slot_states=config.slot_states,
+ )
+
+
+def _track_batch(req, tokens, pool):
+ """Prefill batch whose FLAMetadata carries the hybrid-radix track indices for ``req``."""
+ import freetoken.core as core
+ from freetoken.attention.linear import build_fla_metadata
+ from freetoken.core import Context, set_global_ctx
+
+ core._GLOBAL_CTX = None # test-only: build_fla_metadata reads the state pool off the ctx
+ set_global_ctx(Context(page_size=64, linear_state_pool=pool))
+ batch = _fake_batch([req], decode=False, input_ids=tokens)
+ batch.fla_metadata = build_fla_metadata(batch, torch.device("cpu"))
+ return batch
+
+
+def _tracked_req(table_idx, cached_len, tokens, *, live, ping_pong):
+ req = _req(table_idx, cached_len, tokens, extend_len=len(tokens))
+ req.linear_slot_idx = live
+ req.mamba_ping_pong = ping_pong
+ req.mamba_next_track_idx = 0
+ return req
+
+
+def _no_eos_tokens(n, start=0):
+ return [(t + start) * 13 % (VOCAB - 8) + 8 for t in range(n)]
+
+
+def test_track_snapshot_equals_a_prefill_stopped_at_the_boundary():
+ """The snapshot in the donated slot equals the state a prefill truncated at the boundary leaves."""
+ from freetoken.kernel.fla.chunk import CHUNK_SIZE
+
+ torch.manual_seed(17)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ pool = _state_pool(config)
+ live, dst = 1, 5
+ tokens = _no_eos_tokens(CHUNK_SIZE + 6)
+ req = _tracked_req(0, 0, tokens, live=live, ping_pong=(dst, 6))
+ batch = _track_batch(req, tokens, pool)
+
+ fla = batch.fla_metadata
+ assert fla.track_dst.tolist() == [dst]
+ assert req.mamba_last_track_seqlen == CHUNK_SIZE
+ assert fla.track_boundary_row.tolist() == [CHUNK_SIZE]
+
+ R = torch.randn(len(tokens), args.ple_state_width)
+ slab = pool.slot_state("ple_conv", args.ple_layer_ids[0])
+ layer.forward(R, batch, meta=_meta([tokens], [[EOS, EOS]], slots=[live]), conv_states=slab)
+ got = pool.slot_state("ple_conv", args.ple_layer_ids[0])[dst].clone()
+
+ stopped = torch.zeros_like(slab)
+ _forward(layer, R[:CHUNK_SIZE], _meta([tokens[:CHUNK_SIZE]], [[EOS, EOS]], slots=[live]), stopped)
+ assert torch.equal(got, stopped[live])
+
+
+def test_prefix_hit_matches_the_uncached_run():
+ """A prefix hit that COW-restores the donated snapshot reproduces the tail of an uncached prefill."""
+ from freetoken.kernel.fla.chunk import CHUNK_SIZE
+
+ torch.manual_seed(18)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ pool = _state_pool(config)
+ tokens = _no_eos_tokens(CHUNK_SIZE + 6)
+ context = [[EOS, EOS]]
+ R = torch.randn(len(tokens), args.ple_state_width)
+
+ uncached = _forward(
+ layer, R, _meta([tokens], context, slots=[1]), torch.zeros_like(pool.slot_state("ple_conv", args.ple_layer_ids[0]))
+ )
+
+ live, dst = 1, 5
+ req = _tracked_req(0, 0, tokens, live=live, ping_pong=(dst, 6))
+ batch = _track_batch(req, tokens, pool)
+ layer.forward(R, batch, meta=_meta([tokens], context, slots=[live]), conv_states=pool.slot_state("ple_conv", args.ple_layer_ids[0]))
+
+ resumed_slot = 3
+ pool.copy_from(dst, resumed_slot)
+ tail = tokens[CHUNK_SIZE:]
+ got = _forward(
+ layer,
+ R[CHUNK_SIZE:],
+ _meta([tail], [tokens[CHUNK_SIZE - 2 : CHUNK_SIZE]], slots=[resumed_slot]),
+ pool.slot_state("ple_conv", args.ple_layer_ids[0]),
+ )
+ assert torch.allclose(got, uncached[CHUNK_SIZE:], rtol=1e-5, atol=1e-6)
+
+
+def test_prefill_matches_stepwise_decode():
+ """bs=3 ragged prefill equals feeding the same tokens one decode step at a time."""
+ torch.manual_seed(15)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ sequences = [[3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 12], [2, EOS, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20],
+ [9, 10, 11, 12, 13, 14, EOS, 16, 17, 18, 19, 20]]
+ length = len(sequences[0])
+ contexts = [[EOS, EOS], [21, 22], [EOS, 31]]
+ x = torch.randn(len(sequences) * length, args.ple_state_width)
+ per_req = [x[i * length : (i + 1) * length] for i in range(len(sequences))]
+ zeros = torch.zeros(len(sequences), args.ple_state_width, args.ple_conv_state_len)
+
+ full_states = zeros.clone()
+ full = _forward(layer, x, _meta(sequences, contexts), full_states)
+
+ step_states = zeros.clone()
+ steps = []
+ for t in range(length):
+ window = [(c + s)[t : t + 2] for c, s in zip(contexts, sequences)]
+ steps.append(
+ _forward(
+ layer,
+ torch.stack([r[t] for r in per_req]),
+ _meta([[s[t]] for s in sequences], window, decode=True),
+ step_states,
+ )
+ )
+ for i in range(len(sequences)):
+ got = torch.stack([step[i] for step in steps])
+ assert torch.allclose(got, full[i * length : (i + 1) * length], rtol=1e-4, atol=1e-5)
+ assert torch.allclose(step_states, full_states, rtol=1e-4, atol=1e-5)
+
+
+# --------------------------------------------------------------------------------------
+# full layer vs HF
+# --------------------------------------------------------------------------------------
+
+
+def _layer_ref_inputs(config, layer, tokens=None, hidden=None):
+ args = config.qwen4_args
+ data = {
+ "key_proj": layer.key_proj.weight.float().cpu().numpy(),
+ "value_proj": layer.value_proj.weight.float().cpu().numpy(),
+ "norm_key": layer.norm_key.weight.float().cpu().numpy(),
+ "norm_query": layer.norm_query.weight.float().cpu().numpy(),
+ "norm_conv": layer.norm_conv.weight.float().cpu().numpy(),
+ "conv1d": layer.conv1d.weight.float().cpu().numpy(),
+ "table": layer.ple_embedding.table.weight.float().cpu().numpy(),
+ }
+ if tokens is None:
+ tokens = np.array([[3, 4]], dtype=np.int64)
+ if hidden is None:
+ hidden = np.zeros((1, tokens.shape[1], args.ple_state_width), dtype=np.float32)
+ data["layer_tokens"] = tokens
+ data["hidden"] = hidden
+ return data
+
+
+@requires_cuda
+@requires_hf_ref
+def test_layer_matches_hf(tmp_path):
+ """bf16 PLELayer output matches the fp32 HF Qwen4ExpTextPLELayer within 2e-2."""
+ torch.manual_seed(16)
+ config = _config()
+ args = config.qwen4_args
+ layer = _make_layer(config)
+ tokens = [3, 4, EOS, 5, 6, 8, 9, 2, 4, 5, 6, 12, 13, 14]
+ hidden = (torch.randn(1, len(tokens), args.ple_state_width) * 0.5).numpy()
+ ref = _run_hf_reference(
+ tmp_path,
+ {
+ "hash_tokens": np.array([[EOS, EOS, 3]], dtype=np.int64),
+ **_layer_ref_inputs(config, layer, np.array([tokens], dtype=np.int64), hidden),
+ },
+ )
+ want = torch.as_tensor(ref["layer_out"])[0]
+ assert int(ref["padded_vocab_size"]) == layer.ple_embedding.table.num_rows
+
+ gpu = _make_layer(config, device="cuda", dtype=torch.bfloat16)
+ for name in ("key_proj", "value_proj", "norm_key", "norm_query", "norm_conv"):
+ getattr(gpu, name).weight.copy_(getattr(layer, name).weight)
+ gpu.conv1d.weight.copy_(layer.conv1d.weight)
+ gpu.ple_embedding.attach_table(
+ GpuResidentTable(layer.ple_embedding.table.weight.to("cuda", torch.bfloat16), dtype=torch.bfloat16)
+ )
+ R = torch.as_tensor(hidden)[0].to("cuda", torch.bfloat16)
+ states = torch.zeros(1, args.ple_state_width, args.ple_conv_state_len, device="cuda", dtype=torch.bfloat16)
+ got = _forward(gpu, R, _meta([tokens], [[EOS, EOS]], device="cuda"), states)
+ assert torch.allclose(got.float().cpu(), want, rtol=2e-2, atol=2e-2)
+
+
+# --------------------------------------------------------------------------------------
+# metadata
+# --------------------------------------------------------------------------------------
+
+
+def _fake_batch(reqs, *, decode, input_ids, positions=None, table_idx=None, device="cpu"):
+ return SimpleNamespace(
+ padded_reqs=reqs,
+ reqs=reqs,
+ is_decode=decode,
+ is_prefill=not decode,
+ input_ids=torch.tensor(input_ids, dtype=torch.int64, device=device),
+ positions=None if positions is None else torch.tensor(positions, dtype=torch.int32, device=device),
+ linear_table_idx=(
+ None if table_idx is None else torch.tensor(table_idx, dtype=torch.int32, device=device)
+ ),
+ )
+
+
+def _req(table_idx, cached_len, host_ids, extend_len=1):
+ return SimpleNamespace(
+ table_idx=table_idx,
+ cached_len=cached_len,
+ extend_len=extend_len,
+ linear_slot_idx=None,
+ input_ids=torch.tensor(host_ids, dtype=torch.int64),
+ )
+
+
+def test_commit_writes_the_track_slot_at_the_boundary():
+ """The donated snapshot must carry the context AT the xCHUNK boundary, not the chunk end."""
+ from freetoken.kernel.fla.chunk import CHUNK_SIZE
+
+ args = _config().qwen4_args
+ eos = args.ngram_boundary_token_id
+ ctxp = torch.full((8, 2), eos, dtype=torch.int32)
+ tokens = _no_eos_tokens(CHUNK_SIZE + 6)
+ batch = _fake_batch([_req(1, 0, tokens, extend_len=len(tokens))], decode=False, input_ids=tokens)
+ meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp)
+ fla = SimpleNamespace(
+ track_boundary_row=torch.tensor([CHUNK_SIZE]), track_dst=torch.tensor([5])
+ )
+ commit_ngram_context(meta, fla, ctxp)
+ assert ctxp[1].tolist() == tokens[-2:]
+ assert ctxp[5].tolist() == tokens[CHUNK_SIZE - 2 : CHUNK_SIZE]
+
+
+def test_context_matches_the_token_history_across_chunks_and_decode():
+ """Rolling the slot state chunk by chunk reproduces the last-2-tokens oracle exactly."""
+ args = _config().qwen4_args
+ eos = args.ngram_boundary_token_id
+ ctxp = torch.full((3, 2), 99, dtype=torch.int32) # stale tenant garbage; fresh rows must mask to eos
+ history = _no_eos_tokens(11, start=3)
+ cached = 0
+ for chunk in (3, 1, 2, 5):
+ ids = history[cached : cached + chunk]
+ batch = _fake_batch(
+ [_req(1, cached, history[: cached + chunk], extend_len=chunk)],
+ decode=False, input_ids=ids,
+ )
+ meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp)
+ assert meta.ngram_context.tolist() == [([eos, eos] + history[:cached])[-2:]]
+ commit_ngram_context(meta, None, ctxp)
+ cached += chunk
+ for step in range(3):
+ tok = 200 + step
+ batch = _fake_batch(
+ [_req(1, cached, history + [tok], extend_len=1)],
+ decode=True, input_ids=[tok], positions=[cached], table_idx=[1],
+ )
+ meta = build_ple_metadata(batch, args, torch.device("cpu"), context_pool=ctxp)
+ assert meta.ngram_context.tolist() == [history[-2:]]
+ commit_ngram_context(meta, None, ctxp)
+ history.append(tok)
+ cached += 1
+
+
+# --------------------------------------------------------------------------------------
+# CUDA graph + prefetch overlap
+# --------------------------------------------------------------------------------------
+
+
+@requires_cuda
+def test_decode_graph_replay_matches_eager():
+ """A captured decode PLE forward replays to the eager result, table gather included."""
+ torch.manual_seed(17)
+ config = _config()
+ args = config.qwen4_args
+ rows, bs = 4096, 4
+ layer = _make_layer(config, device="cuda", dtype=torch.bfloat16, rows=rows)
+ bank = _pinned_bank(rows, args.ngram_head_dim, torch.float8_e4m3fn)
+ layer.ple_embedding.attach_table(PinnedUVATable(bank.tensor, 0.05))
+
+ ctxp = torch.full((bs + 1, 2), EOS, dtype=torch.int32, device="cuda")
+ ctxp[1:] = torch.randint(0, VOCAB, (bs, 2), device="cuda", dtype=torch.int32)
+ positions = torch.full((bs,), 8, dtype=torch.int32, device="cuda")
+ slots = torch.arange(1, bs + 1, dtype=torch.int32, device="cuda")
+ batch = SimpleNamespace(
+ padded_reqs=[None] * bs, is_decode=True, is_prefill=False,
+ input_ids=torch.randint(0, VOCAB, (bs,), device="cuda", dtype=torch.int32),
+ positions=positions, linear_table_idx=slots,
+ )
+ R = torch.randn(bs, args.ple_state_width, device="cuda", dtype=torch.bfloat16)
+ states0 = torch.randn(bs + 1, args.ple_state_width, args.ple_conv_state_len,
+ device="cuda", dtype=torch.bfloat16) * 0.1
+ states = states0.clone()
+
+ def step():
+ layer.start_prefetch(batch, build_ple_metadata(batch, args, R.device, context_pool=ctxp))
+ return layer.forward(R, batch, conv_states=states)
+
+ eager = step().clone()
+ eager_states = states.clone()
+
+ warmup = torch.cuda.Stream()
+ warmup.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(warmup):
+ for _ in range(3):
+ states.copy_(states0)
+ step()
+ torch.cuda.current_stream().wait_stream(warmup)
+
+ graph = torch.cuda.CUDAGraph()
+ states.copy_(states0)
+ with torch.cuda.graph(graph):
+ static_out = step()
+ states.copy_(states0)
+ graph.replay()
+ torch.cuda.synchronize()
+ assert torch.equal(static_out, eager)
+ assert torch.equal(states, eager_states)
+
+ # new inputs in the same buffers must flow through the replay
+ ctxp[1:] = torch.randint(0, VOCAB, (bs, 2), device="cuda", dtype=torch.int32)
+ batch.input_ids.copy_(torch.randint(0, VOCAB, (bs,), device="cuda", dtype=torch.int32))
+ states.copy_(states0)
+ graph.replay()
+ replayed = static_out.clone()
+ states.copy_(states0)
+ assert torch.equal(step(), replayed)
+
+ # a bigger eager gather (prefill) must not move the buffer the graph writes into
+ layer.ple_embedding.table.lookup(torch.randint(0, rows, (4096, 16), device="cuda"))
+ states.copy_(states0)
+ graph.replay()
+ torch.cuda.synchronize()
+ assert torch.equal(static_out, replayed)
diff --git a/tests/models/qwen4_exp/test_qsa_backend.py b/tests/models/qwen4_exp/test_qsa_backend.py
new file mode 100644
index 00000000..1d3b944c
--- /dev/null
+++ b/tests/models/qwen4_exp/test_qsa_backend.py
@@ -0,0 +1,250 @@
+"""The QSA backend behind the real Qwen4ExpAttention layer.
+
+(a) dense-oracle equivalence -- while a request sees at most ``index_budget + index_ratio - 1``
+ tokens every complete block is selected, so QSA IS dense attention: the selection must be
+ exactly the causal prefix and the layer output must match ``TorchDenseQSAReference`` (fp32)
+ and a flashinfer dense run over the same pool;
+(b) chunked prefill at unaligned cut points equals one-shot prefill (the dual-source compress);
+(c) a captured decode replay equals the eager decode step.
+"""
+
+from __future__ import annotations
+
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+from .common import Fixture, requires_cuda, parsed_config, selection_spy
+
+QSA_LAYER = 3
+
+
+def _inputs(fixture: Fixture, lengths, extra: int = 0, seed: int = 11):
+ generator = torch.Generator(device=fixture.device).manual_seed(seed)
+ return [
+ torch.randn(
+ n + extra, fixture.config.hidden_size, device=fixture.device,
+ dtype=fixture.dtype, generator=generator,
+ )
+ * 0.5
+ for n in lengths
+ ]
+
+
+def _assert_selection_is_causal_prefix(indices: torch.Tensor, positions: torch.Tensor) -> None:
+ for row, position in enumerate(positions.tolist()):
+ selected = indices[row][indices[row] >= 0]
+ assert torch.equal(
+ selected.sort().values,
+ torch.arange(position + 1, dtype=selected.dtype, device=selected.device),
+ ), f"row {row} (position {position}) did not select its whole causal prefix"
+
+
+@requires_cuda
+def test_prefill_is_dense_below_the_budget(monkeypatch):
+ """bs=3 ragged prefill, longest request exactly at budget + ratio - 1."""
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=128)
+ attn = fixture.layer(QSA_LAYER)
+ lengths = [2051, 1000, 137]
+ inputs = _inputs(fixture, lengths)
+ x = torch.cat([row[:n] for row, n in zip(inputs, lengths)])
+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]
+
+ seen = selection_spy(monkeypatch, fixture.backend)
+ batch = fixture.batch(reqs, "prefill")
+ got = attn.forward(x, batch)
+ _assert_selection_is_causal_prefix(seen["indices"], batch.positions)
+
+ fixture.ctx.attn_backend = _dense_oracle(fixture)
+ reference = attn.forward(x, batch)
+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)
+
+
+def _dense_oracle(fixture: Fixture):
+ from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference
+
+ return TorchDenseQSAReference(
+ fixture.config,
+ num_slots=fixture.num_req_slots,
+ max_len=4096,
+ device=fixture.device,
+ dtype=fixture.dtype,
+ )
+
+
+@requires_cuda
+def test_decode_is_dense_below_the_budget(monkeypatch):
+ """Prefill then five decode steps, sparse path vs the fp32 dense oracle."""
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=128)
+ attn = fixture.layer(QSA_LAYER)
+ lengths, steps = [300, 411, 64], 5
+ inputs = _inputs(fixture, lengths, extra=steps)
+ oracle = _dense_oracle(fixture)
+
+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]
+ seen = selection_spy(monkeypatch, fixture.backend)
+
+ steps_x = [torch.cat([row[:n] for row, n in zip(inputs, lengths)])]
+ steps_x += [
+ torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) for step in range(steps)
+ ]
+ for step, x in enumerate(steps_x):
+ if step:
+ for req in reqs:
+ fixture.step(req)
+ batch = fixture.batch(reqs, "prefill" if step == 0 else "decode")
+ fixture.ctx.attn_backend = fixture.backend
+ got = attn.forward(x, batch)
+ _assert_selection_is_causal_prefix(seen["indices"], batch.positions)
+ fixture.ctx.attn_backend = oracle
+ reference = attn.forward(x, batch)
+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)
+
+
+@requires_cuda
+def test_flashinfer_dense_matches_the_sparse_path():
+ """The engine's dense FULL backend over the same pool, as an independent oracle."""
+ pytest.importorskip("flashinfer")
+ from freetoken.attention.fi import FlashInferBackend
+
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=64)
+ attn = fixture.layer(QSA_LAYER)
+ length = 500
+ x = _inputs(fixture, [length])[0]
+ req = fixture.req(0, 0, length)
+ got = attn.forward(x, fixture.batch([req], "prefill"))
+
+ dense = FlashInferBackend(config)
+ fixture.ctx.attn_backend = SimpleNamespace(
+ qsa_forward=lambda q, k, v, index, layer_id, batch: dense.forward(
+ q, k, v, layer_id, batch
+ )
+ )
+ batch = fixture.batch([req], "prefill")
+ dense.prepare_metadata(batch)
+ reference = attn.forward(x, batch)
+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)
+
+
+@requires_cuda
[email protected]("cut", [1001, 4096, 4097], ids=["unaligned", "page-boundary", "boundary+1"])
+def test_chunked_prefill_matches_one_shot(cut: int):
+ """Cut points that are not multiples of index_ratio exercise the dual-source compress."""
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=512)
+ attn = fixture.layer(QSA_LAYER)
+ length = 5000
+ x = _inputs(fixture, [length])[0]
+
+ one_shot = attn.forward(x, fixture.batch([fixture.req(0, 0, length)], "prefill"))
+ head = fixture.req(1, 0, cut)
+ attn.forward(x[:cut], fixture.batch([head], "prefill"))
+ tail = fixture.req(1, cut, length)
+ got = attn.forward(x[cut:], fixture.batch([tail], "prefill"))
+ assert torch.equal(got, one_shot[cut:])
+
+
+@requires_cuda
+def test_decode_graph_replay_matches_eager():
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=256)
+ attn = fixture.layer(QSA_LAYER)
+ lengths, steps = [300, 411], 4
+ bs = len(lengths)
+ inputs = _inputs(fixture, lengths, extra=steps)
+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]
+ attn.forward(
+ torch.cat([row[:n] for row, n in zip(inputs, lengths)]),
+ fixture.batch(reqs, "prefill"),
+ )
+
+ fixture.backend.init_capture_graph(max_seq_len=fixture.page_table.shape[1], bs_list=[bs])
+ dummy = SimpleNamespace(
+ table_idx=fixture.num_req_slots - 1, cached_len=1, device_len=2, extend_len=1
+ )
+ static = {
+ "x": torch.zeros(bs, config.hidden_size, device=fixture.device, dtype=fixture.dtype),
+ "positions": torch.zeros(bs, dtype=torch.int32, device=fixture.device),
+ "out_loc": torch.zeros(bs, dtype=torch.int32, device=fixture.device),
+ }
+ capture_batch = SimpleNamespace(
+ padded_reqs=[dummy] * bs, reqs=[dummy] * bs, phase="decode", size=bs, padded_size=bs,
+ is_prefill=False, is_decode=True, positions=static["positions"],
+ out_loc=static["out_loc"], attn_metadata=None, active_table_idx=None,
+ )
+ fixture.backend.prepare_for_capture(capture_batch)
+ attn.forward(static["x"], capture_batch) # warmup, same metadata object as the capture
+ torch.cuda.synchronize()
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ captured_out = attn.forward(static["x"], capture_batch)
+ torch.cuda.synchronize()
+
+ for step in range(steps):
+ for req in reqs:
+ fixture.step(req)
+ x = torch.stack([row[n + step] for row, n in zip(inputs, lengths)])
+ batch = fixture.batch(reqs, "decode")
+ static["x"].copy_(x)
+ static["positions"].copy_(batch.positions)
+ static["out_loc"].copy_(batch.out_loc)
+ fixture.backend.prepare_for_replay(batch)
+ # replay must stage into the captured buffers, never reallocate them
+ md = batch.attn_metadata
+ assert md.block_table.data_ptr() == fixture.backend._graph["block_table"].data_ptr()
+ graph.replay()
+ replayed = captured_out.clone()
+ eager = attn.forward(x, fixture.batch(reqs, "decode"))
+ assert torch.equal(replayed, eager), f"graph replay diverged at decode step {step}"
+
+
+@requires_cuda
+def test_row_chunked_scoring_matches_one_chunk(monkeypatch):
+ """The scoring workspace bound splits long prefills into row chunks."""
+ import freetoken.attention.qsa_sparse as qsa_sparse
+
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=64)
+ attn = fixture.layer(QSA_LAYER)
+ length = 600
+ x = _inputs(fixture, [length])[0]
+ whole = attn.forward(x, fixture.batch([fixture.req(0, 0, length)], "prefill"))
+
+ columns = fixture.page_table.shape[1] // config.qwen4_args.index_ratio
+ monkeypatch.setattr(qsa_sparse, "_LOGITS_WORKSPACE_BYTES", 64 * columns * 4)
+ chunked = attn.forward(x, fixture.batch([fixture.req(1, 0, length)], "prefill"))
+ assert torch.equal(chunked, whole)
+
+
+@requires_cuda
+def test_two_qsa_layers_keep_separate_slab_slots(monkeypatch):
+ """Both QSA layers of one forward must hit their own slab slot and ring slice."""
+ config = parsed_config(num_layers=8)
+ assert config.attention_groups[1].layer_ids == (3, 7)
+ fixture = Fixture(config, num_pages=64)
+ layers = [fixture.layer(layer_id, seed=layer_id) for layer_id in (3, 7)]
+ oracle = _dense_oracle(fixture)
+ lengths, steps = [200, 71], 3
+ inputs = _inputs(fixture, lengths, extra=steps)
+ reqs = [fixture.req(i, 0, n) for i, n in enumerate(lengths)]
+
+ xs = [torch.cat([row[:n] for row, n in zip(inputs, lengths)])]
+ xs += [torch.stack([row[n + step] for row, n in zip(inputs, lengths)]) for step in range(steps)]
+ for step, x in enumerate(xs):
+ if step:
+ for req in reqs:
+ fixture.step(req)
+ batch = fixture.batch(reqs, "prefill" if step == 0 else "decode")
+ for attn in layers:
+ fixture.ctx.attn_backend = fixture.backend
+ got = attn.forward(x, batch)
+ fixture.ctx.attn_backend = oracle
+ reference = attn.forward(x, batch)
+ torch.testing.assert_close(got.float(), reference.float(), rtol=2e-2, atol=2e-2)
+
+ slab = fixture.pool.cmp_k_cache
+ assert not torch.equal(slab(0), slab(1))
diff --git a/tests/models/qwen4_exp/test_qsa_hf.py b/tests/models/qwen4_exp/test_qsa_hf.py
new file mode 100644
index 00000000..3d3a8382
--- /dev/null
+++ b/tests/models/qwen4_exp/test_qsa_hf.py
@@ -0,0 +1,253 @@
+"""One QSA layer against the HF reference math at L = 3000.
+
+The fp32 reference here is transcribed from ``modeling_qwen4_exp.py``
+(``Qwen4ExpTextQSAIndexer``:611, ``Qwen4ExpTextAttention``:757): pool a group of raw index
+keys in fp32, ``(1 + w)`` rmsnorm it, rope it at the group's FIRST position, score
+``sum_h relu(<q_h, k_bar_b>) / sqrt(index_head_dim)`` over complete blocks, keep the top
+``budget // ratio``, expand, then attend to that set only.
+
+Two claims: the selected sets agree (ties near the top-k boundary may differ, so the bar is a
+Jaccard floor) and, GIVEN the backend's own selection, the attention output matches. Set
+``FREETOKEN_QWEN4_HF_PYTHON`` to an interpreter whose transformers ships ``qwen4_exp`` to run
+the same comparison against the real HF module in a subprocess.
+"""
+
+from __future__ import annotations
+
+import math
+import os
+import subprocess
+import sys
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from .common import Fixture, requires_cuda, parsed_config, selection_spy
+
+QSA_LAYER = 3
+LENGTH = 3000
+
+
+def _plus_one_rmsnorm(x, weight, eps):
+ xf = x.float()
+ return xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps) * (1.0 + weight.float())
+
+
+def _hf_rope(x, positions, rotary_dim, base):
+ """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims."""
+ inv = 1.0 / (
+ base
+ ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim)
+ )
+ freqs = positions.float().unsqueeze(-1) * inv
+ cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1)
+ sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1)
+ rotated = x[..., :rotary_dim].float()
+ half = rotary_dim // 2
+ swapped = torch.cat([-rotated[..., half:], rotated[..., :half]], dim=-1)
+ return torch.cat([rotated * cos + swapped * sin, x[..., rotary_dim:].float()], dim=-1)
+
+
+def _hf_block_scores(x, indexer, config, positions):
+ """[T, blocks] indexer scores; the pooled block keys do not depend on the query row."""
+ args = config.qwen4_args
+ rotary = config.rotary_config
+ heads, dim, ratio = args.index_n_heads, args.index_head_dim, args.index_ratio
+ qk = F.linear(x.float(), indexer.index_qk_proj.weight.float())
+ q = _plus_one_rmsnorm(
+ qk[:, : heads * dim].view(-1, heads, dim), indexer.q_layernorm.weight, config.rms_norm_eps
+ )
+ q = _hf_rope(q, positions, rotary.rotary_dim, rotary.base)
+ raw = qk[:, heads * dim :]
+ blocks = raw.shape[0] // ratio
+ pooled = raw[: blocks * ratio].view(blocks, ratio, dim).mean(1)
+ pooled = _plus_one_rmsnorm(pooled, indexer.k_layernorm.weight, config.rms_norm_eps)
+ kbar = _hf_rope(
+ pooled.unsqueeze(1), positions[: blocks * ratio : ratio], rotary.rotary_dim, rotary.base
+ ).squeeze(1)
+ return torch.relu(torch.einsum("thd,bd->tbh", q, kbar)).sum(-1) / math.sqrt(dim)
+
+
+def _hf_selection(scores, positions, ratio, budget):
+ """Per-query token ids: the top-(budget // ratio) complete blocks plus the open tail."""
+ offsets = torch.arange(ratio, device=scores.device)
+ selected = []
+ for row, position in enumerate(positions.tolist()):
+ visible = (position + 1) // ratio
+ chosen = torch.empty(0, dtype=torch.int64, device=scores.device)
+ if visible:
+ top = scores[row, :visible].topk(min(budget // ratio, visible)).indices
+ chosen = (top.unsqueeze(-1) * ratio + offsets).flatten()
+ tail = torch.arange(visible * ratio, position + 1, device=scores.device)
+ selected.append(torch.cat([chosen, tail]).sort().values)
+ return selected
+
+
+def _hf_layer_output(x, attn, config, positions, selection):
+ """The HF gated-GQA layer restricted to a given per-query token selection."""
+ rotary = config.rotary_config
+ length, dim = x.shape[0], attn.head_dim
+ qkv = F.linear(x.float(), attn.qkv_proj.weight.float())
+ qg, k, v = qkv.split(attn._qkv_split, dim=-1)
+ qg = qg.view(length, attn.num_q, dim * 2)
+ q = _plus_one_rmsnorm(qg[..., :dim], attn.q_norm.weight, config.rms_norm_eps)
+ q = _hf_rope(q, positions, rotary.rotary_dim, rotary.base)
+ k = _plus_one_rmsnorm(k.view(length, attn.num_kv, dim), attn.k_norm.weight, config.rms_norm_eps)
+ k = _hf_rope(k, positions, rotary.rotary_dim, rotary.base)
+ repeat = attn.num_q // attn.num_kv
+ k = k.repeat_interleave(repeat, dim=1)
+ v = v.view(length, attn.num_kv, dim).repeat_interleave(repeat, dim=1).float()
+ out = torch.zeros(length, attn.num_q, dim, device=x.device, dtype=torch.float32)
+ for row, tokens in enumerate(selection):
+ scores = torch.einsum("hd,khd->hk", q[row], k[tokens]) * dim**-0.5
+ out[row] = torch.einsum("hk,khd->hd", scores.softmax(-1), v[tokens])
+ gate = torch.sigmoid(qg[..., dim:].reshape(length, -1).float())
+ return F.linear(out.reshape(length, -1) * gate, attn.o_proj.weight.float())
+
+
+def _jaccard(indices, selection):
+ scores = []
+ for row, tokens in enumerate(selection):
+ mine = set(indices[row][indices[row] >= 0].tolist())
+ theirs = set(tokens.tolist())
+ scores.append(len(mine & theirs) / max(len(mine | theirs), 1))
+ return torch.tensor(scores)
+
+
+@requires_cuda
+def test_single_layer_matches_hf_reference(monkeypatch):
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=128, max_running_req=4)
+ attn = fixture.layer(QSA_LAYER)
+ generator = torch.Generator(device=fixture.device).manual_seed(13)
+ x = (
+ torch.randn(
+ LENGTH, config.hidden_size, device=fixture.device, dtype=fixture.dtype,
+ generator=generator,
+ )
+ * 0.5
+ )
+ seen = selection_spy(monkeypatch, fixture.backend)
+ batch = fixture.batch([fixture.req(0, 0, LENGTH)], "prefill")
+ got = attn.forward(x, batch)
+ indices = seen["indices"]
+
+ args = config.qwen4_args
+ scores = _hf_block_scores(x, attn.indexer, config, batch.positions)
+ reference_selection = _hf_selection(
+ scores, batch.positions, args.index_ratio, args.index_budget
+ )
+ jaccard = _jaccard(indices, reference_selection)
+ assert jaccard.min() >= 0.97, f"worst-row Jaccard {jaccard.min():.4f}"
+
+ own_selection = [row[row >= 0].long().sort().values for row in indices]
+ reference = _hf_layer_output(x, attn, config, batch.positions, own_selection)
+ torch.testing.assert_close(got.float(), reference, rtol=2e-2, atol=2e-2)
+
+
+_HF_DRIVER = '''
+import sys, torch
+from transformers.models.qwen4_exp.configuration_qwen4_exp import Qwen4ExpTextConfig
+from transformers.models.qwen4_exp.modeling_qwen4_exp import Qwen4ExpTextAttention
+
+payload = torch.load(sys.argv[1], map_location="cuda", weights_only=False)
+meta = payload["meta"]
+config = Qwen4ExpTextConfig(
+ hidden_size=meta["hidden_size"], num_attention_heads=meta["num_q"],
+ num_key_value_heads=meta["num_kv"], head_dim=meta["head_dim"], rms_norm_eps=meta["eps"],
+ max_position_embeddings=meta["max_position"],
+ rope_parameters={"rope_type": "default", "rope_theta": meta["base"],
+ "partial_rotary_factor": meta["rotary_dim"] / meta["head_dim"],
+ "mrope_section": [11, 11, 10]},
+ indexer_n_heads=meta["index_heads"], indexer_kv_heads=1, indexer_head_dim=meta["index_dim"],
+ indexer_budget=meta["budget"], indexer_compress_ratio=meta["ratio"],
+)
+config._attn_implementation = "eager"
+torch.set_grad_enabled(False)
+attn = Qwen4ExpTextAttention(config, layer_idx=0).to("cuda", torch.float32)
+attn.load_state_dict({k: v.to("cuda", torch.float32) for k, v in payload["weights"].items()})
+
+x = payload["x"].to(torch.float32).unsqueeze(0)
+positions = payload["positions"].to("cuda").to(torch.long)
+rotary_dim = meta["rotary_dim"]
+pairs = torch.arange(0, rotary_dim, 2, device="cuda", dtype=torch.float32)
+inv = 1.0 / (meta["base"] ** (pairs / rotary_dim))
+freqs = positions.float().unsqueeze(-1) * inv
+cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(0)
+sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(0)
+length = x.shape[1]
+causal = torch.arange(length, device="cuda")
+mask = torch.zeros(1, 1, length, length, device="cuda", dtype=torch.float32)
+mask.masked_fill_(causal[None, :] > causal[:, None], torch.finfo(torch.float32).min)
+
+out, _ = attn(x, (cos, sin), mask)
+selected = attn.indexer(x, (cos, sin), mask, None)[0, 0] == 0
+torch.save({"out": out[0].cpu(), "selected": selected.cpu()}, sys.argv[2])
+'''
+
+
+@requires_cuda
+ not os.environ.get("FREETOKEN_QWEN4_HF_PYTHON"),
+ reason="set FREETOKEN_QWEN4_HF_PYTHON to a transformers build that ships qwen4_exp",
+)
+def test_single_layer_matches_upstream_hf(tmp_path, monkeypatch):
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=128, max_running_req=4)
+ attn = fixture.layer(QSA_LAYER)
+ generator = torch.Generator(device=fixture.device).manual_seed(13)
+ x = (
+ torch.randn(
+ LENGTH, config.hidden_size, device=fixture.device, dtype=fixture.dtype,
+ generator=generator,
+ )
+ * 0.5
+ )
+ seen = selection_spy(monkeypatch, fixture.backend)
+ batch = fixture.batch([fixture.req(0, 0, LENGTH)], "prefill")
+ got = attn.forward(x, batch)
+
+ args = config.qwen4_args
+ rotary = config.rotary_config
+ q_rows, kv_rows = attn.qo_attn_dim * 2, attn.kv_attn_dim
+ fused = attn.qkv_proj.weight
+ payload = tmp_path / "payload.pt"
+ result = tmp_path / "hf.pt"
+ driver = tmp_path / "driver.py"
+ driver.write_text(_HF_DRIVER)
+ torch.save(
+ {
+ "weights": {
+ "q_proj.weight": fused[:q_rows].cpu(),
+ "k_proj.weight": fused[q_rows : q_rows + kv_rows].cpu(),
+ "v_proj.weight": fused[q_rows + kv_rows :].cpu(),
+ "o_proj.weight": attn.o_proj.weight.cpu(),
+ "q_norm.weight": attn.q_norm.weight.cpu(),
+ "k_norm.weight": attn.k_norm.weight.cpu(),
+ "indexer.index_qk_proj.weight": attn.indexer.index_qk_proj.weight.cpu(),
+ "indexer.q_layernorm.weight": attn.indexer.q_layernorm.weight.cpu(),
+ "indexer.k_layernorm.weight": attn.indexer.k_layernorm.weight.cpu(),
+ },
+ "x": x.cpu(),
+ "positions": batch.positions.cpu(),
+ "meta": {
+ "hidden_size": config.hidden_size, "num_q": attn.num_q, "num_kv": attn.num_kv,
+ "head_dim": attn.head_dim, "eps": config.rms_norm_eps,
+ "max_position": rotary.max_position, "base": rotary.base,
+ "rotary_dim": rotary.rotary_dim, "index_heads": args.index_n_heads,
+ "index_dim": args.index_head_dim, "budget": args.index_budget,
+ "ratio": args.index_ratio,
+ },
+ },
+ payload,
+ )
+ subprocess.run(
+ [os.environ["FREETOKEN_QWEN4_HF_PYTHON"], str(driver), str(payload), str(result)],
+ check=True, stdout=sys.stderr, timeout=1800,
+ )
+ upstream = torch.load(result, map_location=fixture.device, weights_only=False)
+ selection = [row.nonzero().flatten() for row in upstream["selected"].to(fixture.device)]
+ jaccard = _jaccard(seen["indices"], selection)
+ assert jaccard.min() >= 0.97, f"worst-row Jaccard {jaccard.min():.4f}"
+ torch.testing.assert_close(got.float(), upstream["out"].float(), rtol=2e-2, atol=2e-2)
diff --git a/tests/models/qwen4_exp/test_qsa_kernels.py b/tests/models/qwen4_exp/test_qsa_kernels.py
new file mode 100644
index 00000000..4b678b3b
--- /dev/null
+++ b/tests/models/qwen4_exp/test_qsa_kernels.py
@@ -0,0 +1,486 @@
+"""The modified and original QSA Triton kernels against pure-torch references.
+
+Only kernels FreeToken changed or wrote get unit tests: the compression kernel (re-addressed
+pending ring, its own torch check) and the block top-k (original radix select, checked against
+torch.topk and, through the expansion chain, against the vLLM reference semantics). score.py
+and attend.py are vendored from vLLM and are covered by the backend and e2e tests.
+``_qsa_mqa_paged_reference`` / ``_qsa_relative_topk_reference`` / ``_expand_qsa_indices_reference``
+are transcribed from ``vllm/tests/test_qsa_reference.py`` (Apache-2.0).
+"""
+
+from __future__ import annotations
+
+import math
+
+import pytest
+import torch
+
+from .common import Fixture, requires_cuda, parsed_config
+
+PAGE_SIZE = 64
+RATIO = 4
+BUDGET = 2048
+INDEX_DIM = 128
+CMP_PAGE = PAGE_SIZE // RATIO
+
+
+# --------------------------------------------------------------------------------------
+# vLLM pure-torch references (tests/test_qsa_reference.py:87-227)
+# --------------------------------------------------------------------------------------
+
+
+def _qsa_mqa_paged_reference(q, k_cache, page_table, token_to_req, visible_lengths):
+ pages = page_table.index_select(0, token_to_req.long()).long()
+ keys = k_cache[pages, :, 0, :].flatten(1, 2)
+ scores = torch.einsum("rhd,rnd->rnh", q.float(), keys.float())
+ logits = torch.relu(scores).sum(dim=-1) / math.sqrt(q.shape[-1])
+ positions = torch.arange(keys.shape[1], device=q.device).unsqueeze(0)
+ return logits.masked_fill(positions >= visible_lengths.unsqueeze(1), -torch.inf)
+
+
+def _qsa_relative_topk_reference(logits, row_starts, row_ends, topk):
+ output = torch.full((logits.shape[0], topk), -1, dtype=torch.int32, device=logits.device)
+ for row in range(logits.shape[0]):
+ start = int(row_starts[row].item())
+ length = int((row_ends[row] - row_starts[row]).item())
+ width = min(length, topk)
+ if width:
+ output[row, :width] = torch.topk(
+ logits[row, start : start + length], width
+ ).indices.to(torch.int32)
+ return output
+
+
+def _expand_qsa_indices_reference(
+ block_indices, query_positions, sequence_lengths, compress_ratio, token_topk
+):
+ rows = block_indices.shape[0]
+ block_topk = token_topk // compress_ratio
+ output_width = token_topk + compress_ratio - 1
+ offsets = torch.arange(compress_ratio, device=block_indices.device)
+ blocks = block_indices.long()
+ expanded = blocks.unsqueeze(-1) * compress_ratio + offsets
+ expanded = torch.where(
+ blocks.unsqueeze(-1) >= 0, expanded, torch.full_like(expanded, -1)
+ ).reshape(rows, block_topk * compress_ratio)
+ expanded = expanded[:, :token_topk]
+ expanded = torch.where(
+ (expanded >= 0) & (expanded < sequence_lengths.unsqueeze(1)),
+ expanded,
+ torch.full_like(expanded, -1),
+ )
+
+ tail_offsets = torch.arange(compress_ratio - 1, device=block_indices.device)
+ visible_tokens = query_positions + 1
+ tail_start = visible_tokens // compress_ratio * compress_ratio
+ tail = tail_start.unsqueeze(1) + tail_offsets.unsqueeze(0)
+ tail_count = (visible_tokens - tail_start).unsqueeze(1)
+ tail_valid = (tail_offsets.unsqueeze(0) < tail_count) & (
+ tail < sequence_lengths.unsqueeze(1)
+ )
+ tail = torch.where(tail_valid, tail, torch.full_like(tail, -1))
+
+ result = torch.cat((expanded, tail), dim=1)
+ order = torch.arange(output_width, device=result.device).expand(rows, -1)
+ sort_key = torch.where(result >= 0, order, order + output_width)
+ return result.gather(1, torch.argsort(sort_key, dim=1, stable=True)).to(torch.int32)
+
+
+class _Case:
+ def __init__(self, **fields):
+ self.__dict__.update(fields)
+
+
+def _paged_case(length: int, bs: int, rows_per_req: int, seed: int, index_heads: int = 4):
+ """Synthetic paged geometry: shuffled pages, the last ``rows_per_req`` queries per request."""
+ device = torch.device("cuda")
+ torch.manual_seed(seed)
+ generator = torch.Generator(device=device).manual_seed(seed)
+ pages_per_req = -(-length // PAGE_SIZE)
+ total_pages = bs * pages_per_req
+ block_table = (
+ torch.randperm(total_pages, device=device).reshape(bs, pages_per_req).to(torch.int32)
+ )
+ q = torch.randn(
+ bs * rows_per_req, index_heads, INDEX_DIM, device=device, dtype=torch.bfloat16,
+ generator=generator,
+ )
+ token_to_req = torch.repeat_interleave(
+ torch.arange(bs, device=device, dtype=torch.int32), rows_per_req
+ )
+ query_positions = torch.cat(
+ [
+ torch.arange(length - rows_per_req, length, device=device, dtype=torch.int32)
+ for _ in range(bs)
+ ]
+ )
+ seq_lens = torch.full((bs,), length, device=device, dtype=torch.int32)
+ return _Case(
+ device=device,
+ generator=generator,
+ length=length,
+ bs=bs,
+ pages_per_req=pages_per_req,
+ total_pages=total_pages,
+ block_table=block_table,
+ q=q,
+ token_to_req=token_to_req,
+ query_positions=query_positions,
+ seq_lens=seq_lens,
+ )
+
+
+@requires_cuda
[email protected]("length", [20000])
[email protected]("torch_topk", [False, True])
+def test_top_blocks_and_expansion_match_vllm_reference(length: int, bs: int, torch_topk: bool):
+ from freetoken.kernel.triton.qsa import expand_qsa_block_indices, qsa_mqa_paged
+
+ config = parsed_config()
+ fixture = Fixture(config, num_pages=4, max_running_req=2)
+ case = _paged_case(length, bs, rows_per_req=4, seed=7 * length + bs)
+ cache = torch.randn(
+ case.total_pages, CMP_PAGE, 1, INDEX_DIM, device=case.device,
+ dtype=torch.bfloat16, generator=case.generator,
+ )
+ rows, columns = case.q.shape[0], case.pages_per_req * CMP_PAGE
+ logits = torch.empty(rows, columns, dtype=torch.float32, device=case.device)
+ visible = torch.empty(rows, dtype=torch.int32, device=case.device)
+ qsa_mqa_paged(
+ case.q, cache, case.block_table, case.token_to_req, case.query_positions,
+ case.seq_lens, RATIO, logits, visible,
+ )
+ blocks = torch.empty(rows, BUDGET // RATIO, dtype=torch.int32, device=case.device)
+ reference_logits = _qsa_mqa_paged_reference(
+ case.q, cache, case.block_table, case.token_to_req, visible
+ )
+ if torch_topk:
+ fixture.backend._block_topk_kernel = None
+ fixture.backend._top_blocks(logits, visible, blocks)
+
+ expected_blocks = _qsa_relative_topk_reference(
+ reference_logits, torch.zeros_like(visible), visible, BUDGET // RATIO
+ )
+ # Ties between equal scores may land on either index; the SET is what selection means.
+ torch.testing.assert_close(blocks.sort(-1).values, expected_blocks.sort(-1).values)
+
+ row_seq_lens = case.seq_lens.index_select(0, case.token_to_req.long())
+ indices = torch.empty(rows, BUDGET + RATIO - 1, dtype=torch.int32, device=case.device)
+ expand_qsa_block_indices(
+ expected_blocks, case.query_positions, case.seq_lens, case.token_to_req,
+ RATIO, BUDGET, indices,
+ )
+ expected = _expand_qsa_indices_reference(
+ expected_blocks, case.query_positions, row_seq_lens, RATIO, BUDGET
+ )
+ torch.testing.assert_close(indices, expected)
+
+
+@requires_cuda
[email protected]("ring_capacity", [4, 8])
+def test_compression_reads_both_sources(ring_capacity: int):
+ """Members already consumed come from the ring, the rest from this forward's raw rows."""
+ from freetoken.kernel.triton.qsa import qsa_compress_groups, qsa_store_rows
+
+ device = torch.device("cuda")
+ dim, slots = 8, 3
+ pairs = [(0, p) for p in range(2, 9)] + [(1, p) for p in range(5, 11)]
+
+ def key(request: int, position: int) -> torch.Tensor:
+ return (torch.arange(dim, dtype=torch.float32) + request * 1000 + position * 10).to(
+ torch.bfloat16
+ )
+
+ raw = torch.stack([key(*pair) for pair in pairs]).to(device)
+ token_to_req = torch.tensor([r for r, _ in pairs], dtype=torch.int32, device=device)
+ positions = torch.tensor([p for _, p in pairs], dtype=torch.int32, device=device)
+ cu_seqlens = torch.tensor([0, 7, 13], dtype=torch.int32, device=device)
+ ring_slots = torch.tensor([2, 0], dtype=torch.int32, device=device)
+ ring = torch.zeros(slots, ring_capacity, dim, device=device, dtype=torch.bfloat16)
+ for request, position, slot in ((0, 0, 2), (0, 1, 2), (1, 4, 0)):
+ ring[slot, position % ring_capacity] = key(request, position).to(device)
+
+ pooled = torch.empty(len(pairs), dim, device=device, dtype=torch.bfloat16)
+ first = torch.empty(len(pairs), dtype=torch.int32, device=device)
+ qsa_compress_groups(
+ raw, ring, ring_slots, token_to_req, cu_seqlens, positions, RATIO, pooled, first
+ )
+
+ for row, (request, position) in enumerate(pairs):
+ if (position + 1) % RATIO:
+ continue
+ group = torch.stack([key(request, position - RATIO + 1 + k).float() for k in range(RATIO)])
+ expected = group.mean(0).to(torch.bfloat16).to(device)
+ assert torch.equal(pooled[row], expected), (request, position)
+ assert int(first[row]) == position - RATIO + 1
+
+ # The ring keeps only each request's last ring_capacity rows.
+ rows = torch.arange(len(pairs), device=device)
+ ends = cu_seqlens.long().index_select(0, token_to_req.long() + 1)
+ slot = torch.where(
+ rows >= ends - ring_capacity,
+ ring_slots.long().index_select(0, token_to_req.long()) * ring_capacity
+ + positions.long() % ring_capacity,
+ torch.full_like(rows, -1),
+ )
+ qsa_store_rows(ring, slot.to(torch.int32), raw)
+ for request, ring_slot in ((0, 2), (1, 0)):
+ for position in [p for r, p in pairs if r == request][-ring_capacity:]:
+ assert torch.equal(
+ ring[ring_slot, position % ring_capacity], key(request, position).to(device)
+ )
+
+
+# --------------------------------------------------------------------------------------
+# Block top-k (kernel/triton/qsa/topk.py)
+# --------------------------------------------------------------------------------------
+
+TOPK_SHAPES = [(512, 512), (4096, 512)]
+
+
+def _torch_topk_blocks(logits, visible, width):
+ """The torch.topk fallback of ``qsa_sparse._top_blocks``, kept here as the reference."""
+ columns = logits.shape[1]
+ out = torch.full((logits.shape[0], width), -1, dtype=torch.int32, device=logits.device)
+ column = torch.arange(columns, device=logits.device)
+ masked = logits.masked_fill(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf"))
+ take = min(width, columns)
+ values, chosen = torch.topk(masked, take, dim=-1)
+ out[:, :take] = torch.where(values > -float("inf"), chosen.to(torch.int32), -1)
+ return out
+
+
+def _topk_case(n_blocks: int, bs: int, mode: str, seed: int):
+ device = torch.device("cuda")
+ generator = torch.Generator(device=device).manual_seed(seed)
+ logits = torch.randn(bs, n_blocks, device=device, generator=generator)
+ visible = torch.full((bs,), n_blocks, dtype=torch.int32, device=device)
+ if mode == "ties":
+ # Three distinct scores over every column: almost every selection sits on a tie.
+ logits = torch.randint(0, 3, (bs, n_blocks), device=device, generator=generator).float()
+ if mode == "ragged":
+ visible = torch.randint(
+ 0, n_blocks + 1, (bs,), dtype=torch.int32, device=device, generator=generator
+ )
+ if mode == "dead":
+ logits[:, ::5] = -float("inf")
+ return logits, visible
+
+
+@requires_cuda
[email protected]("n_blocks,width", TOPK_SHAPES)
[email protected]("mode", ["random", "ties", "ragged", "dead"])
+def test_block_topk_matches_torch_topk(n_blocks: int, width: int, bs: int, mode: str):
+ from freetoken.kernel.triton.qsa import qsa_block_topk
+
+ logits, visible = _topk_case(n_blocks, bs, mode, seed=31 * n_blocks + 7 * width + bs)
+ blocks = torch.empty(bs, width, dtype=torch.int32, device=logits.device)
+ qsa_block_topk(logits, visible, blocks)
+ expected = _torch_topk_blocks(logits, visible, width)
+
+ # Selection is a set: torch.topk orders by descending score, the kernel by column id.
+ torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values)
+ live = (blocks >= 0).sum(-1)
+ torch.testing.assert_close(live, (expected >= 0).sum(-1))
+ # expand.py reads ranks [0, complete_blocks), so a -1 may only sit in the tail.
+ ranks = torch.arange(width, device=blocks.device)
+ assert torch.equal(blocks >= 0, ranks.unsqueeze(0) < live.unsqueeze(1))
+ assert bool(((blocks[:, 1:] > blocks[:, :-1]) | (blocks[:, 1:] < 0)).all())
+
+
+@requires_cuda
+def test_block_topk_replays_in_a_cuda_graph():
+ """Fixed grid, no host read: one capture serves every later sequence length."""
+ from freetoken.kernel.triton.qsa import qsa_block_topk
+
+ rows, columns, width = 4, 4096, 512
+ device = torch.device("cuda")
+ logits = torch.randn(rows, columns, device=device)
+ visible = torch.full((rows,), columns, dtype=torch.int32, device=device)
+ blocks = torch.empty(rows, width, dtype=torch.int32, device=device)
+
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ qsa_block_topk(logits, visible, blocks)
+ torch.cuda.current_stream().wait_stream(side)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ qsa_block_topk(logits, visible, blocks)
+
+ for lengths in ([columns] * rows, [columns // 2, 900, 37, 0], [4095, 512, 511, 4096]):
+ logits.normal_()
+ visible.copy_(torch.tensor(lengths, dtype=torch.int32, device=device))
+ blocks.fill_(0)
+ graph.replay()
+ expected = _torch_topk_blocks(logits, visible, width)
+ torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values)
+
+
+def test_torch_topk_env_picks_the_fallback(monkeypatch):
+ from freetoken.attention.qsa_sparse import TORCH_TOPK_ENV, _resolve_block_topk
+
+ assert _resolve_block_topk() is not None
+ monkeypatch.setenv(TORCH_TOPK_ENV, "1")
+ assert _resolve_block_topk() is None
+
+
+# --------------------------------------------------------------------------------------
+# Block top-k, split + merge path (wide buffers)
+# --------------------------------------------------------------------------------------
+
+SPLIT_CHUNK = 4096 # _split_plan's chunk for every buffer these tests use
+
+
+def _policy_topk_blocks(logits, visible, width):
+ """The kernel's documented order: highest score wins, lowest column breaks a tie."""
+ columns = logits.shape[1]
+ column = torch.arange(columns, device=logits.device)
+ masked = logits.masked_fill(column.unsqueeze(0) >= visible.unsqueeze(1), -float("inf"))
+ take = min(width, columns)
+ order = masked.argsort(dim=-1, descending=True, stable=True)[:, :take]
+ out = torch.full((logits.shape[0], width), -1, dtype=torch.int32, device=logits.device)
+ out[:, :take] = torch.where(
+ masked.gather(1, order) > -float("inf"), order.to(torch.int32), -1
+ )
+ return out
+
+
+def _split_topk_case(n_blocks: int, width: int, bs: int, mode: str, seed: int):
+ if mode != "boundary":
+ return _topk_case(n_blocks, bs, mode, seed)
+ # width - 212 columns beat the tie, so the 212 remaining winners start 100 columns below
+ # a chunk boundary and run past it into a chunk that is all tie.
+ logits = torch.zeros(bs, n_blocks, device="cuda")
+ for row in range(bs):
+ cut = SPLIT_CHUNK * (1 + row % (n_blocks // SPLIT_CHUNK - 1))
+ logits[row, : width - 212] = 2.0
+ logits[row, cut - 100 :] = 1.0
+ return logits, torch.full((bs,), n_blocks, dtype=torch.int32, device="cuda")
+
+
+@requires_cuda
[email protected]("n_blocks", [65536])
[email protected]("mode", ["random", "boundary", "ragged", "dead"])
+def test_block_topk_split_path_matches_torch_topk(n_blocks: int, bs: int, mode: str):
+ from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width
+
+ width = 512
+ assert qsa_block_topk_scratch_width(n_blocks, width) > 0, "case must take the split path"
+ logits, visible = _split_topk_case(n_blocks, width, bs, mode, seed=n_blocks + bs + len(mode))
+ blocks = torch.empty(bs, width, dtype=torch.int32, device=logits.device)
+ qsa_block_topk(logits, visible, blocks)
+
+ torch.testing.assert_close(
+ blocks.sort(-1).values, _torch_topk_blocks(logits, visible, width).sort(-1).values
+ )
+ # Tie determinism: the winners are the exact set the lowest-column-first policy names,
+ # including the ties that straddle a chunk boundary.
+ torch.testing.assert_close(
+ blocks.sort(-1).values, _policy_topk_blocks(logits, visible, width).sort(-1).values
+ )
+ live = (blocks >= 0).sum(-1)
+ ranks = torch.arange(width, device=blocks.device)
+ assert torch.equal(blocks >= 0, ranks.unsqueeze(0) < live.unsqueeze(1))
+ assert bool(((blocks[:, 1:] > blocks[:, :-1]) | (blocks[:, 1:] < 0)).all())
+
+
+@requires_cuda
[email protected]("preallocated", [False, True])
+def test_block_topk_split_path_replays_in_a_cuda_graph(preallocated: bool):
+ """The split geometry comes from the buffer width, so one capture serves every length."""
+ from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width
+
+ rows, columns, width = 4, 65536, 512
+ device = torch.device("cuda")
+ scratch_width = qsa_block_topk_scratch_width(columns, width)
+ assert scratch_width > 0
+ logits = torch.randn(rows, columns, device=device)
+ visible = torch.full((rows,), columns, dtype=torch.int32, device=device)
+ blocks = torch.empty(rows, width, dtype=torch.int32, device=device)
+ # The scratch never needs clearing: every split rewrites its own slots on every replay.
+ scratch = (
+ torch.empty(rows, scratch_width, dtype=torch.int32, device=device)
+ if preallocated
+ else None
+ )
+
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ qsa_block_topk(logits, visible, blocks, scratch)
+ torch.cuda.current_stream().wait_stream(side)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ qsa_block_topk(logits, visible, blocks, scratch)
+
+ for lengths in ([columns] * rows, [4096, 40000, 0, 65535], [1, 4097, 8192, columns]):
+ logits.normal_()
+ visible.copy_(torch.tensor(lengths, dtype=torch.int32, device=device))
+ blocks.fill_(0)
+ graph.replay()
+ expected = _torch_topk_blocks(logits, visible, width)
+ torch.testing.assert_close(blocks.sort(-1).values, expected.sort(-1).values)
+
+
+@requires_cuda
+def test_block_topk_split_path_cost_tracks_live_blocks():
+ """A wide buffer with a short row must not pay for the splits past its visible tail."""
+ from freetoken.kernel.triton.qsa import qsa_block_topk, qsa_block_topk_scratch_width
+
+ # wide enough that the split work dwarfs the 20-launch floor even at boosted clocks
+ rows, columns, width = 1, 262144, 512
+ device = torch.device("cuda")
+ logits = torch.randn(rows, columns, device=device)
+ visible = torch.full((rows,), columns, dtype=torch.int32, device=device)
+ blocks = torch.empty(rows, width, dtype=torch.int32, device=device)
+ scratch = torch.empty(
+ rows, qsa_block_topk_scratch_width(columns, width), dtype=torch.int32, device=device
+ )
+
+ side = torch.cuda.Stream()
+ side.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(side):
+ qsa_block_topk(logits, visible, blocks, scratch)
+ torch.cuda.current_stream().wait_stream(side)
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ for _ in range(20):
+ qsa_block_topk(logits, visible, blocks, scratch)
+
+ def replay_us(live: int) -> float:
+ visible.fill_(live)
+ best = float("inf")
+ for _ in range(5):
+ start, stop = torch.cuda.Event(True), torch.cuda.Event(True)
+ start.record()
+ graph.replay()
+ stop.record()
+ torch.cuda.synchronize()
+ best = min(best, start.elapsed_time(stop) * 1000.0 / 20)
+ return best
+
+ full, short = replay_us(columns), replay_us(SPLIT_CHUNK)
+ assert full > 2.0 * short, f"{columns} live {full:.1f}us vs {SPLIT_CHUNK} live {short:.1f}us"
+
+
+@requires_cuda
+def test_capture_graph_provisions_the_block_topk_scratch():
+ from freetoken.kernel.triton.qsa import qsa_block_topk_scratch_width
+
+ fixture = Fixture(parsed_config(), num_pages=320, max_running_req=2)
+ backend = fixture.backend
+ table_width = fixture.page_table.shape[1]
+ columns = table_width // PAGE_SIZE * CMP_PAGE
+ width = qsa_block_topk_scratch_width(columns, backend.block_topk)
+ assert width > 0, "the fixture must be wide enough to reach the split path"
+
+ backend.init_capture_graph(table_width, [2])
+ static = backend._graph["topk_scratch"]
+ assert static.shape[1] == width
+ assert backend._scratch("topk_scratch", 2, width, dtype=torch.int32).data_ptr() == (
+ static.data_ptr()
+ )
diff --git a/tests/models/qwen4_exp/test_skeleton.py b/tests/models/qwen4_exp/test_skeleton.py
new file mode 100644
index 00000000..00f7c43f
--- /dev/null
+++ b/tests/models/qwen4_exp/test_skeleton.py
@@ -0,0 +1,503 @@
+"""Skeleton tests: the frozen qwen4_exp interfaces and their torch references.
+
+Everything runs on a scaled-down copy of the real geometry (4 layers, full attention on layer 3,
+PLE on layer 1, hc_count 4, 3-gram hash) so the shapes and the layer split are the shipping ones.
+The hyper-connection and PLE references transcribed here are HF ``modeling_qwen4_exp.py``
+(``Qwen4ExpTextGatedResidual``:941, ``Qwen4ExpTextNGramEmbedding``:1018, ``Qwen4ExpTextPLELayer``
+:1117), so the torch implementations are checked against the math, not against themselves.
+"""
+
+from __future__ import annotations
+
+import math
+from types import SimpleNamespace
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from freetoken.layers import BaseOP, LinearReplicated
+from freetoken.models.config import ModelConfig
+from freetoken.models.qwen4_exp.config import parse_config
+from freetoken.models.qwen4_exp.hc import GatedResidual
+from freetoken.models.qwen4_exp.ple import GpuResidentTable, PLELayer, PLEMetadata
+
+from .common import EOS, hash_constants, requires_cuda, toy_hf_config
+
+
+def _config(num_layers: int = 4) -> ModelConfig:
+ return parse_config(toy_hf_config(num_layers))
+
+
+def _fill(op, gen: torch.Generator, scale: float = 0.05) -> None:
+ """Random floats / zeroed ints for every state-dict tensor of an op tree."""
+ for tensor in op.state_dict().values():
+ if tensor.is_floating_point():
+ tensor.normal_(0.0, scale, generator=gen)
+ else:
+ tensor.zero_()
+
+
+def _group_norm(x, weight, eps, groups):
+ xf = x.float().reshape(*x.shape[:-1], groups, -1)
+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+ return (xf.flatten(-2) * (1.0 + weight.float())).type_as(x)
+
+
+# --------------------------------------------------------------------------------------
+# hyper-connections
+# --------------------------------------------------------------------------------------
+
+
+def _hf_gated_residual(hc, R, w_norm, w_down, w_up, w_inject, hidden, eps):
+ xn = _group_norm(R, w_norm, eps, hc)
+ mix = F.silu(F.linear(xn, w_down) / hc)
+ mix = torch.sigmoid(F.linear(mix, w_up)).unflatten(-1, (hc, hidden))
+ mixed = (mix * xn.unflatten(-1, (hc, hidden))).mean(-2)
+ if w_inject is None:
+ return mixed, None
+ return mixed, 2 * torch.sigmoid(F.linear(xn, w_inject) / hc)
+
+
[email protected]("tokens", [1, 7])
+def test_hc_mix_and_combine_match_hf(tokens: int):
+ torch.manual_seed(0)
+ config = _config()
+ args = config.qwen4_args
+ hc = GatedResidual(config)
+ _fill(hc, torch.Generator().manual_seed(1))
+
+ R = torch.randn(tokens, args.ple_state_width)
+ y = torch.randn(tokens, args.hidden_size)
+ x, s = hc.mix(R)
+ got = hc.combine(R, y, s)
+
+ merged = hc.input_mix_weight_down_block_inject.weight
+ ref_x, ref_inject = _hf_gated_residual(
+ args.hc_count,
+ R,
+ hc.hc_norm.weight,
+ merged[: args.hc_lowrank],
+ hc.input_mix_weight_up.weight,
+ merged[args.hc_lowrank : args.hc_lowrank + args.hc_count],
+ args.hidden_size,
+ config.rms_norm_eps,
+ )
+ ref = R.unflatten(-1, (args.hc_count, args.hidden_size))
+ ref = (ref + y.unsqueeze(-2) * ref_inject.unsqueeze(-1)).flatten(-2)
+
+ assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6)
+ assert torch.allclose(got, ref, rtol=1e-5, atol=1e-6)
+
+
+def test_hc_merged_gemm_layout_and_top_level_mixer():
+ config = _config()
+ args = config.qwen4_args
+ hc = GatedResidual(config)
+ # 320 + 4 lowrank/inject rows padded to a multiple of 16 in the real config
+ assert hc.pad_size == (-(args.hc_lowrank + args.hc_count)) % 16
+ merged = hc.input_mix_weight_down_block_inject.weight
+ assert merged.shape == (
+ args.hc_lowrank + args.hc_count + hc.pad_size,
+ args.ple_state_width,
+ )
+ assert set(hc.state_dict()) == {
+ "hc_norm.weight",
+ "input_mix_weight_down_block_inject.weight",
+ "input_mix_weight_up.weight",
+ }
+
+ mixer = GatedResidual(config, use_combine=False)
+ _fill(mixer, torch.Generator().manual_seed(2))
+ assert set(mixer.state_dict()) == {
+ "hc_norm.weight",
+ "input_mix_weight_down.weight",
+ "input_mix_weight_up.weight",
+ }
+ R = torch.randn(5, args.ple_state_width)
+ x, s = mixer.mix(R)
+ assert s is None
+ ref_x, ref_inject = _hf_gated_residual(
+ args.hc_count,
+ R,
+ mixer.hc_norm.weight,
+ mixer.input_mix_weight_down.weight,
+ mixer.input_mix_weight_up.weight,
+ None,
+ args.hidden_size,
+ config.rms_norm_eps,
+ )
+ assert ref_inject is None
+ assert torch.allclose(x, ref_x, rtol=1e-5, atol=1e-6)
+
+
+# --------------------------------------------------------------------------------------
+# PLE
+# --------------------------------------------------------------------------------------
+
+
+def _hf_shift_right(tokens, shift, eos):
+ if shift == 0:
+ return tokens
+ batch, seq_len = tokens.shape
+ positions = torch.arange(seq_len)
+ eos_positions = torch.where(tokens == eos, positions, torch.full_like(positions, -1))
+ previous = torch.cummax(eos_positions, dim=1).values
+ previous = torch.cat([eos_positions.new_full((batch, 1), -1), previous[:, :-1]], dim=1)
+ in_segment = positions.unsqueeze(0) - (previous + 1)
+ source = positions - shift
+ shifted = tokens.gather(1, source.clamp_min(0).unsqueeze(0).expand(batch, -1))
+ valid = (in_segment >= shift) & (source.unsqueeze(0) >= 0)
+ return torch.where(valid, shifted, tokens.new_full((), eos))
+
+
+def _hf_ngram_ids(tokens, context, args, multipliers, sizes, offsets):
+ """HF Qwen4ExpTextNGramEmbedding id computation over a dense [B, L] batch."""
+ history = torch.cat([context, tokens], dim=-1)
+ shifted = [_hf_shift_right(history, s, args.ngram_boundary_token_id) for s in range(args.ngram_size)]
+ blocks = []
+ for ngram in range(2, args.ngram_size + 1):
+ start = (ngram - 2) * args.heads_per_ngram
+ end = start + args.heads_per_ngram
+ mixed = shifted[0] * multipliers[0]
+ for position in range(1, ngram):
+ mixed = torch.bitwise_xor(mixed, shifted[position] * multipliers[position])
+ ids = torch.remainder(mixed.unsqueeze(-1), sizes[start:end].view(1, 1, -1))
+ blocks.append(ids + offsets[start:end].view(1, 1, -1))
+ return torch.cat(blocks, dim=-1)[:, -tokens.shape[1] :]
+
+
+def _ragged(sequences, contexts, args, device="cpu"):
+ """Ragged PLEMetadata (prefill) for a list of per-request token lists."""
+ lens = [len(s) for s in sequences]
+ cu = torch.tensor([0, *lens], dtype=torch.int64).cumsum(0)
+ return PLEMetadata(
+ input_ids=torch.tensor([t for s in sequences for t in s], dtype=torch.int64, device=device),
+ cu_seqlens=cu.to(device),
+ seq_lens=tuple(lens),
+ ngram_context=torch.tensor(contexts, dtype=torch.int64, device=device),
+ state_slots=torch.arange(len(sequences), dtype=torch.int64, device=device),
+ fresh_slots=None,
+ is_decode=False,
+ )
+
+
+def _make_ple(config, seed: int = 3, rows: int = 4096):
+ args = config.qwen4_args
+ gen = torch.Generator().manual_seed(seed)
+ layer = PLELayer(config, args.ple_layer_ids[0])
+ _fill(layer, gen)
+ multipliers, sizes, offsets = hash_constants(args)
+ layer.ple_embedding.layer_multipliers.copy_(multipliers)
+ layer.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)
+ layer.ple_embedding.ngram_heads_offsets.copy_(offsets)
+ table = torch.randn(rows, args.ngram_head_dim, generator=gen) * 0.05
+ layer.ple_embedding.attach_table(GpuResidentTable(table, dtype=torch.float32))
+ return layer, (multipliers, sizes, offsets)
+
+
+def test_ple_hash_matches_hf():
+ """Ragged hash ids vs the HF dense reference, including eos inside and at the start of a request."""
+ config = _config()
+ args = config.qwen4_args
+ layer, (multipliers, sizes, offsets) = _make_ple(config)
+
+ sequences = [[3, 4, EOS, 5, 6], [EOS, 11, 12], [9]]
+ contexts = [[EOS, EOS], [21, 22], [EOS, 31]]
+ meta = _ragged(sequences, contexts, args)
+ got = layer.ple_embedding.row_ids(meta)
+
+ offset = 0
+ for tokens, context in zip(sequences, contexts):
+ ref = _hf_ngram_ids(
+ torch.tensor([tokens]),
+ torch.tensor([context]),
+ args,
+ multipliers,
+ sizes,
+ offsets,
+ )[0]
+ assert torch.equal(got[offset : offset + len(tokens)], ref)
+ offset += len(tokens)
+ # sequences[0][3] sits right after the boundary token, so its window is cut to eos padding
+ after_eos = layer.ple_embedding.row_ids(_ragged([[5]], [[EOS, EOS]], args))[0]
+ with_history = layer.ple_embedding.row_ids(_ragged([[5]], [[3, 4]], args))[0]
+ assert torch.equal(got[3], after_eos)
+ assert not torch.equal(got[3], with_history)
+
+
+def test_ple_forward_matches_hf():
+ """Full PLE forward (fp32) against the HF gate/norm chain and an explicit conv tap sum."""
+ torch.manual_seed(4)
+ config = _config()
+ args = config.qwen4_args
+ layer, (multipliers, sizes, offsets) = _make_ple(config)
+ hidden, hc = args.hidden_size, args.hc_count
+
+ sequences = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12, 13, 14]]
+ contexts = [[EOS, EOS], [21, 22]]
+ meta = _ragged(sequences, contexts, args)
+ total = sum(len(s) for s in sequences)
+ R = torch.randn(total, args.ple_state_width)
+ states = torch.randn(len(sequences), args.ple_state_width, args.ple_conv_state_len) * 0.1
+ got = layer.forward(R, batch=None, meta=meta, conv_states=states.clone())
+
+ offset = 0
+ for i, (tokens, context) in enumerate(zip(sequences, contexts)):
+ ids = _hf_ngram_ids(
+ torch.tensor([tokens]), torch.tensor([context]), args, multipliers, sizes, offsets
+ )[0]
+ embed = layer.ple_embedding.table.weight[ids.reshape(-1)].view(len(tokens), -1)
+ key = _group_norm(
+ F.linear(embed, layer.key_proj.weight), layer.norm_key.weight, config.rms_norm_eps, hc
+ ).unflatten(-1, (hc, hidden))
+ value = F.linear(embed, layer.value_proj.weight)
+ rows = R[offset : offset + len(tokens)]
+ query = _group_norm(
+ rows, layer.norm_query.weight, config.rms_norm_eps, hc
+ ).unflatten(-1, (hc, hidden))
+ gate = (key * query).sum(-1, keepdim=True) / math.sqrt(hidden)
+ gate = torch.sigmoid(gate.sign() * gate.abs().clamp_min(1e-6).sqrt())
+ gated = (gate * value.unsqueeze(-2)).flatten(-2)
+ normed = _group_norm(gated, layer.norm_conv.weight, config.rms_norm_eps, hc)
+ history = torch.cat([states[i], normed.transpose(0, 1)], dim=-1)
+ taps = sum(
+ layer.conv1d.weight[:, 0, k].unsqueeze(0)
+ * history.transpose(0, 1)[k * args.ple_conv_dilation :][: len(tokens)]
+ for k in range(args.ple_conv_kernel_size)
+ )
+ ref = gated + F.silu(taps)
+ assert torch.allclose(got[offset : offset + len(tokens)], ref, rtol=1e-4, atol=1e-5)
+ offset += len(tokens)
+
+
+def _fresh_ctx(**fields):
+ import freetoken.core as core
+ from freetoken.core import Context, set_global_ctx
+
+ core._GLOBAL_CTX = None # test-only: each scenario builds its own ctx
+ ctx = Context(page_size=64)
+ for name, value in fields.items():
+ setattr(ctx, name, value)
+ set_global_ctx(ctx)
+ return ctx
+
+
+def _plus_one_rmsnorm(x, weight, eps):
+ xf = x.float()
+ xf = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + eps)
+ return (xf * (1.0 + weight.float())).type_as(x)
+
+
+def _hf_rope(x, positions, rotary_dim, base):
+ """HF apply_rotary_pos_emb on [T, H, D], rotating only the first rotary_dim dims."""
+ inv = 1.0 / (
+ base ** (torch.arange(0, rotary_dim, 2, device=x.device, dtype=torch.float32) / rotary_dim)
+ )
+ freqs = positions.float().unsqueeze(-1) * inv
+ cos = torch.cat([freqs.cos(), freqs.cos()], dim=-1).unsqueeze(1)
+ sin = torch.cat([freqs.sin(), freqs.sin()], dim=-1).unsqueeze(1)
+ rot = x[..., :rotary_dim].float()
+ half = rotary_dim // 2
+ rotated = torch.cat([-rot[..., half:], rot[..., :half]], dim=-1)
+ out = (rot * cos + rotated * sin).type_as(x)
+ return torch.cat([out, x[..., rotary_dim:]], dim=-1)
+
+
+def _hf_attention(x, attn, config, positions):
+ """HF Qwen4ExpTextAttention with a dense causal mask (QSA selects every block at this length)."""
+ num_q, num_kv, dim = attn.num_q, attn.num_kv, attn.head_dim
+ qkv = F.linear(x, attn.qkv_proj.weight)
+ qg, k, v = qkv.split(attn._qkv_split, dim=-1)
+ qg = qg.view(-1, num_q, dim * 2)
+ q, gate = qg[..., :dim], qg[..., dim:].reshape(-1, num_q * dim)
+ q = _hf_rope(_plus_one_rmsnorm(q, attn.q_norm.weight, config.rms_norm_eps), positions,
+ config.rotary_config.rotary_dim, config.rotary_config.base)
+ k = _hf_rope(
+ _plus_one_rmsnorm(k.view(-1, num_kv, dim), attn.k_norm.weight, config.rms_norm_eps),
+ positions, config.rotary_config.rotary_dim, config.rotary_config.base,
+ )
+ v = v.view(-1, num_kv, dim)
+ rep = num_q // num_kv
+ scores = torch.einsum("qhd,khd->hqk", q.float(), k.repeat_interleave(rep, 1).float())
+ scores = scores * dim**-0.5
+ mask = torch.arange(x.shape[0], device=x.device) > positions.unsqueeze(-1)
+ out = torch.einsum(
+ "hqk,khd->qhd", scores.masked_fill(mask, float("-inf")).softmax(-1),
+ v.repeat_interleave(rep, 1).float(),
+ ).to(x.dtype)
+ return F.linear(out.reshape(-1, num_q * dim) * torch.sigmoid(gate), attn.o_proj.weight)
+
+
+@requires_cuda
+def test_qsa_layer_matches_hf_dense():
+ """The QSA layer under the dense oracle backend equals HF attention, and freezes what the indexer hands the backend."""
+ from freetoken.models.qwen4_exp.attention import Qwen4ExpAttention, TorchDenseQSAReference
+ from freetoken.utils.torch_utils import torch_dtype
+
+ torch.manual_seed(6)
+ config = _config()
+ device, dtype = torch.device("cuda"), torch.bfloat16
+ with torch.device(device), torch_dtype(dtype):
+ attn = Qwen4ExpAttention(config, layer_id=3)
+ _fill(attn, torch.Generator(device=device).manual_seed(7))
+
+ seq_len = 24
+ x = (torch.randn(seq_len, config.hidden_size, device=device, dtype=dtype) * 0.5)
+ positions = torch.arange(seq_len, device=device, dtype=torch.int64)
+ req = SimpleNamespace(extend_len=seq_len, cached_len=0, table_idx=1)
+ batch = SimpleNamespace(padded_reqs=[req], reqs=[req], positions=positions)
+
+ backend = TorchDenseQSAReference(config, num_slots=4, max_len=64, device=device, dtype=dtype)
+ _fresh_ctx(attn_backend=backend)
+ ref = _hf_attention(x, attn, config, positions)
+ got = attn.forward(x, batch)
+ assert torch.allclose(got.float(), ref.float(), rtol=2e-2, atol=2e-2)
+
+ index = attn.indexer.forward(x)
+ args = config.qwen4_args
+ raw = F.linear(x, attn.indexer.index_qk_proj.weight)
+ assert index.q.shape == (seq_len, args.index_n_heads, args.index_head_dim)
+ assert index.k.shape == (seq_len, args.index_head_dim)
+ assert torch.equal(index.q.reshape(seq_len, -1), raw[:, : args.index_n_heads * args.index_head_dim])
+ assert torch.equal(index.k, raw[:, args.index_n_heads * args.index_head_dim :])
+ assert index.q_norm_weight.data_ptr() == attn.indexer.q_layernorm.weight.data_ptr()
+
+
+class _StubLinearMixer(BaseOP):
+ """Stands in for the GDN layer; same [T, hidden] -> [T, hidden] shape."""
+
+ def __init__(self, config, layer_id):
+ self.out_proj = LinearReplicated(config.hidden_size, config.hidden_size, has_bias=False)
+
+ def forward(self, x):
+ return self.out_proj.forward(x)
+
+
+@requires_cuda
+def test_shared_expert_gate_fusion_matches_eager():
+ """Qwen4ExpMoE only swaps qwen3_5's gemv+sigmoid+mul+add gate chain for two triton kernels."""
+ from freetoken.models.qwen3_5_moe.moe import Qwen3_5MoE
+ from freetoken.models.qwen4_exp.moe import Qwen4ExpMoE
+ from freetoken.moe.fused import FusedMoe
+ from freetoken.utils.torch_utils import torch_dtype
+
+ config = _config()
+ device, dtype = torch.device("cuda"), torch.bfloat16
+ with torch.device(device), torch_dtype(dtype):
+ moe = Qwen4ExpMoE(config, 0)
+ _fill(moe, torch.Generator(device=device).manual_seed(21), scale=0.2)
+ _fresh_ctx(moe_backend=FusedMoe())
+
+ x = torch.randn(6, config.hidden_size, device=device, dtype=dtype) * 0.5
+ fused = moe.forward(x.clone())
+ eager = Qwen3_5MoE.forward(moe, x.clone())
+
+ routed = moe.experts.forward(hidden_states=x.clone(), router_logits=moe.gate.forward(x))
+ gate = torch.sigmoid(x.float() @ moe.shared_expert_gate.weight.float().view(-1))
+ ref = routed.float() + gate.unsqueeze(1) * moe.shared_expert.forward(x).float()
+
+ assert fused.shape == x.shape and fused.dtype == dtype
+ torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2)
+ # The fused gate stays in fp32 where the eager chain rounds the scalar to bf16.
+ assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max()
+
+
+@requires_cuda
[email protected]("dtype", [torch.bfloat16])
[email protected]("num_tokens,hidden", [(1, 2560), (7, 640)])
+def test_shared_gate_kernels_match_torch(num_tokens, hidden, dtype):
+ """Shipping hidden size for the two shared-gate kernels, against the torch chain they replace."""
+ from freetoken.kernel.triton.moe_shared_gate import shared_gate_mul_add, shared_gate_sigmoid
+
+ gen = torch.Generator(device="cuda").manual_seed(hidden + num_tokens)
+ kw = {"generator": gen, "device": "cuda", "dtype": dtype}
+ x = torch.randn(num_tokens, hidden, **kw)
+ weight = torch.randn(1, hidden, **kw) * 0.05
+ shared = torch.randn(num_tokens, hidden, **kw)
+ routed = torch.randn(num_tokens, hidden, **kw)
+
+ fused = shared_gate_mul_add(routed, shared, shared_gate_sigmoid(x, weight.view(-1)))
+ eager = routed + shared * torch.sigmoid(F.linear(x, weight))
+ ref = routed.float() + shared.float() * torch.sigmoid(x.float() @ weight.float().view(-1))[:, None]
+
+ assert fused.dtype == dtype and fused.shape == routed.shape
+ torch.testing.assert_close(fused, eager, rtol=2e-2, atol=2e-2)
+ assert (fused.float() - ref).abs().max() <= (eager.float() - ref).abs().max() + 1e-6
+
+
+@requires_cuda
+def test_decoder_stack_prefill_and_decode(monkeypatch):
+ """Ragged bs=3 prefill then a bs=3 decode step through the whole model with dummy weights."""
+ from freetoken.kvcache.linear_state_pool import LinearStatePool
+ from freetoken.models.qwen4_exp import model as model_module
+ from freetoken.models.qwen4_exp.attention import TorchDenseQSAReference
+ from freetoken.models.qwen4_exp.ple import GpuResidentTable
+ from freetoken.moe.fused import FusedMoe
+ from freetoken.utils.torch_utils import torch_dtype
+
+ torch.manual_seed(8)
+ config = _config()
+ args = config.qwen4_args
+ device, dtype = torch.device("cuda"), torch.bfloat16
+ monkeypatch.setattr(model_module, "build_linear_mixer", _StubLinearMixer)
+
+ with torch.device(device), torch_dtype(dtype):
+ model = model_module.Qwen4ExpForCausalLM(config)
+ gen = torch.Generator(device=device).manual_seed(9)
+ _fill(model, gen)
+ multipliers, sizes, offsets = hash_constants(args)
+ table = torch.randn(4096, args.ngram_head_dim, generator=gen, device=device, dtype=dtype) * 0.05
+ for ple in model.model.ple_layers:
+ ple.ple_embedding.layer_multipliers.copy_(multipliers)
+ ple.ple_embedding.ngram_heads_vocab_sizes.copy_(sizes)
+ ple.ple_embedding.ngram_heads_offsets.copy_(offsets)
+ ple.ple_embedding.attach_table(GpuResidentTable(table, dtype=dtype))
+
+ num_slots, max_len = 4, 64
+ pool = LinearStatePool(
+ config.linear_attention_group(), num_slots, dtype, device,
+ slot_states=config.slot_states,
+ )
+ prompts = [[3, 4, EOS, 5, 6, 8], [2, EOS, 11, 12], [9, 10, 11, 12, 13]]
+ ctx = _fresh_ctx(
+ attn_backend=TorchDenseQSAReference(config, num_slots, max_len, device, dtype),
+ moe_backend=FusedMoe(),
+ linear_state_pool=pool,
+ )
+ reqs = [
+ SimpleNamespace(
+ extend_len=len(p), cached_len=0, table_idx=i + 1, linear_slot_idx=None,
+ input_ids=torch.tensor(p, dtype=torch.int64),
+ )
+ for i, p in enumerate(prompts)
+ ]
+ flat = [t for p in prompts for t in p]
+ last = torch.tensor(
+ [sum(len(p) for p in prompts[: i + 1]) - 1 for i in range(len(prompts))], device=device
+ )
+ batch = SimpleNamespace(
+ padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=True, is_decode=False,
+ input_ids=torch.tensor(flat, dtype=torch.int64, device=device),
+ positions=torch.cat([torch.arange(len(p)) for p in prompts]).to(device),
+ attn_metadata=SimpleNamespace(get_last_indices=lambda bs: last[:bs]),
+ )
+ with ctx.forward_batch(batch):
+ logits = model.forward()
+ assert logits.shape == (len(prompts), config.vocab_size)
+ assert torch.isfinite(logits.float()).all()
+
+ for r, p in zip(reqs, prompts):
+ r.cached_len = len(p)
+ r.extend_len = 1
+ r.input_ids = torch.cat([r.input_ids, torch.tensor([14], dtype=torch.int64)])
+ decode = SimpleNamespace(
+ padded_reqs=reqs, reqs=reqs, size=len(reqs), is_prefill=False, is_decode=True,
+ input_ids=torch.tensor([14] * len(reqs), dtype=torch.int64, device=device),
+ positions=torch.tensor([len(p) for p in prompts], dtype=torch.int64, device=device),
+ attn_metadata=None,
+ )
+ with ctx.forward_batch(decode):
+ decode_logits = model.forward()
+ assert decode_logits.shape == (len(prompts), config.vocab_size)
+ assert torch.isfinite(decode_logits.float()).all()
diff --git a/tests/models/qwen4_exp/test_weight.py b/tests/models/qwen4_exp/test_weight.py
new file mode 100644
index 00000000..b3f1f851
--- /dev/null
+++ b/tests/models/qwen4_exp/test_weight.py
@@ -0,0 +1,410 @@
+"""qwen4_exp weight loading against a synthetic checkpoint shaped like the RadixArk NVFP4 one.
+
+The tensors are tiny but the key names, dtypes and the fusion geometry that matters
+(hc_lowrank=320 + hc_count=4 -> a 12-row zero pad) are the real ones.
+"""
+
+from __future__ import annotations
+
+import random
+from types import SimpleNamespace
+
+import pytest
+import torch
+from safetensors.torch import save_file
+
+from freetoken.distributed import set_tp_info, try_get_tp_info
+from freetoken.kernel.aot_models import SUPPORTED_MODELS, expert_bank_row_bytes
+from freetoken.models.qwen4_exp.weight import (
+ _ZERO_CENTERED_NORM_SUFFIXES,
+ iter_weights,
+ load_ple_table,
+)
+from freetoken.moe.host_banks import HostBank, read_range_into
+
+H = 32 # hidden_size
+HC = 4 # hc_count
+LR = 320 # hc_lowrank; kept real so the merged HC pad is the real (-(320+4)) % 16 = 12
+HCH = HC * H # hyper-connection stream width
+KH, VH, HD = 2, 6, 8 # GDN key / value heads, head dim
+QH, KVH, AHD = 4, 2, 16 # QSA q / kv heads, head dim
+IHD = 8 # indexer head dim
+E, I = 3, 6 # routed experts, moe_intermediate_size
+NGRAM_DIM, NGRAM_ROWS, NGRAM_SHARDS = 4, 7, 4
+
+
[email protected](scope="session", autouse=True)
+def _tp_info():
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+
+def _bf16(*shape: int) -> torch.Tensor:
+ return torch.randn(*shape).to(torch.bfloat16)
+
+
+def _hc_weights(prefix: str, inject: bool) -> dict[str, torch.Tensor]:
+ w = {
+ f"{prefix}.hc_norm.weight": _bf16(HCH),
+ f"{prefix}.input_mix_weight_down.weight": _bf16(LR, HCH),
+ f"{prefix}.input_mix_weight_up.weight": _bf16(HCH, LR),
+ }
+ if inject:
+ w[f"{prefix}.block_inject_weight.weight"] = _bf16(HC, HCH)
+ return w
+
+
+def _raw_checkpoint() -> dict[str, torch.Tensor]:
+ """Layer 0 = GDN + PLE, layer 1 = QSA; plus the mtp / visual / routed-expert noise."""
+ lm = "model.language_model"
+ raw: dict[str, torch.Tensor] = {
+ f"{lm}.embed_tokens.weight": _bf16(11, H),
+ "lm_head.weight": _bf16(11, H),
+ }
+ raw.update(_hc_weights(f"{lm}.hyper_connection_mixer", inject=False))
+ for layer in (0, 1):
+ raw.update(_hc_weights(f"{lm}.layers.{layer}.attn_hyper_connection", inject=True))
+ raw.update(_hc_weights(f"{lm}.layers.{layer}.mlp_hyper_connection", inject=True))
+ raw.update({
+ f"{lm}.layers.{layer}.mlp.gate.weight": _bf16(E, H),
+ f"{lm}.layers.{layer}.mlp.shared_expert.gate_proj.weight": _bf16(I, H),
+ f"{lm}.layers.{layer}.mlp.shared_expert.up_proj.weight": _bf16(I, H),
+ f"{lm}.layers.{layer}.mlp.shared_expert.down_proj.weight": _bf16(H, I),
+ f"{lm}.layers.{layer}.mlp.shared_expert_gate.weight": _bf16(1, H),
+ })
+ for expert in range(E):
+ base = f"{lm}.layers.{layer}.mlp.experts.{expert}"
+ for proj, out, inn in (("gate_proj", I, H), ("up_proj", I, H), ("down_proj", H, I)):
+ raw[f"{base}.{proj}.weight"] = torch.randint(
+ 0, 256, (out, inn // 2), dtype=torch.uint8
+ )
+ raw[f"{base}.{proj}.weight_scale"] = torch.ones(
+ out, inn // 16 or 1, dtype=torch.float8_e4m3fn
+ )
+ raw[f"{base}.{proj}.weight_scale_2"] = torch.tensor(0.5)
+ raw[f"{base}.{proj}.input_scale"] = torch.tensor(0.25)
+ gdn = f"{lm}.layers.0.linear_attn"
+ raw.update({
+ f"{gdn}.in_proj_qkv.weight": _bf16(2 * KH * HD + VH * HD, H),
+ f"{gdn}.in_proj_z.weight": _bf16(VH * HD, H),
+ f"{gdn}.in_proj_b.weight": _bf16(VH, H),
+ f"{gdn}.in_proj_a.weight": _bf16(VH, H),
+ f"{gdn}.conv1d.weight": _bf16(2 * KH * HD + VH * HD, 1, 4),
+ f"{gdn}.A_log": _bf16(VH),
+ f"{gdn}.dt_bias": _bf16(VH),
+ f"{gdn}.norm.weight": _bf16(HD),
+ f"{gdn}.out_proj.weight": _bf16(H, VH * HD),
+ })
+ ple = f"{lm}.layers.0.ple"
+ raw.update({
+ f"{ple}.key_proj.weight": _bf16(HCH, H),
+ f"{ple}.value_proj.weight": _bf16(H, H),
+ f"{ple}.norm_key.weight": _bf16(HCH),
+ f"{ple}.norm_query.weight": _bf16(HCH),
+ f"{ple}.norm_conv.weight": _bf16(HCH),
+ f"{ple}.conv1d.weight": _bf16(HCH, 1, 4),
+ f"{ple}.ple_embedding.layer_multipliers": torch.randint(1, 1 << 40, (3,)),
+ f"{ple}.ple_embedding.ngram_heads_offsets": torch.arange(4),
+ f"{ple}.ple_embedding.ngram_heads_vocab_sizes": torch.full((4,), 5),
+ })
+ attn = f"{lm}.layers.1.self_attn"
+ raw.update({
+ f"{attn}.q_proj.weight": _bf16(2 * QH * AHD, H),
+ f"{attn}.k_proj.weight": _bf16(KVH * AHD, H),
+ f"{attn}.v_proj.weight": _bf16(KVH * AHD, H),
+ f"{attn}.o_proj.weight": _bf16(H, QH * AHD),
+ f"{attn}.q_norm.weight": _bf16(AHD),
+ f"{attn}.k_norm.weight": _bf16(AHD),
+ f"{attn}.indexer.index_qk_proj.weight": _bf16(5 * IHD, H),
+ f"{attn}.indexer.q_layernorm.weight": _bf16(IHD),
+ f"{attn}.indexer.k_layernorm.weight": _bf16(IHD),
+ })
+ raw.update({
+ "mtp.hyper_connection_mixer.hc_norm.weight": _bf16(HCH),
+ "mtp.layers.0.self_attn.q_proj.weight": _bf16(2 * QH * AHD, H),
+ "mtp.layers.0.mlp.experts.gate_up_proj": _bf16(E, 2 * I, H),
+ "mtp.layers.0.mlp.experts.down_proj": _bf16(E, H, I),
+ "model.visual.blocks.0.attn.qkv.weight": _bf16(3 * H, H),
+ "model.visual.merger.norm.weight": _bf16(H),
+ })
+ return raw
+
+
+def _ngram_table() -> tuple[dict[str, torch.Tensor], torch.Tensor]:
+ prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding"
+ shards = {
+ f"{prefix}.shard_{i}.weight": (
+ torch.arange(i * NGRAM_ROWS * NGRAM_DIM, (i + 1) * NGRAM_ROWS * NGRAM_DIM)
+ .remainder(200).to(torch.uint8).view(NGRAM_ROWS, NGRAM_DIM).view(torch.float8_e4m3fn)
+ )
+ for i in range(NGRAM_SHARDS)
+ }
+ scale = torch.tensor([0.125], dtype=torch.bfloat16)
+ shards[f"{prefix}.weight_scale"] = scale
+ return shards, scale
+
+
[email protected](scope="module")
+def checkpoint(tmp_path_factory) -> tuple[str, dict[str, torch.Tensor]]:
+ torch.manual_seed(0)
+ folder = tmp_path_factory.mktemp("qwen4_exp_ckpt")
+ raw = _raw_checkpoint()
+ table, _scale = _ngram_table()
+ # Spread the dense tensors over two shards so the fusion buffer has to survive a file
+ # boundary, and put the n-gram table in its own shards like the real checkpoint does.
+ names = sorted(raw)
+ save_file({n: raw[n] for n in names[::2]}, str(folder / "model-bf16-00001.safetensors"))
+ save_file({n: raw[n] for n in names[1::2]}, str(folder / "model-bf16-00002.safetensors"))
+ shard_names = sorted(table)
+ save_file({n: table[n] for n in shard_names[:2]}, str(folder / "model-plefp8-00000.safetensors"))
+ save_file({n: table[n] for n in shard_names[2:]}, str(folder / "model-plefp8-00001.safetensors"))
+ return str(folder), {**raw, **table}
+
+
[email protected](scope="module")
+def loaded(checkpoint) -> dict[str, torch.Tensor]:
+ folder, _raw = checkpoint
+ return {
+ name: tensor.clone()
+ for name, tensor in iter_weights(
+ folder, torch.device("cpu"), include_moe_experts=True, include_non_moe=True
+ )
+ }
+
+
+def _expected_names() -> set[str]:
+ names = {"model.embed_tokens.weight", "lm_head.weight"}
+ names |= {f"model.hyper_connection_mixer.{leaf}" for leaf in
+ ("hc_norm.weight", "input_mix_weight_down.weight", "input_mix_weight_up.weight")}
+ for layer in (0, 1):
+ for hc in ("attn_hyper_connection", "mlp_hyper_connection"):
+ names |= {f"model.layers.{layer}.{hc}.{leaf}" for leaf in (
+ "hc_norm.weight", "input_mix_weight_down_block_inject.weight",
+ "input_mix_weight_up.weight")}
+ names |= {f"model.layers.{layer}.mlp.{leaf}" for leaf in (
+ "gate.weight", "shared_expert.gate_up_proj.weight",
+ "shared_expert.down_proj.weight", "shared_expert_gate.weight")}
+ names |= {f"model.layers.0.linear_attn.{leaf}" for leaf in (
+ "in_proj.weight", "conv1d.weight", "A_log", "dt_bias", "norm.weight", "out_proj.weight")}
+ names |= {f"model.layers.0.ple.{leaf}" for leaf in (
+ "key_proj.weight", "value_proj.weight", "norm_key.weight", "norm_query.weight",
+ "norm_conv.weight", "conv1d.weight", "ple_embedding.layer_multipliers",
+ "ple_embedding.ngram_heads_offsets", "ple_embedding.ngram_heads_vocab_sizes")}
+ names |= {f"model.layers.1.self_attn.{leaf}" for leaf in (
+ "qkv_proj.weight", "o_proj.weight", "q_norm.weight", "k_norm.weight",
+ "indexer.index_qk_proj.weight", "indexer.q_layernorm.weight",
+ "indexer.k_layernorm.weight")}
+ return names
+
+
+def test_key_map_is_exactly_the_model_state_dict(loaded):
+ assert set(loaded) == _expected_names()
+
+
+def test_mtp_visual_experts_and_table_never_loaded(loaded):
+ for name in loaded:
+ assert not name.startswith(("mtp.", "model.visual."))
+ assert ".mlp.experts." not in name
+ assert "ngram_embedding" not in name
+ assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale"))
+
+
+def test_hc_merge_is_down_then_inject_then_zero_pad(loaded, checkpoint):
+ _folder, raw = checkpoint
+ key = "model.layers.0.attn_hyper_connection.input_mix_weight_down_block_inject.weight"
+ merged = loaded[key]
+ assert merged.shape == (LR + HC + 12, HCH) # pad = (-(320 + 4)) % 16
+ down = raw["model.language_model.layers.0.attn_hyper_connection.input_mix_weight_down.weight"]
+ inject = raw["model.language_model.layers.0.attn_hyper_connection.block_inject_weight.weight"]
+ assert torch.equal(merged[:LR], down)
+ assert torch.equal(merged[LR:LR + HC], inject)
+ assert torch.equal(merged[LR + HC:], torch.zeros(12, HCH, dtype=merged.dtype))
+
+
+def test_top_level_mixer_keeps_the_unmerged_down(loaded, checkpoint):
+ _folder, raw = checkpoint
+ got = loaded["model.hyper_connection_mixer.input_mix_weight_down.weight"]
+ assert got.shape == (LR, HCH)
+ assert torch.equal(
+ got, raw["model.language_model.hyper_connection_mixer.input_mix_weight_down.weight"]
+ )
+ assert torch.equal(
+ loaded["model.hyper_connection_mixer.input_mix_weight_up.weight"],
+ raw["model.language_model.hyper_connection_mixer.input_mix_weight_up.weight"],
+ )
+
+
+def test_qkv_fusion_slices_back_to_q_k_v(loaded, checkpoint):
+ _folder, raw = checkpoint
+ attn = "model.language_model.layers.1.self_attn"
+ parts = [raw[f"{attn}.{p}_proj.weight"] for p in ("q", "k", "v")]
+ fused = loaded["model.layers.1.self_attn.qkv_proj.weight"]
+ assert fused.shape == (2 * QH * AHD + 2 * KVH * AHD, H) # q carries the output gate
+ for part, back in zip(parts, torch.split(fused, [p.shape[0] for p in parts], dim=0)):
+ assert torch.equal(part, back)
+
+
+def test_gdn_in_proj_slices_round_trip(loaded, checkpoint):
+ _folder, raw = checkpoint
+ gdn = "model.language_model.layers.0.linear_attn"
+ parts = [raw[f"{gdn}.in_proj_{p}.weight"] for p in ("qkv", "z", "b", "a")]
+ fused = loaded["model.layers.0.linear_attn.in_proj.weight"]
+ assert fused.shape == (sum(p.shape[0] for p in parts), H)
+ splits = torch.split(fused, [p.shape[0] for p in parts], dim=0)
+ for part, back in zip(parts, splits):
+ assert torch.equal(part, back)
+
+
+def test_shared_expert_gate_up_merge(loaded, checkpoint):
+ _folder, raw = checkpoint
+ base = "model.language_model.layers.1.mlp.shared_expert"
+ merged = loaded["model.layers.1.mlp.shared_expert.gate_up_proj.weight"]
+ assert torch.equal(merged[:I], raw[f"{base}.gate_proj.weight"])
+ assert torch.equal(merged[I:], raw[f"{base}.up_proj.weight"])
+
+
+ZERO_CENTERED = (
+ "model.layers.0.attn_hyper_connection.hc_norm.weight",
+ "model.layers.0.mlp_hyper_connection.hc_norm.weight",
+ "model.hyper_connection_mixer.hc_norm.weight",
+ "model.layers.0.ple.norm_key.weight",
+ "model.layers.0.ple.norm_query.weight",
+ "model.layers.0.ple.norm_conv.weight",
+ "model.layers.1.self_attn.q_norm.weight",
+ "model.layers.1.self_attn.k_norm.weight",
+ "model.layers.1.self_attn.indexer.q_layernorm.weight",
+ "model.layers.1.self_attn.indexer.k_layernorm.weight",
+)
+
+
+def test_zero_centered_norms_are_loaded_raw(loaded, checkpoint):
+ """(1+w) is applied at runtime in fp32, so the loader must not fold it into the bf16 weight."""
+ _folder, raw = checkpoint
+ for name in ZERO_CENTERED:
+ raw_name = name.replace("model.", "model.language_model.", 1)
+ assert torch.equal(loaded[name], raw[raw_name]), name
+
+
+def test_the_zero_centered_suffix_list_covers_every_such_norm():
+ assert {n for n in ZERO_CENTERED if n.endswith(_ZERO_CENTERED_NORM_SUFFIXES)} == set(ZERO_CENTERED)
+ assert not "model.layers.0.linear_attn.norm.weight".endswith(_ZERO_CENTERED_NORM_SUFFIXES)
+
+
+def test_gdn_gated_norm_passes_through(loaded, checkpoint):
+ _folder, raw = checkpoint
+ assert torch.equal(
+ loaded["model.layers.0.linear_attn.norm.weight"],
+ raw["model.language_model.layers.0.linear_attn.norm.weight"],
+ )
+
+
+def test_hash_constants_stay_int64(loaded):
+ for leaf in ("layer_multipliers", "ngram_heads_offsets", "ngram_heads_vocab_sizes"):
+ assert loaded[f"model.layers.0.ple.ple_embedding.{leaf}"].dtype is torch.int64
+
+
+def test_load_ple_table_concatenates_shards_in_index_order(checkpoint):
+ folder, raw = checkpoint
+ args = SimpleNamespace(split_ngram_parts=NGRAM_SHARDS, ngram_head_dim=NGRAM_DIM)
+ table = load_ple_table(folder, args, pin=False)
+ assert table.tensor.shape == (NGRAM_SHARDS * NGRAM_ROWS, NGRAM_DIM)
+ assert table.tensor.dtype is torch.float8_e4m3fn
+ prefix = "model.language_model.layers.0.ple.ple_embedding.ngram_embedding"
+ for shard in range(NGRAM_SHARDS):
+ rows = table.tensor[shard * NGRAM_ROWS: (shard + 1) * NGRAM_ROWS]
+ assert torch.equal(rows.view(torch.uint8),
+ raw[f"{prefix}.shard_{shard}.weight"].view(torch.uint8))
+ assert table.weight_scale.dtype is torch.bfloat16
+ assert float(table.weight_scale) == 0.125
+
+
+def test_load_ple_table_rejects_a_shard_count_mismatch(checkpoint):
+ folder, _raw = checkpoint
+ args = SimpleNamespace(split_ngram_parts=NGRAM_SHARDS + 1, ngram_head_dim=NGRAM_DIM)
+ with pytest.raises(ValueError, match="shards 0"):
+ load_ple_table(folder, args, pin=False)
+
+
+# ======================================================================================
+# read_range_into: the O_DIRECT byte-range read the PLE table load is built on
+# ======================================================================================
+
+
[email protected](scope="module")
+def blob(tmp_path_factory) -> tuple[str, bytes]:
+ data = random.Random(7).randbytes(5_000_003)
+ path = tmp_path_factory.mktemp("blob") / "data.bin"
+ path.write_bytes(data)
+ return str(path), data
+
+
[email protected]("file_offset, nbytes, dest_offset", [
+ (1, 4095, 0), # sub-block, unaligned source
+ (2239, 1_000_000, 0), # the real checkpoint's header-end phase
+ (4095, 4097, 1), # straddles two block boundaries
+ (4_999_000, 1003, 123_456), # runs to EOF
+])
+def test_read_range_into_matches_the_file(blob, file_offset, nbytes, dest_offset):
+ path, data = blob
+ bank = HostBank((6_000_000,), torch.uint8)
+ view = bank.memoryview()
+ got = read_range_into(view, path, file_offset=file_offset, nbytes=nbytes,
+ dest_offset=dest_offset, chunk=1 << 20)
+ assert got == nbytes
+ assert bytes(view[dest_offset:dest_offset + nbytes]) == data[file_offset:file_offset + nbytes]
+
+
+def test_read_range_into_is_chunk_and_thread_safe(blob):
+ path, data = blob
+ bank = HostBank((6_000_000,), torch.uint8)
+ view = bank.memoryview()
+ read_range_into(view, path, file_offset=2239, nbytes=4_000_000, dest_offset=1024,
+ workers=8, chunk=64 << 10)
+ assert bytes(view[1024:1024 + 4_000_000]) == data[2239:2239 + 4_000_000]
+
+
+def test_read_range_into_rejects_a_short_destination(blob):
+ path, _data = blob
+ bank = HostBank((1024,), torch.uint8)
+ with pytest.raises(ValueError, match="destination holds"):
+ read_range_into(bank.memoryview(), path, file_offset=0, nbytes=1 << 20)
+
+
+# ======================================================================================
+# AOT shape table
+# ======================================================================================
+
+
+def test_aot_entry_carries_the_checkpoint_geometry():
+ entry = next(m for m in SUPPORTED_MODELS
+ if m.architecture == "Qwen4ExpForConditionalGeneration")
+ assert (entry.hidden_size, entry.moe_intermediate_size, entry.top_k) == (2560, 640, 10)
+ assert entry.kv_groups == ((2, 256),)
+ rows = expert_bank_row_bytes("nvfp4", entry.hidden_size, entry.moe_intermediate_size)
+ assert set(rows) == {"gate_up_packed", "gate_up_scale", "gate_up_global",
+ "down_packed", "down_scale", "down_global"}
+ for name, nbytes in rows.items():
+ assert nbytes % 16 == 0, name # fused multi-bank copy only engages on 16B multiples
+
+
+def test_every_registry_architecture_is_claimed_by_an_aot_entry():
+ from freetoken.models.register import _MODEL_REGISTRY
+
+ claimed = {m.architecture for m in SUPPORTED_MODELS}
+ claimed |= {a for m in SUPPORTED_MODELS for a in m.arch_aliases}
+ assert "Qwen4ExpForConditionalGeneration" in claimed
+ assert set(_MODEL_REGISTRY) - claimed == set()
+
+
[email protected](not torch.cuda.is_available(), reason="needs cuda")
+def test_fusion_pad_rides_the_tensor_device():
+ """safetensors loads straight to cuda; a cpu-allocated pad row would break torch.cat."""
+ from freetoken.models.qwen4_exp.weight import _try_fuse
+
+ buf = {}
+ down = torch.randn(320, 64, device="cuda", dtype=torch.bfloat16)
+ inject = torch.randn(4, 64, device="cuda", dtype=torch.bfloat16)
+ assert _try_fuse("model.layers.0.attn_hyper_connection.input_mix_weight_down.weight", down, buf) == ()
+ key, fused = _try_fuse("model.layers.0.attn_hyper_connection.block_inject_weight.weight", inject, buf)
+ assert fused.device.type == "cuda" and fused.shape[0] == 336
+ assert torch.equal(fused[324:], torch.zeros(12, 64, device="cuda", dtype=torch.bfloat16))
diff --git a/tests/models/qwen4_exp/test_weight_ckpt.py b/tests/models/qwen4_exp/test_weight_ckpt.py
new file mode 100644
index 00000000..747d22f2
--- /dev/null
+++ b/tests/models/qwen4_exp/test_weight_ckpt.py
@@ -0,0 +1,338 @@
+"""qwen4_exp weight loading against the real RadixArk/Qwen3.8-Flash-Next-NVFP4 checkpoint.
+
+Set ``FREETOKEN_QWEN4EXP_MODEL`` to the local checkpoint directory to run these. Everything is
+sampled except the PLE table, which is loaded and pinned in full once (~47.7 GiB) because that
+is the only way to check the shard concatenation and the pin budget.
+"""
+
+from __future__ import annotations
+
+import dataclasses
+import json
+import os
+import random
+from types import SimpleNamespace
+
+import pytest
+import safetensors
+import torch
+
+from freetoken.distributed import set_tp_info, try_get_tp_info
+from freetoken.kernel.aot_models import expert_bank_row_bytes
+from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks
+from freetoken.models.qwen4_exp.config import parse_config
+from freetoken.models.qwen4_exp.weight import (
+ _NVFP4_SOURCE_SPEC,
+ _ZERO_CENTERED_NORM_SUFFIXES,
+ iter_weights,
+ load_ple_table,
+)
+from freetoken.moe.host_banks import HostResidency
+from freetoken.utils import cached_load_hf_config
+
+MODEL_PATH = os.environ.get("FREETOKEN_QWEN4EXP_MODEL")
+pytestmark = [
+ pytest.mark.needs_weights,
+ pytest.mark.skipif(not MODEL_PATH, reason="FREETOKEN_QWEN4EXP_MODEL is not set"),
+]
+
+E, H, I = 512, 2560, 640
+NUM_LAYERS = 48
+PLE_LAYER = 1
+PLE_SHARDS, PLE_ROWS_PER_SHARD, PLE_DIM = 128, 2_500_012, 160
+PLE_BYTES = PLE_SHARDS * PLE_ROWS_PER_SHARD * PLE_DIM
+EXPERT_LAYER_BYTES = E * sum(expert_bank_row_bytes("nvfp4", H, I).values())
+LM = "model.language_model"
+
+
[email protected](scope="session", autouse=True)
+def _tp_info():
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+
+
+class _Reader:
+ """Serves checkpoint tensors by their raw key, through the index shard map."""
+
+ def __init__(self, folder: str):
+ with open(os.path.join(folder, "model.safetensors.index.json"), encoding="utf-8") as fh:
+ self._map = json.load(fh)["weight_map"]
+ self._folder = folder
+ self._handles: dict = {}
+
+ def get(self, name: str) -> torch.Tensor:
+ shard = self._map[name]
+ handle = self._handles.get(shard)
+ if handle is None:
+ handle = safetensors.safe_open(
+ os.path.join(self._folder, shard), framework="pt", device="cpu"
+ ).__enter__()
+ self._handles[shard] = handle
+ return handle.get_tensor(name)
+
+ def close(self) -> None:
+ for handle in self._handles.values():
+ handle.__exit__(None, None, None)
+ self._handles.clear()
+
+
+def _gdn_parts(layer: int) -> list[str]:
+ return [f"{LM}.layers.{layer}.linear_attn.in_proj_{p}.weight" for p in ("qkv", "z", "b", "a")]
+
+
+def _hc_parts(layer: int, hc: str) -> list[str]:
+ return [f"{LM}.layers.{layer}.{hc}.input_mix_weight_down.weight",
+ f"{LM}.layers.{layer}.{hc}.block_inject_weight.weight"]
+
+
+def _qkv_parts(layer: int) -> list[str]:
+ return [f"{LM}.layers.{layer}.self_attn.{p}_proj.weight" for p in ("q", "k", "v")]
+
+
+# (model key, checkpoint keys, mode). "cat16" additionally requires the merged rows to be
+# zero-padded up to a multiple of 16. Zero-centered norms are "same": (1+w) is a runtime op.
+SAMPLES: tuple[tuple[str, list[str], str], ...] = (
+ ("model.embed_tokens.weight", [f"{LM}.embed_tokens.weight"], "same"),
+ ("lm_head.weight", ["lm_head.weight"], "same"),
+ ("model.layers.0.linear_attn.in_proj.weight", _gdn_parts(0), "cat"),
+ ("model.layers.46.linear_attn.in_proj.weight", _gdn_parts(46), "cat"),
+ ("model.layers.0.linear_attn.conv1d.weight", [f"{LM}.layers.0.linear_attn.conv1d.weight"], "same"),
+ ("model.layers.0.linear_attn.A_log", [f"{LM}.layers.0.linear_attn.A_log"], "same"),
+ ("model.layers.0.linear_attn.dt_bias", [f"{LM}.layers.0.linear_attn.dt_bias"], "same"),
+ ("model.layers.0.linear_attn.norm.weight", [f"{LM}.layers.0.linear_attn.norm.weight"], "same"),
+ ("model.layers.0.linear_attn.out_proj.weight", [f"{LM}.layers.0.linear_attn.out_proj.weight"], "same"),
+ ("model.layers.3.self_attn.qkv_proj.weight", _qkv_parts(3), "cat"),
+ ("model.layers.47.self_attn.qkv_proj.weight", _qkv_parts(47), "cat"),
+ ("model.layers.3.self_attn.o_proj.weight", [f"{LM}.layers.3.self_attn.o_proj.weight"], "same"),
+ ("model.layers.3.self_attn.q_norm.weight", [f"{LM}.layers.3.self_attn.q_norm.weight"], "same"),
+ ("model.layers.3.self_attn.k_norm.weight", [f"{LM}.layers.3.self_attn.k_norm.weight"], "same"),
+ ("model.layers.3.self_attn.indexer.index_qk_proj.weight",
+ [f"{LM}.layers.3.self_attn.indexer.index_qk_proj.weight"], "same"),
+ ("model.layers.3.self_attn.indexer.q_layernorm.weight",
+ [f"{LM}.layers.3.self_attn.indexer.q_layernorm.weight"], "same"),
+ ("model.layers.47.self_attn.indexer.k_layernorm.weight",
+ [f"{LM}.layers.47.self_attn.indexer.k_layernorm.weight"], "same"),
+ ("model.layers.7.attn_hyper_connection.hc_norm.weight",
+ [f"{LM}.layers.7.attn_hyper_connection.hc_norm.weight"], "same"),
+ ("model.layers.7.attn_hyper_connection.input_mix_weight_down_block_inject.weight",
+ _hc_parts(7, "attn_hyper_connection"), "cat16"),
+ ("model.layers.7.attn_hyper_connection.input_mix_weight_up.weight",
+ [f"{LM}.layers.7.attn_hyper_connection.input_mix_weight_up.weight"], "same"),
+ ("model.layers.47.mlp_hyper_connection.input_mix_weight_down_block_inject.weight",
+ _hc_parts(47, "mlp_hyper_connection"), "cat16"),
+ ("model.hyper_connection_mixer.hc_norm.weight",
+ [f"{LM}.hyper_connection_mixer.hc_norm.weight"], "same"),
+ ("model.hyper_connection_mixer.input_mix_weight_down.weight",
+ [f"{LM}.hyper_connection_mixer.input_mix_weight_down.weight"], "same"),
+ ("model.hyper_connection_mixer.input_mix_weight_up.weight",
+ [f"{LM}.hyper_connection_mixer.input_mix_weight_up.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.key_proj.weight", [f"{LM}.layers.{PLE_LAYER}.ple.key_proj.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.value_proj.weight", [f"{LM}.layers.{PLE_LAYER}.ple.value_proj.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.norm_key.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_key.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.norm_query.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_query.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.norm_conv.weight", [f"{LM}.layers.{PLE_LAYER}.ple.norm_conv.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.conv1d.weight", [f"{LM}.layers.{PLE_LAYER}.ple.conv1d.weight"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.ple_embedding.layer_multipliers",
+ [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.layer_multipliers"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_offsets",
+ [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_offsets"], "same"),
+ (f"model.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_vocab_sizes",
+ [f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_heads_vocab_sizes"], "same"),
+ ("model.layers.5.mlp.gate.weight", [f"{LM}.layers.5.mlp.gate.weight"], "same"),
+ ("model.layers.5.mlp.shared_expert.gate_up_proj.weight",
+ [f"{LM}.layers.5.mlp.shared_expert.gate_proj.weight",
+ f"{LM}.layers.5.mlp.shared_expert.up_proj.weight"], "cat"),
+ ("model.layers.5.mlp.shared_expert.down_proj.weight",
+ [f"{LM}.layers.5.mlp.shared_expert.down_proj.weight"], "same"),
+ ("model.layers.5.mlp.shared_expert_gate.weight",
+ [f"{LM}.layers.5.mlp.shared_expert_gate.weight"], "same"),
+)
+
+
[email protected](scope="module")
+def reader() -> _Reader:
+ r = _Reader(MODEL_PATH)
+ yield r
+ r.close()
+
+
[email protected](scope="module")
+def dense_pass() -> tuple[list[str], dict[str, torch.Tensor]]:
+ """One full iter_weights sweep: every emitted name, plus a clone of each sampled tensor.
+
+ The zero-centered norms are kept too -- they are tiny, and checking all of them is the
+ cheapest guard against the +1 creeping back into the load path."""
+ wanted = {name for name, _raw, _mode in SAMPLES}
+ names: list[str] = []
+ sampled: dict[str, torch.Tensor] = {}
+ for name, tensor in iter_weights(
+ MODEL_PATH, torch.device("cpu"), include_moe_experts=True, include_non_moe=True
+ ):
+ names.append(name)
+ if name in wanted or name.endswith(_ZERO_CENTERED_NORM_SUFFIXES):
+ sampled[name] = tensor.clone()
+ return names, sampled
+
+
+def test_emitted_names_are_unique_and_complete(dense_pass):
+ names, _sampled = dense_pass
+ assert len(names) == len(set(names))
+ assert len([n for n in names if n.endswith(".linear_attn.in_proj.weight")]) == 36
+ assert len([n for n in names if n.endswith(".self_attn.qkv_proj.weight")]) == 12
+ assert len([n for n in names
+ if n.endswith(".input_mix_weight_down_block_inject.weight")]) == 2 * NUM_LAYERS
+ assert len([n for n in names if ".ple." in n]) == 9
+ assert {"model.embed_tokens.weight", "lm_head.weight",
+ "model.hyper_connection_mixer.input_mix_weight_down.weight"} <= set(names)
+
+
[email protected](scope="module")
+def model_state_dict_keys() -> set[str]:
+ """Keys ``Qwen4ExpForCausalLM`` declares -- the authoritative target the loader must fill."""
+ from freetoken.layers import rotary
+ from freetoken.models.qwen4_exp.model import Qwen4ExpForCausalLM
+
+ config = parse_config(cached_load_hf_config(MODEL_PATH))
+ saved = rotary._ROPE_DEVICE
+ rotary.set_rope_device(torch.device("cpu")) # get_rope refuses to build on meta
+ rotary.get_rope.cache_clear()
+ try:
+ with torch.device("meta"):
+ return set(Qwen4ExpForCausalLM(config).state_dict())
+ finally:
+ rotary.set_rope_device(saved)
+ rotary.get_rope.cache_clear()
+
+
+def test_emitted_names_are_the_model_state_dict(dense_pass, model_state_dict_keys):
+ names, _sampled = dense_pass
+ # The routed NVFP4 experts come from the offload source banks, never from the dense pass.
+ expected = {k for k in model_state_dict_keys
+ if not k.endswith((".mlp.experts.gate_up_proj", ".mlp.experts.down_proj"))}
+ assert set(names) == expected
+
+
+def test_every_zero_centered_norm_is_present_and_raw(dense_pass, reader):
+ names, sampled = dense_pass
+ zero_centered = [n for n in names if n.endswith(_ZERO_CENTERED_NORM_SUFFIXES)]
+ # 2 HC per layer + the top-level mixer + 3 PLE norms + q/k_norm and indexer q/k per QSA layer
+ assert len(zero_centered) == 2 * NUM_LAYERS + 1 + 3 + 4 * 12
+ for name in zero_centered:
+ assert torch.equal(sampled[name], reader.get(name.replace("model.", f"{LM}.", 1))), name
+
+
+def test_no_mtp_visual_expert_or_table_tensor_is_loaded(dense_pass):
+ names, _sampled = dense_pass
+ for name in names:
+ assert not name.startswith(("mtp.", "model.visual.", "visual."))
+ assert ".mlp.experts." not in name
+ assert "ngram_embedding" not in name
+ assert not name.endswith((".weight_scale", ".weight_scale_2", ".input_scale"))
+
+
[email protected]("name, raw_names, mode", SAMPLES, ids=[s[0] for s in SAMPLES])
+def test_sampled_tensor_matches_the_checkpoint(dense_pass, reader, name, raw_names, mode):
+ _names, sampled = dense_pass
+ parts = [reader.get(raw) for raw in raw_names]
+ got = sampled[name]
+ if mode == "same":
+ assert torch.equal(got, parts[0])
+ assert got.dtype is parts[0].dtype
+ else:
+ rows = sum(p.shape[0] for p in parts)
+ assert torch.equal(got[:rows], torch.cat(parts, dim=0))
+ if mode == "cat16":
+ assert got.shape[0] == rows + (-rows) % 16
+ assert not got[rows:].any()
+ else:
+ assert got.shape[0] == rows
+
+
[email protected](not torch.cuda.is_available(), reason="pinning needs CUDA")
+def test_ple_table_loads_pinned_and_matches_the_checkpoint(reader):
+ args = parse_config(cached_load_hf_config(MODEL_PATH)).qwen4_args
+ table = load_ple_table(MODEL_PATH, args)
+ assert table.bank.residency is HostResidency.PINNED
+ assert table.bank.nbytes == PLE_BYTES
+ assert abs(PLE_BYTES / 2**30 - 47.68) < 0.05
+ assert table.tensor.shape == (PLE_SHARDS * PLE_ROWS_PER_SHARD, PLE_DIM)
+ assert table.tensor.dtype is torch.float8_e4m3fn
+
+ prefix = f"{LM}.layers.{PLE_LAYER}.ple.ple_embedding.ngram_embedding"
+ assert torch.equal(table.weight_scale.reshape(1),
+ reader.get(f"{prefix}.weight_scale").reshape(1))
+ rows = random.Random(0).sample(range(PLE_SHARDS * PLE_ROWS_PER_SHARD), 1000)
+ by_shard: dict[int, list[int]] = {}
+ for row in rows:
+ by_shard.setdefault(row // PLE_ROWS_PER_SHARD, []).append(row)
+ got = table.tensor.view(torch.uint8)
+ for shard, shard_rows in by_shard.items():
+ ref = reader.get(f"{prefix}.shard_{shard}.weight").view(torch.uint8)
+ local = torch.tensor([r - shard * PLE_ROWS_PER_SHARD for r in shard_rows])
+ assert torch.equal(got[torch.tensor(shard_rows)], ref[local])
+
+ # The two pinned host allocations the engine must budget for.
+ total = PLE_BYTES + NUM_LAYERS * EXPERT_LAYER_BYTES
+ assert abs(total / 2**30 - 111.14) < 0.05
+
+
[email protected](scope="module")
+def layer0_expert_banks():
+ """The real NVFP4 source-bank loader, restricted to layer 0 (1.32 GiB instead of 63.5)."""
+ if not torch.cuda.is_available():
+ pytest.skip("expert bank pinning needs CUDA")
+ spec = dataclasses.replace(
+ _NVFP4_SOURCE_SPEC, layer_to_bank=lambda layer, config: 0 if layer == 0 else None
+ )
+ config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,
+ num_moe_layers=1)
+ return load_nvfp4_expert_source_banks(
+ MODEL_PATH, config, spec, drop_page_cache=lambda path: None, primary=False
+ )
+
+
+def test_sampled_experts_match_the_checkpoint(layer0_expert_banks, reader):
+ banks = layer0_expert_banks
+ for expert in random.Random(1).sample(range(E), 8):
+ base = f"{LM}.layers.0.mlp.experts.{expert}"
+ assert torch.equal(banks["gate_up_packed"][0][expert, :I],
+ reader.get(f"{base}.gate_proj.weight"))
+ assert torch.equal(banks["gate_up_packed"][0][expert, I:],
+ reader.get(f"{base}.up_proj.weight"))
+ assert torch.equal(banks["down_packed"][0][expert],
+ reader.get(f"{base}.down_proj.weight"))
+ for proj, bank, rows in (("gate_proj", "gate_up_scale", slice(0, I)),
+ ("up_proj", "gate_up_scale", slice(I, 2 * I)),
+ ("down_proj", "down_scale", slice(None))):
+ scale = reader.get(f"{base}.{proj}.weight_scale")
+ assert torch.equal(banks[bank][0][expert][rows].reshape(-1).view(torch.uint8),
+ scale.reshape(-1).view(torch.uint8))
+ gate_g = reader.get(f"{base}.gate_proj.weight_scale_2").to(torch.float16)
+ assert torch.equal(banks["gate_up_global"][0][expert, :I], gate_g.reshape(1).expand(I))
+
+
+def test_expert_bank_bytes_match_the_aot_row_table(layer0_expert_banks):
+ measured = sum(t[0].numel() * t[0].element_size() for t in layer0_expert_banks.values())
+ assert measured == EXPERT_LAYER_BYTES
+ assert abs(NUM_LAYERS * EXPERT_LAYER_BYTES / 2**30 - 63.46) < 0.05
+
+
[email protected](not torch.cuda.is_available(), reason="dummy banks are pinned")
+def test_dummy_expert_sources_have_the_real_bank_shapes(layer0_expert_banks):
+ from freetoken.models.weight import _model_override, dummy_nvfp4_expert_sources
+ from freetoken.models.register import get_model_spec
+
+ spec = get_model_spec("Qwen4ExpForConditionalGeneration")
+ # No dummy_* override, so --use-dummy-weight goes through the generic builders.
+ for hook in ("dummy_nvfp4_expert_sources", "dummy_moe_expert_sources", "dummy_q4_0_expert_sources"):
+ assert _model_override(spec, hook) is None
+
+ config = SimpleNamespace(num_experts=E, hidden_size=H, moe_intermediate_size=I,
+ num_moe_layers=1)
+ dummy = dummy_nvfp4_expert_sources(config)
+ assert set(dummy) == set(layer0_expert_banks)
+ for name, banks in dummy.items():
+ assert banks[0].shape == layer0_expert_banks[name][0].shape
+ assert banks[0].dtype is layer0_expert_banks[name][0].dtype
diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.py
index 787661dc..9cd8c7fb 100644
--- a/tests/models/test_minimax_m3.py
+++ b/tests/models/test_minimax_m3.py
@@ -234,7 +234,7 @@ def test_auto_backend_resolution():
cfg = parse_config(_hf_config())
required = _required_attn_types(cfg)
assert required == frozenset({AttnType.BSA})
- assert _resolve_auto_attention_backend(required, False) == "m3_sparse"
+ assert _resolve_auto_attention_backend(required) == "m3_sparse"
assert attention_backend_info("m3_sparse").page_sizes == (128,)
diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.py
index 4a4457e9..cbd579e4 100644
--- a/tests/models/test_muse_glimmer.py
+++ b/tests/models/test_muse_glimmer.py
@@ -153,7 +153,7 @@ def test_pool_family_and_backend_resolution():
assert required == frozenset({AttnType.FULL, AttnType.SWA})
# SWA restricts serving to the triton backend (the only one in the capability
# matrix that consumes per-call sliding windows), same as gemma4.
- assert _resolve_auto_attention_backend(required, False) == "triton"
+ assert _resolve_auto_attention_backend(required) == "triton"
def test_registry_resolves_architecture():
diff --git a/tests/moe/test_fused_moe.py b/tests/moe/test_fused_moe.py
index 1fd0f2e5..41a75c20 100644
--- a/tests/moe/test_fused_moe.py
+++ b/tests/moe/test_fused_moe.py
@@ -274,3 +274,98 @@ def test_fused_experts_decode_activation_and_router_weight_modes(
torch.cuda.synchronize()
torch.testing.assert_close(output, expected, rtol=5e-2, atol=5e-2)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+def test_fused_topk_non_power_of_2_k_routes_vendored_router():
+ """triton_kernels.topk builds tl.arange(0, k) (power-of-2 only); k=10 must not reach it."""
+ from freetoken.moe.fused import _torch_fused_topk, fused_topk
+
+ gating = torch.randn(5, 64, device="cuda")
+ hidden = torch.randn(5, 8, device="cuda")
+ weights, ids = fused_topk(hidden, gating, 10, renormalize=True)
+ ref_w, ref_i = _torch_fused_topk(gating, 10, True, None)
+ assert torch.equal(ids, ref_i)
+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)
+
+
+# The vendored triton router behind that k=10 branch; fp32 logits keep the reference top-k tie-free.
+# Ties get their own case below, because torch.topk does not break them by expert id.
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
[email protected]("renormalize", [True, False])
+ "num_tokens,num_experts,topk",
+ [(1, 512, 10), (7, 512, 10), (129, 512, 10), (33, 512, 6), (4, 64, 3)],
+)
+def test_fused_topk_softmax_matches_torch_reference(num_tokens, num_experts, topk, renormalize):
+ from freetoken.kernel.triton.moe_router import fused_topk_softmax
+ from freetoken.moe.fused import _torch_fused_topk
+
+ gen = torch.Generator(device="cuda").manual_seed(num_tokens * 31 + topk)
+ gating = torch.randn(num_tokens, num_experts, generator=gen, device="cuda")
+
+ weights, ids = fused_topk_softmax(gating, topk, renormalize)
+ ref_w, ref_i = _torch_fused_topk(gating, topk, renormalize, None)
+
+ assert weights.dtype == torch.float32 and ids.dtype == torch.int32
+ assert torch.equal(ids, ref_i)
+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+def test_fused_topk_softmax_ties_pick_the_lowest_expert_id():
+ from freetoken.kernel.triton.moe_router import fused_topk_softmax
+
+ gating = torch.full((2, 8), -10.0, device="cuda")
+ gating[0, [6, 2, 5]] = 1.0 # three-way tie for two slots
+ gating[1] = 0.0 # whole row tied
+
+ weights, ids = fused_topk_softmax(gating, 3, renormalize=True)
+
+ assert ids[0].tolist() == [2, 5, 6]
+ assert ids[1].tolist() == [0, 1, 2]
+ torch.testing.assert_close(weights, torch.full((2, 3), 1.0 / 3.0, device="cuda"))
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
[email protected]("limit_dtype", [torch.int32, torch.int64])
+def test_fused_topk_softmax_masks_padded_rows(limit_dtype):
+ from freetoken.kernel.triton.moe_router import fused_topk_softmax
+ from freetoken.moe.fused import _torch_fused_topk
+
+ gen = torch.Generator(device="cuda").manual_seed(5)
+ gating = torch.randn(16, 512, generator=gen, device="cuda")
+ limit = torch.tensor(5, dtype=limit_dtype, device="cuda")
+
+ weights, ids = fused_topk_softmax(gating, 10, True, limit)
+ ref_w, ref_i = _torch_fused_topk(gating, 10, True, limit)
+
+ assert (ids[5:] == -1).all()
+ assert torch.equal(ids, ref_i)
+ torch.testing.assert_close(weights, ref_w, rtol=1e-5, atol=1e-6)
+
+
[email protected](not torch.cuda.is_available(), reason="CUDA is required")
+def test_fused_topk_softmax_is_cuda_graph_capturable():
+ """The padded-row limit must come off the device tensor, not a host read baked into the graph."""
+ from freetoken.kernel.triton.moe_router import fused_topk_softmax
+
+ gen = torch.Generator(device="cuda").manual_seed(11)
+ gating = torch.randn(8, 512, generator=gen, device="cuda")
+ limit = torch.tensor(8, dtype=torch.int32, device="cuda")
+
+ stream = torch.cuda.Stream()
+ stream.wait_stream(torch.cuda.current_stream())
+ with torch.cuda.stream(stream):
+ fused_topk_softmax(gating, 10, True, limit)
+ torch.cuda.current_stream().wait_stream(stream)
+
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ _, ids = fused_topk_softmax(gating, 10, True, limit)
+
+ limit.fill_(3)
+ graph.replay()
+ torch.cuda.synchronize()
+ assert (ids[:3] != -1).all()
+ assert (ids[3:] == -1).all()
diff --git a/tests/scheduler/test_abort_inflight_prefill.py b/tests/scheduler/test_abort_inflight_prefill.py
index 1c9a9fba..a50d0d32 100644
--- a/tests/scheduler/test_abort_inflight_prefill.py
+++ b/tests/scheduler/test_abort_inflight_prefill.py
@@ -37,7 +37,7 @@ UID = 2
def _pool(num_slots=16):
g = LinearGatedDeltaGroupConfig(
name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4,
- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,
+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",
)
return LinearStatePool(group=g, num_slots=num_slots, dtype=torch.bfloat16,
device=torch.device("cpu"), tp_size=1)
diff --git a/tests/scheduler/test_hybrid_cache_manager.py b/tests/scheduler/test_hybrid_cache_manager.py
index 19be56f3..080999ad 100644
--- a/tests/scheduler/test_hybrid_cache_manager.py
+++ b/tests/scheduler/test_hybrid_cache_manager.py
@@ -16,7 +16,7 @@ from freetoken.scheduler.cache import CacheManager
def _pool(num_slots=16):
g = LinearGatedDeltaGroupConfig(
name="linear", layer_ids=(0,), num_key_heads=2, num_value_heads=4,
- key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate=True,
+ key_head_dim=16, value_head_dim=16, conv_kernel_dim=4, output_gate="silu",
)
return LinearStatePool(group=g, num_slots=num_slots, dtype=torch.bfloat16,
device=torch.device("cpu"), tp_size=1)
@@ -116,6 +116,43 @@ def test_rebuild_reclaims_donated_gdn_slots():
assert pool.num_free_slots == pool.num_slots - 1 # all GDN slots reclaimed (no leak)
+def test_prefill_chunk_ends_on_a_page_boundary():
+ """A hybrid chunk must end page-aligned: the snapshot commit skips any other boundary."""
+ from freetoken.scheduler.prefill import ChunkedReq, PrefillAdder
+ from freetoken.scheduler.table import TableManager
+ from freetoken.scheduler.utils import PendingReq
+
+ pool = _pool()
+ pt = torch.zeros(4, 512, dtype=torch.int32)
+ cm = CacheManager(64, 64, pt, "hybrid_radix", linear_state_pool=pool)
+ assert cm.prefill_chunk_align == 64
+ tm = TableManager(max_running_reqs=4, page_table=pt)
+ pending = PendingReq(0, torch.arange(300, dtype=torch.int32), SamplingParams(max_tokens=1))
+
+ adder = PrefillAdder(token_budget=100, reserved_size=0, cache_manager=cm, table_manager=tm)
+ req = adder.try_add_one(pending)
+ assert isinstance(req, ChunkedReq) and req.extend_len == 64
+
+ # a budget below one page keeps the unaligned chunk rather than stalling the request
+ adder = PrefillAdder(token_budget=40, reserved_size=0, cache_manager=cm, table_manager=tm)
+ assert adder.try_add_one(pending).extend_len == 40
+
+
+def test_naive_cache_does_not_align_prefill_chunks():
+ """The alignment hook is hybrid-only; every other cache keeps the raw budget chunk."""
+ from freetoken.scheduler.prefill import PrefillAdder
+ from freetoken.scheduler.table import TableManager
+ from freetoken.scheduler.utils import PendingReq
+
+ pt = torch.zeros(4, 512, dtype=torch.int32)
+ cm = CacheManager(64, 64, pt, "radix")
+ assert cm.prefill_chunk_align == 1
+ tm = TableManager(max_running_reqs=4, page_table=pt)
+ adder = PrefillAdder(token_budget=100, reserved_size=0, cache_manager=cm, table_manager=tm)
+ pending = PendingReq(0, torch.arange(300, dtype=torch.int32), SamplingParams(max_tokens=1))
+ assert adder.try_add_one(pending).extend_len == 100
+
+
def test_pool_sizing_covers_4mr_floor():
"""C6: pool must reserve the 4-slot-per-request non-evictable floor even at a tiny ratio."""
from types import SimpleNamespace