Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

Support minimax m3 (#1)

Shuo Yang9dc3ff4ca524
diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.py
index 9339b9bd..746c04c4 100644
--- a/python/freetoken/attention/__init__.py
+++ b/python/freetoken/attention/__init__.py
@@ -116,6 +116,22 @@ def create_dsa_backend(config: ModelConfig):
return DSAAttnBackend(config)
+@SUPPORTED_ATTENTION_BACKENDS.register(
+ "m3_sparse",
+ BackendInfo(
+ supported_types=frozenset({AttnType.BSA}),
+ # One KV page == one 128-token sparse block: the top-k block ids ARE page
+ # indices and the block-base-row addressing needs page-aligned 128-row runs.
+ # Config-time resolution coerces any other page size here.
+ page_sizes=(128,),
+ ),
+)
+def create_m3_sparse_backend(config: ModelConfig):
+ from .m3_sparse import M3SparseAttnBackend
+
+ return M3SparseAttnBackend(config)
+
+
def attention_backend_info(name: str) -> BackendInfo:
return SUPPORTED_ATTENTION_BACKENDS.info(name)
diff --git a/python/freetoken/attention/base.py b/python/freetoken/attention/base.py
index 5f1be8fb..eb39d472 100644
--- a/python/freetoken/attention/base.py
+++ b/python/freetoken/attention/base.py
@@ -21,6 +21,9 @@ class AttnType(str, Enum):
DSA = "dsa" # latent-KV MLA + DSA sparse indexer -> DSAKVCache
DSV4 = "dsv4" # DSV4 window+compressed sparse -> DSV4PagedKVCache
LINEAR = "linear" # GDN/mamba state layers -> LinearStatePool
+ # GQA block-sparse (MiniMax-M3): paged GQA K/V + a per-sparse-layer index-key
+ # slab; the indexer picks top-k 128-token blocks per query -> BSAKVCache
+ BSA = "bsa"
@property
def backend_driven(self) -> bool:
diff --git a/python/freetoken/attention/fi.py b/python/freetoken/attention/fi.py
index 7f4f46cf..c9e58538 100644
--- a/python/freetoken/attention/fi.py
+++ b/python/freetoken/attention/fi.py
@@ -87,12 +87,32 @@ class FlashInferBackend(BaseAttnBackend):
self.config = config
self.kvcache = get_global_ctx().kv_cache
self.device = self.kvcache.device
- # 256 MiB: fa2 split-KV prefill needs num_qo_heads * padded_batch_size *
- # cta_tile_q * head_dim * 4 bytes of tmp_v scratch. For head_dim=256 models
- # (e.g. Qwen3.5/3.6 MoE), a short extend-prefill over a long cached prefix
- # splits KV into many chunks and overflows the previous 128 MiB buffer.
+ # fa2 split-KV prefill needs ``tmp_v <= qo_heads_local * padded_batch_size *
+ # cta_tile_q * head_dim * 4`` bytes of scratch, where flashinfer's scheduler
+ # caps ``padded_batch_size`` at ~``2 * SM / kv_heads_local`` and
+ # ``cta_tile_q`` is 128 (64 at head_dim >= 256); when the cap can't be met
+ # it disables split-KV and allocates no tmp_v at all. The original flat
+ # 128 MiB overflowed on head_dim=256 extend-prefills (Qwen3.5/3.6 MoE) and
+ # the flat 256 MiB on MiniMax-M3's 64-head dense layers (H100, 132 SMs:
+ # 64 heads x ceil(2*132/4)=66 padded rows x 128 x 128 x 4 B = 264 MiB of
+ # tmp_v, over the flat buffer). Derive the bound
+ # from the model's TP-LOCAL geometry + this device's SM count, with slack
+ # for the tmp_s/merge siblings, floored at the old 256 MiB -- geometries
+ # that never exceeded the flat buffer (e.g. GLM-4.7's 96q/8kv) stay at it.
+ tp_size = get_tp_info().size
+ qo_local = div_even(config.num_qo_heads, tp_size)
+ kv_local = div_even(config.num_kv_heads, tp_size, allow_replicate=True)
+ sm_count = (
+ torch.cuda.get_device_properties(self.device).multi_processor_count
+ if self.device.type == "cuda"
+ else 128
+ )
+ cta_tile_q = 64 if config.head_dim >= 256 else 128
+ padded_batch = -(-2 * sm_count // max(1, kv_local))
+ tmp_v_bound = qo_local * padded_batch * cta_tile_q * config.head_dim * 4
+ workspace_bytes = max(256 * 1024 * 1024, tmp_v_bound + 32 * 1024 * 1024)
self.float_workspace_buffer = torch.empty(
- 256 * 1024 * 1024, dtype=torch.uint8, device=self.device
+ workspace_bytes, dtype=torch.uint8, device=self.device
)
self.prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(
self.float_workspace_buffer,
diff --git a/python/freetoken/attention/m3_sparse.py b/python/freetoken/attention/m3_sparse.py
new file mode 100644
index 00000000..19515f90
--- /dev/null
+++ b/python/freetoken/attention/m3_sparse.py
@@ -0,0 +1,443 @@
+"""MiniMax-M3 block-sparse GQA attention backend.
+
+Serves ``AttnType.BSA`` (one attention group over all layers: paged GQA K/V plus the
+sparse layers' index-key slab, ``kvcache/bsa_pool.py``). Two entry points:
+
+* dense leading layers call the generic :meth:`forward` -- delegated to a wrapped
+ FULL backend (fa/fi/triton, picked by availability; page-size agnostic, since every
+ generic backend addresses the pool by per-token rows regardless of the allocator's
+ page size). The wrapped backend owns its own metadata; this backend's metadata
+ carries it and the delegation swaps ``batch.attn_metadata`` around each call.
+* sparse layers call :meth:`bsa_forward` with this token's (index_q, index_k): the
+ backend scatters K/V and the index keys into the pool, scores 128-token blocks per
+ index head (== per KV head), selects the top-``topk_blocks`` (+ forced init/local)
+ blocks per query, and attends only those (``kernel/triton/minimax_m3_sparse.py``).
+
+Addressing: the engine pins ``page_size == 128`` (this backend's ``page_sizes``
+declaration), so one KV page IS one sparse block and the page table's every-128th
+column is the block's base row -- ``block_rows[req, blk] = page_table[t, blk*128]``.
+
+* **decode** stages the per-request block-base rows + live lengths into static
+ buffers (``prepare_for_replay``, dsa/dsv4 precedent) and runs score -> top-k ->
+ attend entirely on device-read lengths with shape-fixed grids, so the whole path
+ lives inside the captured CUDA graph.
+* **prefill/extend** is eager: per request, in query chunks bounded by the fp32
+ score transient's budget (the [chunk, H, n_blocks] block-score tile), score +
+ causal top-k + attend. Short contexts are exact dense attention by construction
+ (top-k covers every visible block when ``kv_len <= topk_blocks * 128``).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import TYPE_CHECKING, List
+
+import torch
+from freetoken.core import Batch, get_global_ctx
+from freetoken.utils import init_logger
+
+from .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadata
+
+logger = init_logger(__name__)
+
+if TYPE_CHECKING:
+ from freetoken.models import ModelConfig
+
+_CPU_PINNED = {"device": "cpu", "dtype": torch.int32, "pin_memory": True}
+# Prefill scoring transient budget: the fp32 block-score tile is
+# [num_index_heads, chunk, n_blocks], so the query chunk shrinks as the context
+# grows. Worst case is bounded by the model's max position (floor 16 queries x
+# 4 heads x 8192 blocks x 4 B = 2 MB per iteration), not open-ended.
+_PREFILL_SCORE_BYTES = 256 << 20
+_PREFILL_SCORE_CHUNK = 4096
+
+
+def _pick_inner_backend(block_size: int) -> str:
+ """FULL backend for the dense leading layers, resolved through the engine's
+ own FULL auto-tree (one resolver, so the two trees cannot diverge), then
+ filtered for page-size compatibility: this pool serves 128-token pages, so a
+ candidate that pins other page sizes (trtllm: 16/32/64) is skipped.
+
+ ``FREETOKEN_M3_INNER_BACKEND`` overrides (e.g. "triton" on a box with a
+ broken flashinfer JIT toolchain); the override is validated so a
+ page-incompatible or non-FULL pick fails at construction with a real
+ message instead of mis-addressing the pool.
+ """
+ import os
+
+ from freetoken.attention import attention_backend_info
+ from freetoken.attention.base import AttnType
+
+ def _page_ok(name: str) -> bool:
+ for part in name.split(","):
+ sizes = attention_backend_info(part).page_sizes
+ if sizes is not None and block_size not in sizes:
+ return False
+ return True
+
+ override = os.getenv("FREETOKEN_M3_INNER_BACKEND")
+ if override:
+ for part in override.split(","):
+ try:
+ info = attention_backend_info(part)
+ except KeyError:
+ raise ValueError(
+ f"FREETOKEN_M3_INNER_BACKEND={override!r}: unknown attention "
+ f"backend {part!r}"
+ ) from None
+ if AttnType.FULL not in info.supported_types:
+ raise ValueError(
+ f"FREETOKEN_M3_INNER_BACKEND={override!r}: {part!r} does not "
+ "serve FULL attention (the dense leading layers)."
+ )
+ if not _page_ok(override):
+ raise ValueError(
+ f"FREETOKEN_M3_INNER_BACKEND={override!r} cannot address the "
+ f"{block_size}-token pages this model's pool uses."
+ )
+ logger.info(f"m3_sparse dense-layer backend: {override} (env override)")
+ return override
+
+ from freetoken.engine.engine import _resolve_auto_attention_backend
+
+ name = _resolve_auto_attention_backend(frozenset({AttnType.FULL}), False)
+ if not _page_ok(name):
+ # trtllm (the sm_100 first pick) pins 16/32/64-token pages; walk the rest
+ # of the SAME tree (same arch gates, same requirement probes) with the
+ # page filter applied.
+ from freetoken.engine.engine import _backend_requirements_met
+ from freetoken.utils.arch import is_sm90_family
+
+ skipped = name
+ for candidate, arch_ok in (("fa,fi", is_sm90_family()), ("fi", True), ("triton", True)):
+ if arch_ok and _page_ok(candidate) and _backend_requirements_met(candidate):
+ name = candidate
+ break
+ else: # pragma: no cover - triton is unconditional
+ name = "triton"
+ logger.info(
+ f"m3_sparse dense-layer backend: {name} (auto; skipped {skipped!r}: "
+ f"it cannot address this pool's {block_size}-token pages)"
+ )
+ else:
+ logger.info(f"m3_sparse dense-layer backend: {name} (auto)")
+ return name
+
+
+@dataclass
+class M3SparseMetadata(BaseAttnMetadata):
+ # fmt: off
+ is_decode: bool
+ last_indices: torch.Tensor # gpu
+ qo_indptr_cpu: torch.Tensor # cpu pinned int32 [bs+1] (prefill request slicing)
+ kv_len_cpu: torch.Tensor # cpu pinned int32 [bs]
+ inner: BaseAttnMetadata # the wrapped FULL backend's metadata
+ # decode addressing: per-request block-base rows (page_table[:, ::128] snapshot)
+ # + live lengths, device-read. None on prefill / until staged.
+ block_rows: torch.Tensor | None = None
+ kvlen: torch.Tensor | None = None
+ # prefill addressing plan, built ONCE at the first sparse layer's forward and
+ # reused by all 57 (the tensors are layer-invariant): per request, its
+ # contiguous block-base rows plus per-query-chunk (slice, cu, seq, prefix,
+ # chunk_len) tuples. None until built / on decode.
+ prefill_plan: list | None = None
+ # fmt: on
+
+ def get_last_indices(self, bs: int) -> torch.Tensor:
+ return self.last_indices[:bs]
+
+
+class M3SparseAttnBackend(BaseAttnBackend):
+ def __init__(self, config: ModelConfig) -> None:
+ from freetoken.kvcache.bsa_pool import BSAKVCache
+
+ args = config.m3_args
+ assert args is not None and args.use_sparse, (
+ "m3_sparse backend needs ModelConfig.m3_args with sparse attention on "
+ "(the dense ablation resolves to AttnType.FULL and a generic backend)"
+ )
+ self.config = config
+ self.args = args
+ self.num_heads = config.num_qo_heads
+ self.num_kv_heads = config.num_kv_heads
+ self.head_dim = config.head_dim
+ self.kv_attn_dim = self.num_kv_heads * self.head_dim
+ self.block_size = args.block_size
+ self.topk_blocks = args.topk_blocks
+ self.sm_scale = config.attn_sm_scale or (config.head_dim**-0.5)
+ self.kvcache = get_global_ctx().kv_cache
+ self.device = self.kvcache.device
+ # The serving switch is the POOL TYPE (parse_config resolved it into the
+ # attention-group spec): this backend requires the index-key slab.
+ assert isinstance(self.kvcache, BSAKVCache), (
+ f"m3_sparse backend needs a BSA pool, got {type(self.kvcache).__name__}"
+ )
+ # layer -> index-slab slot (sparse-layer order; matches the pool's slots).
+ self._idx_slot = {lid: args.sparse_slot(lid) for lid in args.sparse_layer_ids}
+
+ # Wrapped FULL backend for the dense leading layers.
+ from freetoken.attention import create_attention_backend
+
+ self._inner_name = _pick_inner_backend(self.block_size)
+ self.inner = create_attention_backend(self._inner_name, config)
+
+ # decode staging (static buffers under CUDA graphs; eager decode builds
+ # per-forward tensors lazily instead)
+ self._block_rows_buf: torch.Tensor | None = None
+ self._kvlen_buf: torch.Tensor | None = None
+ self.capture_bs: List[int] = []
+
+ # ----- slab views -------------------------------------------------------------------
+ def _k_rows(self, layer_id: int) -> torch.Tensor:
+ cache = self.kvcache.k_cache(layer_id) # [pages, page_size, kvh, d]
+ return cache.view(-1, cache.shape[2], cache.shape[3])
+
+ def _v_rows(self, layer_id: int) -> torch.Tensor:
+ cache = self.kvcache.v_cache(layer_id)
+ return cache.view(-1, cache.shape[2], cache.shape[3])
+
+ # ----- metadata ---------------------------------------------------------------------
+ def prepare_metadata(self, batch: Batch) -> None:
+ # The wrapped backend builds ITS metadata first (it owns the dense layers'
+ # addressing); ours wraps it and the per-call delegation swaps them.
+ self.inner.prepare_metadata(batch)
+ inner_md = batch.attn_metadata
+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
+ seqlens_q = [r.extend_len for r in reqs]
+ seqlens_k = [r.device_len for r in reqs]
+ is_decode = getattr(batch, "phase", None) == "decode"
+ qo_indptr = torch.tensor([0] + seqlens_q, **_CPU_PINNED).cumsum_(0).to(torch.int32)
+ kv_len = torch.tensor(seqlens_k, **_CPU_PINNED)
+ last = (qo_indptr[1:].to(torch.int32) - 1).to(self.device, non_blocking=True)
+ batch.attn_metadata = M3SparseMetadata(
+ is_decode=is_decode,
+ last_indices=last,
+ qo_indptr_cpu=qo_indptr,
+ kv_len_cpu=kv_len,
+ inner=inner_md,
+ )
+ # Decode addressing (block_rows/kvlen) is DEFERRED: a graph-bound step stages
+ # it into the static buffers (prepare_for_replay) and an eager step snapshots
+ # lazily at the first sparse layer's bsa_forward.
+
+ # ----- dense layers (delegated) -----------------------------------------------------
+ def forward(
+ self,
+ q: torch.Tensor,
+ k: torch.Tensor,
+ v: torch.Tensor,
+ layer_id: int,
+ batch: Batch,
+ attn_spec: AttentionSpec | None = None,
+ ) -> torch.Tensor:
+ md = batch.attn_metadata
+ assert isinstance(md, M3SparseMetadata)
+ batch.attn_metadata = md.inner
+ try:
+ # The wrapped backend stores K/V itself (its forward contract).
+ return self.inner.forward(q, k, v, layer_id, batch, attn_spec=attn_spec)
+ finally:
+ batch.attn_metadata = md
+
+ # ----- sparse layers ----------------------------------------------------------------
+ def bsa_forward(
+ self,
+ q: torch.Tensor, # [T, HQ, D]
+ k: torch.Tensor, # [T, KVH * D]
+ v: torch.Tensor, # [T, KVH * D]
+ index_q: torch.Tensor, # [T, Hi, Di]
+ index_k: torch.Tensor, # [T, Di]
+ layer_id: int,
+ batch: Batch,
+ ) -> torch.Tensor:
+ md = batch.attn_metadata
+ assert isinstance(md, M3SparseMetadata)
+ slot = self._idx_slot[layer_id]
+ self.kvcache.store_kv(k, v, batch.out_loc, layer_id)
+ # Scatter index keys unconditionally: short prefills select every block
+ # TODAY, but their keys must exist once decode's history passes the top-k.
+ self.kvcache.store_index_k(index_k, batch.out_loc, slot)
+
+ if md.is_decode:
+ if md.block_rows is None:
+ # Eager decode (not graph-staged): per-request block-base SNAPSHOT
+ # (the live page-table row may mutate for the next batch while this
+ # one runs) + live lengths, once per step at the first sparse layer.
+ md.block_rows = self._decode_block_rows(batch)
+ md.kvlen = md.kv_len_cpu.to(self.device, non_blocking=True)
+ return self._decode(md, layer_id, slot, q, index_q)
+ return self._prefill(md, layer_id, slot, q, index_q, batch)
+
+ # ----- decode (CUDA-graph capturable, single code path) ------------------------------
+ def _decode(self, md, layer_id: int, slot: int, q, index_q) -> torch.Tensor:
+ from freetoken.kernel.triton.minimax_m3_sparse import (
+ m3_index_decode,
+ m3_sparse_attn_decode,
+ )
+
+ block_rows, kvlen = md.block_rows, md.kvlen
+ topk_idx = m3_index_decode(
+ index_q,
+ self.kvcache.index_k_cache(slot),
+ block_rows,
+ kvlen,
+ block_rows.shape[1],
+ self.topk_blocks,
+ self.args.init_blocks,
+ self.args.local_blocks,
+ )
+ out = torch.empty_like(q)
+ m3_sparse_attn_decode(
+ q, self._k_rows(layer_id), self._v_rows(layer_id),
+ topk_idx, block_rows, kvlen, self.sm_scale, out,
+ )
+ return out
+
+ # ----- prefill / extend (eager) -------------------------------------------------------
+ def _prefill_plan(self, md, batch: Batch) -> list:
+ """Per-request addressing plan, built ONCE per forward (layer-invariant --
+ rebuilding it per sparse layer cost ~3 H2D copies x chunks x 57 layers of
+ stream-serializing overhead): ``[(block_rows [1, n_blocks], kv_len,
+ [(slice, cu, seq, prefix, chunk_len), ...]), ...]``."""
+ if md.prefill_plan is not None:
+ return md.prefill_plan
+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs
+ page_table = get_global_ctx().page_table
+ qo = md.qo_indptr_cpu.tolist()
+ plan = []
+ for i, r in enumerate(reqs):
+ m = qo[i + 1] - qo[i]
+ if m == 0:
+ continue
+ kv_len = r.device_len
+ n_blocks = -(-kv_len // self.block_size)
+ # Contiguous per-request block-base rows (the [::block_size] view is
+ # strided; the kernels read unit-stride rows). Tiny: n_blocks ints.
+ block_rows = (
+ page_table[r.table_idx, : kv_len : self.block_size]
+ .to(torch.int32)
+ .contiguous()
+ .view(1, -1)
+ )
+ # Bound the fp32 [H, chunk, n_blocks] score transient (worst case capped
+ # by max_position: 16 x 4 x (1M/128) x 4 B = 2 MB per chunk iteration).
+ per_q = 4 * self.args.num_index_heads * max(n_blocks, 1)
+ chunk = max(16, min(_PREFILL_SCORE_CHUNK, _PREFILL_SCORE_BYTES // per_q))
+ seq = torch.tensor([kv_len], dtype=torch.int32, device=self.device)
+ chunks = []
+ for s0 in range(0, m, chunk):
+ s1 = min(s0 + chunk, m)
+ chunks.append((
+ slice(qo[i] + s0, qo[i] + s1),
+ torch.tensor([0, s1 - s0], dtype=torch.int32, device=self.device),
+ seq,
+ torch.tensor([r.cached_len + s0], dtype=torch.int32, device=self.device),
+ s1 - s0,
+ ))
+ plan.append((block_rows, kv_len, chunks))
+ md.prefill_plan = plan
+ return plan
+
+ def _prefill(self, md, layer_id: int, slot: int, q, index_q, batch: Batch) -> torch.Tensor:
+ from freetoken.kernel.triton.minimax_m3_sparse import (
+ m3_index_score_prefill,
+ m3_index_topk_prefill,
+ m3_sparse_attn_prefill,
+ )
+
+ k_rows = self._k_rows(layer_id)
+ v_rows = self._v_rows(layer_id)
+ ik_rows = self.kvcache.index_k_cache(slot)
+ out = torch.empty_like(q)
+
+ for block_rows, kv_len, chunks in self._prefill_plan(md, batch):
+ for sl, cu, seq, prefix, chunk_len in chunks:
+ score = m3_index_score_prefill(
+ index_q[sl], ik_rows, block_rows, cu, seq, prefix,
+ chunk_len, kv_len,
+ )
+ topk_idx = m3_index_topk_prefill(
+ score, cu, prefix, chunk_len,
+ self.topk_blocks, self.args.init_blocks, self.args.local_blocks,
+ )
+ del score
+ m3_sparse_attn_prefill(
+ q[sl], k_rows, v_rows, topk_idx, block_rows,
+ cu, seq, prefix, chunk_len, self.sm_scale, out[sl],
+ )
+ return out
+
+ # ----- CUDA graph (decode) ------------------------------------------------------------
+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:
+ self.inner.init_capture_graph(max_seq_len, bs_list)
+ self.capture_bs = sorted(bs_list)
+ max_bs = max(bs_list)
+ width = get_global_ctx().page_table.shape[1]
+ n_blocks = -(-width // self.block_size)
+ self._block_rows_buf = torch.zeros(
+ (max_bs, n_blocks), dtype=torch.int32, device=self.device
+ )
+ self._kvlen_buf = torch.zeros(max_bs, dtype=torch.int32, device=self.device)
+
+ def _block_base_view(self) -> torch.Tensor:
+ """Every-``block_size``-th column of the page table: the per-block base
+ rows. A strided VIEW -- gathering rows through it materializes only the
+ [bs, W/128] result, never the full-width [bs, W] table snapshot (2 MiB/req
+ at a 512K-token table; the staging-width class this repo fixed for DSV4)."""
+ return get_global_ctx().page_table[:, :: self.block_size]
+
+ def _decode_block_rows(self, batch: Batch) -> torch.Tensor:
+ """This decode step's per-request block-base rows [bs, W/128], gathered off
+ the scheduler-staged ``active_table_idx`` (a device tensor -- no host loop)."""
+ assert batch.active_table_idx is not None, "decode batch is missing its page-table rows"
+ return (
+ self._block_base_view()
+ .index_select(0, batch.active_table_idx.to(torch.int64))
+ .to(torch.int32)
+ )
+
+ def _stage_decode(self, batch: Batch, bs: int, table_idx: torch.Tensor) -> None:
+ """Copy this step's addressing into the static graph buffers and point the
+ metadata at them (restage-per-replay, dsa precedent)."""
+ md = batch.attn_metadata
+ self._block_rows_buf[:bs].copy_(self._block_base_view().index_select(0, table_idx))
+ self._kvlen_buf[:bs].copy_(md.kv_len_cpu.to(self.device, non_blocking=True))
+ md.block_rows = self._block_rows_buf[:bs]
+ md.kvlen = self._kvlen_buf[:bs]
+
+ def prepare_for_capture(self, batch: Batch) -> None:
+ # The wrapped backend builds its per-bs graph machinery first (its
+ # prepare_for_capture rebuilds batch.attn_metadata as its own), then this
+ # backend re-wraps the metadata and stages the dummy request's rows for
+ # every slot -- replays overwrite with the live rows.
+ self.inner.prepare_for_capture(batch)
+ inner_md = batch.attn_metadata
+ self.prepare_metadata(batch)
+ md = batch.attn_metadata
+ assert isinstance(md, M3SparseMetadata)
+ md.inner = inner_md
+ bs = batch.size
+ dummy = torch.full(
+ (bs,), batch.padded_reqs[0].table_idx, dtype=torch.int64, device=self.device
+ )
+ self._stage_decode(batch, bs, dummy)
+
+ def prepare_for_replay(self, batch: Batch) -> None:
+ md = batch.attn_metadata
+ assert isinstance(md, M3SparseMetadata)
+ assert batch.active_table_idx is not None, "decode batch is missing its page-table rows"
+ batch.attn_metadata = md.inner
+ try:
+ self.inner.prepare_for_replay(batch)
+ finally:
+ batch.attn_metadata = md
+ self._stage_decode(batch, batch.padded_size, batch.active_table_idx.to(torch.int64))
+
+ def reset_capture(self) -> None:
+ super().reset_capture()
+ self.inner.reset_capture()
+ self._block_rows_buf = None
+ self._kvlen_buf = None
+
+
+__all__ = ["M3SparseAttnBackend", "M3SparseMetadata"]
diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.py
index 2f4505ac..ff3c985f 100644
--- a/python/freetoken/engine/engine.py
+++ b/python/freetoken/engine/engine.py
@@ -125,6 +125,8 @@ def _resolve_auto_attention_backend(
candidates.append(("dsv4_sparse", True))
if required & {AttnType.MLA, AttnType.DSA}:
candidates.append(("dsa", True))
+ if AttnType.BSA in required:
+ candidates.append(("m3_sparse", True))
if AttnType.SWA in required:
candidates.append(("triton", True))
if AttnType.FULL in required:
@@ -173,7 +175,7 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Att
if missing:
valid = [
name
- for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse")
+ for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse")
if required <= attention_backend_info(name).supported_types
]
missing_names = "/".join(sorted(t.value for t in missing))
@@ -253,6 +255,13 @@ def _make_dummy_weight_state_dict(
state_dict[key] = t
elif param.dtype.is_floating_point or param.dtype.is_complex:
state_dict[key] = torch.randn(param.shape, dtype=param.dtype, device=device)
+ elif param.dtype == torch.uint8 and key.endswith("weight_scale_inv"):
+ # MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse
+ # every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8
+ # buffers are packed payloads whose bytes mean something else entirely
+ # (GGUF qweight blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they
+ # keep the benign all-zeros fill below.
+ state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)
else:
state_dict[key] = torch.zeros(param.shape, dtype=param.dtype, device=device)
return state_dict
@@ -1154,6 +1163,26 @@ def _adjust_config(config: EngineConfig):
# lists, then validate whatever is now selected (explicit or auto) -- every
# comma part must serve every required type, with packages/arch available.
required_attn_types = _required_attn_types(model_config)
+ _dtype = getattr(config, "dtype", None) # duck-typed test configs omit it
+ if AttnType.BSA in required_attn_types and _dtype is not None and _dtype.itemsize != 2:
+ # Reject at config time: the BSA pool's own assert only fires after the
+ # model is resident (and not at all under `python -O`).
+ raise ValueError(
+ f"--dtype {config.dtype}: block-sparse attention serves 16-bit "
+ "compute only (the index slab budgets 2 bytes/token); use bfloat16 "
+ "or float16."
+ )
+ if _dtype == torch.float16 and "mxfp8" in (
+ getattr(model_config, "attn_quant", "none"),
+ getattr(model_config, "dense_quant", "none"),
+ ):
+ # The MXFP8 GEMV folds the pow2-descaled fp8 weight into the activation
+ # dtype; fp16's narrow exponent can overflow/flush what bf16 represents
+ # exactly, and the combination was never numerically validated.
+ raise ValueError(
+ "--dtype float16 with MXFP8 resident weights is unsupported (the "
+ "W8A16 fold is only validated exact in bfloat16); use bfloat16."
+ )
if config.attention_backend == "auto":
override(
"attention_backend",
@@ -1166,6 +1195,36 @@ def _adjust_config(config: EngineConfig):
total_experts = config.model_config.num_moe_layers * config.model_config.num_experts
override("moe_cache_size", math.ceil(total_experts * config.moe_cache_rate))
+ # The CPU MoE executor supports the silu/gelu family plus the clamped
+ # swigluoai (csrc ActKind; "gpt_oss_swiglu" rides inside the mxfp4 kernel and
+ # swigluoai the generic GEMV epilogue). A model with any other expert
+ # activation cannot decode on the CPU: reject an explicit cpu/hybrid pick at
+ # config time, and keep auto from upgrading offload -> hybrid off the profile.
+ _cpu_moe_acts = (
+ "silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",
+ )
+ # hidden_act (the dense activation) stands proxy for the expert activation --
+ # true for every in-tree model. mxfp4 experts pass regardless: their act runs
+ # inside the mxfp4 kernel, not the generic epilogue.
+ _cpu_moe_act_ok = getattr(model_config, "hidden_act", "silu") in _cpu_moe_acts or (
+ getattr(model_config, "moe_weight_format", None) == "mxfp4"
+ )
+ if (
+ is_moe
+ and not _cpu_moe_act_ok
+ and (config.moe_backend in ("cpu", "hybrid") or config.moe_cpu_layers)
+ ):
+ asked = (
+ f"--moe-cpu-layers={config.moe_cpu_layers!r}"
+ if config.moe_backend not in ("cpu", "hybrid")
+ else f"--moe-backend {config.moe_backend!r}"
+ )
+ raise ValueError(
+ f"{asked}: the CPU MoE executor does not support this model's expert "
+ f"activation {getattr(model_config, 'hidden_act', None)!r}; drop the flag "
+ "and let every layer decode on the GPU offload path instead."
+ )
+
if is_moe and config.moe_backend == "auto":
# A MoE model always defaults to the offload family: experts stream from pinned host
# banks into an auto-sized GPU slot cache, which is the only default that serves a model
@@ -1188,10 +1247,29 @@ def _adjust_config(config: EngineConfig):
gpu_name = torch.cuda.get_device_name(0) if torch.cuda.is_available() else None
if load_backend_recommendation(bench_fmt, gpu_name=gpu_name) == "hybrid":
- default_backend = "hybrid"
- logger.info_rank0(
- f"benchbw profile recommends hybrid for {bench_fmt!r} experts on this GPU"
- )
+ from freetoken.moe.cpu_executor import compiled_extension_supports
+
+ _act = getattr(model_config, "hidden_act", "silu")
+ if not _cpu_moe_act_ok:
+ logger.info_rank0(
+ f"benchbw profile recommends hybrid, but the CPU MoE executor does not "
+ f"support this model's expert activation "
+ f"{getattr(model_config, 'hidden_act', None)!r}; staying on offload"
+ )
+ elif moe_wfmt != "mxfp4" and not compiled_extension_supports(_act):
+ # Stale prebuilt _cpu_moe.so: an explicit cpu/hybrid pick still
+ # hard-fails in the executor, but a default must not turn into a
+ # post-load crash -- degrade to offload.
+ logger.info_rank0(
+ f"benchbw profile recommends hybrid, but the compiled _cpu_moe "
+ f"extension predates activation {_act!r} (rebuild with "
+ f"`python setup.py build_ext --inplace`); staying on offload"
+ )
+ else:
+ default_backend = "hybrid"
+ logger.info_rank0(
+ f"benchbw profile recommends hybrid for {bench_fmt!r} experts on this GPU"
+ )
override("moe_backend", default_backend)
logger.info_rank0(f"Auto-selected MoE backend: {config.moe_backend}")
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 4068263f..3d195a43 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -266,6 +266,25 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
expert_formats=_NVFP4_FORMATS,
aliases=("MiniMaxAI/MiniMax-M2.5",),
),
+ AotModel(
+ # GQA block-sparse (BSA): the BSAKVCache stores K/V through store_cache
+ # (4 kv heads x 128 head_dim), the index-key slab writes via torch scatter.
+ # Routed experts are NVFP4 with swigluoai, which restricts the expert GEMM
+ # to the Triton kernels -- but the banks keep the native "nvfp4" layout,
+ # so its bank rows are what fast_index_copy sees.
+ # The upstream MiniMaxAI/MiniMax-M3 alias covers the attention/embedding
+ # shapes only: its experts are MXFP8, which has no expert-bank provider on
+ # main -- the NVFP4 release is the servable offload path (M2.5 precedent).
+ name="nvidia/MiniMax-M3-NVFP4",
+ architecture="MiniMaxM3SparseForConditionalGeneration",
+ hidden_size=6144,
+ kv_groups=((4, 128),),
+ top_k=4,
+ moe_intermediate_size=3072,
+ expert_formats=("nvfp4",),
+ aliases=("MiniMaxAI/MiniMax-M3",),
+ arch_aliases=("MiniMaxM3SparseForCausalLM",),
+ ),
AotModel(
name="deepseek-ai/DeepSeek-V4-Flash",
architecture="DeepseekV4ForCausalLM",
diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
index 6f734f5c..880e8637 100644
--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp
@@ -67,7 +67,11 @@ inline bf16_t f32_to_bf16(float f) {
return static_cast<bf16_t>(u >> 16);
}
-enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2 };
+// ACT_SWIGLUOAI is the clamped (up + 1) swiglu (gpt-oss "swigluoai" /
+// MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit
+// scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it;
+// the mxfp4 kernel additionally fuses its own copy of the same math).
+enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2, ACT_SWIGLUOAI = 3 };
inline float act_apply(int act, float x) {
if (act == ACT_SILU) return x / (1.0f + std::exp(-x));
@@ -1600,13 +1604,25 @@ struct CpuMoeExecutor {
bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;
const int i0 = static_cast<int>(ib) * IBLK;
const int i1 = std::min(I, i0 + IBLK);
+ const bool swigluoai = act == ACT_SWIGLUOAI;
+ const float lim = swiglu_limit, alpha = swiglu_alpha;
for (int i = i0; i < i1; ++i) {
// gate = row i, up = row I+i
- const float gate =
+ float gate =
gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in;
- const float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,
- xe, xo, xi8, xas) * w_in;
- g_row[i] = f32_to_bf16(act_apply(act, gate) * up);
+ float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,
+ xe, xo, xi8, xas) * w_in;
+ if (swigluoai) {
+ // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1)
+ // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp).
+ if (gate > lim) gate = lim;
+ if (up > lim) up = lim;
+ else if (up < -lim) up = -lim;
+ const float glu = gate / (1.0f + std::exp(-gate * alpha));
+ g_row[i] = f32_to_bf16(glu * (up + 1.0f));
+ } else {
+ g_row[i] = f32_to_bf16(act_apply(act, gate) * up);
+ }
}
}
@@ -2124,4 +2140,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {
py::arg("ready_addr"), py::arg("slot"));
m.def("memop_sync", &cumemop_sync, py::arg("stream"), py::arg("done_addr"),
py::arg("slot"));
+ // ABI capability marker: the highest ActKind this build implements in the
+ // GENERIC epilogue. CpuMoeExecutor.__init__ probes it before requesting an act
+ // id the epilogue must handle -- a prebuilt .so from before ACT_SWIGLUOAI
+ // accepts id 3 without error and silently computes the wrong activation
+ // (act_apply falls through to gelu_tanh); the probe turns a stale extension
+ // into a loud rebuild instruction instead of wrong model outputs.
+ m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLUOAI); });
}
diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.py
index 252a211f..2c38b533 100644
--- a/python/freetoken/kernel/triton/activation.py
+++ b/python/freetoken/kernel/triton/activation.py
@@ -26,6 +26,11 @@ from freetoken.utils.arch import is_sm90_supported
SILU = 0
GELU = 1
GELU_TANH = 2
+# SwiGLU-OAI (gpt-oss / MiniMax-M3 "swigluoai"), UNINTERLEAVED halves (gate [:d],
+# up [d:], matching this file's *_and_mul layout -- the interleaved gpt-oss bank
+# variant lives in triton/mxfp4_moe.py):
+# y = clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)
+SWIGLUOAI = 3
_SQRT_2_OVER_PI = 0.7978845608028654 # sqrt(2/pi)
_GELU_C = 0.044715
@@ -64,6 +69,8 @@ def _act_and_mul_kernel(
out_ptr,
x_ptr,
d,
+ alpha,
+ limit,
ACT: tl.constexpr,
ENABLE_PDL: tl.constexpr,
BLOCK_D: tl.constexpr,
@@ -88,18 +95,30 @@ def _act_and_mul_kernel(
if ACT == 0: # SILU: x / (1 + exp(-x)) via ex2.approx
act = gate / (1.0 + _fast_ex2(-gate * _LOG2E))
+ y = act * up
elif ACT == 2: # GELU_TANH via tanh.approx
inner = 0.7978845608028654 * (gate + 0.044715 * gate * gate * gate)
act = 0.5 * gate * (1.0 + _fast_tanh(inner))
+ y = act * up
+ elif ACT == 3: # SWIGLUOAI: clamped gate/up, sigmoid(alpha*gate), (up + 1) bias
+ gate = tl.minimum(gate, limit)
+ up = tl.minimum(tl.maximum(up, -limit), limit)
+ act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E))
+ y = act * (up + 1.0)
else: # GELU (erf)
act = 0.5 * gate * (1.0 + libdevice.erf(gate * 0.7071067811865476))
-
- y = act * up
+ y = act * up
out_row = out_ptr + row * d
tl.store(out_row + cols, y.to(out_ptr.dtype.element_ty), mask=mask)
-def _act_and_mul(kind: int, x: torch.Tensor, out: torch.Tensor | None):
+def _act_and_mul(
+ kind: int,
+ x: torch.Tensor,
+ out: torch.Tensor | None,
+ alpha: float = 0.0,
+ limit: float = 0.0,
+):
assert x.is_cuda and x.is_contiguous()
d = x.shape[-1] // 2
out_shape = x.shape[:-1] + (d,)
@@ -115,7 +134,7 @@ def _act_and_mul(kind: int, x: torch.Tensor, out: torch.Tensor | None):
block_d = min(triton.next_power_of_2(d), 1024 if M >= 4096 else 512)
num_stages = 2 if block_d == 1024 else 3
_act_and_mul_kernel[grid](
- o2, x2, d, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl,
+ o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl,
BLOCK_D=block_d, num_warps=4, num_stages=num_stages,
)
return out
@@ -133,4 +152,18 @@ def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None) -> torch
return _act_and_mul(GELU_TANH, x, out)
-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul"]
+def swigluoai_and_mul(
+ x: torch.Tensor,
+ out: torch.Tensor | None = None,
+ *,
+ alpha: float = 1.702,
+ limit: float = 7.0,
+) -> torch.Tensor:
+ """SwiGLU-OAI over UNINTERLEAVED halves (gate ``x[..., :d]``, up ``x[..., d:]``):
+ ``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.
+ Same math as gpt-oss's interleaved ``gpt_oss_swiglu_kernel``; this layout matches
+ the [gate; up] halves the NVFP4 expert banks and merged gate_up projections use."""
+ return _act_and_mul(SWIGLUOAI, x, out, alpha=alpha, limit=limit)
+
+
+__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]
diff --git a/python/freetoken/kernel/triton/minimax_m3_sparse.py b/python/freetoken/kernel/triton/minimax_m3_sparse.py
new file mode 100644
index 00000000..d73596c4
--- /dev/null
+++ b/python/freetoken/kernel/triton/minimax_m3_sparse.py
@@ -0,0 +1,1118 @@
+"""Triton kernels for MiniMax-M3 block-sparse GQA attention (indexer + attend).
+
+Ported from the vLLM reference implementation
+(``vllm/models/minimax_m3/common/ops/{index_topk,sparse_attn}.py`` in the vLLM
+tree -- the semantics source of truth), adapted to FreeToken's storage model:
+
+* FreeToken's page table is page_size=1 semantics (one physical TOKEN row per
+ column) and the allocator hands out page-aligned runs, so with page_size == the
+ 128-token sparse block each block's rows are ``base .. base+127`` where ``base =
+ page_table[t, blk * 128]``. The kernels take a per-request ``block_rows`` table
+ of those int32 base rows and address the FLAT slabs directly -- the index-key
+ slab as ``[rows, index_dim]`` (``BSAKVCache.index_k_cache``), K/V as ``[rows,
+ kv_heads, head_dim]`` (the MHA pool's storage view) -- instead of vLLM's
+ ``[num_blocks, ...]`` paged layouts.
+* decode is always one query token per request (no spec decode), so the
+ ``decode_query_len`` machinery is dropped: ``kv_len = seq_lens[req]``.
+* KV is always bf16 (FreeToken pools); the fp8-KV dequant branches are dropped.
+* ``tl.dot`` needs >= 16 rows per operand, so the decode index-score kernel pads
+ the 4 index heads to a masked 16-row tile (the reference relied on a fork that
+ allows narrow dots).
+
+Semantics kept verbatim from the reference:
+
+* per-index-head (== per-KV-head) block scores: ``score[h, q, blk] = max over the
+ block's 128 positions of dot(iq[q, h], ik[pos])``, causal-masked, no scale
+ (only the ordering is consumed).
+* forced blocks: the first ``init_blocks`` score ``1e30`` and the newest
+ ``local_blocks`` score ``1e29`` (visible blocks only), so they always win the
+ top-k; padding slots come back ``-1``.
+* the attend visits the FIRST ``min(topk, num_visible_blocks)`` selected entries
+ (the bitonic top-k sorts descending), applies the causal mask inside the newest
+ block, and runs a base-2 online softmax; decode is split-K over the selected
+ blocks with an LSE merge (flash-decoding).
+
+CUDA-graph safety (decode): every kernel reads the live ``seq_lens`` from device
+memory and the split/chunk counts depend only on staged shapes, so the captured
+grid tracks the live position (same contract as the dsa/dsv4 backends).
+"""
+
+from __future__ import annotations
+
+import torch
+import triton
+import triton.language as tl
+
+# One sparse block == one KV page (the engine pins page_size to this).
+SPARSE_BLOCK_SIZE = 128
+
+
+def _round_up(a: int, b: int) -> int:
+ return (a + b - 1) // b * b
+
+
+# ---------------------------------------------------------------------------
+# Bitonic top-k helpers (verbatim from the reference; layout-agnostic).
+# ---------------------------------------------------------------------------
+def _compare_and_swap(x, ids, flip, i: tl.constexpr, n_dims: tl.constexpr):
+ n_outer: tl.constexpr = x.numel >> n_dims
+ shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]
+ y = tl.reshape(x, shape)
+ mask = tl.arange(0, 2)[None, :, None]
+ left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)
+ right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)
+ left = tl.reshape(left, x.shape)
+ right = tl.reshape(right, x.shape)
+ y_idx = tl.reshape(ids, shape)
+ left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)
+ right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)
+ left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)
+ right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)
+ idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)
+ ileft = left.to(idtype, bitcast=True)
+ iright = right.to(idtype, bitcast=True)
+ ix = x.to(idtype, bitcast=True)
+ cond = (left > right) != flip
+ ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))
+ new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))
+ return ret.to(x.dtype, bitcast=True), new_ids
+
+
+def _bitonic_merge(
+ x, ids, stage: tl.constexpr, order: tl.constexpr, n_dims: tl.constexpr
+):
+ n_outer: tl.constexpr = x.numel >> n_dims
+ tl.static_assert(stage <= n_dims)
+ if order == 2:
+ shape: tl.constexpr = [n_outer * 2 ** (n_dims - 1 - stage), 2, 2**stage]
+ flip = tl.reshape(
+ tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape
+ )
+ else:
+ flip = order
+ for i in tl.static_range(stage):
+ x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)
+ return x, ids
+
+
+# ---------------------------------------------------------------------------
+# Prefill index block-score kernel. score[h, token, block] = max over the
+# 128-token block of (idx_q . index_k), causal-masked; blocks addressed via the
+# per-request base-row table.
+# ---------------------------------------------------------------------------
[email protected](do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
+def _index_block_score_kernel(
+ q_ptr, # idx_q: [total_q, num_idx_heads, head_dim]
+ ik_ptr, # index-K slab, row-flat: [rows, head_dim]
+ score_ptr, # [num_idx_heads, total_q, max_block(stride)]
+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows
+ cu_seqlens, # [batch+1] query start offsets
+ seq_lens, # [batch] total K length
+ prefix_lens, # [batch] context length before this chunk's queries
+ num_idx_heads,
+ head_dim: tl.constexpr,
+ stride_q_n, stride_q_h, stride_q_d,
+ stride_ik_r, stride_ik_d,
+ stride_s_h, stride_s_n, stride_s_k,
+ stride_br_b,
+ BLOCK_SIZE_Q: tl.constexpr,
+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
+):
+ pid_q = tl.program_id(0)
+ pid_bh = tl.program_id(1)
+ pid_b = pid_bh // num_idx_heads
+ pid_h = pid_bh % num_idx_heads
+
+ seq_start = tl.load(cu_seqlens + pid_b)
+ q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start
+ seq_len = tl.load(seq_lens + pid_b)
+ prefix_len = tl.load(prefix_lens + pid_b)
+ if BLOCK_SIZE_Q * pid_q >= q_len:
+ return
+
+ q_ptrs = tl.make_block_ptr(
+ base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,
+ shape=(q_len, head_dim),
+ strides=(stride_q_n, stride_q_d),
+ offsets=(pid_q * BLOCK_SIZE_Q, 0),
+ block_shape=(BLOCK_SIZE_Q, head_dim),
+ order=(1, 0),
+ )
+ q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")
+ q_start = prefix_len + pid_q * BLOCK_SIZE_Q
+
+ off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len
+ off_k = tl.arange(0, BLOCK_SIZE_K)
+ off_d = tl.arange(0, head_dim)
+ br_row = block_rows_ptr + pid_b * stride_br_b
+ # Causal window: only blocks up to the last query token's position.
+ hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)
+ for i in tl.range(0, hi, BLOCK_SIZE_K):
+ blk = i // BLOCK_SIZE_K
+ base = tl.load(br_row + blk).to(tl.int64)
+ base = tl.maximum(base, 0)
+ pos = i + off_k
+ # No masked load: pages are whole 128-row runs, so rows base..base+127
+ # exist, and the index-key slab is ZERO-INITIALIZED by BSAKVCache (unlike
+ # the K/V slabs) -- unwritten tail rows contribute a finite 0-dot that the
+ # qk causal mask below discards. If that zero-init invariant ever changes,
+ # this load (and the decode score kernel's) must gain a pos mask.
+ k = tl.load(
+ ik_ptr
+ + (base + off_k[None, :]) * stride_ik_r
+ + off_d[:, None] * stride_ik_d,
+ )
+ qk = tl.dot(q, k)
+ if q_start < i + BLOCK_SIZE_K:
+ qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))
+ score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q]: max over the block
+ s_ptrs = (
+ score_ptr
+ + pid_h * stride_s_h
+ + (seq_start + pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q))
+ * stride_s_n
+ + blk * stride_s_k
+ )
+ q_store_mask = (pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)) < q_len
+ tl.store(s_ptrs, score, mask=q_store_mask)
+
+
+# ---------------------------------------------------------------------------
+# Prefill top-k over per-token block scores (verbatim reference port; the forced
+# init/local blocks get score boosts, invalid slots come back -1).
+# ---------------------------------------------------------------------------
[email protected]({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})
+ configs=[
+ triton.Config({"BLOCK_SIZE_K": 1024}, num_warps=8, num_stages=2),
+ triton.Config({"BLOCK_SIZE_K": 512}, num_warps=8, num_stages=2),
+ triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),
+ triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),
+ triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),
+ ],
+ key=["BLOCK_SIZE_T"],
+)
[email protected](do_not_specialize_on_alignment=["prefix_lens"])
+def _topk_index_kernel(
+ s_ptr, # [num_heads, total_q, max_block]
+ ti_ptr, # [num_heads, total_q, topk]
+ block_size: tl.constexpr, # sparse block size (128)
+ cu_seqlens,
+ prefix_lens,
+ topk,
+ init_blocks: tl.constexpr,
+ local_blocks: tl.constexpr,
+ stride_s_h, stride_s_n, stride_s_k,
+ stride_ti_h, stride_ti_n, stride_ti_t,
+ BLOCK_SIZE_K: tl.constexpr,
+ BLOCK_SIZE_T: tl.constexpr,
+):
+ tl.static_assert(BLOCK_SIZE_K > BLOCK_SIZE_T)
+ pid_q = tl.program_id(0)
+ pid_b = tl.program_id(1)
+ pid_h = tl.program_id(2)
+ seq_start = tl.load(cu_seqlens + pid_b)
+ q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start
+ prefix_len = tl.load(prefix_lens + pid_b)
+ if pid_q >= q_len:
+ return
+ off_k = tl.arange(0, BLOCK_SIZE_K)
+ off_t = tl.arange(0, BLOCK_SIZE_T)
+ s_ptrs = (
+ s_ptr
+ + (seq_start + pid_q) * stride_s_n
+ + pid_h * stride_s_h
+ + off_k * stride_s_k
+ )
+ topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
+ topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
+ left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
+ valid_blocks = (prefix_len + pid_q + block_size) // block_size
+ for i in tl.range(0, valid_blocks, BLOCK_SIZE_K):
+ causal_mask = i + off_k < valid_blocks
+ local_mask = i + off_k >= max(0, valid_blocks - local_blocks)
+ init_mask = i + off_k < init_blocks
+ score = tl.load(s_ptrs, mask=causal_mask, other=-1e30).to(tl.float32)
+ score = tl.where(score != score, -1e30, score)
+ s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
+ # Force-select init (1e30) and local (1e29) blocks; local applied last so
+ # it wins an overlap, matching the reference's MASK_INIT/MASK_LOCAL=False
+ # order (moot for M3's init_blocks == 0).
+ score = tl.where(causal_mask & init_mask, 1e30, score)
+ score = tl.where(causal_mask & local_mask, 1e29, score)
+ topk_score, last_topk_score = score, topk_score
+ topk_idx, last_topk_idx = (tl.where(causal_mask, i + off_k + 1, 0), topk_idx)
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
+ for j in tl.static_range(1, n_dims):
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score, topk_idx.to(tl.int32), j, 2, n_dims
+ )
+ if i != 0:
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
+ )
+ topk_score_new = last_topk_score * left_half_mask + topk_score * (
+ 1 - left_half_mask
+ )
+ topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
+ 1 - left_half_mask
+ )
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
+ )
+ else:
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
+ )
+ topk_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
+ topk_idx = tl.sum(
+ topk_mask[:, None]
+ * tl.reshape(topk_idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
+ axis=0,
+ )
+ ti_ptrs = (
+ ti_ptr
+ + (seq_start + pid_q) * stride_ti_n
+ + pid_h * stride_ti_h
+ + off_t * stride_ti_t
+ )
+ store_mask = off_t < topk
+ valid_mask = off_t < valid_blocks
+ topk_idx = tl.where(store_mask & valid_mask, topk_idx, -1)
+ tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=store_mask)
+
+
+# ---------------------------------------------------------------------------
+# Decode index-score kernel (split-K over seq blocks; cudagraph-safe). One query
+# per request; the 4 index heads ride a masked 16-row tile for tl.dot.
+# ---------------------------------------------------------------------------
[email protected](do_not_specialize=["num_kv_chunks"])
+def _decode_index_score_kernel(
+ q_ptr, # idx_q: [num_reqs, num_idx_heads, head_dim]
+ ik_ptr, # index-K slab, row-flat: [rows, head_dim]
+ score_ptr, # [num_idx_heads, num_reqs, max_block(stride)]
+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows
+ seq_lens, # [num_reqs] (device-read live lengths)
+ num_idx_heads: tl.constexpr,
+ head_dim: tl.constexpr,
+ init_blocks,
+ local_blocks,
+ stride_q_n, stride_q_h, stride_q_d,
+ stride_ik_r, stride_ik_d,
+ stride_s_h, stride_s_n, stride_s_k,
+ stride_br_b,
+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
+ BLOCK_SIZE_H: tl.constexpr, # >= 16 (padded index-head tile)
+ num_kv_chunks,
+):
+ pid_r = tl.program_id(0)
+ pid_c = tl.program_id(1)
+ off_h = tl.arange(0, BLOCK_SIZE_H)
+ h_mask = off_h < num_idx_heads
+
+ seq_len = tl.load(seq_lens + pid_r)
+ # Padded graph rows carry zero length -> empty range, nothing stored.
+ kv_len = tl.maximum(seq_len, 0)
+ num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
+
+ # Block-aligned fixed-count split: grid independent of seq_len (cuda graph).
+ chunk_size_blocks = (num_blocks + num_kv_chunks - 1) // num_kv_chunks
+ chunk_start_block = pid_c * chunk_size_blocks
+ chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)
+ if chunk_start_block >= chunk_end_block:
+ return
+ off_k = tl.arange(0, BLOCK_SIZE_K)
+ off_d = tl.arange(0, head_dim)
+ br_row = block_rows_ptr + pid_r * stride_br_b
+ local_start = tl.maximum(0, num_blocks - local_blocks)
+ q = tl.load(
+ q_ptr
+ + pid_r * stride_q_n
+ + off_h[None, :] * stride_q_h
+ + off_d[:, None] * stride_q_d,
+ mask=h_mask[None, :],
+ other=0.0,
+ ) # [D, H16]
+ for blk in tl.range(chunk_start_block, chunk_end_block):
+ base = tl.load(br_row + blk).to(tl.int64)
+ base = tl.maximum(base, 0)
+ pos = blk * BLOCK_SIZE_K + off_k
+ pos_mask = pos[:, None] < kv_len
+ # No masked load: the index-key slab is ZERO-INITIALIZED by BSAKVCache
+ # (unlike the K/V slabs), so unwritten tail rows dot to a finite 0 that
+ # the pos_mask below discards. If that invariant changes, mask this load.
+ k = tl.load(
+ ik_ptr
+ + (base + off_k[:, None]) * stride_ik_r
+ + off_d * stride_ik_d,
+ ) # [N, D]
+ kq = tl.dot(k, q, out_dtype=tl.float32) # [N, H16]
+ kq = tl.where(pos_mask & h_mask[None, :], kq, float("-inf"))
+ score = tl.max(kq, axis=0) # [H16]: max over the block's positions
+ # local wins an init/local overlap (reference order; moot for init_blocks=0).
+ is_init = blk < init_blocks
+ is_local = blk >= local_start
+ score = tl.where(is_local, 1e29, tl.where(is_init, 1e30, score))
+ tl.store(
+ score_ptr + off_h * stride_s_h + pid_r * stride_s_n + blk * stride_s_k,
+ score,
+ mask=h_mask,
+ )
+
+
+# ---------------------------------------------------------------------------
+# Decode top-k (split-K): per-chunk partial top-k + merge. Forced init/local
+# blocks are already encoded in the scores. Verbatim reference port minus the
+# spec-decode query-length math (kv_len == seq_lens[req]).
+# ---------------------------------------------------------------------------
[email protected]({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})
+def _topk_index_partial_kernel(
+ s_ptr, # score: [num_idx_heads, num_reqs, max_block]
+ ts_partial_ptr, # partial scores out: [chunks, num_idx_heads, num_reqs, T]
+ ti_partial_ptr, # partial idx out (1-indexed, 0=invalid): same shape
+ seq_lens, # [num_reqs]
+ block_size: tl.constexpr, # sparse block size (128)
+ topk: tl.constexpr,
+ stride_s_h, stride_s_b, stride_s_k,
+ stride_ts_c, stride_ts_h, stride_ts_b, stride_ts_t,
+ stride_ti_c, stride_ti_h, stride_ti_b, stride_ti_t,
+ BLOCK_SIZE_K: tl.constexpr,
+ BLOCK_SIZE_T: tl.constexpr,
+):
+ tl.static_assert(topk < BLOCK_SIZE_K)
+ pid_b = tl.program_id(0) # request id
+ pid_h = tl.program_id(1)
+ pid_chunk = tl.program_id(2)
+
+ kv_len = tl.maximum(tl.load(seq_lens + pid_b), 0)
+ num_blocks = (kv_len + block_size - 1) // block_size
+
+ # Chunk split from the row's LIVE block count (not the staged table width, which
+ # would degenerate short contexts to a serial scan in chunk 0) -- device-read,
+ # so the captured graph's split tracks the live position.
+ num_chunks = tl.num_programs(2)
+ chunk_blocks = (num_blocks + num_chunks - 1) // num_chunks
+ chunk_start = pid_chunk * chunk_blocks
+ chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)
+ chunk_actual = tl.maximum(chunk_end - chunk_start, 0)
+
+ off_k = tl.arange(0, BLOCK_SIZE_K)
+ off_t = tl.arange(0, BLOCK_SIZE_T)
+
+ s_ptrs = (
+ s_ptr
+ + pid_b * stride_s_b
+ + pid_h * stride_s_h
+ + (chunk_start + off_k) * stride_s_k
+ )
+
+ topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)
+ topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)
+ left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2
+
+ # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty
+ # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.
+ for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):
+ mask = off_k < chunk_actual - i
+ score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)
+ score = tl.where(score != score, -1e30, score)
+ s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K
+ topk_score, last_topk_score = score, topk_score
+ topk_idx, last_topk_idx = (
+ tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global
+ topk_idx,
+ )
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
+ for j in tl.static_range(1, n_dims):
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score, topk_idx.to(tl.int32), j, 2, n_dims
+ )
+ if i != 0:
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims
+ )
+ topk_score_new = last_topk_score * left_half_mask + topk_score * (
+ 1 - left_half_mask
+ )
+ topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (
+ 1 - left_half_mask
+ )
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims
+ )
+ else:
+ topk_score, topk_idx = _bitonic_merge(
+ topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims
+ )
+
+ # Extract first BLOCK_SIZE_T entries (top-K of this chunk after the sort).
+ topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
+ final_score = tl.sum(
+ topk_mask_extract[:, None]
+ * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
+ axis=0,
+ )
+ final_idx = tl.sum(
+ topk_mask_extract[:, None]
+ * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
+ axis=0,
+ )
+
+ # Always write all BLOCK_SIZE_T slots -- invalid slots carry -1e30 / 0
+ # sentinels and lose to real scores in the merge stage.
+ ts_ptrs = (
+ ts_partial_ptr
+ + pid_chunk * stride_ts_c
+ + pid_b * stride_ts_b
+ + pid_h * stride_ts_h
+ + off_t * stride_ts_t
+ )
+ ti_ptrs = (
+ ti_partial_ptr
+ + pid_chunk * stride_ti_c
+ + pid_b * stride_ti_b
+ + pid_h * stride_ti_h
+ + off_t * stride_ti_t
+ )
+ tl.store(ts_ptrs, final_score)
+ tl.store(ti_ptrs, final_idx)
+
+
+ {
+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),
+ "BLOCK_SIZE_K": lambda args: triton.next_power_of_2(
+ args["num_topk_chunks"] * triton.next_power_of_2(args["topk"])
+ ),
+ }
+)
[email protected](do_not_specialize=["num_topk_chunks"])
+def _topk_index_merge_kernel(
+ ts_partial_ptr, # partial scores: [chunks, num_idx_heads, num_reqs, T]
+ ti_partial_ptr, # partial idx (1-indexed, 0=invalid): same shape
+ ti_final_ptr, # final idx (0-indexed, -1=invalid): [num_idx_heads, num_reqs, topk]
+ seq_lens, # [num_reqs]
+ block_size: tl.constexpr,
+ topk: tl.constexpr,
+ stride_ts_c, stride_ts_h, stride_ts_b, stride_ts_t,
+ stride_ti_c, stride_ti_h, stride_ti_b, stride_ti_t,
+ stride_tif_h, stride_tif_b, stride_tif_t,
+ num_topk_chunks,
+ BLOCK_SIZE_K: tl.constexpr,
+ BLOCK_SIZE_T: tl.constexpr,
+):
+ pid_b = tl.program_id(0)
+ pid_h = tl.program_id(1)
+
+ kv_len = tl.maximum(tl.load(seq_lens + pid_b), 0)
+ num_blocks = (kv_len + block_size - 1) // block_size
+
+ off = tl.arange(0, BLOCK_SIZE_K)
+ chunk_idx = off // BLOCK_SIZE_T
+ in_chunk_idx = off % BLOCK_SIZE_T
+ valid = chunk_idx < num_topk_chunks
+
+ score_offset = (
+ chunk_idx * stride_ts_c
+ + pid_h * stride_ts_h
+ + pid_b * stride_ts_b
+ + in_chunk_idx * stride_ts_t
+ )
+ idx_offset = (
+ chunk_idx * stride_ti_c
+ + pid_h * stride_ti_h
+ + pid_b * stride_ti_b
+ + in_chunk_idx * stride_ti_t
+ )
+
+ score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(
+ tl.float32
+ )
+ score = tl.where(score != score, -1e30, score)
+ idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)
+
+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)
+ for j in tl.static_range(1, n_dims):
+ score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)
+ score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)
+
+ extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0
+ topk_idx_final = tl.sum(
+ extract_mask[:, None]
+ * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),
+ axis=0,
+ )
+
+ off_t = tl.arange(0, BLOCK_SIZE_T)
+ tif_ptrs = (
+ ti_final_ptr
+ + pid_h * stride_tif_h
+ + pid_b * stride_tif_b
+ + off_t * stride_tif_t
+ )
+ store_mask = off_t < topk
+ topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)
+ tl.store(
+ tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty), mask=store_mask
+ )
+
+
+# ---------------------------------------------------------------------------
+# GQA block-sparse attend, prefill. One program per (query token, kv head,
+# request); the 16-head GQA group rides one tl.dot tile. Base-2 online softmax.
+# ---------------------------------------------------------------------------
+ {
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
+ "BLOCK_SIZE_H": lambda args: max(
+ 16, triton.next_power_of_2(args["gqa_group_size"])
+ ),
+ }
+)
[email protected](do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])
+def _gqa_sparse_fwd_kernel(
+ q_ptr, # [total_q, num_heads, head_dim]
+ k_ptr, # K slab, row-flat: [rows, num_kv_heads, head_dim]
+ v_ptr, # V slab, row-flat: [rows, num_kv_heads, head_dim]
+ t_ptr, # topk_idx: [num_kv_heads, total_q, topk]
+ o_ptr, # [total_q, num_heads, head_dim]
+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows
+ cu_seqlens_q,
+ seq_lens,
+ prefix_lens,
+ num_kv_heads,
+ gqa_group_size,
+ head_dim,
+ max_topk,
+ sm_scale,
+ stride_qn, stride_qh, stride_qd,
+ stride_kr, stride_kh, stride_kd,
+ stride_vr, stride_vh, stride_vd,
+ stride_th, stride_tn, stride_tk,
+ stride_on, stride_oh, stride_od,
+ stride_br_b,
+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
+ BLOCK_SIZE_D: tl.constexpr,
+ BLOCK_SIZE_H: tl.constexpr,
+):
+ sm_scale_log2e = sm_scale * 1.4426950409
+ pid_q = tl.program_id(0)
+ pid_kh = tl.program_id(1)
+ pid_b = tl.program_id(2)
+ pid_h = pid_kh * gqa_group_size
+ q_start = tl.load(cu_seqlens_q + pid_b)
+ q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start
+ seq_len = tl.load(seq_lens + pid_b)
+ prefix_len = tl.load(prefix_lens + pid_b)
+ if pid_q >= q_len:
+ return
+ br_row = block_rows_ptr + pid_b * stride_br_b
+ off_n = tl.arange(0, BLOCK_SIZE_K)
+ off_d = tl.arange(0, BLOCK_SIZE_D)
+ off_h = tl.arange(0, BLOCK_SIZE_H)
+ d_mask = off_d < head_dim
+ h_mask = off_h < gqa_group_size
+
+ t_ptr_j = t_ptr + (q_start + pid_q) * stride_tn + pid_kh * stride_th
+ q_abs = prefix_len + pid_q # absolute position of this query token
+ valid_blocks = (q_abs + BLOCK_SIZE_K) // BLOCK_SIZE_K
+ real_topk = tl.minimum(max_topk, valid_blocks)
+
+ q = tl.load(
+ q_ptr
+ + (q_start + pid_q) * stride_qn
+ + (pid_h + off_h[:, None]) * stride_qh
+ + off_d[None, :] * stride_qd,
+ mask=h_mask[:, None] & d_mask[None, :],
+ other=0.0,
+ ) # [H, D]
+
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
+ lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
+ acc_o = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_D), dtype=tl.float32)
+ for _ in range(real_topk):
+ # `blk` is trusted in-range: selection writes exactly `real_topk` real
+ # entries. A -1 padding slot here would OOB-read br_row before the base
+ # clamp -- preserve that invariant when editing the top-k kernel.
+ blk = tl.load(t_ptr_j).to(tl.int32)
+ t_ptr_j = t_ptr_j + stride_tk
+ c = blk * BLOCK_SIZE_K
+ base = tl.load(br_row + blk).to(tl.int64)
+ base = tl.maximum(base, 0)
+ pos = c + off_n
+ # causal + in-sequence: the newest block is partially visible.
+ pos_mask = (pos <= q_abs) & (pos < seq_len)
+ # K/V loads must be pos-masked: the masks are the correctness contract
+ # for the tail page's unwritten rows (the forced local block visits the
+ # partial newest page nearly every step; unmasked rows would NaN the
+ # whole output row). BSAKVCache's zero-init is defense-in-depth, not a
+ # substitute -- recycled pages get re-dirtied.
+ k = tl.load(
+ k_ptr
+ + (base + off_n[None, :]) * stride_kr
+ + pid_kh * stride_kh
+ + off_d[:, None] * stride_kd,
+ mask=d_mask[:, None] & pos_mask[None, :],
+ other=0.0,
+ ) # [D, N]
+ qk = tl.dot(q, k) * sm_scale_log2e
+ qk += tl.where(pos_mask[None, :], 0, float("-inf"))
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
+ # All-masked rows keep a -inf running max NaN-free.
+ alpha = tl.where(m_ij == float("-inf"), 1.0, tl.exp2(m_i - m_ij))
+ p = tl.where(pos_mask[None, :], tl.exp2(qk - m_ij[:, None]), 0.0)
+ l_ij = tl.sum(p, axis=1)
+ acc_o = acc_o * alpha[:, None]
+ v = tl.load(
+ v_ptr
+ + (base + off_n[:, None]) * stride_vr
+ + pid_kh * stride_vh
+ + off_d[None, :] * stride_vd,
+ mask=pos_mask[:, None] & d_mask[None, :],
+ other=0.0,
+ ) # [N, D]
+ acc_o += tl.dot(p.to(v.dtype), v)
+ m_i = m_ij
+ lse_i = tl.where(
+ m_ij == float("-inf"),
+ lse_i,
+ m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij),
+ )
+ acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]
+ o_ptrs = (
+ o_ptr
+ + (q_start + pid_q) * stride_on
+ + (pid_h + off_h[:, None]) * stride_oh
+ + off_d[None, :] * stride_od
+ )
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), mask=h_mask[:, None] & d_mask[None, :])
+
+
+# ---------------------------------------------------------------------------
+# GQA block-sparse attend, decode (split-K over the selected blocks + LSE merge;
+# flash-decoding). One query per request; cudagraph-safe (device seq_lens).
+# ---------------------------------------------------------------------------
+ {
+ "BLOCK_SIZE_H": lambda args: max(
+ 16, triton.next_power_of_2(args["gqa_group_size"])
+ ),
+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),
+ }
+)
+def _gqa_sparse_decode_kernel(
+ q_ptr, # [num_reqs, num_heads, head_dim]
+ k_ptr, # K slab, row-flat: [rows, num_kv_heads, head_dim]
+ v_ptr, # V slab, row-flat: [rows, num_kv_heads, head_dim]
+ t_ptr, # topk_idx: [num_kv_heads, num_reqs, topk]
+ o_ptr, # partial out: [chunks, num_reqs, num_heads, head_dim]
+ lse_ptr, # partial lse (log2): [chunks, num_reqs, num_heads]
+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows
+ seq_lens, # [num_reqs]
+ total_q,
+ gqa_group_size,
+ head_dim,
+ max_topk,
+ sm_scale,
+ stride_qn, stride_qh, stride_qd,
+ stride_kr, stride_kh, stride_kd,
+ stride_vr, stride_vh, stride_vd,
+ stride_th, stride_tn, stride_tk,
+ stride_o_c, stride_o_b, stride_o_h, stride_o_d,
+ stride_l_c, stride_l_b, stride_l_h,
+ stride_br_b,
+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)
+ NUM_TOPK_CHUNKS: tl.constexpr,
+ BLOCK_SIZE_H: tl.constexpr,
+ BLOCK_SIZE_D: tl.constexpr,
+):
+ sm_scale_log2e = sm_scale * 1.4426950409
+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)
+ pid_b = pid_bc % total_q
+ pid_c = pid_bc // total_q
+ pid_h = pid_kh * gqa_group_size
+ chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS
+ chunk_start_topk = pid_c * chunk_size_topk
+ chunk_end_compiletime = chunk_start_topk + chunk_size_topk
+
+ kv_len = tl.maximum(tl.load(seq_lens + pid_b), 0)
+
+ idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn
+ num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K
+ real_topk = tl.minimum(max_topk, num_blocks)
+ chunk_end_topk = tl.minimum(chunk_end_compiletime, real_topk)
+
+ off_n = tl.arange(0, BLOCK_SIZE_K)
+ off_d = tl.arange(0, BLOCK_SIZE_D)
+ d_mask = off_d < head_dim
+ br_row = block_rows_ptr + pid_b * stride_br_b
+
+ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
+ lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)
+ acc_o = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_D), dtype=tl.float32)
+ q_ptrs = tl.make_block_ptr(
+ base=q_ptr + pid_b * stride_qn + pid_h * stride_qh,
+ shape=(gqa_group_size, head_dim),
+ strides=(stride_qh, stride_qd),
+ offsets=(0, 0),
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
+ order=(1, 0),
+ )
+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")
+
+ cur_idx_ptr = idx_base + chunk_start_topk * stride_tk
+ for _ in tl.range(chunk_start_topk, chunk_end_topk):
+ # `blk` is trusted in-range (selection invariant; see the sequential
+ # attend kernel): a -1 padding slot would OOB-read br_row pre-clamp.
+ blk = tl.load(cur_idx_ptr).to(tl.int32)
+ cur_idx_ptr = cur_idx_ptr + stride_tk
+ c = blk * BLOCK_SIZE_K
+ base = tl.load(br_row + blk).to(tl.int64)
+ base = tl.maximum(base, 0)
+ pos = c + off_n
+ pos_mask = pos < kv_len
+ # K/V loads must be pos-masked (same contract as the sequential attend
+ # kernel): with K masked, the additive -inf lanes stay -inf and
+ # p = exp2(-inf) = 0. Zero-init is defense-in-depth, not a substitute.
+ k = tl.load(
+ k_ptr
+ + (base + off_n[None, :]) * stride_kr
+ + pid_kh * stride_kh
+ + off_d[:, None] * stride_kd,
+ mask=d_mask[:, None] & pos_mask[None, :],
+ other=0.0,
+ )
+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)
+ qk += tl.where(pos_mask[None, :], 0, float("-inf"))
+ qk += tl.dot(q, k) * sm_scale_log2e
+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))
+ p = tl.exp2(qk - m_ij[:, None])
+ l_ij = tl.sum(p, axis=1)
+ acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]
+ v = tl.load(
+ v_ptr
+ + (base + off_n[:, None]) * stride_vr
+ + pid_kh * stride_vh
+ + off_d[None, :] * stride_vd,
+ mask=pos_mask[:, None] & d_mask[None, :],
+ other=0.0,
+ )
+ acc_o += tl.dot(p.to(v.dtype), v)
+ m_i = m_ij
+ lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)
+
+ # Empty chunks for active rows must store zero output; otherwise the merge
+ # can hit 0 * NaN. All-empty padded rows may still produce NaNs in merge.
+ scale = tl.where(lse_i > float("-inf"), tl.exp2(m_i - lse_i), tl.zeros_like(lse_i))
+ acc_o = acc_o * scale[:, None]
+ o_ptrs = tl.make_block_ptr(
+ base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,
+ shape=(gqa_group_size, head_dim),
+ strides=(stride_o_h, stride_o_d),
+ offsets=(0, 0),
+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),
+ order=(1, 0),
+ )
+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))
+ lse_ptrs = tl.make_block_ptr(
+ base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,
+ shape=(gqa_group_size,),
+ strides=(stride_l_h,),
+ offsets=(0,),
+ block_shape=(BLOCK_SIZE_H,),
+ order=(0,),
+ )
+ tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))
+
+
+ {"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"])}
+)
+def _merge_topk_attn_out_kernel(
+ o_ptr, # partials: [chunks, num_reqs, num_heads, head_dim]
+ lse_ptr, # partials (log2): [chunks, num_reqs, num_heads]
+ out_ptr, # merged out: [num_reqs, num_heads, head_dim]
+ head_dim,
+ stride_o_c, stride_o_b, stride_o_h, stride_o_d,
+ stride_l_c, stride_l_b, stride_l_h,
+ stride_out_n, stride_out_h, stride_out_d,
+ NUM_TOPK_CHUNKS: tl.constexpr,
+ BLOCK_SIZE_D: tl.constexpr,
+):
+ pid_b, pid_h = tl.program_id(0), tl.program_id(1)
+
+ off_c = tl.arange(0, NUM_TOPK_CHUNKS)
+ off_d = tl.arange(0, BLOCK_SIZE_D)
+ o_ptrs = tl.make_block_ptr(
+ base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,
+ shape=(NUM_TOPK_CHUNKS, head_dim),
+ strides=(stride_o_c, stride_o_d),
+ offsets=(0, 0),
+ block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),
+ order=(1, 0),
+ )
+ lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c
+ o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")
+ lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0
+ lse_max = tl.max(lse, axis=0)
+ weights = tl.exp2(lse - lse_max)
+ weights = weights / tl.sum(weights, axis=0)
+ o_merged = tl.sum(o * weights[:, None], axis=0)
+ out_ptrs = (
+ out_ptr + pid_b * stride_out_n + pid_h * stride_out_h + off_d * stride_out_d
+ )
+ tl.store(out_ptrs, o_merged.to(out_ptr.dtype.element_ty), mask=off_d < head_dim)
+
+
+# ---------------------------------------------------------------------------
+# Python wrappers
+# ---------------------------------------------------------------------------
+def m3_index_score_prefill(
+ idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]
+ ik_rows: torch.Tensor, # index-K slab, row-flat: [rows, head_dim]
+ block_rows: torch.Tensor, # [batch, max_blocks] int32 base rows
+ cu_seqlens_q: torch.Tensor, # [batch+1] int32
+ seq_lens: torch.Tensor, # [batch] int32
+ prefix_lens: torch.Tensor, # [batch] int32
+ max_query_len: int,
+ max_seq_len: int,
+) -> torch.Tensor:
+ """Per-token index scores for each visible sparse block: score ``[num_idx_heads,
+ total_q, >=max_block]`` fp32, each entry the max over a 128-token block."""
+ total_q, num_idx_heads, head_dim = idx_q.shape
+ batch = cu_seqlens_q.shape[0] - 1
+ max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)
+ # Keep score strides 16-divisible to avoid Triton recompiles.
+ score = torch.empty(
+ (num_idx_heads, total_q, _round_up(max_block, 16)),
+ dtype=torch.float32,
+ device=idx_q.device,
+ )
+ BLOCK_SIZE_Q = 64
+ grid = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads)
+ _index_block_score_kernel[grid](
+ idx_q, ik_rows, score, block_rows,
+ cu_seqlens_q, seq_lens, prefix_lens,
+ num_idx_heads, head_dim,
+ idx_q.stride(0), idx_q.stride(1), idx_q.stride(2),
+ ik_rows.stride(0), ik_rows.stride(1),
+ score.stride(0), score.stride(1), score.stride(2),
+ block_rows.stride(0),
+ BLOCK_SIZE_Q=BLOCK_SIZE_Q,
+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
+ )
+ return score
+
+
+def m3_index_topk_prefill(
+ score: torch.Tensor, # [num_idx_heads, total_q, >=max_block]
+ cu_seqlens_q: torch.Tensor, # [batch+1] int32
+ prefix_lens: torch.Tensor, # [batch] int32
+ max_query_len: int,
+ topk: int,
+ init_blocks: int,
+ local_blocks: int,
+) -> torch.Tensor:
+ """Causal per-token top-k over precomputed block scores -> ``[num_idx_heads,
+ total_q, topk]`` int32 block ids (-1 pad)."""
+ num_idx_heads, total_q = score.shape[0], score.shape[1]
+ batch = cu_seqlens_q.shape[0] - 1
+ topk_idx = torch.empty(
+ (num_idx_heads, total_q, topk), dtype=torch.int32, device=score.device
+ )
+ grid = (max_query_len, batch, num_idx_heads)
+ _topk_index_kernel[grid](
+ score, topk_idx,
+ SPARSE_BLOCK_SIZE,
+ cu_seqlens_q, prefix_lens,
+ topk, init_blocks, local_blocks,
+ score.stride(0), score.stride(1), score.stride(2),
+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),
+ )
+ return topk_idx
+
+
+def m3_index_decode(
+ idx_q: torch.Tensor, # [num_reqs, num_idx_heads, head_dim]
+ ik_rows: torch.Tensor, # index-K slab, row-flat: [rows, head_dim]
+ block_rows: torch.Tensor, # [num_reqs, max_blocks] int32 base rows
+ seq_lens: torch.Tensor, # [num_reqs] int32 (device, live)
+ max_blocks: int, # staged block-table width (static under cuda graphs)
+ topk: int,
+ init_blocks: int,
+ local_blocks: int,
+) -> torch.Tensor:
+ """Decode index block-score + top-k, both split-K (cudagraph-safe). Returns
+ ``[num_idx_heads, num_reqs, topk]`` int32 (0-indexed block ids, -1 pad)."""
+ num_reqs, num_idx_heads, head_dim = idx_q.shape
+ score_stride = _round_up(max_blocks, 16)
+ score = torch.empty(
+ (num_idx_heads, num_reqs, score_stride), dtype=torch.float32, device=idx_q.device
+ )
+ # split-K over seq blocks; chunk count depends only on staged shapes so the
+ # grid is fixed within a cuda graph.
+ TARGET_GRID = 512
+ MAX_NUM_KV_CHUNKS = 256
+ target = max(1, min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, num_reqs)))
+ num_kv_chunks = 1 << (target.bit_length() - 1)
+ _decode_index_score_kernel[(num_reqs, num_kv_chunks)](
+ idx_q, ik_rows, score, block_rows, seq_lens,
+ num_idx_heads, head_dim,
+ init_blocks, local_blocks,
+ idx_q.stride(0), idx_q.stride(1), idx_q.stride(2),
+ ik_rows.stride(0), ik_rows.stride(1),
+ score.stride(0), score.stride(1), score.stride(2),
+ block_rows.stride(0),
+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
+ BLOCK_SIZE_H=max(16, triton.next_power_of_2(num_idx_heads)),
+ num_kv_chunks=num_kv_chunks,
+ num_warps=4, num_stages=2,
+ )
+
+ topk_idx = torch.empty(
+ (num_idx_heads, num_reqs, topk), dtype=torch.int32, device=idx_q.device
+ )
+ TOPK_TARGET_GRID = 64
+ MAX_NUM_TOPK_CHUNKS = 16
+ topk_target = max(
+ 1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, num_reqs * num_idx_heads))
+ )
+ num_topk_chunks = 1 << (topk_target.bit_length() - 1)
+ block_size_t = triton.next_power_of_2(topk)
+ ts_partial = torch.empty(
+ (num_topk_chunks, num_idx_heads, num_reqs, block_size_t),
+ dtype=torch.float32, device=idx_q.device,
+ )
+ ti_partial = torch.empty(
+ (num_topk_chunks, num_idx_heads, num_reqs, block_size_t),
+ dtype=torch.int32, device=idx_q.device,
+ )
+ # The chunk split is computed in-kernel from each row's LIVE block count
+ # (device-read; the staged table width would serialize short contexts).
+ _topk_index_partial_kernel[(num_reqs, num_idx_heads, num_topk_chunks)](
+ score, ts_partial, ti_partial, seq_lens,
+ SPARSE_BLOCK_SIZE, topk,
+ score.stride(0), score.stride(1), score.stride(2),
+ ts_partial.stride(0), ts_partial.stride(1), ts_partial.stride(2), ts_partial.stride(3),
+ ti_partial.stride(0), ti_partial.stride(1), ti_partial.stride(2), ti_partial.stride(3),
+ # Fixed config (no autotune: decode is captured into CUDA graphs, and
+ # autotune would benchmark at run time). 128 matches the reference's
+ # smallest always-valid config.
+ BLOCK_SIZE_K=128,
+ num_warps=4, num_stages=2,
+ )
+ _topk_index_merge_kernel[(num_reqs, num_idx_heads)](
+ ts_partial, ti_partial, topk_idx, seq_lens,
+ SPARSE_BLOCK_SIZE, topk,
+ ts_partial.stride(0), ts_partial.stride(1), ts_partial.stride(2), ts_partial.stride(3),
+ ti_partial.stride(0), ti_partial.stride(1), ti_partial.stride(2), ti_partial.stride(3),
+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),
+ num_topk_chunks=num_topk_chunks,
+ num_warps=2,
+ )
+ return topk_idx
+
+
+def m3_sparse_attn_prefill(
+ q: torch.Tensor, # [total_q, num_heads, head_dim]
+ k_rows: torch.Tensor, # K slab, row-flat: [rows, num_kv_heads, head_dim]
+ v_rows: torch.Tensor, # V slab, row-flat: [rows, num_kv_heads, head_dim]
+ topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]
+ block_rows: torch.Tensor, # [batch, max_blocks] int32 base rows
+ cu_seqlens_q: torch.Tensor, # [batch+1] int32
+ seq_lens: torch.Tensor, # [batch] int32
+ prefix_lens: torch.Tensor, # [batch] int32
+ max_query_len: int,
+ sm_scale: float,
+ output: torch.Tensor, # [total_q, num_heads, head_dim]
+) -> None:
+ """GQA block-sparse attention over the selected blocks (prefill/extend)."""
+ total_q, num_heads, head_dim = q.shape
+ num_kv_heads = k_rows.shape[1]
+ batch = cu_seqlens_q.shape[0] - 1
+ topk = topk_idx.shape[-1]
+ gqa_group_size = num_heads // num_kv_heads
+ grid = (max_query_len, num_kv_heads, batch)
+ _gqa_sparse_fwd_kernel[grid](
+ q, k_rows, v_rows, topk_idx, output, block_rows,
+ cu_seqlens_q, seq_lens, prefix_lens,
+ num_kv_heads, gqa_group_size, head_dim, topk, sm_scale,
+ q.stride(0), q.stride(1), q.stride(2),
+ k_rows.stride(0), k_rows.stride(1), k_rows.stride(2),
+ v_rows.stride(0), v_rows.stride(1), v_rows.stride(2),
+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),
+ output.stride(0), output.stride(1), output.stride(2),
+ block_rows.stride(0),
+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
+ num_warps=4, num_stages=2,
+ )
+
+
+def m3_sparse_attn_decode(
+ q: torch.Tensor, # [num_reqs, num_heads, head_dim]
+ k_rows: torch.Tensor, # K slab, row-flat: [rows, num_kv_heads, head_dim]
+ v_rows: torch.Tensor, # V slab, row-flat: [rows, num_kv_heads, head_dim]
+ topk_idx: torch.Tensor, # [num_kv_heads, num_reqs, topk]
+ block_rows: torch.Tensor, # [num_reqs, max_blocks] int32 base rows
+ seq_lens: torch.Tensor, # [num_reqs] int32 (device, live)
+ sm_scale: float,
+ output: torch.Tensor, # [num_reqs, num_heads, head_dim]
+) -> None:
+ """GQA block-sparse attention for decode (split-K over the top-k blocks +
+ LSE merge; cudagraph-safe)."""
+ num_reqs, num_heads, head_dim = q.shape
+ num_kv_heads = k_rows.shape[1]
+ max_topk = topk_idx.shape[-1]
+ gqa_group_size = num_heads // num_kv_heads
+ TARGET_GRID = 256
+ target = max(1, min(max_topk, TARGET_GRID // max(1, num_reqs * num_kv_heads)))
+ num_topk_chunks = 1 << (target.bit_length() - 1)
+ o_partial = torch.empty(
+ (num_topk_chunks, num_reqs, num_heads, head_dim), dtype=q.dtype, device=q.device
+ )
+ lse_partial = torch.empty(
+ (num_topk_chunks, num_reqs, num_heads), dtype=torch.float32, device=q.device
+ )
+ grid = (num_reqs * num_topk_chunks, num_kv_heads)
+ _gqa_sparse_decode_kernel[grid](
+ q, k_rows, v_rows, topk_idx, o_partial, lse_partial, block_rows, seq_lens,
+ num_reqs, gqa_group_size, head_dim, max_topk, sm_scale,
+ q.stride(0), q.stride(1), q.stride(2),
+ k_rows.stride(0), k_rows.stride(1), k_rows.stride(2),
+ v_rows.stride(0), v_rows.stride(1), v_rows.stride(2),
+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),
+ o_partial.stride(0), o_partial.stride(1), o_partial.stride(2), o_partial.stride(3),
+ lse_partial.stride(0), lse_partial.stride(1), lse_partial.stride(2),
+ block_rows.stride(0),
+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,
+ NUM_TOPK_CHUNKS=num_topk_chunks,
+ num_warps=4, num_stages=2,
+ )
+ _merge_topk_attn_out_kernel[(num_reqs, num_heads)](
+ o_partial, lse_partial, output, head_dim,
+ o_partial.stride(0), o_partial.stride(1), o_partial.stride(2), o_partial.stride(3),
+ lse_partial.stride(0), lse_partial.stride(1), lse_partial.stride(2),
+ output.stride(0), output.stride(1), output.stride(2),
+ NUM_TOPK_CHUNKS=num_topk_chunks,
+ num_warps=4,
+ )
+
+
+__all__ = [
+ "SPARSE_BLOCK_SIZE",
+ "m3_index_score_prefill",
+ "m3_index_topk_prefill",
+ "m3_index_decode",
+ "m3_sparse_attn_prefill",
+ "m3_sparse_attn_decode",
+]
diff --git a/python/freetoken/kernel/triton/mxfp8_linear.py b/python/freetoken/kernel/triton/mxfp8_linear.py
new file mode 100644
index 00000000..98a7001c
--- /dev/null
+++ b/python/freetoken/kernel/triton/mxfp8_linear.py
@@ -0,0 +1,292 @@
+"""MXFP8 (block-32 e8m0-scaled FP8) W8A16 dense linear, shared across models.
+
+MiniMax-M3's modelopt checkpoint quantizes every dense projection (q/k/v/o, dense
+MLPs, shared experts, indexer) to MXFP8: fp8-e4m3 ``weight [N, K]`` plus uint8 e8m0
+exponent codes ``weight_scale_inv [N, K//32]``, dequant multiplier ``2**(code - 127)``.
+
+Decode is weight-bandwidth bound, so small batches read the fp8 weight directly in a
+split-K GEMV (half the traffic of a bf16-resident weight). The block-32 scale is
+loaded once per tile and broadcast in registers; a per-element scale gather + exp2
+costs ~3x the whole GEMV. Past the GEMV cap the forward dequantizes to bf16
+(``fp8 * 2**k`` is exact in bf16) and runs cuBLAS -- a fused inline-dequant GEMM
+never came close to cuBLAS on these shapes, so the transient is the fast option.
+
+The GEMV accumulates in fp32; the cuBLAS path is bf16 with fp32 accumulate, same as
+any other bf16 projection.
+"""
+
+from __future__ import annotations
+
+import os
+
+import torch
+import triton
+import triton.language as tl
+from freetoken.layers.base import BaseOP
+
+from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native_cx, e4m3_u8_to_f32
+
+FP8 = torch.float8_e4m3fn
+MXFP8_BLOCK = 32
+_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}
+
+# Escape hatch: FREETOKEN_DEBUG_MXFP8_REF=1 swaps the kernels for a pure-torch
+# dequant matmul (numeric reference / A-B debugging). Evaluated once, logged once.
+_USE_REF = os.environ.get("FREETOKEN_DEBUG_MXFP8_REF") == "1"
+if _USE_REF:
+ from freetoken.utils import init_logger
+
+ init_logger(__name__).info(
+ "FREETOKEN_DEBUG_MXFP8_REF=1: MXFP8 linears serve the pure-torch fp32 "
+ "dequant reference (slow; debugging only)."
+ )
+
+
+def mxfp8_dequant(weight: torch.Tensor, scale_codes: torch.Tensor,
+ dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:
+ """Dequant: ``w[n, k] * 2**(codes[n, k//32] - 127)`` -> ``dtype``.
+
+ Serves the large-M forward (dequant + cuBLAS), the load-time bf16 ablation
+ path and the kernels' numeric reference in tests. For 16-bit dtypes the
+ whole computation runs in that dtype -- the pow2 descale is lossless there,
+ and fp32 transients would double the peak footprint (which lands in the
+ graph pool when captured). fp32 keeps fp32 compute: it is the reference.
+ """
+ assert weight.shape[-1] % MXFP8_BLOCK == 0
+ assert scale_codes.shape[-1] == weight.shape[-1] // MXFP8_BLOCK
+ compute = torch.float32 if dtype == torch.float32 else dtype
+ descale = torch.exp2(scale_codes.to(torch.float32) - 127.0).to(compute)
+ w = weight.to(compute).view(*weight.shape[:-1], -1, MXFP8_BLOCK)
+ return (w * descale.unsqueeze(-1)).view(weight.shape).to(dtype)
+
+
+# GEMV cap. Must cover the CUDA-graph ladder max (256): a captured dequant+cuBLAS
+# call would bake its full-weight transient into the shared graph pool (~2 GiB
+# across the resident shapes), and the GEMV is also simply faster everywhere in
+# graph range (M=128: ~230us vs ~930us on RTX PRO 6000). M_TILE buckets {16..256};
+# 512 exceeds sm_120's shared-memory budget. Past the cap only prefill lands here,
+# where cuBLAS amortizes the transient.
+_GEMV_MAX_M = 256
+
+
+def _mxfp8_gemv_m1_splitk_kernel(
+ a_ptr, w_ptr, s_ptr, part_ptr, N, K, n_kb, kb_per,
+ stride_ak, stride_wn, stride_wk, stride_sn, stride_sk, stride_pk, stride_pn,
+ BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
+):
+ """M == 1 specialization: a plain fp32 multiply-reduce. The dot tile's 16-row
+ padding costs ~30% at M=1 on RTX PRO 6000 / 5090, hence this kernel -- but on
+ H100 the dot kernel is ~26% faster even at M=1. If sm_90 becomes a serving
+ target, re-benchmark and dispatch by arch (or drop this kernel)."""
+ KB32: tl.constexpr = BLOCK_K // 32
+ pid_n = tl.program_id(0)
+ pid_k = tl.program_id(1)
+ offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ n_mask = offs_n < N
+ off_kb32 = tl.arange(0, KB32)
+ kb_start = pid_k * kb_per
+ acc = tl.zeros((BLOCK_N,), dtype=tl.float32)
+ for i in range(kb_per):
+ kb = kb_start + i
+ if kb < n_kb:
+ offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)
+ k_mask = offs_k < K
+ a = tl.load(a_ptr + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32)
+ if e4m3_native_cx():
+ w = tl.load(
+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
+ mask=n_mask[:, None] & k_mask[None, :], other=0.0,
+ ).to(tl.float32)
+ else:
+ w = e4m3_u8_to_f32(tl.load(
+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
+ mask=n_mask[:, None] & k_mask[None, :], other=0,
+ ))
+ codes = tl.load(
+ s_ptr + offs_n[:, None] * stride_sn
+ + (kb * KB32 + off_kb32[None, :]) * stride_sk,
+ mask=n_mask[:, None] & ((kb * KB32 + off_kb32[None, :]) * 32 < K),
+ other=127,
+ ).to(tl.float32) # [BLOCK_N, KB32]
+ scale = tl.exp2(codes - 127.0)
+ prod = tl.reshape(w * a[None, :], (BLOCK_N, KB32, 32))
+ acc += tl.sum(tl.sum(prod, axis=2) * scale, axis=1)
+ tl.store(part_ptr + pid_k * stride_pk + offs_n * stride_pn, acc, mask=n_mask)
+
+
+def _mxfp8_gemv_splitk_kernel(
+ a_ptr, w_ptr, s_ptr, part_ptr, M, N, K, n_kb, kb_per,
+ stride_am, stride_ak, stride_wn, stride_wk, stride_sn, stride_sk,
+ stride_pk, stride_pm, stride_pn,
+ M_TILE: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,
+):
+ """Each (pid_n, pid_k) accumulates ``kb_per`` BLOCK_K chunks of ``a[:M] @ w^T``
+ for a BLOCK_N slice of outputs. BLOCK_K is a multiple of 32, so each chunk
+ covers ``BLOCK_K // 32`` whole scale blocks: one ``[BLOCK_N, KB32]`` code load
+ + exp2 per tile, broadcast over the 32-wide inner axis via a 3D register view,
+ folded into the fp8 weight BEFORE the dot (pow2 scaling is lossless in bf16)."""
+ KB32: tl.constexpr = BLOCK_K // 32
+ pid_n = tl.program_id(0)
+ pid_k = tl.program_id(1)
+ offs_m = tl.arange(0, M_TILE)
+ m_mask = offs_m < M
+ offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)
+ n_mask = offs_n < N
+ off_kb32 = tl.arange(0, KB32)
+ kb_start = pid_k * kb_per
+ acc = tl.zeros((M_TILE, BLOCK_N), dtype=tl.float32)
+ for i in range(kb_per):
+ kb = kb_start + i
+ if kb < n_kb:
+ offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)
+ k_mask = offs_k < K
+ a = tl.load(
+ a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak,
+ mask=m_mask[:, None] & k_mask[None, :], other=0.0,
+ ) # [M_TILE, BLOCK_K] bf16
+ if e4m3_native_cx():
+ w = tl.load(
+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
+ mask=n_mask[:, None] & k_mask[None, :], other=0.0,
+ ).to(tl.float32)
+ else:
+ w = e4m3_u8_to_f32(tl.load(
+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,
+ mask=n_mask[:, None] & k_mask[None, :], other=0,
+ ))
+ codes = tl.load(
+ s_ptr + offs_n[:, None] * stride_sn
+ + (kb * KB32 + off_kb32[None, :]) * stride_sk,
+ mask=n_mask[:, None] & ((kb * KB32 + off_kb32[None, :]) * 32 < K),
+ other=127,
+ ).to(tl.float32) # [BLOCK_N, KB32]
+ scale = tl.exp2(codes - 127.0)
+ w3 = tl.reshape(w, (BLOCK_N, KB32, 32)) * scale[:, :, None]
+ w_scaled = tl.reshape(w3, (BLOCK_N, BLOCK_K)).to(a.dtype)
+ acc += tl.dot(a, tl.trans(w_scaled), out_dtype=tl.float32)
+ part_base = part_ptr + pid_k * stride_pk
+ tl.store(
+ part_base + offs_m[:, None] * stride_pm + offs_n[None, :] * stride_pn,
+ acc,
+ mask=m_mask[:, None] & n_mask[None, :],
+ )
+
+
+def _splitk_reduce_kernel(
+ part_ptr, out_ptr, N, SPLIT_K: tl.constexpr,
+ stride_pk, stride_pm, stride_pn, stride_om, stride_on,
+ BLOCK: tl.constexpr, OUT: tl.constexpr,
+):
+ pid_m = tl.program_id(1)
+ offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)
+ mask = offs < N
+ acc = tl.zeros((BLOCK,), dtype=tl.float32)
+ for k in tl.static_range(SPLIT_K):
+ acc += tl.load(
+ part_ptr + k * stride_pk + pid_m * stride_pm + offs * stride_pn,
+ mask=mask, other=0.0,
+ )
+ tl.store(out_ptr + pid_m * stride_om + offs * stride_on, acc.to(OUT), mask=mask)
+
+
+def _gemv(a: torch.Tensor, weight: torch.Tensor, scale_codes: torch.Tensor,
+ out_dtype: torch.dtype) -> torch.Tensor:
+ """Small-M split-K GEMV. ``a`` [M, K] bf16 (M <= _GEMV_MAX_M); ``weight``
+ [N, K] fp8; ``scale_codes`` [N, K//32] uint8."""
+ M, K = a.shape
+ N = weight.shape[0]
+ BLOCK_K = 128
+ n_kb = triton.cdiv(K, BLOCK_K)
+ BLOCK_N = 16
+ n_tiles = triton.cdiv(N, BLOCK_N)
+ split_k = max(1, min(1536 // n_tiles, n_kb))
+ split_k = 1 << (split_k.bit_length() - 1) # pow2 -> stable reduction order
+ kb_per = triton.cdiv(n_kb, split_k)
+ part = torch.empty((split_k, M, N), dtype=torch.float32, device=a.device)
+ if M == 1:
+ _mxfp8_gemv_m1_splitk_kernel[(n_tiles, split_k)](
+ a, weight, scale_codes, part, N, K, n_kb, kb_per,
+ a.stride(1), weight.stride(0), weight.stride(1),
+ scale_codes.stride(0), scale_codes.stride(1),
+ part.stride(0), part.stride(2),
+ BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, num_warps=1,
+ )
+ else:
+ # M_TILE buckets to the next pow2 >= 16 (tl.dot minimum); the whole batch
+ # rides one weight pass regardless of bucket, so the only cost of a bigger
+ # tile is row padding, amortized with proportionally more warps.
+ m_tile = max(16, triton.next_power_of_2(M))
+ _mxfp8_gemv_splitk_kernel[(n_tiles, split_k)](
+ a, weight, scale_codes, part, M, N, K, n_kb, kb_per,
+ a.stride(0), a.stride(1), weight.stride(0), weight.stride(1),
+ scale_codes.stride(0), scale_codes.stride(1),
+ part.stride(0), part.stride(1), part.stride(2),
+ M_TILE=m_tile, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K,
+ num_warps=m_tile // 16,
+ )
+ out = torch.empty((M, N), dtype=out_dtype, device=a.device)
+ _splitk_reduce_kernel[(triton.cdiv(N, 256), M)](
+ part, out, N, split_k,
+ part.stride(0), part.stride(1), part.stride(2), out.stride(0), out.stride(1),
+ BLOCK=256, OUT=_TL_DTYPE[out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16],
+ num_warps=2,
+ )
+ return out
+
+
+def mxfp8_linear(
+ x: torch.Tensor, weight: torch.Tensor, scale_codes: torch.Tensor,
+ bias: torch.Tensor | None = None,
+) -> torch.Tensor:
+ """``y = x @ dequant(weight, scale_codes)^T``. Small batches
+ (M <= 256) -> split-K GEMV (one pass over the fp8 weight); larger M -> bf16
+ dequant + cuBLAS (see the module docstring). ``weight`` [N, K] fp8-e4m3;
+ ``scale_codes`` [N, K//32] uint8 e8m0 exponent codes (dequant multiplier
+ ``2**(code - 127)``)."""
+ *lead, K = x.shape
+ N = weight.shape[0]
+ M = x.numel() // K
+ assert K % MXFP8_BLOCK == 0 and scale_codes.shape == (N, K // MXFP8_BLOCK)
+ if _USE_REF: # numeric-reference fallback (debug / A-B)
+ w = mxfp8_dequant(weight, scale_codes, dtype=torch.float32)
+ out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, N)
+ elif M <= _GEMV_MAX_M:
+ w8 = e4m3_kernel_view(weight)
+ out = _gemv(x.reshape(M, K), w8, scale_codes, x.dtype).reshape(*lead, N)
+ else:
+ # Per-call bf16 transient (pow2 descale is lossless in bf16) + cuBLAS.
+ w = mxfp8_dequant(weight, scale_codes, dtype=x.dtype)
+ out = torch.nn.functional.linear(x.reshape(-1, K), w).reshape(*lead, N)
+ if bias is not None:
+ out = out + bias.to(out.dtype)
+ return out
+
+
+# ======================================================================================
+# BaseOP linear layers (TP=1, replicated). Buffers: fp8 ``weight`` + uint8
+# ``weight_scale_inv`` (the checkpoint's e8m0 exponent codes, loaded verbatim).
+# ======================================================================================
+class Mxfp8Linear(BaseOP):
+ """Replicated MXFP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + uint8 e8m0
+ ``weight_scale_inv`` ``[out, in // 32]``. Fused projections concatenate several
+ checkpoint projections along the output dim; the scales are per-output-row so the
+ concatenation is exact."""
+
+ def __init__(self, in_features: int, out_features: int, has_bias: bool = False):
+ assert in_features % MXFP8_BLOCK == 0
+ self.in_features = in_features
+ self.out_features = out_features
+ self.weight = torch.empty(out_features, in_features, dtype=FP8)
+ self.weight_scale_inv = torch.empty(
+ out_features, in_features // MXFP8_BLOCK, dtype=torch.uint8
+ )
+ self.bias = torch.empty(out_features) if has_bias else None
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return mxfp8_linear(x, self.weight, self.weight_scale_inv, self.bias)
+
+
+__all__ = ["FP8", "MXFP8_BLOCK", "Mxfp8Linear", "mxfp8_linear", "mxfp8_dequant"]
diff --git a/python/freetoken/kernel/triton/norm.py b/python/freetoken/kernel/triton/norm.py
index 179678b7..3f95c29f 100644
--- a/python/freetoken/kernel/triton/norm.py
+++ b/python/freetoken/kernel/triton/norm.py
@@ -48,6 +48,7 @@ def _rmsnorm_kernel(
CONTIG: tl.constexpr,
ENABLE_PDL: tl.constexpr,
BLOCK: tl.constexpr,
+ GEMMA: tl.constexpr = False,
):
a = tl.program_id(0)
b = tl.program_id(1)
@@ -65,6 +66,10 @@ def _rmsnorm_kernel(
w = tl.load(W + cols, mask=mask, other=0.0).to(tl.float32)
if ENABLE_PDL:
gdc_launch_dependents()
+ if GEMMA:
+ # Gemma semantics: checkpoint stores (scale - 1); apply (1 + w) in fp32
+ # (matches flashinfer's gemma_rmsnorm, which never rounds 1+w to bf16).
+ w = w + 1.0
inv = tl.math.rsqrt(tl.sum(x * x, axis=0) / H + eps)
y = x * inv * w
tl.store(OUT + ooff + cols, y.to(OUT.dtype.element_ty), mask=mask)
@@ -79,6 +84,7 @@ def _fused_add_rmsnorm_kernel(
CONTIG: tl.constexpr,
ENABLE_PDL: tl.constexpr,
BLOCK: tl.constexpr,
+ GEMMA: tl.constexpr = False,
):
a = tl.program_id(0)
b = tl.program_id(1)
@@ -97,6 +103,8 @@ def _fused_add_rmsnorm_kernel(
w = tl.load(W + cols, mask=mask, other=0.0).to(tl.float32)
if ENABLE_PDL:
gdc_launch_dependents()
+ if GEMMA:
+ w = w + 1.0
z = x + r
tl.store(R + roff + cols, z.to(R.dtype.element_ty), mask=mask)
inv = tl.math.rsqrt(tl.sum(z * z, axis=0) / H + eps)
@@ -112,7 +120,7 @@ def _leading(t):
return t.shape[0], t.shape[1], t.stride(0), t.stride(1)
-def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False):
+def _rmsnorm(input, weight, eps, out, gemma: bool):
import torch
assert weight.ndim == 1 and input.shape[-1] == weight.shape[0]
@@ -122,7 +130,7 @@ def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False
# 2D hidden norms or 3D per-head qk-norms). reshape may copy for a strided
# input, so this path is out-of-place only.
assert out is None, "rmsnorm on <2D/>3D input must be out-of-place"
- return rmsnorm(input.reshape(-1, H), weight, eps).reshape(input.shape)
+ return _rmsnorm(input.reshape(-1, H), weight, eps, None, gemma).reshape(input.shape)
assert input.stride(-1) == 1 # norm axis must be unit-stride (as flashinfer requires)
if out is None:
out = torch.empty_like(input)
@@ -136,13 +144,23 @@ def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False
pdl = contig and is_sm90_supported()
_rmsnorm_kernel[(A, B)](
out, input, weight, eps, H, sxa, sxb, soa, sob,
- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl,
+ CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,
num_warps=_num_warps(A * B), num_stages=1,
)
return out
-def fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bool = False):
+def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False):
+ return _rmsnorm(input, weight, eps, out, gemma=False)
+
+
+def gemma_rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False):
+ """(1 + w)-scaled RMSNorm (Gemma/MiniMax-M3 semantics: the checkpoint stores
+ ``scale - 1``); drop-in for flashinfer's ``norm.gemma_rmsnorm``."""
+ return _rmsnorm(input, weight, eps, out, gemma=True)
+
+
+def _fused_add_rmsnorm(input, residual, weight, eps, gemma: bool):
assert weight.ndim == 1 and input.shape[-1] == weight.shape[0]
assert input.shape == residual.shape
H = input.shape[-1]
@@ -154,9 +172,19 @@ def fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bo
pdl = contig and is_sm90_supported()
_fused_add_rmsnorm_kernel[(A, B)](
input, residual, weight, eps, H, sxa, sxb, sra, srb,
- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl,
+ CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,
num_warps=_num_warps(A * B), num_stages=1,
)
-__all__ = ["rmsnorm", "fused_add_rmsnorm"]
+def fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bool = False):
+ _fused_add_rmsnorm(input, residual, weight, eps, gemma=False)
+
+
+def gemma_fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bool = False):
+ """In-place ``residual += input; input = gemma_rmsnorm(residual)`` (Gemma (1+w)
+ scaling); drop-in for flashinfer's ``norm.gemma_fused_add_rmsnorm``."""
+ _fused_add_rmsnorm(input, residual, weight, eps, gemma=True)
+
+
+__all__ = ["rmsnorm", "fused_add_rmsnorm", "gemma_rmsnorm", "gemma_fused_add_rmsnorm"]
diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.py
index b0f167cb..1bb352b4 100644
--- a/python/freetoken/kvcache/__init__.py
+++ b/python/freetoken/kvcache/__init__.py
@@ -57,6 +57,10 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:
from .dsa_pool import MLAKVCache
return MLAKVCache
+ if AttnType.BSA in types:
+ from .bsa_pool import BSAKVCache
+
+ return BSAKVCache
from .mha_pool import MHAKVCache
return MHAKVCache
@@ -148,6 +152,28 @@ def create_kvcache_pool(
# spec carries indexer dims. The same spec fields drive the KV cost model, so the
# factory and the budget can never disagree.
kv_specs = model_config.kv_cache_group_specs()
+
+ # GQA block-sparse (MiniMax-M3): one full-attention group carrying the index dims
+ # with mla=False -> the MHA pool plus the index-key slab. The same spec fields
+ # drive the KV cost model, so the factory and the budget can never disagree.
+ from freetoken.attention import AttnType as _AttnType
+
+ if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA:
+ from .bsa_pool import BSAKVCache
+
+ spec = kv_specs[0]
+ return BSAKVCache(
+ num_kv_heads=spec.num_kv_heads,
+ num_layers=model_config.num_layers,
+ head_dim=spec.head_dim,
+ num_pages=num_pages,
+ page_size=page_size,
+ dtype=dtype,
+ device=device,
+ index_head_dim=spec.index_head_dim,
+ num_index_layers=spec.num_index_layers,
+ )
+
if len(kv_specs) == 1 and kv_specs[0].mla:
from .dsa_pool import DSAKVCache, MLAKVCache
diff --git a/python/freetoken/kvcache/bsa_pool.py b/python/freetoken/kvcache/bsa_pool.py
new file mode 100644
index 00000000..3eeb4b12
--- /dev/null
+++ b/python/freetoken/kvcache/bsa_pool.py
@@ -0,0 +1,126 @@
+"""GQA block-sparse (BSA) KV pool: paged GQA K/V + the indexer's index-key slab.
+
+MiniMax-M3's sparse layers attend only the top-k 128-token blocks a lightning
+indexer selects, so the pool is the MHA pool (standard paged GQA K/V, page_size
+== the sparse block size, one page per block) plus one bf16 index-key row per
+token per SPARSE layer (``index_head_dim`` wide; M3 has a single shared index
+key head). Slot order = the backend's sparse-layer order, same convention as
+``DSAKVCache``'s full-indexer slots.
+
+Storage lives here -- not in the attention backend -- so the engine's rebuild
+path (fresh allocation, object identity preserved, views re-derived per
+forward) and the KV cost model (which budgets the index-K bytes off the
+attention-group spec via ``spec_kv_bytes_per_token``) stay correct by
+construction. Both slabs resize atomically in ``rebuild`` so the allocator can
+never hand out a row one slab has and the other lacks (DSAKVCache precedent).
+"""
+
+from __future__ import annotations
+
+from typing import Sequence
+
+import torch
+
+from .mha_pool import MHAKVCache
+
+
+class BSAKVCache(MHAKVCache):
+ """MHA paged pool + the block-sparse index-key slab.
+
+ ``index_k_cache(slot)`` is row-flat ``[num_pages * page_size,
+ index_head_dim]`` addressed by the same physical token rows as the K/V
+ slabs (the shared page table is page_size=1 semantics), so the backend's
+ block addressing -- base row of block ``b`` = ``page_table[t, b *
+ page_size]`` -- serves K/V and index keys alike.
+ """
+
+ def __init__(
+ self,
+ num_kv_heads: int,
+ num_layers: int,
+ head_dim: int,
+ num_pages: int,
+ page_size: int,
+ dtype: torch.dtype,
+ device: torch.device,
+ index_head_dim: int,
+ num_index_layers: int,
+ layer_ids: Sequence[int] | None = None,
+ ) -> None:
+ self._index_head_dim = index_head_dim
+ self._num_index_layers = num_index_layers
+ self._page_size = page_size
+ # Index keys ride the compute dtype (bf16/fp16 -- the model's index_q/index_k
+ # are engine-dtype and store_index_k's index_put_ raises on a mismatch). The
+ # KV cost model budgets 2 bytes/token/index-layer for this slab
+ # (base.spec_kv_bytes_per_token); keep the two in lockstep.
+ assert dtype.itemsize == 2, (
+ f"BSA index slab budgets 2 bytes/token (spec_kv_bytes_per_token); got {dtype}"
+ )
+ self._index_dtype = dtype
+ super().__init__(
+ num_kv_heads=num_kv_heads,
+ num_layers=num_layers,
+ head_dim=head_dim,
+ num_pages=num_pages,
+ page_size=page_size,
+ dtype=dtype,
+ device=device,
+ layer_ids=layer_ids,
+ )
+ self._zero_kv_slabs()
+ self._alloc_index_slab(num_pages)
+
+ def _zero_kv_slabs(self) -> None:
+ # Defense-in-depth: the attend kernels pos-mask every K/V load (the real
+ # fix for torch.empty's recycled NaN/Inf bit patterns), but a zeroed slab
+ # keeps any future unmasked read finite instead of model-poisoning. One
+ # memset per (re)allocation -- negligible.
+ self._kv_buffer.zero_()
+
+ def _alloc_index_slab(self, num_pages: int) -> None:
+ # ZERO-initialized -- the index-score kernels load index keys unmasked and
+ # rely on unwritten tail rows dotting to a finite 0 (see the invariant
+ # comments in kernel/triton/minimax_m3_sparse.py).
+ self._index_k_buffer = torch.zeros(
+ self._num_index_layers,
+ num_pages * self._page_size,
+ self._index_head_dim,
+ dtype=self._index_dtype,
+ device=self._device,
+ )
+
+ def rebuild(self, num_pages: int) -> None:
+ # Free the index slab BEFORE the K/V realloc (super().rebuild frees + syncs +
+ # empty_cache), then re-derive it at the new page count. If the index-slab
+ # alloc itself fails (OOM), null the K/V slab too and re-raise: a pool with a
+ # grown K/V slab and no index slab would mis-serve silently, and the engine's
+ # rebuild path pre-validates the budget so this is already exceptional.
+ self._index_k_buffer = None
+ super().rebuild(num_pages)
+ self._zero_kv_slabs()
+ try:
+ self._alloc_index_slab(num_pages)
+ except Exception:
+ self._kv_buffer = None
+ self._k_buffer = None
+ self._v_buffer = None
+ raise
+
+ def unit_bytes(self) -> tuple[int, int]:
+ # The index slab rides the same token budget as the K/V slabs; each slab's
+ # per-token cost is floor-divided on its own, matching the cost model's terms.
+ kv, swa = super().unit_bytes()
+ idx = self._index_k_buffer
+ tokens = int(self._kv_buffer.shape[2]) * int(self._kv_buffer.shape[3])
+ return kv + int(idx.numel() * idx.element_size()) // tokens, swa
+
+ def index_k_cache(self, slot: int) -> torch.Tensor:
+ """Row-flat index keys for a sparse-layer slot: ``[rows, index_head_dim]``."""
+ return self._index_k_buffer[slot]
+
+ def store_index_k(self, k: torch.Tensor, out_loc: torch.Tensor, slot: int) -> None:
+ self._index_k_buffer[slot][out_loc] = k
+
+
+__all__ = ["BSAKVCache"]
diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.py
index fd0231c7..2dbd6373 100644
--- a/python/freetoken/layers/__init__.py
+++ b/python/freetoken/layers/__init__.py
@@ -1,4 +1,4 @@
-from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
+from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, swigluoai_and_mul
from .base import BaseOP, OPList, StateLessOP
from .embedding import ParallelLMHead, VocabParallelEmbedding
from .linear import (
@@ -9,13 +9,20 @@ from .linear import (
LinearRowParallel,
)
from .moe import MoELayer, OffloadMoELayer, make_moe_layer
-from .norm import GemmaRMSNorm, RMSNorm, RMSNormFused
+from .norm import (
+ GemmaPlusOneRMSNorm,
+ GemmaPlusOneRMSNormFused,
+ GemmaRMSNorm,
+ RMSNorm,
+ RMSNormFused,
+)
from .rotary import get_rope, set_rope_device
__all__ = [
"silu_and_mul",
"gelu_and_mul",
"gelu_tanh_and_mul",
+ "swigluoai_and_mul",
"BaseOP",
"StateLessOP",
"OPList",
@@ -28,6 +35,8 @@ __all__ = [
"RMSNorm",
"RMSNormFused",
"GemmaRMSNorm",
+ "GemmaPlusOneRMSNorm",
+ "GemmaPlusOneRMSNormFused",
"get_rope",
"set_rope_device",
"LinearReplicated",
diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.py
index 0a70db91..93602b6c 100644
--- a/python/freetoken/layers/activation.py
+++ b/python/freetoken/layers/activation.py
@@ -40,4 +40,20 @@ def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):
return gelu_tanh_and_mul(x, out=out)
-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul"]
+def swigluoai_and_mul(
+ x: torch.Tensor,
+ out: torch.Tensor | None = None,
+ *,
+ alpha: float = 1.702,
+ limit: float = 7.0,
+):
+ """SwiGLU-OAI (gpt-oss / MiniMax-M3 ``swigluoai``) over UNINTERLEAVED halves
+ (gate ``x[..., :d]``, up ``x[..., d:]``): ``clamp(gate, max=limit) *
+ sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``. Always the in-repo Triton
+ kernel (flashinfer ships no clamped-swiglu *_and_mul)."""
+ from freetoken.kernel.triton.activation import swigluoai_and_mul
+
+ return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)
+
+
+__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]
diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.py
index b3fa7012..d68d8ded 100644
--- a/python/freetoken/layers/moe.py
+++ b/python/freetoken/layers/moe.py
@@ -466,7 +466,13 @@ class OffloadMoELayer(MoELayer):
if fmt == "nvfp4":
# FreeToken's Triton inline-dequant kernels over the native ModelOpt
# rows: the FP4 banks are read directly in the GEMM, no BF16 copy of
- # the experts is ever materialized.
+ # the experts is ever materialized. The swigluoai scalars (MiniMax-M3)
+ # live on the layer via make_moe_layer's extra_attrs (gpt-oss precedent)
+ # and are ignored by the plain *_and_mul activations.
+ act_alpha = getattr(self, "hidden_act_alpha", 1.702)
+ act_limit = getattr(self, "swiglu_limit", None)
+ # None == "no clamp" everywhere else in the repo (mxfp4 maps it to +inf).
+ act_limit = float("inf") if act_limit is None else act_limit
if is_prefill:
from freetoken.moe.fused_nvfp4 import fused_experts_nvfp4
@@ -478,6 +484,8 @@ class OffloadMoELayer(MoELayer):
n,
self.activation,
self.apply_router_weight_on_input,
+ act_alpha,
+ act_limit,
)
# Marlin-style int32 wide-load GEMV (arithmetic dequant, no HW cvt).
# Bit-identical to the byte-at-a-time path; lifts gate/up BW ~43%->51%
@@ -492,6 +500,8 @@ class OffloadMoELayer(MoELayer):
topk_ids,
self.activation,
self.apply_router_weight_on_input,
+ act_alpha,
+ act_limit,
)
if fmt == "fp8_block":
# Block-fp8 experts: fused inline-dequant grouped GEMM reads the routed fp8 rows
diff --git a/python/freetoken/layers/norm.py b/python/freetoken/layers/norm.py
index 95f18e63..df248136 100644
--- a/python/freetoken/layers/norm.py
+++ b/python/freetoken/layers/norm.py
@@ -75,6 +75,78 @@ class GemmaRMSNorm(BaseOP):
return x, residual
+class GemmaPlusOneRMSNorm(BaseOP):
+ """(1 + w)-scaled RMSNorm (Gemma semantics: the checkpoint stores ``scale - 1``
+ and the effective multiplier is ``1 + weight``, added in fp32 at runtime --
+ never folded into the bf16 weight, which would round away the precision the
+ format exists to keep). Used by MiniMax-M3 (``use_gemma_norm``) for the decoder
+ layernorms, the per-head q/k norms, and the indexer q/k norms.
+
+ Per-head 3D inputs are collapsed to 2D before the kernel call: flashinfer's
+ ``gemma_rmsnorm`` CUDA binding is CHECK_DIM(2) (it rejects 3D outright on
+ wheels without the CuTe path), and the per-head weight makes the 2D view
+ exactly equivalent. The M3 call sites pass contiguous buffers, so the views
+ are free; a non-contiguous 3D input is rejected rather than silently copied
+ (an in-place norm on a copy would be dropped).
+ """
+
+ def __init__(self, size: int, eps: float) -> None:
+ from freetoken.kernel.backend import is_flashinfer_installed
+
+ if is_flashinfer_installed():
+ from flashinfer.norm import gemma_rmsnorm
+ else:
+ from freetoken.kernel.triton.norm import gemma_rmsnorm
+
+ self.eps = eps
+ self.size = size
+ self.weight = torch.empty(size)
+ self.gemma_rmsnorm = gemma_rmsnorm
+
+ def _flat(self, x: torch.Tensor) -> torch.Tensor:
+ if x.dim() == 2:
+ return x
+ assert x.is_contiguous(), "per-head gemma norm needs a contiguous buffer"
+ return x.view(-1, self.size)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.gemma_rmsnorm(self._flat(x), self.weight, self.eps).view(x.shape)
+
+ def forward_inplace(self, x: torch.Tensor) -> None:
+ flat = self._flat(x)
+ self.gemma_rmsnorm(flat, self.weight, self.eps, out=flat)
+
+
+class GemmaPlusOneRMSNormFused(BaseOP):
+ """(1 + w)-scaled RMSNorm with the fused-add-residual API of ``RMSNormFused``
+ (Gemma semantics, see :class:`GemmaPlusOneRMSNorm`). Drop-in for the decoder
+ layernorm seam: ``forward(x, residual)`` returns ``(normed, residual)``."""
+
+ def __init__(self, size: int, eps: float) -> None:
+ from freetoken.kernel.backend import is_flashinfer_installed
+
+ if is_flashinfer_installed():
+ from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm
+ else:
+ from freetoken.kernel.triton.norm import (
+ gemma_fused_add_rmsnorm,
+ gemma_rmsnorm,
+ )
+
+ self.eps = eps
+ self.weight = torch.empty(size)
+ self.gemma_rmsnorm = gemma_rmsnorm
+ self.gemma_fused_add_rmsnorm = gemma_fused_add_rmsnorm
+
+ def forward(
+ self, x: torch.Tensor, residual: torch.Tensor | None = None
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ if residual is None:
+ return self.gemma_rmsnorm(x, self.weight, self.eps), x
+ self.gemma_fused_add_rmsnorm(x, residual, self.weight, self.eps)
+ return x, residual
+
+
class RMSNormFused(BaseOP):
def __init__(self, size: int, eps: float) -> None:
from freetoken.kernel.backend import is_flashinfer_installed
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index ef34052f..3dadbdec 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -144,8 +144,11 @@ AttentionGroupConfig: TypeAlias = (
def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType:
- # Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache.
+ # Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache,
+ # GQA (non-mla) + index slab -> BSAKVCache (MiniMax-M3 block-sparse attention).
if not group.mla:
+ if group.index_head_dim > 0 and group.num_index_layers > 0:
+ return AttnType.BSA
return AttnType.FULL
if group.index_head_dim > 0 and group.num_index_layers > 0:
return AttnType.DSA
@@ -234,6 +237,11 @@ class ModelConfig:
# DSA indexer geometry the model module needs. Opaque to model-agnostic engine code;
# None for every other model.
glm_dsa_args: Any | None = None
+ # MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry
+ # (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the
+ # swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine
+ # code; None for every other model.
+ m3_args: Any | None = None
# Generic execution-path capability flags (set by a model's parse_config) so the engine and
# factories stay model-agnostic instead of branching on dsv4_args:
single_stream_only: bool = False # model runs one sequence at a time -> force bs=1
diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.py
new file mode 100644
index 00000000..71c0bb51
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/__init__.py
@@ -0,0 +1,15 @@
+from .config import parse_config
+from .model import MiniMaxM3ForCausalLM
+from .weight import (
+ iter_weights,
+ load_nvfp4_expert_sources,
+ load_nvfp4_expert_sources_parallel,
+)
+
+__all__ = [
+ "MiniMaxM3ForCausalLM",
+ "parse_config",
+ "iter_weights",
+ "load_nvfp4_expert_sources",
+ "load_nvfp4_expert_sources_parallel",
+]
diff --git a/python/freetoken/models/minimax_m3/args.py b/python/freetoken/models/minimax_m3/args.py
new file mode 100644
index 00000000..954dc0b5
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/args.py
@@ -0,0 +1,257 @@
+"""MiniMax-M3 (``minimax_m3``) hyperparameters.
+
+MiniMax-M3 is a 428B-A23B multimodal MoE (FreeToken serves the text tower; the
+``vision_tower.`` stack is skipped like every other checkpoint's). The text tower is
+GQA (64 q heads / 4 kv heads / head_dim 128) with per-head Gemma-style (1+w) q/k
+norms and partial NeoX RoPE (rotary_dim 64); the first ``sparse_attention_freq==0``
+layers run dense attention + a dense SwiGLU-OAI MLP, every later layer runs
+**block-sparse attention** (a lightning indexer scores 128-token blocks and each
+query attends only its top-``topk_blocks`` blocks, per KV head) + a sigmoid-routed
+MoE (128 NVFP4 experts, top-4, + 1 MXFP8 shared expert).
+
+This payload carries the sparse-indexer geometry and the swigluoai/dense-MLP scalars
+the model module and the ``m3_sparse`` attention backend need; it is stashed on
+``ModelConfig.m3_args`` (opaque to the engine). The MoE/router knobs live on
+``ModelConfig`` directly (shared with the glm4_moe/glm_moe_dsa path).
+
+Semantics pinned against the vLLM reference implementation
+(the vLLM tree's ``vllm/models/minimax_m3/``):
+
+* index heads == KV heads (4): each index head selects top-k blocks for ITS kv
+ head's whole GQA group -- there is no cross-head score reduction.
+* one shared index KEY head (``index_k_proj`` is ``[index_dim, hidden]``).
+* block score = max over the block's 128 positions of ``dot(index_q, index_k)``
+ (``sparse_score_type == "max"``), causal-masked, NO softmax scale (only the
+ ordering is consumed).
+* the newest ``local_blocks`` (1) blocks are force-selected; ``init_blocks`` (0)
+ leading blocks would be too. Selection always includes the query's own block.
+* index q/k get the same per-head Gemma norm + partial NeoX RoPE as the main q/k.
+* ``sparse_disable_index_value`` is 1 on every sparse layer: the indexer is
+ score-only (no index value/output projections exist in the checkpoint).
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Any, Tuple
+
+
+@dataclass(frozen=True)
+class MiniMaxM3Args:
+ hidden_size: int
+ num_heads: int
+ num_kv_heads: int
+ head_dim: int
+ norm_eps: float
+ # RoPE (partial NeoX; shared by the main q/k and the indexer q/k)
+ rotary_dim: int
+ rope_theta: float
+ max_position: int
+ # SwiGLU-OAI scalars (routed experts, shared experts and the dense MLPs)
+ swiglu_alpha: float
+ swiglu_limit: float
+ dense_intermediate_size: int
+ shared_intermediate_size: int
+ # Block-sparse attention (all zeros / empty when serving the dense ablation)
+ index_dim: int
+ num_index_heads: int
+ topk_blocks: int
+ block_size: int
+ init_blocks: int
+ local_blocks: int
+ sparse_layer_ids: Tuple[int, ...]
+ # Layers whose FFN is the sparse MoE block (the rest run the dense MLP)
+ moe_layer_ids: Tuple[int, ...]
+
+ @property
+ def use_sparse(self) -> bool:
+ return bool(self.sparse_layer_ids) and self.index_dim > 0 and self.topk_blocks > 0
+
+ def is_sparse_layer(self, layer_id: int) -> bool:
+ return self.use_sparse and layer_id in self.sparse_layer_ids
+
+ def sparse_slot(self, layer_id: int) -> int:
+ """Index-key slab slot for a sparse layer (slot order = sparse-layer order)."""
+ return self.sparse_layer_ids.index(layer_id)
+
+
+def _freq_ids(freq, num_layers: int) -> Tuple[int, ...]:
+ if not freq:
+ return tuple(range(num_layers))
+ return tuple(i for i, f in enumerate(freq[:num_layers]) if f)
+
+
+# Config-shape normalization. Two shapes reach load_args: raw checkpoint dicts
+# (the auto_map remote-code path keeps rope_theta / sparse_attention_config /
+# moe_layer_freq verbatim), and transformers >= 5's native minimax_m3_vl
+# TextConfig, whose __post_init__ moves rope_theta into rope_parameters, pops the
+# sparse config and moe_layer_freq into flat index_* keys + layer_types /
+# mlp_layer_types, and force-sets hidden_act to "silu". Every sibling family
+# carries the same dual-shape shim.
+def _rope_theta(text_config: Any) -> float:
+ theta = getattr(text_config, "rope_theta", None)
+ if theta is None:
+ params = getattr(text_config, "rope_parameters", None) or {}
+ theta = params.get("rope_theta") if isinstance(params, dict) else None
+ return float(theta) if theta is not None else 10000.0
+
+
+def _sparse_geometry(text_config: Any) -> dict:
+ """The sparse-attention geometry in the raw ``sparse_attention_config`` dict
+ shape, whichever config shape arrived. The native class has no flat keys for
+ init_block / score_type / disable_index_value -- it hardcodes those M3
+ semantics, so the translation pins the same values."""
+ raw = getattr(text_config, "sparse_attention_config", None)
+ if raw:
+ return dict(raw)
+ layer_types = getattr(text_config, "layer_types", None) or []
+ if "minimax_m3_sparse" not in layer_types:
+ return {}
+ return {
+ "use_sparse_attention": True,
+ "sparse_attention_freq": [
+ 1 if t == "minimax_m3_sparse" else 0 for t in layer_types
+ ],
+ "sparse_num_index_heads": int(getattr(text_config, "index_n_heads", 4)),
+ "sparse_index_dim": int(getattr(text_config, "index_head_dim", 128)),
+ "sparse_block_size": int(getattr(text_config, "index_block_size", 128)),
+ "sparse_topk_blocks": int(getattr(text_config, "index_topk_blocks", 16)),
+ "sparse_local_block": int(getattr(text_config, "index_local_blocks", 1)),
+ "sparse_init_block": 0,
+ "sparse_score_type": "max",
+ }
+
+
+def _moe_layer_freq(text_config: Any):
+ freq = getattr(text_config, "moe_layer_freq", None)
+ if freq is not None:
+ return freq
+ mlp_types = getattr(text_config, "mlp_layer_types", None)
+ if mlp_types is not None:
+ return [1 if t == "sparse" else 0 for t in mlp_types]
+ return None
+
+
+def load_args(text_config: Any, num_layers: int, *, sparse_enabled: bool) -> MiniMaxM3Args:
+ """Build the payload from the HF ``text_config`` (already unwrapped), resolving
+ the sparse-attention switch ONCE here (``sparse_enabled`` folds FREETOKEN_M3_SPARSE
+ in parse_config): the pool factory, the KV cost model, the backend and the model
+ modules all read the resolved payload / group spec, never the env."""
+ sparse_cfg = _sparse_geometry(text_config)
+ use_sparse = sparse_enabled and bool(sparse_cfg.get("use_sparse_attention", False))
+
+ head_dim_early = getattr(text_config, "head_dim", None) or (
+ text_config.hidden_size // text_config.num_attention_heads
+ )
+ if use_sparse:
+ score_type = sparse_cfg.get("sparse_score_type", "max")
+ assert score_type == "max", (
+ f"MiniMax-M3 sparse attention only implements sparse_score_type='max', "
+ f"got {score_type!r}"
+ )
+ sparse_layer_ids = _freq_ids(sparse_cfg.get("sparse_attention_freq"), num_layers)
+ # Score-only indexer: the checkpoint ships no index value/output projections.
+ disable_value = sparse_cfg.get("sparse_disable_index_value")
+ if disable_value is not None:
+ assert all(
+ bool(disable_value[i]) for i in sparse_layer_ids if i < len(disable_value)
+ ), "MiniMax-M3 sparse layers must be score-only (sparse_disable_index_value)"
+ # ---- geometry invariants the kernels/backend hardcode; fail a variant ----
+ # checkpoint at PARSE time instead of mis-addressing / mis-compiling at
+ # serve time.
+ blk = int(sparse_cfg.get("sparse_block_size", 128))
+ assert blk == 128, (
+ f"MiniMax-M3 kernels and the pinned KV page size assume 128-token "
+ f"sparse blocks, got sparse_block_size={blk}"
+ )
+ idx_dim = int(sparse_cfg.get("sparse_index_dim", 0))
+ assert idx_dim == head_dim_early, (
+ f"the indexer reuses the main rotary (rotary slices by head_dim); "
+ f"sparse_index_dim={idx_dim} != head_dim={head_dim_early}"
+ )
+ topk = int(sparse_cfg.get("sparse_topk_blocks", 0))
+ assert 0 < topk < 64, (
+ f"the decode top-k kernel's merge stage supports topk_blocks < 64, "
+ f"got {topk} (would fail Triton compile at serve time)"
+ )
+ local = int(sparse_cfg.get("sparse_local_block", 0))
+ assert local >= 1, (
+ "the attend kernels rely on the forced local block guaranteeing every "
+ f"query at least one visible position; got sparse_local_block={local}"
+ )
+ else:
+ sparse_layer_ids = ()
+
+ # swigluoai is implemented in four places (triton *_and_mul, fused_nvfp4, the
+ # CPU GEMV epilogue, eager) with the (up + 1) bias hardcoded; a checkpoint
+ # that changes swiglu_beta needs all four updated.
+ beta = float(getattr(text_config, "swiglu_beta", 1.0))
+ assert beta == 1.0, f"swigluoai implementations hardcode beta=1.0, got {beta}"
+ # Plain-rope only: silently ignoring a variant checkpoint's rope_scaling would
+ # mis-position every token past the scaling boundary. Newer HF configs spell
+ # the key `rope_parameters`; check both so a renamed config still trips this.
+ scaling = (getattr(text_config, "rope_scaling", None)
+ or getattr(text_config, "rope_parameters", None) or {})
+ rope_type = scaling.get("rope_type", scaling.get("type", "default")) if scaling else "default"
+ assert rope_type in (None, "default"), (
+ f"MiniMax-M3 support implements plain rope only, got rope_scaling={scaling!r}"
+ )
+ scoring = getattr(text_config, "scoring_func", "sigmoid")
+ assert scoring == "sigmoid", (
+ f"MiniMax-M3 routing implements sigmoid scoring only, got {scoring!r}"
+ )
+
+ assert not bool(getattr(text_config, "attention_output_gate", False)), (
+ "MiniMax-M3 attention_output_gate is not supported (M3 ships it disabled)"
+ )
+ assert getattr(text_config, "qk_norm_type", "per_head") == "per_head"
+ assert bool(getattr(text_config, "use_gemma_norm", True)), (
+ "MiniMax-M3 support assumes Gemma-style (1+w) RMSNorm (use_gemma_norm)"
+ )
+
+ head_dim = getattr(text_config, "head_dim", None) or (
+ text_config.hidden_size // text_config.num_attention_heads
+ )
+ rotary_dim = getattr(text_config, "rotary_dim", None)
+ if rotary_dim is None:
+ rotary_dim = int(head_dim * getattr(text_config, "partial_rotary_factor", 1.0))
+
+ num_index_heads = int(sparse_cfg.get("sparse_num_index_heads", 0)) if use_sparse else 0
+ if use_sparse:
+ # Per-KV-head selection: each index head feeds one kv head's GQA group.
+ assert num_index_heads == text_config.num_key_value_heads, (
+ f"expected sparse_num_index_heads == num_key_value_heads, got "
+ f"{num_index_heads} != {text_config.num_key_value_heads}"
+ )
+
+ return MiniMaxM3Args(
+ hidden_size=text_config.hidden_size,
+ num_heads=text_config.num_attention_heads,
+ num_kv_heads=text_config.num_key_value_heads,
+ head_dim=head_dim,
+ norm_eps=text_config.rms_norm_eps,
+ rotary_dim=rotary_dim,
+ rope_theta=_rope_theta(text_config),
+ max_position=text_config.max_position_embeddings,
+ swiglu_alpha=float(getattr(text_config, "swiglu_alpha", 1.702)),
+ swiglu_limit=float(getattr(text_config, "swiglu_limit", 7.0)),
+ dense_intermediate_size=int(
+ getattr(text_config, "dense_intermediate_size", 0)
+ or text_config.intermediate_size
+ ),
+ shared_intermediate_size=int(
+ getattr(text_config, "shared_intermediate_size", 0)
+ or text_config.intermediate_size
+ ),
+ index_dim=int(sparse_cfg.get("sparse_index_dim", 0)) if use_sparse else 0,
+ num_index_heads=num_index_heads,
+ topk_blocks=int(sparse_cfg.get("sparse_topk_blocks", 0)) if use_sparse else 0,
+ block_size=int(sparse_cfg.get("sparse_block_size", 128)) if use_sparse else 0,
+ init_blocks=int(sparse_cfg.get("sparse_init_block", 0)) if use_sparse else 0,
+ local_blocks=int(sparse_cfg.get("sparse_local_block", 0)) if use_sparse else 0,
+ sparse_layer_ids=sparse_layer_ids,
+ moe_layer_ids=_freq_ids(_moe_layer_freq(text_config), num_layers),
+ )
+
+
+__all__ = ["MiniMaxM3Args", "load_args"]
diff --git a/python/freetoken/models/minimax_m3/attention.py b/python/freetoken/models/minimax_m3/attention.py
new file mode 100644
index 00000000..fdf090b7
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/attention.py
@@ -0,0 +1,122 @@
+"""MiniMax-M3 attention: GQA with per-head Gemma q/k norms, partial NeoX RoPE, and
+(on the trailing layers) the block-sparse lightning-indexer branch.
+
+The indexer's weights live HERE (``index_qk_proj`` + per-head norms); scoring,
+top-k block selection and the sparse attend live in the ``m3_sparse`` backend
+(``attention/m3_sparse.py``), the same model/backend split as GLM's DSA. Sparse
+layers hand the backend this token's (index_q, index_k) via ``bsa_forward``; the
+backend caches the keys in the pool's index slab and selects per-KV-head top-k
+128-token blocks. The leading dense layers go through the generic
+``forward(q, k, v)`` contract, which the backend serves with a wrapped FULL
+backend -- and the FREETOKEN_M3_SPARSE=0 ablation builds no indexer at all, so
+every layer takes that path under a plain FULL backend.
+
+Index q/k get the SAME per-head Gemma (1+w) norm + partial NeoX rope as the main
+q/k (vLLM reference: ``index_rotary_emb = self.rotary_emb``); norms run before
+rope, per head, in fp32 (the fused reference kernel's order).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from freetoken.core import get_global_ctx
+from freetoken.distributed import get_tp_info
+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm
+from freetoken.layers.rotary import get_rope
+from freetoken.utils import nvtx_annotate
+
+from .mlp import make_proj
+
+if TYPE_CHECKING:
+ import torch
+
+ from freetoken.models.config import ModelConfig
+
+
+class MiniMaxM3Attention(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.m3_args
+ assert get_tp_info().size == 1, "MiniMax-M3 currently supports TP=1 only"
+ self.layer_id = layer_id
+ self.num_qo_heads = args.num_heads
+ self.num_kv_heads = args.num_kv_heads
+ self.head_dim = args.head_dim
+ self.qo_attn_dim = self.num_qo_heads * self.head_dim
+ self.kv_attn_dim = self.num_kv_heads * self.head_dim
+
+ quant = config.attn_quant
+ self.qkv_proj = make_proj(
+ quant, args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim
+ )
+ self.o_proj = make_proj(quant, self.qo_attn_dim, args.hidden_size)
+
+ # Per-head Gemma (1+w) q/k norms (qk_norm_type == "per_head").
+ self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=args.norm_eps)
+ self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=args.norm_eps)
+
+ # Partial NeoX rope (rotary_dim 64 of 128); one shared instance (get_rope
+ # caches) serves the main q/k AND the indexer q/k, like the reference.
+ self.rotary = get_rope(
+ head_dim=self.head_dim,
+ rotary_dim=args.rotary_dim,
+ max_position=args.max_position,
+ base=args.rope_theta,
+ )
+
+ self.is_sparse = args.is_sparse_layer(layer_id)
+ if self.is_sparse:
+ self.num_index_heads = args.num_index_heads
+ self.index_dim = args.index_dim
+ self.index_q_dim = self.num_index_heads * self.index_dim
+ # Merged [index_q | index_k] projection (one shared index KEY head).
+ self.index_qk_proj = make_proj(
+ quant, args.hidden_size, self.index_q_dim + self.index_dim
+ )
+ self.index_q_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)
+ self.index_k_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)
+
+ @nvtx_annotate("MHA")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ ctx = get_global_ctx()
+ qkv = self.qkv_proj.forward(x)
+ q, k, v = qkv.split([self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim], dim=-1)
+ # split() returns strided views into the merged qkv buffer; make each part
+ # contiguous (the norm kernel then runs in place on the copy) -- v must be
+ # contiguous for the KV-cache store kernel regardless.
+ q = q.contiguous()
+ k = k.contiguous()
+ v = v.contiguous()
+ self.q_norm.forward_inplace(q.view(-1, self.num_qo_heads, self.head_dim))
+ self.k_norm.forward_inplace(k.view(-1, self.num_kv_heads, self.head_dim))
+ q, k = self.rotary.forward(ctx.batch.positions, q, k)
+
+ if self.is_sparse:
+ iqk = self.index_qk_proj.forward(x)
+ del x
+ iq, ik = iqk.split([self.index_q_dim, self.index_dim], dim=-1)
+ iq = iq.contiguous()
+ ik = ik.contiguous()
+ self.index_q_norm.forward_inplace(
+ iq.view(-1, self.num_index_heads, self.index_dim)
+ )
+ self.index_k_norm.forward_inplace(ik.view(-1, 1, self.index_dim))
+ iq, ik = self.rotary.forward(ctx.batch.positions, iq, ik)
+ o = ctx.attn_backend.bsa_forward(
+ q.view(-1, self.num_qo_heads, self.head_dim),
+ k, v,
+ iq.view(-1, self.num_index_heads, self.index_dim),
+ ik,
+ self.layer_id,
+ ctx.batch,
+ )
+ else:
+ del x
+ o = ctx.attn_backend.forward(
+ q.view(-1, self.num_qo_heads, self.head_dim), k, v,
+ self.layer_id, ctx.batch,
+ )
+ return self.o_proj.forward(o.reshape(-1, self.qo_attn_dim))
+
+
+__all__ = ["MiniMaxM3Attention"]
diff --git a/python/freetoken/models/minimax_m3/config.py b/python/freetoken/models/minimax_m3/config.py
new file mode 100644
index 00000000..906870e3
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/config.py
@@ -0,0 +1,201 @@
+"""Engine-facing config for MiniMax-M3 (``minimax_m3``).
+
+The checkpoint is a multimodal wrapper (``model_type=minimax_m3_vl``): the text tower
+lives in ``text_config`` and the weights carry a ``language_model.`` prefix. FreeToken
+serves the text tower; the ViT vision stack (``vision_tower.`` / projector) is skipped
+at load like the other VL checkpoints' (``VISION_KEY_PREFIXES``) -- multimodal input
+is future work, so ``ModelConfig.vision_config`` stays None here.
+
+Attention is GQA with block-sparse selection on the trailing layers: parse_config
+declares ONE full-attention group over all layers carrying ``mla=False`` plus the
+index dims, which resolves to ``AttnType.BSA`` -- the pool factory builds
+``BSAKVCache`` (paged GQA K/V + the index-key slab), the KV cost model budgets the
+slab from the same spec, and the ``m3_sparse`` backend serves both the dense leading
+layers and the sparse layers. ``FREETOKEN_M3_SPARSE=0`` zeroes the index dims (dense
+ablation: plain MHAKVCache, every layer attends its whole history through a generic
+FULL backend). ``page_size`` is pinned to the 128-token sparse block by the backend's
+``page_sizes`` declaration, so one KV page == one sparse block.
+
+Resident-weight quantization is resolved HERE (from the FREETOKEN_M3_*_MXFP8 env
+switches, default on) into the standard ``ModelConfig`` fields -- ``attn_quant`` /
+``dense_quant`` = ``"mxfp8"`` or ``"none"`` -- and every consumer (the module
+constructors and the weight loader) reads those fields, so the resolved config is the
+single record of what the served weights actually are. The routed experts are NVFP4
+(same ModelOpt layout as MiniMax-M2 / GLM) and always live in the offload cache;
+their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernels
+(``select_nvfp4_backend``). lm_head / embeddings are BF16 in the checkpoint.
+"""
+
+from __future__ import annotations
+
+import os
+from typing import Any
+
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ ModelConfig,
+ RotaryConfig,
+)
+
+from .args import load_args
+
+def _text_config(hf_config: Any) -> Any:
+ return getattr(hf_config, "text_config", None) or hf_config
+
+
+# House rule: an env switch that changes WHAT IS SERVED must leave a server-log
+# trace. parse_config runs a few times per process (engine + weight loader), so
+# each resolved-mode line is logged once per distinct value.
+_LOGGED_MODES: set = set()
+
+
+def _log_mode_once(key: str, message: str) -> None:
+ if key in _LOGGED_MODES:
+ return
+ _LOGGED_MODES.add(key)
+ from freetoken.utils import init_logger
+
+ init_logger(__name__).info(message)
+
+
+def parse_config(hf_config: Any) -> ModelConfig:
+ text = _text_config(hf_config)
+
+ num_layers = text.num_hidden_layers
+ # Dev/testing only: cap the layer count (e.g. FREETOKEN_M3_MAX_LAYERS=5 -> 3 dense
+ # + 2 MoE layers) so the forward path / KV / offload cache can be exercised without
+ # pinning the full ~230 GB of experts. Unset in normal use (GLM-5.2 precedent).
+ _cap = os.environ.get("FREETOKEN_M3_MAX_LAYERS")
+ if _cap and int(_cap) < num_layers: # a cap >= the real count changes nothing
+ num_layers = int(_cap)
+ _log_mode_once(
+ f"cap={num_layers}",
+ f"FREETOKEN_M3_MAX_LAYERS: serving a TRUNCATED model "
+ f"({num_layers}/{text.num_hidden_layers} layers) -- dev/testing only, "
+ "outputs are garbage; unset the env for real serving.",
+ )
+
+ sparse_enabled = os.getenv("FREETOKEN_M3_SPARSE", "1") != "0"
+ if not sparse_enabled:
+ _log_mode_once(
+ "sparse=0",
+ "FREETOKEN_M3_SPARSE=0: serving the DENSE-attention ablation (plain "
+ "MHA pool, no indexer; different math and outputs from the shipped "
+ "block-sparse model).",
+ )
+ args = load_args(text, num_layers, sparse_enabled=sparse_enabled)
+
+ # W8A16 MXFP8 (the checkpoint's native dense quantization) for the resident
+ # weights; decode is weight-bandwidth bound and the freed VRAM densifies the
+ # expert cache. =0 dequantizes to bf16 at load (bring-up / ablation). Read at
+ # PARSE time so the resolved config is the single record of the served weights
+ # -- an FTW checkpoint converted under one setting must be served under the
+ # same one (see weight.py) -- and logged HERE so the FTW serve path (which
+ # never runs iter_weights) still leaves a serve-time record of the modes.
+ attn_mxfp8 = os.getenv("FREETOKEN_M3_ATTN_MXFP8", "1") != "0"
+ mlp_mxfp8 = os.getenv("FREETOKEN_M3_MLP_MXFP8", "1") != "0"
+ _log_mode_once(
+ f"quant={attn_mxfp8}/{mlp_mxfp8}",
+ f"MiniMax-M3 resident quant: attn={'mxfp8' if attn_mxfp8 else 'none'} "
+ f"dense={'mxfp8' if mlp_mxfp8 else 'none'} lm_head=none "
+ "(FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; an FTW checkpoint "
+ "converted under one setting must be served under the same one).",
+ )
+
+ # Leading dense-FFN layers: M3's moe_layer_freq is a contiguous 0-prefix; the
+ # offload cache and the generic num_moe_layers arithmetic rely on that shape.
+ first_k_dense = num_layers - len(args.moe_layer_ids)
+ assert args.moe_layer_ids == tuple(range(first_k_dense, num_layers)), (
+ "MiniMax-M3 expects a contiguous dense prefix in moe_layer_freq, got "
+ f"{args.moe_layer_ids}"
+ )
+
+ rotary_config = RotaryConfig(
+ head_dim=args.head_dim,
+ rotary_dim=args.rotary_dim,
+ max_position=args.max_position,
+ base=args.rope_theta,
+ scaling=None, # plain NeoX partial rope, no scaling
+ )
+
+ # Routed experts: NVFP4 per the modelopt MIXED_PRECISION quantized_layers map
+ # (checked structurally by the bank loader's shape/dtype asserts). The generic
+ # detect_expert_quant would report "mixed_precision", which names the checkpoint,
+ # not the experts -- so the expert format is pinned here (qwen3_5_moe precedent).
+ quant = getattr(hf_config, "quantization_config", None)
+ expert_quant = "nvfp4" if quant is not None else "none"
+
+ # The activation is model-constant swigluoai: assert instead of consume.
+ # Raw checkpoints declare "swigluoai"; the native transformers TextConfig
+ # force-normalizes hidden_act to "silu", which would send the backend
+ # selection down a silu-only path.
+ declared_act = getattr(text, "hidden_act", "swigluoai")
+ assert declared_act in ("swigluoai", "silu"), (
+ f"MiniMax-M3 support implements swigluoai only, got hidden_act={declared_act!r}"
+ )
+
+ return ModelConfig(
+ num_layers=num_layers,
+ num_qo_heads=args.num_heads,
+ num_kv_heads=args.num_kv_heads,
+ head_dim=args.head_dim,
+ hidden_size=args.hidden_size,
+ vocab_size=text.vocab_size,
+ intermediate_size=args.dense_intermediate_size,
+ hidden_act="swigluoai", # pinned; see the declared_act assert above
+ rms_norm_eps=text.rms_norm_eps,
+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
+ rotary_config=rotary_config,
+ # One group over all layers: GQA K/V everywhere, plus the index-key slab for
+ # the sparse layers (mla=False + index dims -> AttnType.BSA -> BSAKVCache).
+ # The same spec drives the pool factory and the KV cost model, so they can
+ # never disagree. FREETOKEN_M3_SPARSE=0 zeroes the index dims (dense ablation:
+ # plain MHAKVCache, generic FULL backends serve every layer).
+ attention_groups=(
+ FullAttentionGroupConfig(
+ name="full",
+ layer_ids=tuple(range(num_layers)),
+ num_kv_heads=args.num_kv_heads,
+ head_dim=args.head_dim,
+ rotary_config=rotary_config,
+ mla=False,
+ index_head_dim=args.index_dim if args.use_sparse else 0,
+ num_index_layers=(
+ sum(1 for lid in args.sparse_layer_ids if lid < num_layers)
+ if args.use_sparse
+ else 0
+ ),
+ ),
+ ),
+ num_experts=int(getattr(text, "num_local_experts", 0)),
+ num_experts_per_tok=text.num_experts_per_tok,
+ # Routed experts use the text tower's intermediate_size (3072); the dense
+ # MLP width (12288) rides ModelConfig.intermediate_size above.
+ moe_intermediate_size=text.intermediate_size,
+ norm_topk_prob=True, # MiniMax always renormalizes the selected expert weights
+ model_type=getattr(hf_config, "model_type", "minimax_m3_vl"),
+ architectures=getattr(
+ hf_config, "architectures", ["MiniMaxM3SparseForConditionalGeneration"]
+ ),
+ moe_enabled=True,
+ expert_quant=expert_quant,
+ first_k_dense_replace=first_k_dense,
+ n_shared_experts=int(getattr(text, "n_shared_experts", 0)),
+ shared_expert_intermediate_size=args.shared_intermediate_size,
+ routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),
+ has_router_bias=bool(getattr(text, "use_routing_bias", False)),
+ use_qk_norm=bool(getattr(text, "use_qk_norm", True)),
+ swiglu_limit=args.swiglu_limit,
+ hidden_act_alpha=args.swiglu_alpha,
+ # Resident-weight quant modes (see the module docstring): recorded here so the
+ # resolved config describes the served weights; the constructors and
+ # iter_weights read these fields, never the env directly. lm_head stays bf16
+ # (the checkpoint excludes it from quantization).
+ attn_quant="mxfp8" if attn_mxfp8 else "none",
+ dense_quant="mxfp8" if mlp_mxfp8 else "none",
+ lm_head_quant="none",
+ m3_args=args,
+ )
+
+
+__all__ = ["parse_config"]
diff --git a/python/freetoken/models/minimax_m3/mlp.py b/python/freetoken/models/minimax_m3/mlp.py
new file mode 100644
index 00000000..b08f6935
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/mlp.py
@@ -0,0 +1,62 @@
+"""SwiGLU-OAI MLP for MiniMax-M3's leading dense layers and per-layer shared experts.
+
+The checkpoint ships these MXFP8 (fp8-e4m3 weight + uint8 e8m0 block-32
+``weight_scale_inv``); serving default keeps them native W8A16
+(``ModelConfig.dense_quant == "mxfp8"``, from FREETOKEN_M3_MLP_MXFP8) -- decode reads
+every shared expert each token, so this halves the resident-MLP weight traffic and
+frees the same VRAM for expert-cache slots. ``FREETOKEN_M3_MLP_MXFP8=0`` dequantizes
+to bf16 at load (bring-up / ablation).
+
+gate/up are stored merged (``gate_up_proj``, [gate; up] halves -- the loader
+concatenates the two checkpoint projections output-wise; MXFP8 scales are per-output-
+row so the fusion is exact), which feeds the same uninterleaved ``swigluoai_and_mul``
+kernel the NVFP4 expert path uses:
+``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from freetoken.layers import BaseOP, LinearReplicated, swigluoai_and_mul
+from freetoken.utils import nvtx_annotate
+
+if TYPE_CHECKING:
+ import torch
+
+
+def make_proj(quant: str, in_features: int, out_features: int) -> BaseOP:
+ """A resident projection in the model's resolved quant mode: ``"mxfp8"`` (W8A16,
+ the checkpoint's native block-32 e8m0 format -- see weight.py) or bf16."""
+ if quant == "mxfp8":
+ from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear
+
+ return Mxfp8Linear(in_features, out_features, has_bias=False)
+ return LinearReplicated(in_features, out_features, has_bias=False)
+
+
+class MiniMaxM3MLP(BaseOP):
+ def __init__(
+ self,
+ hidden_size: int,
+ intermediate_size: int,
+ *,
+ quant: str,
+ alpha: float,
+ limit: float,
+ ):
+ self.gate_up_proj = make_proj(quant, hidden_size, 2 * intermediate_size)
+ self.down_proj = make_proj(quant, intermediate_size, hidden_size)
+ self._alpha = alpha
+ self._limit = limit
+
+ @nvtx_annotate("MLP")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ gate_up = self.gate_up_proj.forward(x)
+ del x
+ y = swigluoai_and_mul(gate_up, alpha=self._alpha, limit=self._limit)
+ del gate_up
+ return self.down_proj.forward(y)
+
+
+__all__ = ["MiniMaxM3MLP", "make_proj"]
diff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.py
new file mode 100644
index 00000000..86c47328
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/model.py
@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.layers import (
+ BaseOP,
+ GemmaPlusOneRMSNormFused,
+ OPList,
+ ParallelLMHead,
+ VocabParallelEmbedding,
+)
+from freetoken.models.blocks import BaseLLMModel
+from freetoken.utils import nvtx_annotate
+
+from .attention import MiniMaxM3Attention
+from .mlp import MiniMaxM3MLP
+from .moe import MiniMaxM3SparseMoeBlock
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class MiniMaxM3DecoderLayer(BaseOP):
+ """Dense layers store the FFN under ``mlp``; MoE layers under
+ ``block_sparse_moe`` -- matching the checkpoint's naming. All layernorms are
+ Gemma-style (1+w) RMSNorm (``use_gemma_norm``)."""
+
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.m3_args
+ self.self_attn = MiniMaxM3Attention(config, layer_id)
+ self.is_moe_layer = layer_id in args.moe_layer_ids
+ if self.is_moe_layer:
+ self.block_sparse_moe: BaseOP = MiniMaxM3SparseMoeBlock(config, layer_id)
+ else:
+ self.mlp = MiniMaxM3MLP(
+ config.hidden_size,
+ args.dense_intermediate_size,
+ quant=config.dense_quant,
+ alpha=args.swiglu_alpha,
+ limit=args.swiglu_limit,
+ )
+ self.input_layernorm = GemmaPlusOneRMSNormFused(
+ size=config.hidden_size, eps=config.rms_norm_eps
+ )
+ self.post_attention_layernorm = GemmaPlusOneRMSNormFused(
+ size=config.hidden_size, eps=config.rms_norm_eps
+ )
+ self._layer_id = layer_id
+
+ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
+ def forward(
+ self, x: torch.Tensor, residual: torch.Tensor | None = None
+ ) -> Tuple[torch.Tensor, torch.Tensor]:
+ x, residual = self.input_layernorm.forward(x, residual)
+ x = self.self_attn.forward(x)
+ x, residual = self.post_attention_layernorm.forward(x, residual)
+ ffn = self.block_sparse_moe if self.is_moe_layer else self.mlp
+ x = ffn.forward(x)
+ return x, residual
+
+
+class MiniMaxM3Model(BaseOP):
+ def __init__(self, config: ModelConfig):
+ self.embed_tokens = VocabParallelEmbedding(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ )
+ self.layers = OPList(
+ [MiniMaxM3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ )
+ self.norm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)
+
+ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+ x = self.embed_tokens.forward(input_ids)
+ residual: torch.Tensor | None = None
+ for layer in self.layers.op_list:
+ x, residual = layer.forward(x, residual)
+ return self.norm.forward(x, residual)[0]
+
+
+class MiniMaxM3ForCausalLM(BaseLLMModel):
+ def __init__(self, config: ModelConfig):
+ self.model = MiniMaxM3Model(config)
+ # lm_head is BF16 in the checkpoint (excluded from quantization), untied.
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ )
+ super().__init__()
+
+ def forward(self) -> torch.Tensor:
+ output = self.model.forward(get_global_ctx().batch.input_ids)
+ return self.lm_head.forward(output)
+
+
+__all__ = ["MiniMaxM3ForCausalLM"]
diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.py
new file mode 100644
index 00000000..9f54c156
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/moe.py
@@ -0,0 +1,92 @@
+"""MiniMax-M3 sparse MoE block.
+
+Routing follows the DeepSeek/GLM family (``GlmMoeDsaSparseBlock``): fp32 sigmoid
+scores + a selection-only ``e_score_correction_bias``, gather the unbiased scores,
+renormalize, scale by ``routed_scaling_factor`` (2.0). No group-limited top-k
+(n_group == 1). The routed experts (NVFP4, swigluoai) go through ``make_moe_layer``
+(the offload family; the swigluoai alpha/limit ride the layer via ``extra_attrs``,
+gpt-oss precedent) and the always-on shared expert (MXFP8, swigluoai) is added.
+
+The checkpoint keeps ``e_score_correction_bias`` fp32 and the reference applies it
+fp32; the bias sits on the top-k boundary, so it is stored fp32 here too (unlike
+GLM's bf16 storage, whose checkpoint ships it fp32 but consumes it off a bf16 cast).
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Tuple
+
+import torch
+import torch.nn.functional as F
+from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
+
+from .mlp import MiniMaxM3MLP
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+TopK = Tuple[torch.Tensor, torch.Tensor]
+
+
+class MiniMaxM3SparseMoeBlock(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int):
+ args = config.m3_args
+ self.top_k = config.num_experts_per_tok
+ self.num_experts = config.num_experts
+ self.norm_topk_prob = config.norm_topk_prob
+ self.routed_scaling_factor = config.routed_scaling_factor
+
+ # Router weights kept fp32 (vLLM stores them fp32 too): both the gate and
+ # the bias sit on the top-4 selection boundary, where a bf16 round can
+ # flip near-tie picks away from the reference. 128 x 6144 x 4 B = ~3.1 MB
+ # per layer, ~180 MB across the 57 MoE layers -- accepted for selection
+ # fidelity.
+ self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
+ self.gate.weight = torch.empty(
+ config.num_experts, config.hidden_size, dtype=torch.float32
+ )
+ # DeepSeek-style selection bias; fp32 in the checkpoint AND here.
+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)
+
+ # The offload cache indexes experts by *MoE* layer (global layer minus
+ # first_k_dense_replace), matching how the loader packs the expert banks.
+ self.experts = make_moe_layer(
+ config,
+ layer_id=layer_id - config.first_k_dense_replace,
+ renormalize=config.norm_topk_prob,
+ activation="swigluoai",
+ extra_attrs={
+ "hidden_act_alpha": args.swiglu_alpha,
+ "swiglu_limit": args.swiglu_limit,
+ },
+ )
+ self.shared_experts = MiniMaxM3MLP(
+ config.hidden_size,
+ config.shared_expert_intermediate_size * max(1, config.n_shared_experts),
+ quant=config.dense_quant,
+ alpha=args.swiglu_alpha,
+ limit=args.swiglu_limit,
+ )
+
+ def _route(self, hidden_states: torch.Tensor) -> TopK:
+ # HF/vLLM compute the router logits in fp32; the gate is stored fp32.
+ logits = F.linear(hidden_states.float(), self.gate.weight)
+ scores = logits.sigmoid()
+ scores_for_choice = scores + self.e_score_correction_bias
+ _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)
+ topk_weights = scores.gather(-1, topk_ids)
+ if self.norm_topk_prob:
+ topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)
+ topk_weights = topk_weights * self.routed_scaling_factor
+ return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()
+
+ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
+ num_tokens, hidden_dim = hidden_states.shape
+ hidden_states = hidden_states.view(-1, hidden_dim)
+ topk_weights, topk_ids = self._route(hidden_states)
+ out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)
+ out = out + self.shared_experts.forward(hidden_states)
+ return out.view(num_tokens, hidden_dim)
+
+
+__all__ = ["MiniMaxM3SparseMoeBlock"]
diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.py
new file mode 100644
index 00000000..e10a7237
--- /dev/null
+++ b/python/freetoken/models/minimax_m3/weight.py
@@ -0,0 +1,283 @@
+"""Weight loading for MiniMax-M3 (``minimax_m3``).
+
+The checkpoint is the multimodal wrapper layout: every text-tower tensor carries a
+``language_model.`` prefix (``language_model.model.layers.N...``,
+``language_model.lm_head.weight``); the ViT stack (``vision_tower.`` /
+``multi_modal_projector.`` / ``patch_merge_mlp.``) is never read (text-only serving).
+
+Resident (non routed-expert) dense projections are MXFP8 in the checkpoint
+(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``). What this loader
+yields follows the quant modes RESOLVED IN ``parse_config`` (``ModelConfig.attn_quant``
+/ ``dense_quant``, from the FREETOKEN_M3_*_MXFP8 switches, default on): in the default
+mode the fp8 weight and its scale codes stream through verbatim (merged output-wise for
+the fused qkv / index-qk / gate-up projections -- the scales are per-output-row, so
+fusion is exact); with a switch off the projections are dequantized to bf16 at load.
+Norms, the router gate, ``e_score_correction_bias`` (kept fp32), embeddings and
+lm_head are unquantized and stream through verbatim. Routed experts are NVFP4
+(``w1/w3/w2`` = gate/up/down, same ModelOpt layout as MiniMax-M2) and go to the
+offload cache via ``load_nvfp4_expert_sources``; expert ``input_scale`` calibration
+tensors are unused (W4A16) and never match the bank spec.
+
+FTW caveat: an FTW checkpoint stores whatever iter_weights yielded at CONVERSION time,
+and the model is built from the env at SERVE time -- the two must agree (a mismatch
+fails loudly in load_state_dict on the ``*.weight_scale_inv`` keys). The active modes
+are logged at load so conversion logs record the choice.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+from typing import Iterator
+
+import safetensors
+import torch
+from freetoken.distributed import get_tp_info
+from freetoken.models.loader import drop_page_cache
+from freetoken.models.nvfp4_banks import (
+ Nvfp4ExpertSourceSpec,
+ load_nvfp4_expert_source_banks,
+)
+from freetoken.utils import cached_load_hf_config, download_hf_weight
+from tqdm import tqdm
+
+from .config import parse_config
+
+_EXPERT_KEY_RE = re.compile(
+ r"^language_model\.model\.layers\.(?P<layer>\d+)\.block_sparse_moe\."
+ r"experts\.(?P<expert>\d+)\.(?P<proj>w1|w2|w3)\."
+ r"(?P<kind>weight|weight_scale|weight_scale_2)$"
+)
+_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(
+ key_pattern=_EXPERT_KEY_RE,
+ proj_to_role={"w1": "gate", "w3": "up", "w2": "down"},
+ # Experts exist only for layers [first_k_dense_replace, num_layers); banks pack by
+ # MoE-layer index so the leading dense layers leave no holes (GLM precedent).
+ layer_to_bank=lambda layer, config: (
+ None
+ if layer < config.first_k_dense_replace or layer >= config.num_layers
+ else layer - config.first_k_dense_replace
+ ),
+ desc="MiniMax-M3 NVFP4 experts",
+)
+
+
+class _ShardReader:
+ """Random-access reader over the safetensors index (GLM-5.2 precedent): the M3
+ dense pass walks layers in model order and fuses q/k/v (etc.) output-wise, which
+ a shard-ordered stream cannot do without buffering whole projections anyway."""
+
+ def __init__(self, folder: str, weight_map: dict, device: torch.device):
+ self._folder = folder
+ self._weight_map = weight_map
+ self._device = device
+ self._handles: dict[str, object] = {}
+
+ def has(self, name: str) -> bool:
+ return name in self._weight_map
+
+ def get(self, name: str) -> torch.Tensor:
+ shard = self._weight_map[name]
+ handle = self._handles.get(shard)
+ if handle is None:
+ handle = safetensors.safe_open(
+ os.path.join(self._folder, shard), framework="pt", device=str(self._device)
+ ).__enter__()
+ self._handles[shard] = handle
+ return handle.get_tensor(name)
+
+ def close(self) -> None:
+ for shard, handle in self._handles.items():
+ try:
+ handle.__exit__(None, None, None)
+ except Exception: # pragma: no cover - best effort
+ pass
+ drop_page_cache(os.path.join(self._folder, shard))
+ self._handles.clear()
+
+
+def _read_proj(reader: _ShardReader, key: str) -> tuple[torch.Tensor, torch.Tensor | None]:
+ """One dense projection as ``(weight, scale_codes | None)``. MXFP8 tensors carry a
+ ``weight_scale_inv`` sibling; a bf16 tensor (nothing quantized under this name)
+ comes back scale-less and passes through every mode unchanged."""
+ w = reader.get(f"{key}.weight")
+ scale_key = f"{key}.weight_scale_inv"
+ if reader.has(scale_key):
+ s = reader.get(scale_key)
+ assert w.dtype == torch.float8_e4m3fn and s.dtype == torch.uint8, (
+ f"unexpected MXFP8 dtypes at {key}: {w.dtype}/{s.dtype}"
+ )
+ # The GEMV folds w * 2**(code-127) into the 16-bit compute dtype, which
+ # is lossless only while the product stays in range; codes above 245
+ # could overflow bf16. Real checkpoints sit near 127 -- pin at load.
+ assert int(s.max()) <= 245, (
+ f"e8m0 scale code {int(s.max())} at {key} exceeds the bf16-exact "
+ "fold bound (245); the W8A16 kernels would overflow"
+ )
+ return w, s
+ return w, None
+
+
+def _emit_fused(
+ reader: _ShardReader,
+ out_key: str,
+ part_keys: list[str],
+ keep_mxfp8: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """Emit one (possibly multi-part output-wise fused) projection in the resolved
+ quant mode. MXFP8 scales are per-output-row / per-32-input-block, so concatenating
+ the parts' weights AND scales along the output dim is exact."""
+ parts = [_read_proj(reader, key) for key in part_keys]
+ if keep_mxfp8 and all(s is not None for _, s in parts):
+ yield f"{out_key}.weight", torch.cat([w for w, _ in parts], dim=0)
+ yield f"{out_key}.weight_scale_inv", torch.cat([s for _, s in parts], dim=0)
+ return
+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant
+
+ dequant = [
+ mxfp8_dequant(w, s, dtype=torch.bfloat16) if s is not None else w
+ for w, s in parts
+ ]
+ yield f"{out_key}.weight", (
+ torch.cat(dequant, dim=0) if len(dequant) > 1 else dequant[0]
+ )
+
+
+def iter_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ assert not include_moe_experts, (
+ "MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "
+ "backend; experts are loaded into the offload cache via load_nvfp4_expert_sources()."
+ )
+ assert include_non_moe
+ config = parse_config(cached_load_hf_config(model_path))
+ args = config.m3_args
+ folder = download_hf_weight(model_path)
+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:
+ weight_map = json.load(f)["weight_map"]
+ reader = _ShardReader(folder, weight_map, device)
+ primary = get_tp_info().is_primary()
+ attn_mx = config.attn_quant == "mxfp8"
+ mlp_mx = config.dense_quant == "mxfp8"
+ if primary:
+ from freetoken.utils import init_logger
+
+ init_logger(__name__).info(
+ f"MiniMax-M3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "
+ f"lm_head={config.lm_head_quant} (FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; "
+ "an FTW conversion records these choices implicitly -- serve with the same flags)"
+ )
+ try:
+ for layer in tqdm(
+ range(config.num_layers),
+ desc="Loading MiniMax-M3 dense weights",
+ disable=not primary,
+ ):
+ src = f"language_model.model.layers.{layer}"
+ dst = f"model.layers.{layer}"
+ a_src, a_dst = f"{src}.self_attn", f"{dst}.self_attn"
+
+ yield from _emit_fused(
+ reader,
+ f"{a_dst}.qkv_proj",
+ [f"{a_src}.q_proj", f"{a_src}.k_proj", f"{a_src}.v_proj"],
+ attn_mx,
+ )
+ yield from _emit_fused(reader, f"{a_dst}.o_proj", [f"{a_src}.o_proj"], attn_mx)
+ for norm in ("q_norm", "k_norm"):
+ yield f"{a_dst}.{norm}.weight", reader.get(f"{a_src}.{norm}.weight")
+
+ # Block-sparse indexer (sparse layers only; the dense ablation builds no
+ # indexer modules, so its tensors are skipped entirely).
+ if args.is_sparse_layer(layer):
+ yield from _emit_fused(
+ reader,
+ f"{a_dst}.index_qk_proj",
+ [f"{a_src}.index_q_proj", f"{a_src}.index_k_proj"],
+ attn_mx,
+ )
+ for norm in ("index_q_norm", "index_k_norm"):
+ yield f"{a_dst}.{norm}.weight", reader.get(f"{a_src}.{norm}.weight")
+
+ for norm in ("input_layernorm", "post_attention_layernorm"):
+ yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight")
+
+ if layer in args.moe_layer_ids:
+ m_src, m_dst = f"{src}.block_sparse_moe", f"{dst}.block_sparse_moe"
+ # fp32 like the bias: both sit on the top-4 selection boundary.
+ yield (
+ f"{m_dst}.gate.weight",
+ reader.get(f"{m_src}.gate.weight").to(torch.float32),
+ )
+ # fp32 in the checkpoint AND the module (top-k selection boundary).
+ yield (
+ f"{m_dst}.e_score_correction_bias",
+ reader.get(f"{m_src}.e_score_correction_bias").to(torch.float32),
+ )
+ s_src, s_dst = f"{m_src}.shared_experts", f"{m_dst}.shared_experts"
+ yield from _emit_fused(
+ reader,
+ f"{s_dst}.gate_up_proj",
+ [f"{s_src}.gate_proj", f"{s_src}.up_proj"],
+ mlp_mx,
+ )
+ yield from _emit_fused(
+ reader, f"{s_dst}.down_proj", [f"{s_src}.down_proj"], mlp_mx
+ )
+ else:
+ yield from _emit_fused(
+ reader,
+ f"{dst}.mlp.gate_up_proj",
+ [f"{src}.mlp.gate_proj", f"{src}.mlp.up_proj"],
+ mlp_mx,
+ )
+ yield from _emit_fused(
+ reader, f"{dst}.mlp.down_proj", [f"{src}.mlp.down_proj"], mlp_mx
+ )
+
+ yield "model.embed_tokens.weight", reader.get("language_model.model.embed_tokens.weight")
+ yield "model.norm.weight", reader.get("language_model.model.norm.weight")
+ yield "lm_head.weight", reader.get("language_model.lm_head.weight")
+ finally:
+ reader.close()
+
+
+def load_nvfp4_expert_sources(
+ model_path: str, config, *, layer_sink=None
+) -> dict[str, list[torch.Tensor]]:
+ """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""
+ return load_nvfp4_expert_source_banks(
+ model_path,
+ config,
+ _NVFP4_SOURCE_SPEC,
+ drop_page_cache=drop_page_cache,
+ primary=get_tp_info().is_primary(),
+ layer_sink=layer_sink,
+ )
+
+
+def load_nvfp4_expert_sources_parallel(
+ model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None
+):
+ """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""
+ from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel
+
+ return load_nvfp4_expert_source_banks_parallel(
+ model_path,
+ config,
+ _NVFP4_SOURCE_SPEC,
+ drop_page_cache=drop_page_cache,
+ primary=get_tp_info().is_primary(),
+ workers=workers,
+ chunk=chunk,
+ layer_sink=layer_sink,
+ )
+
+
+__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 00ff1412..1e015187 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -37,6 +37,19 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"freetoken.models.minimax_m2",
"MiniMaxM2ForCausalLM",
),
+ # MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in
+ # text_config, weights under language_model.); served text-only. GQA + block-sparse
+ # attention (lightning indexer, top-k 128-token blocks) on the trailing layers,
+ # sigmoid/bias-routed NVFP4 experts + MXFP8 shared expert, swigluoai activation.
+ "MiniMaxM3SparseForConditionalGeneration": ModelSpec(
+ "freetoken.models.minimax_m3",
+ "MiniMaxM3ForCausalLM",
+ ),
+ # Text-only sibling (the text_config's own architectures entry).
+ "MiniMaxM3SparseForCausalLM": ModelSpec(
+ "freetoken.models.minimax_m3",
+ "MiniMaxM3ForCausalLM",
+ ),
"DeepseekV4ForCausalLM": ModelSpec(
"freetoken.models.deepseek_v4",
"DeepseekV4ForCausalLM",
diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.py
index 6adf194f..b96205aa 100644
--- a/python/freetoken/moe/cpu_executor.py
+++ b/python/freetoken/moe/cpu_executor.py
@@ -54,9 +54,9 @@ _FLAG_SYNC = os.getenv("FREETOKEN_CPU_MOE_FLAG_SYNC", "1") != "0"
# just keeps the host-func path for the extra combos.
_FLAG_SLOTS_PER_LAYER = 16
-# Activation ids must match ActKind in csrc/cpu_moe/cpu_moe_ext.cpp. "gpt_oss_swiglu"
-# is handled entirely inside the mxfp4 kernel (clamped swiglu + bias), so its id is a
-# placeholder that act_apply never sees.
+# Activation ids must match ActKind in csrc/cpu_moe/cpu_moe_ext.cpp. Id 3 is the
+# clamped (up + 1) swiglu: "swigluoai" runs it in the generic GEMV epilogue,
+# "gpt_oss_swiglu" is the same math fused inside the mxfp4 kernel.
_ACT_IDS = {
"silu": 0,
"swish": 0,
@@ -64,12 +64,30 @@ _ACT_IDS = {
"gelu_tanh": 2,
"gelu_pytorch_tanh": 2,
"gpt_oss_swiglu": 3,
+ "swigluoai": 3,
}
# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp.
_WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4}
+def compiled_extension_supports(activation: str) -> bool:
+ """Whether the compiled ``_cpu_moe`` extension can serve ``activation``
+ through its generic epilogue. A stale prebuilt .so accepts newer act ids
+ while silently computing the wrong math; the executor hard-errors on that,
+ but the engine's auto offload->hybrid upgrade consults this first so a
+ default boot degrades to offload instead of crashing after weight load."""
+ if activation not in _ACT_IDS:
+ return False
+ if _ACT_IDS[activation] < 3:
+ return True
+ try:
+ from freetoken.kernel import _cpu_moe
+ except ImportError:
+ return False
+ return _ACT_IDS[activation] <= getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()
+
+
def physical_core_cpus() -> list[int]:
"""One logical CPU per physical core, restricted to this process's affinity.
@@ -150,6 +168,19 @@ class CpuMoeExecutor:
)
if activation not in _ACT_IDS:
raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")
+ # ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without
+ # error and silently computes the wrong activation in the generic
+ # epilogue -- fail loudly with the rebuild instruction instead. (mxfp4
+ # handles its act inside the kernel and predates the marker.)
+ if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":
+ supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()
+ if _ACT_IDS[activation] > supported:
+ raise RuntimeError(
+ f"the compiled _cpu_moe extension predates activation "
+ f"{activation!r} (max generic act id {supported}); rebuild it "
+ "with `python setup.py build_ext --inplace` (or reinstall the "
+ "wheel) before serving this model on the cpu/hybrid backend."
+ )
self.num_layers = int(cache.num_layers)
self.num_experts = int(cache.num_experts)
diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.py
index d3e6df99..e456c931 100644
--- a/python/freetoken/moe/expert_banks.py
+++ b/python/freetoken/moe/expert_banks.py
@@ -178,7 +178,8 @@ def _nvfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False,
backend = None
if not native:
backend = select_nvfp4_backend(device, getattr(model_config, "moe_intermediate_size", None),
- getattr(model_config, "nvfp4_backend", "auto"))
+ getattr(model_config, "nvfp4_backend", "auto"),
+ activation=getattr(model_config, "hidden_act", "silu"))
native = backend == "triton"
repack_sink = None
diff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.py
index 4c04192d..5ed8d766 100644
--- a/python/freetoken/moe/fused_nvfp4.py
+++ b/python/freetoken/moe/fused_nvfp4.py
@@ -21,11 +21,33 @@ from freetoken.kernel.triton.nvfp4_fused_moe import (
_e2m1_lut,
_prefill_nvfp4_moe_kernel,
)
-from freetoken.layers import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul
+from freetoken.layers import (
+ gelu_and_mul,
+ gelu_tanh_and_mul,
+ silu_and_mul,
+ swigluoai_and_mul,
+)
from freetoken.moe.fused import moe_align_block_size
_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}
+
+def _run_act(
+ activation: str,
+ gate_up: torch.Tensor,
+ out: torch.Tensor,
+ act_alpha: float,
+ act_limit: float,
+) -> None:
+ """gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped
+ gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) carries the
+ per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds
+ ignore them."""
+ if activation == "swigluoai":
+ swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)
+ return
+ _ACT[activation](gate_up, out)
+
# Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune,
# which benchmarks at run time). Tuned offline against the NVFP4 decode kernels.
# These drive the original LUT-gather decode (_decode_gemm), kept only for A/B.
@@ -142,6 +164,8 @@ def _fused_experts_decode_nvfp4(
topk_ids: torch.Tensor,
activation: str,
apply_router_weight_on_input: bool,
+ act_alpha: float = 1.702,
+ act_limit: float = 7.0,
) -> torch.Tensor:
"""Shared decode body (gemm1 -> act -> gemm2 -> sum-reduce); ``gemm_fn`` is either
the marlin-style int32 GEMV (:func:`_decode_gemm_marlin`) or the original LUT-gather
@@ -158,7 +182,7 @@ def _fused_experts_decode_nvfp4(
ic1, topk_weights, topk_ids, apply_router_weight_on_input, False,
)
ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)
- _ACT[activation](ic1.view(-1, two_i), ic2)
+ _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)
ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)
gemm_fn(
ic2, down_packed, down_scale, down_global,
@@ -181,6 +205,8 @@ def fused_experts_decode_nvfp4_marlin(
topk_ids: torch.Tensor,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
+ act_alpha: float = 1.702,
+ act_limit: float = 7.0,
) -> torch.Tensor:
"""Decode inline-NVFP4 MoE using the Marlin-style int32 wide-load GEMV."""
return _fused_experts_decode_nvfp4(
@@ -188,6 +214,7 @@ def fused_experts_decode_nvfp4_marlin(
hidden_states, gate_up_packed, gate_up_scale, gate_up_global,
down_packed, down_scale, down_global,
topk_weights, topk_ids, activation, apply_router_weight_on_input,
+ act_alpha, act_limit,
)
@@ -203,6 +230,8 @@ def fused_experts_decode_nvfp4_serial(
topk_ids: torch.Tensor,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
+ act_alpha: float = 1.702,
+ act_limit: float = 7.0,
) -> torch.Tensor:
"""Original LUT-gather decode (one program per route, full K reduction). Retained for
A/B benchmarking against the marlin decode path; not on the production decode path."""
@@ -211,6 +240,7 @@ def fused_experts_decode_nvfp4_serial(
hidden_states, gate_up_packed, gate_up_scale, gate_up_global,
down_packed, down_scale, down_global,
topk_weights, topk_ids, activation, apply_router_weight_on_input,
+ act_alpha, act_limit,
)
@@ -278,6 +308,8 @@ def fused_experts_nvfp4(
num_experts: int,
activation: str = "silu",
apply_router_weight_on_input: bool = False,
+ act_alpha: float = 1.702,
+ act_limit: float = 7.0,
) -> torch.Tensor:
"""Prefill inline-NVFP4 MoE. ``topk_ids`` index rows of the bank tensors in
``[0, num_experts)``: full-layer banks with position == expert id (the
@@ -300,7 +332,7 @@ def fused_experts_nvfp4(
apply_router_weight_on_input, cfg,
)
ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)
- _ACT[activation](ic1.view(-1, two_i), ic2)
+ _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)
ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)
_prefill_gemm(
ic2, down_packed, down_scale, down_global, ic3,
diff --git a/python/freetoken/moe/nvfp4_backends.py b/python/freetoken/moe/nvfp4_backends.py
index cd715553..e8a3c6f6 100644
--- a/python/freetoken/moe/nvfp4_backends.py
+++ b/python/freetoken/moe/nvfp4_backends.py
@@ -181,7 +181,10 @@ def _b12x_min_intermediate() -> int:
def select_nvfp4_backend(
- device: torch.device, intermediate_size: int | None = None, requested: str = "auto"
+ device: torch.device,
+ intermediate_size: int | None = None,
+ requested: str = "auto",
+ activation: str = "silu",
) -> str:
"""Pick the NVFP4 expert-GEMM backend for ``device`` (and, in ``auto``, ``intermediate_size``).
@@ -195,6 +198,11 @@ def select_nvfp4_backend(
* ``marlin`` / ``flashinfer`` / ``triton`` -- force that backend (and its bank layout),
raising if it cannot run rather than degrading.
+ ``activation`` is the model's routed-expert activation: the borrowed marlin/b12x
+ kernels hard-code silu (their fused epilogue), so any other activation (MiniMax-M3's
+ ``swigluoai``) resolves ``auto`` to the Triton kernels -- which dispatch the
+ activation as a separate elementwise op -- and rejects a forced marlin/flashinfer.
+
Returns the internal backend name (``marlin`` / ``b12x`` / ``triton``); ``flashinfer``
maps to ``b12x``.
"""
@@ -207,6 +215,18 @@ def select_nvfp4_backend(
)
if requested == "triton":
return "triton"
+ if activation != "silu":
+ if requested != "auto":
+ raise RuntimeError(
+ f"--nvfp4-backend={requested} only supports silu routed experts (its fused "
+ f"epilogue); this model's experts use {activation!r} -- use "
+ "--nvfp4-backend triton (or auto)."
+ )
+ logger.info(
+ f"NVFP4 auto backend: routed experts use {activation!r}, which only the "
+ "Triton kernels support; using the Triton inline-dequant kernels"
+ )
+ return "triton"
if requested == "marlin":
if device.type != "cuda":
raise RuntimeError("--nvfp4-backend=marlin requires a CUDA device")
diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.py
index 3feaa63a..e535ae53 100644
--- a/python/freetoken/server/anthropic_api.py
+++ b/python/freetoken/server/anthropic_api.py
@@ -113,7 +113,10 @@ async def handle_anthropic_messages(
model_sampling: dict[str, Any],
):
try:
- spec = convert_anthropic_to_genspec(req, model_sampling)
+ spec = convert_anthropic_to_genspec(
+ req, model_sampling,
+ reasoning_parser=getattr(state.config, "reasoning_parser", None),
+ )
uid = await submit_generation(spec, state)
except ValueError as exc:
return _anthropic_error_response(400, "invalid_request_error", str(exc))
@@ -145,7 +148,9 @@ async def handle_anthropic_count_tokens(req: AnthropicCountTokensRequest, state:
# a checkpoint whose chat template raises ValueError is a server fault, not a bad request,
# so it must not fall into the convert/empty-prompt ValueError branch.
try:
- messages, template_tools, _, ctk = convert_anthropic_prompt(req)
+ messages, template_tools, _, ctk = convert_anthropic_prompt(
+ req, reasoning_parser=getattr(state.config, "reasoning_parser", None)
+ )
except ValueError as exc:
return _anthropic_error_response(400, "invalid_request_error", str(exc))
if not messages:
@@ -172,6 +177,7 @@ async def handle_anthropic_count_tokens(req: AnthropicCountTokensRequest, state:
# --------------------------------------------------------------------------- #
def convert_anthropic_prompt(
req: AnthropicMessagesRequest | AnthropicCountTokensRequest,
+ reasoning_parser: str | None = None,
) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None, list[dict[str, Any]] | None, dict[str, Any]]:
"""(messages, template_tools, parser_tools, chat_template_kwargs) — the prompt
side of the conversion, shared by /v1/messages and /v1/messages/count_tokens so
@@ -278,23 +284,29 @@ def convert_anthropic_prompt(
selected = req.tool_choice.name if (req.tool_choice and req.tool_choice.type == "tool") else None
template_tools, parser_tools = split_tool_lists(raw_tools, selected)
- # Native extended-thinking toggle -> template kwargs: "enabled" turns thinking
- # on for templates that default it off (gemma4); an explicit "disabled" is
- # forwarded so templates that default ON can honor it.
+ # Native extended-thinking toggle -> template kwargs, through the per-family
+ # mapping in model_meta (a bare enable_thinking bool is inert for templates
+ # that read a different knob, e.g. M3's thinking_mode).
+ from .model_meta import think_toggle_kwargs
+
ctk: dict[str, Any] = {}
if req.thinking:
if req.thinking.get("type") == "enabled":
- ctk = {"enable_thinking": True}
+ ctk = think_toggle_kwargs(reasoning_parser, True)
elif req.thinking.get("type") == "disabled":
- ctk = {"enable_thinking": False}
+ ctk = think_toggle_kwargs(reasoning_parser, False)
return render_messages(messages), template_tools, parser_tools, ctk
def convert_anthropic_to_genspec(
- req: AnthropicMessagesRequest, model_sampling: dict[str, Any]
+ req: AnthropicMessagesRequest,
+ model_sampling: dict[str, Any],
+ reasoning_parser: str | None = None,
) -> GenSpec:
- messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(req)
+ messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(
+ req, reasoning_parser=reasoning_parser
+ )
return GenSpec(
messages=messages,
sampling_params=resolve_sampling(
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index c2bb118e..7bc3593c 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -128,6 +128,10 @@ def parse_args(
marker = " ".join(candidates).lower()
if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker:
return "gpt_oss"
+ # M3 first: its marker also contains the bare "minimax" substring, but the
+ # namespaced tool grammar is a different protocol from M2's.
+ if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker:
+ return "minimax_m3"
if "minimax" in marker:
return "minimax"
if "gemma4" in marker:
@@ -177,6 +181,10 @@ def parse_args(
return "qwen3"
if "glm" in marker:
return "glm"
+ # M3 first ("minimax" is a substring): <mm:think> tags + 3 thinking gears,
+ # not M2's always-on implicit <think>.
+ if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker:
+ return "minimax_m3"
if "minimax" in marker:
return "minimax"
if "gemma4" in marker:
@@ -407,6 +415,7 @@ def parse_args(
"gemma4",
"glm47",
"minimax",
+ "minimax_m3",
"gpt_oss",
"gpt-oss",
],
@@ -417,11 +426,15 @@ def parse_args(
"--reasoning-parser",
type=str,
default="auto",
- choices=["auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm", "minimax", "gemma4"],
+ choices=[
+ "auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm",
+ "minimax", "minimax_m3", "gemma4",
+ ],
help=(
"Reasoning parser that splits chain-of-thought into reasoning_content "
"for OpenAI responses. 'auto' selects per model family (gpt-oss Harmony, "
- "<think> for qwen3/glm/minimax, gemma thought, dsv4); 'off' disables it."
+ "<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, gemma thought, "
+ "dsv4); 'off' disables it."
),
)
diff --git a/python/freetoken/server/function_call_parser.py b/python/freetoken/server/function_call_parser.py
index 6981b476..0c22b8e0 100644
--- a/python/freetoken/server/function_call_parser.py
+++ b/python/freetoken/server/function_call_parser.py
@@ -62,6 +62,7 @@ TOOLS_TAG_LIST = [
"<function=",
"<tool_call>",
"<minimax:tool_call>",
+ "]<]minimax[>[<tool_call>",
"<|python_tag|>",
"[TOOL_CALLS]",
"<|DSML|function_calls>",
@@ -2386,6 +2387,473 @@ class MiniMaxDetector(InvokeParamStreamMixin, BaseFormatDetector):
+class MiniMaxM3Detector(BaseFormatDetector):
+ """FreeToken serving adapter for MiniMax-M3 namespace-delimited XML tool calls.
+
+ M3 is NOT M2 with renamed tags (vLLM's Rust ``MinimaxM3ToolParser`` grammar is
+ the reference): every structural tag is prefixed with the ``]<]minimax[>[``
+ namespace marker, one wrapper block holds multiple ``<invoke>`` tags, and the
+ arguments are parameter-NAME XML elements rendered recursively -- objects nest
+ tags, arrays repeat ``<item>``::
+
+ ]<]minimax[>[<tool_call>
+ ]<]minimax[>[<invoke name="create_order">
+ ]<]minimax[>[<user_id>42]<]minimax[>[</user_id>
+ ]<]minimax[>[<shipping>
+ ]<]minimax[>[<city>Singapore]<]minimax[>[</city>
+ ]<]minimax[>[</shipping>
+ ]<]minimax[>[</invoke>
+ ]<]minimax[>[</tool_call>
+
+ Leaf values are typed from the tool schema at every nesting level (loose JSON
+ when undeclared). Element scanning is lenient -- malformed tags and stray text
+ never void the parameters that did parse, and a truncated trailing element
+ salvages every complete sibling (which is what makes ``recover_truncated_call``
+ work for calls cut off by max_tokens).
+
+ Streaming: text outside the wrapper streams live (holding back a partial
+ marker suffix); inside the wrapper each completed ``</invoke>`` emits its call
+ in one piece, wire order preserved. The buffer keeps the wrapper opener while
+ inside a block so ``finish_streaming`` suppresses truncated-markup residue and
+ ``recover_truncated_call`` can re-parse it.
+ """
+
+ NS = "]<]minimax[>["
+
+ def __init__(self):
+ super().__init__()
+ self.bot_token = self.NS + "<tool_call>"
+ self.eot_token = self.NS + "</tool_call>"
+ # The template renders double quotes but models emit single quotes too.
+ self.invoke_open_re = re.compile(
+ re.escape(self.NS) + r'<invoke\s+name=["\']([^"\']+)["\']\s*>'
+ )
+ self._invoke_close = self.NS + "</invoke>"
+ # While True, self._buffer starts with bot_token (the retained opener).
+ self._m3_in_block = False
+
+ def has_tool_call(self, text: str) -> bool:
+ return self.bot_token in text
+
+ def _partial_marker_suffix(self, buffer: str) -> int:
+ """Length of the LONGEST buffer suffix that is a proper prefix of the
+ wrapper opener. The base ``_ends_with_partial_token`` returns the shortest
+ match, which under-holds for this marker: ``]`` recurs inside
+ ``]<]minimax[>[...``, so a buffer ending ``]<]`` would hold only 1 char and
+ leak ``]<`` as content, breaking the marker forever."""
+ for i in range(min(len(buffer), len(self.bot_token) - 1), 0, -1):
+ if self.bot_token.startswith(buffer[-i:]):
+ return i
+ return 0
+
+ # ---- lenient recursive parameter-XML scanning ------------------------------------
+ _TAG_BAD_CHARS = frozenset(' "<>/')
+
+ def _scan_elements(self, text: str) -> tuple:
+ """Scan ``text`` for ``NS<k>...NS</k>`` elements; returns ``(items, stray)``
+ where ``stray`` is the inter-element text (surfaced as ``"$text"`` for
+ mixed content). Lenient: malformed tags and dangling closers are stepped
+ over without dropping later siblings, an unterminated trailing element
+ salvages everything before it, and same-name nesting is depth-matched.
+
+ Known leniency trade-off: a leaf value quoting a well-formed element pair
+ parses as structure, and one quoting its own closer ends the element
+ early -- the strict alternative voided every parameter over one stray
+ character. (Quoted invoke/wrapper markers are handled structurally by
+ ``_scan_invoke_interior``.)
+ """
+ items: List[tuple] = []
+ stray_parts: List[str] = []
+ pos, n = 0, len(text)
+ while pos < n:
+ nxt = text.find(self.NS + "<", pos)
+ if nxt == -1:
+ stray_parts.append(text[pos:])
+ break
+ stray_parts.append(text[pos:nxt])
+ pos = nxt
+ if text.startswith(self.NS + "</", pos):
+ # A close tag we did not open: model noise -- skip the marker and
+ # keep scanning (aborting dropped every later sibling).
+ pos += len(self.NS) + 2
+ continue
+ tag_start = pos + len(self.NS) + 1
+ gt = text.find(">", tag_start)
+ if gt == -1:
+ break # truncated open tag
+ name = text[tag_start:gt]
+ if not name or any(c in self._TAG_BAD_CHARS for c in name):
+ pos = gt + 1 # malformed tag (e.g. an <invoke ...> echo): step over
+ continue
+ open_tag = self.NS + "<" + name + ">"
+ close_tag = self.NS + "</" + name + ">"
+ depth, search, inner_end = 1, gt + 1, -1
+ while depth:
+ cpos = text.find(close_tag, search)
+ if cpos == -1:
+ # unterminated element: salvage what parsed
+ return items, "".join(stray_parts)
+ opos = text.find(open_tag, search)
+ if opos != -1 and opos < cpos:
+ depth += 1
+ search = opos + len(open_tag)
+ else:
+ depth -= 1
+ inner_end = cpos
+ search = cpos + len(close_tag)
+ items.append((name, text[gt + 1 : inner_end]))
+ pos = search
+ return items, "".join(stray_parts)
+
+ def _typed_leaf(self, raw: str, prop: Any) -> Any:
+ """Leaf dequoting. The template renders leaf values verbatim, so string
+ leaves round-trip exactly (stripping would corrupt multi-line arguments).
+ Declared non-string types tolerate surrounding whitespace; undeclared
+ leaves fall back to loose JSON but keep the verbatim text when the parse
+ yields a string anyway."""
+ if prop is not None:
+ if isinstance(prop, dict) and "type" not in prop:
+ subs = prop.get("anyOf") or prop.get("oneOf")
+ if isinstance(subs, list) and subs:
+ return self._union_leaf(raw, subs)
+ ptype = (
+ str(prop.get("type", "string")).strip().lower()
+ if isinstance(prop, dict)
+ else "string"
+ )
+ if ptype in ("string", "str", "enum"):
+ return raw
+ if ptype in ("number", "float", "double"):
+ # Integer literals stay int; "5.0" stays 5.0.
+ s = raw.strip()
+ try:
+ return int(s)
+ except ValueError:
+ try:
+ return float(s)
+ except ValueError:
+ return raw
+ if ptype in ("object", "array") and raw.strip() == "":
+ # Empty container-typed element -> typed empty container.
+ return {} if ptype == "object" else []
+ return self._convert_param_value(raw.strip(), "_", {"_": prop}, "")
+ if raw.strip() == "":
+ # An empty element is the empty STRING; loose-JSON would make it {}.
+ return ""
+ value = _parse_loose_json_value(raw.strip())
+ return raw if isinstance(value, str) else value
+
+ def _union_leaf(self, raw: str, subs: list) -> Any:
+ """``anyOf``/``oneOf``: try each member's strict coercion in declared
+ order; verbatim text when only the string member (or nothing) matches."""
+ s = raw.strip()
+ for sub in subs:
+ t = str(sub.get("type", "")).strip().lower() if isinstance(sub, dict) else ""
+ if t in ("integer", "int"):
+ try:
+ return int(s)
+ except ValueError:
+ continue
+ if t in ("number", "float", "double"):
+ try:
+ return int(s)
+ except ValueError:
+ try:
+ return float(s)
+ except ValueError:
+ continue
+ if t in ("boolean", "bool") and s.lower() in ("true", "false"):
+ return s.lower() == "true"
+ if t == "null" and s.lower() == "null":
+ return None
+ if t in ("object", "array"):
+ try:
+ v = json.loads(s)
+ except (json.JSONDecodeError, ValueError):
+ continue
+ if isinstance(v, dict if t == "object" else list):
+ return v
+ return raw
+
+ def _nested_value(self, raw: str, schema: Any = None) -> Any:
+ items, stray = self._scan_elements(raw)
+ if not items:
+ return self._typed_leaf(raw, schema)
+ return self._structure(items, schema, stray=stray)
+
+ def _structure(self, items: List[tuple], schema: Any = None, stray: str = "") -> Any:
+ """Composite value from scanned elements, threading the JSON schema down
+ (``properties`` / ``items`` / ``additionalProperties``). Only ``<item>``
+ children render as a bare array -- the template's array convention;
+ repeated siblings under any other tag stay an object with an array-valued
+ key. Mixed text+children keeps the text under ``"$text"``."""
+ props = schema.get("properties") if isinstance(schema, dict) else None
+ item_schema = schema.get("items") if isinstance(schema, dict) else None
+
+ def _sub_schema(key: str) -> Any:
+ sub = props.get(key) if isinstance(props, dict) else None
+ if sub is None and isinstance(schema, dict):
+ ap = schema.get("additionalProperties")
+ if isinstance(ap, dict):
+ sub = ap
+ return sub
+
+ names = {k for k, _ in items}
+ if names == {"item"}:
+ return [self._nested_value(raw, item_schema) for _, raw in items]
+ counts: Dict[str, int] = {}
+ for k, _ in items:
+ counts[k] = counts.get(k, 0) + 1
+ out: Dict[str, Any] = {}
+ for key, raw in items:
+ sub = _sub_schema(key)
+ # A repeated key's elements are the array's members: unwrap its array
+ # schema to the item schema. A singleton key keeps the array schema
+ # (its <item> children unwrap inside).
+ if (
+ counts[key] > 1
+ and isinstance(sub, dict)
+ and str(sub.get("type", "")).lower() == "array"
+ ):
+ sub = sub.get("items")
+ value = self._nested_value(raw, sub)
+ if key in out:
+ prev = out[key]
+ out[key] = (prev if isinstance(prev, list) else [prev]) + [value]
+ else:
+ out[key] = value
+ if stray.strip():
+ out["$text"] = stray.strip()
+ return out
+
+ def _args_from_items(
+ self, func_name: str, items: List[tuple], tools: List[Tool]
+ ) -> Dict:
+ config = self._get_param_config(func_name, tools)
+ args: Dict[str, Any] = {}
+ for key, raw in items:
+ prop = config.get(key) if isinstance(config, dict) and key in config else None
+ nested, stray = self._scan_elements(raw)
+ if nested:
+ value: Any = self._structure(nested, prop, stray=stray)
+ else:
+ value = self._typed_leaf(raw, prop)
+ if key in args:
+ prev = args[key]
+ args[key] = (prev if isinstance(prev, list) else [prev]) + [value]
+ else:
+ args[key] = value
+ return args
+
+ def _scan_invoke_interior(self, text: str, pos: int) -> tuple:
+ """Collect parameter elements from ``pos`` (just past an invoke opener)
+ until a top-level ``NS</invoke>`` / ``NS</tool_call>`` / end of text.
+ Close markers are only interpreted between elements, so a value quoting
+ wire syntax can neither end the invoke early nor truncate the block.
+ Returns ``(items, end_pos, closer)`` with ``closer`` in
+ {"invoke", "eot", None}; ``end_pos`` points at the closer marker
+ (``len(text)`` when the text ran out -- truncation salvage)."""
+ items: List[tuple] = []
+ n = len(text)
+ while pos < n:
+ nxt = text.find(self.NS + "<", pos)
+ if nxt == -1:
+ return items, n, None
+ if text.startswith(self._invoke_close, nxt):
+ return items, nxt, "invoke"
+ if text.startswith(self.eot_token, nxt):
+ return items, nxt, "eot"
+ pos = nxt
+ if text.startswith(self.NS + "</", pos):
+ # Dangling closer we did not open: model noise -- skip it and
+ # keep collecting (the references keep well-formed siblings).
+ pos += len(self.NS) + 2
+ continue
+ tag_start = pos + len(self.NS) + 1
+ gt = text.find(">", tag_start)
+ if gt == -1:
+ return items, n, None # truncated open tag
+ name = text[tag_start:gt]
+ if not name or any(c in self._TAG_BAD_CHARS for c in name):
+ pos = gt + 1 # malformed tag (e.g. a quoted <invoke ...> echo)
+ continue
+ open_tag = self.NS + "<" + name + ">"
+ close_tag = self.NS + "</" + name + ">"
+ depth, search, inner_end = 1, gt + 1, -1
+ while depth:
+ cpos = text.find(close_tag, search)
+ if cpos == -1:
+ return items, n, None # unterminated element: salvage
+ opos = text.find(open_tag, search)
+ if opos != -1 and opos < cpos:
+ depth += 1
+ search = opos + len(open_tag)
+ else:
+ depth -= 1
+ inner_end = cpos
+ search = cpos + len(close_tag)
+ items.append((name, text[gt + 1 : inner_end]))
+ pos = search
+ return items, n, None
+
+ def _parse_block(
+ self, text: str, pos: int, tools: List[Tool], calls: List[ToolCallItem]
+ ) -> int:
+ """Parse a wrapper interior starting at ``pos`` (just past ``bot_token``),
+ appending calls; returns the position just past the block's
+ ``NS</tool_call>``, or ``len(text)`` when the wrapper never closes
+ (end-of-generation truncation -- a truncated trailing invoke still
+ salvages its complete parameters, the recover_truncated_call path)."""
+ tool_indices = self._get_tool_indices(tools)
+ n = len(text)
+ while pos < n:
+ nxt = text.find(self.NS + "<", pos)
+ if nxt == -1:
+ return n
+ if text.startswith(self.eot_token, nxt):
+ return nxt + len(self.eot_token)
+ m = self.invoke_open_re.match(text, nxt)
+ if m is None:
+ pos = nxt + len(self.NS) + 1 # stray marker at block level
+ continue
+ items, end, closer = self._scan_invoke_interior(text, m.end())
+ func_name = m.group(1)
+ if func_name in tool_indices or _should_forward_unknown_tool(func_name):
+ args = self._args_from_items(func_name, items, tools)
+ calls.append(
+ ToolCallItem(
+ tool_index=len(calls),
+ name=func_name,
+ parameters=json.dumps(args, ensure_ascii=False),
+ )
+ )
+ else:
+ logger.warning(f"Model attempted to call undefined function: {func_name}")
+ if closer == "invoke":
+ pos = end + len(self._invoke_close)
+ elif closer == "eot":
+ return end + len(self.eot_token)
+ else:
+ return n
+ return n
+
+ # ---- one-shot -------------------------------------------------------------------
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ """Parse EVERY wrapper block (text between and after blocks stays content).
+ A block without ``</tool_call>`` (end-of-generation truncation) still
+ yields its parseable calls -- recover_truncated_call appends the closer
+ and re-enters here."""
+ calls: List[ToolCallItem] = []
+ normal_parts: List[str] = []
+ pos = 0
+ while True:
+ b = text.find(self.bot_token, pos)
+ if b == -1:
+ normal_parts.append(text[pos:])
+ break
+ normal_parts.append(text[pos:b])
+ pos = self._parse_block(text, b + len(self.bot_token), tools, calls)
+ return StreamingParseResult(
+ normal_text="".join(normal_parts).strip(), calls=calls
+ )
+
+ # ---- streaming ------------------------------------------------------------------
+ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult:
+ self._buffer += new_text
+ calls: List[ToolCallItem] = []
+ normal_parts: List[str] = []
+ tool_indices = self._get_tool_indices(tools)
+
+ while True:
+ if not self._m3_in_block:
+ pos = self._buffer.find(self.bot_token)
+ if pos == -1:
+ hold = self._partial_marker_suffix(self._buffer)
+ release = self._buffer[: len(self._buffer) - hold]
+ if release and calls:
+ break # wire order: text after a call defers to the next step
+ self._buffer = self._buffer[len(release) :]
+ if release:
+ normal_parts.append(release)
+ break
+ if pos > 0:
+ if calls:
+ break # wire order (text precedes a SECOND block)
+ normal_parts.append(self._buffer[:pos])
+ self._buffer = self._buffer[pos:]
+ self._m3_in_block = True
+ continue
+
+ # In-block: the buffer keeps the opener (finish_streaming and
+ # recover_truncated_call key off has_tool_call(buffer)). Markers are
+ # interpreted structurally, same as detect_and_parse: the first
+ # top-level invoke opener or wrapper closer wins.
+ body = self._buffer[len(self.bot_token) :]
+ action = None
+ scan = 0
+ while True:
+ nxt = body.find(self.NS + "<", scan)
+ if nxt == -1:
+ break
+ if body.startswith(self.eot_token, nxt):
+ action = ("eot", nxt, None)
+ break
+ m = self.invoke_open_re.match(body, nxt)
+ if m is not None:
+ action = ("invoke", nxt, m)
+ break
+ scan = nxt + len(self.NS) + 1 # stray marker at block level
+ if action is None:
+ break # inside the block, nothing complete yet -> hold
+ if action[0] == "eot":
+ # Wrapper closed: drop the block, loop back to the idle scan so the
+ # residue gets partial-marker holds and second-block detection
+ # instead of leaking raw.
+ self._buffer = body[action[1] + len(self.eot_token) :]
+ self._m3_in_block = False
+ continue
+ m = action[2]
+ items, end, closer = self._scan_invoke_interior(body, m.end())
+ if closer != "invoke":
+ # Invoke still streaming (or eot-truncated: left to
+ # finish_streaming / recover_truncated_call) -> hold.
+ break
+ func_name = m.group(1)
+ self._buffer = self.bot_token + body[end + len(self._invoke_close) :]
+ if func_name not in tool_indices and not _should_forward_unknown_tool(
+ func_name
+ ):
+ logger.warning(
+ f"Model attempted to call undefined function: {func_name}"
+ )
+ continue
+ args = self._args_from_items(func_name, items, tools)
+ args_json = json.dumps(args, ensure_ascii=False)
+ self.current_tool_id += 1
+ while len(self.prev_tool_call_arr) <= self.current_tool_id:
+ self.prev_tool_call_arr.append({})
+ while len(self.streamed_args_for_tool) <= self.current_tool_id:
+ self.streamed_args_for_tool.append("")
+ self.prev_tool_call_arr[self.current_tool_id] = {
+ "name": func_name,
+ "arguments": args,
+ }
+ calls.append(
+ ToolCallItem(
+ tool_index=self.current_tool_id, name=func_name, parameters=""
+ )
+ )
+ calls.append(
+ ToolCallItem(
+ tool_index=self.current_tool_id, name=None, parameters=args_json
+ )
+ )
+ self.streamed_args_for_tool[self.current_tool_id] = args_json
+ continue
+
+ return StreamingParseResult(normal_text="".join(normal_parts), calls=calls)
+
+
class GptOssDetector(BaseFormatDetector):
"""FreeToken serving adapter for Harmony ``to=functions.*`` tool calls.
@@ -2591,6 +3059,7 @@ class FunctionCallParser:
"glm47": Glm47Detector,
"llama3": Llama32Detector,
"minimax": MiniMaxDetector,
+ "minimax_m3": MiniMaxM3Detector,
"mistral": MistralDetector,
"qwen": Qwen25Detector,
"qwen25": Qwen25Detector,
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index 6fb40948..bd8741b8 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -327,6 +327,11 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:
or bool(ctk.get("enable_thinking"))
or bool(ctk.get("thinking"))
)
+ elif parser_name == "minimax_m3":
+ # M3's template pre-opens <mm:think> only in thinking_mode "enabled" (the
+ # model then emits just the closing tag); "adaptive" (default) leaves the
+ # model to open the tag itself and "disabled" pre-closes it.
+ force_reasoning = (spec.chat_template_kwargs or {}).get("thinking_mode") == "enabled"
else:
force_reasoning = (
resolve_thinking_mode(spec.chat_template_kwargs, spec.template_tools) == "thinking"
diff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.py
index 18e4952c..d976c924 100644
--- a/python/freetoken/server/model_meta.py
+++ b/python/freetoken/server/model_meta.py
@@ -22,6 +22,10 @@ def think_spec(reasoning_parser: str | None) -> Tuple[Tuple[str, ...], str | Non
return ("off", "on", "max"), "off" # thinking on/off + a max-effort gear
if reasoning_parser == "minimax":
return ("on",), "on" # template always opens a think block; no off path
+ if reasoning_parser == "minimax_m3":
+ # M3's template takes thinking_mode disabled/adaptive/enabled; adaptive
+ # (the template's own default) lets the model decide per turn.
+ return ("off", "adaptive", "on"), "adaptive"
if reasoning_parser == "gemma4":
return ("off", "on"), "off" # gemma's template defaults thinking off
if reasoning_parser in ("qwen3", "glm"):
@@ -41,9 +45,28 @@ def think_chat_template_kwargs(reasoning_parser: str | None, gear: str | None) -
return {"enable_thinking": gear == "on"}
if reasoning_parser == "minimax":
return {} # always thinks; its template reads no knob
+ if reasoning_parser == "minimax_m3":
+ mode = {"off": "disabled", "adaptive": "adaptive", "on": "enabled"}[gear]
+ return {"thinking_mode": mode}
return {"enable_thinking": gear == "on"} # qwen3, glm, gemma4
+def think_toggle_kwargs(reasoning_parser: str | None, enabled: bool) -> dict:
+ """``chat_template_kwargs`` for a protocol-level thinking on/off toggle
+ (Anthropic ``thinking.type``, Responses ``reasoning.effort``), routed through
+ the same per-family mapping as the chat-completions gears -- a hardcoded
+ ``enable_thinking`` is inert for templates that read a different knob (M3's
+ ``thinking_mode``). A family without the requested direction returns ``{}``;
+ with no configured parser the protocol-generic key is kept."""
+ gears, _default = think_spec(reasoning_parser)
+ if not gears:
+ return {"enable_thinking": enabled}
+ gear = "on" if enabled else "off"
+ if gear not in gears:
+ return {}
+ return think_chat_template_kwargs(reasoning_parser, gear)
+
+
def moe_total_experts(config: Any) -> int:
"""Total routed-expert slots the model has: experts per layer x MoE layers. Matches the
engine's own basis (``Engine._resolve_auto_moe_cache_size``), so a residency rate derived
diff --git a/python/freetoken/server/reasoning_parser.py b/python/freetoken/server/reasoning_parser.py
index db31658e..2d5e3d29 100644
--- a/python/freetoken/server/reasoning_parser.py
+++ b/python/freetoken/server/reasoning_parser.py
@@ -411,6 +411,126 @@ class ThinkReasoningParser(BaseReasoningParser):
)
+class MiniMaxM3ReasoningParser(BaseReasoningParser):
+ """Reasoning parser for MiniMax-M3's ``<mm:think>...</mm:think>`` protocol.
+
+ M3's template has three thinking modes: "enabled" pre-opens ``<mm:think>`` at
+ the generation prompt (the model emits only the closing tag -- the caller
+ passes ``force_reasoning=True``), "adaptive" leaves the model to open the tag
+ itself, and "disabled" pre-closes it (no reasoning). The tool block's
+ namespaced opener ends reasoning when a (malformed) turn skips
+ ``</mm:think>`` and runs straight into a tool call (dsv4 precedent).
+
+ Adaptive quirk (matches vLLM / llama.cpp): a NON-thinking adaptive turn
+ starts with a bare ``</mm:think>`` written by the model itself ("thinking
+ off for this turn"). Without special handling that literal marker would
+ stream into content on the DEFAULT gear's most common path, so a single
+ leading closer (whitespace-tolerant head: the detokenizer may open with a
+ newline before the marker; anything else before a closer keeps it visible)
+ is stripped in both one-shot and streaming modes -- never when
+ ``force_reasoning`` is set, where the closer genuinely terminates the
+ template-opened think block.
+ """
+
+ THINK_START = "<mm:think>"
+ THINK_END = "</mm:think>"
+
+ def __init__(self, force_reasoning: bool = False, stream_reasoning: bool = True) -> None:
+ super().__init__(
+ think_start_token=self.THINK_START,
+ think_end_token=self.THINK_END,
+ force_reasoning=force_reasoning,
+ stream_reasoning=stream_reasoning,
+ tool_start_token="]<]minimax[>[<tool_call>",
+ )
+ # Streaming state for the leading-bare-closer check: hold the head of the
+ # stream while it is still a prefix of ``</mm:think>``.
+ self._leading_closer_pending = not force_reasoning
+ self._head_buffer = ""
+
+ def detect_and_parse(self, text: str) -> ReasoningParseResult:
+ """One-shot parse, positional and verbatim, matching the streaming path:
+ reasoning is anchored at the first opener the model wrote, prose before
+ it stays content, later marker occurrences are data, and nothing is
+ whitespace-trimmed. (The base one-shot relabels prose before a
+ mid-content opener and strips both sides -- wrong for this grammar.)"""
+ end = self.think_end_token
+ if self.force_reasoning:
+ # enabled gear: the template pre-opened the think block.
+ reasoning, sep, rest = text.partition(end)
+ if sep:
+ return ReasoningParseResult(reasoning_text=reasoning, normal_text=rest)
+ # No closer: a malformed turn running straight into a tool block
+ # ends reasoning there (dsv4 precedent); else truncated reasoning.
+ if self.tool_start_token is not None and self.tool_start_token in text:
+ t = text.find(self.tool_start_token)
+ return ReasoningParseResult(
+ reasoning_text=text[:t], normal_text=text[t:]
+ )
+ return ReasoningParseResult(reasoning_text=text)
+ # adaptive: a leading bare closer is "thinking off for this turn"
+ # (whitespace-tolerant head, same as streaming).
+ head = text.lstrip()
+ if head.startswith(end):
+ return ReasoningParseResult(normal_text=head[len(end) :])
+ start = text.find(self.think_start_token)
+ if start == -1:
+ return ReasoningParseResult(normal_text=text)
+ before = text[:start]
+ body = text[start + len(self.think_start_token) :]
+ reasoning, sep, rest = body.partition(end)
+ if sep:
+ return ReasoningParseResult(
+ reasoning_text=reasoning, normal_text=before + rest
+ )
+ if self.tool_start_token is not None and self.tool_start_token in body:
+ t = body.find(self.tool_start_token)
+ return ReasoningParseResult(
+ reasoning_text=body[:t], normal_text=before + body[t:]
+ )
+ return ReasoningParseResult(reasoning_text=body)
+
+ def parse_streaming_increment(self, new_text: str) -> ReasoningParseResult:
+ if self._leading_closer_pending:
+ self._head_buffer += new_text
+ # Whitespace-tolerant head: the detokenizer may open with a newline
+ # before the model's bare closer.
+ head = self._head_buffer.lstrip()
+ if head.startswith(self.think_end_token):
+ # Bare leading closer: strip it once, everything after is content.
+ self._leading_closer_pending = False
+ self._head_buffer = ""
+ self._in_reasoning = False
+ rest = head[len(self.think_end_token) :]
+ return (
+ super().parse_streaming_increment(rest)
+ if rest
+ else ReasoningParseResult()
+ )
+ if not head or self.think_end_token.startswith(head):
+ return ReasoningParseResult() # still a (whitespace+) prefix: hold
+ # Diverged: not a leading closer -- replay the held head as normal
+ # (whitespace included; it was real output).
+ replay, self._head_buffer = self._head_buffer, ""
+ self._leading_closer_pending = False
+ return super().parse_streaming_increment(replay)
+ return super().parse_streaming_increment(new_text)
+
+ def flush(self) -> ReasoningParseResult:
+ if self._leading_closer_pending and self._head_buffer:
+ # Stream ended while the head was still a closer prefix (e.g. "</mm:t"):
+ # it was never a bare closer, replay it before the base flush.
+ head, self._head_buffer = self._head_buffer, ""
+ self._leading_closer_pending = False
+ first = super().parse_streaming_increment(head)
+ rest = super().flush()
+ return ReasoningParseResult(
+ reasoning_text=first.reasoning_text + rest.reasoning_text,
+ normal_text=first.normal_text + rest.normal_text,
+ )
+ return super().flush()
+
+
class GemmaThoughtReasoningParser(BaseReasoningParser):
"""Reasoning parser for Gemma-4's thought channel: the model emits its thought,
then a closing ``<channel|>`` marker, then the visible answer. The opening
@@ -435,6 +555,7 @@ class ReasoningParser:
"qwen3": ThinkReasoningParser,
"glm": ThinkReasoningParser,
"minimax": ThinkReasoningParser,
+ "minimax_m3": MiniMaxM3ReasoningParser,
"gemma4": GemmaThoughtReasoningParser,
}
diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.py
index 7255f7cb..31a316f8 100644
--- a/python/freetoken/server/responses_api.py
+++ b/python/freetoken/server/responses_api.py
@@ -152,7 +152,10 @@ async def handle_responses(
return _error_response(400, "max_output_tokens must be a positive integer")
default_max = getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENS
try:
- spec = convert_responses_to_genspec(req, model_sampling, default_max_tokens=default_max)
+ spec = convert_responses_to_genspec(
+ req, model_sampling, default_max_tokens=default_max,
+ reasoning_parser=getattr(state.config, "reasoning_parser", None),
+ )
uid = await submit_generation(spec, state)
except ValueError as exc:
return _error_response(400, str(exc))
@@ -184,6 +187,7 @@ def convert_responses_to_genspec(
req: ResponsesRequest,
model_sampling: dict[str, Any],
default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,
+ reasoning_parser: str | None = None,
) -> GenSpec:
# Collect every system/developer text — the top-level `instructions` PLUS any
# system/developer-role input items (codex sends both: a system prompt as `instructions`
@@ -226,19 +230,19 @@ def convert_responses_to_genspec(
else:
template_tools, parser_tools = split_tool_lists(raw_tools, selected)
- # Thinking toggle: an explicit chat_template_kwargs extra field wins; else the
- # protocol-native `reasoning` object (codex sends {"effort": ...}) drives the
- # template's thinking mode. effort "none" explicitly DISABLES thinking (vLLM
- # maps it the same way); other/unset efforts enable it.
+ # Thinking toggle: an explicit chat_template_kwargs extra field wins; else
+ # the protocol-native `reasoning` object drives the template through the
+ # per-family mapping in model_meta. effort "none" disables thinking (vLLM
+ # semantics); other efforts enable it and are forwarded for templates that
+ # grade them (gpt-oss).
+ from .model_meta import think_toggle_kwargs
+
ctk = dict(getattr(req, "chat_template_kwargs", None) or {})
if req.reasoning and not ctk:
effort = req.reasoning.get("effort")
- if effort == "none":
- ctk = {"enable_thinking": False}
- else:
- ctk = {"enable_thinking": True}
- if effort:
- ctk["reasoning_effort"] = effort
+ ctk = dict(think_toggle_kwargs(reasoning_parser, effort != "none"))
+ if effort and effort != "none":
+ ctk.setdefault("reasoning_effort", effort)
return GenSpec(
messages=render_messages(messages),
diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.py
index fe0cae67..f27640b9 100644
--- a/tests/engine/test_attention_backend_matrix.py
+++ b/tests/engine/test_attention_backend_matrix.py
@@ -64,6 +64,9 @@ def _model_config(kind):
elif kind == "dsv4":
mc.dsv4_args = SimpleNamespace(window_size=128)
specs = (_spec("dsv4", AttnType.DSV4, sliding_window=128),)
+ elif kind == "bsa":
+ # MiniMax-M3 shape: one FULL-family group, mla=False + index dims -> BSA.
+ specs = (_spec("full", AttnType.BSA, index_head_dim=128),)
elif kind == "linear_hybrid":
mc.has_linear_attention = True
specs = (_spec("full", AttnType.FULL),)
@@ -105,6 +108,7 @@ def _patch_env(monkeypatch, *, major=9, flashinfer=True, sgl=True):
("mla", "dsa"), # plain latent MLA
("dsa", "dsa"), # MLA + DSA indexer (GLM-5.2 shape)
("dsv4", "dsv4_sparse"),
+ ("bsa", "m3_sparse"), # MiniMax-M3 block-sparse GQA
],
)
def test_auto_resolves_per_type(monkeypatch, kind, expected):
@@ -116,6 +120,29 @@ def test_auto_resolves_per_type(monkeypatch, kind, expected):
assert config.attention_backend == expected
+def test_auto_bsa_sets_block_page_size(monkeypatch):
+ # m3_sparse declares page_sizes=(128,): one KV page == one sparse block, and
+ # config-time resolution must coerce the page size to match.
+ from freetoken.engine.engine import _adjust_config
+
+ _patch_env(monkeypatch)
+ config = _config("bsa", attention_backend="auto")
+ _adjust_config(config)
+ assert config.page_size == 128
+
+
+def test_bsa_rejects_float32_dtype(monkeypatch):
+ # --dtype float32 used to pass config validation and die on the pool's
+ # itemsize==2 assert only after the model was resident.
+ from freetoken.engine.engine import _adjust_config
+
+ _patch_env(monkeypatch)
+ config = _config("bsa", attention_backend="auto")
+ object.__setattr__(config, "dtype", torch.float32)
+ with pytest.raises(ValueError, match="16-bit"):
+ _adjust_config(config)
+
+
def test_auto_dsv4_sets_window_page_size(monkeypatch):
from freetoken.engine.engine import _adjust_config
@@ -131,7 +158,11 @@ def test_auto_dsv4_sets_window_page_size(monkeypatch):
# reverse gates: type-specific backends on models without the type
("full", "dsa"),
("full", "dsv4_sparse"),
+ ("full", "m3_sparse"),
("swa", "dsa"),
+ # forward gates: generic backends on the BSA-locked model
+ ("bsa", "fi"),
+ ("bsa", "triton"),
# forward gates: generic backends on type-locked models
("mla", "fi"),
("mla", "triton"),
diff --git a/tests/kernels/test_minimax_m3_sparse.py b/tests/kernels/test_minimax_m3_sparse.py
new file mode 100644
index 00000000..9a56d8fe
--- /dev/null
+++ b/tests/kernels/test_minimax_m3_sparse.py
@@ -0,0 +1,437 @@
+"""Parity tests for the MiniMax-M3 block-sparse attention kernels.
+
+The pure-torch reference below implements the semantics pinned against the vLLM
+reference implementation (see the vLLM tree's ``vllm/models/minimax_m3/`` and
+``kernel/triton/minimax_m3_sparse.py``'s module docstring):
+
+* per-index-head block scores: max over the block's 128 positions of
+ ``dot(index_q, index_k)``, causal-masked, no scale;
+* top-k selection with forced init/local blocks (score boosts 1e30/1e29, local
+ wins overlaps), -1 padding past the visible block count;
+* GQA attend over the selected blocks only (each kv head follows its own index
+ head), causal inside the newest block.
+
+The physical layout mirrors serving: a row-flat slab addressed through a
+per-request ``block_rows`` base-row table built from a random page permutation,
+so the tests exercise the gather addressing, not just the math.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+if not torch.cuda.is_available(): # pragma: no cover
+ pytest.skip("CUDA required for the M3 sparse kernels", allow_module_level=True)
+
+from freetoken.kernel.triton.minimax_m3_sparse import (
+ SPARSE_BLOCK_SIZE,
+ m3_index_decode,
+ m3_index_score_prefill,
+ m3_index_topk_prefill,
+ m3_sparse_attn_decode,
+ m3_sparse_attn_prefill,
+)
+
+BLK = SPARSE_BLOCK_SIZE
+DEV = "cuda"
+
+
+# ---------------------------------------------------------------------------
+# Torch reference
+# ---------------------------------------------------------------------------
+def ref_block_scores(
+ iq: torch.Tensor, # [T, H, D] queries at absolute positions q_pos
+ ik: torch.Tensor, # [S, D] index keys for positions 0..S-1
+ q_pos: torch.Tensor, # [T] absolute positions
+) -> torch.Tensor:
+ """[H, T, nblocks] fp32; max over each 128-block, causal, -inf invisible."""
+ scores = torch.einsum("thd,sd->hts", iq.float(), ik.float())
+ s_pos = torch.arange(ik.shape[0], device=iq.device)
+ scores = scores.masked_fill(s_pos[None, None, :] > q_pos[None, :, None], float("-inf"))
+ nblocks = (ik.shape[0] + BLK - 1) // BLK
+ pad = nblocks * BLK - ik.shape[0]
+ if pad:
+ scores = torch.nn.functional.pad(scores, (0, pad), value=float("-inf"))
+ return scores.view(*scores.shape[:2], nblocks, BLK).amax(dim=-1)
+
+
+def ref_select(
+ block_scores: torch.Tensor, # [H, T, nblocks]
+ q_pos: torch.Tensor, # [T]
+ topk: int,
+ init_blocks: int,
+ local_blocks: int,
+) -> list[list[set[int]]]:
+ """Per (head, token) SET of selected block ids (order is score-descending in
+ the kernels, but only membership is contract -- the attend softmax is
+ order-invariant)."""
+ H, T, nblocks = block_scores.shape
+ out: list[list[set[int]]] = []
+ for h in range(H):
+ rows = []
+ for t in range(T):
+ nb = int(q_pos[t]) // BLK + 1
+ s = block_scores[h, t, :nb].clone()
+ if init_blocks:
+ s[: min(init_blocks, nb)] = 1e30
+ if local_blocks:
+ s[max(0, nb - local_blocks):] = 1e29
+ k = min(topk, nb)
+ rows.append(set(torch.topk(s, k).indices.tolist()))
+ out.append(rows)
+ return out
+
+
+def ref_sparse_attend(
+ q: torch.Tensor, # [T, HQ, D] queries at q_pos
+ k: torch.Tensor, # [S, KVH, D]
+ v: torch.Tensor, # [S, KVH, D]
+ selection: list[list[set[int]]], # [KVH][T] block-id sets
+ q_pos: torch.Tensor, # [T]
+ sm_scale: float,
+) -> torch.Tensor:
+ T, HQ, D = q.shape
+ S, KVH, _ = k.shape
+ group = HQ // KVH
+ out = torch.empty(T, HQ, D, dtype=torch.float32, device=q.device)
+ s_pos = torch.arange(S, device=q.device)
+ for t in range(T):
+ for kh in range(KVH):
+ sel = sorted(selection[kh][t])
+ pos_mask = torch.zeros(S, dtype=torch.bool, device=q.device)
+ for b in sel:
+ pos_mask[b * BLK : min((b + 1) * BLK, S)] = True
+ pos_mask &= s_pos <= int(q_pos[t])
+ idx = pos_mask.nonzero(as_tuple=True)[0]
+ kk = k[idx, kh].float() # [n, D]
+ vv = v[idx, kh].float()
+ for g in range(group):
+ h = kh * group + g
+ logits = (q[t, h].float() @ kk.t()) * sm_scale
+ out[t, h] = torch.softmax(logits, dim=-1) @ vv
+ return out
+
+
+def make_layout(nblocks: int, seed: int = 0):
+ """A shuffled physical layout: logical block i lives at page perm[i]."""
+ g = torch.Generator(device="cpu").manual_seed(seed)
+ perm = torch.randperm(nblocks + 3, generator=g)[:nblocks] # +3 spare pages
+ total_pages = nblocks + 3
+ block_rows = (perm * BLK).to(torch.int32).to(DEV)
+ return block_rows, total_pages
+
+
+def scatter_rows(
+ x: torch.Tensor, block_rows: torch.Tensor, total_pages: int, fill: float = float("nan")
+):
+ """Place logical rows [S, ...] into a physical slab [total_pages*BLK, ...].
+
+ Unwritten rows default to NaN, modeling the serving pool's ``torch.empty``
+ recycled-allocator garbage: the attend kernels MUST pos-mask every K/V load
+ or the forced local (partial) block poisons the output -- zero-filled slabs
+ hid exactly that bug from the original tests. The index-key slab passes
+ ``fill=0.0`` (BSAKVCache zero-inits it; the score kernels rely on that).
+ """
+ S = x.shape[0]
+ slab = torch.full(
+ (total_pages * BLK, *x.shape[1:]), fill, dtype=x.dtype, device=x.device
+ )
+ for b in range((S + BLK - 1) // BLK):
+ n = min(BLK, S - b * BLK)
+ base = int(block_rows[b])
+ slab[base : base + n] = x[b * BLK : b * BLK + n]
+ return slab
+
+
+# ---------------------------------------------------------------------------
+# Tests
+# ---------------------------------------------------------------------------
+H_IDX, D_IDX = 4, 128
+HQ, KVH, D = 16, 4, 128 # scaled-down GQA (group 4) to keep the reference fast
+TOPK, INIT, LOCAL = 4, 0, 1
+SCALE = D**-0.5
+
+
[email protected]("topk", [4, 16])
[email protected]("kv_len,q_len", [(BLK * 7 + 55, 300), (BLK * 2, BLK * 2), (90, 90)])
+def test_prefill_index_score_and_topk(kv_len: int, q_len: int, topk: int):
+ torch.manual_seed(0)
+ prefix = kv_len - q_len
+ nblocks = (kv_len + BLK - 1) // BLK
+ iq = torch.randn(q_len, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)
+ ik = torch.randn(kv_len, D_IDX, device=DEV, dtype=torch.bfloat16)
+ q_pos = torch.arange(prefix, kv_len, device=DEV)
+
+ block_rows, total_pages = make_layout(nblocks)
+ ik_slab = scatter_rows(ik, block_rows, total_pages, fill=0.0)
+
+ cu = torch.tensor([0, q_len], dtype=torch.int32, device=DEV)
+ seq = torch.tensor([kv_len], dtype=torch.int32, device=DEV)
+ pre = torch.tensor([prefix], dtype=torch.int32, device=DEV)
+ score = m3_index_score_prefill(
+ iq, ik_slab, block_rows.view(1, -1), cu, seq, pre, q_len, kv_len
+ )
+ ref = ref_block_scores(iq, ik, q_pos)
+ for t in range(q_len):
+ nb = int(q_pos[t]) // BLK + 1
+ got = score[:, t, :nb].float()
+ want = ref[:, t, :nb]
+ assert torch.allclose(got, want, atol=0.35, rtol=0.02), (t, (got - want).abs().max())
+
+ topk_idx = m3_index_topk_prefill(score, cu, pre, q_len, topk, INIT, LOCAL)
+ ref_sel = ref_select(ref, q_pos, topk, INIT, LOCAL)
+ for h in range(H_IDX):
+ for t in range(q_len):
+ nb = int(q_pos[t]) // BLK + 1
+ k = min(topk, nb)
+ got = topk_idx[h, t].tolist()
+ assert set(got[:k]) == ref_sel[h][t], (h, t, got, ref_sel[h][t])
+ assert all(i == -1 for i in got[k:]), (h, t, got)
+
+
[email protected]("topk", [4, 16])
[email protected]("kv_lens", [[BLK * 6 + 17, 70, BLK * 3]])
+def test_decode_index_topk(kv_lens: list[int], topk: int):
+ torch.manual_seed(1)
+ bs = len(kv_lens)
+ max_nb = max((L + BLK - 1) // BLK for L in kv_lens)
+ iq = torch.randn(bs, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)
+ block_rows = torch.zeros(bs, max_nb, dtype=torch.int32, device=DEV)
+ slabs = []
+ total = 0
+ layouts = []
+ for i, L in enumerate(kv_lens):
+ nb = (L + BLK - 1) // BLK
+ br, pages = make_layout(nb, seed=i)
+ layouts.append((br, pages))
+ block_rows[i, :nb] = br + total * BLK
+ total += pages
+ slab = torch.zeros(total * BLK, D_IDX, device=DEV, dtype=torch.bfloat16)
+ iks = []
+ off = 0
+ for i, L in enumerate(kv_lens):
+ ik = torch.randn(L, D_IDX, device=DEV, dtype=torch.bfloat16)
+ iks.append(ik)
+ br, pages = layouts[i]
+ sub = scatter_rows(ik, br, pages, fill=0.0)
+ slab[off * BLK : (off + pages) * BLK] = sub
+ off += pages
+ seq = torch.tensor(kv_lens, dtype=torch.int32, device=DEV)
+ topk_idx = m3_index_decode(iq, slab, block_rows, seq, max_nb, topk, INIT, LOCAL)
+ for i, L in enumerate(kv_lens):
+ q_pos = torch.tensor([L - 1], device=DEV)
+ ref = ref_block_scores(iq[i : i + 1], iks[i], q_pos)
+ sel = ref_select(ref, q_pos, topk, INIT, LOCAL)
+ nb = (L + BLK - 1) // BLK
+ k = min(topk, nb)
+ for h in range(H_IDX):
+ got = topk_idx[h, i].tolist()
+ assert set(got[:k]) == sel[h][0], (i, h, got, sel[h][0])
+ assert all(x == -1 for x in got[k:]), (i, h, got)
+
+
+def _attend_case(kv_len: int, q_len: int, seed: int):
+ torch.manual_seed(seed)
+ prefix = kv_len - q_len
+ nblocks = (kv_len + BLK - 1) // BLK
+ q = torch.randn(q_len, HQ, D, device=DEV, dtype=torch.bfloat16)
+ k = torch.randn(kv_len, KVH, D, device=DEV, dtype=torch.bfloat16)
+ v = torch.randn(kv_len, KVH, D, device=DEV, dtype=torch.bfloat16)
+ iq = torch.randn(q_len, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)
+ ik = torch.randn(kv_len, D_IDX, device=DEV, dtype=torch.bfloat16)
+ q_pos = torch.arange(prefix, kv_len, device=DEV)
+ block_rows, total_pages = make_layout(nblocks, seed=seed)
+ return q, k, v, iq, ik, q_pos, block_rows, total_pages, nblocks, prefix
+
+
[email protected]("topk", [4, 16])
[email protected]("kv_len,q_len", [(BLK * 7 + 55, 200), (BLK * 2, BLK * 2), (77, 77)])
+def test_prefill_attend(kv_len: int, q_len: int, topk: int):
+ q, k, v, iq, ik, q_pos, block_rows, total_pages, nblocks, prefix = _attend_case(
+ kv_len, q_len, seed=2
+ )
+ k_slab = scatter_rows(k, block_rows, total_pages)
+ v_slab = scatter_rows(v, block_rows, total_pages)
+ ik_slab = scatter_rows(ik, block_rows, total_pages, fill=0.0)
+
+ cu = torch.tensor([0, q_len], dtype=torch.int32, device=DEV)
+ seq = torch.tensor([kv_len], dtype=torch.int32, device=DEV)
+ pre = torch.tensor([prefix], dtype=torch.int32, device=DEV)
+ score = m3_index_score_prefill(
+ iq, ik_slab, block_rows.view(1, -1), cu, seq, pre, q_len, kv_len
+ )
+ topk_idx = m3_index_topk_prefill(score, cu, pre, q_len, topk, INIT, LOCAL)
+
+ out = torch.empty_like(q)
+ m3_sparse_attn_prefill(
+ q, k_slab, v_slab, topk_idx, block_rows.view(1, -1), cu, seq, pre,
+ q_len, SCALE, out,
+ )
+
+ ref_scores = ref_block_scores(iq, ik, q_pos)
+ sel = ref_select(ref_scores, q_pos, topk, INIT, LOCAL)
+ ref = ref_sparse_attend(q, k, v, sel, q_pos, SCALE)
+ err = (out.float() - ref).abs().max().item()
+ assert err < 3e-2, err
+
+
+def test_prefill_attend_equals_dense_when_topk_covers():
+ """kv_len <= topk * 128 -> the selection covers every visible block, so the
+ sparse attend must equal dense causal attention exactly."""
+ kv_len = q_len = BLK * TOPK # 4 blocks, topk 4
+ q, k, v, iq, ik, q_pos, block_rows, total_pages, nblocks, prefix = _attend_case(
+ kv_len, q_len, seed=3
+ )
+ k_slab = scatter_rows(k, block_rows, total_pages)
+ v_slab = scatter_rows(v, block_rows, total_pages)
+ ik_slab = scatter_rows(ik, block_rows, total_pages, fill=0.0)
+ cu = torch.tensor([0, q_len], dtype=torch.int32, device=DEV)
+ seq = torch.tensor([kv_len], dtype=torch.int32, device=DEV)
+ pre = torch.tensor([prefix], dtype=torch.int32, device=DEV)
+ score = m3_index_score_prefill(
+ iq, ik_slab, block_rows.view(1, -1), cu, seq, pre, q_len, kv_len
+ )
+ topk_idx = m3_index_topk_prefill(score, cu, pre, q_len, TOPK, INIT, LOCAL)
+ out = torch.empty_like(q)
+ m3_sparse_attn_prefill(
+ q, k_slab, v_slab, topk_idx, block_rows.view(1, -1), cu, seq, pre,
+ q_len, SCALE, out,
+ )
+ # dense causal reference
+ group = HQ // KVH
+ ref = torch.empty(q_len, HQ, D, device=DEV, dtype=torch.float32)
+ s_pos = torch.arange(kv_len, device=DEV)
+ for h in range(HQ):
+ kh = h // group
+ logits = (q[:, h].float() @ k[:, kh].float().t()) * SCALE
+ logits = logits.masked_fill(s_pos[None, :] > q_pos[:, None], float("-inf"))
+ ref[:, h] = torch.softmax(logits, dim=-1) @ v[:, kh].float()
+ err = (out.float() - ref).abs().max().item()
+ assert err < 3e-2, err
+
+
[email protected]("topk", [4, 16])
[email protected]("kv_lens", [[BLK * 6 + 17, 70, BLK * 3, BLK * 9]])
+def test_decode_attend(kv_lens: list[int], topk: int):
+ torch.manual_seed(4)
+ bs = len(kv_lens)
+ max_nb = max((L + BLK - 1) // BLK for L in kv_lens)
+ q = torch.randn(bs, HQ, D, device=DEV, dtype=torch.bfloat16)
+ iq = torch.randn(bs, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)
+ block_rows = torch.zeros(bs, max_nb, dtype=torch.int32, device=DEV)
+ total = 0
+ layouts = []
+ for i, L in enumerate(kv_lens):
+ nb = (L + BLK - 1) // BLK
+ br, pages = make_layout(nb, seed=10 + i)
+ layouts.append((br, pages))
+ block_rows[i, :nb] = br + total * BLK
+ total += pages
+ # K/V gaps carry NaN (the serving pool is torch.empty; the kernels must
+ # pos-mask); the index slab stays zeroed (BSAKVCache invariant).
+ k_slab = torch.full((total * BLK, KVH, D), float("nan"), device=DEV, dtype=torch.bfloat16)
+ v_slab = torch.full_like(k_slab, float("nan"))
+ ik_slab = torch.zeros(total * BLK, D_IDX, device=DEV, dtype=torch.bfloat16)
+ ks, vs, iks = [], [], []
+ off = 0
+ for i, L in enumerate(kv_lens):
+ kk = torch.randn(L, KVH, D, device=DEV, dtype=torch.bfloat16)
+ vv = torch.randn(L, KVH, D, device=DEV, dtype=torch.bfloat16)
+ ik = torch.randn(L, D_IDX, device=DEV, dtype=torch.bfloat16)
+ ks.append(kk); vs.append(vv); iks.append(ik)
+ br, pages = layouts[i]
+ k_slab[off * BLK : (off + pages) * BLK] = scatter_rows(kk, br, pages)
+ v_slab[off * BLK : (off + pages) * BLK] = scatter_rows(vv, br, pages)
+ ik_slab[off * BLK : (off + pages) * BLK] = scatter_rows(ik, br, pages, fill=0.0)
+ off += pages
+ seq = torch.tensor(kv_lens, dtype=torch.int32, device=DEV)
+ topk_idx = m3_index_decode(iq, ik_slab, block_rows, seq, max_nb, topk, INIT, LOCAL)
+ out = torch.empty_like(q)
+ m3_sparse_attn_decode(q, k_slab, v_slab, topk_idx, block_rows, seq, SCALE, out)
+
+ for i, L in enumerate(kv_lens):
+ q_pos = torch.tensor([L - 1], device=DEV)
+ ref_scores = ref_block_scores(iq[i : i + 1], iks[i], q_pos)
+ sel = ref_select(ref_scores, q_pos, topk, INIT, LOCAL)
+ ref = ref_sparse_attend(q[i : i + 1], ks[i], vs[i], sel, q_pos, SCALE)
+ err = (out[i].float() - ref[0]).abs().max().item()
+ assert err < 3e-2, (i, err)
+
+
+def test_decode_attend_capture_replay_mutated_lengths():
+ """The decode index+attend pair under CUDA-graph capture, replayed with
+ mutated kv lengths (1 / 129 / 8191 rotated across requests) and refreshed
+ K/V/index data: the split-K partition and every length-dependent mask must
+ be replay-safe, with nothing but ``max_nb`` baked into the capture -- the
+ production graph contract."""
+ topk = 16
+ LENS = [1, 129, 8191]
+ bs = len(LENS)
+ MAXL = max(LENS)
+ max_nb = (MAXL + BLK - 1) // BLK
+ block_rows = torch.zeros(bs, max_nb, dtype=torch.int32, device=DEV)
+ total = 0
+ layouts = []
+ for i in range(bs):
+ br, pages = make_layout(max_nb, seed=20 + i)
+ layouts.append((br, pages))
+ block_rows[i] = br + total * BLK
+ total += pages
+ k_slab = torch.full(
+ (total * BLK, KVH, D), float("nan"), device=DEV, dtype=torch.bfloat16
+ )
+ v_slab = torch.full_like(k_slab, float("nan"))
+ ik_slab = torch.zeros(total * BLK, D_IDX, device=DEV, dtype=torch.bfloat16)
+
+ def refresh(seed):
+ torch.manual_seed(seed)
+ ks, vs, iks = [], [], []
+ off = 0
+ for i in range(bs):
+ kk = torch.randn(MAXL, KVH, D, device=DEV, dtype=torch.bfloat16)
+ vv = torch.randn(MAXL, KVH, D, device=DEV, dtype=torch.bfloat16)
+ ik = torch.randn(MAXL, D_IDX, device=DEV, dtype=torch.bfloat16)
+ ks.append(kk); vs.append(vv); iks.append(ik)
+ br, pages = layouts[i]
+ k_slab[off * BLK : (off + pages) * BLK] = scatter_rows(kk, br, pages)
+ v_slab[off * BLK : (off + pages) * BLK] = scatter_rows(vv, br, pages)
+ ik_slab[off * BLK : (off + pages) * BLK] = scatter_rows(ik, br, pages, fill=0.0)
+ off += pages
+ return ks, vs, iks
+
+ q = torch.randn(bs, HQ, D, device=DEV, dtype=torch.bfloat16)
+ iq = torch.randn(bs, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)
+ seq = torch.tensor(LENS, dtype=torch.int32, device=DEV)
+ out = torch.empty_like(q)
+ data = refresh(100)
+ # warmup outside capture (Triton JIT), then capture the pair
+ ti = m3_index_decode(iq, ik_slab, block_rows, seq, max_nb, topk, INIT, LOCAL)
+ m3_sparse_attn_decode(q, k_slab, v_slab, ti, block_rows, seq, SCALE, out)
+ torch.cuda.synchronize()
+ graph = torch.cuda.CUDAGraph()
+ with torch.cuda.graph(graph):
+ ti = m3_index_decode(iq, ik_slab, block_rows, seq, max_nb, topk, INIT, LOCAL)
+ m3_sparse_attn_decode(q, k_slab, v_slab, ti, block_rows, seq, SCALE, out)
+
+ def check(lens, ks, vs, iks):
+ for i, L in enumerate(lens):
+ q_pos = torch.tensor([L - 1], device=DEV)
+ ref_scores = ref_block_scores(iq[i : i + 1], iks[i][:L], q_pos)
+ sel = ref_select(ref_scores, q_pos, topk, INIT, LOCAL)
+ ref = ref_sparse_attend(q[i : i + 1], ks[i][:L], vs[i][:L], sel, q_pos, SCALE)
+ err = (out[i].float() - ref[0]).abs().max().item()
+ assert err < 3e-2, (lens, i, err)
+ assert torch.isfinite(out[i].float()).all(), (lens, i)
+
+ graph.replay()
+ torch.cuda.synchronize()
+ check(LENS, *data)
+ for step, new_lens in enumerate(([129, 8191, 1], [8191, 1, 129])):
+ data = refresh(200 + step)
+ seq.copy_(torch.tensor(new_lens, dtype=torch.int32, device=DEV))
+ q.normal_()
+ iq.normal_()
+ graph.replay()
+ torch.cuda.synchronize()
+ check(new_lens, *data)
diff --git a/tests/kernels/test_mxfp8_linear.py b/tests/kernels/test_mxfp8_linear.py
new file mode 100644
index 00000000..1f1557d4
--- /dev/null
+++ b/tests/kernels/test_mxfp8_linear.py
@@ -0,0 +1,102 @@
+"""MXFP8 (block-32 e8m0) W8A16 linear: kernels vs the dequant reference, plus the
+Gemma (1+w) norm and swigluoai activation the MiniMax-M3 modules ride on."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+if not torch.cuda.is_available(): # pragma: no cover
+ pytest.skip("CUDA required", allow_module_level=True)
+
+DEV = "cuda"
+
+
+def _make_mxfp8(N: int, K: int, seed: int = 0):
+ torch.manual_seed(seed)
+ wf = torch.randn(N, K, device=DEV) * 0.05
+ codes = torch.randint(110, 132, (N, K // 32), device=DEV, dtype=torch.uint8)
+ descale = torch.exp2(codes.float() - 127.0)
+ w8 = (
+ (wf.view(N, -1, 32) / descale.unsqueeze(-1)).clamp(-448, 448).view(N, K)
+ ).to(torch.float8_e4m3fn)
+ return w8, codes
+
+
+# 1 = m1 kernel; 2..256 = dot GEMV across its M_TILE buckets {16,32,64,128,256}
+# (17/33/129 exercise row-padding masks); 257/300 = dequant+cuBLAS past
+# _GEMV_MAX_M (257 pins the boundary).
[email protected]("M", [1, 2, 8, 16, 17, 33, 64, 129, 256, 257, 300])
+# (511, 6144): N not a BLOCK_N multiple (n-mask tail); (640, 6112): K a multiple
+# of the 32-wide scale block but not of BLOCK_K=128 (k-mask + OOB scale codes).
[email protected]("N,K", [(512, 6144), (9216, 6144), (640, 6144), (511, 6144), (640, 6112)])
+def test_mxfp8_linear_matches_dequant_reference(M: int, N: int, K: int):
+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant, mxfp8_linear
+
+ w8, codes = _make_mxfp8(N, K, seed=M)
+ ref_w = mxfp8_dequant(w8, codes, torch.float32)
+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)
+ y = mxfp8_linear(x, w8, codes)
+ y_ref = (x.float() @ ref_w.t()).to(torch.bfloat16)
+ rel = (y.float() - y_ref.float()).abs().max() / y_ref.float().abs().max().clamp(min=1e-6)
+ assert rel.item() < 2e-2, rel.item()
+
+
+def test_mxfp8_module_shapes_and_forward():
+ from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear
+
+ lin = Mxfp8Linear(6144, 512)
+ assert lin.weight.shape == (512, 6144) and lin.weight.dtype == torch.float8_e4m3fn
+ assert lin.weight_scale_inv.shape == (512, 192) and lin.weight_scale_inv.dtype == torch.uint8
+ w8, codes = _make_mxfp8(512, 6144, seed=42)
+ lin.weight, lin.weight_scale_inv = w8, codes
+ x = torch.randn(3, 6144, device=DEV, dtype=torch.bfloat16)
+ assert lin.forward(x).shape == (3, 512)
+
+
+def test_gemma_plus_one_norm_matches_flashinfer_semantics():
+ """Triton fallback vs the (1+w) definition; per-head 3D strided in-place."""
+ from freetoken.kernel.triton.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm
+
+ torch.manual_seed(0)
+ x = torch.randn(64, 6144, device=DEV, dtype=torch.bfloat16)
+ w = torch.randn(6144, device=DEV, dtype=torch.bfloat16) * 0.1
+
+ def ref(v):
+ vf = v.float()
+ inv = torch.rsqrt(vf.pow(2).mean(-1, keepdim=True) + 1e-6)
+ return (vf * inv * (1.0 + w.float())).to(torch.bfloat16)
+
+ out = gemma_rmsnorm(x, w, 1e-6)
+ assert (out.float() - ref(x).float()).abs().max().item() < 2e-2
+
+ a, r = x.clone(), x.clone()
+ gemma_fused_add_rmsnorm(a, r, w, 1e-6)
+ assert torch.equal(r, (x.float() + x.float()).to(torch.bfloat16))
+ assert (a.float() - ref(r).float()).abs().max().item() < 2e-2
+
+ # per-head strided in-place (the fused-qkv slice pattern)
+ q = torch.randn(16, 9216, device=DEV, dtype=torch.bfloat16)
+ wq = torch.randn(128, device=DEV, dtype=torch.bfloat16) * 0.1
+ qh = q[:, :8192].view(16, 64, 128)
+ ref3 = torch.empty_like(qh)
+ for h in range(64):
+ vf = qh[:, h].float()
+ inv = torch.rsqrt(vf.pow(2).mean(-1, keepdim=True) + 1e-6)
+ ref3[:, h] = (vf * inv * (1.0 + wq.float())).to(torch.bfloat16)
+ gemma_rmsnorm(qh, wq, 1e-6, out=qh)
+ assert (qh.float() - ref3.float()).abs().max().item() < 2e-2
+
+
+def test_swigluoai_and_mul_uninterleaved():
+ from freetoken.layers import swigluoai_and_mul
+
+ torch.manual_seed(1)
+ d = 3072
+ x = torch.randn(500, 2 * d, device=DEV, dtype=torch.bfloat16) * 3
+ gate, up = x[:, :d].float(), x[:, d:].float()
+ alpha, limit = 1.702, 7.0
+ g = gate.clamp(max=limit)
+ ref = g * torch.sigmoid(g * alpha) * (up.clamp(-limit, limit) + 1.0)
+ out = swigluoai_and_mul(x, alpha=alpha, limit=limit)
+ assert (out.float() - ref).abs().max().item() < 0.15
diff --git a/tests/kvcache/test_bsa_pool.py b/tests/kvcache/test_bsa_pool.py
new file mode 100644
index 00000000..73abe270
--- /dev/null
+++ b/tests/kvcache/test_bsa_pool.py
@@ -0,0 +1,116 @@
+"""BSAKVCache: paged GQA K/V + index-key slab (MiniMax-M3 block-sparse pool)."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+if not torch.cuda.is_available(): # pragma: no cover
+ pytest.skip("CUDA required", allow_module_level=True)
+
+from freetoken.kvcache.bsa_pool import BSAKVCache
+
+DEV = torch.device("cuda")
+
+
[email protected](autouse=True)
+def _tp(monkeypatch):
+ from freetoken.distributed.info import DistributedInfo
+
+ monkeypatch.setattr(
+ "freetoken.kvcache.mha_pool.get_tp_info",
+ lambda: DistributedInfo(rank=0, size=1),
+ )
+
+
+def _pool(num_pages=8, page_size=128):
+ return BSAKVCache(
+ num_kv_heads=4,
+ num_layers=5,
+ head_dim=128,
+ num_pages=num_pages,
+ page_size=page_size,
+ dtype=torch.bfloat16,
+ device=DEV,
+ index_head_dim=128,
+ num_index_layers=2,
+ )
+
+
+def test_store_and_views():
+ pool = _pool()
+ assert pool.k_cache(0).shape == (8, 128, 4, 128)
+ assert pool.index_k_cache(0).shape == (8 * 128, 128)
+ assert pool.index_k_cache(1).shape == (8 * 128, 128)
+
+ t = 6
+ k = torch.randn(t, 4 * 128, device=DEV, dtype=torch.bfloat16)
+ v = torch.randn(t, 4 * 128, device=DEV, dtype=torch.bfloat16)
+ ik = torch.randn(t, 128, device=DEV, dtype=torch.bfloat16)
+ out_loc = torch.tensor([128, 129, 130, 200, 201, 202], device=DEV, dtype=torch.int64)
+ pool.store_kv(k, v, out_loc.to(torch.int32), layer_id=3)
+ pool.store_index_k(ik, out_loc, slot=1)
+
+ k_rows = pool.k_cache(3).view(-1, 4, 128)
+ assert torch.equal(k_rows[128], k[0].view(4, 128))
+ assert torch.equal(k_rows[202], k[5].view(4, 128))
+ assert torch.equal(pool.index_k_cache(1)[200], ik[3])
+ # the other slot stays untouched (zero-initialized)
+ assert pool.index_k_cache(0)[200].abs().sum().item() == 0.0
+
+
+def test_rebuild_resizes_both_slabs_atomically():
+ pool = _pool(num_pages=8)
+ pool.rebuild(16)
+ assert pool.k_cache(0).shape == (16, 128, 4, 128)
+ assert pool.index_k_cache(0).shape == (16 * 128, 128)
+ assert pool.index_k_cache(1).shape == (16 * 128, 128)
+
+
+def test_unit_bytes_matches_cost_model():
+ pool = _pool()
+ kv_bytes, swa_bytes = pool.unit_bytes()
+ # 5 layers x 2 slabs x 4 heads x 128 x 2 B + 2 index layers x 128 x 2 B
+ assert kv_bytes == 5 * 2 * 4 * 128 * 2 + 2 * 128 * 2
+ assert swa_bytes == 0
+
+
+def test_factory_builds_bsa_pool():
+ from freetoken.kvcache import create_kvcache_pool
+ from freetoken.models.config import FullAttentionGroupConfig, ModelConfig, RotaryConfig
+
+ rotary = RotaryConfig(head_dim=128, rotary_dim=64, max_position=4096, base=1e4, scaling=None)
+ cfg = ModelConfig(
+ num_layers=5,
+ num_qo_heads=64,
+ num_kv_heads=4,
+ head_dim=128,
+ hidden_size=6144,
+ vocab_size=1000,
+ intermediate_size=1024,
+ rms_norm_eps=1e-6,
+ rotary_config=rotary,
+ hidden_act="swigluoai",
+ tie_word_embeddings=False,
+ num_experts=8,
+ num_experts_per_tok=2,
+ moe_intermediate_size=512,
+ norm_topk_prob=True,
+ model_type="minimax_m3_vl",
+ architectures=["MiniMaxM3SparseForConditionalGeneration"],
+ attention_groups=(
+ FullAttentionGroupConfig(
+ name="full",
+ layer_ids=tuple(range(5)),
+ num_kv_heads=4,
+ head_dim=128,
+ rotary_config=rotary,
+ mla=False,
+ index_head_dim=128,
+ num_index_layers=2,
+ ),
+ ),
+ )
+ pool = create_kvcache_pool(cfg, num_pages=4, page_size=128, dtype=torch.bfloat16, device=DEV)
+ assert isinstance(pool, BSAKVCache)
+ assert pool.index_k_cache(0).shape == (4 * 128, 128)
diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.py
new file mode 100644
index 00000000..787661dc
--- /dev/null
+++ b/tests/models/test_minimax_m3.py
@@ -0,0 +1,263 @@
+"""MiniMax-M3 config parsing, payload semantics and serving-mode resolution.
+
+Drives ``minimax_m3.parse_config`` off a synthetic HF config shaped exactly like the
+nvidia/MiniMax-M3-NVFP4 checkpoint's (multimodal wrapper: text tower in
+``text_config``, quantization_config present) and checks every decision the plan
+pinned: the BSA attention group (pool family + KV cost), the sparse indexer
+geometry, the MoE/router knobs, the resolved quant modes and the env switches.
+"""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from freetoken.attention.base import AttnType
+from freetoken.models.minimax_m3.config import parse_config
+
+
+class _Cfg:
+ """Attribute-access shim over a dict (what AutoConfig hands parse_config)."""
+
+ def __init__(self, data: dict):
+ for k, v in data.items():
+ setattr(self, k, _Cfg(v) if isinstance(v, dict) and k == "text_config" else v)
+
+
+def _hf_config(num_layers: int = 60) -> _Cfg:
+ dense = min(3, num_layers)
+ moe_freq = [0] * dense + [1] * (num_layers - dense)
+ return _Cfg(
+ {
+ "architectures": ["MiniMaxM3SparseForConditionalGeneration"],
+ "model_type": "minimax_m3_vl",
+ "quantization_config": {"quant_algo": "MIXED_PRECISION", "quant_method": "modelopt"},
+ "text_config": {
+ "hidden_size": 6144,
+ "intermediate_size": 3072,
+ "dense_intermediate_size": 12288,
+ "shared_intermediate_size": 3072,
+ "num_hidden_layers": num_layers,
+ "num_attention_heads": 64,
+ "num_key_value_heads": 4,
+ "head_dim": 128,
+ "vocab_size": 200064,
+ "max_position_embeddings": 1048576,
+ "rms_norm_eps": 1e-6,
+ "use_gemma_norm": True,
+ "attention_output_gate": False,
+ "rope_theta": 5000000,
+ "rotary_dim": 64,
+ "partial_rotary_factor": 0.5,
+ "hidden_act": "swigluoai",
+ "swiglu_alpha": 1.702,
+ "swiglu_limit": 7.0,
+ "use_qk_norm": True,
+ "qk_norm_type": "per_head",
+ "num_local_experts": 128,
+ "num_experts_per_tok": 4,
+ "n_shared_experts": 1,
+ "scoring_func": "sigmoid",
+ "use_routing_bias": True,
+ "routed_scaling_factor": 2.0,
+ "moe_layer_freq": moe_freq,
+ "tie_word_embeddings": False,
+ "sparse_attention_config": {
+ "use_sparse_attention": True,
+ "sparse_index_dim": 128,
+ "sparse_num_index_heads": 4,
+ "sparse_topk_blocks": 16,
+ "sparse_block_size": 128,
+ "sparse_disable_index_value": moe_freq,
+ "sparse_score_type": "max",
+ "sparse_init_block": 0,
+ "sparse_local_block": 1,
+ "sparse_attention_freq": moe_freq,
+ },
+ },
+ }
+ )
+
+
+def _hf_config_native(num_layers: int = 60) -> _Cfg:
+ """The transformers >= 5 NATIVE minimax_m3_vl TextConfig shape: __post_init__
+ pops rope_theta into rope_parameters, sparse_attention_config/moe_layer_freq
+ into flat index_* keys + layer_types/mlp_layer_types, and force-sets
+ hidden_act to "silu". parse_config must resolve both shapes identically
+ (verified empirically against transformers 5.14's class)."""
+ cfg = _hf_config(num_layers)
+ text = cfg.text_config
+ dense = min(3, num_layers)
+ delattr(text, "rope_theta")
+ text.rope_parameters = {
+ "rope_theta": 5000000,
+ "partial_rotary_factor": 0.5,
+ "rope_type": "default",
+ }
+ text.hidden_act = "silu"
+ delattr(text, "sparse_attention_config")
+ text.layer_types = ["full_attention"] * dense + ["minimax_m3_sparse"] * (
+ num_layers - dense
+ )
+ text.index_n_heads = 4
+ text.index_head_dim = 128
+ text.index_block_size = 128
+ text.index_topk_blocks = 16
+ text.index_local_blocks = 1
+ delattr(text, "moe_layer_freq")
+ text.mlp_layer_types = ["dense"] * dense + ["sparse"] * (num_layers - dense)
+ return cfg
+
+
+def test_parse_config_native_shape_matches_raw(monkeypatch):
+ """The native shape silently produced rope base 10000, use_sparse=False and
+ hidden_act='silu' before the dual-shape shim -- all three load-bearing reads
+ must resolve identically to the raw-dict shape."""
+ monkeypatch.delenv("FREETOKEN_M3_MAX_LAYERS", raising=False)
+ monkeypatch.delenv("FREETOKEN_M3_SPARSE", raising=False)
+ raw = parse_config(_hf_config())
+ native = parse_config(_hf_config_native())
+
+ assert native.rotary_config.base == raw.rotary_config.base == 5000000.0
+ assert native.hidden_act == raw.hidden_act == "swigluoai"
+ assert native.first_k_dense_replace == raw.first_k_dense_replace == 3
+ a, b = native.m3_args, raw.m3_args
+ assert a.use_sparse and a.sparse_layer_ids == b.sparse_layer_ids
+ assert a.moe_layer_ids == b.moe_layer_ids
+ assert (a.index_dim, a.num_index_heads, a.topk_blocks, a.block_size) == (
+ b.index_dim, b.num_index_heads, b.topk_blocks, b.block_size,
+ )
+ assert (a.init_blocks, a.local_blocks) == (b.init_blocks, b.local_blocks)
+ assert (a.rope_theta, a.rotary_dim) == (b.rope_theta, b.rotary_dim)
+
+
+def test_parse_config_full_model(monkeypatch):
+ monkeypatch.delenv("FREETOKEN_M3_MAX_LAYERS", raising=False)
+ monkeypatch.delenv("FREETOKEN_M3_SPARSE", raising=False)
+ cfg = parse_config(_hf_config())
+
+ assert cfg.num_layers == 60
+ assert (cfg.num_qo_heads, cfg.num_kv_heads, cfg.head_dim) == (64, 4, 128)
+ assert cfg.rotary_config.rotary_dim == 64 and cfg.rotary_config.base == 5000000.0
+ assert cfg.first_k_dense_replace == 3 and cfg.num_moe_layers == 57
+ assert (cfg.num_experts, cfg.num_experts_per_tok) == (128, 4)
+ assert cfg.moe_intermediate_size == 3072 and cfg.intermediate_size == 12288
+ assert cfg.n_shared_experts == 1 and cfg.shared_expert_intermediate_size == 3072
+ assert cfg.routed_scaling_factor == 2.0 and cfg.norm_topk_prob
+ assert cfg.hidden_act == "swigluoai"
+ assert cfg.hidden_act_alpha == pytest.approx(1.702)
+ assert cfg.swiglu_limit == pytest.approx(7.0)
+ assert cfg.expert_quant == "nvfp4"
+ # Default env: MXFP8 dense native, bf16 lm_head.
+ assert cfg.attn_quant == "mxfp8" and cfg.dense_quant == "mxfp8"
+ assert cfg.lm_head_quant == "none"
+
+ # One BSA group over all layers, index slab for the 57 sparse layers.
+ (spec,) = cfg.kv_cache_group_specs()
+ assert spec.attn_type == AttnType.BSA
+ assert not spec.mla
+ assert spec.index_head_dim == 128 and spec.num_index_layers == 57
+
+ args = cfg.m3_args
+ assert args.use_sparse and args.topk_blocks == 16 and args.block_size == 128
+ assert (args.init_blocks, args.local_blocks) == (0, 1)
+ assert args.num_index_heads == 4 and args.index_dim == 128
+ assert args.sparse_layer_ids == tuple(range(3, 60))
+ assert args.moe_layer_ids == tuple(range(3, 60))
+ assert args.sparse_slot(3) == 0 and args.sparse_slot(59) == 56
+
+
+def test_pool_family_and_kv_cost():
+ cfg = parse_config(_hf_config())
+ from freetoken.kvcache import resolve_pool_class
+ from freetoken.kvcache.base import spec_kv_bytes_per_token
+ from freetoken.kvcache.bsa_pool import BSAKVCache
+
+ assert resolve_pool_class(cfg) is BSAKVCache
+
+ class _Budget:
+ model_config = cfg
+ dtype = torch.bfloat16
+ page_size = 128
+
+ class tp_info:
+ size = 1
+
+ (spec,) = cfg.kv_cache_group_specs()
+ # 60 layers x 2 slabs x 4 heads x 128 dims x 2 B + 57 index layers x 128 x 2 B
+ assert spec_kv_bytes_per_token(spec, _Budget) == 60 * 2 * 4 * 128 * 2 + 57 * 128 * 2
+
+
+def test_layer_cap_env(monkeypatch):
+ monkeypatch.setenv("FREETOKEN_M3_MAX_LAYERS", "5")
+ cfg = parse_config(_hf_config())
+ assert cfg.num_layers == 5
+ assert cfg.first_k_dense_replace == 3 and cfg.num_moe_layers == 2
+ (spec,) = cfg.kv_cache_group_specs()
+ assert spec.num_index_layers == 2
+ assert cfg.m3_args.sparse_layer_ids[:2] == (3, 4)
+
+
+def test_sparse_ablation_env(monkeypatch):
+ monkeypatch.setenv("FREETOKEN_M3_SPARSE", "0")
+ cfg = parse_config(_hf_config())
+ (spec,) = cfg.kv_cache_group_specs()
+ assert spec.attn_type == AttnType.FULL
+ assert spec.index_head_dim == 0 and spec.num_index_layers == 0
+ assert not cfg.m3_args.use_sparse
+
+ from freetoken.kvcache import resolve_pool_class
+ from freetoken.kvcache.mha_pool import MHAKVCache
+
+ assert resolve_pool_class(cfg) is MHAKVCache
+
+
+def test_mxfp8_ablation_env(monkeypatch):
+ monkeypatch.setenv("FREETOKEN_M3_ATTN_MXFP8", "0")
+ monkeypatch.setenv("FREETOKEN_M3_MLP_MXFP8", "0")
+ cfg = parse_config(_hf_config())
+ assert cfg.attn_quant == "none" and cfg.dense_quant == "none"
+
+
+def test_registry_resolves_both_architectures():
+ from freetoken.models.register import get_model_spec
+
+ for arch in ("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3SparseForCausalLM"):
+ spec = get_model_spec(arch)
+ assert spec.module == "freetoken.models.minimax_m3"
+
+
+def test_auto_backend_resolution():
+ from freetoken.attention import attention_backend_info
+ from freetoken.engine.engine import _required_attn_types, _resolve_auto_attention_backend
+
+ cfg = parse_config(_hf_config())
+ required = _required_attn_types(cfg)
+ assert required == frozenset({AttnType.BSA})
+ assert _resolve_auto_attention_backend(required, False) == "m3_sparse"
+ assert attention_backend_info("m3_sparse").page_sizes == (128,)
+
+
+def test_nvfp4_backend_restricted_to_triton_for_swigluoai():
+ from freetoken.moe.nvfp4_backends import select_nvfp4_backend
+
+ dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")
+ assert select_nvfp4_backend(dev, 3072, "auto", activation="swigluoai") == "triton"
+ with pytest.raises(RuntimeError):
+ select_nvfp4_backend(dev, 3072, "marlin", activation="swigluoai")
+
+
+def test_expert_source_spec_layer_to_bank():
+ from freetoken.models.minimax_m3.weight import _EXPERT_KEY_RE, _NVFP4_SOURCE_SPEC
+
+ cfg = parse_config(_hf_config())
+ m = _EXPERT_KEY_RE.match(
+ "language_model.model.layers.7.block_sparse_moe.experts.42.w1.weight_scale_2"
+ )
+ assert m and m.group("layer") == "7" and m.group("expert") == "42"
+ assert m.group("proj") == "w1" and m.group("kind") == "weight_scale_2"
+ assert _NVFP4_SOURCE_SPEC.proj_to_role == {"w1": "gate", "w3": "up", "w2": "down"}
+ # Dense prefix leaves no bank holes; MoE layer i maps to bank i-3.
+ assert _NVFP4_SOURCE_SPEC.layer_to_bank(3, cfg) == 0
+ assert _NVFP4_SOURCE_SPEC.layer_to_bank(59, cfg) == 56
+ assert _NVFP4_SOURCE_SPEC.layer_to_bank(0, cfg) is None
diff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.py
index ccc6b7a2..f496de31 100644
--- a/tests/moe/test_cpu_moe.py
+++ b/tests/moe/test_cpu_moe.py
@@ -164,6 +164,94 @@ def test_cpu_decode_nvfp4_matches_dequant_then_gpu(bs):
assert rel < 3e-2, f"nvfp4 bs={bs} rel err {rel.item()}"
[email protected]("bs", [1, 4])
+def test_cpu_decode_nvfp4_swigluoai_matches_dequant_reference(bs):
+ """MiniMax-M3's swigluoai routed experts on the CPU executor (ActKind 3 through
+ the generic NVFP4 GEMV epilogue, with the layer's alpha/limit scalars) vs. a
+ pure-torch dequant reference."""
+ from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4
+ from freetoken.moe.cpu_executor import CpuMoeExecutor
+
+ torch.manual_seed(300 + bs)
+ L, E, H, I, top_k = 2, 16, 1024, 512, 4
+ layer = 1
+ alpha, limit = 1.702, 7.0
+ dev = torch.device("cuda")
+ cache = _make_nvfp4_cache(L, E, H, I, seed=17)
+
+ ex = CpuMoeExecutor(
+ cache,
+ top_k=top_k,
+ activation="swigluoai",
+ apply_router_weight_on_input=False,
+ num_threads=0,
+ max_tokens=bs,
+ device=dev,
+ swiglu_alpha=alpha,
+ swiglu_limit=limit,
+ )
+
+ hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)
+ ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)
+ w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)
+
+ cpu_out = ex.decode(layer, hidden, w, ids).float()
+ torch.cuda.synchronize()
+
+ b = cache.bank_sources
+ slots = torch.arange(E, device=dev, dtype=torch.int32)
+ gate_up_layer = dequant_nvfp4(
+ b["gate_up_packed"][layer].to(dev), b["gate_up_scale"][layer].to(dev),
+ b["gate_up_global"][layer].to(dev), slots, dtype=torch.bfloat16,
+ ).float() # [E, 2I, H]
+ down_layer = dequant_nvfp4(
+ b["down_packed"][layer].to(dev), b["down_scale"][layer].to(dev),
+ b["down_global"][layer].to(dev), slots, dtype=torch.bfloat16,
+ ).float() # [E, H, I]
+
+ ref = torch.zeros(bs, H, device=dev, dtype=torch.float32)
+ x = hidden.float()
+ for t in range(bs):
+ for k in range(top_k):
+ e = int(ids[t, k])
+ h = gate_up_layer[e] @ x[t]
+ gate = h[:I].clamp(max=limit)
+ up = h[I:].clamp(-limit, limit)
+ act = gate * torch.sigmoid(gate * alpha) * (up + 1.0)
+ ref[t] += float(w[t, k]) * (down_layer[e] @ act)
+
+ rel = (cpu_out - ref).abs().max() / (ref.abs().max() + 1e-6)
+ assert rel < 3e-2, f"nvfp4 swigluoai bs={bs} rel err {rel.item()}"
+
+
+def test_stale_extension_rejected_for_swigluoai(monkeypatch):
+ """A prebuilt _cpu_moe.so from before ACT_SWIGLUOAI accepts act id 3 without
+ error and silently computes the wrong activation in the generic epilogue.
+ The executor probes the extension's `max_generic_act_id` marker -- absent on
+ stale builds -- and must fail loudly with the rebuild instruction."""
+ from freetoken.kernel import _cpu_moe
+ from freetoken.moe.cpu_executor import CpuMoeExecutor, compiled_extension_supports
+
+ # raising=False: on a GENUINELY stale extension the attribute is already
+ # absent, and this test must still run (it is the test for that case).
+ monkeypatch.delattr(_cpu_moe, "max_generic_act_id", raising=False)
+ assert not compiled_extension_supports("swigluoai")
+ assert compiled_extension_supports("silu") # silu family predates the marker
+ cache = _make_nvfp4_cache(1, 4, 256, 128)
+ with pytest.raises(RuntimeError, match="rebuild"):
+ CpuMoeExecutor(
+ cache, top_k=2, activation="swigluoai",
+ apply_router_weight_on_input=False, num_threads=0, max_tokens=1,
+ device=torch.device("cuda"), swiglu_alpha=1.702, swiglu_limit=7.0,
+ )
+ # silu predates the marker and must keep working on a stale build.
+ CpuMoeExecutor(
+ cache, top_k=2, activation="silu",
+ apply_router_weight_on_input=False, num_threads=0, max_tokens=1,
+ device=torch.device("cuda"),
+ )
+
+
def _pack_mxfp4_transposed(codes: torch.Tensor) -> torch.Tensor:
"""Pack [S, K, N] codes -> transposed blocks [S, K//2, N] (low nibble = even K)."""
lo = codes[:, 0::2, :]
diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.py
index 5541138f..7005a1b4 100644
--- a/tests/moe/test_nvfp4_backends.py
+++ b/tests/moe/test_nvfp4_backends.py
@@ -84,8 +84,15 @@ def _assert_close(out: torch.Tensor, ref: torch.Tensor) -> None:
torch.testing.assert_close(out.float(), ref, rtol=3e-2, atol=tol)
-def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids) -> torch.Tensor:
- """Dequant + dense per-token reference for the gated-silu MoE."""
+def _swigluoai_ref(h: torch.Tensor, alpha: float = 1.702, limit: float = 7.0) -> torch.Tensor:
+ """MiniMax-M3 / gpt-oss clamped swiglu over UNINTERLEAVED [gate; up] halves."""
+ gate = h[:I].clamp(max=limit)
+ up = h[I:].clamp(-limit, limit)
+ return gate * torch.sigmoid(gate * alpha) * (up + 1.0)
+
+
+def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids, activation="silu") -> torch.Tensor:
+ """Dequant + dense per-token reference for the gated MoE (silu or swigluoai)."""
out = torch.zeros(hidden.shape, dtype=torch.float32, device=hidden.device)
x = hidden.float()
for t in range(hidden.size(0)):
@@ -102,7 +109,10 @@ def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids) -> torch.Tensor:
sources["down_global"][layer_id][e].to(hidden.device),
)
h = gu @ x[t]
- act = torch.nn.functional.silu(h[:I]) * h[I:]
+ if activation == "swigluoai":
+ act = _swigluoai_ref(h)
+ else:
+ act = torch.nn.functional.silu(h[:I]) * h[I:]
out[t] += float(topk_weights[t, j]) * (dn @ act)
return out
@@ -287,6 +297,47 @@ def test_triton_overlap_prefill_matches_dequant_reference():
cache.release_prefill_layer(layer_id)
+@cuda
+def test_triton_swigluoai_matches_dequant_reference():
+ """MiniMax-M3's swigluoai routed experts through the Triton prefill grouped GEMM
+ and the marlin-style decode GEMV: same banks, the clamped (up+1) swiglu instead
+ of silu, alpha/limit threaded through the fused entry points."""
+ from freetoken.moe.fused_nvfp4 import (
+ fused_experts_decode_nvfp4_marlin,
+ fused_experts_nvfp4,
+ )
+
+ device = torch.device("cuda")
+ sources = _make_native_sources(device, seed=11)
+ torch.manual_seed(12)
+ M = 8
+ hidden = torch.randn(M, H, dtype=torch.bfloat16, device=device) / 4
+ topk_ids = torch.randint(0, E, (M, TOPK), dtype=torch.int32, device=device)
+ topk_weights = torch.rand(M, TOPK, dtype=torch.float32, device=device)
+ layer_id = 0
+ banks = [
+ sources[name][layer_id].to(device)
+ for name in (
+ "gate_up_packed", "gate_up_scale", "gate_up_global",
+ "down_packed", "down_scale", "down_global",
+ )
+ ]
+ ref = _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids, activation="swigluoai")
+ out = fused_experts_nvfp4(
+ hidden, *banks, topk_weights, topk_ids, E, "swigluoai", False, 1.702, 7.0
+ )
+ _assert_close(out, ref)
+
+ dec_hidden = hidden[:1]
+ dec_ids = topk_ids[:1]
+ dec_weights = topk_weights[:1]
+ ref = _ref_moe(sources, layer_id, dec_hidden, dec_weights, dec_ids, activation="swigluoai")
+ out = fused_experts_decode_nvfp4_marlin(
+ dec_hidden, *banks, dec_weights, dec_ids, "swigluoai", False, 1.702, 7.0
+ )
+ _assert_close(out, ref)
+
+
def _triton_cache(device, *, cache_size=S, prefill_overlap=False):
"""Native 6-bank NVFP4 cache (no repack), consumed directly by the Triton kernels.
The banks are not transformed, so ``sources`` doubles as the dequant reference."""
diff --git a/tests/server/test_anthropic_api.py b/tests/server/test_anthropic_api.py
index 2478b17a..d8eb0169 100644
--- a/tests/server/test_anthropic_api.py
+++ b/tests/server/test_anthropic_api.py
@@ -158,6 +158,33 @@ def test_convert_system_role_message_and_unknown_block():
assert spec.messages[2]["content"] == "hi"
+def test_convert_thinking_toggle_routes_through_family_mapping():
+ """A hardcoded {"enable_thinking": bool} is inert for M3, whose template
+ reads thinking_mode -- the toggle must go through the same model_meta
+ mapping as the chat-completions gears."""
+ def _req(ttype):
+ return AnthropicMessagesRequest.model_validate(
+ {
+ "model": "m", "max_tokens": 16,
+ "messages": [{"role": "user", "content": "hi"}],
+ "thinking": {"type": ttype, "budget_tokens": 1024},
+ }
+ )
+
+ spec = A.convert_anthropic_to_genspec(_req("enabled"), {}, reasoning_parser="minimax_m3")
+ assert spec.chat_template_kwargs == {"thinking_mode": "enabled"}
+ spec = A.convert_anthropic_to_genspec(_req("disabled"), {}, reasoning_parser="minimax_m3")
+ assert spec.chat_template_kwargs == {"thinking_mode": "disabled"}
+ # enable_thinking families keep their key; no parser keeps the generic key
+ spec = A.convert_anthropic_to_genspec(_req("enabled"), {}, reasoning_parser="qwen3")
+ assert spec.chat_template_kwargs == {"enable_thinking": True}
+ spec = A.convert_anthropic_to_genspec(_req("disabled"), {})
+ assert spec.chat_template_kwargs == {"enable_thinking": False}
+ # a family without the requested direction sets nothing (minimax has no off)
+ spec = A.convert_anthropic_to_genspec(_req("disabled"), {}, reasoning_parser="minimax")
+ assert spec.chat_template_kwargs == {}
+
+
def test_convert_thinking_only_assistant_message_keeps_empty_content():
# Truncated-then-resent turn: survives with content="" for templates that
# concatenate message.content unconditionally.
diff --git a/tests/server/test_minimax_m3_parsers.py b/tests/server/test_minimax_m3_parsers.py
new file mode 100644
index 00000000..e7980903
--- /dev/null
+++ b/tests/server/test_minimax_m3_parsers.py
@@ -0,0 +1,597 @@
+"""MiniMax-M3 reasoning parser (<mm:think>, 3 thinking modes) and tool-call
+detector (namespace-delimited recursive XML) -- one-shot and streaming."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from freetoken.server.function_call_parser import (
+ Function,
+ MiniMaxM3Detector,
+ Tool,
+)
+from freetoken.server.reasoning_parser import MiniMaxM3ReasoningParser, ReasoningParser
+
+NS = "]<]minimax[>["
+
+
+# ---------------------------------------------------------------------------
+# Reasoning parser
+# ---------------------------------------------------------------------------
+def test_reasoning_registered():
+ assert ReasoningParser.ReasoningParserEnum["minimax_m3"] is MiniMaxM3ReasoningParser
+
+
+def test_reasoning_adaptive_with_tags():
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ r = p.detect_and_parse("<mm:think>chain of thought</mm:think>The answer is 4.")
+ assert r.reasoning_text == "chain of thought"
+ assert r.normal_text == "The answer is 4."
+
+
+def test_reasoning_adaptive_without_tags():
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ r = p.detect_and_parse("Plain answer, the model chose not to think.")
+ assert r.reasoning_text == ""
+ assert r.normal_text == "Plain answer, the model chose not to think."
+
+
+def test_reasoning_enabled_implicit_open():
+ # thinking_mode=enabled: the template pre-opens <mm:think>, the model emits
+ # only the closing tag -> the caller constructs with force_reasoning=True.
+ p = MiniMaxM3ReasoningParser(force_reasoning=True)
+ r = p.detect_and_parse("thinking hard</mm:think>final answer")
+ assert r.reasoning_text == "thinking hard"
+ assert r.normal_text == "final answer"
+
+
+def test_reasoning_streaming_split_marker():
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ reasoning, content = [], []
+ for chunk in ["<mm:th", "ink>let me ", "think</mm:t", "hink>done: 42"]:
+ r = p.parse_streaming_increment(chunk)
+ reasoning.append(r.reasoning_text)
+ content.append(r.normal_text)
+ r = p.flush()
+ reasoning.append(r.reasoning_text)
+ content.append(r.normal_text)
+ assert "".join(reasoning) == "let me think"
+ assert "".join(content) == "done: 42"
+
+
+def test_reasoning_tool_block_ends_missing_close():
+ # A malformed turn skipping </mm:think> straight into a tool block: the block
+ # must land in normal_text for the tool parser (dsv4 precedent).
+ p = MiniMaxM3ReasoningParser(force_reasoning=True)
+ text = f"half a thought{NS}<tool_call>...{NS}</tool_call>"
+ r = p.detect_and_parse(text)
+ assert r.reasoning_text == "half a thought"
+ assert r.normal_text.startswith(f"{NS}<tool_call>")
+
+
+# ---------------------------------------------------------------------------
+# Tool-call detector
+# ---------------------------------------------------------------------------
+def _tools():
+ return [
+ Tool(
+ function=Function(
+ name="create_order",
+ parameters={
+ "type": "object",
+ "properties": {
+ "user_id": {"type": "integer"},
+ "note": {"type": "string"},
+ "shipping": {"type": "object"},
+ "items": {"type": "array"},
+ },
+ },
+ )
+ ),
+ Tool(function=Function(name="get_weather", parameters={
+ "type": "object", "properties": {"city": {"type": "string"}},
+ })),
+ ]
+
+
+def _block(inner: str) -> str:
+ return f"{NS}<tool_call>\n{inner}{NS}</tool_call>"
+
+
+_NESTED_INVOKE = (
+ f'{NS}<invoke name="create_order">'
+ f"{NS}<user_id>42{NS}</user_id>"
+ f"{NS}<note>two books{NS}</note>"
+ f"{NS}<shipping>"
+ f"{NS}<city>Singapore{NS}</city>"
+ f"{NS}<zip>018956{NS}</zip>"
+ f"{NS}</shipping>"
+ f"{NS}<items>"
+ f"{NS}<item>"
+ f"{NS}<sku>book-001{NS}</sku>"
+ f"{NS}<qty>2{NS}</qty>"
+ f"{NS}</item>"
+ f"{NS}</items>"
+ f"{NS}</invoke>\n"
+)
+
+
+def test_detect_and_parse_recursive_args():
+ det = MiniMaxM3Detector()
+ text = "I'll place the order." + _block(_NESTED_INVOKE)
+ assert det.has_tool_call(text)
+ res = det.detect_and_parse(text, _tools())
+ assert res.normal_text == "I'll place the order."
+ assert len(res.calls) == 1
+ call = res.calls[0]
+ assert call.name == "create_order"
+ args = json.loads(call.parameters)
+ assert args["user_id"] == 42 # schema-typed integer
+ assert args["note"] == "two books" # schema-typed string
+ # nested leaves are loose-JSON typed: numbers parse, but a leading-zero token
+ # like "018956" is not valid JSON and stays a string (never silently mangled)
+ assert args["shipping"] == {"city": "Singapore", "zip": "018956"}
+ assert args["items"] == [{"sku": "book-001", "qty": 2}]
+
+
+def test_detect_and_parse_multiple_invokes():
+ det = MiniMaxM3Detector()
+ two = (
+ _NESTED_INVOKE
+ + f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n'
+ )
+ res = det.detect_and_parse(_block(two), _tools())
+ assert [c.name for c in res.calls] == ["create_order", "get_weather"]
+ assert json.loads(res.calls[1].parameters) == {"city": "Paris"}
+
+
+def test_detect_and_parse_unknown_tool_follows_forwarding_policy():
+ # FreeToken forwards unknown tool names by default (FORWARD_UNKNOWN_TOOLS);
+ # the detector must honor the same policy switch as every other family.
+ import freetoken.server.function_call_parser as fcp
+
+ det = MiniMaxM3Detector()
+ text = _block(f'{NS}<invoke name="nope">{NS}<a>1{NS}</a>{NS}</invoke>\n')
+ res = det.detect_and_parse(text, _tools())
+ assert [c.name for c in res.calls] == ["nope"] # default: forwarded
+
+ orig = fcp.FORWARD_UNKNOWN_TOOLS
+ try:
+ fcp.FORWARD_UNKNOWN_TOOLS = False
+ res = MiniMaxM3Detector().detect_and_parse(text, _tools())
+ assert res.calls == [] # strict mode: dropped with a warning
+ finally:
+ fcp.FORWARD_UNKNOWN_TOOLS = orig
+
+
+def test_quoted_invoke_opener_is_data_not_call():
+ """A parameter value quoting the invoke wire syntax must parse as data --
+ a raw regex over the block used to spawn a phantom call and diverge from
+ streaming."""
+ quoted = f'see {NS}<invoke name="write_file"> for the syntax'
+ text = _block(
+ f'{NS}<invoke name="get_weather">{NS}<city>{quoted}{NS}</city>{NS}</invoke>\n'
+ )
+ res = MiniMaxM3Detector().detect_and_parse(text, _tools())
+ assert [c.name for c in res.calls] == ["get_weather"]
+ assert json.loads(res.calls[0].parameters) == {"city": quoted}
+ # streaming agrees (it already produced one call before the fix; pin it)
+ det = MiniMaxM3Detector()
+ r = det.parse_streaming_increment(text, _tools())
+ assert [c.name for c in r.calls if c.name is not None] == ["get_weather"]
+
+
+def test_quoted_wrapper_closer_is_data_not_block_end():
+ """A value quoting NS</tool_call> must not end the wrapper: before the fix
+ the parameter was dropped ({}) and the rest of the block leaked as raw
+ markup content."""
+ quoted = f"wrap calls in {NS}</tool_call> at the end"
+ inner = f'{NS}<invoke name="get_weather">{NS}<city>{quoted}{NS}</city>{NS}</invoke>\n'
+ text = _block(inner) + " done."
+ res = MiniMaxM3Detector().detect_and_parse(text, _tools())
+ assert [c.name for c in res.calls] == ["get_weather"]
+ assert json.loads(res.calls[0].parameters) == {"city": quoted}
+ assert res.normal_text == "done."
+ # chunked streaming: same call, no markup leak
+ det = MiniMaxM3Detector()
+ calls, texts = [], []
+ for i in range(0, len(text), 7):
+ r = det.parse_streaming_increment(text[i : i + 7], _tools())
+ calls += [c for c in r.calls if c.name is not None]
+ texts.append(r.normal_text)
+ assert [c.name for c in calls] == ["get_weather"]
+ assert NS not in "".join(texts)
+
+
+def test_multiline_string_argument_verbatim():
+ """The template renders leaf values verbatim; the round-trip must preserve
+ them verbatim too (stripping ate the trailing newline of multi-line string
+ arguments)."""
+ tools = [
+ Tool(function=Function(name="write_file", parameters={
+ "type": "object",
+ "properties": {"path": {"type": "string"}, "content": {"type": "string"}},
+ }))
+ ]
+ content = "line1\nline2\n"
+ text = _block(
+ f'{NS}<invoke name="write_file">'
+ f"{NS}<path>/tmp/x{NS}</path>"
+ f"{NS}<content>{content}{NS}</content>"
+ f"{NS}</invoke>\n"
+ )
+ res = MiniMaxM3Detector().detect_and_parse(text, tools)
+ assert json.loads(res.calls[0].parameters) == {"path": "/tmp/x", "content": content}
+
+
+def test_nested_schema_types_string_leaves():
+ """Nested leaves honor the declared schema (string-typed fields stay
+ strings instead of loose-JSON numbers/bools); undeclared nested leaves keep
+ the loose-JSON fallback."""
+ tools = [
+ Tool(function=Function(name="create_order", parameters={
+ "type": "object",
+ "properties": {
+ "shipping": {
+ "type": "object",
+ "properties": {
+ "zip": {"type": "string"},
+ "note": {"type": "string"},
+ "floor": {"type": "integer"},
+ },
+ },
+ "tags": {"type": "array", "items": {"type": "string"}},
+ },
+ }))
+ ]
+ text = _block(
+ f'{NS}<invoke name="create_order">'
+ f"{NS}<shipping>"
+ f"{NS}<zip>94107{NS}</zip>"
+ f"{NS}<note>true{NS}</note>"
+ f"{NS}<floor>3{NS}</floor>"
+ f"{NS}</shipping>"
+ f"{NS}<tags>"
+ f"{NS}<item>1{NS}</item>"
+ f"{NS}<item>two{NS}</item>"
+ f"{NS}</tags>"
+ f"{NS}</invoke>\n"
+ )
+ args = json.loads(MiniMaxM3Detector().detect_and_parse(text, tools).calls[0].parameters)
+ assert args["shipping"] == {"zip": "94107", "note": "true", "floor": 3}
+ assert args["tags"] == ["1", "two"]
+
+
+def test_streaming_text_then_call():
+ det = MiniMaxM3Detector()
+ full = "Let me check." + _block(
+ f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n'
+ )
+ # Chop into small chunks that split the namespace marker mid-token.
+ chunks = [full[i : i + 7] for i in range(0, len(full), 7)]
+ normal, calls = "", []
+ for ch in chunks:
+ r = det.parse_streaming_increment(ch, _tools())
+ normal += r.normal_text
+ calls.extend(r.calls)
+ assert normal == "Let me check."
+ named = [c for c in calls if c.name]
+ frags = [c for c in calls if c.name is None and c.parameters]
+ assert [c.name for c in named] == ["get_weather"]
+ assert json.loads("".join(c.parameters for c in frags)) == {"city": "Paris"}
+ # ledgers the serving layer reads at stream end
+ assert det.prev_tool_call_arr[0]["name"] == "get_weather"
+ assert det.prev_tool_call_arr[0]["arguments"] == {"city": "Paris"}
+
+
+def test_streaming_multiple_invokes_and_indices():
+ det = MiniMaxM3Detector()
+ two = (
+ _NESTED_INVOKE
+ + f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n'
+ )
+ full = _block(two)
+ chunks = [full[i : i + 13] for i in range(0, len(full), 13)]
+ calls = []
+ for ch in chunks:
+ calls.extend(det.parse_streaming_increment(ch, _tools()).calls)
+ named = [(c.tool_index, c.name) for c in calls if c.name]
+ assert named == [(0, "create_order"), (1, "get_weather")]
+
+
+def test_streaming_plain_text_never_held():
+ det = MiniMaxM3Detector()
+ r = det.parse_streaming_increment("Just a normal answer, no tools.", _tools())
+ assert r.normal_text == "Just a normal answer, no tools."
+ assert r.calls == []
+
+
+def test_auto_selection_picks_minimax_m3():
+ from unittest.mock import patch
+
+ from freetoken.server.args import parse_args
+
+ class _Config:
+ def to_dict(self):
+ return {
+ "architectures": ["MiniMaxM3SparseForConditionalGeneration"],
+ "model_type": "minimax_m3_vl",
+ "text_config": {"model_type": "minimax_m2"},
+ }
+
+ with patch("freetoken.utils.cached_load_hf_config", lambda _p: _Config()):
+ args, _ = parse_args(["--model", "/models/anon"])
+ assert args.tool_call_parser == "minimax_m3"
+ assert args.reasoning_parser == "minimax_m3"
+
+
+# ---------------------------------------------------------------------------
+# Review regressions
+# ---------------------------------------------------------------------------
+def test_reasoning_adaptive_leading_bare_closer_one_shot():
+ # Adaptive non-thinking turns START with a bare </mm:think> written by the
+ # model; it must never leak into content (position 0 only).
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ r = p.detect_and_parse("</mm:think>Here is the answer.")
+ assert r.reasoning_text == ""
+ assert r.normal_text == "Here is the answer."
+
+
+def test_reasoning_adaptive_leading_bare_closer_streaming_split():
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ normal = ""
+ for chunk in ["</mm:th", "ink>Hello,", " world!"]:
+ r = p.parse_streaming_increment(chunk)
+ assert r.reasoning_text == ""
+ normal += r.normal_text
+ normal += p.flush().normal_text
+ assert normal == "Hello, world!"
+
+
+def test_reasoning_leading_closer_after_whitespace():
+ # The detokenizer may open with a newline before the model's bare closer;
+ # the whitespace-tolerant head still strips it (one-shot and streaming).
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ r = p.detect_and_parse("\n</mm:think>Here is the answer.")
+ assert r.reasoning_text == "" and r.normal_text == "Here is the answer."
+
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ normal = ""
+ for chunk in ["\n", "</mm:think>", "Hi."]:
+ out = p.parse_streaming_increment(chunk)
+ assert out.reasoning_text == ""
+ normal += out.normal_text
+ normal += p.flush().normal_text
+ assert normal == "Hi."
+
+
+def test_reasoning_later_closer_stays_visible():
+ # Only a position-0 closer is stripped; one appearing later is content.
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ r = p.detect_and_parse("The tag </mm:think> is literal here.")
+ assert "</mm:think>" in r.normal_text
+
+
+def test_reasoning_forced_mode_closer_not_stripped():
+ # thinking_mode=enabled: the template pre-opened the block, so a leading
+ # closer legitimately ends (empty) reasoning -- content follows.
+ p = MiniMaxM3ReasoningParser(force_reasoning=True)
+ r = p.detect_and_parse("</mm:think>answer")
+ assert r.reasoning_text == "" and r.normal_text == "answer"
+
+
+def test_reasoning_streaming_head_prefix_replayed_on_divergence():
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ out = p.parse_streaming_increment("</mm") # closer prefix: held
+ assert out.normal_text == "" and out.reasoning_text == ""
+ out = p.parse_streaming_increment("ory says hi") # diverges: replayed
+ assert out.normal_text == "</mmory says hi"
+
+
+def test_streaming_wire_order_trailing_text_defers():
+ # "text + block + trailing text" in ONE chunk: the trailing text must not be
+ # emitted in the same increment as (i.e. wire-ahead of) the tool call.
+ det = MiniMaxM3Detector()
+ chunk = (
+ "Before. "
+ + _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')
+ + " After."
+ )
+ r1 = det.parse_streaming_increment(chunk, _tools())
+ assert r1.normal_text == "Before. "
+ assert [c.name for c in r1.calls if c.name] == ["get_weather"]
+ r2 = det.parse_streaming_increment("", _tools())
+ assert r2.normal_text == " After." and r2.calls == []
+
+
+def test_streaming_second_block_in_same_chunk():
+ det = MiniMaxM3Detector()
+ one = _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')
+ two = _block(f'{NS}<invoke name="get_weather">{NS}<city>Tokyo{NS}</city>{NS}</invoke>\n')
+ calls = []
+ calls += det.parse_streaming_increment(one + two, _tools()).calls
+ calls += det.parse_streaming_increment("", _tools()).calls
+ args = [json.loads(c.parameters) for c in calls if c.name is None and c.parameters]
+ assert args == [{"city": "Paris"}, {"city": "Tokyo"}]
+
+
+def test_streaming_residue_after_close_holds_partial_marker():
+ # After a closed wrapper, a split second-block marker in the residue must be
+ # held (idle-path hold), never leaked as content.
+ det = MiniMaxM3Detector()
+ one = _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')
+ r1 = det.parse_streaming_increment(one + "]<]mini", _tools())
+ assert "]<]mini" not in r1.normal_text
+ r2 = det.parse_streaming_increment(
+ f'max[>[<tool_call>\n{NS}<invoke name="get_weather">{NS}<city>Rome{NS}</city>'
+ f"{NS}</invoke>\n{NS}</tool_call>",
+ _tools(),
+ )
+ frags = [c.parameters for c in r2.calls if c.name is None and c.parameters]
+ assert json.loads(frags[-1]) == {"city": "Rome"}
+
+
+def test_streaming_truncated_call_suppressed_and_recovered():
+ # Generation cut mid-invoke (max_tokens): finish_streaming must not leak the
+ # raw markup, and recover_truncated_call must salvage the complete params.
+ from freetoken.server.function_call_parser import FunctionCallParser
+
+ parser = FunctionCallParser(_tools(), "minimax_m3")
+ det = parser.detector
+ truncated = (
+ "Let me order. " + NS + "<tool_call>\n"
+ + f'{NS}<invoke name="create_order">{NS}<user_id>42{NS}</user_id>{NS}<note>hi'
+ )
+ normal = ""
+ for i in range(0, len(truncated), 11):
+ r = det.parse_streaming_increment(truncated[i : i + 11], _tools())
+ normal += r.normal_text
+ assert not r.calls
+ assert normal == "Let me order. "
+ # Serving order (generation.py): recover first (consumes the buffer on
+ # success), then finish_stream drains what's left.
+ recovered = parser.recover_truncated_call()
+ assert [c.name for c in recovered] == ["create_order"]
+ args = json.loads(recovered[0].parameters)
+ assert args["user_id"] == 42 # the complete param survives; truncated one dropped
+ assert det.finish_streaming() == "" # raw markup never reaches the client
+
+
+def test_args_stray_text_keeps_parsed_params():
+ det = MiniMaxM3Detector()
+ body = f"{NS}<city>Paris{NS}</city>zzz-stray"
+ text = _block(f'{NS}<invoke name="get_weather">{body}{NS}</invoke>\n')
+ res = det.detect_and_parse(text, _tools())
+ assert json.loads(res.calls[0].parameters) == {"city": "Paris"}
+
+
+def test_args_empty_value_is_empty_string():
+ det = MiniMaxM3Detector()
+ text = _block(f'{NS}<invoke name="get_weather">{NS}<city>{NS}</city>{NS}</invoke>\n')
+ res = det.detect_and_parse(text, _tools())
+ assert json.loads(res.calls[0].parameters) == {"city": ""}
+
+
+def test_args_repeated_siblings_keep_parent_key():
+ """Only <item> children render as a bare array; repeated same-name siblings
+ under any other tag stay an object with an array-valued key (the bare-list
+ collapse dropped the parent key the references keep)."""
+ det = MiniMaxM3Detector()
+ body = (
+ f"{NS}<items>{NS}<sku>a{NS}</sku>{NS}<sku>b{NS}</sku>{NS}</items>"
+ f"{NS}<note>x{NS}</note>{NS}<note>y{NS}</note>"
+ )
+ text = _block(f'{NS}<invoke name="create_order">{body}{NS}</invoke>\n')
+ res = det.detect_and_parse(text, _tools())
+ args = json.loads(res.calls[0].parameters)
+ assert args["items"] == {"sku": ["a", "b"]} # parent key kept, values aggregate
+ assert args["note"] == ["x", "y"] # repeated top-level params still aggregate
+
+
+def test_element_semantics_reference_batch():
+ """Element-grammar batch: empty container-typed params, anyOf coercion,
+ dangling-closer leniency, single-quoted invoke name, $text for mixed
+ content, and float preservation."""
+ tools = [
+ Tool(function=Function(name="t", parameters={
+ "type": "object",
+ "properties": {
+ "obj": {"type": "object"},
+ "arr": {"type": "array"},
+ "uni": {"anyOf": [{"type": "integer"}, {"type": "string"}]},
+ "price": {"type": "number"},
+ },
+ }))
+ ]
+ det = MiniMaxM3Detector()
+ body = (
+ f"{NS}<obj>{NS}</obj>"
+ f"{NS}<arr>{NS}</arr>"
+ f"{NS}<uni>42{NS}</uni>"
+ f"{NS}<price>5.0{NS}</price>"
+ )
+ text = _block(f'{NS}<invoke name="t">{body}{NS}</invoke>\n')
+ args = json.loads(det.detect_and_parse(text, tools).calls[0].parameters)
+ assert args["obj"] == {} and args["arr"] == [] # typed empty containers
+ assert args["uni"] == 42 # anyOf: integer member coerces
+ assert args["price"] == 5.0 and isinstance(args["price"], float) # not 5
+
+ # anyOf falls back to the string member verbatim
+ text2 = _block(f'{NS}<invoke name="t">{NS}<uni>hello{NS}</uni>{NS}</invoke>\n')
+ args2 = json.loads(MiniMaxM3Detector().detect_and_parse(text2, tools).calls[0].parameters)
+ assert args2["uni"] == "hello"
+
+ # a dangling closer no longer drops the well-formed siblings after it
+ body3 = f"{NS}</ghost>{NS}<uni>7{NS}</uni>"
+ text3 = _block(f'{NS}<invoke name="t">{body3}{NS}</invoke>\n')
+ args3 = json.loads(MiniMaxM3Detector().detect_and_parse(text3, tools).calls[0].parameters)
+ assert args3["uni"] == 7
+
+ # single-quoted name= parses (double quotes are the template's rendering,
+ # but models emit single quotes too)
+ text4 = _block(f"{NS}<invoke name='t'>{NS}<uni>1{NS}</uni>{NS}</invoke>\n")
+ res4 = MiniMaxM3Detector().detect_and_parse(text4, tools)
+ assert [c.name for c in res4.calls] == ["t"]
+
+ # mixed text+children keeps the text under $text
+ body5 = f"{NS}<obj>freeform {NS}<k>v{NS}</k>{NS}</obj>"
+ text5 = _block(f'{NS}<invoke name="t">{body5}{NS}</invoke>\n')
+ args5 = json.loads(MiniMaxM3Detector().detect_and_parse(text5, tools).calls[0].parameters)
+ assert args5["obj"] == {"k": "v", "$text": "freeform"}
+
+
+def test_reasoning_one_shot_matches_streaming_positionally():
+ """One-shot and streaming must agree positionally: prose before a
+ mid-content <mm:think> stays content, and a later quoted marker occurrence
+ is data."""
+ text = "intro <mm:think>deep thought</mm:think> answer quoting <mm:think> literally"
+ one = MiniMaxM3ReasoningParser(force_reasoning=False).detect_and_parse(text)
+ assert one.reasoning_text == "deep thought"
+ assert one.normal_text == "intro answer quoting <mm:think> literally"
+
+ p = MiniMaxM3ReasoningParser(force_reasoning=False)
+ reasoning, content = [], []
+ for i in range(0, len(text), 3):
+ r = p.parse_streaming_increment(text[i : i + 3])
+ reasoning.append(r.reasoning_text)
+ content.append(r.normal_text)
+ r = p.flush()
+ reasoning.append(r.reasoning_text)
+ content.append(r.normal_text)
+ assert "".join(reasoning) == one.reasoning_text
+ assert "".join(content) == one.normal_text
+
+
+def test_reasoning_one_shot_verbatim_no_strip():
+ # whitespace around reasoning/content is data (references are verbatim)
+ r = MiniMaxM3ReasoningParser(force_reasoning=False).detect_and_parse(
+ "<mm:think>\nthink\n</mm:think>\nanswer\n"
+ )
+ assert r.reasoning_text == "\nthink\n"
+ assert r.normal_text == "\nanswer\n"
+ r2 = MiniMaxM3ReasoningParser(force_reasoning=True).detect_and_parse(
+ "th\n</mm:think>\nans\n"
+ )
+ assert r2.reasoning_text == "th\n" and r2.normal_text == "\nans\n"
+
+
+def test_detect_and_parse_multiple_wrappers_and_inter_block_text():
+ det = MiniMaxM3Detector()
+ one = _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')
+ two = _block(f'{NS}<invoke name="get_weather">{NS}<city>Rome{NS}</city>{NS}</invoke>\n')
+ res = det.detect_and_parse("A. " + one + " middle " + two + " B.", _tools())
+ assert [json.loads(c.parameters)["city"] for c in res.calls] == ["Paris", "Rome"]
+ assert "middle" in res.normal_text and "A." in res.normal_text and "B." in res.normal_text
+
+
+ "gear,mode", [("off", "disabled"), ("adaptive", "adaptive"), ("on", "enabled")]
+)
+def test_think_gears(gear, mode):
+ from freetoken.server.model_meta import think_chat_template_kwargs, think_spec
+
+ gears, default = think_spec("minimax_m3")
+ assert gears == ("off", "adaptive", "on") and default == "adaptive"
+ assert think_chat_template_kwargs("minimax_m3", gear) == {"thinking_mode": mode}
diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.py
index 549d925a..ef9e602d 100644
--- a/tests/server/test_responses_api.py
+++ b/tests/server/test_responses_api.py
@@ -888,3 +888,23 @@ def test_convert_reasoning_effort_none_disables_thinking():
assert RP.convert_responses_to_genspec(req, {}).chat_template_kwargs == {
"enable_thinking": False
}
+
+
+def test_convert_reasoning_toggle_routes_through_family_mapping():
+ """The toggle goes through model_meta's per-family mapping -- for M3 that is
+ thinking_mode, not the (inert) enable_thinking key."""
+ req = ResponsesRequest.model_validate(
+ {"model": "m", "input": "hi", "reasoning": {"effort": "high"}}
+ )
+ spec = RP.convert_responses_to_genspec(req, {}, reasoning_parser="minimax_m3")
+ assert spec.chat_template_kwargs == {
+ "thinking_mode": "enabled", "reasoning_effort": "high",
+ }
+ off = ResponsesRequest.model_validate(
+ {"model": "m", "input": "hi", "reasoning": {"effort": "none"}}
+ )
+ spec = RP.convert_responses_to_genspec(off, {}, reasoning_parser="minimax_m3")
+ assert spec.chat_template_kwargs == {"thinking_mode": "disabled"}
+ # gpt-oss: the template grades effort and has no off gear
+ spec = RP.convert_responses_to_genspec(req, {}, reasoning_parser="gpt_oss")
+ assert spec.chat_template_kwargs == {"reasoning_effort": "high"}
diff --git a/tests/server/test_streaming_model_matrix.py b/tests/server/test_streaming_model_matrix.py
index 5f0b8590..a0c94a5d 100644
--- a/tests/server/test_streaming_model_matrix.py
+++ b/tests/server/test_streaming_model_matrix.py
@@ -80,6 +80,13 @@ CALL_BLOCKS = {
'<minimax:tool_call><invoke name="read"><parameter name="filePath">'
"/tmp/test_calc.py</parameter></invoke></minimax:tool_call>"
),
+ "minimax_m3": (
+ "]<]minimax[>[<tool_call>\n"
+ ']<]minimax[>[<invoke name="read">'
+ "]<]minimax[>[<filePath>/tmp/test_calc.py]<]minimax[>[</filePath>"
+ "]<]minimax[>[</invoke>\n"
+ "]<]minimax[>[</tool_call>"
+ ),
"mistral": '[TOOL_CALLS] [{"name": "read", "arguments": {"filePath": "/tmp/test_calc.py"}}]',
"llama3": '<|python_tag|>{"name": "read", "arguments": {"filePath": "/tmp/test_calc.py"}}',
"gpt_oss": (
@@ -96,6 +103,7 @@ MARKUP_MARKERS = {
"glm47": ["<arg_key>", "<arg_value>", "</tool_call>"],
"gemma4": ["<|tool_call>", "<tool_call|>"],
"minimax": ["<minimax:tool_call>", "<invoke"],
+ "minimax_m3": ["]<]minimax[>[", "<invoke"],
"mistral": ["[TOOL_CALLS]"],
"llama3": ["<|python_tag|>"],
"gpt_oss": ["<|channel|>", "to=functions."],
@@ -110,6 +118,9 @@ REASONING_FAMILIES = {
"qwen": ("qwen25", "qwen3", "", "</think>"),
"glm4.7": ("glm47", "glm", "", "</think>"),
"minimax-m2": ("minimax", "minimax", "", "</think>"),
+ # M3 adaptive mode: the model opens <mm:think> itself (enabled mode pre-opens it
+ # in the template; the parser then runs with force_reasoning=True instead).
+ "minimax-m3": ("minimax_m3", "minimax_m3", "<mm:think>", "</mm:think>"),
"gemma4": ("gemma4", "gemma4", "<|channel>thought\n", "<channel|>"),
"gpt-oss": ("gpt_oss", "gpt_oss", None, None), # harmony channels; custom fixture
}
@@ -629,7 +640,9 @@ def test_empty_arguments_call_emitted_exactly_once(tool, block):
@pytest.mark.parametrize(
- "family", ["qwen25", "qwen3_coder", "glm47", "gemma4", "minimax", "deepseekv32", "gpt_oss"]
+ "family",
+ ["qwen25", "qwen3_coder", "glm47", "gemma4", "minimax", "minimax_m3",
+ "deepseekv32", "gpt_oss"],
)
def test_call_then_trailing_text_in_one_chunk_keeps_order(family):
# Worst case: the whole generation arrives as ONE chunk. Text after the call