FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
Support minimax m3 (#1)
Shuo Yang9dc3ff4ca524
diff --git a/python/freetoken/attention/__init__.py b/python/freetoken/attention/__init__.pyindex 9339b9bd..746c04c4 100644--- a/python/freetoken/attention/__init__.py+++ b/python/freetoken/attention/__init__.py@@ -116,6 +116,22 @@ def create_dsa_backend(config: ModelConfig):return DSAAttnBackend(config)+@SUPPORTED_ATTENTION_BACKENDS.register(+ "m3_sparse",+ BackendInfo(+ supported_types=frozenset({AttnType.BSA}),+ # One KV page == one 128-token sparse block: the top-k block ids ARE page+ # indices and the block-base-row addressing needs page-aligned 128-row runs.+ # Config-time resolution coerces any other page size here.+ page_sizes=(128,),+ ),+)+def create_m3_sparse_backend(config: ModelConfig):+ from .m3_sparse import M3SparseAttnBackend++ return M3SparseAttnBackend(config)++def attention_backend_info(name: str) -> BackendInfo:return SUPPORTED_ATTENTION_BACKENDS.info(name)diff --git a/python/freetoken/attention/base.py b/python/freetoken/attention/base.pyindex 5f1be8fb..eb39d472 100644--- a/python/freetoken/attention/base.py+++ b/python/freetoken/attention/base.py@@ -21,6 +21,9 @@ class AttnType(str, Enum):DSA = "dsa" # latent-KV MLA + DSA sparse indexer -> DSAKVCacheDSV4 = "dsv4" # DSV4 window+compressed sparse -> DSV4PagedKVCacheLINEAR = "linear" # GDN/mamba state layers -> LinearStatePool+ # GQA block-sparse (MiniMax-M3): paged GQA K/V + a per-sparse-layer index-key+ # slab; the indexer picks top-k 128-token blocks per query -> BSAKVCache+ BSA = "bsa"@propertydef backend_driven(self) -> bool:diff --git a/python/freetoken/attention/fi.py b/python/freetoken/attention/fi.pyindex 7f4f46cf..c9e58538 100644--- a/python/freetoken/attention/fi.py+++ b/python/freetoken/attention/fi.py@@ -87,12 +87,32 @@ class FlashInferBackend(BaseAttnBackend):self.config = configself.kvcache = get_global_ctx().kv_cacheself.device = self.kvcache.device- # 256 MiB: fa2 split-KV prefill needs num_qo_heads * padded_batch_size *- # cta_tile_q * head_dim * 4 bytes of tmp_v scratch. For head_dim=256 models- # (e.g. Qwen3.5/3.6 MoE), a short extend-prefill over a long cached prefix- # splits KV into many chunks and overflows the previous 128 MiB buffer.+ # fa2 split-KV prefill needs ``tmp_v <= qo_heads_local * padded_batch_size *+ # cta_tile_q * head_dim * 4`` bytes of scratch, where flashinfer's scheduler+ # caps ``padded_batch_size`` at ~``2 * SM / kv_heads_local`` and+ # ``cta_tile_q`` is 128 (64 at head_dim >= 256); when the cap can't be met+ # it disables split-KV and allocates no tmp_v at all. The original flat+ # 128 MiB overflowed on head_dim=256 extend-prefills (Qwen3.5/3.6 MoE) and+ # the flat 256 MiB on MiniMax-M3's 64-head dense layers (H100, 132 SMs:+ # 64 heads x ceil(2*132/4)=66 padded rows x 128 x 128 x 4 B = 264 MiB of+ # tmp_v, over the flat buffer). Derive the bound+ # from the model's TP-LOCAL geometry + this device's SM count, with slack+ # for the tmp_s/merge siblings, floored at the old 256 MiB -- geometries+ # that never exceeded the flat buffer (e.g. GLM-4.7's 96q/8kv) stay at it.+ tp_size = get_tp_info().size+ qo_local = div_even(config.num_qo_heads, tp_size)+ kv_local = div_even(config.num_kv_heads, tp_size, allow_replicate=True)+ sm_count = (+ torch.cuda.get_device_properties(self.device).multi_processor_count+ if self.device.type == "cuda"+ else 128+ )+ cta_tile_q = 64 if config.head_dim >= 256 else 128+ padded_batch = -(-2 * sm_count // max(1, kv_local))+ tmp_v_bound = qo_local * padded_batch * cta_tile_q * config.head_dim * 4+ workspace_bytes = max(256 * 1024 * 1024, tmp_v_bound + 32 * 1024 * 1024)self.float_workspace_buffer = torch.empty(- 256 * 1024 * 1024, dtype=torch.uint8, device=self.device+ workspace_bytes, dtype=torch.uint8, device=self.device)self.prefill_wrapper = BatchPrefillWithPagedKVCacheWrapper(self.float_workspace_buffer,diff --git a/python/freetoken/attention/m3_sparse.py b/python/freetoken/attention/m3_sparse.pynew file mode 100644index 00000000..19515f90--- /dev/null+++ b/python/freetoken/attention/m3_sparse.py@@ -0,0 +1,443 @@+"""MiniMax-M3 block-sparse GQA attention backend.++Serves ``AttnType.BSA`` (one attention group over all layers: paged GQA K/V plus the+sparse layers' index-key slab, ``kvcache/bsa_pool.py``). Two entry points:++* dense leading layers call the generic :meth:`forward` -- delegated to a wrapped+ FULL backend (fa/fi/triton, picked by availability; page-size agnostic, since every+ generic backend addresses the pool by per-token rows regardless of the allocator's+ page size). The wrapped backend owns its own metadata; this backend's metadata+ carries it and the delegation swaps ``batch.attn_metadata`` around each call.+* sparse layers call :meth:`bsa_forward` with this token's (index_q, index_k): the+ backend scatters K/V and the index keys into the pool, scores 128-token blocks per+ index head (== per KV head), selects the top-``topk_blocks`` (+ forced init/local)+ blocks per query, and attends only those (``kernel/triton/minimax_m3_sparse.py``).++Addressing: the engine pins ``page_size == 128`` (this backend's ``page_sizes``+declaration), so one KV page IS one sparse block and the page table's every-128th+column is the block's base row -- ``block_rows[req, blk] = page_table[t, blk*128]``.++* **decode** stages the per-request block-base rows + live lengths into static+ buffers (``prepare_for_replay``, dsa/dsv4 precedent) and runs score -> top-k ->+ attend entirely on device-read lengths with shape-fixed grids, so the whole path+ lives inside the captured CUDA graph.+* **prefill/extend** is eager: per request, in query chunks bounded by the fp32+ score transient's budget (the [chunk, H, n_blocks] block-score tile), score ++ causal top-k + attend. Short contexts are exact dense attention by construction+ (top-k covers every visible block when ``kv_len <= topk_blocks * 128``).+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import TYPE_CHECKING, List++import torch+from freetoken.core import Batch, get_global_ctx+from freetoken.utils import init_logger++from .base import AttentionSpec, BaseAttnBackend, BaseAttnMetadata++logger = init_logger(__name__)++if TYPE_CHECKING:+ from freetoken.models import ModelConfig++_CPU_PINNED = {"device": "cpu", "dtype": torch.int32, "pin_memory": True}+# Prefill scoring transient budget: the fp32 block-score tile is+# [num_index_heads, chunk, n_blocks], so the query chunk shrinks as the context+# grows. Worst case is bounded by the model's max position (floor 16 queries x+# 4 heads x 8192 blocks x 4 B = 2 MB per iteration), not open-ended.+_PREFILL_SCORE_BYTES = 256 << 20+_PREFILL_SCORE_CHUNK = 4096+++def _pick_inner_backend(block_size: int) -> str:+ """FULL backend for the dense leading layers, resolved through the engine's+ own FULL auto-tree (one resolver, so the two trees cannot diverge), then+ filtered for page-size compatibility: this pool serves 128-token pages, so a+ candidate that pins other page sizes (trtllm: 16/32/64) is skipped.++ ``FREETOKEN_M3_INNER_BACKEND`` overrides (e.g. "triton" on a box with a+ broken flashinfer JIT toolchain); the override is validated so a+ page-incompatible or non-FULL pick fails at construction with a real+ message instead of mis-addressing the pool.+ """+ import os++ from freetoken.attention import attention_backend_info+ from freetoken.attention.base import AttnType++ def _page_ok(name: str) -> bool:+ for part in name.split(","):+ sizes = attention_backend_info(part).page_sizes+ if sizes is not None and block_size not in sizes:+ return False+ return True++ override = os.getenv("FREETOKEN_M3_INNER_BACKEND")+ if override:+ for part in override.split(","):+ try:+ info = attention_backend_info(part)+ except KeyError:+ raise ValueError(+ f"FREETOKEN_M3_INNER_BACKEND={override!r}: unknown attention "+ f"backend {part!r}"+ ) from None+ if AttnType.FULL not in info.supported_types:+ raise ValueError(+ f"FREETOKEN_M3_INNER_BACKEND={override!r}: {part!r} does not "+ "serve FULL attention (the dense leading layers)."+ )+ if not _page_ok(override):+ raise ValueError(+ f"FREETOKEN_M3_INNER_BACKEND={override!r} cannot address the "+ f"{block_size}-token pages this model's pool uses."+ )+ logger.info(f"m3_sparse dense-layer backend: {override} (env override)")+ return override++ from freetoken.engine.engine import _resolve_auto_attention_backend++ name = _resolve_auto_attention_backend(frozenset({AttnType.FULL}), False)+ if not _page_ok(name):+ # trtllm (the sm_100 first pick) pins 16/32/64-token pages; walk the rest+ # of the SAME tree (same arch gates, same requirement probes) with the+ # page filter applied.+ from freetoken.engine.engine import _backend_requirements_met+ from freetoken.utils.arch import is_sm90_family++ skipped = name+ for candidate, arch_ok in (("fa,fi", is_sm90_family()), ("fi", True), ("triton", True)):+ if arch_ok and _page_ok(candidate) and _backend_requirements_met(candidate):+ name = candidate+ break+ else: # pragma: no cover - triton is unconditional+ name = "triton"+ logger.info(+ f"m3_sparse dense-layer backend: {name} (auto; skipped {skipped!r}: "+ f"it cannot address this pool's {block_size}-token pages)"+ )+ else:+ logger.info(f"m3_sparse dense-layer backend: {name} (auto)")+ return name+++@dataclass+class M3SparseMetadata(BaseAttnMetadata):+ # fmt: off+ is_decode: bool+ last_indices: torch.Tensor # gpu+ qo_indptr_cpu: torch.Tensor # cpu pinned int32 [bs+1] (prefill request slicing)+ kv_len_cpu: torch.Tensor # cpu pinned int32 [bs]+ inner: BaseAttnMetadata # the wrapped FULL backend's metadata+ # decode addressing: per-request block-base rows (page_table[:, ::128] snapshot)+ # + live lengths, device-read. None on prefill / until staged.+ block_rows: torch.Tensor | None = None+ kvlen: torch.Tensor | None = None+ # prefill addressing plan, built ONCE at the first sparse layer's forward and+ # reused by all 57 (the tensors are layer-invariant): per request, its+ # contiguous block-base rows plus per-query-chunk (slice, cu, seq, prefix,+ # chunk_len) tuples. None until built / on decode.+ prefill_plan: list | None = None+ # fmt: on++ def get_last_indices(self, bs: int) -> torch.Tensor:+ return self.last_indices[:bs]+++class M3SparseAttnBackend(BaseAttnBackend):+ def __init__(self, config: ModelConfig) -> None:+ from freetoken.kvcache.bsa_pool import BSAKVCache++ args = config.m3_args+ assert args is not None and args.use_sparse, (+ "m3_sparse backend needs ModelConfig.m3_args with sparse attention on "+ "(the dense ablation resolves to AttnType.FULL and a generic backend)"+ )+ self.config = config+ self.args = args+ self.num_heads = config.num_qo_heads+ self.num_kv_heads = config.num_kv_heads+ self.head_dim = config.head_dim+ self.kv_attn_dim = self.num_kv_heads * self.head_dim+ self.block_size = args.block_size+ self.topk_blocks = args.topk_blocks+ self.sm_scale = config.attn_sm_scale or (config.head_dim**-0.5)+ self.kvcache = get_global_ctx().kv_cache+ self.device = self.kvcache.device+ # The serving switch is the POOL TYPE (parse_config resolved it into the+ # attention-group spec): this backend requires the index-key slab.+ assert isinstance(self.kvcache, BSAKVCache), (+ f"m3_sparse backend needs a BSA pool, got {type(self.kvcache).__name__}"+ )+ # layer -> index-slab slot (sparse-layer order; matches the pool's slots).+ self._idx_slot = {lid: args.sparse_slot(lid) for lid in args.sparse_layer_ids}++ # Wrapped FULL backend for the dense leading layers.+ from freetoken.attention import create_attention_backend++ self._inner_name = _pick_inner_backend(self.block_size)+ self.inner = create_attention_backend(self._inner_name, config)++ # decode staging (static buffers under CUDA graphs; eager decode builds+ # per-forward tensors lazily instead)+ self._block_rows_buf: torch.Tensor | None = None+ self._kvlen_buf: torch.Tensor | None = None+ self.capture_bs: List[int] = []++ # ----- slab views -------------------------------------------------------------------+ def _k_rows(self, layer_id: int) -> torch.Tensor:+ cache = self.kvcache.k_cache(layer_id) # [pages, page_size, kvh, d]+ return cache.view(-1, cache.shape[2], cache.shape[3])++ def _v_rows(self, layer_id: int) -> torch.Tensor:+ cache = self.kvcache.v_cache(layer_id)+ return cache.view(-1, cache.shape[2], cache.shape[3])++ # ----- metadata ---------------------------------------------------------------------+ def prepare_metadata(self, batch: Batch) -> None:+ # The wrapped backend builds ITS metadata first (it owns the dense layers'+ # addressing); ours wraps it and the per-call delegation swaps them.+ self.inner.prepare_metadata(batch)+ inner_md = batch.attn_metadata+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs+ seqlens_q = [r.extend_len for r in reqs]+ seqlens_k = [r.device_len for r in reqs]+ is_decode = getattr(batch, "phase", None) == "decode"+ qo_indptr = torch.tensor([0] + seqlens_q, **_CPU_PINNED).cumsum_(0).to(torch.int32)+ kv_len = torch.tensor(seqlens_k, **_CPU_PINNED)+ last = (qo_indptr[1:].to(torch.int32) - 1).to(self.device, non_blocking=True)+ batch.attn_metadata = M3SparseMetadata(+ is_decode=is_decode,+ last_indices=last,+ qo_indptr_cpu=qo_indptr,+ kv_len_cpu=kv_len,+ inner=inner_md,+ )+ # Decode addressing (block_rows/kvlen) is DEFERRED: a graph-bound step stages+ # it into the static buffers (prepare_for_replay) and an eager step snapshots+ # lazily at the first sparse layer's bsa_forward.++ # ----- dense layers (delegated) -----------------------------------------------------+ def forward(+ self,+ q: torch.Tensor,+ k: torch.Tensor,+ v: torch.Tensor,+ layer_id: int,+ batch: Batch,+ attn_spec: AttentionSpec | None = None,+ ) -> torch.Tensor:+ md = batch.attn_metadata+ assert isinstance(md, M3SparseMetadata)+ batch.attn_metadata = md.inner+ try:+ # The wrapped backend stores K/V itself (its forward contract).+ return self.inner.forward(q, k, v, layer_id, batch, attn_spec=attn_spec)+ finally:+ batch.attn_metadata = md++ # ----- sparse layers ----------------------------------------------------------------+ def bsa_forward(+ self,+ q: torch.Tensor, # [T, HQ, D]+ k: torch.Tensor, # [T, KVH * D]+ v: torch.Tensor, # [T, KVH * D]+ index_q: torch.Tensor, # [T, Hi, Di]+ index_k: torch.Tensor, # [T, Di]+ layer_id: int,+ batch: Batch,+ ) -> torch.Tensor:+ md = batch.attn_metadata+ assert isinstance(md, M3SparseMetadata)+ slot = self._idx_slot[layer_id]+ self.kvcache.store_kv(k, v, batch.out_loc, layer_id)+ # Scatter index keys unconditionally: short prefills select every block+ # TODAY, but their keys must exist once decode's history passes the top-k.+ self.kvcache.store_index_k(index_k, batch.out_loc, slot)++ if md.is_decode:+ if md.block_rows is None:+ # Eager decode (not graph-staged): per-request block-base SNAPSHOT+ # (the live page-table row may mutate for the next batch while this+ # one runs) + live lengths, once per step at the first sparse layer.+ md.block_rows = self._decode_block_rows(batch)+ md.kvlen = md.kv_len_cpu.to(self.device, non_blocking=True)+ return self._decode(md, layer_id, slot, q, index_q)+ return self._prefill(md, layer_id, slot, q, index_q, batch)++ # ----- decode (CUDA-graph capturable, single code path) ------------------------------+ def _decode(self, md, layer_id: int, slot: int, q, index_q) -> torch.Tensor:+ from freetoken.kernel.triton.minimax_m3_sparse import (+ m3_index_decode,+ m3_sparse_attn_decode,+ )++ block_rows, kvlen = md.block_rows, md.kvlen+ topk_idx = m3_index_decode(+ index_q,+ self.kvcache.index_k_cache(slot),+ block_rows,+ kvlen,+ block_rows.shape[1],+ self.topk_blocks,+ self.args.init_blocks,+ self.args.local_blocks,+ )+ out = torch.empty_like(q)+ m3_sparse_attn_decode(+ q, self._k_rows(layer_id), self._v_rows(layer_id),+ topk_idx, block_rows, kvlen, self.sm_scale, out,+ )+ return out++ # ----- prefill / extend (eager) -------------------------------------------------------+ def _prefill_plan(self, md, batch: Batch) -> list:+ """Per-request addressing plan, built ONCE per forward (layer-invariant --+ rebuilding it per sparse layer cost ~3 H2D copies x chunks x 57 layers of+ stream-serializing overhead): ``[(block_rows [1, n_blocks], kv_len,+ [(slice, cu, seq, prefix, chunk_len), ...]), ...]``."""+ if md.prefill_plan is not None:+ return md.prefill_plan+ reqs = batch.padded_reqs if hasattr(batch, "padded_reqs") else batch.reqs+ page_table = get_global_ctx().page_table+ qo = md.qo_indptr_cpu.tolist()+ plan = []+ for i, r in enumerate(reqs):+ m = qo[i + 1] - qo[i]+ if m == 0:+ continue+ kv_len = r.device_len+ n_blocks = -(-kv_len // self.block_size)+ # Contiguous per-request block-base rows (the [::block_size] view is+ # strided; the kernels read unit-stride rows). Tiny: n_blocks ints.+ block_rows = (+ page_table[r.table_idx, : kv_len : self.block_size]+ .to(torch.int32)+ .contiguous()+ .view(1, -1)+ )+ # Bound the fp32 [H, chunk, n_blocks] score transient (worst case capped+ # by max_position: 16 x 4 x (1M/128) x 4 B = 2 MB per chunk iteration).+ per_q = 4 * self.args.num_index_heads * max(n_blocks, 1)+ chunk = max(16, min(_PREFILL_SCORE_CHUNK, _PREFILL_SCORE_BYTES // per_q))+ seq = torch.tensor([kv_len], dtype=torch.int32, device=self.device)+ chunks = []+ for s0 in range(0, m, chunk):+ s1 = min(s0 + chunk, m)+ chunks.append((+ slice(qo[i] + s0, qo[i] + s1),+ torch.tensor([0, s1 - s0], dtype=torch.int32, device=self.device),+ seq,+ torch.tensor([r.cached_len + s0], dtype=torch.int32, device=self.device),+ s1 - s0,+ ))+ plan.append((block_rows, kv_len, chunks))+ md.prefill_plan = plan+ return plan++ def _prefill(self, md, layer_id: int, slot: int, q, index_q, batch: Batch) -> torch.Tensor:+ from freetoken.kernel.triton.minimax_m3_sparse import (+ m3_index_score_prefill,+ m3_index_topk_prefill,+ m3_sparse_attn_prefill,+ )++ k_rows = self._k_rows(layer_id)+ v_rows = self._v_rows(layer_id)+ ik_rows = self.kvcache.index_k_cache(slot)+ out = torch.empty_like(q)++ for block_rows, kv_len, chunks in self._prefill_plan(md, batch):+ for sl, cu, seq, prefix, chunk_len in chunks:+ score = m3_index_score_prefill(+ index_q[sl], ik_rows, block_rows, cu, seq, prefix,+ chunk_len, kv_len,+ )+ topk_idx = m3_index_topk_prefill(+ score, cu, prefix, chunk_len,+ self.topk_blocks, self.args.init_blocks, self.args.local_blocks,+ )+ del score+ m3_sparse_attn_prefill(+ q[sl], k_rows, v_rows, topk_idx, block_rows,+ cu, seq, prefix, chunk_len, self.sm_scale, out[sl],+ )+ return out++ # ----- CUDA graph (decode) ------------------------------------------------------------+ def init_capture_graph(self, max_seq_len: int, bs_list: List[int]) -> None:+ self.inner.init_capture_graph(max_seq_len, bs_list)+ self.capture_bs = sorted(bs_list)+ max_bs = max(bs_list)+ width = get_global_ctx().page_table.shape[1]+ n_blocks = -(-width // self.block_size)+ self._block_rows_buf = torch.zeros(+ (max_bs, n_blocks), dtype=torch.int32, device=self.device+ )+ self._kvlen_buf = torch.zeros(max_bs, dtype=torch.int32, device=self.device)++ def _block_base_view(self) -> torch.Tensor:+ """Every-``block_size``-th column of the page table: the per-block base+ rows. A strided VIEW -- gathering rows through it materializes only the+ [bs, W/128] result, never the full-width [bs, W] table snapshot (2 MiB/req+ at a 512K-token table; the staging-width class this repo fixed for DSV4)."""+ return get_global_ctx().page_table[:, :: self.block_size]++ def _decode_block_rows(self, batch: Batch) -> torch.Tensor:+ """This decode step's per-request block-base rows [bs, W/128], gathered off+ the scheduler-staged ``active_table_idx`` (a device tensor -- no host loop)."""+ assert batch.active_table_idx is not None, "decode batch is missing its page-table rows"+ return (+ self._block_base_view()+ .index_select(0, batch.active_table_idx.to(torch.int64))+ .to(torch.int32)+ )++ def _stage_decode(self, batch: Batch, bs: int, table_idx: torch.Tensor) -> None:+ """Copy this step's addressing into the static graph buffers and point the+ metadata at them (restage-per-replay, dsa precedent)."""+ md = batch.attn_metadata+ self._block_rows_buf[:bs].copy_(self._block_base_view().index_select(0, table_idx))+ self._kvlen_buf[:bs].copy_(md.kv_len_cpu.to(self.device, non_blocking=True))+ md.block_rows = self._block_rows_buf[:bs]+ md.kvlen = self._kvlen_buf[:bs]++ def prepare_for_capture(self, batch: Batch) -> None:+ # The wrapped backend builds its per-bs graph machinery first (its+ # prepare_for_capture rebuilds batch.attn_metadata as its own), then this+ # backend re-wraps the metadata and stages the dummy request's rows for+ # every slot -- replays overwrite with the live rows.+ self.inner.prepare_for_capture(batch)+ inner_md = batch.attn_metadata+ self.prepare_metadata(batch)+ md = batch.attn_metadata+ assert isinstance(md, M3SparseMetadata)+ md.inner = inner_md+ bs = batch.size+ dummy = torch.full(+ (bs,), batch.padded_reqs[0].table_idx, dtype=torch.int64, device=self.device+ )+ self._stage_decode(batch, bs, dummy)++ def prepare_for_replay(self, batch: Batch) -> None:+ md = batch.attn_metadata+ assert isinstance(md, M3SparseMetadata)+ assert batch.active_table_idx is not None, "decode batch is missing its page-table rows"+ batch.attn_metadata = md.inner+ try:+ self.inner.prepare_for_replay(batch)+ finally:+ batch.attn_metadata = md+ self._stage_decode(batch, batch.padded_size, batch.active_table_idx.to(torch.int64))++ def reset_capture(self) -> None:+ super().reset_capture()+ self.inner.reset_capture()+ self._block_rows_buf = None+ self._kvlen_buf = None+++__all__ = ["M3SparseAttnBackend", "M3SparseMetadata"]diff --git a/python/freetoken/engine/engine.py b/python/freetoken/engine/engine.pyindex 2f4505ac..ff3c985f 100644--- a/python/freetoken/engine/engine.py+++ b/python/freetoken/engine/engine.py@@ -125,6 +125,8 @@ def _resolve_auto_attention_backend(candidates.append(("dsv4_sparse", True))if required & {AttnType.MLA, AttnType.DSA}:candidates.append(("dsa", True))+ if AttnType.BSA in required:+ candidates.append(("m3_sparse", True))if AttnType.SWA in required:candidates.append(("triton", True))if AttnType.FULL in required:@@ -173,7 +175,7 @@ def _validate_attention_backend_choice(config, override, required: frozenset[Attif missing:valid = [name- for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse")+ for name in ("fa", "fi", "trtllm", "triton", "dsa", "dsv4_sparse", "m3_sparse")if required <= attention_backend_info(name).supported_types]missing_names = "/".join(sorted(t.value for t in missing))@@ -253,6 +255,13 @@ def _make_dummy_weight_state_dict(state_dict[key] = telif param.dtype.is_floating_point or param.dtype.is_complex:state_dict[key] = torch.randn(param.shape, dtype=param.dtype, device=device)+ elif param.dtype == torch.uint8 and key.endswith("weight_scale_inv"):+ # MXFP8 e8m0 exponent codes: 127 encodes scale 1.0; zeros would collapse+ # every scale to 2^-127 and zero the model. Scoped BY NAME: other uint8+ # buffers are packed payloads whose bytes mean something else entirely+ # (GGUF qweight blocks embed fp16 scales -- 0x7F7F is fp16 NaN), so they+ # keep the benign all-zeros fill below.+ state_dict[key] = torch.full(param.shape, 127, dtype=param.dtype, device=device)else:state_dict[key] = torch.zeros(param.shape, dtype=param.dtype, device=device)return state_dict@@ -1154,6 +1163,26 @@ def _adjust_config(config: EngineConfig):# lists, then validate whatever is now selected (explicit or auto) -- every# comma part must serve every required type, with packages/arch available.required_attn_types = _required_attn_types(model_config)+ _dtype = getattr(config, "dtype", None) # duck-typed test configs omit it+ if AttnType.BSA in required_attn_types and _dtype is not None and _dtype.itemsize != 2:+ # Reject at config time: the BSA pool's own assert only fires after the+ # model is resident (and not at all under `python -O`).+ raise ValueError(+ f"--dtype {config.dtype}: block-sparse attention serves 16-bit "+ "compute only (the index slab budgets 2 bytes/token); use bfloat16 "+ "or float16."+ )+ if _dtype == torch.float16 and "mxfp8" in (+ getattr(model_config, "attn_quant", "none"),+ getattr(model_config, "dense_quant", "none"),+ ):+ # The MXFP8 GEMV folds the pow2-descaled fp8 weight into the activation+ # dtype; fp16's narrow exponent can overflow/flush what bf16 represents+ # exactly, and the combination was never numerically validated.+ raise ValueError(+ "--dtype float16 with MXFP8 resident weights is unsupported (the "+ "W8A16 fold is only validated exact in bfloat16); use bfloat16."+ )if config.attention_backend == "auto":override("attention_backend",@@ -1166,6 +1195,36 @@ def _adjust_config(config: EngineConfig):total_experts = config.model_config.num_moe_layers * config.model_config.num_expertsoverride("moe_cache_size", math.ceil(total_experts * config.moe_cache_rate))+ # The CPU MoE executor supports the silu/gelu family plus the clamped+ # swigluoai (csrc ActKind; "gpt_oss_swiglu" rides inside the mxfp4 kernel and+ # swigluoai the generic GEMV epilogue). A model with any other expert+ # activation cannot decode on the CPU: reject an explicit cpu/hybrid pick at+ # config time, and keep auto from upgrading offload -> hybrid off the profile.+ _cpu_moe_acts = (+ "silu", "swish", "gelu", "gelu_tanh", "gelu_pytorch_tanh", "swigluoai",+ )+ # hidden_act (the dense activation) stands proxy for the expert activation --+ # true for every in-tree model. mxfp4 experts pass regardless: their act runs+ # inside the mxfp4 kernel, not the generic epilogue.+ _cpu_moe_act_ok = getattr(model_config, "hidden_act", "silu") in _cpu_moe_acts or (+ getattr(model_config, "moe_weight_format", None) == "mxfp4"+ )+ if (+ is_moe+ and not _cpu_moe_act_ok+ and (config.moe_backend in ("cpu", "hybrid") or config.moe_cpu_layers)+ ):+ asked = (+ f"--moe-cpu-layers={config.moe_cpu_layers!r}"+ if config.moe_backend not in ("cpu", "hybrid")+ else f"--moe-backend {config.moe_backend!r}"+ )+ raise ValueError(+ f"{asked}: the CPU MoE executor does not support this model's expert "+ f"activation {getattr(model_config, 'hidden_act', None)!r}; drop the flag "+ "and let every layer decode on the GPU offload path instead."+ )+if is_moe and config.moe_backend == "auto":# A MoE model always defaults to the offload family: experts stream from pinned host# banks into an auto-sized GPU slot cache, which is the only default that serves a model@@ -1188,10 +1247,29 @@ def _adjust_config(config: EngineConfig):gpu_name = torch.cuda.get_device_name(0) if torch.cuda.is_available() else Noneif load_backend_recommendation(bench_fmt, gpu_name=gpu_name) == "hybrid":- default_backend = "hybrid"- logger.info_rank0(- f"benchbw profile recommends hybrid for {bench_fmt!r} experts on this GPU"- )+ from freetoken.moe.cpu_executor import compiled_extension_supports++ _act = getattr(model_config, "hidden_act", "silu")+ if not _cpu_moe_act_ok:+ logger.info_rank0(+ f"benchbw profile recommends hybrid, but the CPU MoE executor does not "+ f"support this model's expert activation "+ f"{getattr(model_config, 'hidden_act', None)!r}; staying on offload"+ )+ elif moe_wfmt != "mxfp4" and not compiled_extension_supports(_act):+ # Stale prebuilt _cpu_moe.so: an explicit cpu/hybrid pick still+ # hard-fails in the executor, but a default must not turn into a+ # post-load crash -- degrade to offload.+ logger.info_rank0(+ f"benchbw profile recommends hybrid, but the compiled _cpu_moe "+ f"extension predates activation {_act!r} (rebuild with "+ f"`python setup.py build_ext --inplace`); staying on offload"+ )+ else:+ default_backend = "hybrid"+ logger.info_rank0(+ f"benchbw profile recommends hybrid for {bench_fmt!r} experts on this GPU"+ )override("moe_backend", default_backend)logger.info_rank0(f"Auto-selected MoE backend: {config.moe_backend}")diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 4068263f..3d195a43 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -266,6 +266,25 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (expert_formats=_NVFP4_FORMATS,aliases=("MiniMaxAI/MiniMax-M2.5",),),+ AotModel(+ # GQA block-sparse (BSA): the BSAKVCache stores K/V through store_cache+ # (4 kv heads x 128 head_dim), the index-key slab writes via torch scatter.+ # Routed experts are NVFP4 with swigluoai, which restricts the expert GEMM+ # to the Triton kernels -- but the banks keep the native "nvfp4" layout,+ # so its bank rows are what fast_index_copy sees.+ # The upstream MiniMaxAI/MiniMax-M3 alias covers the attention/embedding+ # shapes only: its experts are MXFP8, which has no expert-bank provider on+ # main -- the NVFP4 release is the servable offload path (M2.5 precedent).+ name="nvidia/MiniMax-M3-NVFP4",+ architecture="MiniMaxM3SparseForConditionalGeneration",+ hidden_size=6144,+ kv_groups=((4, 128),),+ top_k=4,+ moe_intermediate_size=3072,+ expert_formats=("nvfp4",),+ aliases=("MiniMaxAI/MiniMax-M3",),+ arch_aliases=("MiniMaxM3SparseForCausalLM",),+ ),AotModel(name="deepseek-ai/DeepSeek-V4-Flash",architecture="DeepseekV4ForCausalLM",diff --git a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cppindex 6f734f5c..880e8637 100644--- a/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp+++ b/python/freetoken/kernel/csrc/cpu_moe/cpu_moe_ext.cpp@@ -67,7 +67,11 @@ inline bf16_t f32_to_bf16(float f) {return static_cast<bf16_t>(u >> 16);}-enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2 };+// ACT_SWIGLUOAI is the clamped (up + 1) swiglu (gpt-oss "swigluoai" /+// MiniMax-M3): gate/up are combined jointly with the runtime alpha/limit+// scalars, so it is handled in the do_pass1 epilogue (act_apply never sees it;+// the mxfp4 kernel additionally fuses its own copy of the same math).+enum ActKind { ACT_SILU = 0, ACT_GELU = 1, ACT_GELU_TANH = 2, ACT_SWIGLUOAI = 3 };inline float act_apply(int act, float x) {if (act == ACT_SILU) return x / (1.0f + std::exp(-x));@@ -1600,13 +1604,25 @@ struct CpuMoeExecutor {bf16_t* g_row = g_scratch.data() + ((size_t)tok * top_k + k) * I;const int i0 = static_cast<int>(ib) * IBLK;const int i1 = std::min(I, i0 + IBLK);+ const bool swigluoai = act == ACT_SWIGLUOAI;+ const float lim = swiglu_limit, alpha = swiglu_alpha;for (int i = i0; i < i1; ++i) {// gate = row i, up = row I+i- const float gate =+ float gate =gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, i, x_row, xe, xo, xi8, xas) * w_in;- const float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,- xe, xo, xi8, xas) * w_in;- g_row[i] = f32_to_bf16(act_apply(act, gate) * up);+ float up = gemm1_dot(gate_up_l, gu_packed_l, gu_scale_l, gu_global_l, e, I + i, x_row,+ xe, xo, xi8, xas) * w_in;+ if (swigluoai) {+ // clamp(gate, max=lim) * sigmoid(alpha * gate) * (clamp(up, +-lim) + 1)+ // -- same math as the mxfp4 kernel's fused epilogue (lim == +inf: no clamp).+ if (gate > lim) gate = lim;+ if (up > lim) up = lim;+ else if (up < -lim) up = -lim;+ const float glu = gate / (1.0f + std::exp(-gate * alpha));+ g_row[i] = f32_to_bf16(glu * (up + 1.0f));+ } else {+ g_row[i] = f32_to_bf16(act_apply(act, gate) * up);+ }}}@@ -2124,4 +2140,11 @@ PYBIND11_MODULE(TORCH_EXTENSION_NAME, m) {py::arg("ready_addr"), py::arg("slot"));m.def("memop_sync", &cumemop_sync, py::arg("stream"), py::arg("done_addr"),py::arg("slot"));+ // ABI capability marker: the highest ActKind this build implements in the+ // GENERIC epilogue. CpuMoeExecutor.__init__ probes it before requesting an act+ // id the epilogue must handle -- a prebuilt .so from before ACT_SWIGLUOAI+ // accepts id 3 without error and silently computes the wrong activation+ // (act_apply falls through to gelu_tanh); the probe turns a stale extension+ // into a loud rebuild instruction instead of wrong model outputs.+ m.def("max_generic_act_id", []() { return static_cast<int>(ACT_SWIGLUOAI); });}diff --git a/python/freetoken/kernel/triton/activation.py b/python/freetoken/kernel/triton/activation.pyindex 252a211f..2c38b533 100644--- a/python/freetoken/kernel/triton/activation.py+++ b/python/freetoken/kernel/triton/activation.py@@ -26,6 +26,11 @@ from freetoken.utils.arch import is_sm90_supportedSILU = 0GELU = 1GELU_TANH = 2+# SwiGLU-OAI (gpt-oss / MiniMax-M3 "swigluoai"), UNINTERLEAVED halves (gate [:d],+# up [d:], matching this file's *_and_mul layout -- the interleaved gpt-oss bank+# variant lives in triton/mxfp4_moe.py):+# y = clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)+SWIGLUOAI = 3_SQRT_2_OVER_PI = 0.7978845608028654 # sqrt(2/pi)_GELU_C = 0.044715@@ -64,6 +69,8 @@ def _act_and_mul_kernel(out_ptr,x_ptr,d,+ alpha,+ limit,ACT: tl.constexpr,ENABLE_PDL: tl.constexpr,BLOCK_D: tl.constexpr,@@ -88,18 +95,30 @@ def _act_and_mul_kernel(if ACT == 0: # SILU: x / (1 + exp(-x)) via ex2.approxact = gate / (1.0 + _fast_ex2(-gate * _LOG2E))+ y = act * upelif ACT == 2: # GELU_TANH via tanh.approxinner = 0.7978845608028654 * (gate + 0.044715 * gate * gate * gate)act = 0.5 * gate * (1.0 + _fast_tanh(inner))+ y = act * up+ elif ACT == 3: # SWIGLUOAI: clamped gate/up, sigmoid(alpha*gate), (up + 1) bias+ gate = tl.minimum(gate, limit)+ up = tl.minimum(tl.maximum(up, -limit), limit)+ act = gate / (1.0 + _fast_ex2(-gate * alpha * _LOG2E))+ y = act * (up + 1.0)else: # GELU (erf)act = 0.5 * gate * (1.0 + libdevice.erf(gate * 0.7071067811865476))-- y = act * up+ y = act * upout_row = out_ptr + row * dtl.store(out_row + cols, y.to(out_ptr.dtype.element_ty), mask=mask)-def _act_and_mul(kind: int, x: torch.Tensor, out: torch.Tensor | None):+def _act_and_mul(+ kind: int,+ x: torch.Tensor,+ out: torch.Tensor | None,+ alpha: float = 0.0,+ limit: float = 0.0,+):assert x.is_cuda and x.is_contiguous()d = x.shape[-1] // 2out_shape = x.shape[:-1] + (d,)@@ -115,7 +134,7 @@ def _act_and_mul(kind: int, x: torch.Tensor, out: torch.Tensor | None):block_d = min(triton.next_power_of_2(d), 1024 if M >= 4096 else 512)num_stages = 2 if block_d == 1024 else 3_act_and_mul_kernel[grid](- o2, x2, d, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl,+ o2, x2, d, alpha, limit, ACT=kind, ENABLE_PDL=pdl, launch_pdl=pdl,BLOCK_D=block_d, num_warps=4, num_stages=num_stages,)return out@@ -133,4 +152,18 @@ def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None) -> torchreturn _act_and_mul(GELU_TANH, x, out)-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul"]+def swigluoai_and_mul(+ x: torch.Tensor,+ out: torch.Tensor | None = None,+ *,+ alpha: float = 1.702,+ limit: float = 7.0,+) -> torch.Tensor:+ """SwiGLU-OAI over UNINTERLEAVED halves (gate ``x[..., :d]``, up ``x[..., d:]``):+ ``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.+ Same math as gpt-oss's interleaved ``gpt_oss_swiglu_kernel``; this layout matches+ the [gate; up] halves the NVFP4 expert banks and merged gate_up projections use."""+ return _act_and_mul(SWIGLUOAI, x, out, alpha=alpha, limit=limit)+++__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]diff --git a/python/freetoken/kernel/triton/minimax_m3_sparse.py b/python/freetoken/kernel/triton/minimax_m3_sparse.pynew file mode 100644index 00000000..d73596c4--- /dev/null+++ b/python/freetoken/kernel/triton/minimax_m3_sparse.py@@ -0,0 +1,1118 @@+"""Triton kernels for MiniMax-M3 block-sparse GQA attention (indexer + attend).++Ported from the vLLM reference implementation+(``vllm/models/minimax_m3/common/ops/{index_topk,sparse_attn}.py`` in the vLLM+tree -- the semantics source of truth), adapted to FreeToken's storage model:++* FreeToken's page table is page_size=1 semantics (one physical TOKEN row per+ column) and the allocator hands out page-aligned runs, so with page_size == the+ 128-token sparse block each block's rows are ``base .. base+127`` where ``base =+ page_table[t, blk * 128]``. The kernels take a per-request ``block_rows`` table+ of those int32 base rows and address the FLAT slabs directly -- the index-key+ slab as ``[rows, index_dim]`` (``BSAKVCache.index_k_cache``), K/V as ``[rows,+ kv_heads, head_dim]`` (the MHA pool's storage view) -- instead of vLLM's+ ``[num_blocks, ...]`` paged layouts.+* decode is always one query token per request (no spec decode), so the+ ``decode_query_len`` machinery is dropped: ``kv_len = seq_lens[req]``.+* KV is always bf16 (FreeToken pools); the fp8-KV dequant branches are dropped.+* ``tl.dot`` needs >= 16 rows per operand, so the decode index-score kernel pads+ the 4 index heads to a masked 16-row tile (the reference relied on a fork that+ allows narrow dots).++Semantics kept verbatim from the reference:++* per-index-head (== per-KV-head) block scores: ``score[h, q, blk] = max over the+ block's 128 positions of dot(iq[q, h], ik[pos])``, causal-masked, no scale+ (only the ordering is consumed).+* forced blocks: the first ``init_blocks`` score ``1e30`` and the newest+ ``local_blocks`` score ``1e29`` (visible blocks only), so they always win the+ top-k; padding slots come back ``-1``.+* the attend visits the FIRST ``min(topk, num_visible_blocks)`` selected entries+ (the bitonic top-k sorts descending), applies the causal mask inside the newest+ block, and runs a base-2 online softmax; decode is split-K over the selected+ blocks with an LSE merge (flash-decoding).++CUDA-graph safety (decode): every kernel reads the live ``seq_lens`` from device+memory and the split/chunk counts depend only on staged shapes, so the captured+grid tracks the live position (same contract as the dsa/dsv4 backends).+"""++from __future__ import annotations++import torch+import triton+import triton.language as tl++# One sparse block == one KV page (the engine pins page_size to this).+SPARSE_BLOCK_SIZE = 128+++def _round_up(a: int, b: int) -> int:+ return (a + b - 1) // b * b+++# ---------------------------------------------------------------------------+# Bitonic top-k helpers (verbatim from the reference; layout-agnostic).+# ---------------------------------------------------------------------------+def _compare_and_swap(x, ids, flip, i: tl.constexpr, n_dims: tl.constexpr):+ n_outer: tl.constexpr = x.numel >> n_dims+ shape: tl.constexpr = [n_outer * 2**i, 2, 2 ** (n_dims - i - 1)]+ y = tl.reshape(x, shape)+ mask = tl.arange(0, 2)[None, :, None]+ left = tl.broadcast_to(tl.sum(y * (1 - mask), 1)[:, None, :], shape).to(y.dtype)+ right = tl.broadcast_to(tl.sum(y * mask, 1)[:, None, :], shape).to(y.dtype)+ left = tl.reshape(left, x.shape)+ right = tl.reshape(right, x.shape)+ y_idx = tl.reshape(ids, shape)+ left_idx = tl.broadcast_to(tl.sum(y_idx * (1 - mask), 1)[:, None, :], shape)+ right_idx = tl.broadcast_to(tl.sum(y_idx * mask, 1)[:, None, :], shape)+ left_idx = tl.reshape(left_idx, x.shape).to(y_idx.dtype)+ right_idx = tl.reshape(right_idx, x.shape).to(y_idx.dtype)+ idtype = tl.core.get_int_dtype(bitwidth=x.dtype.primitive_bitwidth, signed=True)+ ileft = left.to(idtype, bitcast=True)+ iright = right.to(idtype, bitcast=True)+ ix = x.to(idtype, bitcast=True)+ cond = (left > right) != flip+ ret = ix ^ tl.where(cond, ileft ^ iright, tl.zeros_like(ix))+ new_ids = ids ^ tl.where(cond, left_idx ^ right_idx, tl.zeros_like(ids))+ return ret.to(x.dtype, bitcast=True), new_ids+++def _bitonic_merge(+ x, ids, stage: tl.constexpr, order: tl.constexpr, n_dims: tl.constexpr+):+ n_outer: tl.constexpr = x.numel >> n_dims+ tl.static_assert(stage <= n_dims)+ if order == 2:+ shape: tl.constexpr = [n_outer * 2 ** (n_dims - 1 - stage), 2, 2**stage]+ flip = tl.reshape(+ tl.broadcast_to(tl.arange(0, 2)[None, :, None], shape), x.shape+ )+ else:+ flip = order+ for i in tl.static_range(stage):+ x, ids = _compare_and_swap(x, ids, flip, i + (n_dims - stage), n_dims)+ return x, ids+++# ---------------------------------------------------------------------------+# Prefill index block-score kernel. score[h, token, block] = max over the+# 128-token block of (idx_q . index_k), causal-masked; blocks addressed via the+# per-request base-row table.+# ---------------------------------------------------------------------------[email protected](do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])+def _index_block_score_kernel(+ q_ptr, # idx_q: [total_q, num_idx_heads, head_dim]+ ik_ptr, # index-K slab, row-flat: [rows, head_dim]+ score_ptr, # [num_idx_heads, total_q, max_block(stride)]+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows+ cu_seqlens, # [batch+1] query start offsets+ seq_lens, # [batch] total K length+ prefix_lens, # [batch] context length before this chunk's queries+ num_idx_heads,+ head_dim: tl.constexpr,+ stride_q_n, stride_q_h, stride_q_d,+ stride_ik_r, stride_ik_d,+ stride_s_h, stride_s_n, stride_s_k,+ stride_br_b,+ BLOCK_SIZE_Q: tl.constexpr,+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)+):+ pid_q = tl.program_id(0)+ pid_bh = tl.program_id(1)+ pid_b = pid_bh // num_idx_heads+ pid_h = pid_bh % num_idx_heads++ seq_start = tl.load(cu_seqlens + pid_b)+ q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start+ seq_len = tl.load(seq_lens + pid_b)+ prefix_len = tl.load(prefix_lens + pid_b)+ if BLOCK_SIZE_Q * pid_q >= q_len:+ return++ q_ptrs = tl.make_block_ptr(+ base=q_ptr + seq_start * stride_q_n + pid_h * stride_q_h,+ shape=(q_len, head_dim),+ strides=(stride_q_n, stride_q_d),+ offsets=(pid_q * BLOCK_SIZE_Q, 0),+ block_shape=(BLOCK_SIZE_Q, head_dim),+ order=(1, 0),+ )+ q = tl.load(q_ptrs, boundary_check=(0,), padding_option="zero")+ q_start = prefix_len + pid_q * BLOCK_SIZE_Q++ off_q = tl.arange(0, BLOCK_SIZE_Q) + pid_q * BLOCK_SIZE_Q + prefix_len+ off_k = tl.arange(0, BLOCK_SIZE_K)+ off_d = tl.arange(0, head_dim)+ br_row = block_rows_ptr + pid_b * stride_br_b+ # Causal window: only blocks up to the last query token's position.+ hi = min(seq_len, prefix_len + (pid_q + 1) * BLOCK_SIZE_Q)+ for i in tl.range(0, hi, BLOCK_SIZE_K):+ blk = i // BLOCK_SIZE_K+ base = tl.load(br_row + blk).to(tl.int64)+ base = tl.maximum(base, 0)+ pos = i + off_k+ # No masked load: pages are whole 128-row runs, so rows base..base+127+ # exist, and the index-key slab is ZERO-INITIALIZED by BSAKVCache (unlike+ # the K/V slabs) -- unwritten tail rows contribute a finite 0-dot that the+ # qk causal mask below discards. If that zero-init invariant ever changes,+ # this load (and the decode score kernel's) must gain a pos mask.+ k = tl.load(+ ik_ptr+ + (base + off_k[None, :]) * stride_ik_r+ + off_d[:, None] * stride_ik_d,+ )+ qk = tl.dot(q, k)+ if q_start < i + BLOCK_SIZE_K:+ qk = tl.where(off_q[:, None] >= pos[None, :], qk, float("-inf"))+ score = tl.max(qk, axis=1) # [BLOCK_SIZE_Q]: max over the block+ s_ptrs = (+ score_ptr+ + pid_h * stride_s_h+ + (seq_start + pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q))+ * stride_s_n+ + blk * stride_s_k+ )+ q_store_mask = (pid_q * BLOCK_SIZE_Q + tl.arange(0, BLOCK_SIZE_Q)) < q_len+ tl.store(s_ptrs, score, mask=q_store_mask)+++# ---------------------------------------------------------------------------+# Prefill top-k over per-token block scores (verbatim reference port; the forced+# init/local blocks get score boosts, invalid slots come back -1).+# ---------------------------------------------------------------------------[email protected]({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})+ configs=[+ triton.Config({"BLOCK_SIZE_K": 1024}, num_warps=8, num_stages=2),+ triton.Config({"BLOCK_SIZE_K": 512}, num_warps=8, num_stages=2),+ triton.Config({"BLOCK_SIZE_K": 256}, num_warps=8, num_stages=2),+ triton.Config({"BLOCK_SIZE_K": 128}, num_warps=4, num_stages=2),+ triton.Config({"BLOCK_SIZE_K": 64}, num_warps=2, num_stages=2),+ ],+ key=["BLOCK_SIZE_T"],+)[email protected](do_not_specialize_on_alignment=["prefix_lens"])+def _topk_index_kernel(+ s_ptr, # [num_heads, total_q, max_block]+ ti_ptr, # [num_heads, total_q, topk]+ block_size: tl.constexpr, # sparse block size (128)+ cu_seqlens,+ prefix_lens,+ topk,+ init_blocks: tl.constexpr,+ local_blocks: tl.constexpr,+ stride_s_h, stride_s_n, stride_s_k,+ stride_ti_h, stride_ti_n, stride_ti_t,+ BLOCK_SIZE_K: tl.constexpr,+ BLOCK_SIZE_T: tl.constexpr,+):+ tl.static_assert(BLOCK_SIZE_K > BLOCK_SIZE_T)+ pid_q = tl.program_id(0)+ pid_b = tl.program_id(1)+ pid_h = tl.program_id(2)+ seq_start = tl.load(cu_seqlens + pid_b)+ q_len = tl.load(cu_seqlens + pid_b + 1) - seq_start+ prefix_len = tl.load(prefix_lens + pid_b)+ if pid_q >= q_len:+ return+ off_k = tl.arange(0, BLOCK_SIZE_K)+ off_t = tl.arange(0, BLOCK_SIZE_T)+ s_ptrs = (+ s_ptr+ + (seq_start + pid_q) * stride_s_n+ + pid_h * stride_s_h+ + off_k * stride_s_k+ )+ topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)+ topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)+ left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2+ valid_blocks = (prefix_len + pid_q + block_size) // block_size+ for i in tl.range(0, valid_blocks, BLOCK_SIZE_K):+ causal_mask = i + off_k < valid_blocks+ local_mask = i + off_k >= max(0, valid_blocks - local_blocks)+ init_mask = i + off_k < init_blocks+ score = tl.load(s_ptrs, mask=causal_mask, other=-1e30).to(tl.float32)+ score = tl.where(score != score, -1e30, score)+ s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K+ # Force-select init (1e30) and local (1e29) blocks; local applied last so+ # it wins an overlap, matching the reference's MASK_INIT/MASK_LOCAL=False+ # order (moot for M3's init_blocks == 0).+ score = tl.where(causal_mask & init_mask, 1e30, score)+ score = tl.where(causal_mask & local_mask, 1e29, score)+ topk_score, last_topk_score = score, topk_score+ topk_idx, last_topk_idx = (tl.where(causal_mask, i + off_k + 1, 0), topk_idx)+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)+ for j in tl.static_range(1, n_dims):+ topk_score, topk_idx = _bitonic_merge(+ topk_score, topk_idx.to(tl.int32), j, 2, n_dims+ )+ if i != 0:+ topk_score, topk_idx = _bitonic_merge(+ topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims+ )+ topk_score_new = last_topk_score * left_half_mask + topk_score * (+ 1 - left_half_mask+ )+ topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (+ 1 - left_half_mask+ )+ topk_score, topk_idx = _bitonic_merge(+ topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims+ )+ else:+ topk_score, topk_idx = _bitonic_merge(+ topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims+ )+ topk_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0+ topk_idx = tl.sum(+ topk_mask[:, None]+ * tl.reshape(topk_idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),+ axis=0,+ )+ ti_ptrs = (+ ti_ptr+ + (seq_start + pid_q) * stride_ti_n+ + pid_h * stride_ti_h+ + off_t * stride_ti_t+ )+ store_mask = off_t < topk+ valid_mask = off_t < valid_blocks+ topk_idx = tl.where(store_mask & valid_mask, topk_idx, -1)+ tl.store(ti_ptrs, topk_idx.to(ti_ptrs.dtype.element_ty), mask=store_mask)+++# ---------------------------------------------------------------------------+# Decode index-score kernel (split-K over seq blocks; cudagraph-safe). One query+# per request; the 4 index heads ride a masked 16-row tile for tl.dot.+# ---------------------------------------------------------------------------[email protected](do_not_specialize=["num_kv_chunks"])+def _decode_index_score_kernel(+ q_ptr, # idx_q: [num_reqs, num_idx_heads, head_dim]+ ik_ptr, # index-K slab, row-flat: [rows, head_dim]+ score_ptr, # [num_idx_heads, num_reqs, max_block(stride)]+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows+ seq_lens, # [num_reqs] (device-read live lengths)+ num_idx_heads: tl.constexpr,+ head_dim: tl.constexpr,+ init_blocks,+ local_blocks,+ stride_q_n, stride_q_h, stride_q_d,+ stride_ik_r, stride_ik_d,+ stride_s_h, stride_s_n, stride_s_k,+ stride_br_b,+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)+ BLOCK_SIZE_H: tl.constexpr, # >= 16 (padded index-head tile)+ num_kv_chunks,+):+ pid_r = tl.program_id(0)+ pid_c = tl.program_id(1)+ off_h = tl.arange(0, BLOCK_SIZE_H)+ h_mask = off_h < num_idx_heads++ seq_len = tl.load(seq_lens + pid_r)+ # Padded graph rows carry zero length -> empty range, nothing stored.+ kv_len = tl.maximum(seq_len, 0)+ num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K++ # Block-aligned fixed-count split: grid independent of seq_len (cuda graph).+ chunk_size_blocks = (num_blocks + num_kv_chunks - 1) // num_kv_chunks+ chunk_start_block = pid_c * chunk_size_blocks+ chunk_end_block = tl.minimum(chunk_start_block + chunk_size_blocks, num_blocks)+ if chunk_start_block >= chunk_end_block:+ return+ off_k = tl.arange(0, BLOCK_SIZE_K)+ off_d = tl.arange(0, head_dim)+ br_row = block_rows_ptr + pid_r * stride_br_b+ local_start = tl.maximum(0, num_blocks - local_blocks)+ q = tl.load(+ q_ptr+ + pid_r * stride_q_n+ + off_h[None, :] * stride_q_h+ + off_d[:, None] * stride_q_d,+ mask=h_mask[None, :],+ other=0.0,+ ) # [D, H16]+ for blk in tl.range(chunk_start_block, chunk_end_block):+ base = tl.load(br_row + blk).to(tl.int64)+ base = tl.maximum(base, 0)+ pos = blk * BLOCK_SIZE_K + off_k+ pos_mask = pos[:, None] < kv_len+ # No masked load: the index-key slab is ZERO-INITIALIZED by BSAKVCache+ # (unlike the K/V slabs), so unwritten tail rows dot to a finite 0 that+ # the pos_mask below discards. If that invariant changes, mask this load.+ k = tl.load(+ ik_ptr+ + (base + off_k[:, None]) * stride_ik_r+ + off_d * stride_ik_d,+ ) # [N, D]+ kq = tl.dot(k, q, out_dtype=tl.float32) # [N, H16]+ kq = tl.where(pos_mask & h_mask[None, :], kq, float("-inf"))+ score = tl.max(kq, axis=0) # [H16]: max over the block's positions+ # local wins an init/local overlap (reference order; moot for init_blocks=0).+ is_init = blk < init_blocks+ is_local = blk >= local_start+ score = tl.where(is_local, 1e29, tl.where(is_init, 1e30, score))+ tl.store(+ score_ptr + off_h * stride_s_h + pid_r * stride_s_n + blk * stride_s_k,+ score,+ mask=h_mask,+ )+++# ---------------------------------------------------------------------------+# Decode top-k (split-K): per-chunk partial top-k + merge. Forced init/local+# blocks are already encoded in the scores. Verbatim reference port minus the+# spec-decode query-length math (kv_len == seq_lens[req]).+# ---------------------------------------------------------------------------[email protected]({"BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"])})+def _topk_index_partial_kernel(+ s_ptr, # score: [num_idx_heads, num_reqs, max_block]+ ts_partial_ptr, # partial scores out: [chunks, num_idx_heads, num_reqs, T]+ ti_partial_ptr, # partial idx out (1-indexed, 0=invalid): same shape+ seq_lens, # [num_reqs]+ block_size: tl.constexpr, # sparse block size (128)+ topk: tl.constexpr,+ stride_s_h, stride_s_b, stride_s_k,+ stride_ts_c, stride_ts_h, stride_ts_b, stride_ts_t,+ stride_ti_c, stride_ti_h, stride_ti_b, stride_ti_t,+ BLOCK_SIZE_K: tl.constexpr,+ BLOCK_SIZE_T: tl.constexpr,+):+ tl.static_assert(topk < BLOCK_SIZE_K)+ pid_b = tl.program_id(0) # request id+ pid_h = tl.program_id(1)+ pid_chunk = tl.program_id(2)++ kv_len = tl.maximum(tl.load(seq_lens + pid_b), 0)+ num_blocks = (kv_len + block_size - 1) // block_size++ # Chunk split from the row's LIVE block count (not the staged table width, which+ # would degenerate short contexts to a serial scan in chunk 0) -- device-read,+ # so the captured graph's split tracks the live position.+ num_chunks = tl.num_programs(2)+ chunk_blocks = (num_blocks + num_chunks - 1) // num_chunks+ chunk_start = pid_chunk * chunk_blocks+ chunk_end = tl.minimum(chunk_start + chunk_blocks, num_blocks)+ chunk_actual = tl.maximum(chunk_end - chunk_start, 0)++ off_k = tl.arange(0, BLOCK_SIZE_K)+ off_t = tl.arange(0, BLOCK_SIZE_T)++ s_ptrs = (+ s_ptr+ + pid_b * stride_s_b+ + pid_h * stride_s_h+ + (chunk_start + off_k) * stride_s_k+ )++ topk_score = tl.full((BLOCK_SIZE_K,), -1e30, dtype=tl.float32)+ topk_idx = tl.full((BLOCK_SIZE_K,), 0, dtype=tl.int32)+ left_half_mask = tl.arange(0, BLOCK_SIZE_K) < BLOCK_SIZE_K // 2++ # Streaming top-K within this chunk. tl.range(0, 0) is a no-op so empty+ # chunks (chunk_actual == 0) skip the body and store sentinel -1e30 / 0.+ for i in tl.range(0, chunk_actual, BLOCK_SIZE_K):+ mask = off_k < chunk_actual - i+ score = tl.load(s_ptrs, mask=mask, other=-1e30).to(tl.float32)+ score = tl.where(score != score, -1e30, score)+ s_ptrs = s_ptrs + stride_s_k * BLOCK_SIZE_K+ topk_score, last_topk_score = score, topk_score+ topk_idx, last_topk_idx = (+ tl.where(mask, chunk_start + i + off_k + 1, 0), # 1-indexed global+ topk_idx,+ )+ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)+ for j in tl.static_range(1, n_dims):+ topk_score, topk_idx = _bitonic_merge(+ topk_score, topk_idx.to(tl.int32), j, 2, n_dims+ )+ if i != 0:+ topk_score, topk_idx = _bitonic_merge(+ topk_score, topk_idx.to(tl.int32), n_dims, False, n_dims+ )+ topk_score_new = last_topk_score * left_half_mask + topk_score * (+ 1 - left_half_mask+ )+ topk_idx_new = last_topk_idx * left_half_mask + topk_idx * (+ 1 - left_half_mask+ )+ topk_score, topk_idx = _bitonic_merge(+ topk_score_new, topk_idx_new.to(tl.int32), n_dims, True, n_dims+ )+ else:+ topk_score, topk_idx = _bitonic_merge(+ topk_score, topk_idx.to(tl.int32), n_dims, True, n_dims+ )++ # Extract first BLOCK_SIZE_T entries (top-K of this chunk after the sort).+ topk_mask_extract = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0+ final_score = tl.sum(+ topk_mask_extract[:, None]+ * tl.reshape(topk_score, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),+ axis=0,+ )+ final_idx = tl.sum(+ topk_mask_extract[:, None]+ * tl.reshape(topk_idx, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),+ axis=0,+ )++ # Always write all BLOCK_SIZE_T slots -- invalid slots carry -1e30 / 0+ # sentinels and lose to real scores in the merge stage.+ ts_ptrs = (+ ts_partial_ptr+ + pid_chunk * stride_ts_c+ + pid_b * stride_ts_b+ + pid_h * stride_ts_h+ + off_t * stride_ts_t+ )+ ti_ptrs = (+ ti_partial_ptr+ + pid_chunk * stride_ti_c+ + pid_b * stride_ti_b+ + pid_h * stride_ti_h+ + off_t * stride_ti_t+ )+ tl.store(ts_ptrs, final_score)+ tl.store(ti_ptrs, final_idx)+++ {+ "BLOCK_SIZE_T": lambda args: triton.next_power_of_2(args["topk"]),+ "BLOCK_SIZE_K": lambda args: triton.next_power_of_2(+ args["num_topk_chunks"] * triton.next_power_of_2(args["topk"])+ ),+ }+)[email protected](do_not_specialize=["num_topk_chunks"])+def _topk_index_merge_kernel(+ ts_partial_ptr, # partial scores: [chunks, num_idx_heads, num_reqs, T]+ ti_partial_ptr, # partial idx (1-indexed, 0=invalid): same shape+ ti_final_ptr, # final idx (0-indexed, -1=invalid): [num_idx_heads, num_reqs, topk]+ seq_lens, # [num_reqs]+ block_size: tl.constexpr,+ topk: tl.constexpr,+ stride_ts_c, stride_ts_h, stride_ts_b, stride_ts_t,+ stride_ti_c, stride_ti_h, stride_ti_b, stride_ti_t,+ stride_tif_h, stride_tif_b, stride_tif_t,+ num_topk_chunks,+ BLOCK_SIZE_K: tl.constexpr,+ BLOCK_SIZE_T: tl.constexpr,+):+ pid_b = tl.program_id(0)+ pid_h = tl.program_id(1)++ kv_len = tl.maximum(tl.load(seq_lens + pid_b), 0)+ num_blocks = (kv_len + block_size - 1) // block_size++ off = tl.arange(0, BLOCK_SIZE_K)+ chunk_idx = off // BLOCK_SIZE_T+ in_chunk_idx = off % BLOCK_SIZE_T+ valid = chunk_idx < num_topk_chunks++ score_offset = (+ chunk_idx * stride_ts_c+ + pid_h * stride_ts_h+ + pid_b * stride_ts_b+ + in_chunk_idx * stride_ts_t+ )+ idx_offset = (+ chunk_idx * stride_ti_c+ + pid_h * stride_ti_h+ + pid_b * stride_ti_b+ + in_chunk_idx * stride_ti_t+ )++ score = tl.load(ts_partial_ptr + score_offset, mask=valid, other=-1e30).to(+ tl.float32+ )+ score = tl.where(score != score, -1e30, score)+ idx = tl.load(ti_partial_ptr + idx_offset, mask=valid, other=0).to(tl.int32)++ n_dims: tl.constexpr = tl.standard._log2(BLOCK_SIZE_K)+ for j in tl.static_range(1, n_dims):+ score, idx = _bitonic_merge(score, idx.to(tl.int32), j, 2, n_dims)+ score, idx = _bitonic_merge(score, idx.to(tl.int32), n_dims, True, n_dims)++ extract_mask = tl.arange(0, BLOCK_SIZE_K // BLOCK_SIZE_T) == 0+ topk_idx_final = tl.sum(+ extract_mask[:, None]+ * tl.reshape(idx - 1, [BLOCK_SIZE_K // BLOCK_SIZE_T, BLOCK_SIZE_T]),+ axis=0,+ )++ off_t = tl.arange(0, BLOCK_SIZE_T)+ tif_ptrs = (+ ti_final_ptr+ + pid_h * stride_tif_h+ + pid_b * stride_tif_b+ + off_t * stride_tif_t+ )+ store_mask = off_t < topk+ topk_idx_final = tl.where(off_t < tl.minimum(topk, num_blocks), topk_idx_final, -1)+ tl.store(+ tif_ptrs, topk_idx_final.to(ti_final_ptr.dtype.element_ty), mask=store_mask+ )+++# ---------------------------------------------------------------------------+# GQA block-sparse attend, prefill. One program per (query token, kv head,+# request); the 16-head GQA group rides one tl.dot tile. Base-2 online softmax.+# ---------------------------------------------------------------------------+ {+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),+ "BLOCK_SIZE_H": lambda args: max(+ 16, triton.next_power_of_2(args["gqa_group_size"])+ ),+ }+)[email protected](do_not_specialize_on_alignment=["seq_lens", "prefix_lens"])+def _gqa_sparse_fwd_kernel(+ q_ptr, # [total_q, num_heads, head_dim]+ k_ptr, # K slab, row-flat: [rows, num_kv_heads, head_dim]+ v_ptr, # V slab, row-flat: [rows, num_kv_heads, head_dim]+ t_ptr, # topk_idx: [num_kv_heads, total_q, topk]+ o_ptr, # [total_q, num_heads, head_dim]+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows+ cu_seqlens_q,+ seq_lens,+ prefix_lens,+ num_kv_heads,+ gqa_group_size,+ head_dim,+ max_topk,+ sm_scale,+ stride_qn, stride_qh, stride_qd,+ stride_kr, stride_kh, stride_kd,+ stride_vr, stride_vh, stride_vd,+ stride_th, stride_tn, stride_tk,+ stride_on, stride_oh, stride_od,+ stride_br_b,+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)+ BLOCK_SIZE_D: tl.constexpr,+ BLOCK_SIZE_H: tl.constexpr,+):+ sm_scale_log2e = sm_scale * 1.4426950409+ pid_q = tl.program_id(0)+ pid_kh = tl.program_id(1)+ pid_b = tl.program_id(2)+ pid_h = pid_kh * gqa_group_size+ q_start = tl.load(cu_seqlens_q + pid_b)+ q_len = tl.load(cu_seqlens_q + pid_b + 1) - q_start+ seq_len = tl.load(seq_lens + pid_b)+ prefix_len = tl.load(prefix_lens + pid_b)+ if pid_q >= q_len:+ return+ br_row = block_rows_ptr + pid_b * stride_br_b+ off_n = tl.arange(0, BLOCK_SIZE_K)+ off_d = tl.arange(0, BLOCK_SIZE_D)+ off_h = tl.arange(0, BLOCK_SIZE_H)+ d_mask = off_d < head_dim+ h_mask = off_h < gqa_group_size++ t_ptr_j = t_ptr + (q_start + pid_q) * stride_tn + pid_kh * stride_th+ q_abs = prefix_len + pid_q # absolute position of this query token+ valid_blocks = (q_abs + BLOCK_SIZE_K) // BLOCK_SIZE_K+ real_topk = tl.minimum(max_topk, valid_blocks)++ q = tl.load(+ q_ptr+ + (q_start + pid_q) * stride_qn+ + (pid_h + off_h[:, None]) * stride_qh+ + off_d[None, :] * stride_qd,+ mask=h_mask[:, None] & d_mask[None, :],+ other=0.0,+ ) # [H, D]++ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)+ lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)+ acc_o = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_D), dtype=tl.float32)+ for _ in range(real_topk):+ # `blk` is trusted in-range: selection writes exactly `real_topk` real+ # entries. A -1 padding slot here would OOB-read br_row before the base+ # clamp -- preserve that invariant when editing the top-k kernel.+ blk = tl.load(t_ptr_j).to(tl.int32)+ t_ptr_j = t_ptr_j + stride_tk+ c = blk * BLOCK_SIZE_K+ base = tl.load(br_row + blk).to(tl.int64)+ base = tl.maximum(base, 0)+ pos = c + off_n+ # causal + in-sequence: the newest block is partially visible.+ pos_mask = (pos <= q_abs) & (pos < seq_len)+ # K/V loads must be pos-masked: the masks are the correctness contract+ # for the tail page's unwritten rows (the forced local block visits the+ # partial newest page nearly every step; unmasked rows would NaN the+ # whole output row). BSAKVCache's zero-init is defense-in-depth, not a+ # substitute -- recycled pages get re-dirtied.+ k = tl.load(+ k_ptr+ + (base + off_n[None, :]) * stride_kr+ + pid_kh * stride_kh+ + off_d[:, None] * stride_kd,+ mask=d_mask[:, None] & pos_mask[None, :],+ other=0.0,+ ) # [D, N]+ qk = tl.dot(q, k) * sm_scale_log2e+ qk += tl.where(pos_mask[None, :], 0, float("-inf"))+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))+ # All-masked rows keep a -inf running max NaN-free.+ alpha = tl.where(m_ij == float("-inf"), 1.0, tl.exp2(m_i - m_ij))+ p = tl.where(pos_mask[None, :], tl.exp2(qk - m_ij[:, None]), 0.0)+ l_ij = tl.sum(p, axis=1)+ acc_o = acc_o * alpha[:, None]+ v = tl.load(+ v_ptr+ + (base + off_n[:, None]) * stride_vr+ + pid_kh * stride_vh+ + off_d[None, :] * stride_vd,+ mask=pos_mask[:, None] & d_mask[None, :],+ other=0.0,+ ) # [N, D]+ acc_o += tl.dot(p.to(v.dtype), v)+ m_i = m_ij+ lse_i = tl.where(+ m_ij == float("-inf"),+ lse_i,+ m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij),+ )+ acc_o = acc_o * tl.exp2(m_i - lse_i)[:, None]+ o_ptrs = (+ o_ptr+ + (q_start + pid_q) * stride_on+ + (pid_h + off_h[:, None]) * stride_oh+ + off_d[None, :] * stride_od+ )+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), mask=h_mask[:, None] & d_mask[None, :])+++# ---------------------------------------------------------------------------+# GQA block-sparse attend, decode (split-K over the selected blocks + LSE merge;+# flash-decoding). One query per request; cudagraph-safe (device seq_lens).+# ---------------------------------------------------------------------------+ {+ "BLOCK_SIZE_H": lambda args: max(+ 16, triton.next_power_of_2(args["gqa_group_size"])+ ),+ "BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"]),+ }+)+def _gqa_sparse_decode_kernel(+ q_ptr, # [num_reqs, num_heads, head_dim]+ k_ptr, # K slab, row-flat: [rows, num_kv_heads, head_dim]+ v_ptr, # V slab, row-flat: [rows, num_kv_heads, head_dim]+ t_ptr, # topk_idx: [num_kv_heads, num_reqs, topk]+ o_ptr, # partial out: [chunks, num_reqs, num_heads, head_dim]+ lse_ptr, # partial lse (log2): [chunks, num_reqs, num_heads]+ block_rows_ptr, # [num_reqs, max_blocks] int32 base rows+ seq_lens, # [num_reqs]+ total_q,+ gqa_group_size,+ head_dim,+ max_topk,+ sm_scale,+ stride_qn, stride_qh, stride_qd,+ stride_kr, stride_kh, stride_kd,+ stride_vr, stride_vh, stride_vd,+ stride_th, stride_tn, stride_tk,+ stride_o_c, stride_o_b, stride_o_h, stride_o_d,+ stride_l_c, stride_l_b, stride_l_h,+ stride_br_b,+ BLOCK_SIZE_K: tl.constexpr, # == SPARSE_BLOCK_SIZE (128)+ NUM_TOPK_CHUNKS: tl.constexpr,+ BLOCK_SIZE_H: tl.constexpr,+ BLOCK_SIZE_D: tl.constexpr,+):+ sm_scale_log2e = sm_scale * 1.4426950409+ pid_bc, pid_kh = tl.program_id(0), tl.program_id(1)+ pid_b = pid_bc % total_q+ pid_c = pid_bc // total_q+ pid_h = pid_kh * gqa_group_size+ chunk_size_topk = (max_topk + NUM_TOPK_CHUNKS - 1) // NUM_TOPK_CHUNKS+ chunk_start_topk = pid_c * chunk_size_topk+ chunk_end_compiletime = chunk_start_topk + chunk_size_topk++ kv_len = tl.maximum(tl.load(seq_lens + pid_b), 0)++ idx_base = t_ptr + pid_kh * stride_th + pid_b * stride_tn+ num_blocks = (kv_len + BLOCK_SIZE_K - 1) // BLOCK_SIZE_K+ real_topk = tl.minimum(max_topk, num_blocks)+ chunk_end_topk = tl.minimum(chunk_end_compiletime, real_topk)++ off_n = tl.arange(0, BLOCK_SIZE_K)+ off_d = tl.arange(0, BLOCK_SIZE_D)+ d_mask = off_d < head_dim+ br_row = block_rows_ptr + pid_b * stride_br_b++ m_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)+ lse_i = tl.full((BLOCK_SIZE_H,), float("-inf"), dtype=tl.float32)+ acc_o = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_D), dtype=tl.float32)+ q_ptrs = tl.make_block_ptr(+ base=q_ptr + pid_b * stride_qn + pid_h * stride_qh,+ shape=(gqa_group_size, head_dim),+ strides=(stride_qh, stride_qd),+ offsets=(0, 0),+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),+ order=(1, 0),+ )+ q = tl.load(q_ptrs, boundary_check=(0, 1), padding_option="zero")++ cur_idx_ptr = idx_base + chunk_start_topk * stride_tk+ for _ in tl.range(chunk_start_topk, chunk_end_topk):+ # `blk` is trusted in-range (selection invariant; see the sequential+ # attend kernel): a -1 padding slot would OOB-read br_row pre-clamp.+ blk = tl.load(cur_idx_ptr).to(tl.int32)+ cur_idx_ptr = cur_idx_ptr + stride_tk+ c = blk * BLOCK_SIZE_K+ base = tl.load(br_row + blk).to(tl.int64)+ base = tl.maximum(base, 0)+ pos = c + off_n+ pos_mask = pos < kv_len+ # K/V loads must be pos-masked (same contract as the sequential attend+ # kernel): with K masked, the additive -inf lanes stay -inf and+ # p = exp2(-inf) = 0. Zero-init is defense-in-depth, not a substitute.+ k = tl.load(+ k_ptr+ + (base + off_n[None, :]) * stride_kr+ + pid_kh * stride_kh+ + off_d[:, None] * stride_kd,+ mask=d_mask[:, None] & pos_mask[None, :],+ other=0.0,+ )+ qk = tl.zeros((BLOCK_SIZE_H, BLOCK_SIZE_K), dtype=tl.float32)+ qk += tl.where(pos_mask[None, :], 0, float("-inf"))+ qk += tl.dot(q, k) * sm_scale_log2e+ m_ij = tl.maximum(m_i, tl.max(qk, axis=1))+ p = tl.exp2(qk - m_ij[:, None])+ l_ij = tl.sum(p, axis=1)+ acc_o = acc_o * tl.exp2(m_i - m_ij)[:, None]+ v = tl.load(+ v_ptr+ + (base + off_n[:, None]) * stride_vr+ + pid_kh * stride_vh+ + off_d[None, :] * stride_vd,+ mask=pos_mask[:, None] & d_mask[None, :],+ other=0.0,+ )+ acc_o += tl.dot(p.to(v.dtype), v)+ m_i = m_ij+ lse_i = m_ij + tl.log2(tl.exp2(lse_i - m_ij) + l_ij)++ # Empty chunks for active rows must store zero output; otherwise the merge+ # can hit 0 * NaN. All-empty padded rows may still produce NaNs in merge.+ scale = tl.where(lse_i > float("-inf"), tl.exp2(m_i - lse_i), tl.zeros_like(lse_i))+ acc_o = acc_o * scale[:, None]+ o_ptrs = tl.make_block_ptr(+ base=o_ptr + pid_c * stride_o_c + pid_b * stride_o_b + pid_h * stride_o_h,+ shape=(gqa_group_size, head_dim),+ strides=(stride_o_h, stride_o_d),+ offsets=(0, 0),+ block_shape=(BLOCK_SIZE_H, BLOCK_SIZE_D),+ order=(1, 0),+ )+ tl.store(o_ptrs, acc_o.to(o_ptr.dtype.element_ty), boundary_check=(0, 1))+ lse_ptrs = tl.make_block_ptr(+ base=lse_ptr + pid_c * stride_l_c + pid_b * stride_l_b + pid_h * stride_l_h,+ shape=(gqa_group_size,),+ strides=(stride_l_h,),+ offsets=(0,),+ block_shape=(BLOCK_SIZE_H,),+ order=(0,),+ )+ tl.store(lse_ptrs, lse_i.to(lse_ptr.dtype.element_ty), boundary_check=(0,))+++ {"BLOCK_SIZE_D": lambda args: triton.next_power_of_2(args["head_dim"])}+)+def _merge_topk_attn_out_kernel(+ o_ptr, # partials: [chunks, num_reqs, num_heads, head_dim]+ lse_ptr, # partials (log2): [chunks, num_reqs, num_heads]+ out_ptr, # merged out: [num_reqs, num_heads, head_dim]+ head_dim,+ stride_o_c, stride_o_b, stride_o_h, stride_o_d,+ stride_l_c, stride_l_b, stride_l_h,+ stride_out_n, stride_out_h, stride_out_d,+ NUM_TOPK_CHUNKS: tl.constexpr,+ BLOCK_SIZE_D: tl.constexpr,+):+ pid_b, pid_h = tl.program_id(0), tl.program_id(1)++ off_c = tl.arange(0, NUM_TOPK_CHUNKS)+ off_d = tl.arange(0, BLOCK_SIZE_D)+ o_ptrs = tl.make_block_ptr(+ base=o_ptr + pid_b * stride_o_b + pid_h * stride_o_h,+ shape=(NUM_TOPK_CHUNKS, head_dim),+ strides=(stride_o_c, stride_o_d),+ offsets=(0, 0),+ block_shape=(NUM_TOPK_CHUNKS, BLOCK_SIZE_D),+ order=(1, 0),+ )+ lse_ptrs = lse_ptr + pid_b * stride_l_b + pid_h * stride_l_h + off_c * stride_l_c+ o = tl.load(o_ptrs, boundary_check=(0, 1), padding_option="zero")+ lse = tl.load(lse_ptrs) # empty chunks contribute -inf -> weight 0+ lse_max = tl.max(lse, axis=0)+ weights = tl.exp2(lse - lse_max)+ weights = weights / tl.sum(weights, axis=0)+ o_merged = tl.sum(o * weights[:, None], axis=0)+ out_ptrs = (+ out_ptr + pid_b * stride_out_n + pid_h * stride_out_h + off_d * stride_out_d+ )+ tl.store(out_ptrs, o_merged.to(out_ptr.dtype.element_ty), mask=off_d < head_dim)+++# ---------------------------------------------------------------------------+# Python wrappers+# ---------------------------------------------------------------------------[email protected]_grad()+def m3_index_score_prefill(+ idx_q: torch.Tensor, # [total_q, num_idx_heads, head_dim]+ ik_rows: torch.Tensor, # index-K slab, row-flat: [rows, head_dim]+ block_rows: torch.Tensor, # [batch, max_blocks] int32 base rows+ cu_seqlens_q: torch.Tensor, # [batch+1] int32+ seq_lens: torch.Tensor, # [batch] int32+ prefix_lens: torch.Tensor, # [batch] int32+ max_query_len: int,+ max_seq_len: int,+) -> torch.Tensor:+ """Per-token index scores for each visible sparse block: score ``[num_idx_heads,+ total_q, >=max_block]`` fp32, each entry the max over a 128-token block."""+ total_q, num_idx_heads, head_dim = idx_q.shape+ batch = cu_seqlens_q.shape[0] - 1+ max_block = triton.cdiv(max_seq_len, SPARSE_BLOCK_SIZE)+ # Keep score strides 16-divisible to avoid Triton recompiles.+ score = torch.empty(+ (num_idx_heads, total_q, _round_up(max_block, 16)),+ dtype=torch.float32,+ device=idx_q.device,+ )+ BLOCK_SIZE_Q = 64+ grid = (triton.cdiv(max_query_len, BLOCK_SIZE_Q), batch * num_idx_heads)+ _index_block_score_kernel[grid](+ idx_q, ik_rows, score, block_rows,+ cu_seqlens_q, seq_lens, prefix_lens,+ num_idx_heads, head_dim,+ idx_q.stride(0), idx_q.stride(1), idx_q.stride(2),+ ik_rows.stride(0), ik_rows.stride(1),+ score.stride(0), score.stride(1), score.stride(2),+ block_rows.stride(0),+ BLOCK_SIZE_Q=BLOCK_SIZE_Q,+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,+ )+ return score++[email protected]_grad()+def m3_index_topk_prefill(+ score: torch.Tensor, # [num_idx_heads, total_q, >=max_block]+ cu_seqlens_q: torch.Tensor, # [batch+1] int32+ prefix_lens: torch.Tensor, # [batch] int32+ max_query_len: int,+ topk: int,+ init_blocks: int,+ local_blocks: int,+) -> torch.Tensor:+ """Causal per-token top-k over precomputed block scores -> ``[num_idx_heads,+ total_q, topk]`` int32 block ids (-1 pad)."""+ num_idx_heads, total_q = score.shape[0], score.shape[1]+ batch = cu_seqlens_q.shape[0] - 1+ topk_idx = torch.empty(+ (num_idx_heads, total_q, topk), dtype=torch.int32, device=score.device+ )+ grid = (max_query_len, batch, num_idx_heads)+ _topk_index_kernel[grid](+ score, topk_idx,+ SPARSE_BLOCK_SIZE,+ cu_seqlens_q, prefix_lens,+ topk, init_blocks, local_blocks,+ score.stride(0), score.stride(1), score.stride(2),+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),+ )+ return topk_idx++[email protected]_grad()+def m3_index_decode(+ idx_q: torch.Tensor, # [num_reqs, num_idx_heads, head_dim]+ ik_rows: torch.Tensor, # index-K slab, row-flat: [rows, head_dim]+ block_rows: torch.Tensor, # [num_reqs, max_blocks] int32 base rows+ seq_lens: torch.Tensor, # [num_reqs] int32 (device, live)+ max_blocks: int, # staged block-table width (static under cuda graphs)+ topk: int,+ init_blocks: int,+ local_blocks: int,+) -> torch.Tensor:+ """Decode index block-score + top-k, both split-K (cudagraph-safe). Returns+ ``[num_idx_heads, num_reqs, topk]`` int32 (0-indexed block ids, -1 pad)."""+ num_reqs, num_idx_heads, head_dim = idx_q.shape+ score_stride = _round_up(max_blocks, 16)+ score = torch.empty(+ (num_idx_heads, num_reqs, score_stride), dtype=torch.float32, device=idx_q.device+ )+ # split-K over seq blocks; chunk count depends only on staged shapes so the+ # grid is fixed within a cuda graph.+ TARGET_GRID = 512+ MAX_NUM_KV_CHUNKS = 256+ target = max(1, min(MAX_NUM_KV_CHUNKS, TARGET_GRID // max(1, num_reqs)))+ num_kv_chunks = 1 << (target.bit_length() - 1)+ _decode_index_score_kernel[(num_reqs, num_kv_chunks)](+ idx_q, ik_rows, score, block_rows, seq_lens,+ num_idx_heads, head_dim,+ init_blocks, local_blocks,+ idx_q.stride(0), idx_q.stride(1), idx_q.stride(2),+ ik_rows.stride(0), ik_rows.stride(1),+ score.stride(0), score.stride(1), score.stride(2),+ block_rows.stride(0),+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,+ BLOCK_SIZE_H=max(16, triton.next_power_of_2(num_idx_heads)),+ num_kv_chunks=num_kv_chunks,+ num_warps=4, num_stages=2,+ )++ topk_idx = torch.empty(+ (num_idx_heads, num_reqs, topk), dtype=torch.int32, device=idx_q.device+ )+ TOPK_TARGET_GRID = 64+ MAX_NUM_TOPK_CHUNKS = 16+ topk_target = max(+ 1, min(MAX_NUM_TOPK_CHUNKS, TOPK_TARGET_GRID // max(1, num_reqs * num_idx_heads))+ )+ num_topk_chunks = 1 << (topk_target.bit_length() - 1)+ block_size_t = triton.next_power_of_2(topk)+ ts_partial = torch.empty(+ (num_topk_chunks, num_idx_heads, num_reqs, block_size_t),+ dtype=torch.float32, device=idx_q.device,+ )+ ti_partial = torch.empty(+ (num_topk_chunks, num_idx_heads, num_reqs, block_size_t),+ dtype=torch.int32, device=idx_q.device,+ )+ # The chunk split is computed in-kernel from each row's LIVE block count+ # (device-read; the staged table width would serialize short contexts).+ _topk_index_partial_kernel[(num_reqs, num_idx_heads, num_topk_chunks)](+ score, ts_partial, ti_partial, seq_lens,+ SPARSE_BLOCK_SIZE, topk,+ score.stride(0), score.stride(1), score.stride(2),+ ts_partial.stride(0), ts_partial.stride(1), ts_partial.stride(2), ts_partial.stride(3),+ ti_partial.stride(0), ti_partial.stride(1), ti_partial.stride(2), ti_partial.stride(3),+ # Fixed config (no autotune: decode is captured into CUDA graphs, and+ # autotune would benchmark at run time). 128 matches the reference's+ # smallest always-valid config.+ BLOCK_SIZE_K=128,+ num_warps=4, num_stages=2,+ )+ _topk_index_merge_kernel[(num_reqs, num_idx_heads)](+ ts_partial, ti_partial, topk_idx, seq_lens,+ SPARSE_BLOCK_SIZE, topk,+ ts_partial.stride(0), ts_partial.stride(1), ts_partial.stride(2), ts_partial.stride(3),+ ti_partial.stride(0), ti_partial.stride(1), ti_partial.stride(2), ti_partial.stride(3),+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),+ num_topk_chunks=num_topk_chunks,+ num_warps=2,+ )+ return topk_idx++[email protected]_grad()+def m3_sparse_attn_prefill(+ q: torch.Tensor, # [total_q, num_heads, head_dim]+ k_rows: torch.Tensor, # K slab, row-flat: [rows, num_kv_heads, head_dim]+ v_rows: torch.Tensor, # V slab, row-flat: [rows, num_kv_heads, head_dim]+ topk_idx: torch.Tensor, # [num_kv_heads, total_q, topk]+ block_rows: torch.Tensor, # [batch, max_blocks] int32 base rows+ cu_seqlens_q: torch.Tensor, # [batch+1] int32+ seq_lens: torch.Tensor, # [batch] int32+ prefix_lens: torch.Tensor, # [batch] int32+ max_query_len: int,+ sm_scale: float,+ output: torch.Tensor, # [total_q, num_heads, head_dim]+) -> None:+ """GQA block-sparse attention over the selected blocks (prefill/extend)."""+ total_q, num_heads, head_dim = q.shape+ num_kv_heads = k_rows.shape[1]+ batch = cu_seqlens_q.shape[0] - 1+ topk = topk_idx.shape[-1]+ gqa_group_size = num_heads // num_kv_heads+ grid = (max_query_len, num_kv_heads, batch)+ _gqa_sparse_fwd_kernel[grid](+ q, k_rows, v_rows, topk_idx, output, block_rows,+ cu_seqlens_q, seq_lens, prefix_lens,+ num_kv_heads, gqa_group_size, head_dim, topk, sm_scale,+ q.stride(0), q.stride(1), q.stride(2),+ k_rows.stride(0), k_rows.stride(1), k_rows.stride(2),+ v_rows.stride(0), v_rows.stride(1), v_rows.stride(2),+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),+ output.stride(0), output.stride(1), output.stride(2),+ block_rows.stride(0),+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,+ num_warps=4, num_stages=2,+ )++[email protected]_grad()+def m3_sparse_attn_decode(+ q: torch.Tensor, # [num_reqs, num_heads, head_dim]+ k_rows: torch.Tensor, # K slab, row-flat: [rows, num_kv_heads, head_dim]+ v_rows: torch.Tensor, # V slab, row-flat: [rows, num_kv_heads, head_dim]+ topk_idx: torch.Tensor, # [num_kv_heads, num_reqs, topk]+ block_rows: torch.Tensor, # [num_reqs, max_blocks] int32 base rows+ seq_lens: torch.Tensor, # [num_reqs] int32 (device, live)+ sm_scale: float,+ output: torch.Tensor, # [num_reqs, num_heads, head_dim]+) -> None:+ """GQA block-sparse attention for decode (split-K over the top-k blocks ++ LSE merge; cudagraph-safe)."""+ num_reqs, num_heads, head_dim = q.shape+ num_kv_heads = k_rows.shape[1]+ max_topk = topk_idx.shape[-1]+ gqa_group_size = num_heads // num_kv_heads+ TARGET_GRID = 256+ target = max(1, min(max_topk, TARGET_GRID // max(1, num_reqs * num_kv_heads)))+ num_topk_chunks = 1 << (target.bit_length() - 1)+ o_partial = torch.empty(+ (num_topk_chunks, num_reqs, num_heads, head_dim), dtype=q.dtype, device=q.device+ )+ lse_partial = torch.empty(+ (num_topk_chunks, num_reqs, num_heads), dtype=torch.float32, device=q.device+ )+ grid = (num_reqs * num_topk_chunks, num_kv_heads)+ _gqa_sparse_decode_kernel[grid](+ q, k_rows, v_rows, topk_idx, o_partial, lse_partial, block_rows, seq_lens,+ num_reqs, gqa_group_size, head_dim, max_topk, sm_scale,+ q.stride(0), q.stride(1), q.stride(2),+ k_rows.stride(0), k_rows.stride(1), k_rows.stride(2),+ v_rows.stride(0), v_rows.stride(1), v_rows.stride(2),+ topk_idx.stride(0), topk_idx.stride(1), topk_idx.stride(2),+ o_partial.stride(0), o_partial.stride(1), o_partial.stride(2), o_partial.stride(3),+ lse_partial.stride(0), lse_partial.stride(1), lse_partial.stride(2),+ block_rows.stride(0),+ BLOCK_SIZE_K=SPARSE_BLOCK_SIZE,+ NUM_TOPK_CHUNKS=num_topk_chunks,+ num_warps=4, num_stages=2,+ )+ _merge_topk_attn_out_kernel[(num_reqs, num_heads)](+ o_partial, lse_partial, output, head_dim,+ o_partial.stride(0), o_partial.stride(1), o_partial.stride(2), o_partial.stride(3),+ lse_partial.stride(0), lse_partial.stride(1), lse_partial.stride(2),+ output.stride(0), output.stride(1), output.stride(2),+ NUM_TOPK_CHUNKS=num_topk_chunks,+ num_warps=4,+ )+++__all__ = [+ "SPARSE_BLOCK_SIZE",+ "m3_index_score_prefill",+ "m3_index_topk_prefill",+ "m3_index_decode",+ "m3_sparse_attn_prefill",+ "m3_sparse_attn_decode",+]diff --git a/python/freetoken/kernel/triton/mxfp8_linear.py b/python/freetoken/kernel/triton/mxfp8_linear.pynew file mode 100644index 00000000..98a7001c--- /dev/null+++ b/python/freetoken/kernel/triton/mxfp8_linear.py@@ -0,0 +1,292 @@+"""MXFP8 (block-32 e8m0-scaled FP8) W8A16 dense linear, shared across models.++MiniMax-M3's modelopt checkpoint quantizes every dense projection (q/k/v/o, dense+MLPs, shared experts, indexer) to MXFP8: fp8-e4m3 ``weight [N, K]`` plus uint8 e8m0+exponent codes ``weight_scale_inv [N, K//32]``, dequant multiplier ``2**(code - 127)``.++Decode is weight-bandwidth bound, so small batches read the fp8 weight directly in a+split-K GEMV (half the traffic of a bf16-resident weight). The block-32 scale is+loaded once per tile and broadcast in registers; a per-element scale gather + exp2+costs ~3x the whole GEMV. Past the GEMV cap the forward dequantizes to bf16+(``fp8 * 2**k`` is exact in bf16) and runs cuBLAS -- a fused inline-dequant GEMM+never came close to cuBLAS on these shapes, so the transient is the fast option.++The GEMV accumulates in fp32; the cuBLAS path is bf16 with fp32 accumulate, same as+any other bf16 projection.+"""++from __future__ import annotations++import os++import torch+import triton+import triton.language as tl+from freetoken.layers.base import BaseOP++from freetoken.kernel.triton.e4m3_compat import e4m3_kernel_view, e4m3_native_cx, e4m3_u8_to_f32++FP8 = torch.float8_e4m3fn+MXFP8_BLOCK = 32+_TL_DTYPE = {torch.bfloat16: tl.bfloat16, torch.float16: tl.float16, torch.float32: tl.float32}++# Escape hatch: FREETOKEN_DEBUG_MXFP8_REF=1 swaps the kernels for a pure-torch+# dequant matmul (numeric reference / A-B debugging). Evaluated once, logged once.+_USE_REF = os.environ.get("FREETOKEN_DEBUG_MXFP8_REF") == "1"+if _USE_REF:+ from freetoken.utils import init_logger++ init_logger(__name__).info(+ "FREETOKEN_DEBUG_MXFP8_REF=1: MXFP8 linears serve the pure-torch fp32 "+ "dequant reference (slow; debugging only)."+ )+++def mxfp8_dequant(weight: torch.Tensor, scale_codes: torch.Tensor,+ dtype: torch.dtype = torch.bfloat16) -> torch.Tensor:+ """Dequant: ``w[n, k] * 2**(codes[n, k//32] - 127)`` -> ``dtype``.++ Serves the large-M forward (dequant + cuBLAS), the load-time bf16 ablation+ path and the kernels' numeric reference in tests. For 16-bit dtypes the+ whole computation runs in that dtype -- the pow2 descale is lossless there,+ and fp32 transients would double the peak footprint (which lands in the+ graph pool when captured). fp32 keeps fp32 compute: it is the reference.+ """+ assert weight.shape[-1] % MXFP8_BLOCK == 0+ assert scale_codes.shape[-1] == weight.shape[-1] // MXFP8_BLOCK+ compute = torch.float32 if dtype == torch.float32 else dtype+ descale = torch.exp2(scale_codes.to(torch.float32) - 127.0).to(compute)+ w = weight.to(compute).view(*weight.shape[:-1], -1, MXFP8_BLOCK)+ return (w * descale.unsqueeze(-1)).view(weight.shape).to(dtype)+++# GEMV cap. Must cover the CUDA-graph ladder max (256): a captured dequant+cuBLAS+# call would bake its full-weight transient into the shared graph pool (~2 GiB+# across the resident shapes), and the GEMV is also simply faster everywhere in+# graph range (M=128: ~230us vs ~930us on RTX PRO 6000). M_TILE buckets {16..256};+# 512 exceeds sm_120's shared-memory budget. Past the cap only prefill lands here,+# where cuBLAS amortizes the transient.+_GEMV_MAX_M = 256+++def _mxfp8_gemv_m1_splitk_kernel(+ a_ptr, w_ptr, s_ptr, part_ptr, N, K, n_kb, kb_per,+ stride_ak, stride_wn, stride_wk, stride_sn, stride_sk, stride_pk, stride_pn,+ BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,+):+ """M == 1 specialization: a plain fp32 multiply-reduce. The dot tile's 16-row+ padding costs ~30% at M=1 on RTX PRO 6000 / 5090, hence this kernel -- but on+ H100 the dot kernel is ~26% faster even at M=1. If sm_90 becomes a serving+ target, re-benchmark and dispatch by arch (or drop this kernel)."""+ KB32: tl.constexpr = BLOCK_K // 32+ pid_n = tl.program_id(0)+ pid_k = tl.program_id(1)+ offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)+ n_mask = offs_n < N+ off_kb32 = tl.arange(0, KB32)+ kb_start = pid_k * kb_per+ acc = tl.zeros((BLOCK_N,), dtype=tl.float32)+ for i in range(kb_per):+ kb = kb_start + i+ if kb < n_kb:+ offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)+ k_mask = offs_k < K+ a = tl.load(a_ptr + offs_k * stride_ak, mask=k_mask, other=0.0).to(tl.float32)+ if e4m3_native_cx():+ w = tl.load(+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,+ mask=n_mask[:, None] & k_mask[None, :], other=0.0,+ ).to(tl.float32)+ else:+ w = e4m3_u8_to_f32(tl.load(+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,+ mask=n_mask[:, None] & k_mask[None, :], other=0,+ ))+ codes = tl.load(+ s_ptr + offs_n[:, None] * stride_sn+ + (kb * KB32 + off_kb32[None, :]) * stride_sk,+ mask=n_mask[:, None] & ((kb * KB32 + off_kb32[None, :]) * 32 < K),+ other=127,+ ).to(tl.float32) # [BLOCK_N, KB32]+ scale = tl.exp2(codes - 127.0)+ prod = tl.reshape(w * a[None, :], (BLOCK_N, KB32, 32))+ acc += tl.sum(tl.sum(prod, axis=2) * scale, axis=1)+ tl.store(part_ptr + pid_k * stride_pk + offs_n * stride_pn, acc, mask=n_mask)+++def _mxfp8_gemv_splitk_kernel(+ a_ptr, w_ptr, s_ptr, part_ptr, M, N, K, n_kb, kb_per,+ stride_am, stride_ak, stride_wn, stride_wk, stride_sn, stride_sk,+ stride_pk, stride_pm, stride_pn,+ M_TILE: tl.constexpr, BLOCK_N: tl.constexpr, BLOCK_K: tl.constexpr,+):+ """Each (pid_n, pid_k) accumulates ``kb_per`` BLOCK_K chunks of ``a[:M] @ w^T``+ for a BLOCK_N slice of outputs. BLOCK_K is a multiple of 32, so each chunk+ covers ``BLOCK_K // 32`` whole scale blocks: one ``[BLOCK_N, KB32]`` code load+ + exp2 per tile, broadcast over the 32-wide inner axis via a 3D register view,+ folded into the fp8 weight BEFORE the dot (pow2 scaling is lossless in bf16)."""+ KB32: tl.constexpr = BLOCK_K // 32+ pid_n = tl.program_id(0)+ pid_k = tl.program_id(1)+ offs_m = tl.arange(0, M_TILE)+ m_mask = offs_m < M+ offs_n = pid_n * BLOCK_N + tl.arange(0, BLOCK_N)+ n_mask = offs_n < N+ off_kb32 = tl.arange(0, KB32)+ kb_start = pid_k * kb_per+ acc = tl.zeros((M_TILE, BLOCK_N), dtype=tl.float32)+ for i in range(kb_per):+ kb = kb_start + i+ if kb < n_kb:+ offs_k = kb * BLOCK_K + tl.arange(0, BLOCK_K)+ k_mask = offs_k < K+ a = tl.load(+ a_ptr + offs_m[:, None] * stride_am + offs_k[None, :] * stride_ak,+ mask=m_mask[:, None] & k_mask[None, :], other=0.0,+ ) # [M_TILE, BLOCK_K] bf16+ if e4m3_native_cx():+ w = tl.load(+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,+ mask=n_mask[:, None] & k_mask[None, :], other=0.0,+ ).to(tl.float32)+ else:+ w = e4m3_u8_to_f32(tl.load(+ w_ptr + offs_n[:, None] * stride_wn + offs_k[None, :] * stride_wk,+ mask=n_mask[:, None] & k_mask[None, :], other=0,+ ))+ codes = tl.load(+ s_ptr + offs_n[:, None] * stride_sn+ + (kb * KB32 + off_kb32[None, :]) * stride_sk,+ mask=n_mask[:, None] & ((kb * KB32 + off_kb32[None, :]) * 32 < K),+ other=127,+ ).to(tl.float32) # [BLOCK_N, KB32]+ scale = tl.exp2(codes - 127.0)+ w3 = tl.reshape(w, (BLOCK_N, KB32, 32)) * scale[:, :, None]+ w_scaled = tl.reshape(w3, (BLOCK_N, BLOCK_K)).to(a.dtype)+ acc += tl.dot(a, tl.trans(w_scaled), out_dtype=tl.float32)+ part_base = part_ptr + pid_k * stride_pk+ tl.store(+ part_base + offs_m[:, None] * stride_pm + offs_n[None, :] * stride_pn,+ acc,+ mask=m_mask[:, None] & n_mask[None, :],+ )+++def _splitk_reduce_kernel(+ part_ptr, out_ptr, N, SPLIT_K: tl.constexpr,+ stride_pk, stride_pm, stride_pn, stride_om, stride_on,+ BLOCK: tl.constexpr, OUT: tl.constexpr,+):+ pid_m = tl.program_id(1)+ offs = tl.program_id(0) * BLOCK + tl.arange(0, BLOCK)+ mask = offs < N+ acc = tl.zeros((BLOCK,), dtype=tl.float32)+ for k in tl.static_range(SPLIT_K):+ acc += tl.load(+ part_ptr + k * stride_pk + pid_m * stride_pm + offs * stride_pn,+ mask=mask, other=0.0,+ )+ tl.store(out_ptr + pid_m * stride_om + offs * stride_on, acc.to(OUT), mask=mask)+++def _gemv(a: torch.Tensor, weight: torch.Tensor, scale_codes: torch.Tensor,+ out_dtype: torch.dtype) -> torch.Tensor:+ """Small-M split-K GEMV. ``a`` [M, K] bf16 (M <= _GEMV_MAX_M); ``weight``+ [N, K] fp8; ``scale_codes`` [N, K//32] uint8."""+ M, K = a.shape+ N = weight.shape[0]+ BLOCK_K = 128+ n_kb = triton.cdiv(K, BLOCK_K)+ BLOCK_N = 16+ n_tiles = triton.cdiv(N, BLOCK_N)+ split_k = max(1, min(1536 // n_tiles, n_kb))+ split_k = 1 << (split_k.bit_length() - 1) # pow2 -> stable reduction order+ kb_per = triton.cdiv(n_kb, split_k)+ part = torch.empty((split_k, M, N), dtype=torch.float32, device=a.device)+ if M == 1:+ _mxfp8_gemv_m1_splitk_kernel[(n_tiles, split_k)](+ a, weight, scale_codes, part, N, K, n_kb, kb_per,+ a.stride(1), weight.stride(0), weight.stride(1),+ scale_codes.stride(0), scale_codes.stride(1),+ part.stride(0), part.stride(2),+ BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K, num_warps=1,+ )+ else:+ # M_TILE buckets to the next pow2 >= 16 (tl.dot minimum); the whole batch+ # rides one weight pass regardless of bucket, so the only cost of a bigger+ # tile is row padding, amortized with proportionally more warps.+ m_tile = max(16, triton.next_power_of_2(M))+ _mxfp8_gemv_splitk_kernel[(n_tiles, split_k)](+ a, weight, scale_codes, part, M, N, K, n_kb, kb_per,+ a.stride(0), a.stride(1), weight.stride(0), weight.stride(1),+ scale_codes.stride(0), scale_codes.stride(1),+ part.stride(0), part.stride(1), part.stride(2),+ M_TILE=m_tile, BLOCK_N=BLOCK_N, BLOCK_K=BLOCK_K,+ num_warps=m_tile // 16,+ )+ out = torch.empty((M, N), dtype=out_dtype, device=a.device)+ _splitk_reduce_kernel[(triton.cdiv(N, 256), M)](+ part, out, N, split_k,+ part.stride(0), part.stride(1), part.stride(2), out.stride(0), out.stride(1),+ BLOCK=256, OUT=_TL_DTYPE[out_dtype if out_dtype in _TL_DTYPE else torch.bfloat16],+ num_warps=2,+ )+ return out+++def mxfp8_linear(+ x: torch.Tensor, weight: torch.Tensor, scale_codes: torch.Tensor,+ bias: torch.Tensor | None = None,+) -> torch.Tensor:+ """``y = x @ dequant(weight, scale_codes)^T``. Small batches+ (M <= 256) -> split-K GEMV (one pass over the fp8 weight); larger M -> bf16+ dequant + cuBLAS (see the module docstring). ``weight`` [N, K] fp8-e4m3;+ ``scale_codes`` [N, K//32] uint8 e8m0 exponent codes (dequant multiplier+ ``2**(code - 127)``)."""+ *lead, K = x.shape+ N = weight.shape[0]+ M = x.numel() // K+ assert K % MXFP8_BLOCK == 0 and scale_codes.shape == (N, K // MXFP8_BLOCK)+ if _USE_REF: # numeric-reference fallback (debug / A-B)+ w = mxfp8_dequant(weight, scale_codes, dtype=torch.float32)+ out = (x.reshape(-1, K).float() @ w.t()).to(x.dtype).reshape(*lead, N)+ elif M <= _GEMV_MAX_M:+ w8 = e4m3_kernel_view(weight)+ out = _gemv(x.reshape(M, K), w8, scale_codes, x.dtype).reshape(*lead, N)+ else:+ # Per-call bf16 transient (pow2 descale is lossless in bf16) + cuBLAS.+ w = mxfp8_dequant(weight, scale_codes, dtype=x.dtype)+ out = torch.nn.functional.linear(x.reshape(-1, K), w).reshape(*lead, N)+ if bias is not None:+ out = out + bias.to(out.dtype)+ return out+++# ======================================================================================+# BaseOP linear layers (TP=1, replicated). Buffers: fp8 ``weight`` + uint8+# ``weight_scale_inv`` (the checkpoint's e8m0 exponent codes, loaded verbatim).+# ======================================================================================+class Mxfp8Linear(BaseOP):+ """Replicated MXFP8 linear: fp8-e4m3 ``weight`` ``[out, in]`` + uint8 e8m0+ ``weight_scale_inv`` ``[out, in // 32]``. Fused projections concatenate several+ checkpoint projections along the output dim; the scales are per-output-row so the+ concatenation is exact."""++ def __init__(self, in_features: int, out_features: int, has_bias: bool = False):+ assert in_features % MXFP8_BLOCK == 0+ self.in_features = in_features+ self.out_features = out_features+ self.weight = torch.empty(out_features, in_features, dtype=FP8)+ self.weight_scale_inv = torch.empty(+ out_features, in_features // MXFP8_BLOCK, dtype=torch.uint8+ )+ self.bias = torch.empty(out_features) if has_bias else None++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return mxfp8_linear(x, self.weight, self.weight_scale_inv, self.bias)+++__all__ = ["FP8", "MXFP8_BLOCK", "Mxfp8Linear", "mxfp8_linear", "mxfp8_dequant"]diff --git a/python/freetoken/kernel/triton/norm.py b/python/freetoken/kernel/triton/norm.pyindex 179678b7..3f95c29f 100644--- a/python/freetoken/kernel/triton/norm.py+++ b/python/freetoken/kernel/triton/norm.py@@ -48,6 +48,7 @@ def _rmsnorm_kernel(CONTIG: tl.constexpr,ENABLE_PDL: tl.constexpr,BLOCK: tl.constexpr,+ GEMMA: tl.constexpr = False,):a = tl.program_id(0)b = tl.program_id(1)@@ -65,6 +66,10 @@ def _rmsnorm_kernel(w = tl.load(W + cols, mask=mask, other=0.0).to(tl.float32)if ENABLE_PDL:gdc_launch_dependents()+ if GEMMA:+ # Gemma semantics: checkpoint stores (scale - 1); apply (1 + w) in fp32+ # (matches flashinfer's gemma_rmsnorm, which never rounds 1+w to bf16).+ w = w + 1.0inv = tl.math.rsqrt(tl.sum(x * x, axis=0) / H + eps)y = x * inv * wtl.store(OUT + ooff + cols, y.to(OUT.dtype.element_ty), mask=mask)@@ -79,6 +84,7 @@ def _fused_add_rmsnorm_kernel(CONTIG: tl.constexpr,ENABLE_PDL: tl.constexpr,BLOCK: tl.constexpr,+ GEMMA: tl.constexpr = False,):a = tl.program_id(0)b = tl.program_id(1)@@ -97,6 +103,8 @@ def _fused_add_rmsnorm_kernel(w = tl.load(W + cols, mask=mask, other=0.0).to(tl.float32)if ENABLE_PDL:gdc_launch_dependents()+ if GEMMA:+ w = w + 1.0z = x + rtl.store(R + roff + cols, z.to(R.dtype.element_ty), mask=mask)inv = tl.math.rsqrt(tl.sum(z * z, axis=0) / H + eps)@@ -112,7 +120,7 @@ def _leading(t):return t.shape[0], t.shape[1], t.stride(0), t.stride(1)-def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False):+def _rmsnorm(input, weight, eps, out, gemma: bool):import torchassert weight.ndim == 1 and input.shape[-1] == weight.shape[0]@@ -122,7 +130,7 @@ def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False# 2D hidden norms or 3D per-head qk-norms). reshape may copy for a strided# input, so this path is out-of-place only.assert out is None, "rmsnorm on <2D/>3D input must be out-of-place"- return rmsnorm(input.reshape(-1, H), weight, eps).reshape(input.shape)+ return _rmsnorm(input.reshape(-1, H), weight, eps, None, gemma).reshape(input.shape)assert input.stride(-1) == 1 # norm axis must be unit-stride (as flashinfer requires)if out is None:out = torch.empty_like(input)@@ -136,13 +144,23 @@ def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = Falsepdl = contig and is_sm90_supported()_rmsnorm_kernel[(A, B)](out, input, weight, eps, H, sxa, sxb, soa, sob,- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl,+ CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,num_warps=_num_warps(A * B), num_stages=1,)return out-def fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bool = False):+def rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False):+ return _rmsnorm(input, weight, eps, out, gemma=False)+++def gemma_rmsnorm(input, weight, eps: float = 1e-6, out=None, enable_pdl: bool = False):+ """(1 + w)-scaled RMSNorm (Gemma/MiniMax-M3 semantics: the checkpoint stores+ ``scale - 1``); drop-in for flashinfer's ``norm.gemma_rmsnorm``."""+ return _rmsnorm(input, weight, eps, out, gemma=True)+++def _fused_add_rmsnorm(input, residual, weight, eps, gemma: bool):assert weight.ndim == 1 and input.shape[-1] == weight.shape[0]assert input.shape == residual.shapeH = input.shape[-1]@@ -154,9 +172,19 @@ def fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bopdl = contig and is_sm90_supported()_fused_add_rmsnorm_kernel[(A, B)](input, residual, weight, eps, H, sxa, sxb, sra, srb,- CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl,+ CONTIG=contig, ENABLE_PDL=pdl, launch_pdl=pdl, GEMMA=gemma,num_warps=_num_warps(A * B), num_stages=1,)-__all__ = ["rmsnorm", "fused_add_rmsnorm"]+def fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bool = False):+ _fused_add_rmsnorm(input, residual, weight, eps, gemma=False)+++def gemma_fused_add_rmsnorm(input, residual, weight, eps: float = 1e-6, enable_pdl: bool = False):+ """In-place ``residual += input; input = gemma_rmsnorm(residual)`` (Gemma (1+w)+ scaling); drop-in for flashinfer's ``norm.gemma_fused_add_rmsnorm``."""+ _fused_add_rmsnorm(input, residual, weight, eps, gemma=True)+++__all__ = ["rmsnorm", "fused_add_rmsnorm", "gemma_rmsnorm", "gemma_fused_add_rmsnorm"]diff --git a/python/freetoken/kvcache/__init__.py b/python/freetoken/kvcache/__init__.pyindex b0f167cb..1bb352b4 100644--- a/python/freetoken/kvcache/__init__.py+++ b/python/freetoken/kvcache/__init__.py@@ -57,6 +57,10 @@ def resolve_pool_class(model_config: ModelConfig) -> type[BaseKVCachePool]:from .dsa_pool import MLAKVCachereturn MLAKVCache+ if AttnType.BSA in types:+ from .bsa_pool import BSAKVCache++ return BSAKVCachefrom .mha_pool import MHAKVCachereturn MHAKVCache@@ -148,6 +152,28 @@ def create_kvcache_pool(# spec carries indexer dims. The same spec fields drive the KV cost model, so the# factory and the budget can never disagree.kv_specs = model_config.kv_cache_group_specs()++ # GQA block-sparse (MiniMax-M3): one full-attention group carrying the index dims+ # with mla=False -> the MHA pool plus the index-key slab. The same spec fields+ # drive the KV cost model, so the factory and the budget can never disagree.+ from freetoken.attention import AttnType as _AttnType++ if len(kv_specs) == 1 and kv_specs[0].attn_type == _AttnType.BSA:+ from .bsa_pool import BSAKVCache++ spec = kv_specs[0]+ return BSAKVCache(+ num_kv_heads=spec.num_kv_heads,+ num_layers=model_config.num_layers,+ head_dim=spec.head_dim,+ num_pages=num_pages,+ page_size=page_size,+ dtype=dtype,+ device=device,+ index_head_dim=spec.index_head_dim,+ num_index_layers=spec.num_index_layers,+ )+if len(kv_specs) == 1 and kv_specs[0].mla:from .dsa_pool import DSAKVCache, MLAKVCachediff --git a/python/freetoken/kvcache/bsa_pool.py b/python/freetoken/kvcache/bsa_pool.pynew file mode 100644index 00000000..3eeb4b12--- /dev/null+++ b/python/freetoken/kvcache/bsa_pool.py@@ -0,0 +1,126 @@+"""GQA block-sparse (BSA) KV pool: paged GQA K/V + the indexer's index-key slab.++MiniMax-M3's sparse layers attend only the top-k 128-token blocks a lightning+indexer selects, so the pool is the MHA pool (standard paged GQA K/V, page_size+== the sparse block size, one page per block) plus one bf16 index-key row per+token per SPARSE layer (``index_head_dim`` wide; M3 has a single shared index+key head). Slot order = the backend's sparse-layer order, same convention as+``DSAKVCache``'s full-indexer slots.++Storage lives here -- not in the attention backend -- so the engine's rebuild+path (fresh allocation, object identity preserved, views re-derived per+forward) and the KV cost model (which budgets the index-K bytes off the+attention-group spec via ``spec_kv_bytes_per_token``) stay correct by+construction. Both slabs resize atomically in ``rebuild`` so the allocator can+never hand out a row one slab has and the other lacks (DSAKVCache precedent).+"""++from __future__ import annotations++from typing import Sequence++import torch++from .mha_pool import MHAKVCache+++class BSAKVCache(MHAKVCache):+ """MHA paged pool + the block-sparse index-key slab.++ ``index_k_cache(slot)`` is row-flat ``[num_pages * page_size,+ index_head_dim]`` addressed by the same physical token rows as the K/V+ slabs (the shared page table is page_size=1 semantics), so the backend's+ block addressing -- base row of block ``b`` = ``page_table[t, b *+ page_size]`` -- serves K/V and index keys alike.+ """++ def __init__(+ self,+ num_kv_heads: int,+ num_layers: int,+ head_dim: int,+ num_pages: int,+ page_size: int,+ dtype: torch.dtype,+ device: torch.device,+ index_head_dim: int,+ num_index_layers: int,+ layer_ids: Sequence[int] | None = None,+ ) -> None:+ self._index_head_dim = index_head_dim+ self._num_index_layers = num_index_layers+ self._page_size = page_size+ # Index keys ride the compute dtype (bf16/fp16 -- the model's index_q/index_k+ # are engine-dtype and store_index_k's index_put_ raises on a mismatch). The+ # KV cost model budgets 2 bytes/token/index-layer for this slab+ # (base.spec_kv_bytes_per_token); keep the two in lockstep.+ assert dtype.itemsize == 2, (+ f"BSA index slab budgets 2 bytes/token (spec_kv_bytes_per_token); got {dtype}"+ )+ self._index_dtype = dtype+ super().__init__(+ num_kv_heads=num_kv_heads,+ num_layers=num_layers,+ head_dim=head_dim,+ num_pages=num_pages,+ page_size=page_size,+ dtype=dtype,+ device=device,+ layer_ids=layer_ids,+ )+ self._zero_kv_slabs()+ self._alloc_index_slab(num_pages)++ def _zero_kv_slabs(self) -> None:+ # Defense-in-depth: the attend kernels pos-mask every K/V load (the real+ # fix for torch.empty's recycled NaN/Inf bit patterns), but a zeroed slab+ # keeps any future unmasked read finite instead of model-poisoning. One+ # memset per (re)allocation -- negligible.+ self._kv_buffer.zero_()++ def _alloc_index_slab(self, num_pages: int) -> None:+ # ZERO-initialized -- the index-score kernels load index keys unmasked and+ # rely on unwritten tail rows dotting to a finite 0 (see the invariant+ # comments in kernel/triton/minimax_m3_sparse.py).+ self._index_k_buffer = torch.zeros(+ self._num_index_layers,+ num_pages * self._page_size,+ self._index_head_dim,+ dtype=self._index_dtype,+ device=self._device,+ )++ def rebuild(self, num_pages: int) -> None:+ # Free the index slab BEFORE the K/V realloc (super().rebuild frees + syncs ++ # empty_cache), then re-derive it at the new page count. If the index-slab+ # alloc itself fails (OOM), null the K/V slab too and re-raise: a pool with a+ # grown K/V slab and no index slab would mis-serve silently, and the engine's+ # rebuild path pre-validates the budget so this is already exceptional.+ self._index_k_buffer = None+ super().rebuild(num_pages)+ self._zero_kv_slabs()+ try:+ self._alloc_index_slab(num_pages)+ except Exception:+ self._kv_buffer = None+ self._k_buffer = None+ self._v_buffer = None+ raise++ def unit_bytes(self) -> tuple[int, int]:+ # The index slab rides the same token budget as the K/V slabs; each slab's+ # per-token cost is floor-divided on its own, matching the cost model's terms.+ kv, swa = super().unit_bytes()+ idx = self._index_k_buffer+ tokens = int(self._kv_buffer.shape[2]) * int(self._kv_buffer.shape[3])+ return kv + int(idx.numel() * idx.element_size()) // tokens, swa++ def index_k_cache(self, slot: int) -> torch.Tensor:+ """Row-flat index keys for a sparse-layer slot: ``[rows, index_head_dim]``."""+ return self._index_k_buffer[slot]++ def store_index_k(self, k: torch.Tensor, out_loc: torch.Tensor, slot: int) -> None:+ self._index_k_buffer[slot][out_loc] = k+++__all__ = ["BSAKVCache"]diff --git a/python/freetoken/layers/__init__.py b/python/freetoken/layers/__init__.pyindex fd0231c7..2dbd6373 100644--- a/python/freetoken/layers/__init__.py+++ b/python/freetoken/layers/__init__.py@@ -1,4 +1,4 @@-from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul+from .activation import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul, swigluoai_and_mulfrom .base import BaseOP, OPList, StateLessOPfrom .embedding import ParallelLMHead, VocabParallelEmbeddingfrom .linear import (@@ -9,13 +9,20 @@ from .linear import (LinearRowParallel,)from .moe import MoELayer, OffloadMoELayer, make_moe_layer-from .norm import GemmaRMSNorm, RMSNorm, RMSNormFused+from .norm import (+ GemmaPlusOneRMSNorm,+ GemmaPlusOneRMSNormFused,+ GemmaRMSNorm,+ RMSNorm,+ RMSNormFused,+)from .rotary import get_rope, set_rope_device__all__ = ["silu_and_mul","gelu_and_mul","gelu_tanh_and_mul",+ "swigluoai_and_mul","BaseOP","StateLessOP","OPList",@@ -28,6 +35,8 @@ __all__ = ["RMSNorm","RMSNormFused","GemmaRMSNorm",+ "GemmaPlusOneRMSNorm",+ "GemmaPlusOneRMSNormFused","get_rope","set_rope_device","LinearReplicated",diff --git a/python/freetoken/layers/activation.py b/python/freetoken/layers/activation.pyindex 0a70db91..93602b6c 100644--- a/python/freetoken/layers/activation.py+++ b/python/freetoken/layers/activation.py@@ -40,4 +40,20 @@ def gelu_tanh_and_mul(x: torch.Tensor, out: torch.Tensor | None = None):return gelu_tanh_and_mul(x, out=out)-__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul"]+def swigluoai_and_mul(+ x: torch.Tensor,+ out: torch.Tensor | None = None,+ *,+ alpha: float = 1.702,+ limit: float = 7.0,+):+ """SwiGLU-OAI (gpt-oss / MiniMax-M3 ``swigluoai``) over UNINTERLEAVED halves+ (gate ``x[..., :d]``, up ``x[..., d:]``): ``clamp(gate, max=limit) *+ sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``. Always the in-repo Triton+ kernel (flashinfer ships no clamped-swiglu *_and_mul)."""+ from freetoken.kernel.triton.activation import swigluoai_and_mul++ return swigluoai_and_mul(x, out=out, alpha=alpha, limit=limit)+++__all__ = ["silu_and_mul", "gelu_and_mul", "gelu_tanh_and_mul", "swigluoai_and_mul"]diff --git a/python/freetoken/layers/moe.py b/python/freetoken/layers/moe.pyindex b3fa7012..d68d8ded 100644--- a/python/freetoken/layers/moe.py+++ b/python/freetoken/layers/moe.py@@ -466,7 +466,13 @@ class OffloadMoELayer(MoELayer):if fmt == "nvfp4":# FreeToken's Triton inline-dequant kernels over the native ModelOpt# rows: the FP4 banks are read directly in the GEMM, no BF16 copy of- # the experts is ever materialized.+ # the experts is ever materialized. The swigluoai scalars (MiniMax-M3)+ # live on the layer via make_moe_layer's extra_attrs (gpt-oss precedent)+ # and are ignored by the plain *_and_mul activations.+ act_alpha = getattr(self, "hidden_act_alpha", 1.702)+ act_limit = getattr(self, "swiglu_limit", None)+ # None == "no clamp" everywhere else in the repo (mxfp4 maps it to +inf).+ act_limit = float("inf") if act_limit is None else act_limitif is_prefill:from freetoken.moe.fused_nvfp4 import fused_experts_nvfp4@@ -478,6 +484,8 @@ class OffloadMoELayer(MoELayer):n,self.activation,self.apply_router_weight_on_input,+ act_alpha,+ act_limit,)# Marlin-style int32 wide-load GEMV (arithmetic dequant, no HW cvt).# Bit-identical to the byte-at-a-time path; lifts gate/up BW ~43%->51%@@ -492,6 +500,8 @@ class OffloadMoELayer(MoELayer):topk_ids,self.activation,self.apply_router_weight_on_input,+ act_alpha,+ act_limit,)if fmt == "fp8_block":# Block-fp8 experts: fused inline-dequant grouped GEMM reads the routed fp8 rowsdiff --git a/python/freetoken/layers/norm.py b/python/freetoken/layers/norm.pyindex 95f18e63..df248136 100644--- a/python/freetoken/layers/norm.py+++ b/python/freetoken/layers/norm.py@@ -75,6 +75,78 @@ class GemmaRMSNorm(BaseOP):return x, residual+class GemmaPlusOneRMSNorm(BaseOP):+ """(1 + w)-scaled RMSNorm (Gemma semantics: the checkpoint stores ``scale - 1``+ and the effective multiplier is ``1 + weight``, added in fp32 at runtime --+ never folded into the bf16 weight, which would round away the precision the+ format exists to keep). Used by MiniMax-M3 (``use_gemma_norm``) for the decoder+ layernorms, the per-head q/k norms, and the indexer q/k norms.++ Per-head 3D inputs are collapsed to 2D before the kernel call: flashinfer's+ ``gemma_rmsnorm`` CUDA binding is CHECK_DIM(2) (it rejects 3D outright on+ wheels without the CuTe path), and the per-head weight makes the 2D view+ exactly equivalent. The M3 call sites pass contiguous buffers, so the views+ are free; a non-contiguous 3D input is rejected rather than silently copied+ (an in-place norm on a copy would be dropped).+ """++ def __init__(self, size: int, eps: float) -> None:+ from freetoken.kernel.backend import is_flashinfer_installed++ if is_flashinfer_installed():+ from flashinfer.norm import gemma_rmsnorm+ else:+ from freetoken.kernel.triton.norm import gemma_rmsnorm++ self.eps = eps+ self.size = size+ self.weight = torch.empty(size)+ self.gemma_rmsnorm = gemma_rmsnorm++ def _flat(self, x: torch.Tensor) -> torch.Tensor:+ if x.dim() == 2:+ return x+ assert x.is_contiguous(), "per-head gemma norm needs a contiguous buffer"+ return x.view(-1, self.size)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return self.gemma_rmsnorm(self._flat(x), self.weight, self.eps).view(x.shape)++ def forward_inplace(self, x: torch.Tensor) -> None:+ flat = self._flat(x)+ self.gemma_rmsnorm(flat, self.weight, self.eps, out=flat)+++class GemmaPlusOneRMSNormFused(BaseOP):+ """(1 + w)-scaled RMSNorm with the fused-add-residual API of ``RMSNormFused``+ (Gemma semantics, see :class:`GemmaPlusOneRMSNorm`). Drop-in for the decoder+ layernorm seam: ``forward(x, residual)`` returns ``(normed, residual)``."""++ def __init__(self, size: int, eps: float) -> None:+ from freetoken.kernel.backend import is_flashinfer_installed++ if is_flashinfer_installed():+ from flashinfer.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm+ else:+ from freetoken.kernel.triton.norm import (+ gemma_fused_add_rmsnorm,+ gemma_rmsnorm,+ )++ self.eps = eps+ self.weight = torch.empty(size)+ self.gemma_rmsnorm = gemma_rmsnorm+ self.gemma_fused_add_rmsnorm = gemma_fused_add_rmsnorm++ def forward(+ self, x: torch.Tensor, residual: torch.Tensor | None = None+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ if residual is None:+ return self.gemma_rmsnorm(x, self.weight, self.eps), x+ self.gemma_fused_add_rmsnorm(x, residual, self.weight, self.eps)+ return x, residual++class RMSNormFused(BaseOP):def __init__(self, size: int, eps: float) -> None:from freetoken.kernel.backend import is_flashinfer_installeddiff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex ef34052f..3dadbdec 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -144,8 +144,11 @@ AttentionGroupConfig: TypeAlias = (def _full_group_attn_type(group: FullAttentionGroupConfig) -> AttnType:- # Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache.+ # Mirrors the pool-factory split: mla + index slab -> DSAKVCache, mla -> MLAKVCache,+ # GQA (non-mla) + index slab -> BSAKVCache (MiniMax-M3 block-sparse attention).if not group.mla:+ if group.index_head_dim > 0 and group.num_index_layers > 0:+ return AttnType.BSAreturn AttnType.FULLif group.index_head_dim > 0 and group.num_index_layers > 0:return AttnType.DSA@@ -234,6 +237,11 @@ class ModelConfig:# DSA indexer geometry the model module needs. Opaque to model-agnostic engine code;# None for every other model.glm_dsa_args: Any | None = None+ # MiniMax-M3 (minimax_m3) payload (MiniMaxM3Args): the block-sparse indexer geometry+ # (index heads/dim, top-k blocks, init/local blocks, sparse layer set) plus the+ # swigluoai/dense-MLP scalars the model module needs. Opaque to model-agnostic engine+ # code; None for every other model.+ m3_args: Any | None = None# Generic execution-path capability flags (set by a model's parse_config) so the engine and# factories stay model-agnostic instead of branching on dsv4_args:single_stream_only: bool = False # model runs one sequence at a time -> force bs=1diff --git a/python/freetoken/models/minimax_m3/__init__.py b/python/freetoken/models/minimax_m3/__init__.pynew file mode 100644index 00000000..71c0bb51--- /dev/null+++ b/python/freetoken/models/minimax_m3/__init__.py@@ -0,0 +1,15 @@+from .config import parse_config+from .model import MiniMaxM3ForCausalLM+from .weight import (+ iter_weights,+ load_nvfp4_expert_sources,+ load_nvfp4_expert_sources_parallel,+)++__all__ = [+ "MiniMaxM3ForCausalLM",+ "parse_config",+ "iter_weights",+ "load_nvfp4_expert_sources",+ "load_nvfp4_expert_sources_parallel",+]diff --git a/python/freetoken/models/minimax_m3/args.py b/python/freetoken/models/minimax_m3/args.pynew file mode 100644index 00000000..954dc0b5--- /dev/null+++ b/python/freetoken/models/minimax_m3/args.py@@ -0,0 +1,257 @@+"""MiniMax-M3 (``minimax_m3``) hyperparameters.++MiniMax-M3 is a 428B-A23B multimodal MoE (FreeToken serves the text tower; the+``vision_tower.`` stack is skipped like every other checkpoint's). The text tower is+GQA (64 q heads / 4 kv heads / head_dim 128) with per-head Gemma-style (1+w) q/k+norms and partial NeoX RoPE (rotary_dim 64); the first ``sparse_attention_freq==0``+layers run dense attention + a dense SwiGLU-OAI MLP, every later layer runs+**block-sparse attention** (a lightning indexer scores 128-token blocks and each+query attends only its top-``topk_blocks`` blocks, per KV head) + a sigmoid-routed+MoE (128 NVFP4 experts, top-4, + 1 MXFP8 shared expert).++This payload carries the sparse-indexer geometry and the swigluoai/dense-MLP scalars+the model module and the ``m3_sparse`` attention backend need; it is stashed on+``ModelConfig.m3_args`` (opaque to the engine). The MoE/router knobs live on+``ModelConfig`` directly (shared with the glm4_moe/glm_moe_dsa path).++Semantics pinned against the vLLM reference implementation+(the vLLM tree's ``vllm/models/minimax_m3/``):++* index heads == KV heads (4): each index head selects top-k blocks for ITS kv+ head's whole GQA group -- there is no cross-head score reduction.+* one shared index KEY head (``index_k_proj`` is ``[index_dim, hidden]``).+* block score = max over the block's 128 positions of ``dot(index_q, index_k)``+ (``sparse_score_type == "max"``), causal-masked, NO softmax scale (only the+ ordering is consumed).+* the newest ``local_blocks`` (1) blocks are force-selected; ``init_blocks`` (0)+ leading blocks would be too. Selection always includes the query's own block.+* index q/k get the same per-head Gemma norm + partial NeoX RoPE as the main q/k.+* ``sparse_disable_index_value`` is 1 on every sparse layer: the indexer is+ score-only (no index value/output projections exist in the checkpoint).+"""++from __future__ import annotations++from dataclasses import dataclass+from typing import Any, Tuple+++@dataclass(frozen=True)+class MiniMaxM3Args:+ hidden_size: int+ num_heads: int+ num_kv_heads: int+ head_dim: int+ norm_eps: float+ # RoPE (partial NeoX; shared by the main q/k and the indexer q/k)+ rotary_dim: int+ rope_theta: float+ max_position: int+ # SwiGLU-OAI scalars (routed experts, shared experts and the dense MLPs)+ swiglu_alpha: float+ swiglu_limit: float+ dense_intermediate_size: int+ shared_intermediate_size: int+ # Block-sparse attention (all zeros / empty when serving the dense ablation)+ index_dim: int+ num_index_heads: int+ topk_blocks: int+ block_size: int+ init_blocks: int+ local_blocks: int+ sparse_layer_ids: Tuple[int, ...]+ # Layers whose FFN is the sparse MoE block (the rest run the dense MLP)+ moe_layer_ids: Tuple[int, ...]++ @property+ def use_sparse(self) -> bool:+ return bool(self.sparse_layer_ids) and self.index_dim > 0 and self.topk_blocks > 0++ def is_sparse_layer(self, layer_id: int) -> bool:+ return self.use_sparse and layer_id in self.sparse_layer_ids++ def sparse_slot(self, layer_id: int) -> int:+ """Index-key slab slot for a sparse layer (slot order = sparse-layer order)."""+ return self.sparse_layer_ids.index(layer_id)+++def _freq_ids(freq, num_layers: int) -> Tuple[int, ...]:+ if not freq:+ return tuple(range(num_layers))+ return tuple(i for i, f in enumerate(freq[:num_layers]) if f)+++# Config-shape normalization. Two shapes reach load_args: raw checkpoint dicts+# (the auto_map remote-code path keeps rope_theta / sparse_attention_config /+# moe_layer_freq verbatim), and transformers >= 5's native minimax_m3_vl+# TextConfig, whose __post_init__ moves rope_theta into rope_parameters, pops the+# sparse config and moe_layer_freq into flat index_* keys + layer_types /+# mlp_layer_types, and force-sets hidden_act to "silu". Every sibling family+# carries the same dual-shape shim.+def _rope_theta(text_config: Any) -> float:+ theta = getattr(text_config, "rope_theta", None)+ if theta is None:+ params = getattr(text_config, "rope_parameters", None) or {}+ theta = params.get("rope_theta") if isinstance(params, dict) else None+ return float(theta) if theta is not None else 10000.0+++def _sparse_geometry(text_config: Any) -> dict:+ """The sparse-attention geometry in the raw ``sparse_attention_config`` dict+ shape, whichever config shape arrived. The native class has no flat keys for+ init_block / score_type / disable_index_value -- it hardcodes those M3+ semantics, so the translation pins the same values."""+ raw = getattr(text_config, "sparse_attention_config", None)+ if raw:+ return dict(raw)+ layer_types = getattr(text_config, "layer_types", None) or []+ if "minimax_m3_sparse" not in layer_types:+ return {}+ return {+ "use_sparse_attention": True,+ "sparse_attention_freq": [+ 1 if t == "minimax_m3_sparse" else 0 for t in layer_types+ ],+ "sparse_num_index_heads": int(getattr(text_config, "index_n_heads", 4)),+ "sparse_index_dim": int(getattr(text_config, "index_head_dim", 128)),+ "sparse_block_size": int(getattr(text_config, "index_block_size", 128)),+ "sparse_topk_blocks": int(getattr(text_config, "index_topk_blocks", 16)),+ "sparse_local_block": int(getattr(text_config, "index_local_blocks", 1)),+ "sparse_init_block": 0,+ "sparse_score_type": "max",+ }+++def _moe_layer_freq(text_config: Any):+ freq = getattr(text_config, "moe_layer_freq", None)+ if freq is not None:+ return freq+ mlp_types = getattr(text_config, "mlp_layer_types", None)+ if mlp_types is not None:+ return [1 if t == "sparse" else 0 for t in mlp_types]+ return None+++def load_args(text_config: Any, num_layers: int, *, sparse_enabled: bool) -> MiniMaxM3Args:+ """Build the payload from the HF ``text_config`` (already unwrapped), resolving+ the sparse-attention switch ONCE here (``sparse_enabled`` folds FREETOKEN_M3_SPARSE+ in parse_config): the pool factory, the KV cost model, the backend and the model+ modules all read the resolved payload / group spec, never the env."""+ sparse_cfg = _sparse_geometry(text_config)+ use_sparse = sparse_enabled and bool(sparse_cfg.get("use_sparse_attention", False))++ head_dim_early = getattr(text_config, "head_dim", None) or (+ text_config.hidden_size // text_config.num_attention_heads+ )+ if use_sparse:+ score_type = sparse_cfg.get("sparse_score_type", "max")+ assert score_type == "max", (+ f"MiniMax-M3 sparse attention only implements sparse_score_type='max', "+ f"got {score_type!r}"+ )+ sparse_layer_ids = _freq_ids(sparse_cfg.get("sparse_attention_freq"), num_layers)+ # Score-only indexer: the checkpoint ships no index value/output projections.+ disable_value = sparse_cfg.get("sparse_disable_index_value")+ if disable_value is not None:+ assert all(+ bool(disable_value[i]) for i in sparse_layer_ids if i < len(disable_value)+ ), "MiniMax-M3 sparse layers must be score-only (sparse_disable_index_value)"+ # ---- geometry invariants the kernels/backend hardcode; fail a variant ----+ # checkpoint at PARSE time instead of mis-addressing / mis-compiling at+ # serve time.+ blk = int(sparse_cfg.get("sparse_block_size", 128))+ assert blk == 128, (+ f"MiniMax-M3 kernels and the pinned KV page size assume 128-token "+ f"sparse blocks, got sparse_block_size={blk}"+ )+ idx_dim = int(sparse_cfg.get("sparse_index_dim", 0))+ assert idx_dim == head_dim_early, (+ f"the indexer reuses the main rotary (rotary slices by head_dim); "+ f"sparse_index_dim={idx_dim} != head_dim={head_dim_early}"+ )+ topk = int(sparse_cfg.get("sparse_topk_blocks", 0))+ assert 0 < topk < 64, (+ f"the decode top-k kernel's merge stage supports topk_blocks < 64, "+ f"got {topk} (would fail Triton compile at serve time)"+ )+ local = int(sparse_cfg.get("sparse_local_block", 0))+ assert local >= 1, (+ "the attend kernels rely on the forced local block guaranteeing every "+ f"query at least one visible position; got sparse_local_block={local}"+ )+ else:+ sparse_layer_ids = ()++ # swigluoai is implemented in four places (triton *_and_mul, fused_nvfp4, the+ # CPU GEMV epilogue, eager) with the (up + 1) bias hardcoded; a checkpoint+ # that changes swiglu_beta needs all four updated.+ beta = float(getattr(text_config, "swiglu_beta", 1.0))+ assert beta == 1.0, f"swigluoai implementations hardcode beta=1.0, got {beta}"+ # Plain-rope only: silently ignoring a variant checkpoint's rope_scaling would+ # mis-position every token past the scaling boundary. Newer HF configs spell+ # the key `rope_parameters`; check both so a renamed config still trips this.+ scaling = (getattr(text_config, "rope_scaling", None)+ or getattr(text_config, "rope_parameters", None) or {})+ rope_type = scaling.get("rope_type", scaling.get("type", "default")) if scaling else "default"+ assert rope_type in (None, "default"), (+ f"MiniMax-M3 support implements plain rope only, got rope_scaling={scaling!r}"+ )+ scoring = getattr(text_config, "scoring_func", "sigmoid")+ assert scoring == "sigmoid", (+ f"MiniMax-M3 routing implements sigmoid scoring only, got {scoring!r}"+ )++ assert not bool(getattr(text_config, "attention_output_gate", False)), (+ "MiniMax-M3 attention_output_gate is not supported (M3 ships it disabled)"+ )+ assert getattr(text_config, "qk_norm_type", "per_head") == "per_head"+ assert bool(getattr(text_config, "use_gemma_norm", True)), (+ "MiniMax-M3 support assumes Gemma-style (1+w) RMSNorm (use_gemma_norm)"+ )++ head_dim = getattr(text_config, "head_dim", None) or (+ text_config.hidden_size // text_config.num_attention_heads+ )+ rotary_dim = getattr(text_config, "rotary_dim", None)+ if rotary_dim is None:+ rotary_dim = int(head_dim * getattr(text_config, "partial_rotary_factor", 1.0))++ num_index_heads = int(sparse_cfg.get("sparse_num_index_heads", 0)) if use_sparse else 0+ if use_sparse:+ # Per-KV-head selection: each index head feeds one kv head's GQA group.+ assert num_index_heads == text_config.num_key_value_heads, (+ f"expected sparse_num_index_heads == num_key_value_heads, got "+ f"{num_index_heads} != {text_config.num_key_value_heads}"+ )++ return MiniMaxM3Args(+ hidden_size=text_config.hidden_size,+ num_heads=text_config.num_attention_heads,+ num_kv_heads=text_config.num_key_value_heads,+ head_dim=head_dim,+ norm_eps=text_config.rms_norm_eps,+ rotary_dim=rotary_dim,+ rope_theta=_rope_theta(text_config),+ max_position=text_config.max_position_embeddings,+ swiglu_alpha=float(getattr(text_config, "swiglu_alpha", 1.702)),+ swiglu_limit=float(getattr(text_config, "swiglu_limit", 7.0)),+ dense_intermediate_size=int(+ getattr(text_config, "dense_intermediate_size", 0)+ or text_config.intermediate_size+ ),+ shared_intermediate_size=int(+ getattr(text_config, "shared_intermediate_size", 0)+ or text_config.intermediate_size+ ),+ index_dim=int(sparse_cfg.get("sparse_index_dim", 0)) if use_sparse else 0,+ num_index_heads=num_index_heads,+ topk_blocks=int(sparse_cfg.get("sparse_topk_blocks", 0)) if use_sparse else 0,+ block_size=int(sparse_cfg.get("sparse_block_size", 128)) if use_sparse else 0,+ init_blocks=int(sparse_cfg.get("sparse_init_block", 0)) if use_sparse else 0,+ local_blocks=int(sparse_cfg.get("sparse_local_block", 0)) if use_sparse else 0,+ sparse_layer_ids=sparse_layer_ids,+ moe_layer_ids=_freq_ids(_moe_layer_freq(text_config), num_layers),+ )+++__all__ = ["MiniMaxM3Args", "load_args"]diff --git a/python/freetoken/models/minimax_m3/attention.py b/python/freetoken/models/minimax_m3/attention.pynew file mode 100644index 00000000..fdf090b7--- /dev/null+++ b/python/freetoken/models/minimax_m3/attention.py@@ -0,0 +1,122 @@+"""MiniMax-M3 attention: GQA with per-head Gemma q/k norms, partial NeoX RoPE, and+(on the trailing layers) the block-sparse lightning-indexer branch.++The indexer's weights live HERE (``index_qk_proj`` + per-head norms); scoring,+top-k block selection and the sparse attend live in the ``m3_sparse`` backend+(``attention/m3_sparse.py``), the same model/backend split as GLM's DSA. Sparse+layers hand the backend this token's (index_q, index_k) via ``bsa_forward``; the+backend caches the keys in the pool's index slab and selects per-KV-head top-k+128-token blocks. The leading dense layers go through the generic+``forward(q, k, v)`` contract, which the backend serves with a wrapped FULL+backend -- and the FREETOKEN_M3_SPARSE=0 ablation builds no indexer at all, so+every layer takes that path under a plain FULL backend.++Index q/k get the SAME per-head Gemma (1+w) norm + partial NeoX rope as the main+q/k (vLLM reference: ``index_rotary_emb = self.rotary_emb``); norms run before+rope, per head, in fp32 (the fused reference kernel's order).+"""++from __future__ import annotations++from typing import TYPE_CHECKING++from freetoken.core import get_global_ctx+from freetoken.distributed import get_tp_info+from freetoken.layers import BaseOP, GemmaPlusOneRMSNorm+from freetoken.layers.rotary import get_rope+from freetoken.utils import nvtx_annotate++from .mlp import make_proj++if TYPE_CHECKING:+ import torch++ from freetoken.models.config import ModelConfig+++class MiniMaxM3Attention(BaseOP):+ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.m3_args+ assert get_tp_info().size == 1, "MiniMax-M3 currently supports TP=1 only"+ self.layer_id = layer_id+ self.num_qo_heads = args.num_heads+ self.num_kv_heads = args.num_kv_heads+ self.head_dim = args.head_dim+ self.qo_attn_dim = self.num_qo_heads * self.head_dim+ self.kv_attn_dim = self.num_kv_heads * self.head_dim++ quant = config.attn_quant+ self.qkv_proj = make_proj(+ quant, args.hidden_size, self.qo_attn_dim + 2 * self.kv_attn_dim+ )+ self.o_proj = make_proj(quant, self.qo_attn_dim, args.hidden_size)++ # Per-head Gemma (1+w) q/k norms (qk_norm_type == "per_head").+ self.q_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=args.norm_eps)+ self.k_norm = GemmaPlusOneRMSNorm(self.head_dim, eps=args.norm_eps)++ # Partial NeoX rope (rotary_dim 64 of 128); one shared instance (get_rope+ # caches) serves the main q/k AND the indexer q/k, like the reference.+ self.rotary = get_rope(+ head_dim=self.head_dim,+ rotary_dim=args.rotary_dim,+ max_position=args.max_position,+ base=args.rope_theta,+ )++ self.is_sparse = args.is_sparse_layer(layer_id)+ if self.is_sparse:+ self.num_index_heads = args.num_index_heads+ self.index_dim = args.index_dim+ self.index_q_dim = self.num_index_heads * self.index_dim+ # Merged [index_q | index_k] projection (one shared index KEY head).+ self.index_qk_proj = make_proj(+ quant, args.hidden_size, self.index_q_dim + self.index_dim+ )+ self.index_q_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)+ self.index_k_norm = GemmaPlusOneRMSNorm(self.index_dim, eps=args.norm_eps)++ @nvtx_annotate("MHA")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ ctx = get_global_ctx()+ qkv = self.qkv_proj.forward(x)+ q, k, v = qkv.split([self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim], dim=-1)+ # split() returns strided views into the merged qkv buffer; make each part+ # contiguous (the norm kernel then runs in place on the copy) -- v must be+ # contiguous for the KV-cache store kernel regardless.+ q = q.contiguous()+ k = k.contiguous()+ v = v.contiguous()+ self.q_norm.forward_inplace(q.view(-1, self.num_qo_heads, self.head_dim))+ self.k_norm.forward_inplace(k.view(-1, self.num_kv_heads, self.head_dim))+ q, k = self.rotary.forward(ctx.batch.positions, q, k)++ if self.is_sparse:+ iqk = self.index_qk_proj.forward(x)+ del x+ iq, ik = iqk.split([self.index_q_dim, self.index_dim], dim=-1)+ iq = iq.contiguous()+ ik = ik.contiguous()+ self.index_q_norm.forward_inplace(+ iq.view(-1, self.num_index_heads, self.index_dim)+ )+ self.index_k_norm.forward_inplace(ik.view(-1, 1, self.index_dim))+ iq, ik = self.rotary.forward(ctx.batch.positions, iq, ik)+ o = ctx.attn_backend.bsa_forward(+ q.view(-1, self.num_qo_heads, self.head_dim),+ k, v,+ iq.view(-1, self.num_index_heads, self.index_dim),+ ik,+ self.layer_id,+ ctx.batch,+ )+ else:+ del x+ o = ctx.attn_backend.forward(+ q.view(-1, self.num_qo_heads, self.head_dim), k, v,+ self.layer_id, ctx.batch,+ )+ return self.o_proj.forward(o.reshape(-1, self.qo_attn_dim))+++__all__ = ["MiniMaxM3Attention"]diff --git a/python/freetoken/models/minimax_m3/config.py b/python/freetoken/models/minimax_m3/config.pynew file mode 100644index 00000000..906870e3--- /dev/null+++ b/python/freetoken/models/minimax_m3/config.py@@ -0,0 +1,201 @@+"""Engine-facing config for MiniMax-M3 (``minimax_m3``).++The checkpoint is a multimodal wrapper (``model_type=minimax_m3_vl``): the text tower+lives in ``text_config`` and the weights carry a ``language_model.`` prefix. FreeToken+serves the text tower; the ViT vision stack (``vision_tower.`` / projector) is skipped+at load like the other VL checkpoints' (``VISION_KEY_PREFIXES``) -- multimodal input+is future work, so ``ModelConfig.vision_config`` stays None here.++Attention is GQA with block-sparse selection on the trailing layers: parse_config+declares ONE full-attention group over all layers carrying ``mla=False`` plus the+index dims, which resolves to ``AttnType.BSA`` -- the pool factory builds+``BSAKVCache`` (paged GQA K/V + the index-key slab), the KV cost model budgets the+slab from the same spec, and the ``m3_sparse`` backend serves both the dense leading+layers and the sparse layers. ``FREETOKEN_M3_SPARSE=0`` zeroes the index dims (dense+ablation: plain MHAKVCache, every layer attends its whole history through a generic+FULL backend). ``page_size`` is pinned to the 128-token sparse block by the backend's+``page_sizes`` declaration, so one KV page == one sparse block.++Resident-weight quantization is resolved HERE (from the FREETOKEN_M3_*_MXFP8 env+switches, default on) into the standard ``ModelConfig`` fields -- ``attn_quant`` /+``dense_quant`` = ``"mxfp8"`` or ``"none"`` -- and every consumer (the module+constructors and the weight loader) reads those fields, so the resolved config is the+single record of what the served weights actually are. The routed experts are NVFP4+(same ModelOpt layout as MiniMax-M2 / GLM) and always live in the offload cache;+their swigluoai activation restricts the NVFP4 GEMM backend to the Triton kernels+(``select_nvfp4_backend``). lm_head / embeddings are BF16 in the checkpoint.+"""++from __future__ import annotations++import os+from typing import Any++from freetoken.models.config import (+ FullAttentionGroupConfig,+ ModelConfig,+ RotaryConfig,+)++from .args import load_args++def _text_config(hf_config: Any) -> Any:+ return getattr(hf_config, "text_config", None) or hf_config+++# House rule: an env switch that changes WHAT IS SERVED must leave a server-log+# trace. parse_config runs a few times per process (engine + weight loader), so+# each resolved-mode line is logged once per distinct value.+_LOGGED_MODES: set = set()+++def _log_mode_once(key: str, message: str) -> None:+ if key in _LOGGED_MODES:+ return+ _LOGGED_MODES.add(key)+ from freetoken.utils import init_logger++ init_logger(__name__).info(message)+++def parse_config(hf_config: Any) -> ModelConfig:+ text = _text_config(hf_config)++ num_layers = text.num_hidden_layers+ # Dev/testing only: cap the layer count (e.g. FREETOKEN_M3_MAX_LAYERS=5 -> 3 dense+ # + 2 MoE layers) so the forward path / KV / offload cache can be exercised without+ # pinning the full ~230 GB of experts. Unset in normal use (GLM-5.2 precedent).+ _cap = os.environ.get("FREETOKEN_M3_MAX_LAYERS")+ if _cap and int(_cap) < num_layers: # a cap >= the real count changes nothing+ num_layers = int(_cap)+ _log_mode_once(+ f"cap={num_layers}",+ f"FREETOKEN_M3_MAX_LAYERS: serving a TRUNCATED model "+ f"({num_layers}/{text.num_hidden_layers} layers) -- dev/testing only, "+ "outputs are garbage; unset the env for real serving.",+ )++ sparse_enabled = os.getenv("FREETOKEN_M3_SPARSE", "1") != "0"+ if not sparse_enabled:+ _log_mode_once(+ "sparse=0",+ "FREETOKEN_M3_SPARSE=0: serving the DENSE-attention ablation (plain "+ "MHA pool, no indexer; different math and outputs from the shipped "+ "block-sparse model).",+ )+ args = load_args(text, num_layers, sparse_enabled=sparse_enabled)++ # W8A16 MXFP8 (the checkpoint's native dense quantization) for the resident+ # weights; decode is weight-bandwidth bound and the freed VRAM densifies the+ # expert cache. =0 dequantizes to bf16 at load (bring-up / ablation). Read at+ # PARSE time so the resolved config is the single record of the served weights+ # -- an FTW checkpoint converted under one setting must be served under the+ # same one (see weight.py) -- and logged HERE so the FTW serve path (which+ # never runs iter_weights) still leaves a serve-time record of the modes.+ attn_mxfp8 = os.getenv("FREETOKEN_M3_ATTN_MXFP8", "1") != "0"+ mlp_mxfp8 = os.getenv("FREETOKEN_M3_MLP_MXFP8", "1") != "0"+ _log_mode_once(+ f"quant={attn_mxfp8}/{mlp_mxfp8}",+ f"MiniMax-M3 resident quant: attn={'mxfp8' if attn_mxfp8 else 'none'} "+ f"dense={'mxfp8' if mlp_mxfp8 else 'none'} lm_head=none "+ "(FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; an FTW checkpoint "+ "converted under one setting must be served under the same one).",+ )++ # Leading dense-FFN layers: M3's moe_layer_freq is a contiguous 0-prefix; the+ # offload cache and the generic num_moe_layers arithmetic rely on that shape.+ first_k_dense = num_layers - len(args.moe_layer_ids)+ assert args.moe_layer_ids == tuple(range(first_k_dense, num_layers)), (+ "MiniMax-M3 expects a contiguous dense prefix in moe_layer_freq, got "+ f"{args.moe_layer_ids}"+ )++ rotary_config = RotaryConfig(+ head_dim=args.head_dim,+ rotary_dim=args.rotary_dim,+ max_position=args.max_position,+ base=args.rope_theta,+ scaling=None, # plain NeoX partial rope, no scaling+ )++ # Routed experts: NVFP4 per the modelopt MIXED_PRECISION quantized_layers map+ # (checked structurally by the bank loader's shape/dtype asserts). The generic+ # detect_expert_quant would report "mixed_precision", which names the checkpoint,+ # not the experts -- so the expert format is pinned here (qwen3_5_moe precedent).+ quant = getattr(hf_config, "quantization_config", None)+ expert_quant = "nvfp4" if quant is not None else "none"++ # The activation is model-constant swigluoai: assert instead of consume.+ # Raw checkpoints declare "swigluoai"; the native transformers TextConfig+ # force-normalizes hidden_act to "silu", which would send the backend+ # selection down a silu-only path.+ declared_act = getattr(text, "hidden_act", "swigluoai")+ assert declared_act in ("swigluoai", "silu"), (+ f"MiniMax-M3 support implements swigluoai only, got hidden_act={declared_act!r}"+ )++ return ModelConfig(+ num_layers=num_layers,+ num_qo_heads=args.num_heads,+ num_kv_heads=args.num_kv_heads,+ head_dim=args.head_dim,+ hidden_size=args.hidden_size,+ vocab_size=text.vocab_size,+ intermediate_size=args.dense_intermediate_size,+ hidden_act="swigluoai", # pinned; see the declared_act assert above+ rms_norm_eps=text.rms_norm_eps,+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),+ rotary_config=rotary_config,+ # One group over all layers: GQA K/V everywhere, plus the index-key slab for+ # the sparse layers (mla=False + index dims -> AttnType.BSA -> BSAKVCache).+ # The same spec drives the pool factory and the KV cost model, so they can+ # never disagree. FREETOKEN_M3_SPARSE=0 zeroes the index dims (dense ablation:+ # plain MHAKVCache, generic FULL backends serve every layer).+ attention_groups=(+ FullAttentionGroupConfig(+ name="full",+ layer_ids=tuple(range(num_layers)),+ num_kv_heads=args.num_kv_heads,+ head_dim=args.head_dim,+ rotary_config=rotary_config,+ mla=False,+ index_head_dim=args.index_dim if args.use_sparse else 0,+ num_index_layers=(+ sum(1 for lid in args.sparse_layer_ids if lid < num_layers)+ if args.use_sparse+ else 0+ ),+ ),+ ),+ num_experts=int(getattr(text, "num_local_experts", 0)),+ num_experts_per_tok=text.num_experts_per_tok,+ # Routed experts use the text tower's intermediate_size (3072); the dense+ # MLP width (12288) rides ModelConfig.intermediate_size above.+ moe_intermediate_size=text.intermediate_size,+ norm_topk_prob=True, # MiniMax always renormalizes the selected expert weights+ model_type=getattr(hf_config, "model_type", "minimax_m3_vl"),+ architectures=getattr(+ hf_config, "architectures", ["MiniMaxM3SparseForConditionalGeneration"]+ ),+ moe_enabled=True,+ expert_quant=expert_quant,+ first_k_dense_replace=first_k_dense,+ n_shared_experts=int(getattr(text, "n_shared_experts", 0)),+ shared_expert_intermediate_size=args.shared_intermediate_size,+ routed_scaling_factor=float(getattr(text, "routed_scaling_factor", 1.0)),+ has_router_bias=bool(getattr(text, "use_routing_bias", False)),+ use_qk_norm=bool(getattr(text, "use_qk_norm", True)),+ swiglu_limit=args.swiglu_limit,+ hidden_act_alpha=args.swiglu_alpha,+ # Resident-weight quant modes (see the module docstring): recorded here so the+ # resolved config describes the served weights; the constructors and+ # iter_weights read these fields, never the env directly. lm_head stays bf16+ # (the checkpoint excludes it from quantization).+ attn_quant="mxfp8" if attn_mxfp8 else "none",+ dense_quant="mxfp8" if mlp_mxfp8 else "none",+ lm_head_quant="none",+ m3_args=args,+ )+++__all__ = ["parse_config"]diff --git a/python/freetoken/models/minimax_m3/mlp.py b/python/freetoken/models/minimax_m3/mlp.pynew file mode 100644index 00000000..b08f6935--- /dev/null+++ b/python/freetoken/models/minimax_m3/mlp.py@@ -0,0 +1,62 @@+"""SwiGLU-OAI MLP for MiniMax-M3's leading dense layers and per-layer shared experts.++The checkpoint ships these MXFP8 (fp8-e4m3 weight + uint8 e8m0 block-32+``weight_scale_inv``); serving default keeps them native W8A16+(``ModelConfig.dense_quant == "mxfp8"``, from FREETOKEN_M3_MLP_MXFP8) -- decode reads+every shared expert each token, so this halves the resident-MLP weight traffic and+frees the same VRAM for expert-cache slots. ``FREETOKEN_M3_MLP_MXFP8=0`` dequantizes+to bf16 at load (bring-up / ablation).++gate/up are stored merged (``gate_up_proj``, [gate; up] halves -- the loader+concatenates the two checkpoint projections output-wise; MXFP8 scales are per-output-+row so the fusion is exact), which feeds the same uninterleaved ``swigluoai_and_mul``+kernel the NVFP4 expert path uses:+``clamp(gate, max=limit) * sigmoid(alpha * gate) * (clamp(up, +-limit) + 1)``.+"""++from __future__ import annotations++from typing import TYPE_CHECKING++from freetoken.layers import BaseOP, LinearReplicated, swigluoai_and_mul+from freetoken.utils import nvtx_annotate++if TYPE_CHECKING:+ import torch+++def make_proj(quant: str, in_features: int, out_features: int) -> BaseOP:+ """A resident projection in the model's resolved quant mode: ``"mxfp8"`` (W8A16,+ the checkpoint's native block-32 e8m0 format -- see weight.py) or bf16."""+ if quant == "mxfp8":+ from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear++ return Mxfp8Linear(in_features, out_features, has_bias=False)+ return LinearReplicated(in_features, out_features, has_bias=False)+++class MiniMaxM3MLP(BaseOP):+ def __init__(+ self,+ hidden_size: int,+ intermediate_size: int,+ *,+ quant: str,+ alpha: float,+ limit: float,+ ):+ self.gate_up_proj = make_proj(quant, hidden_size, 2 * intermediate_size)+ self.down_proj = make_proj(quant, intermediate_size, hidden_size)+ self._alpha = alpha+ self._limit = limit++ @nvtx_annotate("MLP")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ gate_up = self.gate_up_proj.forward(x)+ del x+ y = swigluoai_and_mul(gate_up, alpha=self._alpha, limit=self._limit)+ del gate_up+ return self.down_proj.forward(y)+++__all__ = ["MiniMaxM3MLP", "make_proj"]diff --git a/python/freetoken/models/minimax_m3/model.py b/python/freetoken/models/minimax_m3/model.pynew file mode 100644index 00000000..86c47328--- /dev/null+++ b/python/freetoken/models/minimax_m3/model.py@@ -0,0 +1,100 @@+from __future__ import annotations++from typing import TYPE_CHECKING, Tuple++import torch+from freetoken.core import get_global_ctx+from freetoken.layers import (+ BaseOP,+ GemmaPlusOneRMSNormFused,+ OPList,+ ParallelLMHead,+ VocabParallelEmbedding,+)+from freetoken.models.blocks import BaseLLMModel+from freetoken.utils import nvtx_annotate++from .attention import MiniMaxM3Attention+from .mlp import MiniMaxM3MLP+from .moe import MiniMaxM3SparseMoeBlock++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class MiniMaxM3DecoderLayer(BaseOP):+ """Dense layers store the FFN under ``mlp``; MoE layers under+ ``block_sparse_moe`` -- matching the checkpoint's naming. All layernorms are+ Gemma-style (1+w) RMSNorm (``use_gemma_norm``)."""++ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.m3_args+ self.self_attn = MiniMaxM3Attention(config, layer_id)+ self.is_moe_layer = layer_id in args.moe_layer_ids+ if self.is_moe_layer:+ self.block_sparse_moe: BaseOP = MiniMaxM3SparseMoeBlock(config, layer_id)+ else:+ self.mlp = MiniMaxM3MLP(+ config.hidden_size,+ args.dense_intermediate_size,+ quant=config.dense_quant,+ alpha=args.swiglu_alpha,+ limit=args.swiglu_limit,+ )+ self.input_layernorm = GemmaPlusOneRMSNormFused(+ size=config.hidden_size, eps=config.rms_norm_eps+ )+ self.post_attention_layernorm = GemmaPlusOneRMSNormFused(+ size=config.hidden_size, eps=config.rms_norm_eps+ )+ self._layer_id = layer_id++ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")+ def forward(+ self, x: torch.Tensor, residual: torch.Tensor | None = None+ ) -> Tuple[torch.Tensor, torch.Tensor]:+ x, residual = self.input_layernorm.forward(x, residual)+ x = self.self_attn.forward(x)+ x, residual = self.post_attention_layernorm.forward(x, residual)+ ffn = self.block_sparse_moe if self.is_moe_layer else self.mlp+ x = ffn.forward(x)+ return x, residual+++class MiniMaxM3Model(BaseOP):+ def __init__(self, config: ModelConfig):+ self.embed_tokens = VocabParallelEmbedding(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ )+ self.layers = OPList(+ [MiniMaxM3DecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ )+ self.norm = GemmaPlusOneRMSNormFused(size=config.hidden_size, eps=config.rms_norm_eps)++ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:+ x = self.embed_tokens.forward(input_ids)+ residual: torch.Tensor | None = None+ for layer in self.layers.op_list:+ x, residual = layer.forward(x, residual)+ return self.norm.forward(x, residual)[0]+++class MiniMaxM3ForCausalLM(BaseLLMModel):+ def __init__(self, config: ModelConfig):+ self.model = MiniMaxM3Model(config)+ # lm_head is BF16 in the checkpoint (excluded from quantization), untied.+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ )+ super().__init__()++ def forward(self) -> torch.Tensor:+ output = self.model.forward(get_global_ctx().batch.input_ids)+ return self.lm_head.forward(output)+++__all__ = ["MiniMaxM3ForCausalLM"]diff --git a/python/freetoken/models/minimax_m3/moe.py b/python/freetoken/models/minimax_m3/moe.pynew file mode 100644index 00000000..9f54c156--- /dev/null+++ b/python/freetoken/models/minimax_m3/moe.py@@ -0,0 +1,92 @@+"""MiniMax-M3 sparse MoE block.++Routing follows the DeepSeek/GLM family (``GlmMoeDsaSparseBlock``): fp32 sigmoid+scores + a selection-only ``e_score_correction_bias``, gather the unbiased scores,+renormalize, scale by ``routed_scaling_factor`` (2.0). No group-limited top-k+(n_group == 1). The routed experts (NVFP4, swigluoai) go through ``make_moe_layer``+(the offload family; the swigluoai alpha/limit ride the layer via ``extra_attrs``,+gpt-oss precedent) and the always-on shared expert (MXFP8, swigluoai) is added.++The checkpoint keeps ``e_score_correction_bias`` fp32 and the reference applies it+fp32; the bias sits on the top-k boundary, so it is stored fp32 here too (unlike+GLM's bf16 storage, whose checkpoint ships it fp32 but consumes it off a bf16 cast).+"""++from __future__ import annotations++from typing import TYPE_CHECKING, Tuple++import torch+import torch.nn.functional as F+from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer++from .mlp import MiniMaxM3MLP++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig++TopK = Tuple[torch.Tensor, torch.Tensor]+++class MiniMaxM3SparseMoeBlock(BaseOP):+ def __init__(self, config: ModelConfig, layer_id: int):+ args = config.m3_args+ self.top_k = config.num_experts_per_tok+ self.num_experts = config.num_experts+ self.norm_topk_prob = config.norm_topk_prob+ self.routed_scaling_factor = config.routed_scaling_factor++ # Router weights kept fp32 (vLLM stores them fp32 too): both the gate and+ # the bias sit on the top-4 selection boundary, where a bf16 round can+ # flip near-tie picks away from the reference. 128 x 6144 x 4 B = ~3.1 MB+ # per layer, ~180 MB across the 57 MoE layers -- accepted for selection+ # fidelity.+ self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)+ self.gate.weight = torch.empty(+ config.num_experts, config.hidden_size, dtype=torch.float32+ )+ # DeepSeek-style selection bias; fp32 in the checkpoint AND here.+ self.e_score_correction_bias = torch.empty(config.num_experts, dtype=torch.float32)++ # The offload cache indexes experts by *MoE* layer (global layer minus+ # first_k_dense_replace), matching how the loader packs the expert banks.+ self.experts = make_moe_layer(+ config,+ layer_id=layer_id - config.first_k_dense_replace,+ renormalize=config.norm_topk_prob,+ activation="swigluoai",+ extra_attrs={+ "hidden_act_alpha": args.swiglu_alpha,+ "swiglu_limit": args.swiglu_limit,+ },+ )+ self.shared_experts = MiniMaxM3MLP(+ config.hidden_size,+ config.shared_expert_intermediate_size * max(1, config.n_shared_experts),+ quant=config.dense_quant,+ alpha=args.swiglu_alpha,+ limit=args.swiglu_limit,+ )++ def _route(self, hidden_states: torch.Tensor) -> TopK:+ # HF/vLLM compute the router logits in fp32; the gate is stored fp32.+ logits = F.linear(hidden_states.float(), self.gate.weight)+ scores = logits.sigmoid()+ scores_for_choice = scores + self.e_score_correction_bias+ _, topk_ids = torch.topk(scores_for_choice, self.top_k, dim=-1)+ topk_weights = scores.gather(-1, topk_ids)+ if self.norm_topk_prob:+ topk_weights = topk_weights / (topk_weights.sum(dim=-1, keepdim=True) + 1e-20)+ topk_weights = topk_weights * self.routed_scaling_factor+ return topk_weights.to(torch.float32).contiguous(), topk_ids.to(torch.int32).contiguous()++ def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:+ num_tokens, hidden_dim = hidden_states.shape+ hidden_states = hidden_states.view(-1, hidden_dim)+ topk_weights, topk_ids = self._route(hidden_states)+ out = self.experts.routed_forward(hidden_states, topk_weights, topk_ids)+ out = out + self.shared_experts.forward(hidden_states)+ return out.view(num_tokens, hidden_dim)+++__all__ = ["MiniMaxM3SparseMoeBlock"]diff --git a/python/freetoken/models/minimax_m3/weight.py b/python/freetoken/models/minimax_m3/weight.pynew file mode 100644index 00000000..e10a7237--- /dev/null+++ b/python/freetoken/models/minimax_m3/weight.py@@ -0,0 +1,283 @@+"""Weight loading for MiniMax-M3 (``minimax_m3``).++The checkpoint is the multimodal wrapper layout: every text-tower tensor carries a+``language_model.`` prefix (``language_model.model.layers.N...``,+``language_model.lm_head.weight``); the ViT stack (``vision_tower.`` /+``multi_modal_projector.`` / ``patch_merge_mlp.``) is never read (text-only serving).++Resident (non routed-expert) dense projections are MXFP8 in the checkpoint+(fp8-e4m3 ``weight`` + uint8 e8m0 block-32 ``weight_scale_inv``). What this loader+yields follows the quant modes RESOLVED IN ``parse_config`` (``ModelConfig.attn_quant``+/ ``dense_quant``, from the FREETOKEN_M3_*_MXFP8 switches, default on): in the default+mode the fp8 weight and its scale codes stream through verbatim (merged output-wise for+the fused qkv / index-qk / gate-up projections -- the scales are per-output-row, so+fusion is exact); with a switch off the projections are dequantized to bf16 at load.+Norms, the router gate, ``e_score_correction_bias`` (kept fp32), embeddings and+lm_head are unquantized and stream through verbatim. Routed experts are NVFP4+(``w1/w3/w2`` = gate/up/down, same ModelOpt layout as MiniMax-M2) and go to the+offload cache via ``load_nvfp4_expert_sources``; expert ``input_scale`` calibration+tensors are unused (W4A16) and never match the bank spec.++FTW caveat: an FTW checkpoint stores whatever iter_weights yielded at CONVERSION time,+and the model is built from the env at SERVE time -- the two must agree (a mismatch+fails loudly in load_state_dict on the ``*.weight_scale_inv`` keys). The active modes+are logged at load so conversion logs record the choice.+"""++from __future__ import annotations++import json+import os+import re+from typing import Iterator++import safetensors+import torch+from freetoken.distributed import get_tp_info+from freetoken.models.loader import drop_page_cache+from freetoken.models.nvfp4_banks import (+ Nvfp4ExpertSourceSpec,+ load_nvfp4_expert_source_banks,+)+from freetoken.utils import cached_load_hf_config, download_hf_weight+from tqdm import tqdm++from .config import parse_config++_EXPERT_KEY_RE = re.compile(+ r"^language_model\.model\.layers\.(?P<layer>\d+)\.block_sparse_moe\."+ r"experts\.(?P<expert>\d+)\.(?P<proj>w1|w2|w3)\."+ r"(?P<kind>weight|weight_scale|weight_scale_2)$"+)+_NVFP4_SOURCE_SPEC = Nvfp4ExpertSourceSpec(+ key_pattern=_EXPERT_KEY_RE,+ proj_to_role={"w1": "gate", "w3": "up", "w2": "down"},+ # Experts exist only for layers [first_k_dense_replace, num_layers); banks pack by+ # MoE-layer index so the leading dense layers leave no holes (GLM precedent).+ layer_to_bank=lambda layer, config: (+ None+ if layer < config.first_k_dense_replace or layer >= config.num_layers+ else layer - config.first_k_dense_replace+ ),+ desc="MiniMax-M3 NVFP4 experts",+)+++class _ShardReader:+ """Random-access reader over the safetensors index (GLM-5.2 precedent): the M3+ dense pass walks layers in model order and fuses q/k/v (etc.) output-wise, which+ a shard-ordered stream cannot do without buffering whole projections anyway."""++ def __init__(self, folder: str, weight_map: dict, device: torch.device):+ self._folder = folder+ self._weight_map = weight_map+ self._device = device+ self._handles: dict[str, object] = {}++ def has(self, name: str) -> bool:+ return name in self._weight_map++ def get(self, name: str) -> torch.Tensor:+ shard = self._weight_map[name]+ handle = self._handles.get(shard)+ if handle is None:+ handle = safetensors.safe_open(+ os.path.join(self._folder, shard), framework="pt", device=str(self._device)+ ).__enter__()+ self._handles[shard] = handle+ return handle.get_tensor(name)++ def close(self) -> None:+ for shard, handle in self._handles.items():+ try:+ handle.__exit__(None, None, None)+ except Exception: # pragma: no cover - best effort+ pass+ drop_page_cache(os.path.join(self._folder, shard))+ self._handles.clear()+++def _read_proj(reader: _ShardReader, key: str) -> tuple[torch.Tensor, torch.Tensor | None]:+ """One dense projection as ``(weight, scale_codes | None)``. MXFP8 tensors carry a+ ``weight_scale_inv`` sibling; a bf16 tensor (nothing quantized under this name)+ comes back scale-less and passes through every mode unchanged."""+ w = reader.get(f"{key}.weight")+ scale_key = f"{key}.weight_scale_inv"+ if reader.has(scale_key):+ s = reader.get(scale_key)+ assert w.dtype == torch.float8_e4m3fn and s.dtype == torch.uint8, (+ f"unexpected MXFP8 dtypes at {key}: {w.dtype}/{s.dtype}"+ )+ # The GEMV folds w * 2**(code-127) into the 16-bit compute dtype, which+ # is lossless only while the product stays in range; codes above 245+ # could overflow bf16. Real checkpoints sit near 127 -- pin at load.+ assert int(s.max()) <= 245, (+ f"e8m0 scale code {int(s.max())} at {key} exceeds the bf16-exact "+ "fold bound (245); the W8A16 kernels would overflow"+ )+ return w, s+ return w, None+++def _emit_fused(+ reader: _ShardReader,+ out_key: str,+ part_keys: list[str],+ keep_mxfp8: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ """Emit one (possibly multi-part output-wise fused) projection in the resolved+ quant mode. MXFP8 scales are per-output-row / per-32-input-block, so concatenating+ the parts' weights AND scales along the output dim is exact."""+ parts = [_read_proj(reader, key) for key in part_keys]+ if keep_mxfp8 and all(s is not None for _, s in parts):+ yield f"{out_key}.weight", torch.cat([w for w, _ in parts], dim=0)+ yield f"{out_key}.weight_scale_inv", torch.cat([s for _, s in parts], dim=0)+ return+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant++ dequant = [+ mxfp8_dequant(w, s, dtype=torch.bfloat16) if s is not None else w+ for w, s in parts+ ]+ yield f"{out_key}.weight", (+ torch.cat(dequant, dim=0) if len(dequant) > 1 else dequant[0]+ )+++def iter_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ assert not include_moe_experts, (+ "MiniMax-M3 stores routed experts as NVFP4 and only supports the offload MoE "+ "backend; experts are loaded into the offload cache via load_nvfp4_expert_sources()."+ )+ assert include_non_moe+ config = parse_config(cached_load_hf_config(model_path))+ args = config.m3_args+ folder = download_hf_weight(model_path)+ with open(os.path.join(folder, "model.safetensors.index.json")) as f:+ weight_map = json.load(f)["weight_map"]+ reader = _ShardReader(folder, weight_map, device)+ primary = get_tp_info().is_primary()+ attn_mx = config.attn_quant == "mxfp8"+ mlp_mx = config.dense_quant == "mxfp8"+ if primary:+ from freetoken.utils import init_logger++ init_logger(__name__).info(+ f"MiniMax-M3 resident quant: attn={config.attn_quant} dense={config.dense_quant} "+ f"lm_head={config.lm_head_quant} (FREETOKEN_M3_ATTN_MXFP8/FREETOKEN_M3_MLP_MXFP8; "+ "an FTW conversion records these choices implicitly -- serve with the same flags)"+ )+ try:+ for layer in tqdm(+ range(config.num_layers),+ desc="Loading MiniMax-M3 dense weights",+ disable=not primary,+ ):+ src = f"language_model.model.layers.{layer}"+ dst = f"model.layers.{layer}"+ a_src, a_dst = f"{src}.self_attn", f"{dst}.self_attn"++ yield from _emit_fused(+ reader,+ f"{a_dst}.qkv_proj",+ [f"{a_src}.q_proj", f"{a_src}.k_proj", f"{a_src}.v_proj"],+ attn_mx,+ )+ yield from _emit_fused(reader, f"{a_dst}.o_proj", [f"{a_src}.o_proj"], attn_mx)+ for norm in ("q_norm", "k_norm"):+ yield f"{a_dst}.{norm}.weight", reader.get(f"{a_src}.{norm}.weight")++ # Block-sparse indexer (sparse layers only; the dense ablation builds no+ # indexer modules, so its tensors are skipped entirely).+ if args.is_sparse_layer(layer):+ yield from _emit_fused(+ reader,+ f"{a_dst}.index_qk_proj",+ [f"{a_src}.index_q_proj", f"{a_src}.index_k_proj"],+ attn_mx,+ )+ for norm in ("index_q_norm", "index_k_norm"):+ yield f"{a_dst}.{norm}.weight", reader.get(f"{a_src}.{norm}.weight")++ for norm in ("input_layernorm", "post_attention_layernorm"):+ yield f"{dst}.{norm}.weight", reader.get(f"{src}.{norm}.weight")++ if layer in args.moe_layer_ids:+ m_src, m_dst = f"{src}.block_sparse_moe", f"{dst}.block_sparse_moe"+ # fp32 like the bias: both sit on the top-4 selection boundary.+ yield (+ f"{m_dst}.gate.weight",+ reader.get(f"{m_src}.gate.weight").to(torch.float32),+ )+ # fp32 in the checkpoint AND the module (top-k selection boundary).+ yield (+ f"{m_dst}.e_score_correction_bias",+ reader.get(f"{m_src}.e_score_correction_bias").to(torch.float32),+ )+ s_src, s_dst = f"{m_src}.shared_experts", f"{m_dst}.shared_experts"+ yield from _emit_fused(+ reader,+ f"{s_dst}.gate_up_proj",+ [f"{s_src}.gate_proj", f"{s_src}.up_proj"],+ mlp_mx,+ )+ yield from _emit_fused(+ reader, f"{s_dst}.down_proj", [f"{s_src}.down_proj"], mlp_mx+ )+ else:+ yield from _emit_fused(+ reader,+ f"{dst}.mlp.gate_up_proj",+ [f"{src}.mlp.gate_proj", f"{src}.mlp.up_proj"],+ mlp_mx,+ )+ yield from _emit_fused(+ reader, f"{dst}.mlp.down_proj", [f"{src}.mlp.down_proj"], mlp_mx+ )++ yield "model.embed_tokens.weight", reader.get("language_model.model.embed_tokens.weight")+ yield "model.norm.weight", reader.get("language_model.model.norm.weight")+ yield "lm_head.weight", reader.get("language_model.lm_head.weight")+ finally:+ reader.close()+++def load_nvfp4_expert_sources(+ model_path: str, config, *, layer_sink=None+) -> dict[str, list[torch.Tensor]]:+ """CPU NVFP4 expert source banks for the offload cache; see load_nvfp4_expert_source_banks."""+ return load_nvfp4_expert_source_banks(+ model_path,+ config,+ _NVFP4_SOURCE_SPEC,+ drop_page_cache=drop_page_cache,+ primary=get_tp_info().is_primary(),+ layer_sink=layer_sink,+ )+++def load_nvfp4_expert_sources_parallel(+ model_path: str, config, *, workers: int = 8, chunk: int = 8 << 20, layer_sink=None+):+ """parallel: same NVFP4 source banks via the common chunked multi-threaded O_DIRECT reader."""+ from freetoken.models.nvfp4_banks import load_nvfp4_expert_source_banks_parallel++ return load_nvfp4_expert_source_banks_parallel(+ model_path,+ config,+ _NVFP4_SOURCE_SPEC,+ drop_page_cache=drop_page_cache,+ primary=get_tp_info().is_primary(),+ workers=workers,+ chunk=chunk,+ layer_sink=layer_sink,+ )+++__all__ = ["iter_weights", "load_nvfp4_expert_sources", "load_nvfp4_expert_sources_parallel"]diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 00ff1412..1e015187 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -37,6 +37,19 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"freetoken.models.minimax_m2","MiniMaxM2ForCausalLM",),+ # MiniMax-M3 (model_type minimax_m3_vl): multimodal wrapper config (text tower in+ # text_config, weights under language_model.); served text-only. GQA + block-sparse+ # attention (lightning indexer, top-k 128-token blocks) on the trailing layers,+ # sigmoid/bias-routed NVFP4 experts + MXFP8 shared expert, swigluoai activation.+ "MiniMaxM3SparseForConditionalGeneration": ModelSpec(+ "freetoken.models.minimax_m3",+ "MiniMaxM3ForCausalLM",+ ),+ # Text-only sibling (the text_config's own architectures entry).+ "MiniMaxM3SparseForCausalLM": ModelSpec(+ "freetoken.models.minimax_m3",+ "MiniMaxM3ForCausalLM",+ ),"DeepseekV4ForCausalLM": ModelSpec("freetoken.models.deepseek_v4","DeepseekV4ForCausalLM",diff --git a/python/freetoken/moe/cpu_executor.py b/python/freetoken/moe/cpu_executor.pyindex 6adf194f..b96205aa 100644--- a/python/freetoken/moe/cpu_executor.py+++ b/python/freetoken/moe/cpu_executor.py@@ -54,9 +54,9 @@ _FLAG_SYNC = os.getenv("FREETOKEN_CPU_MOE_FLAG_SYNC", "1") != "0"# just keeps the host-func path for the extra combos._FLAG_SLOTS_PER_LAYER = 16-# Activation ids must match ActKind in csrc/cpu_moe/cpu_moe_ext.cpp. "gpt_oss_swiglu"-# is handled entirely inside the mxfp4 kernel (clamped swiglu + bias), so its id is a-# placeholder that act_apply never sees.+# Activation ids must match ActKind in csrc/cpu_moe/cpu_moe_ext.cpp. Id 3 is the+# clamped (up + 1) swiglu: "swigluoai" runs it in the generic GEMV epilogue,+# "gpt_oss_swiglu" is the same math fused inside the mxfp4 kernel._ACT_IDS = {"silu": 0,"swish": 0,@@ -64,12 +64,30 @@ _ACT_IDS = {"gelu_tanh": 2,"gelu_pytorch_tanh": 2,"gpt_oss_swiglu": 3,+ "swigluoai": 3,}# Weight-format ids must match WFmt in csrc/cpu_moe/cpu_moe_ext.cpp._WFMT_IDS = {"bf16": 0, "nvfp4": 1, "mxfp4_triton": 2, "ds_fp4": 3, "q4_0": 4}+def compiled_extension_supports(activation: str) -> bool:+ """Whether the compiled ``_cpu_moe`` extension can serve ``activation``+ through its generic epilogue. A stale prebuilt .so accepts newer act ids+ while silently computing the wrong math; the executor hard-errors on that,+ but the engine's auto offload->hybrid upgrade consults this first so a+ default boot degrades to offload instead of crashing after weight load."""+ if activation not in _ACT_IDS:+ return False+ if _ACT_IDS[activation] < 3:+ return True+ try:+ from freetoken.kernel import _cpu_moe+ except ImportError:+ return False+ return _ACT_IDS[activation] <= getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()++def physical_core_cpus() -> list[int]:"""One logical CPU per physical core, restricted to this process's affinity.@@ -150,6 +168,19 @@ class CpuMoeExecutor:)if activation not in _ACT_IDS:raise NotImplementedError(f"CPU MoE backend: unsupported activation {activation!r}")+ # ABI probe: a stale prebuilt _cpu_moe.so accepts newer act ids without+ # error and silently computes the wrong activation in the generic+ # epilogue -- fail loudly with the rebuild instruction instead. (mxfp4+ # handles its act inside the kernel and predates the marker.)+ if _ACT_IDS[activation] >= 3 and fmt != "mxfp4_triton":+ supported = getattr(_cpu_moe, "max_generic_act_id", lambda: 2)()+ if _ACT_IDS[activation] > supported:+ raise RuntimeError(+ f"the compiled _cpu_moe extension predates activation "+ f"{activation!r} (max generic act id {supported}); rebuild it "+ "with `python setup.py build_ext --inplace` (or reinstall the "+ "wheel) before serving this model on the cpu/hybrid backend."+ )self.num_layers = int(cache.num_layers)self.num_experts = int(cache.num_experts)diff --git a/python/freetoken/moe/expert_banks.py b/python/freetoken/moe/expert_banks.pyindex d3e6df99..e456c931 100644--- a/python/freetoken/moe/expert_banks.py+++ b/python/freetoken/moe/expert_banks.py@@ -178,7 +178,8 @@ def _nvfp4_banks(model_path, model_config, device, dtype, dummy, parallel=False,backend = Noneif not native:backend = select_nvfp4_backend(device, getattr(model_config, "moe_intermediate_size", None),- getattr(model_config, "nvfp4_backend", "auto"))+ getattr(model_config, "nvfp4_backend", "auto"),+ activation=getattr(model_config, "hidden_act", "silu"))native = backend == "triton"repack_sink = Nonediff --git a/python/freetoken/moe/fused_nvfp4.py b/python/freetoken/moe/fused_nvfp4.pyindex 4c04192d..5ed8d766 100644--- a/python/freetoken/moe/fused_nvfp4.py+++ b/python/freetoken/moe/fused_nvfp4.py@@ -21,11 +21,33 @@ from freetoken.kernel.triton.nvfp4_fused_moe import (_e2m1_lut,_prefill_nvfp4_moe_kernel,)-from freetoken.layers import gelu_and_mul, gelu_tanh_and_mul, silu_and_mul+from freetoken.layers import (+ gelu_and_mul,+ gelu_tanh_and_mul,+ silu_and_mul,+ swigluoai_and_mul,+)from freetoken.moe.fused import moe_align_block_size_ACT = {"silu": silu_and_mul, "gelu": gelu_and_mul, "gelu_tanh": gelu_tanh_and_mul}++def _run_act(+ activation: str,+ gate_up: torch.Tensor,+ out: torch.Tensor,+ act_alpha: float,+ act_limit: float,+) -> None:+ """gemm1 -> gemm2 activation dispatch. ``swigluoai`` (MiniMax-M3, clamped+ gpt-oss swiglu over the banks' uninterleaved [gate; up] halves) carries the+ per-model ``act_alpha``/``act_limit`` scalars; the plain *_and_mul kinds+ ignore them."""+ if activation == "swigluoai":+ swigluoai_and_mul(gate_up, out, alpha=act_alpha, limit=act_limit)+ return+ _ACT[activation](gate_up, out)+# Decode is captured into a CUDA graph, so the config must be fixed (no triton.autotune,# which benchmarks at run time). Tuned offline against the NVFP4 decode kernels.# These drive the original LUT-gather decode (_decode_gemm), kept only for A/B.@@ -142,6 +164,8 @@ def _fused_experts_decode_nvfp4(topk_ids: torch.Tensor,activation: str,apply_router_weight_on_input: bool,+ act_alpha: float = 1.702,+ act_limit: float = 7.0,) -> torch.Tensor:"""Shared decode body (gemm1 -> act -> gemm2 -> sum-reduce); ``gemm_fn`` is eitherthe marlin-style int32 GEMV (:func:`_decode_gemm_marlin`) or the original LUT-gather@@ -158,7 +182,7 @@ def _fused_experts_decode_nvfp4(ic1, topk_weights, topk_ids, apply_router_weight_on_input, False,)ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)- _ACT[activation](ic1.view(-1, two_i), ic2)+ _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)gemm_fn(ic2, down_packed, down_scale, down_global,@@ -181,6 +205,8 @@ def fused_experts_decode_nvfp4_marlin(topk_ids: torch.Tensor,activation: str = "silu",apply_router_weight_on_input: bool = False,+ act_alpha: float = 1.702,+ act_limit: float = 7.0,) -> torch.Tensor:"""Decode inline-NVFP4 MoE using the Marlin-style int32 wide-load GEMV."""return _fused_experts_decode_nvfp4(@@ -188,6 +214,7 @@ def fused_experts_decode_nvfp4_marlin(hidden_states, gate_up_packed, gate_up_scale, gate_up_global,down_packed, down_scale, down_global,topk_weights, topk_ids, activation, apply_router_weight_on_input,+ act_alpha, act_limit,)@@ -203,6 +230,8 @@ def fused_experts_decode_nvfp4_serial(topk_ids: torch.Tensor,activation: str = "silu",apply_router_weight_on_input: bool = False,+ act_alpha: float = 1.702,+ act_limit: float = 7.0,) -> torch.Tensor:"""Original LUT-gather decode (one program per route, full K reduction). Retained forA/B benchmarking against the marlin decode path; not on the production decode path."""@@ -211,6 +240,7 @@ def fused_experts_decode_nvfp4_serial(hidden_states, gate_up_packed, gate_up_scale, gate_up_global,down_packed, down_scale, down_global,topk_weights, topk_ids, activation, apply_router_weight_on_input,+ act_alpha, act_limit,)@@ -278,6 +308,8 @@ def fused_experts_nvfp4(num_experts: int,activation: str = "silu",apply_router_weight_on_input: bool = False,+ act_alpha: float = 1.702,+ act_limit: float = 7.0,) -> torch.Tensor:"""Prefill inline-NVFP4 MoE. ``topk_ids`` index rows of the bank tensors in``[0, num_experts)``: full-layer banks with position == expert id (the@@ -300,7 +332,7 @@ def fused_experts_nvfp4(apply_router_weight_on_input, cfg,)ic2 = torch.empty((M * top_k, inter), device=dev, dtype=dt)- _ACT[activation](ic1.view(-1, two_i), ic2)+ _run_act(activation, ic1.view(-1, two_i), ic2, act_alpha, act_limit)ic3 = torch.empty((M, top_k, H), device=dev, dtype=dt)_prefill_gemm(ic2, down_packed, down_scale, down_global, ic3,diff --git a/python/freetoken/moe/nvfp4_backends.py b/python/freetoken/moe/nvfp4_backends.pyindex cd715553..e8a3c6f6 100644--- a/python/freetoken/moe/nvfp4_backends.py+++ b/python/freetoken/moe/nvfp4_backends.py@@ -181,7 +181,10 @@ def _b12x_min_intermediate() -> int:def select_nvfp4_backend(- device: torch.device, intermediate_size: int | None = None, requested: str = "auto"+ device: torch.device,+ intermediate_size: int | None = None,+ requested: str = "auto",+ activation: str = "silu",) -> str:"""Pick the NVFP4 expert-GEMM backend for ``device`` (and, in ``auto``, ``intermediate_size``).@@ -195,6 +198,11 @@ def select_nvfp4_backend(* ``marlin`` / ``flashinfer`` / ``triton`` -- force that backend (and its bank layout),raising if it cannot run rather than degrading.+ ``activation`` is the model's routed-expert activation: the borrowed marlin/b12x+ kernels hard-code silu (their fused epilogue), so any other activation (MiniMax-M3's+ ``swigluoai``) resolves ``auto`` to the Triton kernels -- which dispatch the+ activation as a separate elementwise op -- and rejects a forced marlin/flashinfer.+Returns the internal backend name (``marlin`` / ``b12x`` / ``triton``); ``flashinfer``maps to ``b12x``."""@@ -207,6 +215,18 @@ def select_nvfp4_backend()if requested == "triton":return "triton"+ if activation != "silu":+ if requested != "auto":+ raise RuntimeError(+ f"--nvfp4-backend={requested} only supports silu routed experts (its fused "+ f"epilogue); this model's experts use {activation!r} -- use "+ "--nvfp4-backend triton (or auto)."+ )+ logger.info(+ f"NVFP4 auto backend: routed experts use {activation!r}, which only the "+ "Triton kernels support; using the Triton inline-dequant kernels"+ )+ return "triton"if requested == "marlin":if device.type != "cuda":raise RuntimeError("--nvfp4-backend=marlin requires a CUDA device")diff --git a/python/freetoken/server/anthropic_api.py b/python/freetoken/server/anthropic_api.pyindex 3feaa63a..e535ae53 100644--- a/python/freetoken/server/anthropic_api.py+++ b/python/freetoken/server/anthropic_api.py@@ -113,7 +113,10 @@ async def handle_anthropic_messages(model_sampling: dict[str, Any],):try:- spec = convert_anthropic_to_genspec(req, model_sampling)+ spec = convert_anthropic_to_genspec(+ req, model_sampling,+ reasoning_parser=getattr(state.config, "reasoning_parser", None),+ )uid = await submit_generation(spec, state)except ValueError as exc:return _anthropic_error_response(400, "invalid_request_error", str(exc))@@ -145,7 +148,9 @@ async def handle_anthropic_count_tokens(req: AnthropicCountTokensRequest, state:# a checkpoint whose chat template raises ValueError is a server fault, not a bad request,# so it must not fall into the convert/empty-prompt ValueError branch.try:- messages, template_tools, _, ctk = convert_anthropic_prompt(req)+ messages, template_tools, _, ctk = convert_anthropic_prompt(+ req, reasoning_parser=getattr(state.config, "reasoning_parser", None)+ )except ValueError as exc:return _anthropic_error_response(400, "invalid_request_error", str(exc))if not messages:@@ -172,6 +177,7 @@ async def handle_anthropic_count_tokens(req: AnthropicCountTokensRequest, state:# --------------------------------------------------------------------------- #def convert_anthropic_prompt(req: AnthropicMessagesRequest | AnthropicCountTokensRequest,+ reasoning_parser: str | None = None,) -> tuple[list[dict[str, Any]], list[dict[str, Any]] | None, list[dict[str, Any]] | None, dict[str, Any]]:"""(messages, template_tools, parser_tools, chat_template_kwargs) — the promptside of the conversion, shared by /v1/messages and /v1/messages/count_tokens so@@ -278,23 +284,29 @@ def convert_anthropic_prompt(selected = req.tool_choice.name if (req.tool_choice and req.tool_choice.type == "tool") else Nonetemplate_tools, parser_tools = split_tool_lists(raw_tools, selected)- # Native extended-thinking toggle -> template kwargs: "enabled" turns thinking- # on for templates that default it off (gemma4); an explicit "disabled" is- # forwarded so templates that default ON can honor it.+ # Native extended-thinking toggle -> template kwargs, through the per-family+ # mapping in model_meta (a bare enable_thinking bool is inert for templates+ # that read a different knob, e.g. M3's thinking_mode).+ from .model_meta import think_toggle_kwargs+ctk: dict[str, Any] = {}if req.thinking:if req.thinking.get("type") == "enabled":- ctk = {"enable_thinking": True}+ ctk = think_toggle_kwargs(reasoning_parser, True)elif req.thinking.get("type") == "disabled":- ctk = {"enable_thinking": False}+ ctk = think_toggle_kwargs(reasoning_parser, False)return render_messages(messages), template_tools, parser_tools, ctkdef convert_anthropic_to_genspec(- req: AnthropicMessagesRequest, model_sampling: dict[str, Any]+ req: AnthropicMessagesRequest,+ model_sampling: dict[str, Any],+ reasoning_parser: str | None = None,) -> GenSpec:- messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(req)+ messages, template_tools, parser_tools, ctk = convert_anthropic_prompt(+ req, reasoning_parser=reasoning_parser+ )return GenSpec(messages=messages,sampling_params=resolve_sampling(diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex c2bb118e..7bc3593c 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -128,6 +128,10 @@ def parse_args(marker = " ".join(candidates).lower()if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker:return "gpt_oss"+ # M3 first: its marker also contains the bare "minimax" substring, but the+ # namespaced tool grammar is a different protocol from M2's.+ if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker:+ return "minimax_m3"if "minimax" in marker:return "minimax"if "gemma4" in marker:@@ -177,6 +181,10 @@ def parse_args(return "qwen3"if "glm" in marker:return "glm"+ # M3 first ("minimax" is a substring): <mm:think> tags + 3 thinking gears,+ # not M2's always-on implicit <think>.+ if "minimax_m3" in marker or "minimax-m3" in marker or "minimaxm3" in marker:+ return "minimax_m3"if "minimax" in marker:return "minimax"if "gemma4" in marker:@@ -407,6 +415,7 @@ def parse_args("gemma4","glm47","minimax",+ "minimax_m3","gpt_oss","gpt-oss",],@@ -417,11 +426,15 @@ def parse_args("--reasoning-parser",type=str,default="auto",- choices=["auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm", "minimax", "gemma4"],+ choices=[+ "auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm",+ "minimax", "minimax_m3", "gemma4",+ ],help=("Reasoning parser that splits chain-of-thought into reasoning_content ""for OpenAI responses. 'auto' selects per model family (gpt-oss Harmony, "- "<think> for qwen3/glm/minimax, gemma thought, dsv4); 'off' disables it."+ "<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, gemma thought, "+ "dsv4); 'off' disables it."),)diff --git a/python/freetoken/server/function_call_parser.py b/python/freetoken/server/function_call_parser.pyindex 6981b476..0c22b8e0 100644--- a/python/freetoken/server/function_call_parser.py+++ b/python/freetoken/server/function_call_parser.py@@ -62,6 +62,7 @@ TOOLS_TAG_LIST = ["<function=","<tool_call>","<minimax:tool_call>",+ "]<]minimax[>[<tool_call>","<|python_tag|>","[TOOL_CALLS]","<|DSML|function_calls>",@@ -2386,6 +2387,473 @@ class MiniMaxDetector(InvokeParamStreamMixin, BaseFormatDetector):+class MiniMaxM3Detector(BaseFormatDetector):+ """FreeToken serving adapter for MiniMax-M3 namespace-delimited XML tool calls.++ M3 is NOT M2 with renamed tags (vLLM's Rust ``MinimaxM3ToolParser`` grammar is+ the reference): every structural tag is prefixed with the ``]<]minimax[>[``+ namespace marker, one wrapper block holds multiple ``<invoke>`` tags, and the+ arguments are parameter-NAME XML elements rendered recursively -- objects nest+ tags, arrays repeat ``<item>``::++ ]<]minimax[>[<tool_call>+ ]<]minimax[>[<invoke name="create_order">+ ]<]minimax[>[<user_id>42]<]minimax[>[</user_id>+ ]<]minimax[>[<shipping>+ ]<]minimax[>[<city>Singapore]<]minimax[>[</city>+ ]<]minimax[>[</shipping>+ ]<]minimax[>[</invoke>+ ]<]minimax[>[</tool_call>++ Leaf values are typed from the tool schema at every nesting level (loose JSON+ when undeclared). Element scanning is lenient -- malformed tags and stray text+ never void the parameters that did parse, and a truncated trailing element+ salvages every complete sibling (which is what makes ``recover_truncated_call``+ work for calls cut off by max_tokens).++ Streaming: text outside the wrapper streams live (holding back a partial+ marker suffix); inside the wrapper each completed ``</invoke>`` emits its call+ in one piece, wire order preserved. The buffer keeps the wrapper opener while+ inside a block so ``finish_streaming`` suppresses truncated-markup residue and+ ``recover_truncated_call`` can re-parse it.+ """++ NS = "]<]minimax[>["++ def __init__(self):+ super().__init__()+ self.bot_token = self.NS + "<tool_call>"+ self.eot_token = self.NS + "</tool_call>"+ # The template renders double quotes but models emit single quotes too.+ self.invoke_open_re = re.compile(+ re.escape(self.NS) + r'<invoke\s+name=["\']([^"\']+)["\']\s*>'+ )+ self._invoke_close = self.NS + "</invoke>"+ # While True, self._buffer starts with bot_token (the retained opener).+ self._m3_in_block = False++ def has_tool_call(self, text: str) -> bool:+ return self.bot_token in text++ def _partial_marker_suffix(self, buffer: str) -> int:+ """Length of the LONGEST buffer suffix that is a proper prefix of the+ wrapper opener. The base ``_ends_with_partial_token`` returns the shortest+ match, which under-holds for this marker: ``]`` recurs inside+ ``]<]minimax[>[...``, so a buffer ending ``]<]`` would hold only 1 char and+ leak ``]<`` as content, breaking the marker forever."""+ for i in range(min(len(buffer), len(self.bot_token) - 1), 0, -1):+ if self.bot_token.startswith(buffer[-i:]):+ return i+ return 0++ # ---- lenient recursive parameter-XML scanning ------------------------------------+ _TAG_BAD_CHARS = frozenset(' "<>/')++ def _scan_elements(self, text: str) -> tuple:+ """Scan ``text`` for ``NS<k>...NS</k>`` elements; returns ``(items, stray)``+ where ``stray`` is the inter-element text (surfaced as ``"$text"`` for+ mixed content). Lenient: malformed tags and dangling closers are stepped+ over without dropping later siblings, an unterminated trailing element+ salvages everything before it, and same-name nesting is depth-matched.++ Known leniency trade-off: a leaf value quoting a well-formed element pair+ parses as structure, and one quoting its own closer ends the element+ early -- the strict alternative voided every parameter over one stray+ character. (Quoted invoke/wrapper markers are handled structurally by+ ``_scan_invoke_interior``.)+ """+ items: List[tuple] = []+ stray_parts: List[str] = []+ pos, n = 0, len(text)+ while pos < n:+ nxt = text.find(self.NS + "<", pos)+ if nxt == -1:+ stray_parts.append(text[pos:])+ break+ stray_parts.append(text[pos:nxt])+ pos = nxt+ if text.startswith(self.NS + "</", pos):+ # A close tag we did not open: model noise -- skip the marker and+ # keep scanning (aborting dropped every later sibling).+ pos += len(self.NS) + 2+ continue+ tag_start = pos + len(self.NS) + 1+ gt = text.find(">", tag_start)+ if gt == -1:+ break # truncated open tag+ name = text[tag_start:gt]+ if not name or any(c in self._TAG_BAD_CHARS for c in name):+ pos = gt + 1 # malformed tag (e.g. an <invoke ...> echo): step over+ continue+ open_tag = self.NS + "<" + name + ">"+ close_tag = self.NS + "</" + name + ">"+ depth, search, inner_end = 1, gt + 1, -1+ while depth:+ cpos = text.find(close_tag, search)+ if cpos == -1:+ # unterminated element: salvage what parsed+ return items, "".join(stray_parts)+ opos = text.find(open_tag, search)+ if opos != -1 and opos < cpos:+ depth += 1+ search = opos + len(open_tag)+ else:+ depth -= 1+ inner_end = cpos+ search = cpos + len(close_tag)+ items.append((name, text[gt + 1 : inner_end]))+ pos = search+ return items, "".join(stray_parts)++ def _typed_leaf(self, raw: str, prop: Any) -> Any:+ """Leaf dequoting. The template renders leaf values verbatim, so string+ leaves round-trip exactly (stripping would corrupt multi-line arguments).+ Declared non-string types tolerate surrounding whitespace; undeclared+ leaves fall back to loose JSON but keep the verbatim text when the parse+ yields a string anyway."""+ if prop is not None:+ if isinstance(prop, dict) and "type" not in prop:+ subs = prop.get("anyOf") or prop.get("oneOf")+ if isinstance(subs, list) and subs:+ return self._union_leaf(raw, subs)+ ptype = (+ str(prop.get("type", "string")).strip().lower()+ if isinstance(prop, dict)+ else "string"+ )+ if ptype in ("string", "str", "enum"):+ return raw+ if ptype in ("number", "float", "double"):+ # Integer literals stay int; "5.0" stays 5.0.+ s = raw.strip()+ try:+ return int(s)+ except ValueError:+ try:+ return float(s)+ except ValueError:+ return raw+ if ptype in ("object", "array") and raw.strip() == "":+ # Empty container-typed element -> typed empty container.+ return {} if ptype == "object" else []+ return self._convert_param_value(raw.strip(), "_", {"_": prop}, "")+ if raw.strip() == "":+ # An empty element is the empty STRING; loose-JSON would make it {}.+ return ""+ value = _parse_loose_json_value(raw.strip())+ return raw if isinstance(value, str) else value++ def _union_leaf(self, raw: str, subs: list) -> Any:+ """``anyOf``/``oneOf``: try each member's strict coercion in declared+ order; verbatim text when only the string member (or nothing) matches."""+ s = raw.strip()+ for sub in subs:+ t = str(sub.get("type", "")).strip().lower() if isinstance(sub, dict) else ""+ if t in ("integer", "int"):+ try:+ return int(s)+ except ValueError:+ continue+ if t in ("number", "float", "double"):+ try:+ return int(s)+ except ValueError:+ try:+ return float(s)+ except ValueError:+ continue+ if t in ("boolean", "bool") and s.lower() in ("true", "false"):+ return s.lower() == "true"+ if t == "null" and s.lower() == "null":+ return None+ if t in ("object", "array"):+ try:+ v = json.loads(s)+ except (json.JSONDecodeError, ValueError):+ continue+ if isinstance(v, dict if t == "object" else list):+ return v+ return raw++ def _nested_value(self, raw: str, schema: Any = None) -> Any:+ items, stray = self._scan_elements(raw)+ if not items:+ return self._typed_leaf(raw, schema)+ return self._structure(items, schema, stray=stray)++ def _structure(self, items: List[tuple], schema: Any = None, stray: str = "") -> Any:+ """Composite value from scanned elements, threading the JSON schema down+ (``properties`` / ``items`` / ``additionalProperties``). Only ``<item>``+ children render as a bare array -- the template's array convention;+ repeated siblings under any other tag stay an object with an array-valued+ key. Mixed text+children keeps the text under ``"$text"``."""+ props = schema.get("properties") if isinstance(schema, dict) else None+ item_schema = schema.get("items") if isinstance(schema, dict) else None++ def _sub_schema(key: str) -> Any:+ sub = props.get(key) if isinstance(props, dict) else None+ if sub is None and isinstance(schema, dict):+ ap = schema.get("additionalProperties")+ if isinstance(ap, dict):+ sub = ap+ return sub++ names = {k for k, _ in items}+ if names == {"item"}:+ return [self._nested_value(raw, item_schema) for _, raw in items]+ counts: Dict[str, int] = {}+ for k, _ in items:+ counts[k] = counts.get(k, 0) + 1+ out: Dict[str, Any] = {}+ for key, raw in items:+ sub = _sub_schema(key)+ # A repeated key's elements are the array's members: unwrap its array+ # schema to the item schema. A singleton key keeps the array schema+ # (its <item> children unwrap inside).+ if (+ counts[key] > 1+ and isinstance(sub, dict)+ and str(sub.get("type", "")).lower() == "array"+ ):+ sub = sub.get("items")+ value = self._nested_value(raw, sub)+ if key in out:+ prev = out[key]+ out[key] = (prev if isinstance(prev, list) else [prev]) + [value]+ else:+ out[key] = value+ if stray.strip():+ out["$text"] = stray.strip()+ return out++ def _args_from_items(+ self, func_name: str, items: List[tuple], tools: List[Tool]+ ) -> Dict:+ config = self._get_param_config(func_name, tools)+ args: Dict[str, Any] = {}+ for key, raw in items:+ prop = config.get(key) if isinstance(config, dict) and key in config else None+ nested, stray = self._scan_elements(raw)+ if nested:+ value: Any = self._structure(nested, prop, stray=stray)+ else:+ value = self._typed_leaf(raw, prop)+ if key in args:+ prev = args[key]+ args[key] = (prev if isinstance(prev, list) else [prev]) + [value]+ else:+ args[key] = value+ return args++ def _scan_invoke_interior(self, text: str, pos: int) -> tuple:+ """Collect parameter elements from ``pos`` (just past an invoke opener)+ until a top-level ``NS</invoke>`` / ``NS</tool_call>`` / end of text.+ Close markers are only interpreted between elements, so a value quoting+ wire syntax can neither end the invoke early nor truncate the block.+ Returns ``(items, end_pos, closer)`` with ``closer`` in+ {"invoke", "eot", None}; ``end_pos`` points at the closer marker+ (``len(text)`` when the text ran out -- truncation salvage)."""+ items: List[tuple] = []+ n = len(text)+ while pos < n:+ nxt = text.find(self.NS + "<", pos)+ if nxt == -1:+ return items, n, None+ if text.startswith(self._invoke_close, nxt):+ return items, nxt, "invoke"+ if text.startswith(self.eot_token, nxt):+ return items, nxt, "eot"+ pos = nxt+ if text.startswith(self.NS + "</", pos):+ # Dangling closer we did not open: model noise -- skip it and+ # keep collecting (the references keep well-formed siblings).+ pos += len(self.NS) + 2+ continue+ tag_start = pos + len(self.NS) + 1+ gt = text.find(">", tag_start)+ if gt == -1:+ return items, n, None # truncated open tag+ name = text[tag_start:gt]+ if not name or any(c in self._TAG_BAD_CHARS for c in name):+ pos = gt + 1 # malformed tag (e.g. a quoted <invoke ...> echo)+ continue+ open_tag = self.NS + "<" + name + ">"+ close_tag = self.NS + "</" + name + ">"+ depth, search, inner_end = 1, gt + 1, -1+ while depth:+ cpos = text.find(close_tag, search)+ if cpos == -1:+ return items, n, None # unterminated element: salvage+ opos = text.find(open_tag, search)+ if opos != -1 and opos < cpos:+ depth += 1+ search = opos + len(open_tag)+ else:+ depth -= 1+ inner_end = cpos+ search = cpos + len(close_tag)+ items.append((name, text[gt + 1 : inner_end]))+ pos = search+ return items, n, None++ def _parse_block(+ self, text: str, pos: int, tools: List[Tool], calls: List[ToolCallItem]+ ) -> int:+ """Parse a wrapper interior starting at ``pos`` (just past ``bot_token``),+ appending calls; returns the position just past the block's+ ``NS</tool_call>``, or ``len(text)`` when the wrapper never closes+ (end-of-generation truncation -- a truncated trailing invoke still+ salvages its complete parameters, the recover_truncated_call path)."""+ tool_indices = self._get_tool_indices(tools)+ n = len(text)+ while pos < n:+ nxt = text.find(self.NS + "<", pos)+ if nxt == -1:+ return n+ if text.startswith(self.eot_token, nxt):+ return nxt + len(self.eot_token)+ m = self.invoke_open_re.match(text, nxt)+ if m is None:+ pos = nxt + len(self.NS) + 1 # stray marker at block level+ continue+ items, end, closer = self._scan_invoke_interior(text, m.end())+ func_name = m.group(1)+ if func_name in tool_indices or _should_forward_unknown_tool(func_name):+ args = self._args_from_items(func_name, items, tools)+ calls.append(+ ToolCallItem(+ tool_index=len(calls),+ name=func_name,+ parameters=json.dumps(args, ensure_ascii=False),+ )+ )+ else:+ logger.warning(f"Model attempted to call undefined function: {func_name}")+ if closer == "invoke":+ pos = end + len(self._invoke_close)+ elif closer == "eot":+ return end + len(self.eot_token)+ else:+ return n+ return n++ # ---- one-shot -------------------------------------------------------------------+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:+ """Parse EVERY wrapper block (text between and after blocks stays content).+ A block without ``</tool_call>`` (end-of-generation truncation) still+ yields its parseable calls -- recover_truncated_call appends the closer+ and re-enters here."""+ calls: List[ToolCallItem] = []+ normal_parts: List[str] = []+ pos = 0+ while True:+ b = text.find(self.bot_token, pos)+ if b == -1:+ normal_parts.append(text[pos:])+ break+ normal_parts.append(text[pos:b])+ pos = self._parse_block(text, b + len(self.bot_token), tools, calls)+ return StreamingParseResult(+ normal_text="".join(normal_parts).strip(), calls=calls+ )++ # ---- streaming ------------------------------------------------------------------+ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult:+ self._buffer += new_text+ calls: List[ToolCallItem] = []+ normal_parts: List[str] = []+ tool_indices = self._get_tool_indices(tools)++ while True:+ if not self._m3_in_block:+ pos = self._buffer.find(self.bot_token)+ if pos == -1:+ hold = self._partial_marker_suffix(self._buffer)+ release = self._buffer[: len(self._buffer) - hold]+ if release and calls:+ break # wire order: text after a call defers to the next step+ self._buffer = self._buffer[len(release) :]+ if release:+ normal_parts.append(release)+ break+ if pos > 0:+ if calls:+ break # wire order (text precedes a SECOND block)+ normal_parts.append(self._buffer[:pos])+ self._buffer = self._buffer[pos:]+ self._m3_in_block = True+ continue++ # In-block: the buffer keeps the opener (finish_streaming and+ # recover_truncated_call key off has_tool_call(buffer)). Markers are+ # interpreted structurally, same as detect_and_parse: the first+ # top-level invoke opener or wrapper closer wins.+ body = self._buffer[len(self.bot_token) :]+ action = None+ scan = 0+ while True:+ nxt = body.find(self.NS + "<", scan)+ if nxt == -1:+ break+ if body.startswith(self.eot_token, nxt):+ action = ("eot", nxt, None)+ break+ m = self.invoke_open_re.match(body, nxt)+ if m is not None:+ action = ("invoke", nxt, m)+ break+ scan = nxt + len(self.NS) + 1 # stray marker at block level+ if action is None:+ break # inside the block, nothing complete yet -> hold+ if action[0] == "eot":+ # Wrapper closed: drop the block, loop back to the idle scan so the+ # residue gets partial-marker holds and second-block detection+ # instead of leaking raw.+ self._buffer = body[action[1] + len(self.eot_token) :]+ self._m3_in_block = False+ continue+ m = action[2]+ items, end, closer = self._scan_invoke_interior(body, m.end())+ if closer != "invoke":+ # Invoke still streaming (or eot-truncated: left to+ # finish_streaming / recover_truncated_call) -> hold.+ break+ func_name = m.group(1)+ self._buffer = self.bot_token + body[end + len(self._invoke_close) :]+ if func_name not in tool_indices and not _should_forward_unknown_tool(+ func_name+ ):+ logger.warning(+ f"Model attempted to call undefined function: {func_name}"+ )+ continue+ args = self._args_from_items(func_name, items, tools)+ args_json = json.dumps(args, ensure_ascii=False)+ self.current_tool_id += 1+ while len(self.prev_tool_call_arr) <= self.current_tool_id:+ self.prev_tool_call_arr.append({})+ while len(self.streamed_args_for_tool) <= self.current_tool_id:+ self.streamed_args_for_tool.append("")+ self.prev_tool_call_arr[self.current_tool_id] = {+ "name": func_name,+ "arguments": args,+ }+ calls.append(+ ToolCallItem(+ tool_index=self.current_tool_id, name=func_name, parameters=""+ )+ )+ calls.append(+ ToolCallItem(+ tool_index=self.current_tool_id, name=None, parameters=args_json+ )+ )+ self.streamed_args_for_tool[self.current_tool_id] = args_json+ continue++ return StreamingParseResult(normal_text="".join(normal_parts), calls=calls)++class GptOssDetector(BaseFormatDetector):"""FreeToken serving adapter for Harmony ``to=functions.*`` tool calls.@@ -2591,6 +3059,7 @@ class FunctionCallParser:"glm47": Glm47Detector,"llama3": Llama32Detector,"minimax": MiniMaxDetector,+ "minimax_m3": MiniMaxM3Detector,"mistral": MistralDetector,"qwen": Qwen25Detector,"qwen25": Qwen25Detector,diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex 6fb40948..bd8741b8 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -327,6 +327,11 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:or bool(ctk.get("enable_thinking"))or bool(ctk.get("thinking")))+ elif parser_name == "minimax_m3":+ # M3's template pre-opens <mm:think> only in thinking_mode "enabled" (the+ # model then emits just the closing tag); "adaptive" (default) leaves the+ # model to open the tag itself and "disabled" pre-closes it.+ force_reasoning = (spec.chat_template_kwargs or {}).get("thinking_mode") == "enabled"else:force_reasoning = (resolve_thinking_mode(spec.chat_template_kwargs, spec.template_tools) == "thinking"diff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.pyindex 18e4952c..d976c924 100644--- a/python/freetoken/server/model_meta.py+++ b/python/freetoken/server/model_meta.py@@ -22,6 +22,10 @@ def think_spec(reasoning_parser: str | None) -> Tuple[Tuple[str, ...], str | Nonreturn ("off", "on", "max"), "off" # thinking on/off + a max-effort gearif reasoning_parser == "minimax":return ("on",), "on" # template always opens a think block; no off path+ if reasoning_parser == "minimax_m3":+ # M3's template takes thinking_mode disabled/adaptive/enabled; adaptive+ # (the template's own default) lets the model decide per turn.+ return ("off", "adaptive", "on"), "adaptive"if reasoning_parser == "gemma4":return ("off", "on"), "off" # gemma's template defaults thinking offif reasoning_parser in ("qwen3", "glm"):@@ -41,9 +45,28 @@ def think_chat_template_kwargs(reasoning_parser: str | None, gear: str | None) -return {"enable_thinking": gear == "on"}if reasoning_parser == "minimax":return {} # always thinks; its template reads no knob+ if reasoning_parser == "minimax_m3":+ mode = {"off": "disabled", "adaptive": "adaptive", "on": "enabled"}[gear]+ return {"thinking_mode": mode}return {"enable_thinking": gear == "on"} # qwen3, glm, gemma4+def think_toggle_kwargs(reasoning_parser: str | None, enabled: bool) -> dict:+ """``chat_template_kwargs`` for a protocol-level thinking on/off toggle+ (Anthropic ``thinking.type``, Responses ``reasoning.effort``), routed through+ the same per-family mapping as the chat-completions gears -- a hardcoded+ ``enable_thinking`` is inert for templates that read a different knob (M3's+ ``thinking_mode``). A family without the requested direction returns ``{}``;+ with no configured parser the protocol-generic key is kept."""+ gears, _default = think_spec(reasoning_parser)+ if not gears:+ return {"enable_thinking": enabled}+ gear = "on" if enabled else "off"+ if gear not in gears:+ return {}+ return think_chat_template_kwargs(reasoning_parser, gear)++def moe_total_experts(config: Any) -> int:"""Total routed-expert slots the model has: experts per layer x MoE layers. Matches theengine's own basis (``Engine._resolve_auto_moe_cache_size``), so a residency rate deriveddiff --git a/python/freetoken/server/reasoning_parser.py b/python/freetoken/server/reasoning_parser.pyindex db31658e..2d5e3d29 100644--- a/python/freetoken/server/reasoning_parser.py+++ b/python/freetoken/server/reasoning_parser.py@@ -411,6 +411,126 @@ class ThinkReasoningParser(BaseReasoningParser):)+class MiniMaxM3ReasoningParser(BaseReasoningParser):+ """Reasoning parser for MiniMax-M3's ``<mm:think>...</mm:think>`` protocol.++ M3's template has three thinking modes: "enabled" pre-opens ``<mm:think>`` at+ the generation prompt (the model emits only the closing tag -- the caller+ passes ``force_reasoning=True``), "adaptive" leaves the model to open the tag+ itself, and "disabled" pre-closes it (no reasoning). The tool block's+ namespaced opener ends reasoning when a (malformed) turn skips+ ``</mm:think>`` and runs straight into a tool call (dsv4 precedent).++ Adaptive quirk (matches vLLM / llama.cpp): a NON-thinking adaptive turn+ starts with a bare ``</mm:think>`` written by the model itself ("thinking+ off for this turn"). Without special handling that literal marker would+ stream into content on the DEFAULT gear's most common path, so a single+ leading closer (whitespace-tolerant head: the detokenizer may open with a+ newline before the marker; anything else before a closer keeps it visible)+ is stripped in both one-shot and streaming modes -- never when+ ``force_reasoning`` is set, where the closer genuinely terminates the+ template-opened think block.+ """++ THINK_START = "<mm:think>"+ THINK_END = "</mm:think>"++ def __init__(self, force_reasoning: bool = False, stream_reasoning: bool = True) -> None:+ super().__init__(+ think_start_token=self.THINK_START,+ think_end_token=self.THINK_END,+ force_reasoning=force_reasoning,+ stream_reasoning=stream_reasoning,+ tool_start_token="]<]minimax[>[<tool_call>",+ )+ # Streaming state for the leading-bare-closer check: hold the head of the+ # stream while it is still a prefix of ``</mm:think>``.+ self._leading_closer_pending = not force_reasoning+ self._head_buffer = ""++ def detect_and_parse(self, text: str) -> ReasoningParseResult:+ """One-shot parse, positional and verbatim, matching the streaming path:+ reasoning is anchored at the first opener the model wrote, prose before+ it stays content, later marker occurrences are data, and nothing is+ whitespace-trimmed. (The base one-shot relabels prose before a+ mid-content opener and strips both sides -- wrong for this grammar.)"""+ end = self.think_end_token+ if self.force_reasoning:+ # enabled gear: the template pre-opened the think block.+ reasoning, sep, rest = text.partition(end)+ if sep:+ return ReasoningParseResult(reasoning_text=reasoning, normal_text=rest)+ # No closer: a malformed turn running straight into a tool block+ # ends reasoning there (dsv4 precedent); else truncated reasoning.+ if self.tool_start_token is not None and self.tool_start_token in text:+ t = text.find(self.tool_start_token)+ return ReasoningParseResult(+ reasoning_text=text[:t], normal_text=text[t:]+ )+ return ReasoningParseResult(reasoning_text=text)+ # adaptive: a leading bare closer is "thinking off for this turn"+ # (whitespace-tolerant head, same as streaming).+ head = text.lstrip()+ if head.startswith(end):+ return ReasoningParseResult(normal_text=head[len(end) :])+ start = text.find(self.think_start_token)+ if start == -1:+ return ReasoningParseResult(normal_text=text)+ before = text[:start]+ body = text[start + len(self.think_start_token) :]+ reasoning, sep, rest = body.partition(end)+ if sep:+ return ReasoningParseResult(+ reasoning_text=reasoning, normal_text=before + rest+ )+ if self.tool_start_token is not None and self.tool_start_token in body:+ t = body.find(self.tool_start_token)+ return ReasoningParseResult(+ reasoning_text=body[:t], normal_text=before + body[t:]+ )+ return ReasoningParseResult(reasoning_text=body)++ def parse_streaming_increment(self, new_text: str) -> ReasoningParseResult:+ if self._leading_closer_pending:+ self._head_buffer += new_text+ # Whitespace-tolerant head: the detokenizer may open with a newline+ # before the model's bare closer.+ head = self._head_buffer.lstrip()+ if head.startswith(self.think_end_token):+ # Bare leading closer: strip it once, everything after is content.+ self._leading_closer_pending = False+ self._head_buffer = ""+ self._in_reasoning = False+ rest = head[len(self.think_end_token) :]+ return (+ super().parse_streaming_increment(rest)+ if rest+ else ReasoningParseResult()+ )+ if not head or self.think_end_token.startswith(head):+ return ReasoningParseResult() # still a (whitespace+) prefix: hold+ # Diverged: not a leading closer -- replay the held head as normal+ # (whitespace included; it was real output).+ replay, self._head_buffer = self._head_buffer, ""+ self._leading_closer_pending = False+ return super().parse_streaming_increment(replay)+ return super().parse_streaming_increment(new_text)++ def flush(self) -> ReasoningParseResult:+ if self._leading_closer_pending and self._head_buffer:+ # Stream ended while the head was still a closer prefix (e.g. "</mm:t"):+ # it was never a bare closer, replay it before the base flush.+ head, self._head_buffer = self._head_buffer, ""+ self._leading_closer_pending = False+ first = super().parse_streaming_increment(head)+ rest = super().flush()+ return ReasoningParseResult(+ reasoning_text=first.reasoning_text + rest.reasoning_text,+ normal_text=first.normal_text + rest.normal_text,+ )+ return super().flush()++class GemmaThoughtReasoningParser(BaseReasoningParser):"""Reasoning parser for Gemma-4's thought channel: the model emits its thought,then a closing ``<channel|>`` marker, then the visible answer. The opening@@ -435,6 +555,7 @@ class ReasoningParser:"qwen3": ThinkReasoningParser,"glm": ThinkReasoningParser,"minimax": ThinkReasoningParser,+ "minimax_m3": MiniMaxM3ReasoningParser,"gemma4": GemmaThoughtReasoningParser,}diff --git a/python/freetoken/server/responses_api.py b/python/freetoken/server/responses_api.pyindex 7255f7cb..31a316f8 100644--- a/python/freetoken/server/responses_api.py+++ b/python/freetoken/server/responses_api.py@@ -152,7 +152,10 @@ async def handle_responses(return _error_response(400, "max_output_tokens must be a positive integer")default_max = getattr(state.config, "max_output_tokens", None) or DEFAULT_MAX_OUTPUT_TOKENStry:- spec = convert_responses_to_genspec(req, model_sampling, default_max_tokens=default_max)+ spec = convert_responses_to_genspec(+ req, model_sampling, default_max_tokens=default_max,+ reasoning_parser=getattr(state.config, "reasoning_parser", None),+ )uid = await submit_generation(spec, state)except ValueError as exc:return _error_response(400, str(exc))@@ -184,6 +187,7 @@ def convert_responses_to_genspec(req: ResponsesRequest,model_sampling: dict[str, Any],default_max_tokens: int = DEFAULT_MAX_OUTPUT_TOKENS,+ reasoning_parser: str | None = None,) -> GenSpec:# Collect every system/developer text — the top-level `instructions` PLUS any# system/developer-role input items (codex sends both: a system prompt as `instructions`@@ -226,19 +230,19 @@ def convert_responses_to_genspec(else:template_tools, parser_tools = split_tool_lists(raw_tools, selected)- # Thinking toggle: an explicit chat_template_kwargs extra field wins; else the- # protocol-native `reasoning` object (codex sends {"effort": ...}) drives the- # template's thinking mode. effort "none" explicitly DISABLES thinking (vLLM- # maps it the same way); other/unset efforts enable it.+ # Thinking toggle: an explicit chat_template_kwargs extra field wins; else+ # the protocol-native `reasoning` object drives the template through the+ # per-family mapping in model_meta. effort "none" disables thinking (vLLM+ # semantics); other efforts enable it and are forwarded for templates that+ # grade them (gpt-oss).+ from .model_meta import think_toggle_kwargs+ctk = dict(getattr(req, "chat_template_kwargs", None) or {})if req.reasoning and not ctk:effort = req.reasoning.get("effort")- if effort == "none":- ctk = {"enable_thinking": False}- else:- ctk = {"enable_thinking": True}- if effort:- ctk["reasoning_effort"] = effort+ ctk = dict(think_toggle_kwargs(reasoning_parser, effort != "none"))+ if effort and effort != "none":+ ctk.setdefault("reasoning_effort", effort)return GenSpec(messages=render_messages(messages),diff --git a/tests/engine/test_attention_backend_matrix.py b/tests/engine/test_attention_backend_matrix.pyindex fe0cae67..f27640b9 100644--- a/tests/engine/test_attention_backend_matrix.py+++ b/tests/engine/test_attention_backend_matrix.py@@ -64,6 +64,9 @@ def _model_config(kind):elif kind == "dsv4":mc.dsv4_args = SimpleNamespace(window_size=128)specs = (_spec("dsv4", AttnType.DSV4, sliding_window=128),)+ elif kind == "bsa":+ # MiniMax-M3 shape: one FULL-family group, mla=False + index dims -> BSA.+ specs = (_spec("full", AttnType.BSA, index_head_dim=128),)elif kind == "linear_hybrid":mc.has_linear_attention = Truespecs = (_spec("full", AttnType.FULL),)@@ -105,6 +108,7 @@ def _patch_env(monkeypatch, *, major=9, flashinfer=True, sgl=True):("mla", "dsa"), # plain latent MLA("dsa", "dsa"), # MLA + DSA indexer (GLM-5.2 shape)("dsv4", "dsv4_sparse"),+ ("bsa", "m3_sparse"), # MiniMax-M3 block-sparse GQA],)def test_auto_resolves_per_type(monkeypatch, kind, expected):@@ -116,6 +120,29 @@ def test_auto_resolves_per_type(monkeypatch, kind, expected):assert config.attention_backend == expected+def test_auto_bsa_sets_block_page_size(monkeypatch):+ # m3_sparse declares page_sizes=(128,): one KV page == one sparse block, and+ # config-time resolution must coerce the page size to match.+ from freetoken.engine.engine import _adjust_config++ _patch_env(monkeypatch)+ config = _config("bsa", attention_backend="auto")+ _adjust_config(config)+ assert config.page_size == 128+++def test_bsa_rejects_float32_dtype(monkeypatch):+ # --dtype float32 used to pass config validation and die on the pool's+ # itemsize==2 assert only after the model was resident.+ from freetoken.engine.engine import _adjust_config++ _patch_env(monkeypatch)+ config = _config("bsa", attention_backend="auto")+ object.__setattr__(config, "dtype", torch.float32)+ with pytest.raises(ValueError, match="16-bit"):+ _adjust_config(config)++def test_auto_dsv4_sets_window_page_size(monkeypatch):from freetoken.engine.engine import _adjust_config@@ -131,7 +158,11 @@ def test_auto_dsv4_sets_window_page_size(monkeypatch):# reverse gates: type-specific backends on models without the type("full", "dsa"),("full", "dsv4_sparse"),+ ("full", "m3_sparse"),("swa", "dsa"),+ # forward gates: generic backends on the BSA-locked model+ ("bsa", "fi"),+ ("bsa", "triton"),# forward gates: generic backends on type-locked models("mla", "fi"),("mla", "triton"),diff --git a/tests/kernels/test_minimax_m3_sparse.py b/tests/kernels/test_minimax_m3_sparse.pynew file mode 100644index 00000000..9a56d8fe--- /dev/null+++ b/tests/kernels/test_minimax_m3_sparse.py@@ -0,0 +1,437 @@+"""Parity tests for the MiniMax-M3 block-sparse attention kernels.++The pure-torch reference below implements the semantics pinned against the vLLM+reference implementation (see the vLLM tree's ``vllm/models/minimax_m3/`` and+``kernel/triton/minimax_m3_sparse.py``'s module docstring):++* per-index-head block scores: max over the block's 128 positions of+ ``dot(index_q, index_k)``, causal-masked, no scale;+* top-k selection with forced init/local blocks (score boosts 1e30/1e29, local+ wins overlaps), -1 padding past the visible block count;+* GQA attend over the selected blocks only (each kv head follows its own index+ head), causal inside the newest block.++The physical layout mirrors serving: a row-flat slab addressed through a+per-request ``block_rows`` base-row table built from a random page permutation,+so the tests exercise the gather addressing, not just the math.+"""++from __future__ import annotations++import pytest+import torch++if not torch.cuda.is_available(): # pragma: no cover+ pytest.skip("CUDA required for the M3 sparse kernels", allow_module_level=True)++from freetoken.kernel.triton.minimax_m3_sparse import (+ SPARSE_BLOCK_SIZE,+ m3_index_decode,+ m3_index_score_prefill,+ m3_index_topk_prefill,+ m3_sparse_attn_decode,+ m3_sparse_attn_prefill,+)++BLK = SPARSE_BLOCK_SIZE+DEV = "cuda"+++# ---------------------------------------------------------------------------+# Torch reference+# ---------------------------------------------------------------------------+def ref_block_scores(+ iq: torch.Tensor, # [T, H, D] queries at absolute positions q_pos+ ik: torch.Tensor, # [S, D] index keys for positions 0..S-1+ q_pos: torch.Tensor, # [T] absolute positions+) -> torch.Tensor:+ """[H, T, nblocks] fp32; max over each 128-block, causal, -inf invisible."""+ scores = torch.einsum("thd,sd->hts", iq.float(), ik.float())+ s_pos = torch.arange(ik.shape[0], device=iq.device)+ scores = scores.masked_fill(s_pos[None, None, :] > q_pos[None, :, None], float("-inf"))+ nblocks = (ik.shape[0] + BLK - 1) // BLK+ pad = nblocks * BLK - ik.shape[0]+ if pad:+ scores = torch.nn.functional.pad(scores, (0, pad), value=float("-inf"))+ return scores.view(*scores.shape[:2], nblocks, BLK).amax(dim=-1)+++def ref_select(+ block_scores: torch.Tensor, # [H, T, nblocks]+ q_pos: torch.Tensor, # [T]+ topk: int,+ init_blocks: int,+ local_blocks: int,+) -> list[list[set[int]]]:+ """Per (head, token) SET of selected block ids (order is score-descending in+ the kernels, but only membership is contract -- the attend softmax is+ order-invariant)."""+ H, T, nblocks = block_scores.shape+ out: list[list[set[int]]] = []+ for h in range(H):+ rows = []+ for t in range(T):+ nb = int(q_pos[t]) // BLK + 1+ s = block_scores[h, t, :nb].clone()+ if init_blocks:+ s[: min(init_blocks, nb)] = 1e30+ if local_blocks:+ s[max(0, nb - local_blocks):] = 1e29+ k = min(topk, nb)+ rows.append(set(torch.topk(s, k).indices.tolist()))+ out.append(rows)+ return out+++def ref_sparse_attend(+ q: torch.Tensor, # [T, HQ, D] queries at q_pos+ k: torch.Tensor, # [S, KVH, D]+ v: torch.Tensor, # [S, KVH, D]+ selection: list[list[set[int]]], # [KVH][T] block-id sets+ q_pos: torch.Tensor, # [T]+ sm_scale: float,+) -> torch.Tensor:+ T, HQ, D = q.shape+ S, KVH, _ = k.shape+ group = HQ // KVH+ out = torch.empty(T, HQ, D, dtype=torch.float32, device=q.device)+ s_pos = torch.arange(S, device=q.device)+ for t in range(T):+ for kh in range(KVH):+ sel = sorted(selection[kh][t])+ pos_mask = torch.zeros(S, dtype=torch.bool, device=q.device)+ for b in sel:+ pos_mask[b * BLK : min((b + 1) * BLK, S)] = True+ pos_mask &= s_pos <= int(q_pos[t])+ idx = pos_mask.nonzero(as_tuple=True)[0]+ kk = k[idx, kh].float() # [n, D]+ vv = v[idx, kh].float()+ for g in range(group):+ h = kh * group + g+ logits = (q[t, h].float() @ kk.t()) * sm_scale+ out[t, h] = torch.softmax(logits, dim=-1) @ vv+ return out+++def make_layout(nblocks: int, seed: int = 0):+ """A shuffled physical layout: logical block i lives at page perm[i]."""+ g = torch.Generator(device="cpu").manual_seed(seed)+ perm = torch.randperm(nblocks + 3, generator=g)[:nblocks] # +3 spare pages+ total_pages = nblocks + 3+ block_rows = (perm * BLK).to(torch.int32).to(DEV)+ return block_rows, total_pages+++def scatter_rows(+ x: torch.Tensor, block_rows: torch.Tensor, total_pages: int, fill: float = float("nan")+):+ """Place logical rows [S, ...] into a physical slab [total_pages*BLK, ...].++ Unwritten rows default to NaN, modeling the serving pool's ``torch.empty``+ recycled-allocator garbage: the attend kernels MUST pos-mask every K/V load+ or the forced local (partial) block poisons the output -- zero-filled slabs+ hid exactly that bug from the original tests. The index-key slab passes+ ``fill=0.0`` (BSAKVCache zero-inits it; the score kernels rely on that).+ """+ S = x.shape[0]+ slab = torch.full(+ (total_pages * BLK, *x.shape[1:]), fill, dtype=x.dtype, device=x.device+ )+ for b in range((S + BLK - 1) // BLK):+ n = min(BLK, S - b * BLK)+ base = int(block_rows[b])+ slab[base : base + n] = x[b * BLK : b * BLK + n]+ return slab+++# ---------------------------------------------------------------------------+# Tests+# ---------------------------------------------------------------------------+H_IDX, D_IDX = 4, 128+HQ, KVH, D = 16, 4, 128 # scaled-down GQA (group 4) to keep the reference fast+TOPK, INIT, LOCAL = 4, 0, 1+SCALE = D**-0.5++[email protected]("topk", [4, 16])[email protected]("kv_len,q_len", [(BLK * 7 + 55, 300), (BLK * 2, BLK * 2), (90, 90)])+def test_prefill_index_score_and_topk(kv_len: int, q_len: int, topk: int):+ torch.manual_seed(0)+ prefix = kv_len - q_len+ nblocks = (kv_len + BLK - 1) // BLK+ iq = torch.randn(q_len, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)+ ik = torch.randn(kv_len, D_IDX, device=DEV, dtype=torch.bfloat16)+ q_pos = torch.arange(prefix, kv_len, device=DEV)++ block_rows, total_pages = make_layout(nblocks)+ ik_slab = scatter_rows(ik, block_rows, total_pages, fill=0.0)++ cu = torch.tensor([0, q_len], dtype=torch.int32, device=DEV)+ seq = torch.tensor([kv_len], dtype=torch.int32, device=DEV)+ pre = torch.tensor([prefix], dtype=torch.int32, device=DEV)+ score = m3_index_score_prefill(+ iq, ik_slab, block_rows.view(1, -1), cu, seq, pre, q_len, kv_len+ )+ ref = ref_block_scores(iq, ik, q_pos)+ for t in range(q_len):+ nb = int(q_pos[t]) // BLK + 1+ got = score[:, t, :nb].float()+ want = ref[:, t, :nb]+ assert torch.allclose(got, want, atol=0.35, rtol=0.02), (t, (got - want).abs().max())++ topk_idx = m3_index_topk_prefill(score, cu, pre, q_len, topk, INIT, LOCAL)+ ref_sel = ref_select(ref, q_pos, topk, INIT, LOCAL)+ for h in range(H_IDX):+ for t in range(q_len):+ nb = int(q_pos[t]) // BLK + 1+ k = min(topk, nb)+ got = topk_idx[h, t].tolist()+ assert set(got[:k]) == ref_sel[h][t], (h, t, got, ref_sel[h][t])+ assert all(i == -1 for i in got[k:]), (h, t, got)++[email protected]("topk", [4, 16])[email protected]("kv_lens", [[BLK * 6 + 17, 70, BLK * 3]])+def test_decode_index_topk(kv_lens: list[int], topk: int):+ torch.manual_seed(1)+ bs = len(kv_lens)+ max_nb = max((L + BLK - 1) // BLK for L in kv_lens)+ iq = torch.randn(bs, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)+ block_rows = torch.zeros(bs, max_nb, dtype=torch.int32, device=DEV)+ slabs = []+ total = 0+ layouts = []+ for i, L in enumerate(kv_lens):+ nb = (L + BLK - 1) // BLK+ br, pages = make_layout(nb, seed=i)+ layouts.append((br, pages))+ block_rows[i, :nb] = br + total * BLK+ total += pages+ slab = torch.zeros(total * BLK, D_IDX, device=DEV, dtype=torch.bfloat16)+ iks = []+ off = 0+ for i, L in enumerate(kv_lens):+ ik = torch.randn(L, D_IDX, device=DEV, dtype=torch.bfloat16)+ iks.append(ik)+ br, pages = layouts[i]+ sub = scatter_rows(ik, br, pages, fill=0.0)+ slab[off * BLK : (off + pages) * BLK] = sub+ off += pages+ seq = torch.tensor(kv_lens, dtype=torch.int32, device=DEV)+ topk_idx = m3_index_decode(iq, slab, block_rows, seq, max_nb, topk, INIT, LOCAL)+ for i, L in enumerate(kv_lens):+ q_pos = torch.tensor([L - 1], device=DEV)+ ref = ref_block_scores(iq[i : i + 1], iks[i], q_pos)+ sel = ref_select(ref, q_pos, topk, INIT, LOCAL)+ nb = (L + BLK - 1) // BLK+ k = min(topk, nb)+ for h in range(H_IDX):+ got = topk_idx[h, i].tolist()+ assert set(got[:k]) == sel[h][0], (i, h, got, sel[h][0])+ assert all(x == -1 for x in got[k:]), (i, h, got)+++def _attend_case(kv_len: int, q_len: int, seed: int):+ torch.manual_seed(seed)+ prefix = kv_len - q_len+ nblocks = (kv_len + BLK - 1) // BLK+ q = torch.randn(q_len, HQ, D, device=DEV, dtype=torch.bfloat16)+ k = torch.randn(kv_len, KVH, D, device=DEV, dtype=torch.bfloat16)+ v = torch.randn(kv_len, KVH, D, device=DEV, dtype=torch.bfloat16)+ iq = torch.randn(q_len, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)+ ik = torch.randn(kv_len, D_IDX, device=DEV, dtype=torch.bfloat16)+ q_pos = torch.arange(prefix, kv_len, device=DEV)+ block_rows, total_pages = make_layout(nblocks, seed=seed)+ return q, k, v, iq, ik, q_pos, block_rows, total_pages, nblocks, prefix++[email protected]("topk", [4, 16])[email protected]("kv_len,q_len", [(BLK * 7 + 55, 200), (BLK * 2, BLK * 2), (77, 77)])+def test_prefill_attend(kv_len: int, q_len: int, topk: int):+ q, k, v, iq, ik, q_pos, block_rows, total_pages, nblocks, prefix = _attend_case(+ kv_len, q_len, seed=2+ )+ k_slab = scatter_rows(k, block_rows, total_pages)+ v_slab = scatter_rows(v, block_rows, total_pages)+ ik_slab = scatter_rows(ik, block_rows, total_pages, fill=0.0)++ cu = torch.tensor([0, q_len], dtype=torch.int32, device=DEV)+ seq = torch.tensor([kv_len], dtype=torch.int32, device=DEV)+ pre = torch.tensor([prefix], dtype=torch.int32, device=DEV)+ score = m3_index_score_prefill(+ iq, ik_slab, block_rows.view(1, -1), cu, seq, pre, q_len, kv_len+ )+ topk_idx = m3_index_topk_prefill(score, cu, pre, q_len, topk, INIT, LOCAL)++ out = torch.empty_like(q)+ m3_sparse_attn_prefill(+ q, k_slab, v_slab, topk_idx, block_rows.view(1, -1), cu, seq, pre,+ q_len, SCALE, out,+ )++ ref_scores = ref_block_scores(iq, ik, q_pos)+ sel = ref_select(ref_scores, q_pos, topk, INIT, LOCAL)+ ref = ref_sparse_attend(q, k, v, sel, q_pos, SCALE)+ err = (out.float() - ref).abs().max().item()+ assert err < 3e-2, err+++def test_prefill_attend_equals_dense_when_topk_covers():+ """kv_len <= topk * 128 -> the selection covers every visible block, so the+ sparse attend must equal dense causal attention exactly."""+ kv_len = q_len = BLK * TOPK # 4 blocks, topk 4+ q, k, v, iq, ik, q_pos, block_rows, total_pages, nblocks, prefix = _attend_case(+ kv_len, q_len, seed=3+ )+ k_slab = scatter_rows(k, block_rows, total_pages)+ v_slab = scatter_rows(v, block_rows, total_pages)+ ik_slab = scatter_rows(ik, block_rows, total_pages, fill=0.0)+ cu = torch.tensor([0, q_len], dtype=torch.int32, device=DEV)+ seq = torch.tensor([kv_len], dtype=torch.int32, device=DEV)+ pre = torch.tensor([prefix], dtype=torch.int32, device=DEV)+ score = m3_index_score_prefill(+ iq, ik_slab, block_rows.view(1, -1), cu, seq, pre, q_len, kv_len+ )+ topk_idx = m3_index_topk_prefill(score, cu, pre, q_len, TOPK, INIT, LOCAL)+ out = torch.empty_like(q)+ m3_sparse_attn_prefill(+ q, k_slab, v_slab, topk_idx, block_rows.view(1, -1), cu, seq, pre,+ q_len, SCALE, out,+ )+ # dense causal reference+ group = HQ // KVH+ ref = torch.empty(q_len, HQ, D, device=DEV, dtype=torch.float32)+ s_pos = torch.arange(kv_len, device=DEV)+ for h in range(HQ):+ kh = h // group+ logits = (q[:, h].float() @ k[:, kh].float().t()) * SCALE+ logits = logits.masked_fill(s_pos[None, :] > q_pos[:, None], float("-inf"))+ ref[:, h] = torch.softmax(logits, dim=-1) @ v[:, kh].float()+ err = (out.float() - ref).abs().max().item()+ assert err < 3e-2, err++[email protected]("topk", [4, 16])[email protected]("kv_lens", [[BLK * 6 + 17, 70, BLK * 3, BLK * 9]])+def test_decode_attend(kv_lens: list[int], topk: int):+ torch.manual_seed(4)+ bs = len(kv_lens)+ max_nb = max((L + BLK - 1) // BLK for L in kv_lens)+ q = torch.randn(bs, HQ, D, device=DEV, dtype=torch.bfloat16)+ iq = torch.randn(bs, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)+ block_rows = torch.zeros(bs, max_nb, dtype=torch.int32, device=DEV)+ total = 0+ layouts = []+ for i, L in enumerate(kv_lens):+ nb = (L + BLK - 1) // BLK+ br, pages = make_layout(nb, seed=10 + i)+ layouts.append((br, pages))+ block_rows[i, :nb] = br + total * BLK+ total += pages+ # K/V gaps carry NaN (the serving pool is torch.empty; the kernels must+ # pos-mask); the index slab stays zeroed (BSAKVCache invariant).+ k_slab = torch.full((total * BLK, KVH, D), float("nan"), device=DEV, dtype=torch.bfloat16)+ v_slab = torch.full_like(k_slab, float("nan"))+ ik_slab = torch.zeros(total * BLK, D_IDX, device=DEV, dtype=torch.bfloat16)+ ks, vs, iks = [], [], []+ off = 0+ for i, L in enumerate(kv_lens):+ kk = torch.randn(L, KVH, D, device=DEV, dtype=torch.bfloat16)+ vv = torch.randn(L, KVH, D, device=DEV, dtype=torch.bfloat16)+ ik = torch.randn(L, D_IDX, device=DEV, dtype=torch.bfloat16)+ ks.append(kk); vs.append(vv); iks.append(ik)+ br, pages = layouts[i]+ k_slab[off * BLK : (off + pages) * BLK] = scatter_rows(kk, br, pages)+ v_slab[off * BLK : (off + pages) * BLK] = scatter_rows(vv, br, pages)+ ik_slab[off * BLK : (off + pages) * BLK] = scatter_rows(ik, br, pages, fill=0.0)+ off += pages+ seq = torch.tensor(kv_lens, dtype=torch.int32, device=DEV)+ topk_idx = m3_index_decode(iq, ik_slab, block_rows, seq, max_nb, topk, INIT, LOCAL)+ out = torch.empty_like(q)+ m3_sparse_attn_decode(q, k_slab, v_slab, topk_idx, block_rows, seq, SCALE, out)++ for i, L in enumerate(kv_lens):+ q_pos = torch.tensor([L - 1], device=DEV)+ ref_scores = ref_block_scores(iq[i : i + 1], iks[i], q_pos)+ sel = ref_select(ref_scores, q_pos, topk, INIT, LOCAL)+ ref = ref_sparse_attend(q[i : i + 1], ks[i], vs[i], sel, q_pos, SCALE)+ err = (out[i].float() - ref[0]).abs().max().item()+ assert err < 3e-2, (i, err)+++def test_decode_attend_capture_replay_mutated_lengths():+ """The decode index+attend pair under CUDA-graph capture, replayed with+ mutated kv lengths (1 / 129 / 8191 rotated across requests) and refreshed+ K/V/index data: the split-K partition and every length-dependent mask must+ be replay-safe, with nothing but ``max_nb`` baked into the capture -- the+ production graph contract."""+ topk = 16+ LENS = [1, 129, 8191]+ bs = len(LENS)+ MAXL = max(LENS)+ max_nb = (MAXL + BLK - 1) // BLK+ block_rows = torch.zeros(bs, max_nb, dtype=torch.int32, device=DEV)+ total = 0+ layouts = []+ for i in range(bs):+ br, pages = make_layout(max_nb, seed=20 + i)+ layouts.append((br, pages))+ block_rows[i] = br + total * BLK+ total += pages+ k_slab = torch.full(+ (total * BLK, KVH, D), float("nan"), device=DEV, dtype=torch.bfloat16+ )+ v_slab = torch.full_like(k_slab, float("nan"))+ ik_slab = torch.zeros(total * BLK, D_IDX, device=DEV, dtype=torch.bfloat16)++ def refresh(seed):+ torch.manual_seed(seed)+ ks, vs, iks = [], [], []+ off = 0+ for i in range(bs):+ kk = torch.randn(MAXL, KVH, D, device=DEV, dtype=torch.bfloat16)+ vv = torch.randn(MAXL, KVH, D, device=DEV, dtype=torch.bfloat16)+ ik = torch.randn(MAXL, D_IDX, device=DEV, dtype=torch.bfloat16)+ ks.append(kk); vs.append(vv); iks.append(ik)+ br, pages = layouts[i]+ k_slab[off * BLK : (off + pages) * BLK] = scatter_rows(kk, br, pages)+ v_slab[off * BLK : (off + pages) * BLK] = scatter_rows(vv, br, pages)+ ik_slab[off * BLK : (off + pages) * BLK] = scatter_rows(ik, br, pages, fill=0.0)+ off += pages+ return ks, vs, iks++ q = torch.randn(bs, HQ, D, device=DEV, dtype=torch.bfloat16)+ iq = torch.randn(bs, H_IDX, D_IDX, device=DEV, dtype=torch.bfloat16)+ seq = torch.tensor(LENS, dtype=torch.int32, device=DEV)+ out = torch.empty_like(q)+ data = refresh(100)+ # warmup outside capture (Triton JIT), then capture the pair+ ti = m3_index_decode(iq, ik_slab, block_rows, seq, max_nb, topk, INIT, LOCAL)+ m3_sparse_attn_decode(q, k_slab, v_slab, ti, block_rows, seq, SCALE, out)+ torch.cuda.synchronize()+ graph = torch.cuda.CUDAGraph()+ with torch.cuda.graph(graph):+ ti = m3_index_decode(iq, ik_slab, block_rows, seq, max_nb, topk, INIT, LOCAL)+ m3_sparse_attn_decode(q, k_slab, v_slab, ti, block_rows, seq, SCALE, out)++ def check(lens, ks, vs, iks):+ for i, L in enumerate(lens):+ q_pos = torch.tensor([L - 1], device=DEV)+ ref_scores = ref_block_scores(iq[i : i + 1], iks[i][:L], q_pos)+ sel = ref_select(ref_scores, q_pos, topk, INIT, LOCAL)+ ref = ref_sparse_attend(q[i : i + 1], ks[i][:L], vs[i][:L], sel, q_pos, SCALE)+ err = (out[i].float() - ref[0]).abs().max().item()+ assert err < 3e-2, (lens, i, err)+ assert torch.isfinite(out[i].float()).all(), (lens, i)++ graph.replay()+ torch.cuda.synchronize()+ check(LENS, *data)+ for step, new_lens in enumerate(([129, 8191, 1], [8191, 1, 129])):+ data = refresh(200 + step)+ seq.copy_(torch.tensor(new_lens, dtype=torch.int32, device=DEV))+ q.normal_()+ iq.normal_()+ graph.replay()+ torch.cuda.synchronize()+ check(new_lens, *data)diff --git a/tests/kernels/test_mxfp8_linear.py b/tests/kernels/test_mxfp8_linear.pynew file mode 100644index 00000000..1f1557d4--- /dev/null+++ b/tests/kernels/test_mxfp8_linear.py@@ -0,0 +1,102 @@+"""MXFP8 (block-32 e8m0) W8A16 linear: kernels vs the dequant reference, plus the+Gemma (1+w) norm and swigluoai activation the MiniMax-M3 modules ride on."""++from __future__ import annotations++import pytest+import torch++if not torch.cuda.is_available(): # pragma: no cover+ pytest.skip("CUDA required", allow_module_level=True)++DEV = "cuda"+++def _make_mxfp8(N: int, K: int, seed: int = 0):+ torch.manual_seed(seed)+ wf = torch.randn(N, K, device=DEV) * 0.05+ codes = torch.randint(110, 132, (N, K // 32), device=DEV, dtype=torch.uint8)+ descale = torch.exp2(codes.float() - 127.0)+ w8 = (+ (wf.view(N, -1, 32) / descale.unsqueeze(-1)).clamp(-448, 448).view(N, K)+ ).to(torch.float8_e4m3fn)+ return w8, codes+++# 1 = m1 kernel; 2..256 = dot GEMV across its M_TILE buckets {16,32,64,128,256}+# (17/33/129 exercise row-padding masks); 257/300 = dequant+cuBLAS past+# _GEMV_MAX_M (257 pins the boundary).[email protected]("M", [1, 2, 8, 16, 17, 33, 64, 129, 256, 257, 300])+# (511, 6144): N not a BLOCK_N multiple (n-mask tail); (640, 6112): K a multiple+# of the 32-wide scale block but not of BLOCK_K=128 (k-mask + OOB scale codes).[email protected]("N,K", [(512, 6144), (9216, 6144), (640, 6144), (511, 6144), (640, 6112)])+def test_mxfp8_linear_matches_dequant_reference(M: int, N: int, K: int):+ from freetoken.kernel.triton.mxfp8_linear import mxfp8_dequant, mxfp8_linear++ w8, codes = _make_mxfp8(N, K, seed=M)+ ref_w = mxfp8_dequant(w8, codes, torch.float32)+ x = torch.randn(M, K, device=DEV, dtype=torch.bfloat16)+ y = mxfp8_linear(x, w8, codes)+ y_ref = (x.float() @ ref_w.t()).to(torch.bfloat16)+ rel = (y.float() - y_ref.float()).abs().max() / y_ref.float().abs().max().clamp(min=1e-6)+ assert rel.item() < 2e-2, rel.item()+++def test_mxfp8_module_shapes_and_forward():+ from freetoken.kernel.triton.mxfp8_linear import Mxfp8Linear++ lin = Mxfp8Linear(6144, 512)+ assert lin.weight.shape == (512, 6144) and lin.weight.dtype == torch.float8_e4m3fn+ assert lin.weight_scale_inv.shape == (512, 192) and lin.weight_scale_inv.dtype == torch.uint8+ w8, codes = _make_mxfp8(512, 6144, seed=42)+ lin.weight, lin.weight_scale_inv = w8, codes+ x = torch.randn(3, 6144, device=DEV, dtype=torch.bfloat16)+ assert lin.forward(x).shape == (3, 512)+++def test_gemma_plus_one_norm_matches_flashinfer_semantics():+ """Triton fallback vs the (1+w) definition; per-head 3D strided in-place."""+ from freetoken.kernel.triton.norm import gemma_fused_add_rmsnorm, gemma_rmsnorm++ torch.manual_seed(0)+ x = torch.randn(64, 6144, device=DEV, dtype=torch.bfloat16)+ w = torch.randn(6144, device=DEV, dtype=torch.bfloat16) * 0.1++ def ref(v):+ vf = v.float()+ inv = torch.rsqrt(vf.pow(2).mean(-1, keepdim=True) + 1e-6)+ return (vf * inv * (1.0 + w.float())).to(torch.bfloat16)++ out = gemma_rmsnorm(x, w, 1e-6)+ assert (out.float() - ref(x).float()).abs().max().item() < 2e-2++ a, r = x.clone(), x.clone()+ gemma_fused_add_rmsnorm(a, r, w, 1e-6)+ assert torch.equal(r, (x.float() + x.float()).to(torch.bfloat16))+ assert (a.float() - ref(r).float()).abs().max().item() < 2e-2++ # per-head strided in-place (the fused-qkv slice pattern)+ q = torch.randn(16, 9216, device=DEV, dtype=torch.bfloat16)+ wq = torch.randn(128, device=DEV, dtype=torch.bfloat16) * 0.1+ qh = q[:, :8192].view(16, 64, 128)+ ref3 = torch.empty_like(qh)+ for h in range(64):+ vf = qh[:, h].float()+ inv = torch.rsqrt(vf.pow(2).mean(-1, keepdim=True) + 1e-6)+ ref3[:, h] = (vf * inv * (1.0 + wq.float())).to(torch.bfloat16)+ gemma_rmsnorm(qh, wq, 1e-6, out=qh)+ assert (qh.float() - ref3.float()).abs().max().item() < 2e-2+++def test_swigluoai_and_mul_uninterleaved():+ from freetoken.layers import swigluoai_and_mul++ torch.manual_seed(1)+ d = 3072+ x = torch.randn(500, 2 * d, device=DEV, dtype=torch.bfloat16) * 3+ gate, up = x[:, :d].float(), x[:, d:].float()+ alpha, limit = 1.702, 7.0+ g = gate.clamp(max=limit)+ ref = g * torch.sigmoid(g * alpha) * (up.clamp(-limit, limit) + 1.0)+ out = swigluoai_and_mul(x, alpha=alpha, limit=limit)+ assert (out.float() - ref).abs().max().item() < 0.15diff --git a/tests/kvcache/test_bsa_pool.py b/tests/kvcache/test_bsa_pool.pynew file mode 100644index 00000000..73abe270--- /dev/null+++ b/tests/kvcache/test_bsa_pool.py@@ -0,0 +1,116 @@+"""BSAKVCache: paged GQA K/V + index-key slab (MiniMax-M3 block-sparse pool)."""++from __future__ import annotations++import pytest+import torch++if not torch.cuda.is_available(): # pragma: no cover+ pytest.skip("CUDA required", allow_module_level=True)++from freetoken.kvcache.bsa_pool import BSAKVCache++DEV = torch.device("cuda")++[email protected](autouse=True)+def _tp(monkeypatch):+ from freetoken.distributed.info import DistributedInfo++ monkeypatch.setattr(+ "freetoken.kvcache.mha_pool.get_tp_info",+ lambda: DistributedInfo(rank=0, size=1),+ )+++def _pool(num_pages=8, page_size=128):+ return BSAKVCache(+ num_kv_heads=4,+ num_layers=5,+ head_dim=128,+ num_pages=num_pages,+ page_size=page_size,+ dtype=torch.bfloat16,+ device=DEV,+ index_head_dim=128,+ num_index_layers=2,+ )+++def test_store_and_views():+ pool = _pool()+ assert pool.k_cache(0).shape == (8, 128, 4, 128)+ assert pool.index_k_cache(0).shape == (8 * 128, 128)+ assert pool.index_k_cache(1).shape == (8 * 128, 128)++ t = 6+ k = torch.randn(t, 4 * 128, device=DEV, dtype=torch.bfloat16)+ v = torch.randn(t, 4 * 128, device=DEV, dtype=torch.bfloat16)+ ik = torch.randn(t, 128, device=DEV, dtype=torch.bfloat16)+ out_loc = torch.tensor([128, 129, 130, 200, 201, 202], device=DEV, dtype=torch.int64)+ pool.store_kv(k, v, out_loc.to(torch.int32), layer_id=3)+ pool.store_index_k(ik, out_loc, slot=1)++ k_rows = pool.k_cache(3).view(-1, 4, 128)+ assert torch.equal(k_rows[128], k[0].view(4, 128))+ assert torch.equal(k_rows[202], k[5].view(4, 128))+ assert torch.equal(pool.index_k_cache(1)[200], ik[3])+ # the other slot stays untouched (zero-initialized)+ assert pool.index_k_cache(0)[200].abs().sum().item() == 0.0+++def test_rebuild_resizes_both_slabs_atomically():+ pool = _pool(num_pages=8)+ pool.rebuild(16)+ assert pool.k_cache(0).shape == (16, 128, 4, 128)+ assert pool.index_k_cache(0).shape == (16 * 128, 128)+ assert pool.index_k_cache(1).shape == (16 * 128, 128)+++def test_unit_bytes_matches_cost_model():+ pool = _pool()+ kv_bytes, swa_bytes = pool.unit_bytes()+ # 5 layers x 2 slabs x 4 heads x 128 x 2 B + 2 index layers x 128 x 2 B+ assert kv_bytes == 5 * 2 * 4 * 128 * 2 + 2 * 128 * 2+ assert swa_bytes == 0+++def test_factory_builds_bsa_pool():+ from freetoken.kvcache import create_kvcache_pool+ from freetoken.models.config import FullAttentionGroupConfig, ModelConfig, RotaryConfig++ rotary = RotaryConfig(head_dim=128, rotary_dim=64, max_position=4096, base=1e4, scaling=None)+ cfg = ModelConfig(+ num_layers=5,+ num_qo_heads=64,+ num_kv_heads=4,+ head_dim=128,+ hidden_size=6144,+ vocab_size=1000,+ intermediate_size=1024,+ rms_norm_eps=1e-6,+ rotary_config=rotary,+ hidden_act="swigluoai",+ tie_word_embeddings=False,+ num_experts=8,+ num_experts_per_tok=2,+ moe_intermediate_size=512,+ norm_topk_prob=True,+ model_type="minimax_m3_vl",+ architectures=["MiniMaxM3SparseForConditionalGeneration"],+ attention_groups=(+ FullAttentionGroupConfig(+ name="full",+ layer_ids=tuple(range(5)),+ num_kv_heads=4,+ head_dim=128,+ rotary_config=rotary,+ mla=False,+ index_head_dim=128,+ num_index_layers=2,+ ),+ ),+ )+ pool = create_kvcache_pool(cfg, num_pages=4, page_size=128, dtype=torch.bfloat16, device=DEV)+ assert isinstance(pool, BSAKVCache)+ assert pool.index_k_cache(0).shape == (4 * 128, 128)diff --git a/tests/models/test_minimax_m3.py b/tests/models/test_minimax_m3.pynew file mode 100644index 00000000..787661dc--- /dev/null+++ b/tests/models/test_minimax_m3.py@@ -0,0 +1,263 @@+"""MiniMax-M3 config parsing, payload semantics and serving-mode resolution.++Drives ``minimax_m3.parse_config`` off a synthetic HF config shaped exactly like the+nvidia/MiniMax-M3-NVFP4 checkpoint's (multimodal wrapper: text tower in+``text_config``, quantization_config present) and checks every decision the plan+pinned: the BSA attention group (pool family + KV cost), the sparse indexer+geometry, the MoE/router knobs, the resolved quant modes and the env switches.+"""++from __future__ import annotations++import pytest+import torch++from freetoken.attention.base import AttnType+from freetoken.models.minimax_m3.config import parse_config+++class _Cfg:+ """Attribute-access shim over a dict (what AutoConfig hands parse_config)."""++ def __init__(self, data: dict):+ for k, v in data.items():+ setattr(self, k, _Cfg(v) if isinstance(v, dict) and k == "text_config" else v)+++def _hf_config(num_layers: int = 60) -> _Cfg:+ dense = min(3, num_layers)+ moe_freq = [0] * dense + [1] * (num_layers - dense)+ return _Cfg(+ {+ "architectures": ["MiniMaxM3SparseForConditionalGeneration"],+ "model_type": "minimax_m3_vl",+ "quantization_config": {"quant_algo": "MIXED_PRECISION", "quant_method": "modelopt"},+ "text_config": {+ "hidden_size": 6144,+ "intermediate_size": 3072,+ "dense_intermediate_size": 12288,+ "shared_intermediate_size": 3072,+ "num_hidden_layers": num_layers,+ "num_attention_heads": 64,+ "num_key_value_heads": 4,+ "head_dim": 128,+ "vocab_size": 200064,+ "max_position_embeddings": 1048576,+ "rms_norm_eps": 1e-6,+ "use_gemma_norm": True,+ "attention_output_gate": False,+ "rope_theta": 5000000,+ "rotary_dim": 64,+ "partial_rotary_factor": 0.5,+ "hidden_act": "swigluoai",+ "swiglu_alpha": 1.702,+ "swiglu_limit": 7.0,+ "use_qk_norm": True,+ "qk_norm_type": "per_head",+ "num_local_experts": 128,+ "num_experts_per_tok": 4,+ "n_shared_experts": 1,+ "scoring_func": "sigmoid",+ "use_routing_bias": True,+ "routed_scaling_factor": 2.0,+ "moe_layer_freq": moe_freq,+ "tie_word_embeddings": False,+ "sparse_attention_config": {+ "use_sparse_attention": True,+ "sparse_index_dim": 128,+ "sparse_num_index_heads": 4,+ "sparse_topk_blocks": 16,+ "sparse_block_size": 128,+ "sparse_disable_index_value": moe_freq,+ "sparse_score_type": "max",+ "sparse_init_block": 0,+ "sparse_local_block": 1,+ "sparse_attention_freq": moe_freq,+ },+ },+ }+ )+++def _hf_config_native(num_layers: int = 60) -> _Cfg:+ """The transformers >= 5 NATIVE minimax_m3_vl TextConfig shape: __post_init__+ pops rope_theta into rope_parameters, sparse_attention_config/moe_layer_freq+ into flat index_* keys + layer_types/mlp_layer_types, and force-sets+ hidden_act to "silu". parse_config must resolve both shapes identically+ (verified empirically against transformers 5.14's class)."""+ cfg = _hf_config(num_layers)+ text = cfg.text_config+ dense = min(3, num_layers)+ delattr(text, "rope_theta")+ text.rope_parameters = {+ "rope_theta": 5000000,+ "partial_rotary_factor": 0.5,+ "rope_type": "default",+ }+ text.hidden_act = "silu"+ delattr(text, "sparse_attention_config")+ text.layer_types = ["full_attention"] * dense + ["minimax_m3_sparse"] * (+ num_layers - dense+ )+ text.index_n_heads = 4+ text.index_head_dim = 128+ text.index_block_size = 128+ text.index_topk_blocks = 16+ text.index_local_blocks = 1+ delattr(text, "moe_layer_freq")+ text.mlp_layer_types = ["dense"] * dense + ["sparse"] * (num_layers - dense)+ return cfg+++def test_parse_config_native_shape_matches_raw(monkeypatch):+ """The native shape silently produced rope base 10000, use_sparse=False and+ hidden_act='silu' before the dual-shape shim -- all three load-bearing reads+ must resolve identically to the raw-dict shape."""+ monkeypatch.delenv("FREETOKEN_M3_MAX_LAYERS", raising=False)+ monkeypatch.delenv("FREETOKEN_M3_SPARSE", raising=False)+ raw = parse_config(_hf_config())+ native = parse_config(_hf_config_native())++ assert native.rotary_config.base == raw.rotary_config.base == 5000000.0+ assert native.hidden_act == raw.hidden_act == "swigluoai"+ assert native.first_k_dense_replace == raw.first_k_dense_replace == 3+ a, b = native.m3_args, raw.m3_args+ assert a.use_sparse and a.sparse_layer_ids == b.sparse_layer_ids+ assert a.moe_layer_ids == b.moe_layer_ids+ assert (a.index_dim, a.num_index_heads, a.topk_blocks, a.block_size) == (+ b.index_dim, b.num_index_heads, b.topk_blocks, b.block_size,+ )+ assert (a.init_blocks, a.local_blocks) == (b.init_blocks, b.local_blocks)+ assert (a.rope_theta, a.rotary_dim) == (b.rope_theta, b.rotary_dim)+++def test_parse_config_full_model(monkeypatch):+ monkeypatch.delenv("FREETOKEN_M3_MAX_LAYERS", raising=False)+ monkeypatch.delenv("FREETOKEN_M3_SPARSE", raising=False)+ cfg = parse_config(_hf_config())++ assert cfg.num_layers == 60+ assert (cfg.num_qo_heads, cfg.num_kv_heads, cfg.head_dim) == (64, 4, 128)+ assert cfg.rotary_config.rotary_dim == 64 and cfg.rotary_config.base == 5000000.0+ assert cfg.first_k_dense_replace == 3 and cfg.num_moe_layers == 57+ assert (cfg.num_experts, cfg.num_experts_per_tok) == (128, 4)+ assert cfg.moe_intermediate_size == 3072 and cfg.intermediate_size == 12288+ assert cfg.n_shared_experts == 1 and cfg.shared_expert_intermediate_size == 3072+ assert cfg.routed_scaling_factor == 2.0 and cfg.norm_topk_prob+ assert cfg.hidden_act == "swigluoai"+ assert cfg.hidden_act_alpha == pytest.approx(1.702)+ assert cfg.swiglu_limit == pytest.approx(7.0)+ assert cfg.expert_quant == "nvfp4"+ # Default env: MXFP8 dense native, bf16 lm_head.+ assert cfg.attn_quant == "mxfp8" and cfg.dense_quant == "mxfp8"+ assert cfg.lm_head_quant == "none"++ # One BSA group over all layers, index slab for the 57 sparse layers.+ (spec,) = cfg.kv_cache_group_specs()+ assert spec.attn_type == AttnType.BSA+ assert not spec.mla+ assert spec.index_head_dim == 128 and spec.num_index_layers == 57++ args = cfg.m3_args+ assert args.use_sparse and args.topk_blocks == 16 and args.block_size == 128+ assert (args.init_blocks, args.local_blocks) == (0, 1)+ assert args.num_index_heads == 4 and args.index_dim == 128+ assert args.sparse_layer_ids == tuple(range(3, 60))+ assert args.moe_layer_ids == tuple(range(3, 60))+ assert args.sparse_slot(3) == 0 and args.sparse_slot(59) == 56+++def test_pool_family_and_kv_cost():+ cfg = parse_config(_hf_config())+ from freetoken.kvcache import resolve_pool_class+ from freetoken.kvcache.base import spec_kv_bytes_per_token+ from freetoken.kvcache.bsa_pool import BSAKVCache++ assert resolve_pool_class(cfg) is BSAKVCache++ class _Budget:+ model_config = cfg+ dtype = torch.bfloat16+ page_size = 128++ class tp_info:+ size = 1++ (spec,) = cfg.kv_cache_group_specs()+ # 60 layers x 2 slabs x 4 heads x 128 dims x 2 B + 57 index layers x 128 x 2 B+ assert spec_kv_bytes_per_token(spec, _Budget) == 60 * 2 * 4 * 128 * 2 + 57 * 128 * 2+++def test_layer_cap_env(monkeypatch):+ monkeypatch.setenv("FREETOKEN_M3_MAX_LAYERS", "5")+ cfg = parse_config(_hf_config())+ assert cfg.num_layers == 5+ assert cfg.first_k_dense_replace == 3 and cfg.num_moe_layers == 2+ (spec,) = cfg.kv_cache_group_specs()+ assert spec.num_index_layers == 2+ assert cfg.m3_args.sparse_layer_ids[:2] == (3, 4)+++def test_sparse_ablation_env(monkeypatch):+ monkeypatch.setenv("FREETOKEN_M3_SPARSE", "0")+ cfg = parse_config(_hf_config())+ (spec,) = cfg.kv_cache_group_specs()+ assert spec.attn_type == AttnType.FULL+ assert spec.index_head_dim == 0 and spec.num_index_layers == 0+ assert not cfg.m3_args.use_sparse++ from freetoken.kvcache import resolve_pool_class+ from freetoken.kvcache.mha_pool import MHAKVCache++ assert resolve_pool_class(cfg) is MHAKVCache+++def test_mxfp8_ablation_env(monkeypatch):+ monkeypatch.setenv("FREETOKEN_M3_ATTN_MXFP8", "0")+ monkeypatch.setenv("FREETOKEN_M3_MLP_MXFP8", "0")+ cfg = parse_config(_hf_config())+ assert cfg.attn_quant == "none" and cfg.dense_quant == "none"+++def test_registry_resolves_both_architectures():+ from freetoken.models.register import get_model_spec++ for arch in ("MiniMaxM3SparseForConditionalGeneration", "MiniMaxM3SparseForCausalLM"):+ spec = get_model_spec(arch)+ assert spec.module == "freetoken.models.minimax_m3"+++def test_auto_backend_resolution():+ from freetoken.attention import attention_backend_info+ from freetoken.engine.engine import _required_attn_types, _resolve_auto_attention_backend++ cfg = parse_config(_hf_config())+ required = _required_attn_types(cfg)+ assert required == frozenset({AttnType.BSA})+ assert _resolve_auto_attention_backend(required, False) == "m3_sparse"+ assert attention_backend_info("m3_sparse").page_sizes == (128,)+++def test_nvfp4_backend_restricted_to_triton_for_swigluoai():+ from freetoken.moe.nvfp4_backends import select_nvfp4_backend++ dev = torch.device("cuda" if torch.cuda.is_available() else "cpu")+ assert select_nvfp4_backend(dev, 3072, "auto", activation="swigluoai") == "triton"+ with pytest.raises(RuntimeError):+ select_nvfp4_backend(dev, 3072, "marlin", activation="swigluoai")+++def test_expert_source_spec_layer_to_bank():+ from freetoken.models.minimax_m3.weight import _EXPERT_KEY_RE, _NVFP4_SOURCE_SPEC++ cfg = parse_config(_hf_config())+ m = _EXPERT_KEY_RE.match(+ "language_model.model.layers.7.block_sparse_moe.experts.42.w1.weight_scale_2"+ )+ assert m and m.group("layer") == "7" and m.group("expert") == "42"+ assert m.group("proj") == "w1" and m.group("kind") == "weight_scale_2"+ assert _NVFP4_SOURCE_SPEC.proj_to_role == {"w1": "gate", "w3": "up", "w2": "down"}+ # Dense prefix leaves no bank holes; MoE layer i maps to bank i-3.+ assert _NVFP4_SOURCE_SPEC.layer_to_bank(3, cfg) == 0+ assert _NVFP4_SOURCE_SPEC.layer_to_bank(59, cfg) == 56+ assert _NVFP4_SOURCE_SPEC.layer_to_bank(0, cfg) is Nonediff --git a/tests/moe/test_cpu_moe.py b/tests/moe/test_cpu_moe.pyindex ccc6b7a2..f496de31 100644--- a/tests/moe/test_cpu_moe.py+++ b/tests/moe/test_cpu_moe.py@@ -164,6 +164,94 @@ def test_cpu_decode_nvfp4_matches_dequant_then_gpu(bs):assert rel < 3e-2, f"nvfp4 bs={bs} rel err {rel.item()}"[email protected]("bs", [1, 4])+def test_cpu_decode_nvfp4_swigluoai_matches_dequant_reference(bs):+ """MiniMax-M3's swigluoai routed experts on the CPU executor (ActKind 3 through+ the generic NVFP4 GEMV epilogue, with the layer's alpha/limit scalars) vs. a+ pure-torch dequant reference."""+ from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4+ from freetoken.moe.cpu_executor import CpuMoeExecutor++ torch.manual_seed(300 + bs)+ L, E, H, I, top_k = 2, 16, 1024, 512, 4+ layer = 1+ alpha, limit = 1.702, 7.0+ dev = torch.device("cuda")+ cache = _make_nvfp4_cache(L, E, H, I, seed=17)++ ex = CpuMoeExecutor(+ cache,+ top_k=top_k,+ activation="swigluoai",+ apply_router_weight_on_input=False,+ num_threads=0,+ max_tokens=bs,+ device=dev,+ swiglu_alpha=alpha,+ swiglu_limit=limit,+ )++ hidden = torch.randn(bs, H, device=dev, dtype=torch.bfloat16)+ ids = torch.stack([torch.randperm(E, device=dev)[:top_k] for _ in range(bs)]).to(torch.int32)+ w = torch.rand(bs, top_k, device=dev, dtype=torch.float32)++ cpu_out = ex.decode(layer, hidden, w, ids).float()+ torch.cuda.synchronize()++ b = cache.bank_sources+ slots = torch.arange(E, device=dev, dtype=torch.int32)+ gate_up_layer = dequant_nvfp4(+ b["gate_up_packed"][layer].to(dev), b["gate_up_scale"][layer].to(dev),+ b["gate_up_global"][layer].to(dev), slots, dtype=torch.bfloat16,+ ).float() # [E, 2I, H]+ down_layer = dequant_nvfp4(+ b["down_packed"][layer].to(dev), b["down_scale"][layer].to(dev),+ b["down_global"][layer].to(dev), slots, dtype=torch.bfloat16,+ ).float() # [E, H, I]++ ref = torch.zeros(bs, H, device=dev, dtype=torch.float32)+ x = hidden.float()+ for t in range(bs):+ for k in range(top_k):+ e = int(ids[t, k])+ h = gate_up_layer[e] @ x[t]+ gate = h[:I].clamp(max=limit)+ up = h[I:].clamp(-limit, limit)+ act = gate * torch.sigmoid(gate * alpha) * (up + 1.0)+ ref[t] += float(w[t, k]) * (down_layer[e] @ act)++ rel = (cpu_out - ref).abs().max() / (ref.abs().max() + 1e-6)+ assert rel < 3e-2, f"nvfp4 swigluoai bs={bs} rel err {rel.item()}"+++def test_stale_extension_rejected_for_swigluoai(monkeypatch):+ """A prebuilt _cpu_moe.so from before ACT_SWIGLUOAI accepts act id 3 without+ error and silently computes the wrong activation in the generic epilogue.+ The executor probes the extension's `max_generic_act_id` marker -- absent on+ stale builds -- and must fail loudly with the rebuild instruction."""+ from freetoken.kernel import _cpu_moe+ from freetoken.moe.cpu_executor import CpuMoeExecutor, compiled_extension_supports++ # raising=False: on a GENUINELY stale extension the attribute is already+ # absent, and this test must still run (it is the test for that case).+ monkeypatch.delattr(_cpu_moe, "max_generic_act_id", raising=False)+ assert not compiled_extension_supports("swigluoai")+ assert compiled_extension_supports("silu") # silu family predates the marker+ cache = _make_nvfp4_cache(1, 4, 256, 128)+ with pytest.raises(RuntimeError, match="rebuild"):+ CpuMoeExecutor(+ cache, top_k=2, activation="swigluoai",+ apply_router_weight_on_input=False, num_threads=0, max_tokens=1,+ device=torch.device("cuda"), swiglu_alpha=1.702, swiglu_limit=7.0,+ )+ # silu predates the marker and must keep working on a stale build.+ CpuMoeExecutor(+ cache, top_k=2, activation="silu",+ apply_router_weight_on_input=False, num_threads=0, max_tokens=1,+ device=torch.device("cuda"),+ )++def _pack_mxfp4_transposed(codes: torch.Tensor) -> torch.Tensor:"""Pack [S, K, N] codes -> transposed blocks [S, K//2, N] (low nibble = even K)."""lo = codes[:, 0::2, :]diff --git a/tests/moe/test_nvfp4_backends.py b/tests/moe/test_nvfp4_backends.pyindex 5541138f..7005a1b4 100644--- a/tests/moe/test_nvfp4_backends.py+++ b/tests/moe/test_nvfp4_backends.py@@ -84,8 +84,15 @@ def _assert_close(out: torch.Tensor, ref: torch.Tensor) -> None:torch.testing.assert_close(out.float(), ref, rtol=3e-2, atol=tol)-def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids) -> torch.Tensor:- """Dequant + dense per-token reference for the gated-silu MoE."""+def _swigluoai_ref(h: torch.Tensor, alpha: float = 1.702, limit: float = 7.0) -> torch.Tensor:+ """MiniMax-M3 / gpt-oss clamped swiglu over UNINTERLEAVED [gate; up] halves."""+ gate = h[:I].clamp(max=limit)+ up = h[I:].clamp(-limit, limit)+ return gate * torch.sigmoid(gate * alpha) * (up + 1.0)+++def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids, activation="silu") -> torch.Tensor:+ """Dequant + dense per-token reference for the gated MoE (silu or swigluoai)."""out = torch.zeros(hidden.shape, dtype=torch.float32, device=hidden.device)x = hidden.float()for t in range(hidden.size(0)):@@ -102,7 +109,10 @@ def _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids) -> torch.Tensor:sources["down_global"][layer_id][e].to(hidden.device),)h = gu @ x[t]- act = torch.nn.functional.silu(h[:I]) * h[I:]+ if activation == "swigluoai":+ act = _swigluoai_ref(h)+ else:+ act = torch.nn.functional.silu(h[:I]) * h[I:]out[t] += float(topk_weights[t, j]) * (dn @ act)return out@@ -287,6 +297,47 @@ def test_triton_overlap_prefill_matches_dequant_reference():cache.release_prefill_layer(layer_id)+@cuda+def test_triton_swigluoai_matches_dequant_reference():+ """MiniMax-M3's swigluoai routed experts through the Triton prefill grouped GEMM+ and the marlin-style decode GEMV: same banks, the clamped (up+1) swiglu instead+ of silu, alpha/limit threaded through the fused entry points."""+ from freetoken.moe.fused_nvfp4 import (+ fused_experts_decode_nvfp4_marlin,+ fused_experts_nvfp4,+ )++ device = torch.device("cuda")+ sources = _make_native_sources(device, seed=11)+ torch.manual_seed(12)+ M = 8+ hidden = torch.randn(M, H, dtype=torch.bfloat16, device=device) / 4+ topk_ids = torch.randint(0, E, (M, TOPK), dtype=torch.int32, device=device)+ topk_weights = torch.rand(M, TOPK, dtype=torch.float32, device=device)+ layer_id = 0+ banks = [+ sources[name][layer_id].to(device)+ for name in (+ "gate_up_packed", "gate_up_scale", "gate_up_global",+ "down_packed", "down_scale", "down_global",+ )+ ]+ ref = _ref_moe(sources, layer_id, hidden, topk_weights, topk_ids, activation="swigluoai")+ out = fused_experts_nvfp4(+ hidden, *banks, topk_weights, topk_ids, E, "swigluoai", False, 1.702, 7.0+ )+ _assert_close(out, ref)++ dec_hidden = hidden[:1]+ dec_ids = topk_ids[:1]+ dec_weights = topk_weights[:1]+ ref = _ref_moe(sources, layer_id, dec_hidden, dec_weights, dec_ids, activation="swigluoai")+ out = fused_experts_decode_nvfp4_marlin(+ dec_hidden, *banks, dec_weights, dec_ids, "swigluoai", False, 1.702, 7.0+ )+ _assert_close(out, ref)++def _triton_cache(device, *, cache_size=S, prefill_overlap=False):"""Native 6-bank NVFP4 cache (no repack), consumed directly by the Triton kernels.The banks are not transformed, so ``sources`` doubles as the dequant reference."""diff --git a/tests/server/test_anthropic_api.py b/tests/server/test_anthropic_api.pyindex 2478b17a..d8eb0169 100644--- a/tests/server/test_anthropic_api.py+++ b/tests/server/test_anthropic_api.py@@ -158,6 +158,33 @@ def test_convert_system_role_message_and_unknown_block():assert spec.messages[2]["content"] == "hi"+def test_convert_thinking_toggle_routes_through_family_mapping():+ """A hardcoded {"enable_thinking": bool} is inert for M3, whose template+ reads thinking_mode -- the toggle must go through the same model_meta+ mapping as the chat-completions gears."""+ def _req(ttype):+ return AnthropicMessagesRequest.model_validate(+ {+ "model": "m", "max_tokens": 16,+ "messages": [{"role": "user", "content": "hi"}],+ "thinking": {"type": ttype, "budget_tokens": 1024},+ }+ )++ spec = A.convert_anthropic_to_genspec(_req("enabled"), {}, reasoning_parser="minimax_m3")+ assert spec.chat_template_kwargs == {"thinking_mode": "enabled"}+ spec = A.convert_anthropic_to_genspec(_req("disabled"), {}, reasoning_parser="minimax_m3")+ assert spec.chat_template_kwargs == {"thinking_mode": "disabled"}+ # enable_thinking families keep their key; no parser keeps the generic key+ spec = A.convert_anthropic_to_genspec(_req("enabled"), {}, reasoning_parser="qwen3")+ assert spec.chat_template_kwargs == {"enable_thinking": True}+ spec = A.convert_anthropic_to_genspec(_req("disabled"), {})+ assert spec.chat_template_kwargs == {"enable_thinking": False}+ # a family without the requested direction sets nothing (minimax has no off)+ spec = A.convert_anthropic_to_genspec(_req("disabled"), {}, reasoning_parser="minimax")+ assert spec.chat_template_kwargs == {}++def test_convert_thinking_only_assistant_message_keeps_empty_content():# Truncated-then-resent turn: survives with content="" for templates that# concatenate message.content unconditionally.diff --git a/tests/server/test_minimax_m3_parsers.py b/tests/server/test_minimax_m3_parsers.pynew file mode 100644index 00000000..e7980903--- /dev/null+++ b/tests/server/test_minimax_m3_parsers.py@@ -0,0 +1,597 @@+"""MiniMax-M3 reasoning parser (<mm:think>, 3 thinking modes) and tool-call+detector (namespace-delimited recursive XML) -- one-shot and streaming."""++from __future__ import annotations++import json++import pytest++from freetoken.server.function_call_parser import (+ Function,+ MiniMaxM3Detector,+ Tool,+)+from freetoken.server.reasoning_parser import MiniMaxM3ReasoningParser, ReasoningParser++NS = "]<]minimax[>["+++# ---------------------------------------------------------------------------+# Reasoning parser+# ---------------------------------------------------------------------------+def test_reasoning_registered():+ assert ReasoningParser.ReasoningParserEnum["minimax_m3"] is MiniMaxM3ReasoningParser+++def test_reasoning_adaptive_with_tags():+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ r = p.detect_and_parse("<mm:think>chain of thought</mm:think>The answer is 4.")+ assert r.reasoning_text == "chain of thought"+ assert r.normal_text == "The answer is 4."+++def test_reasoning_adaptive_without_tags():+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ r = p.detect_and_parse("Plain answer, the model chose not to think.")+ assert r.reasoning_text == ""+ assert r.normal_text == "Plain answer, the model chose not to think."+++def test_reasoning_enabled_implicit_open():+ # thinking_mode=enabled: the template pre-opens <mm:think>, the model emits+ # only the closing tag -> the caller constructs with force_reasoning=True.+ p = MiniMaxM3ReasoningParser(force_reasoning=True)+ r = p.detect_and_parse("thinking hard</mm:think>final answer")+ assert r.reasoning_text == "thinking hard"+ assert r.normal_text == "final answer"+++def test_reasoning_streaming_split_marker():+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ reasoning, content = [], []+ for chunk in ["<mm:th", "ink>let me ", "think</mm:t", "hink>done: 42"]:+ r = p.parse_streaming_increment(chunk)+ reasoning.append(r.reasoning_text)+ content.append(r.normal_text)+ r = p.flush()+ reasoning.append(r.reasoning_text)+ content.append(r.normal_text)+ assert "".join(reasoning) == "let me think"+ assert "".join(content) == "done: 42"+++def test_reasoning_tool_block_ends_missing_close():+ # A malformed turn skipping </mm:think> straight into a tool block: the block+ # must land in normal_text for the tool parser (dsv4 precedent).+ p = MiniMaxM3ReasoningParser(force_reasoning=True)+ text = f"half a thought{NS}<tool_call>...{NS}</tool_call>"+ r = p.detect_and_parse(text)+ assert r.reasoning_text == "half a thought"+ assert r.normal_text.startswith(f"{NS}<tool_call>")+++# ---------------------------------------------------------------------------+# Tool-call detector+# ---------------------------------------------------------------------------+def _tools():+ return [+ Tool(+ function=Function(+ name="create_order",+ parameters={+ "type": "object",+ "properties": {+ "user_id": {"type": "integer"},+ "note": {"type": "string"},+ "shipping": {"type": "object"},+ "items": {"type": "array"},+ },+ },+ )+ ),+ Tool(function=Function(name="get_weather", parameters={+ "type": "object", "properties": {"city": {"type": "string"}},+ })),+ ]+++def _block(inner: str) -> str:+ return f"{NS}<tool_call>\n{inner}{NS}</tool_call>"+++_NESTED_INVOKE = (+ f'{NS}<invoke name="create_order">'+ f"{NS}<user_id>42{NS}</user_id>"+ f"{NS}<note>two books{NS}</note>"+ f"{NS}<shipping>"+ f"{NS}<city>Singapore{NS}</city>"+ f"{NS}<zip>018956{NS}</zip>"+ f"{NS}</shipping>"+ f"{NS}<items>"+ f"{NS}<item>"+ f"{NS}<sku>book-001{NS}</sku>"+ f"{NS}<qty>2{NS}</qty>"+ f"{NS}</item>"+ f"{NS}</items>"+ f"{NS}</invoke>\n"+)+++def test_detect_and_parse_recursive_args():+ det = MiniMaxM3Detector()+ text = "I'll place the order." + _block(_NESTED_INVOKE)+ assert det.has_tool_call(text)+ res = det.detect_and_parse(text, _tools())+ assert res.normal_text == "I'll place the order."+ assert len(res.calls) == 1+ call = res.calls[0]+ assert call.name == "create_order"+ args = json.loads(call.parameters)+ assert args["user_id"] == 42 # schema-typed integer+ assert args["note"] == "two books" # schema-typed string+ # nested leaves are loose-JSON typed: numbers parse, but a leading-zero token+ # like "018956" is not valid JSON and stays a string (never silently mangled)+ assert args["shipping"] == {"city": "Singapore", "zip": "018956"}+ assert args["items"] == [{"sku": "book-001", "qty": 2}]+++def test_detect_and_parse_multiple_invokes():+ det = MiniMaxM3Detector()+ two = (+ _NESTED_INVOKE+ + f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n'+ )+ res = det.detect_and_parse(_block(two), _tools())+ assert [c.name for c in res.calls] == ["create_order", "get_weather"]+ assert json.loads(res.calls[1].parameters) == {"city": "Paris"}+++def test_detect_and_parse_unknown_tool_follows_forwarding_policy():+ # FreeToken forwards unknown tool names by default (FORWARD_UNKNOWN_TOOLS);+ # the detector must honor the same policy switch as every other family.+ import freetoken.server.function_call_parser as fcp++ det = MiniMaxM3Detector()+ text = _block(f'{NS}<invoke name="nope">{NS}<a>1{NS}</a>{NS}</invoke>\n')+ res = det.detect_and_parse(text, _tools())+ assert [c.name for c in res.calls] == ["nope"] # default: forwarded++ orig = fcp.FORWARD_UNKNOWN_TOOLS+ try:+ fcp.FORWARD_UNKNOWN_TOOLS = False+ res = MiniMaxM3Detector().detect_and_parse(text, _tools())+ assert res.calls == [] # strict mode: dropped with a warning+ finally:+ fcp.FORWARD_UNKNOWN_TOOLS = orig+++def test_quoted_invoke_opener_is_data_not_call():+ """A parameter value quoting the invoke wire syntax must parse as data --+ a raw regex over the block used to spawn a phantom call and diverge from+ streaming."""+ quoted = f'see {NS}<invoke name="write_file"> for the syntax'+ text = _block(+ f'{NS}<invoke name="get_weather">{NS}<city>{quoted}{NS}</city>{NS}</invoke>\n'+ )+ res = MiniMaxM3Detector().detect_and_parse(text, _tools())+ assert [c.name for c in res.calls] == ["get_weather"]+ assert json.loads(res.calls[0].parameters) == {"city": quoted}+ # streaming agrees (it already produced one call before the fix; pin it)+ det = MiniMaxM3Detector()+ r = det.parse_streaming_increment(text, _tools())+ assert [c.name for c in r.calls if c.name is not None] == ["get_weather"]+++def test_quoted_wrapper_closer_is_data_not_block_end():+ """A value quoting NS</tool_call> must not end the wrapper: before the fix+ the parameter was dropped ({}) and the rest of the block leaked as raw+ markup content."""+ quoted = f"wrap calls in {NS}</tool_call> at the end"+ inner = f'{NS}<invoke name="get_weather">{NS}<city>{quoted}{NS}</city>{NS}</invoke>\n'+ text = _block(inner) + " done."+ res = MiniMaxM3Detector().detect_and_parse(text, _tools())+ assert [c.name for c in res.calls] == ["get_weather"]+ assert json.loads(res.calls[0].parameters) == {"city": quoted}+ assert res.normal_text == "done."+ # chunked streaming: same call, no markup leak+ det = MiniMaxM3Detector()+ calls, texts = [], []+ for i in range(0, len(text), 7):+ r = det.parse_streaming_increment(text[i : i + 7], _tools())+ calls += [c for c in r.calls if c.name is not None]+ texts.append(r.normal_text)+ assert [c.name for c in calls] == ["get_weather"]+ assert NS not in "".join(texts)+++def test_multiline_string_argument_verbatim():+ """The template renders leaf values verbatim; the round-trip must preserve+ them verbatim too (stripping ate the trailing newline of multi-line string+ arguments)."""+ tools = [+ Tool(function=Function(name="write_file", parameters={+ "type": "object",+ "properties": {"path": {"type": "string"}, "content": {"type": "string"}},+ }))+ ]+ content = "line1\nline2\n"+ text = _block(+ f'{NS}<invoke name="write_file">'+ f"{NS}<path>/tmp/x{NS}</path>"+ f"{NS}<content>{content}{NS}</content>"+ f"{NS}</invoke>\n"+ )+ res = MiniMaxM3Detector().detect_and_parse(text, tools)+ assert json.loads(res.calls[0].parameters) == {"path": "/tmp/x", "content": content}+++def test_nested_schema_types_string_leaves():+ """Nested leaves honor the declared schema (string-typed fields stay+ strings instead of loose-JSON numbers/bools); undeclared nested leaves keep+ the loose-JSON fallback."""+ tools = [+ Tool(function=Function(name="create_order", parameters={+ "type": "object",+ "properties": {+ "shipping": {+ "type": "object",+ "properties": {+ "zip": {"type": "string"},+ "note": {"type": "string"},+ "floor": {"type": "integer"},+ },+ },+ "tags": {"type": "array", "items": {"type": "string"}},+ },+ }))+ ]+ text = _block(+ f'{NS}<invoke name="create_order">'+ f"{NS}<shipping>"+ f"{NS}<zip>94107{NS}</zip>"+ f"{NS}<note>true{NS}</note>"+ f"{NS}<floor>3{NS}</floor>"+ f"{NS}</shipping>"+ f"{NS}<tags>"+ f"{NS}<item>1{NS}</item>"+ f"{NS}<item>two{NS}</item>"+ f"{NS}</tags>"+ f"{NS}</invoke>\n"+ )+ args = json.loads(MiniMaxM3Detector().detect_and_parse(text, tools).calls[0].parameters)+ assert args["shipping"] == {"zip": "94107", "note": "true", "floor": 3}+ assert args["tags"] == ["1", "two"]+++def test_streaming_text_then_call():+ det = MiniMaxM3Detector()+ full = "Let me check." + _block(+ f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n'+ )+ # Chop into small chunks that split the namespace marker mid-token.+ chunks = [full[i : i + 7] for i in range(0, len(full), 7)]+ normal, calls = "", []+ for ch in chunks:+ r = det.parse_streaming_increment(ch, _tools())+ normal += r.normal_text+ calls.extend(r.calls)+ assert normal == "Let me check."+ named = [c for c in calls if c.name]+ frags = [c for c in calls if c.name is None and c.parameters]+ assert [c.name for c in named] == ["get_weather"]+ assert json.loads("".join(c.parameters for c in frags)) == {"city": "Paris"}+ # ledgers the serving layer reads at stream end+ assert det.prev_tool_call_arr[0]["name"] == "get_weather"+ assert det.prev_tool_call_arr[0]["arguments"] == {"city": "Paris"}+++def test_streaming_multiple_invokes_and_indices():+ det = MiniMaxM3Detector()+ two = (+ _NESTED_INVOKE+ + f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n'+ )+ full = _block(two)+ chunks = [full[i : i + 13] for i in range(0, len(full), 13)]+ calls = []+ for ch in chunks:+ calls.extend(det.parse_streaming_increment(ch, _tools()).calls)+ named = [(c.tool_index, c.name) for c in calls if c.name]+ assert named == [(0, "create_order"), (1, "get_weather")]+++def test_streaming_plain_text_never_held():+ det = MiniMaxM3Detector()+ r = det.parse_streaming_increment("Just a normal answer, no tools.", _tools())+ assert r.normal_text == "Just a normal answer, no tools."+ assert r.calls == []+++def test_auto_selection_picks_minimax_m3():+ from unittest.mock import patch++ from freetoken.server.args import parse_args++ class _Config:+ def to_dict(self):+ return {+ "architectures": ["MiniMaxM3SparseForConditionalGeneration"],+ "model_type": "minimax_m3_vl",+ "text_config": {"model_type": "minimax_m2"},+ }++ with patch("freetoken.utils.cached_load_hf_config", lambda _p: _Config()):+ args, _ = parse_args(["--model", "/models/anon"])+ assert args.tool_call_parser == "minimax_m3"+ assert args.reasoning_parser == "minimax_m3"+++# ---------------------------------------------------------------------------+# Review regressions+# ---------------------------------------------------------------------------+def test_reasoning_adaptive_leading_bare_closer_one_shot():+ # Adaptive non-thinking turns START with a bare </mm:think> written by the+ # model; it must never leak into content (position 0 only).+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ r = p.detect_and_parse("</mm:think>Here is the answer.")+ assert r.reasoning_text == ""+ assert r.normal_text == "Here is the answer."+++def test_reasoning_adaptive_leading_bare_closer_streaming_split():+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ normal = ""+ for chunk in ["</mm:th", "ink>Hello,", " world!"]:+ r = p.parse_streaming_increment(chunk)+ assert r.reasoning_text == ""+ normal += r.normal_text+ normal += p.flush().normal_text+ assert normal == "Hello, world!"+++def test_reasoning_leading_closer_after_whitespace():+ # The detokenizer may open with a newline before the model's bare closer;+ # the whitespace-tolerant head still strips it (one-shot and streaming).+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ r = p.detect_and_parse("\n</mm:think>Here is the answer.")+ assert r.reasoning_text == "" and r.normal_text == "Here is the answer."++ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ normal = ""+ for chunk in ["\n", "</mm:think>", "Hi."]:+ out = p.parse_streaming_increment(chunk)+ assert out.reasoning_text == ""+ normal += out.normal_text+ normal += p.flush().normal_text+ assert normal == "Hi."+++def test_reasoning_later_closer_stays_visible():+ # Only a position-0 closer is stripped; one appearing later is content.+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ r = p.detect_and_parse("The tag </mm:think> is literal here.")+ assert "</mm:think>" in r.normal_text+++def test_reasoning_forced_mode_closer_not_stripped():+ # thinking_mode=enabled: the template pre-opened the block, so a leading+ # closer legitimately ends (empty) reasoning -- content follows.+ p = MiniMaxM3ReasoningParser(force_reasoning=True)+ r = p.detect_and_parse("</mm:think>answer")+ assert r.reasoning_text == "" and r.normal_text == "answer"+++def test_reasoning_streaming_head_prefix_replayed_on_divergence():+ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ out = p.parse_streaming_increment("</mm") # closer prefix: held+ assert out.normal_text == "" and out.reasoning_text == ""+ out = p.parse_streaming_increment("ory says hi") # diverges: replayed+ assert out.normal_text == "</mmory says hi"+++def test_streaming_wire_order_trailing_text_defers():+ # "text + block + trailing text" in ONE chunk: the trailing text must not be+ # emitted in the same increment as (i.e. wire-ahead of) the tool call.+ det = MiniMaxM3Detector()+ chunk = (+ "Before. "+ + _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')+ + " After."+ )+ r1 = det.parse_streaming_increment(chunk, _tools())+ assert r1.normal_text == "Before. "+ assert [c.name for c in r1.calls if c.name] == ["get_weather"]+ r2 = det.parse_streaming_increment("", _tools())+ assert r2.normal_text == " After." and r2.calls == []+++def test_streaming_second_block_in_same_chunk():+ det = MiniMaxM3Detector()+ one = _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')+ two = _block(f'{NS}<invoke name="get_weather">{NS}<city>Tokyo{NS}</city>{NS}</invoke>\n')+ calls = []+ calls += det.parse_streaming_increment(one + two, _tools()).calls+ calls += det.parse_streaming_increment("", _tools()).calls+ args = [json.loads(c.parameters) for c in calls if c.name is None and c.parameters]+ assert args == [{"city": "Paris"}, {"city": "Tokyo"}]+++def test_streaming_residue_after_close_holds_partial_marker():+ # After a closed wrapper, a split second-block marker in the residue must be+ # held (idle-path hold), never leaked as content.+ det = MiniMaxM3Detector()+ one = _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')+ r1 = det.parse_streaming_increment(one + "]<]mini", _tools())+ assert "]<]mini" not in r1.normal_text+ r2 = det.parse_streaming_increment(+ f'max[>[<tool_call>\n{NS}<invoke name="get_weather">{NS}<city>Rome{NS}</city>'+ f"{NS}</invoke>\n{NS}</tool_call>",+ _tools(),+ )+ frags = [c.parameters for c in r2.calls if c.name is None and c.parameters]+ assert json.loads(frags[-1]) == {"city": "Rome"}+++def test_streaming_truncated_call_suppressed_and_recovered():+ # Generation cut mid-invoke (max_tokens): finish_streaming must not leak the+ # raw markup, and recover_truncated_call must salvage the complete params.+ from freetoken.server.function_call_parser import FunctionCallParser++ parser = FunctionCallParser(_tools(), "minimax_m3")+ det = parser.detector+ truncated = (+ "Let me order. " + NS + "<tool_call>\n"+ + f'{NS}<invoke name="create_order">{NS}<user_id>42{NS}</user_id>{NS}<note>hi'+ )+ normal = ""+ for i in range(0, len(truncated), 11):+ r = det.parse_streaming_increment(truncated[i : i + 11], _tools())+ normal += r.normal_text+ assert not r.calls+ assert normal == "Let me order. "+ # Serving order (generation.py): recover first (consumes the buffer on+ # success), then finish_stream drains what's left.+ recovered = parser.recover_truncated_call()+ assert [c.name for c in recovered] == ["create_order"]+ args = json.loads(recovered[0].parameters)+ assert args["user_id"] == 42 # the complete param survives; truncated one dropped+ assert det.finish_streaming() == "" # raw markup never reaches the client+++def test_args_stray_text_keeps_parsed_params():+ det = MiniMaxM3Detector()+ body = f"{NS}<city>Paris{NS}</city>zzz-stray"+ text = _block(f'{NS}<invoke name="get_weather">{body}{NS}</invoke>\n')+ res = det.detect_and_parse(text, _tools())+ assert json.loads(res.calls[0].parameters) == {"city": "Paris"}+++def test_args_empty_value_is_empty_string():+ det = MiniMaxM3Detector()+ text = _block(f'{NS}<invoke name="get_weather">{NS}<city>{NS}</city>{NS}</invoke>\n')+ res = det.detect_and_parse(text, _tools())+ assert json.loads(res.calls[0].parameters) == {"city": ""}+++def test_args_repeated_siblings_keep_parent_key():+ """Only <item> children render as a bare array; repeated same-name siblings+ under any other tag stay an object with an array-valued key (the bare-list+ collapse dropped the parent key the references keep)."""+ det = MiniMaxM3Detector()+ body = (+ f"{NS}<items>{NS}<sku>a{NS}</sku>{NS}<sku>b{NS}</sku>{NS}</items>"+ f"{NS}<note>x{NS}</note>{NS}<note>y{NS}</note>"+ )+ text = _block(f'{NS}<invoke name="create_order">{body}{NS}</invoke>\n')+ res = det.detect_and_parse(text, _tools())+ args = json.loads(res.calls[0].parameters)+ assert args["items"] == {"sku": ["a", "b"]} # parent key kept, values aggregate+ assert args["note"] == ["x", "y"] # repeated top-level params still aggregate+++def test_element_semantics_reference_batch():+ """Element-grammar batch: empty container-typed params, anyOf coercion,+ dangling-closer leniency, single-quoted invoke name, $text for mixed+ content, and float preservation."""+ tools = [+ Tool(function=Function(name="t", parameters={+ "type": "object",+ "properties": {+ "obj": {"type": "object"},+ "arr": {"type": "array"},+ "uni": {"anyOf": [{"type": "integer"}, {"type": "string"}]},+ "price": {"type": "number"},+ },+ }))+ ]+ det = MiniMaxM3Detector()+ body = (+ f"{NS}<obj>{NS}</obj>"+ f"{NS}<arr>{NS}</arr>"+ f"{NS}<uni>42{NS}</uni>"+ f"{NS}<price>5.0{NS}</price>"+ )+ text = _block(f'{NS}<invoke name="t">{body}{NS}</invoke>\n')+ args = json.loads(det.detect_and_parse(text, tools).calls[0].parameters)+ assert args["obj"] == {} and args["arr"] == [] # typed empty containers+ assert args["uni"] == 42 # anyOf: integer member coerces+ assert args["price"] == 5.0 and isinstance(args["price"], float) # not 5++ # anyOf falls back to the string member verbatim+ text2 = _block(f'{NS}<invoke name="t">{NS}<uni>hello{NS}</uni>{NS}</invoke>\n')+ args2 = json.loads(MiniMaxM3Detector().detect_and_parse(text2, tools).calls[0].parameters)+ assert args2["uni"] == "hello"++ # a dangling closer no longer drops the well-formed siblings after it+ body3 = f"{NS}</ghost>{NS}<uni>7{NS}</uni>"+ text3 = _block(f'{NS}<invoke name="t">{body3}{NS}</invoke>\n')+ args3 = json.loads(MiniMaxM3Detector().detect_and_parse(text3, tools).calls[0].parameters)+ assert args3["uni"] == 7++ # single-quoted name= parses (double quotes are the template's rendering,+ # but models emit single quotes too)+ text4 = _block(f"{NS}<invoke name='t'>{NS}<uni>1{NS}</uni>{NS}</invoke>\n")+ res4 = MiniMaxM3Detector().detect_and_parse(text4, tools)+ assert [c.name for c in res4.calls] == ["t"]++ # mixed text+children keeps the text under $text+ body5 = f"{NS}<obj>freeform {NS}<k>v{NS}</k>{NS}</obj>"+ text5 = _block(f'{NS}<invoke name="t">{body5}{NS}</invoke>\n')+ args5 = json.loads(MiniMaxM3Detector().detect_and_parse(text5, tools).calls[0].parameters)+ assert args5["obj"] == {"k": "v", "$text": "freeform"}+++def test_reasoning_one_shot_matches_streaming_positionally():+ """One-shot and streaming must agree positionally: prose before a+ mid-content <mm:think> stays content, and a later quoted marker occurrence+ is data."""+ text = "intro <mm:think>deep thought</mm:think> answer quoting <mm:think> literally"+ one = MiniMaxM3ReasoningParser(force_reasoning=False).detect_and_parse(text)+ assert one.reasoning_text == "deep thought"+ assert one.normal_text == "intro answer quoting <mm:think> literally"++ p = MiniMaxM3ReasoningParser(force_reasoning=False)+ reasoning, content = [], []+ for i in range(0, len(text), 3):+ r = p.parse_streaming_increment(text[i : i + 3])+ reasoning.append(r.reasoning_text)+ content.append(r.normal_text)+ r = p.flush()+ reasoning.append(r.reasoning_text)+ content.append(r.normal_text)+ assert "".join(reasoning) == one.reasoning_text+ assert "".join(content) == one.normal_text+++def test_reasoning_one_shot_verbatim_no_strip():+ # whitespace around reasoning/content is data (references are verbatim)+ r = MiniMaxM3ReasoningParser(force_reasoning=False).detect_and_parse(+ "<mm:think>\nthink\n</mm:think>\nanswer\n"+ )+ assert r.reasoning_text == "\nthink\n"+ assert r.normal_text == "\nanswer\n"+ r2 = MiniMaxM3ReasoningParser(force_reasoning=True).detect_and_parse(+ "th\n</mm:think>\nans\n"+ )+ assert r2.reasoning_text == "th\n" and r2.normal_text == "\nans\n"+++def test_detect_and_parse_multiple_wrappers_and_inter_block_text():+ det = MiniMaxM3Detector()+ one = _block(f'{NS}<invoke name="get_weather">{NS}<city>Paris{NS}</city>{NS}</invoke>\n')+ two = _block(f'{NS}<invoke name="get_weather">{NS}<city>Rome{NS}</city>{NS}</invoke>\n')+ res = det.detect_and_parse("A. " + one + " middle " + two + " B.", _tools())+ assert [json.loads(c.parameters)["city"] for c in res.calls] == ["Paris", "Rome"]+ assert "middle" in res.normal_text and "A." in res.normal_text and "B." in res.normal_text+++ "gear,mode", [("off", "disabled"), ("adaptive", "adaptive"), ("on", "enabled")]+)+def test_think_gears(gear, mode):+ from freetoken.server.model_meta import think_chat_template_kwargs, think_spec++ gears, default = think_spec("minimax_m3")+ assert gears == ("off", "adaptive", "on") and default == "adaptive"+ assert think_chat_template_kwargs("minimax_m3", gear) == {"thinking_mode": mode}diff --git a/tests/server/test_responses_api.py b/tests/server/test_responses_api.pyindex 549d925a..ef9e602d 100644--- a/tests/server/test_responses_api.py+++ b/tests/server/test_responses_api.py@@ -888,3 +888,23 @@ def test_convert_reasoning_effort_none_disables_thinking():assert RP.convert_responses_to_genspec(req, {}).chat_template_kwargs == {"enable_thinking": False}+++def test_convert_reasoning_toggle_routes_through_family_mapping():+ """The toggle goes through model_meta's per-family mapping -- for M3 that is+ thinking_mode, not the (inert) enable_thinking key."""+ req = ResponsesRequest.model_validate(+ {"model": "m", "input": "hi", "reasoning": {"effort": "high"}}+ )+ spec = RP.convert_responses_to_genspec(req, {}, reasoning_parser="minimax_m3")+ assert spec.chat_template_kwargs == {+ "thinking_mode": "enabled", "reasoning_effort": "high",+ }+ off = ResponsesRequest.model_validate(+ {"model": "m", "input": "hi", "reasoning": {"effort": "none"}}+ )+ spec = RP.convert_responses_to_genspec(off, {}, reasoning_parser="minimax_m3")+ assert spec.chat_template_kwargs == {"thinking_mode": "disabled"}+ # gpt-oss: the template grades effort and has no off gear+ spec = RP.convert_responses_to_genspec(req, {}, reasoning_parser="gpt_oss")+ assert spec.chat_template_kwargs == {"reasoning_effort": "high"}diff --git a/tests/server/test_streaming_model_matrix.py b/tests/server/test_streaming_model_matrix.pyindex 5f0b8590..a0c94a5d 100644--- a/tests/server/test_streaming_model_matrix.py+++ b/tests/server/test_streaming_model_matrix.py@@ -80,6 +80,13 @@ CALL_BLOCKS = {'<minimax:tool_call><invoke name="read"><parameter name="filePath">'"/tmp/test_calc.py</parameter></invoke></minimax:tool_call>"),+ "minimax_m3": (+ "]<]minimax[>[<tool_call>\n"+ ']<]minimax[>[<invoke name="read">'+ "]<]minimax[>[<filePath>/tmp/test_calc.py]<]minimax[>[</filePath>"+ "]<]minimax[>[</invoke>\n"+ "]<]minimax[>[</tool_call>"+ ),"mistral": '[TOOL_CALLS] [{"name": "read", "arguments": {"filePath": "/tmp/test_calc.py"}}]',"llama3": '<|python_tag|>{"name": "read", "arguments": {"filePath": "/tmp/test_calc.py"}}',"gpt_oss": (@@ -96,6 +103,7 @@ MARKUP_MARKERS = {"glm47": ["<arg_key>", "<arg_value>", "</tool_call>"],"gemma4": ["<|tool_call>", "<tool_call|>"],"minimax": ["<minimax:tool_call>", "<invoke"],+ "minimax_m3": ["]<]minimax[>[", "<invoke"],"mistral": ["[TOOL_CALLS]"],"llama3": ["<|python_tag|>"],"gpt_oss": ["<|channel|>", "to=functions."],@@ -110,6 +118,9 @@ REASONING_FAMILIES = {"qwen": ("qwen25", "qwen3", "", "</think>"),"glm4.7": ("glm47", "glm", "", "</think>"),"minimax-m2": ("minimax", "minimax", "", "</think>"),+ # M3 adaptive mode: the model opens <mm:think> itself (enabled mode pre-opens it+ # in the template; the parser then runs with force_reasoning=True instead).+ "minimax-m3": ("minimax_m3", "minimax_m3", "<mm:think>", "</mm:think>"),"gemma4": ("gemma4", "gemma4", "<|channel>thought\n", "<channel|>"),"gpt-oss": ("gpt_oss", "gpt_oss", None, None), # harmony channels; custom fixture}@@ -629,7 +640,9 @@ def test_empty_arguments_call_emitted_exactly_once(tool, block):@pytest.mark.parametrize(- "family", ["qwen25", "qwen3_coder", "glm47", "gemma4", "minimax", "deepseekv32", "gpt_oss"]+ "family",+ ["qwen25", "qwen3_coder", "glm47", "gemma4", "minimax", "minimax_m3",+ "deepseekv32", "gpt_oss"],)def test_call_then_trailing_text_in_one_chunk_keeps_order(family):# Worst case: the whole generation arrives as ONE chunk. Text after the call