FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
Support muse glimmer (#4)
Shuo Yang6223cda1b0f1
diff --git a/docs/models.md b/docs/models.mdindex fd4e0a7e..e4850a12 100644--- a/docs/models.md+++ b/docs/models.md@@ -15,6 +15,7 @@ for them; other checkpoints of the same architectures work too.| gpt-oss | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b), [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) || Gemma-4 | [google/gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it), [nvidia/Gemma-4-26B-A4B-NVFP4](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4), [google/gemma-4-12B-it](https://huggingface.co/google/gemma-4-12B-it), [nvidia/Gemma-4-31B-IT-NVFP4](https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4) .. || MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) |+| Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |## MoE backendsdiff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 3d195a43..a00154d0 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -316,6 +316,13 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (hidden_size=5376,kv_groups=((4, 512), (16, 256)),),+ AotModel(+ name="meta-models/Muse-Glimmer-30B",+ architecture="MuseGlimmerForConditionalGeneration",+ hidden_size=6656,+ kv_groups=((2, 128),), # sliding and full (NoPE) layers share the same geometry+ aliases=("RedHatAI/Muse-Glimmer-30B-NVFP4",),+ ),AotModel(name="meta-llama/Llama-3.1-8B-Instruct",architecture="LlamaForCausalLM",diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.pyindex 3dadbdec..f6105e1f 100644--- a/python/freetoken/models/config.py+++ b/python/freetoken/models/config.py@@ -34,6 +34,46 @@ def detect_expert_quant(hf_config: Any) -> str:return "nvfp4" if "fp4" in str(algo).lower() else str(algo).lower()+def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool:+ """Detect a compressed-tensors NVFP4 (W4A16) checkpoint (dense Qwen3.6-27B,+ Muse-Glimmer-30B-NVFP4, ...).++ llm-compressor stores ``quant_method == "compressed-tensors"`` with a ``config_groups``+ map whose ``weights`` are ``num_bits=4, type="float", group_size=16,+ strategy="tensor_group"`` -- the NVFP4 layout (``weight_packed`` uint8 ++ ``weight_scale`` fp8 block + scalar ``weight_global_scale``). All ``targets:+ ["Linear"]`` are NVFP4 except the per-module ``ignore`` list. A 4-bit float scheme+ with a DIFFERENT geometry (MXFP4: group_size 32, real since LLM Compressor 0.9)+ raises instead of routing into the NVFP4 loader and dying in a shape assert."""+ quant = getattr(hf_config, "quantization_config", None)+ if quant is None:+ return False+ get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))+ if str(get("quant_method") or "").lower() != "compressed-tensors":+ return False+ groups = get("config_groups") or {}+ # Verdicts are collected across ALL groups before returning: an early return on+ # the first NVFP4 group would accept a mixed {nvfp4, mxfp4} checkpoint (and the+ # error would depend on the groups' key order).+ saw_nvfp4 = False+ for g in (groups.values() if isinstance(groups, dict) else []):+ w = (g or {}).get("weights") or {}+ if int(w.get("num_bits", 0) or 0) != 4 or str(w.get("type", "")).lower() != "float":+ continue+ # vLLM's _is_nvfp4_format gates on the same two fields.+ group_size = int(w.get("group_size", 0) or 0)+ strategy = str(w.get("strategy", "")).lower()+ if group_size == 16 and strategy == "tensor_group":+ saw_nvfp4 = True+ continue+ raise ValueError(+ "unsupported compressed-tensors 4-bit float scheme "+ f"(group_size={group_size}, strategy={strategy!r}); FreeToken serves "+ "NVFP4 (group_size=16, strategy=tensor_group) only"+ )+ return saw_nvfp4++@dataclass(frozen=True)class RotaryConfig:head_dim: int@@ -221,6 +261,12 @@ class ModelConfig:attn_sm_scale: float | None = None # None -> 1/sqrt(head_dim)final_logit_softcapping: float | None = Noneembedding_scale: float | None = None+ # Second RMSNorm eps for models whose post-sublayer norms use a different eps than the+ # pre-sublayer ones (muse_glimmer: post_attention/post_feedforward at 1e-8 vs 1e-5).+ post_norm_eps: float | None = None+ # Scalar the raw lm_head logits are multiplied by BEFORE final_logit_softcapping+ # (muse_glimmer: 1/sqrt(hidden/256)); None leaves the logits unscaled.+ output_multiplier: float | None = Nonevision_config: Any | None = Noneimage_token_id: int | None = Noneattention_groups: Tuple[AttentionGroupConfig, ...] = ()diff --git a/python/freetoken/models/loader.py b/python/freetoken/models/loader.pyindex 5d4571eb..49419364 100644--- a/python/freetoken/models/loader.py+++ b/python/freetoken/models/loader.py@@ -133,6 +133,125 @@ def iter_merged_tensors()+# ---------------------------------------------------------------------------------+# compressed-tensors NVFP4 (llm-compressor) dense-weight helpers, shared by the+# models that serve such checkpoints natively (qwen3_5_moe, muse_glimmer). Storage:+# ``weight_packed`` (uint8 [O, IN//2]) + ``weight_scale`` (fp8-e4m3 block [O, IN//16])+# + a scalar ``weight_global_scale``. The stored global is the *quant-side* scale, so+# the dequant/native global is its reciprocal (vLLM inverts it identically).+# ---------------------------------------------------------------------------------++# Quant scales consumed with their ``weight_packed`` (or unused: the input scales are+# for W4A4 activation quant, which FreeToken does not run).+CT_SCALE_SUFFIXES = (+ ".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale",+)+++class ShardReader:+ """Serves tensors by name across safetensors shards (handles opened lazily).++ The quant scales of a ``weight_packed`` can land in a DIFFERENT shard than the+ packed weight itself (Muse-Glimmer-30B-NVFP4 splits layer 49's down_proj across+ the shard boundary), so sibling lookups must go through the index's weight_map+ rather than the shard the packed weight came from."""++ def __init__(self, model_path: str, device: torch.device):+ folder = download_hf_weight(model_path)+ index = os.path.join(folder, "model.safetensors.index.json")+ if os.path.exists(index):+ with open(index, encoding="utf-8") as f:+ weight_map = json.load(f)["weight_map"]+ self._map = {+ name: os.path.join(folder, shard) for name, shard in weight_map.items()+ }+ else: # single-file checkpoint+ import safetensors++ self._map = {}+ for file in iter_weight_files(model_path):+ with safetensors.safe_open(file, framework="pt", device="cpu") as f:+ for name in f.keys():+ self._map[name] = file+ self._device = str(device)+ self._handles: dict[str, object] = {}++ def files(self) -> list[str]:+ return sorted(set(self._map.values()))++ def names_in(self, file: str) -> list[str]:+ return [name for name, shard in self._map.items() if shard == file]++ def get_tensor(self, name: str) -> torch.Tensor:+ import safetensors++ file = self._map[name]+ h = self._handles.get(file)+ if h is None:+ h = safetensors.safe_open(file, framework="pt", device=self._device).__enter__()+ self._handles[file] = h+ return h.get_tensor(name)++ def close(self) -> None:+ for h in self._handles.values():+ try:+ h.__exit__(None, None, None)+ except Exception: # noqa: BLE001 -- best-effort handle cleanup+ pass+ self._handles.clear()+++def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16],+ per-output-row global fp16 [O])`` for the W4A16 kernels."""+ w = f.get_tensor(raw_base + ".weight_packed")+ s = f.get_tensor(raw_base + ".weight_scale")+ wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)+ g = (1.0 / wg).to(torch.float16).expand(w.shape[0]).contiguous()+ return w, s, g+++def ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict, groups: dict[str, tuple[str, ...]]):+ """Buffer a native NVFP4 fusion part ``(w, s, g)``; emit the concatenated native parts+ (``.weight``/``.weight_scale``/``.weight_global``, output-dim concat with each part+ keeping its own scales, so the fused FP4 weight is exact) once complete, ``[]`` while+ incomplete, ``None`` if ``base`` is not a fusion part of any group in ``groups``."""+ for fused_suffix, parts in groups.items():+ for idx, part in enumerate(parts):+ if base.endswith(part):+ key = base[: -len(part)] + fused_suffix+ slots = buf.setdefault(key, {})+ slots[idx] = parts_tuple+ if len(slots) < len(parts):+ return []+ del buf[key]+ ws = [slots[i][0] for i in range(len(parts))]+ ss = [slots[i][1] for i in range(len(parts))]+ gs = [slots[i][2] for i in range(len(parts))]+ return [+ (key + ".weight", torch.cat(ws, dim=0)),+ (key + ".weight_scale", torch.cat(ss, dim=0)),+ (key + ".weight_global", torch.cat(gs, dim=0)),+ ]+ return None+++def ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict[str, tuple[str, ...]]):+ """Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]``+ while incomplete, ``None`` if ``base`` is not a part of any group in ``groups``."""+ for fused_suffix, parts in groups.items():+ for idx, part in enumerate(parts):+ if base.endswith(part):+ key = base[: -len(part)] + fused_suffix+ slots = buf.setdefault(key, {})+ slots[idx] = tensor+ if len(slots) < len(parts):+ return []+ del buf[key]+ return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))]+ return None++def _expert_stack_info(key: str, expert_pattern: re.Pattern[str]) -> tuple[str, int] | None:match = expert_pattern.match(key)if match is None:diff --git a/python/freetoken/models/muse_glimmer/__init__.py b/python/freetoken/models/muse_glimmer/__init__.pynew file mode 100644index 00000000..307d2866--- /dev/null+++ b/python/freetoken/models/muse_glimmer/__init__.py@@ -0,0 +1,11 @@+from .attention import MuseGlimmerAttention+from .config import parse_config+from .model import MuseGlimmerForCausalLM+from .weight import iter_weights++__all__ = [+ "MuseGlimmerAttention",+ "MuseGlimmerForCausalLM",+ "parse_config",+ "iter_weights",+]diff --git a/python/freetoken/models/muse_glimmer/attention.py b/python/freetoken/models/muse_glimmer/attention.pynew file mode 100644index 00000000..29764dcf--- /dev/null+++ b/python/freetoken/models/muse_glimmer/attention.py@@ -0,0 +1,100 @@+from __future__ import annotations++from typing import TYPE_CHECKING++import torch+from freetoken.attention import AttentionSpec+from freetoken.core import get_global_ctx+from freetoken.layers import BaseOP, GemmaRMSNorm+from freetoken.layers.rotary import get_rope+from freetoken.models.config import SWAAttentionGroupConfig+from freetoken.utils import nvtx_annotate++from freetoken.models.quant_linear import make_col_merged, make_replicated++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class MuseGlimmerAttention(BaseOP):+ """Gated GQA for one sliding-window or full-attention (NoPE) layer.++ q, k, v, gate = split(qkvg_proj(x))+ q = qk_norm(q); k = qk_norm(k) # weightless per-head RMSNorm+ q, k = rope(q, k) # sliding layers only; full layers are NoPE+ attn = paged_attention(q, k, v) # sm_scale carries qk_scale_factor/sqrt(d)+ out = o_proj(attn * sigmoid(gate))++ The reference multiplies q by ``qk_scale_factor`` before rope; rope is a rotation, so+ the factor commutes and is folded into ``AttentionSpec.sm_scale`` instead (parse_config+ sets ``attn_sm_scale = qk_scale_factor / sqrt(head_dim)``). The gate is computed from+ the same normed layer input as q/k/v, so it rides the fused projection.+ """++ def __init__(self, config: ModelConfig, layer_id: int):+ self.layer_id = layer_id+ group = config.attention_group_for_layer(layer_id)+ self.is_swa = isinstance(group, SWAAttentionGroupConfig)+ head_dim = group.head_dim+ self.head_dim = head_dim+ self.num_q = config.num_qo_heads+ self.num_kv = group.num_kv_heads+ self.qo_attn_dim = self.num_q * head_dim+ self.kv_attn_dim = self.num_kv * head_dim++ self._qkvg_split = [self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim, self.qo_attn_dim]+ self.qkvg_proj = make_col_merged(+ config, config.hidden_size, self._qkvg_split, has_bias=False+ )+ self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False)+ # Weightless: the checkpoint carries no q/k norm weights; the runtime ones vector+ # is intentionally not part of state_dict.+ self.qk_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps, with_scale=False)+ self.attn_spec = AttentionSpec(+ sliding_window=group.sliding_window if self.is_swa else None,+ sm_scale=config.attn_sm_scale,+ )+ # base == 0.0 marks the NoPE full-attention layers; RotaryEmbedding must never+ # see it (0 ** x in the inv_freq computation).+ rotary_config = group.rotary_config+ self.rotary = (+ get_rope(+ head_dim=head_dim,+ rotary_dim=rotary_config.rotary_dim,+ max_position=rotary_config.max_position,+ base=rotary_config.base,+ )+ if rotary_config.base > 0.0+ else None+ )++ @nvtx_annotate("MHA")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ ctx = get_global_ctx()+ T = x.shape[0]++ qkvg = self.qkvg_proj.forward(x)+ q, k, v, gate = torch.split(qkvg, self._qkvg_split, dim=-1)+ del qkvg+ q = self.qk_norm.forward(q.contiguous().view(T, self.num_q, self.head_dim))+ k = self.qk_norm.forward(k.contiguous().view(T, self.num_kv, self.head_dim))+ v = v.contiguous() # the split view has the qkvg row stride; the KV store needs contiguous++ q = q.reshape(T, self.qo_attn_dim)+ k = k.reshape(T, self.kv_attn_dim)+ if self.rotary is not None:+ q, k = self.rotary.forward(ctx.batch.positions, q, k)++ o = ctx.attn_backend.forward(+ q.view(T, self.num_q, self.head_dim),+ k,+ v,+ self.layer_id,+ ctx.batch,+ attn_spec=self.attn_spec,+ )+ gated = o.reshape(T, self.qo_attn_dim) * torch.sigmoid(gate)+ return self.o_proj.forward(gated)+++__all__ = ["MuseGlimmerAttention"]diff --git a/python/freetoken/models/muse_glimmer/config.py b/python/freetoken/models/muse_glimmer/config.pynew file mode 100644index 00000000..4cc756cb--- /dev/null+++ b/python/freetoken/models/muse_glimmer/config.py@@ -0,0 +1,136 @@+from __future__ import annotations++from typing import Any++from freetoken.models.config import (+ FullAttentionGroupConfig,+ ModelConfig,+ RotaryConfig,+ SWAAttentionGroupConfig,+ detect_compressed_tensors_nvfp4,+)++_SWA_TYPE = "sliding_attention"+_FULL_TYPE = "full_attention"+++def _text_config(hf_config: Any) -> Any:+ """Muse Glimmer ships as a multimodal wrapper (MuseGlimmerForConditionalGeneration):+ the text tower lives in ``text_config`` and the weights carry a ``language_model.``+ prefix. Served text-only -- the ~1.8B ViT perception encoder is never built, so+ ``ModelConfig.vision_config`` stays None and the loader drops the vision tensors."""+ text = getattr(hf_config, "text_config", None)+ return text if text is not None else hf_config+++def _group_rope_theta(text: Any, layer_ids: tuple[int, ...], default: float) -> float:+ """Rope theta of one layer group. ``layer_rope_theta`` is the per-layer source of+ truth (0 marks the NoPE full-attention layers); absent, fall back to ``default``+ (the shared ``rope_parameters.rope_theta``)."""+ per_layer = getattr(text, "layer_rope_theta", None)+ if not per_layer:+ return default+ thetas = {float(per_layer[i]) for i in layer_ids}+ assert len(thetas) == 1, f"mixed rope thetas within one attention group: {thetas}"+ return thetas.pop()+++def parse_config(hf_config: Any) -> ModelConfig:+ text = _text_config(hf_config)++ head_dim = getattr(text, "head_dim", None) or text.hidden_size // text.num_attention_heads+ num_kv_heads = getattr(text, "num_key_value_heads", text.num_attention_heads)+ max_position = text.max_position_embeddings++ rope_params = getattr(text, "rope_parameters", None) or {}+ base_theta = float(rope_params.get("rope_theta", getattr(text, "rope_theta", 500000.0)))++ layer_types = list(text.layer_types)+ swa_ids = tuple(i for i, t in enumerate(layer_types) if t == _SWA_TYPE)+ full_ids = tuple(i for i, t in enumerate(layer_types) if t == _FULL_TYPE)+ assert len(swa_ids) + len(full_ids) == len(layer_types), (+ f"unknown layer types in {set(layer_types)}"+ )++ swa_rotary = RotaryConfig(+ head_dim=head_dim,+ rotary_dim=head_dim,+ max_position=max_position,+ base=_group_rope_theta(text, swa_ids, base_theta),+ scaling=None,+ )+ # Full-attention layers are NoPE (layer_rope_theta == 0): base 0.0 is the marker the+ # attention module reads to skip rope entirely. It must never reach RotaryEmbedding+ # (0 ** x); MuseGlimmerAttention guards on it.+ full_rotary = RotaryConfig(+ head_dim=head_dim,+ rotary_dim=head_dim,+ max_position=max_position,+ base=_group_rope_theta(text, full_ids, 0.0),+ scaling=None,+ )++ # The reference applies a weightless per-head RMSNorm to q and k, then multiplies q by+ # qk_scale_factor and runs standard 1/sqrt(head_dim) attention. q is not cached, so the+ # factor folds exactly into the softmax scale.+ qk_scale_factor = float(getattr(text, "qk_scale_factor", 1.0))+ attn_sm_scale = qk_scale_factor * head_dim**-0.5++ # RedHatAI/Muse-Glimmer-30B-NVFP4: every text-tower Linear (q/k/v/o + the attention+ # gate + the MLP) is compressed-tensors NVFP4 (W4A16); lm_head, embeddings, norms and+ # the (unserved) vision tower stay bf16.+ nvfp4 = detect_compressed_tensors_nvfp4(hf_config)+ quant = "nvfp4" if nvfp4 else "none"++ return ModelConfig(+ num_layers=text.num_hidden_layers,+ num_qo_heads=text.num_attention_heads,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ hidden_size=text.hidden_size,+ vocab_size=text.vocab_size,+ intermediate_size=text.intermediate_size,+ hidden_act=getattr(text, "hidden_activation", "silu"),+ rms_norm_eps=text.rms_norm_eps,+ post_norm_eps=float(getattr(text, "post_norm_eps", text.rms_norm_eps)),+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),+ rotary_config=swa_rotary,+ num_experts=0,+ num_experts_per_tok=0,+ moe_intermediate_size=0,+ norm_topk_prob=False,+ model_type=getattr(hf_config, "model_type", "muse_glimmer"),+ architectures=(+ getattr(hf_config, "architectures", None)+ or ["MuseGlimmerForConditionalGeneration"]+ ),+ use_qk_norm=True,+ attn_sm_scale=attn_sm_scale,+ final_logit_softcapping=getattr(text, "final_logit_softcapping", None),+ output_multiplier=getattr(text, "output_multiplier", None),+ vision_config=None, # served text-only+ image_token_id=getattr(hf_config, "image_token_id", None),+ attn_quant=quant,+ dense_quant=quant,+ lm_head_quant="none",+ attention_groups=(+ FullAttentionGroupConfig(+ name="full",+ layer_ids=full_ids,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ rotary_config=full_rotary,+ ),+ SWAAttentionGroupConfig(+ name="swa",+ layer_ids=swa_ids,+ num_kv_heads=num_kv_heads,+ head_dim=head_dim,+ rotary_config=swa_rotary,+ sliding_window=text.sliding_window,+ ),+ ),+ )+++__all__ = ["parse_config"]diff --git a/python/freetoken/models/muse_glimmer/model.py b/python/freetoken/models/muse_glimmer/model.pynew file mode 100644index 00000000..ba632c0b--- /dev/null+++ b/python/freetoken/models/muse_glimmer/model.py@@ -0,0 +1,134 @@+from __future__ import annotations++from typing import TYPE_CHECKING++import torch+from freetoken.core import get_global_ctx+from freetoken.layers import (+ BaseOP,+ GemmaPlusOneRMSNorm,+ GemmaPlusOneRMSNormFused,+ GemmaRMSNorm,+ OPList,+ ParallelLMHead,+ RMSNorm,+ VocabParallelEmbedding,+ silu_and_mul,+)+from freetoken.models.blocks import BaseLLMModel+from freetoken.utils import nvtx_annotate++from .attention import MuseGlimmerAttention+from freetoken.models.quant_linear import make_col_merged, make_replicated++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+++class MuseGlimmerMLP(BaseOP):+ """SwiGLU MLP (gate|up fused), NVFP4 (W4A16) on the quantized checkpoint else bf16."""++ def __init__(self, config: ModelConfig):+ self.gate_up_proj = make_col_merged(+ config,+ config.hidden_size,+ [config.intermediate_size, config.intermediate_size],+ has_bias=False,+ )+ self.down_proj = make_replicated(+ config, config.intermediate_size, config.hidden_size, has_bias=False+ )++ @nvtx_annotate("MLP")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return self.down_proj.forward(silu_and_mul(self.gate_up_proj.forward(x)))+++class MuseGlimmerDecoderLayer(BaseOP):+ """Sandwich block with centered (1+w) norms:++ h = post_attention_layernorm(attn(input_layernorm(x))); x = x + h+ h = post_feedforward_layernorm(mlp(pre_feedforward_layernorm(x))); x = x + h++ The pre-norms use ``rms_norm_eps``, the post-norms the tighter ``post_norm_eps``+ (1e-8). The (1+w) scaling stays a runtime fp32 add (GemmaPlusOneRMSNorm) -- baking+ +1 into the bf16 weight would round away exactly the precision the centered form+ exists to keep.+ """++ def __init__(self, config: ModelConfig, layer_id: int):+ self._layer_id = layer_id+ self.self_attn = MuseGlimmerAttention(config, layer_id)+ self.mlp = MuseGlimmerMLP(config)+ H = config.hidden_size+ post_eps = config.post_norm_eps if config.post_norm_eps is not None else config.rms_norm_eps+ self.input_layernorm = GemmaPlusOneRMSNorm(H, eps=config.rms_norm_eps)+ self.post_attention_layernorm = GemmaPlusOneRMSNorm(H, eps=post_eps)+ self.pre_feedforward_layernorm = GemmaPlusOneRMSNormFused(H, eps=config.rms_norm_eps)+ self.post_feedforward_layernorm = GemmaPlusOneRMSNorm(H, eps=post_eps)++ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")+ def forward(self, x: torch.Tensor) -> torch.Tensor:+ residual = x+ h = self.input_layernorm.forward(x)+ h = self.self_attn.forward(h)+ h = self.post_attention_layernorm.forward(h)+ # Fused add + norm: residual becomes x + h, pre_ff its norm.+ pre_ff, residual = self.pre_feedforward_layernorm.forward(h, residual)+ h = self.mlp.forward(pre_ff)+ h = self.post_feedforward_layernorm.forward(h)+ return residual + h+++class MuseGlimmerModel(BaseOP):+ def __init__(self, config: ModelConfig):+ self.embed_tokens = VocabParallelEmbedding(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ )+ # NormedEmbedding: a weightless RMSNorm on top of the embeddings. Kept separate+ # from the embedding matrix (the reference cannot fold it either -- the DFlash+ # drafter embeds without the norm); no weight, so it never appears in state_dict.+ self.embed_norm = GemmaRMSNorm(+ config.hidden_size, eps=config.rms_norm_eps, with_scale=False+ )+ self.layers = OPList(+ [MuseGlimmerDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]+ )+ # Final norm scales by the raw checkpoint weight (plain RMSNorm, not the+ # centered (1+w) form the decoder norms use).+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)++ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:+ x = self.embed_norm.forward(self.embed_tokens.forward(input_ids))+ for layer in self.layers.op_list:+ x = layer.forward(x)+ return self.norm.forward(x)+++class MuseGlimmerForCausalLM(BaseLLMModel):+ def __init__(self, config: ModelConfig):+ self.model = MuseGlimmerModel(config)+ self.lm_head = ParallelLMHead(+ num_embeddings=config.vocab_size,+ embedding_dim=config.hidden_size,+ tie_word_embeddings=config.tie_word_embeddings,+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,+ )+ self._output_multiplier = config.output_multiplier+ self._final_logit_softcapping = config.final_logit_softcapping+ super().__init__()++ def forward(self) -> torch.Tensor:+ output = self.model.forward(get_global_ctx().batch.input_ids)+ logits = self.lm_head.forward(output)+ # Gemma-style logit post-processing, with a pre-scale: T * tanh(logits * mult / T).+ if self._output_multiplier is not None:+ logits = logits * self._output_multiplier+ if self._final_logit_softcapping is not None:+ cap = self._final_logit_softcapping+ logits = torch.tanh(logits / cap) * cap+ return logits+++__all__ = ["MuseGlimmerForCausalLM"]diff --git a/python/freetoken/models/muse_glimmer/weight.py b/python/freetoken/models/muse_glimmer/weight.pynew file mode 100644index 00000000..8a613225--- /dev/null+++ b/python/freetoken/models/muse_glimmer/weight.py@@ -0,0 +1,138 @@+from __future__ import annotations++from typing import Iterator++import safetensors+import torch+from freetoken.distributed import get_tp_info+from freetoken.models.config import detect_compressed_tensors_nvfp4+from freetoken.models.loader import (+ CT_SCALE_SUFFIXES,+ ShardReader,+ ct_bf16_fuse,+ ct_nvfp4_fuse,+ iter_weight_files,+ nvfp4_parts_ct,+)+from freetoken.utils import cached_load_hf_config+from tqdm import tqdm++# Vision stack of the multimodal wrapper -- served text-only, always dropped.+_VISION_PREFIXES = (+ "model.vision_tower.",+ "model.vision_adapter.",+ "model.vision_projection.",+ "vision_tower.",+ "vision_adapter.",+ "vision_projection.",+)++# Fused projections, concatenated on the output dim in this exact order to match the+# model's merged-linear splits. The attention gate rides the q/k/v fusion (it is computed+# from the same layer input); ``.self_attn.gate_proj`` and the SwiGLU ``.mlp.gate_proj``+# are disambiguated by the full suffix.+_FUSIONS: dict[str, tuple[str, ...]] = {+ ".self_attn.qkvg_proj": (+ ".self_attn.q_proj", ".self_attn.k_proj", ".self_attn.v_proj", ".self_attn.gate_proj",+ ),+ ".mlp.gate_up_proj": (".mlp.gate_proj", ".mlp.up_proj"),+}+++def _rename(raw_name: str) -> str | None:+ """HF key -> FreeToken state-dict key, or None to skip (vision stack)."""+ if raw_name.startswith(_VISION_PREFIXES):+ return None+ if raw_name.startswith("model.language_model."):+ return "model." + raw_name[len("model.language_model.") :]+ if raw_name.startswith("language_model."):+ return "model." + raw_name[len("language_model.") :]+ return raw_name # lm_head.weight+++def iter_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ if not include_non_moe:+ return # dense model: there is no experts-only pass+ if get_tp_info().size > 1:+ raise NotImplementedError("muse_glimmer weight loading currently supports TP=1 only")++ if detect_compressed_tensors_nvfp4(cached_load_hf_config(model_path)):+ yield from _iter_weights_compressed_tensors(model_path, device)+ return++ fuse_buf: dict = {}+ for file in tqdm(+ iter_weight_files(model_path),+ desc="Loading weights",+ disable=not get_tp_info().is_primary(),+ ):+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:+ for raw_name in f.keys():+ name = _rename(raw_name)+ if name is None:+ continue+ tensor = f.get_tensor(raw_name)+ if name.endswith(".weight"):+ emit = ct_bf16_fuse(name[: -len(".weight")], tensor, fuse_buf, _FUSIONS)+ if emit is not None:+ yield from emit+ continue+ # All norms pass through raw: the decoder's centered (1+w) norms apply the+ # +1 at runtime (GemmaPlusOneRMSNorm), the final norm / qk norms need none.+ yield name, tensor++ assert not fuse_buf, f"Incomplete projection fusions: {list(fuse_buf.keys())}"+++def _iter_weights_compressed_tensors(+ model_path: str, device: torch.device+) -> Iterator[tuple[str, torch.Tensor]]:+ """Dense pass for the compressed-tensors NVFP4 checkpoint.++ Every text Linear (q/k/v/o, the attention gate, the MLP) is kept native NVFP4+ (W4A16): ``.weight`` (uint8 packed) + ``.weight_scale`` (fp8 block) + ``.weight_global``+ (fp16 per-row, the reciprocal of the stored quant-side global). q/k/v/gate fuse into+ ``qkvg_proj`` and gate/up into ``gate_up_proj`` on the output dim, each part keeping+ its own scales, so the fused FP4 weights are exact. Embeddings, norms and lm_head are+ bf16 (the checkpoint's ignore list). Scale lookups go through the shard-map reader:+ they can land in a different shard than their weight_packed."""+ nvfp4_buf: dict = {}+ reader = ShardReader(model_path, device)+ try:+ for file in tqdm(+ reader.files(),+ desc="Loading compressed-tensors weights",+ disable=not get_tp_info().is_primary(),+ ):+ for raw_name in reader.names_in(file):+ if raw_name.endswith(CT_SCALE_SUFFIXES):+ continue # consumed with their weight_packed+ name = _rename(raw_name)+ if name is None:+ continue+ if raw_name.endswith(".weight_packed"):+ base = name[: -len(".weight_packed")]+ parts = nvfp4_parts_ct(reader, raw_name[: -len(".weight_packed")])+ emit = ct_nvfp4_fuse(base, parts, nvfp4_buf, _FUSIONS)+ if emit is not None:+ yield from emit+ else: # standalone: o_proj, down_proj+ w, s, g = parts+ yield base + ".weight", w+ yield base + ".weight_scale", s+ yield base + ".weight_global", g+ continue+ yield name, reader.get_tensor(raw_name)+ finally:+ reader.close()++ assert not nvfp4_buf, f"Incomplete NVFP4 fusions: {list(nvfp4_buf.keys())}"+++__all__ = ["iter_weights"]diff --git a/python/freetoken/models/quant_linear.py b/python/freetoken/models/quant_linear.pynew file mode 100644index 00000000..6ba67a9e--- /dev/null+++ b/python/freetoken/models/quant_linear.py@@ -0,0 +1,78 @@+"""Quant-aware dense-linear factories, shared by the models that serve quantized+dense projections (qwen3_5_moe, muse_glimmer).++Maps the model's quant config (``expert_quant`` for the dense MLP / shared-expert path,+``attn_quant`` for attention + GatedDeltaNet projections) to the right ``BaseOP`` linear:+block-FP8, per-tensor-FP8 and NVFP4 implementations live under ``freetoken.kernel.triton``;+the bf16 fallback is the framework's TP-aware ``freetoken.layers``. Only the *dispatch*+(config -> layer class) lives here.+"""++from __future__ import annotations+++def make_col_merged_quant(expert_quant: str, attn_quant: str, in_f: int,+ output_sizes: list[int], has_bias: bool = False):+ """Column-merged linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""+ if expert_quant == "fp8_block":+ from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged++ return Fp8BlockColMerged(in_f, output_sizes, has_bias)+ if attn_quant == "fp8_pertensor":+ from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged++ return Fp8PerTensorColMerged(in_f, output_sizes, has_bias)+ if attn_quant == "nvfp4": # compressed-tensors W4A16 attention (q/k/v fused)+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged++ return Nvfp4DenseColMerged(in_f, output_sizes, has_bias)+ from freetoken.layers import LinearColParallelMerged++ return LinearColParallelMerged(in_f, output_sizes, has_bias=has_bias)+++def make_replicated_quant(expert_quant: str, attn_quant: str, in_f: int, out_f: int,+ has_bias: bool = False):+ """Replicated linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""+ if expert_quant == "fp8_block":+ from freetoken.kernel.triton.fp8_block_linear import Fp8BlockLinear++ return Fp8BlockLinear(in_f, out_f, has_bias)+ if attn_quant == "fp8_pertensor":+ from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear++ return Fp8PerTensorLinear(in_f, out_f, has_bias)+ if attn_quant == "nvfp4": # compressed-tensors W4A16 attention o_proj / GDN out_proj+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseLinear++ return Nvfp4DenseLinear(in_f, out_f, has_bias)+ from freetoken.layers import LinearReplicated++ return LinearReplicated(in_f, out_f, has_bias=has_bias)+++def make_replicated(config, in_f: int, out_f: int, has_bias: bool = False):+ """Config-driven replicated linear: ``Fp8BlockLinear`` under block-fp8, ``Fp8PerTensorLinear``+ under per-tensor-fp8 attention, ``Nvfp4DenseLinear`` under nvfp4, else ``LinearReplicated``."""+ return make_replicated_quant(+ getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),+ in_f, out_f, has_bias,+ )+++def make_col_merged(config, in_f: int, output_sizes: list[int], has_bias: bool = False):+ """Config-driven column-merged linear: ``Fp8BlockColMerged`` under block-fp8,+ ``Fp8PerTensorColMerged`` under per-tensor-fp8 attention, ``Nvfp4DenseColMerged`` under+ nvfp4, else ``LinearColParallelMerged``."""+ return make_col_merged_quant(+ getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),+ in_f, output_sizes, has_bias,+ )+++__all__ = [+ "make_col_merged_quant",+ "make_replicated_quant",+ "make_replicated",+ "make_col_merged",+]diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.pyindex 74bef262..dc16cfff 100644--- a/python/freetoken/models/qwen3_5_moe/config.py+++ b/python/freetoken/models/qwen3_5_moe/config.py@@ -7,6 +7,7 @@ from freetoken.models.config import (LinearGatedDeltaGroupConfig,ModelConfig,RotaryConfig,+ detect_compressed_tensors_nvfp4,)@@ -62,25 +63,9 @@ def _expert_quant(hf_config: Any) -> str:return "none"-def _compressed_tensors_nvfp4(hf_config: Any) -> bool:- """Detect a compressed-tensors NVFP4 (W4A16) checkpoint (e.g. dense Qwen3.6-27B).-- llm-compressor stores ``quant_method == "compressed-tensors"`` with a ``config_groups``- map whose ``weights`` are ``num_bits=4, type="float", group_size=16`` -- the NVFP4 layout- (``weight_packed`` uint8 + ``weight_scale`` fp8 block + scalar ``weight_global_scale``).- All ``targets: ["Linear"]`` are NVFP4 except the per-module ``ignore`` list (lm_head,- GDN in_proj_*, vision, mtp)."""- get = _quant_accessor(hf_config)- if get is None:- return False- if str(get("quant_method") or "").lower() != "compressed-tensors":- return False- groups = get("config_groups") or {}- for g in (groups.values() if isinstance(groups, dict) else []):- w = (g or {}).get("weights") or {}- if int(w.get("num_bits", 0) or 0) == 4 and str(w.get("type", "")).lower() == "float":- return True- return False+# Detection now lives in models/config.py (shared with muse_glimmer); weight.py imports+# it under this name.+_compressed_tensors_nvfp4 = detect_compressed_tensors_nvfp4def _lm_head_quant(hf_config: Any) -> str:diff --git a/python/freetoken/models/qwen3_5_moe/quant_linear.py b/python/freetoken/models/qwen3_5_moe/quant_linear.pyindex e57a5393..cd0443b6 100644--- a/python/freetoken/models/qwen3_5_moe/quant_linear.py+++ b/python/freetoken/models/qwen3_5_moe/quant_linear.py@@ -1,73 +1,15 @@-"""Quant-aware dense-linear factories for Qwen3.5/3.6.+"""Re-export of the shared quant-aware dense-linear factories (models/quant_linear.py).-Maps the model's quant config (``expert_quant`` for the dense MLP / shared-expert path,-``attn_quant`` for attention + GatedDeltaNet projections) to the right ``BaseOP`` linear:-block-FP8, per-tensor-FP8 and NVFP4 implementations live under ``freetoken.kernel.triton``-(shared across models); the bf16 fallback is the framework's TP-aware ``freetoken.layers``.-Only the *dispatch* (config -> layer class) is model glue and stays here.+The dispatch used to live here; muse_glimmer needing the same NVFP4/bf16 pair promoted+it to a model-agnostic home. Kept as a module so existing imports stay valid."""-from __future__ import annotations---def make_col_merged_quant(expert_quant: str, attn_quant: str, in_f: int,- output_sizes: list[int], has_bias: bool = False):- """Column-merged linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""- if expert_quant == "fp8_block":- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged-- return Fp8BlockColMerged(in_f, output_sizes, has_bias)- if attn_quant == "fp8_pertensor":- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged-- return Fp8PerTensorColMerged(in_f, output_sizes, has_bias)- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention (q/k/v fused)- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged-- return Nvfp4DenseColMerged(in_f, output_sizes, has_bias)- from freetoken.layers import LinearColParallelMerged-- return LinearColParallelMerged(in_f, output_sizes, has_bias=has_bias)---def make_replicated_quant(expert_quant: str, attn_quant: str, in_f: int, out_f: int,- has_bias: bool = False):- """Replicated linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""- if expert_quant == "fp8_block":- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockLinear-- return Fp8BlockLinear(in_f, out_f, has_bias)- if attn_quant == "fp8_pertensor":- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear-- return Fp8PerTensorLinear(in_f, out_f, has_bias)- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention o_proj / GDN out_proj- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseLinear-- return Nvfp4DenseLinear(in_f, out_f, has_bias)- from freetoken.layers import LinearReplicated-- return LinearReplicated(in_f, out_f, has_bias=has_bias)---def make_replicated(config, in_f: int, out_f: int, has_bias: bool = False):- """Config-driven replicated linear: ``Fp8BlockLinear`` under block-fp8, ``Fp8PerTensorLinear``- under per-tensor-fp8 attention, ``Nvfp4DenseLinear`` under nvfp4, else ``LinearReplicated``."""- return make_replicated_quant(- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),- in_f, out_f, has_bias,- )---def make_col_merged(config, in_f: int, output_sizes: list[int], has_bias: bool = False):- """Config-driven column-merged linear: ``Fp8BlockColMerged`` under block-fp8,- ``Fp8PerTensorColMerged`` under per-tensor-fp8 attention, ``Nvfp4DenseColMerged`` under- nvfp4, else ``LinearColParallelMerged``."""- return make_col_merged_quant(- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),- in_f, output_sizes, has_bias,- )-+from freetoken.models.quant_linear import (+ make_col_merged,+ make_col_merged_quant,+ make_replicated,+ make_replicated_quant,+)__all__ = ["make_col_merged_quant",diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.pyindex 79622c32..926df7c8 100644--- a/python/freetoken/models/qwen3_5_moe/weight.py+++ b/python/freetoken/models/qwen3_5_moe/weight.py@@ -9,7 +9,15 @@ import safetensorsimport torchfrom freetoken.distributed import get_tp_infofrom freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4-from freetoken.models.loader import drop_page_cache, iter_weight_files+from freetoken.models.loader import (+ CT_SCALE_SUFFIXES,+ ShardReader,+ ct_bf16_fuse,+ ct_nvfp4_fuse,+ drop_page_cache,+ iter_weight_files,+ nvfp4_parts_ct,+)from freetoken.models.nvfp4_banks import (Nvfp4ExpertSourceSpec,load_nvfp4_expert_source_banks,@@ -525,59 +533,15 @@ _CT_BF16_FUSE: dict[str, tuple[str, ...]] = {".linear_attn.in_proj_b", ".linear_attn.in_proj_a",),}-# Activation/quant scales consumed with weight_packed (or unused: this is W4A16, the input-# scales are for W4A4 which FreeToken does not run).-_CT_SCALE_SUFFIXES = (".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale")---def _nvfp4_parts_ct(f, raw_base: str):- """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16],- per-output-row global fp16 [O])`` for the W4A16 kernels. The native/dequant global is- ``1/weight_global_scale`` (compressed-tensors stores the quant-side global)."""- w = f.get_tensor(raw_base + ".weight_packed")- s = f.get_tensor(raw_base + ".weight_scale")- wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)- g = (1.0 / wg).to(torch.float16).expand(w.shape[0]).contiguous()- return w, s, g+# The scale suffixes and parts/fuse machinery are shared with muse_glimmer and live+# in models/loader.py.+_CT_SCALE_SUFFIXES = CT_SCALE_SUFFIXES+_nvfp4_parts_ct = nvfp4_parts_ct+_ct_bf16_fuse = ct_bf16_fusedef _ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict):- """Buffer a native NVFP4 fusion part ``(w, s, g)``; emit the concatenated native parts once- complete, ``[]`` while incomplete, ``None`` if ``base`` is not a fusion part."""- for fused_suffix, parts in _CT_NVFP4_FUSE.items():- for idx, part in enumerate(parts):- if base.endswith(part):- key = base[: -len(part)] + fused_suffix- slots = buf.setdefault(key, {})- slots[idx] = parts_tuple- if len(slots) < len(parts):- return []- del buf[key]- ws = [slots[i][0] for i in range(len(parts))]- ss = [slots[i][1] for i in range(len(parts))]- gs = [slots[i][2] for i in range(len(parts))]- return [- (key + ".weight", torch.cat(ws, dim=0)),- (key + ".weight_scale", torch.cat(ss, dim=0)),- (key + ".weight_global", torch.cat(gs, dim=0)),- ]- return None---def _ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict):- """Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]`` while- incomplete, ``None`` if ``base`` is not a part of any group in ``groups``."""- for fused_suffix, parts in groups.items():- for idx, part in enumerate(parts):- if base.endswith(part):- key = base[: -len(part)] + fused_suffix- slots = buf.setdefault(key, {})- slots[idx] = tensor- if len(slots) < len(parts):- return []- del buf[key]- return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))]- return None+ return ct_nvfp4_fuse(base, parts_tuple, buf, _CT_NVFP4_FUSE)def _iter_weights_compressed_tensors(@@ -612,13 +576,17 @@ def _iter_weights_compressed_tensors(tensor = tensor + 1.0 # (1 + weight) baked into the stored norm weightyield name, tensor- for file in tqdm(- iter_weight_files(model_path),- desc="Loading compressed-tensors weights",- disable=not tp_info.is_primary(),- ):- with safetensors.safe_open(file, framework="pt", device=str(device)) as f:- for raw_name in f.keys():+ # Scale lookups go through the shard-map reader: a weight_packed's quant scales+ # can land in a different shard than the packed weight (the Muse-Glimmer layer-49+ # shape; nothing prevents an llm-compressor Qwen export from splitting the same way).+ reader = ShardReader(model_path, device)+ try:+ for file in tqdm(+ reader.files(),+ desc="Loading compressed-tensors weights",+ disable=not tp_info.is_primary(),+ ):+ for raw_name in reader.names_in(file):if raw_name.startswith(("mtp.", "model.visual.", "visual.")):continueif raw_name.endswith(_CT_SCALE_SUFFIXES):@@ -631,7 +599,7 @@ def _iter_weights_compressed_tensors(if raw_name.endswith(".weight_packed"): # NVFP4 projectionbase = name[: -len(".weight_packed")]raw_base = raw_name[: -len(".weight_packed")]- w, s, g = _nvfp4_parts_ct(f, raw_base)+ w, s, g = _nvfp4_parts_ct(reader, raw_base)# GDN in_proj_* compute in bf16 (model contract) but some checkpoints# (e.g. sakamakismile/Qwen3.6-27B-NVFP4) quantize them too: dequant to# bf16 here and let the bf16 fusion assemble ``in_proj`` as usual.@@ -659,11 +627,13 @@ def _iter_weights_compressed_tensors(continueif name.endswith(".weight"):- yield from _emit_bf16_weight(name, f.get_tensor(raw_name))+ yield from _emit_bf16_weight(name, reader.get_tensor(raw_name))continue# A_log / dt_bias (kept fp32 by the model; the load downcast exempts them).- yield name, f.get_tensor(raw_name)+ yield name, reader.get_tensor(raw_name)+ finally:+ reader.close()assert not nvfp4_buf, f"Incomplete NVFP4 fusions: {list(nvfp4_buf.keys())}"assert not bf16_buf, f"Incomplete bf16 fusions: {list(bf16_buf.keys())}"diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 1e015187..0c033ca0 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -65,6 +65,15 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"freetoken.models.qwen3_5_moe","Qwen3_5MoEForCausalLM",),+ # Muse-Glimmer-30B (model_type muse_glimmer): multimodal wrapper config (text tower in+ # text_config, weights under model.language_model.); served text-only. Dense gated GQA+ # with a [SWA x3, full] pattern -- full layers are NoPE -- weightless qk norms, centered+ # (1+w) sandwich norms and softcapped logits; the NVFP4 release is compressed-tensors+ # W4A16 on every text Linear.+ "MuseGlimmerForConditionalGeneration": ModelSpec(+ "freetoken.models.muse_glimmer",+ "MuseGlimmerForCausalLM",+ ),"MistralForCausalLM": ModelSpec("freetoken.models.mistral","MistralForCausalLM",diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.pyindex 7bc3593c..b2857b75 100644--- a/python/freetoken/server/args.py+++ b/python/freetoken/server/args.py@@ -134,6 +134,8 @@ def parse_args(return "minimax_m3"if "minimax" in marker:return "minimax"+ if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker:+ return "muse_glimmer"if "gemma4" in marker:return "gemma4"if (@@ -187,6 +189,8 @@ def parse_args(return "minimax_m3"if "minimax" in marker:return "minimax"+ if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker:+ return "muse_glimmer"if "gemma4" in marker:return "gemma4"return None@@ -416,6 +420,7 @@ def parse_args("glm47","minimax","minimax_m3",+ "muse_glimmer","gpt_oss","gpt-oss",],@@ -428,13 +433,13 @@ def parse_args(default="auto",choices=["auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm",- "minimax", "minimax_m3", "gemma4",+ "minimax", "minimax_m3", "muse_glimmer", "gemma4",],help=("Reasoning parser that splits chain-of-thought into reasoning_content ""for OpenAI responses. 'auto' selects per model family (gpt-oss Harmony, "- "<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, gemma thought, "- "dsv4); 'off' disables it."+ "<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, ATEM to=self "+ "channels for muse-glimmer, gemma thought, dsv4); 'off' disables it."),)diff --git a/python/freetoken/server/function_call_parser.py b/python/freetoken/server/function_call_parser.pyindex 0c22b8e0..0804dcc3 100644--- a/python/freetoken/server/function_call_parser.py+++ b/python/freetoken/server/function_call_parser.py@@ -27,6 +27,16 @@ from partial_json_parser.core.options import Allowfrom pydantic import BaseModelfrom .api_models import Function, Tool+from .reasoning_parser import (+ ATEM_CLOSING_TOKENS,+ ATEM_HEADER_SPAN,+ ATEM_INLINE_HEADER_RE,+ ATEM_MESSAGE,+ ATEM_RECIPIENT_RE,+ ATEM_START,+ atem_hold_len,+ atem_marker_inside,+)try:import orjson@@ -68,6 +78,7 @@ TOOLS_TAG_LIST = ["<|DSML|function_calls>","<|DSML|tool_calls>","<|DSML|invoke",+ "<atem:function_calls>",]@@ -680,6 +691,11 @@ class InvokeParamStreamMixin:return self._convert_param_value(raw, key, self._ps_param_config, "")return _parse_loose_json_value(raw)+ def _ps_canonical_name(self, name: str) -> str:+ """Hook: normalize an invoke's function name before validation (identity by+ default; muse_glimmer collapses template-doubled ``name.name`` recipients)."""+ return name+def _ps_trim_leading(self, text: str) -> str:if self._ps_trim_single:return text[1:] if text[:1] and text[:1] in self._ps_trim else text@@ -754,7 +770,7 @@ class InvokeParamStreamMixin:m = self._ps_invoke_open_re.search(buf, inv)if m is None:break # invoke tag still streaming- func_name = m.group(1).strip()+ func_name = self._ps_canonical_name(m.group(1).strip())if self.current_tool_id == -1:self.current_tool_id = 0while len(self.prev_tool_call_arr) <= self.current_tool_id:@@ -3042,6 +3058,460 @@ class GptOssDetector(BaseFormatDetector):+class MuseGlimmerDetector(InvokeParamStreamMixin, BaseFormatDetector):+ """Detector for Muse Glimmer's ATEM tool-call protocol.++ Wire format (inside a ``<|start|>assistant to=<tool><|message|> ... <|eot|>``+ channel; the ``MuseGlimmerReasoningParser`` upstream preserves such blocks+ verbatim and unwraps everything else):++ ```+ <atem:function_calls>+ <atem:invoke name="tool.fn">+ <atem:parameter name="param">value</atem:parameter>+ </atem:invoke>+ </atem:function_calls>+ ```++ The invoke/parameter body is the InvokeParamStreamMixin grammar; per the+ template's own contract values are parsed with regexes (not strict XML) and+ string values are NOT whitespace-trimmed (``_ps_trim = ""``). A channel layer+ around the mixin swallows the ``<|start|>...<|message|>`` headers and the+ ``<|eot|>/<|eom|>`` terminators, streams ``to=user`` bodies as text, and drops+ ``to=self`` bodies (they only reach this detector when no reasoning parser+ runs upstream). Channels also open with a mid-stream headerless+ ``to=X<|message|>`` switch (the model leaves a channel without ``<|eom|>``).++ ``header_open`` initializes the detector from the prompt it continues:+ True when the detector receives the raw turn bytes directly (no muse+ reasoning parser stacked above), whose templated prompt ends with+ ``<|start|>assistant`` -- a synthetic ``<|start|>`` is seeded so the bare+ first header goes through the ordinary full-header machinery. False (the+ default) downstream of ``MuseGlimmerReasoningParser``, which delivers tool+ slices with their full headers and everything else already classified.++ ATEM markup is EXECUTED only inside a tool-recipient channel (vLLM's rule).+ A block quoted in a ``to=user`` body -- or the system prompt's own ATEM+ example echoed back -- renders as plain text instead of becoming a real+ call.+ """++ # <atem:function_calls> is not a unique tool opener under the channel-scoped+ # execution rule (a to=user body may quote it), so no checkpoint anchor.+ toolcall_opener = None+ _ps_trim = "" # "spaces for string values are not stripped" (chat-template contract)+ _ps_missing_type = "string"+ accepts_header_open = True # FunctionCallParser passes the turn-start state++ def __init__(self, header_open: bool = False):+ super().__init__()+ self.bot_token = "<atem:function_calls>"+ self.eot_token = "</atem:function_calls>"+ self.tool_call_separator = "\n"++ # InvokeParamStreamMixin grammar+ self._ps_outer_open = "<atem:function_calls>"+ self._ps_outer_close = "</atem:function_calls>"+ self._ps_invoke_open_prefix = "<atem:invoke"+ self._ps_invoke_open_re = re.compile(r'<atem:invoke name="([^"]*)">')+ self._ps_invoke_close = "</atem:invoke>"+ self._ps_param_open_prefix = "<atem:parameter"+ self._ps_param_open_re = re.compile(r'<atem:parameter name="([^"]*)">')+ self._ps_param_close = "</atem:parameter>"+ self._ps_reset()++ # Channel layer state: "text" (to=user / outside channels) | "skip"+ # (non-user, non-tool channel body) | "tool" (tool channel body,+ # delegated to the mixin).+ self._ch_mode = "text"+ self._header_open = header_open+ # The seed is synthetic: if it is ever ruled a non-header, the marker+ # text itself is not delivered -- the model never emitted it.+ self._buffer = ATEM_START if header_open else ""+ self._synthetic_open = header_open+ # A channel boundary fired mid-invoke: until the NEXT boundary, incoming+ # text is that broken channel's residue (raw ATEM markup), not content.+ self._truncated_channel = False+ # Dropped tool-channel prose, accumulated for ONE warning per channel+ # (a per-fragment warning logs once per generated character).+ self._dropped_prose: List[str] = []++ def has_tool_call(self, text: str) -> bool:+ return self.bot_token in text++ def block_close_tokens(self) -> tuple:+ return (self.eot_token,) + ATEM_CLOSING_TOKENS++ def _ps_canonical_name(self, name: str) -> str:+ """The template renders a bare-name tool's recipient namespace as+ ``name.*``, so the model emits ``get_weather.get_weather``: collapse the+ doubled form to the registered head (never when the doubled form itself+ is a registered tool)."""+ head, dot, tail = name.partition(".")+ indices = getattr(self, "_tool_indices", {})+ if dot and head == tail and head in indices and name not in indices:+ return head+ return name++ # ------------------------------------------------------------------+ # streaming+ # ------------------------------------------------------------------+ def _enter_recipient(self, recipient: str) -> None:+ if recipient == "self":+ self._ch_mode = "skip"+ elif recipient == "user":+ self._ch_mode = "text"+ else:+ self._ch_mode = "tool"++ @staticmethod+ def _channel_boundary(buf: str):+ """Earliest channel boundary in ``buf``: ``(pos, kind, payload)`` with kind+ "closer" (payload = token), "start", or "inline" (payload = the headerless+ ``to=X<|message|>`` match); ``(-1, None, None)`` when none is present."""+ best = (len(buf) + 1, None, None)+ for tok in ATEM_CLOSING_TOKENS:+ pos = buf.find(tok)+ if pos != -1 and pos < best[0]:+ best = (pos, "closer", tok)+ pos = buf.find(ATEM_START)+ if pos != -1 and pos < best[0]:+ best = (pos, "start", None)+ m = ATEM_INLINE_HEADER_RE.search(buf)+ if m is not None and m.start() < best[0]:+ best = (m.start(), "inline", m)+ if best[1] is None:+ return -1, None, None+ return best++ def _finalize_truncated_invoke(self, reason: str = "tool channel closed") -> List[ToolCallItem]:+ """The channel (or the whole stream) ended while the ATEM machinery was+ mid-flight. If an invoke already streamed its Start, close its argument+ JSON so the streamed fragments stay valid (never emit broken arguments),+ ledger the completed parameters, and advance the ordinal so the next+ call cannot merge into this one. Anything less than an open invoke is+ markup debris: warn and reset."""+ self._warn_dropped_prose()+ mode = getattr(self, "_ps_mode", "idle")+ if mode == "idle":+ return []+ self._truncated_channel = True # what follows is the broken channel's residue+ calls: List[ToolCallItem] = []+ if mode in ("invoke", "pstr", "pbuf"):+ ledger = self.streamed_args_for_tool[self.current_tool_id]+ if mode == "pstr":+ frag = '"}' # the ledger ends inside an open string value+ elif ledger:+ frag = "}" # ends after a complete "key": value pair+ else:+ frag = "{}" # invoke opened, no parameter emitted yet+ self.streamed_args_for_tool[self.current_tool_id] += frag+ calls.append(+ ToolCallItem(tool_index=self.current_tool_id, name=None, parameters=frag)+ )+ try:+ parsed = json.loads(self.streamed_args_for_tool[self.current_tool_id])+ except (json.JSONDecodeError, ValueError):+ parsed = {}+ self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (+ parsed if isinstance(parsed, dict) else {}+ )+ self.streamed_args_for_tool[self.current_tool_id] = ""+ self.current_tool_id += 1+ while len(self.streamed_args_for_tool) <= self.current_tool_id:+ self.streamed_args_for_tool.append("")+ logger.warning("muse_glimmer: %s mid-invoke; arguments truncated", reason)+ else: # block / invoke_skip / pskip: no open call, just partial markup+ logger.warning("muse_glimmer: %s mid-block; partial tool markup dropped", reason)+ self._ps_reset()+ return calls++ def finalize_streaming(self) -> List[ToolCallItem]:+ """End-of-stream hook (FunctionCallParser.finalize_stream): the generation+ ran out (max_tokens) while an invoke was still streaming -- close it the+ same way a channel boundary would, so the client's concatenated argument+ fragments end as valid JSON instead of an unterminated string."""+ return self._finalize_truncated_invoke("generation ended")++ # The truncated channel's trailing-markup shapes: what a broken-off invoke+ # leaves behind when the model closes the tags it opened.+ _CLOSING_MARKUP = ("</atem:parameter>", "</atem:invoke>", "</atem:function_calls>")++ @classmethod+ def _channel_residue_end(cls, buf: str) -> tuple[int, bool]:+ """Length of the leading run of ATEM closing markup (closing tags plus+ whitespace) in ``buf``, and whether the scan DECIDED. False means the+ buffer ends inside the run or a partial tag: hold for more input.++ Known cost of shape-based dropping: a reply that literally BEGINS with a+ complete closing tag ("</atem:parameter> is the closing tag") loses that+ leading tag. Partial-tag-shaped prose ("</a", "</atem") is held and+ released intact, so no ordinary word can be eaten -- only a verbatim+ full tag right after a truncated invoke, which is indistinguishable from+ the residue this filter exists to drop."""+ i = 0+ while i < len(buf):+ if buf[i] in " \t\r\n":+ i += 1+ continue+ matched = False+ for tag in cls._CLOSING_MARKUP:+ if buf.startswith(tag, i):+ i += len(tag)+ matched = True+ break+ if matched:+ continue+ rest = buf[i:]+ if any(tag.startswith(rest) for tag in cls._CLOSING_MARKUP):+ return i, False # a partial closing tag at the end: hold+ return i, True # first non-markup character: the residue ended+ return i, False # consumed the whole buffer; the run may continue++ def _drop_channel_prose(self, text: str) -> None:+ """Text inside a tool channel that is not ATEM markup is discarded (only+ tool-recipient markup is executed; the template puts nothing else there).+ Accumulated and logged ONCE per channel by ``_warn_dropped_prose`` --+ token-by-token serving would otherwise warn per generated character."""+ if text and text.strip():+ self._dropped_prose.append(text)++ def _warn_dropped_prose(self) -> None:+ if not self._dropped_prose:+ return+ dropped = "".join(self._dropped_prose)+ self._dropped_prose = []+ logger.warning(+ "muse_glimmer: dropped %d chars of non-markup text inside a tool channel: %.120r",+ len(dropped),+ dropped,+ )++ def _run_mixin(self, atem_part: str, remainder: str, tools: List[Tool]) -> StreamingParseResult:+ """Feed ``atem_part`` through the invoke/parameter machinery; whatever the+ mixin leaves unconsumed stays buffered ahead of ``remainder``."""+ self._buffer = atem_part+ result = InvokeParamStreamMixin.parse_streaming_increment(self, "", tools)+ self._buffer += remainder+ return result++ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult:+ self._buffer += new_text+ if not hasattr(self, "_tool_indices"):+ self._tool_indices = self._get_tool_indices(tools)+ normal_parts: List[str] = []+ calls: List[ToolCallItem] = []+ while True:+ buf = self._buffer+ if not buf:+ break++ if self._ch_mode == "text":+ if calls:+ break # text after a call defers to the next step (wire order)+ if self._truncated_channel:+ # After a mid-invoke channel break, drop the broken channel's+ # trailing markup BY SHAPE (closing tags + whitespace) and+ # clear at the first non-markup character -- a real reply+ # following the break must flow, not be swallowed until some+ # later boundary that the production pipeline never delivers.+ consumed, decided = self._channel_residue_end(buf)+ if consumed:+ self._buffer = buf[consumed:]+ if not decided:+ break+ self._truncated_channel = False+ self._warn_dropped_prose()+ continue+ pos, kind, payload = self._channel_boundary(buf)+ if pos == -1:+ hold = atem_hold_len(buf)+ release = buf[: len(buf) - hold] if hold else buf+ if release:+ normal_parts.append(release)+ self._buffer = buf[len(release):]+ break+ if pos > 0:+ normal_parts.append(buf[:pos])+ self._buffer = buf[pos:]+ continue+ if kind == "closer":+ self._buffer = buf[len(payload):] # stray terminator: drop+ continue+ if kind == "inline":+ self._enter_recipient(payload.group(1))+ self._buffer = buf[payload.end():]+ continue+ # kind == "start": parse the channel header. While the header-open+ # seed is unconsumed it is the leftmost marker, so each consuming+ # branch owns the synthetic flag: a synthetic marker ruled a+ # non-header is dropped, never delivered (the model never emitted it).+ synthetic = self._synthetic_open+ msg = buf.find(ATEM_MESSAGE)+ if atem_marker_inside(+ buf, len(ATEM_START), msg if msg != -1 else len(buf)+ ):+ # A control token inside the candidate: headers never contain+ # markers, so this <|start|> is literal text (a synthetic+ # seed is simply dropped) -- mirrors the reasoning parser.+ if not synthetic:+ normal_parts.append(ATEM_START)+ self._synthetic_open = False+ self._buffer = buf[len(ATEM_START):]+ continue+ if msg == -1:+ # +len(<|message|>) slack: a protocol-legal header whose marker+ # is still mid-arrival must not be cut at the nominal span.+ if len(buf) > len(ATEM_START) + ATEM_HEADER_SPAN + len(ATEM_MESSAGE):+ # Past a plausible header span the marker cannot open a+ # header anymore: release it as literal text and move on+ # (mirrors the reasoning parser's bound).+ if not synthetic:+ normal_parts.append(ATEM_START)+ self._synthetic_open = False+ self._buffer = buf[len(ATEM_START):]+ continue+ break # header still streaming+ if msg - len(ATEM_START) > ATEM_HEADER_SPAN:+ # The found <|message|> is too far away to belong to THIS+ # marker (a stray literal <|start|> followed by junk, then the+ # NEXT segment's real header): the marker is literal text.+ if not synthetic:+ normal_parts.append(ATEM_START)+ self._synthetic_open = False+ self._buffer = buf[len(ATEM_START):]+ continue+ m = ATEM_RECIPIENT_RE.search(buf[len(ATEM_START):msg])+ self._enter_recipient(m.group(1) if m else "user")+ self._synthetic_open = False+ self._buffer = buf[msg + len(ATEM_MESSAGE):]+ continue++ pos, kind, payload = self._channel_boundary(buf)++ if self._ch_mode == "skip":+ if pos == -1:+ hold = atem_hold_len(buf)+ self._buffer = buf[len(buf) - hold:] if hold else ""+ break+ # Drop the skipped body; transition exactly like text mode would.+ self._buffer = buf[pos:]+ self._ch_mode = "text"+ continue++ # self._ch_mode == "tool": the body up to the channel boundary belongs+ # to the ATEM machinery; text the mixin releases inside the channel is+ # not executed (only tool-recipient markup is) and is dropped.+ if pos == -1:+ hold = atem_hold_len(buf)+ atem_part = buf[: len(buf) - hold] if hold else buf+ if not atem_part:+ break+ result = self._run_mixin(atem_part, buf[len(atem_part):], tools)+ self._drop_channel_prose(result.normal_text)+ calls.extend(result.calls)+ if self._buffer == buf:+ break # no progress: the mixin is holding a partial marker+ continue+ result = self._run_mixin(buf[:pos], buf[pos:], tools)+ self._drop_channel_prose(result.normal_text)+ calls.extend(result.calls)+ if self._buffer != buf:+ # The mixin advanced (it stops after each call's close, wire-order):+ # more blocks may still precede the boundary -- a second invoke block+ # in the same undrained buffer must parse, not vanish as debris.+ continue+ # Inert: nothing before the boundary the machinery can still consume.+ # Finalize a mid-flight invoke (its Start already streamed) instead of+ # letting the next channel's call merge into it, then transition; any+ # residue ahead of the boundary is markup debris and drops with it.+ calls.extend(self._finalize_truncated_invoke())+ if kind == "closer":+ self._buffer = buf[pos + len(payload):]+ self._ch_mode = "text"+ # NOTE: the truncation mark (when finalize set it) survives the+ # closer on purpose: on the production pipeline the broken+ # channel's markup residue arrives AFTER the reasoning parser's+ # synthetic terminator. Text mode clears the mark by SHAPE -- at+ # the first non-markup character -- so a real reply flows+ # immediately while trailing closing tags are dropped.+ elif kind == "inline":+ self._enter_recipient(payload.group(1))+ self._buffer = buf[payload.end():]+ else: # abutting <|start|>: reprocess it in text mode+ self._buffer = buf[pos:]+ self._ch_mode = "text"+ continue+ return StreamingParseResult(normal_text="".join(normal_parts), calls=calls)++ def finish_streaming(self) -> str:+ self._warn_dropped_prose()+ residual, self._buffer = self._buffer, ATEM_START if self._header_open else ""+ ch_mode, self._ch_mode = self._ch_mode, "text"+ synthetic, self._synthetic_open = self._synthetic_open, self._header_open+ truncated, self._truncated_channel = self._truncated_channel, False+ ps_mode = getattr(self, "_ps_mode", "idle")+ self._ps_reset()+ if ch_mode != "text" or ps_mode != "idle" or truncated:+ return ""+ if synthetic:+ residual = residual[len(ATEM_START):] # the seed: never model output+ # At end of stream a <|start|> that never received its <|message|> is NOT+ # a header: deliver the text, drop only the marker(s). Any capped discard+ # here diverged from the layer above -- its span runs on raw bytes while+ # this layer sees closer-stripped text, so the two edges can never agree;+ # removing the drop window entirely makes the agreement trivial.+ residual = residual.replace(ATEM_START, "")+ for tok in ATEM_CLOSING_TOKENS:+ residual = residual.replace(tok, "")+ if self.prev_tool_call_arr and residual.strip() == "":+ return ""+ return residual++ # ------------------------------------------------------------------+ # one-shot+ # ------------------------------------------------------------------+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:+ """One-shot parse by replaying the streaming machinery on a fresh clone,+ so both paths share one definition of channel classification (a raw+ ``to=self`` body never leaks into content), execution scoping and value+ semantics. Streamed fragments are reassembled into complete calls; a+ call truncated by end-of-input closes from the parse ledger, exactly as+ the serving layer would."""+ clone = type(self)(header_open=self._header_open)+ raw_calls: List[ToolCallItem] = []+ normal_parts: List[str] = []+ result = clone.parse_streaming_increment(text, tools)+ normal_parts.append(result.normal_text)+ raw_calls.extend(result.calls)+ while True: # drain wire-order deferrals (one call boundary per step)+ result = clone.parse_streaming_increment("", tools)+ if not result.normal_text and not result.calls:+ break+ normal_parts.append(result.normal_text)+ raw_calls.extend(result.calls)+ raw_calls.extend(clone._finalize_truncated_invoke("input ended")) # mid-invoke+ normal_parts.append(clone.finish_streaming())++ order: List[int] = []+ by_index: Dict[int, Dict[str, str]] = {}+ for item in raw_calls:+ if item.name is not None and item.tool_index not in by_index:+ by_index[item.tool_index] = {"name": item.name, "params": ""}+ order.append(item.tool_index)+ if item.parameters and item.tool_index in by_index:+ by_index[item.tool_index]["params"] += item.parameters+ calls = [+ ToolCallItem(+ tool_index=ordinal,+ name=by_index[idx]["name"],+ parameters=by_index[idx]["params"] or "{}",+ )+ for ordinal, idx in enumerate(order)+ ]+ return StreamingParseResult(normal_text="".join(normal_parts).strip(), calls=calls)++class FunctionCallParser:"""Parser for function/tool calls in model outputs.@@ -3061,18 +3531,24 @@ class FunctionCallParser:"minimax": MiniMaxDetector,"minimax_m3": MiniMaxM3Detector,"mistral": MistralDetector,+ "muse_glimmer": MuseGlimmerDetector,"qwen": Qwen25Detector,"qwen25": Qwen25Detector,"qwen3_coder": Qwen3CoderDetector,}- def __init__(self, tools: List[Tool], tool_call_parser: str):+ def __init__(self, tools: List[Tool], tool_call_parser: str, turn_starts_open: bool = False):detector: Type[BaseFormatDetector] = Nonedetector_class = self.ToolCallParserEnum.get(tool_call_parser)- if detector_class:- detector = detector_class()- else:+ if detector_class is None:raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")+ if getattr(detector_class, "accepts_header_open", False):+ # Turn-start parse state read from the prompt: the templated chat+ # prompt ends inside a channel header (``<|start|>assistant``), so a+ # detector that receives the raw turn bytes starts header-open.+ detector = detector_class(header_open=turn_starts_open)+ else:+ detector = detector_class()self.detector = detectorself.tools = _coerce_tools(tools)@@ -3211,10 +3687,21 @@ class FunctionCallParser:arr = self.detector.prev_tool_call_arrif 0 <= tool_ordinal < len(arr):args = arr[tool_ordinal].get("arguments")- if args:+ # `is not None`, not truthiness: {} is a legitimate ledger entry (an+ # empty-argument call) and must serialize instead of falling through.+ if args is not None:return json.dumps(args, ensure_ascii=False)return None+ def finalize_stream(self) -> List[ToolCallItem]:+ """End-of-stream finalize: argument fragments that CLOSE a call cut off+ mid-arguments, from detectors that can (muse_glimmer closes the streamed+ JSON so the fragments the client concatenated stay valid). The serving+ layer routes these before it closes the open call; [] for detectors+ without the hook."""+ finalize = getattr(self.detector, "finalize_streaming", None)+ return finalize() if finalize is not None else []+SUPPORTED_TOOL_CALL_PARSERS = list(FunctionCallParser.ToolCallParserEnum.keys())diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.pyindex 1fc2b02b..d6689680 100644--- a/python/freetoken/server/generation.py+++ b/python/freetoken/server/generation.py@@ -388,6 +388,18 @@ def _leaked_special_tokens(state: Any) -> list[str]:return DSV4_SPECIAL_TOKENS if getattr(state.config, "reasoning_parser", None) == "deepseekv32" else []+def _make_tool_parser(spec: GenSpec, state: Any) -> FunctionCallParser:+ """Build the tool-call parser with its turn-start state read from the prompt:+ a muse detector receives the raw turn bytes (opened by the template's+ ``<|start|>assistant``) only when its reasoning parser is not stacked above,+ which otherwise delivers tool slices with full headers."""+ return FunctionCallParser(+ spec.parser_tools or [],+ getattr(state.config, "tool_call_parser", "llama3"),+ turn_starts_open=getattr(state.config, "reasoning_parser", None) != "muse_glimmer",+ )++def _parse_tool_response(text: str,spec: GenSpec,@@ -397,7 +409,7 @@ def _parse_tool_response(return Noneif not any(tag in text for tag in TOOLS_TAG_LIST):return None- parser = FunctionCallParser(spec.parser_tools or [], getattr(state.config, "tool_call_parser", "llama3"))+ parser = _make_tool_parser(spec, state)result = parser.parse_non_stream(text)if not result.calls:return None@@ -548,9 +560,7 @@ async def _generate_events_impl(uid: int, spec: GenSpec, state: Any) -> AsyncItetool_parser: FunctionCallParser | None = Noneif parse_tools:try:- candidate = FunctionCallParser(- spec.parser_tools or [], getattr(state.config, "tool_call_parser", "llama3")- )+ candidate = _make_tool_parser(spec, state)except ValueError:candidate = None # unsupported parser name: keep the buffered path's behaviorif candidate is not None and candidate.supports_streaming():@@ -678,9 +688,15 @@ async def _generate_events_impl(uid: int, spec: GenSpec, state: Any) -> AsyncItefinish_reason = engine_finish_reason or "stop"if tool_parser is not None:- # End-of-stream drain: close a call whose end marker never arrived (truncated- # generation), best-effort recover a call cut off inside an unterminated tag- # block, then release text still held back for tag disambiguation.+ # End-of-stream drain: let the detector finalize a call cut off mid-arguments+ # (closing fragments keep the client's concatenated JSON valid), close a call+ # whose end marker never arrived (truncated generation), best-effort recover a+ # call cut off inside an unterminated tag block, then release text still held+ # back for tag disambiguation.+ for frag in tool_parser.finalize_stream():+ if open_call is not None and frag.parameters:+ open_call["params"] += frag.parameters+ yield ToolCallArgsDelta(tool_index=open_call["ordinal"], fragment=frag.parameters)done = _close_open_call()if done is not None:yield donediff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.pyindex 0045cd7f..8ce8009a 100644--- a/python/freetoken/server/model_meta.py+++ b/python/freetoken/server/model_meta.py@@ -13,11 +13,11 @@ from typing import Any, Tuplefrom freetoken.tokenizer.effort import (EFFORT_SCALE,- OPENAI_EFFORT_TRIPLE,THINKING_ADAPTIVE_KWARGS,THINKING_OFF_KWARGS,THINKING_ON_KWARGS,ThinkingProfile,+ effective_efforts,)@@ -38,8 +38,8 @@ def derive_think_gears(controls -- the checkpoint owns this knowledge; nothing here is keyed bymodel family. ``None`` when there is nothing controllable to offer.- A template that grades effort without validating it gets the OpenAI triple- (the only vocabulary such a template is known to understand); an always-on+ A template that grades effort without validating it gets the graded ladder+ (its trained levels; never-advertised dialects quantize onto it); an always-onmodel with a reasoning parser but no observable knob shows a single "on"gear so clients can still label the state."""efforts = profile.efforts@@ -53,11 +53,10 @@ def derive_think_gears(kwargs["adaptive"] = dict(THINKING_ADAPTIVE_KWARGS)if efforts.consumes_effort:- names = (- [n for n in efforts.supported if n in EFFORT_SCALE]- if efforts.validates- else list(OPENAI_EFFORT_TRIPLE)- )+ # The served vocabulary: a validating template's own probed set; a+ # non-validating grader gets the graded ladder (incl. xhigh -- muse's+ # model card recommends it for agentic use) rather than the OpenAI triple.+ names = [n for n in effective_efforts(efforts) if n in EFFORT_SCALE]for name in sorted(names, key=lambda n: EFFORT_SCALE[n]):gears.append(name)gear_kwargs = dict(THINKING_ON_KWARGS) if profile.toggleable else {}diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.pyindex 3fb74eec..b4becd26 100644--- a/python/freetoken/server/openai_api.py+++ b/python/freetoken/server/openai_api.py@@ -663,9 +663,12 @@ async def _effort_fields(state: Any) -> tuple[list[str] | None, str | None]:profile = await asyncio.to_thread(manager.effort_profile)except Exception: # noqa: BLE001 -- metadata only; the generation path reports real faultsreturn None, None- if not profile.consumes_effort:+ from freetoken.tokenizer.effort import effective_efforts++ served = effective_efforts(profile)+ if not served:return None, None- ordered = sorted(profile.supported, key=lambda name: -EFFORT_SCALE.get(name, 0.0))+ ordered = sorted(served, key=lambda name: -EFFORT_SCALE.get(name, 0.0))return ordered, profile.defaultdiff --git a/python/freetoken/server/reasoning_parser.py b/python/freetoken/server/reasoning_parser.pyindex 2d5e3d29..6788675c 100644--- a/python/freetoken/server/reasoning_parser.py+++ b/python/freetoken/server/reasoning_parser.py@@ -18,6 +18,7 @@ preserves the block for the tool-call parser.from __future__ import annotations+import refrom dataclasses import dataclassfrom typing import Dict, List, Optional, Tuple, Type@@ -546,6 +547,329 @@ class GemmaThoughtReasoningParser(BaseReasoningParser):)+# Muse Glimmer ATEM protocol control tokens. Generation starts right after the+# template's ``<|start|>assistant``, so the FIRST segment arrives as a bare header+# continuation (`` to=self<|message|>...``) -- the parser is constructed+# header-open and seeds a synthetic ``<|start|>`` so those bytes go through the+# ordinary full-header machinery instead of a duplicate guessing path. Later+# segments open with a full ``<|start|>assistant to=X<|message|>`` -- or, when+# the model leaves a channel without emitting ``<|eom|>`` first, with a bare+# ``to=X<|message|>`` (a complete match of that shape is a segment boundary;+# vLLM applies the same rule). ``<|eom|>`` separates segments within one turn;+# ``<|eot|>`` / ``<|end_of_text|>`` are the stops.+ATEM_START = "<|start|>"+ATEM_MESSAGE = "<|message|>"+ATEM_CLOSING_TOKENS = ("<|eot|>", "<|eom|>", "<|end_of_text|>")+ATEM_ALL_TOKENS = (ATEM_START, ATEM_MESSAGE) + ATEM_CLOSING_TOKENS+# Recipient names cap at 64 chars in EVERY header shape (inline switches and the+# bare stream-start header use the same bound below), so streaming and one-shot+# agree on which recipients are valid and >64 degrades identically everywhere.+ATEM_RECIPIENT_RE = re.compile(r"to=([^\s<]{1,64})")+# A complete headerless channel switch: ``to=<name>`` abutting ``<|message|>``.+# The name length is capped so the streaming hold-back below can bound how much+# tail it withholds while a potential switch is still arriving.+ATEM_INLINE_HEADER_RE = re.compile(r"to=([^\s<]{1,64})<\|message\|>")+# Lookback window for the streaming hold-back's inline-switch scan: a headerless+# ``to=<name><|message|>`` switch can occupy at most this many trailing bytes.+_ATEM_HOLD_WINDOW = 96+# Longest span after "<|start|>" a real header can occupy before its <|message|>+# arrives ("assistant to=<name>" + slack). Past this, the marker cannot open a+# header anymore and is released as literal content instead of held forever.+ATEM_HEADER_SPAN = 128+++def atem_marker_inside(text: str, start: int, end: int) -> bool:+ """True when a complete ATEM control token lies fully inside ``text[start:end]``.+ A channel header can never contain one, so a header candidate with a marker+ before its ``<|message|>`` is not a header."""+ for tok in ATEM_ALL_TOKENS:+ if tok == ATEM_MESSAGE:+ continue # the terminator being sought, not a header byte+ if text.find(tok, start, end) != -1:+ return True+ return False+++def _longest_atem_partial_suffix(text: str) -> int:+ """Length of the longest suffix of ``text`` that is a *proper* prefix of any ATEM+ control token (the Harmony hold-back, for Muse Glimmer's marker set)."""+ best = 0+ for tok in ATEM_ALL_TOKENS:+ for k in range(min(len(tok) - 1, len(text)), best, -1):+ if text.endswith(tok[:k]):+ best = k+ break+ return best+++def atem_hold_len(text: str) -> int:+ """Streaming hold-back: the longest suffix of ``text`` that could still grow into+ an ATEM marker OR a headerless ``to=<name><|message|>`` channel switch. Withheld+ text is re-examined on the next chunk, so an emitted body never has to shrink+ when the boundary completes."""+ best = _longest_atem_partial_suffix(text)+ window = text[-_ATEM_HOLD_WINDOW:]+ i = window.rfind("to=")+ if i != -1:+ tail = window[i:]+ if re.fullmatch(r"to=[^\s<]{0,64}", tail):+ best = max(best, len(tail)) # name (or "to=") still streaming+ else:+ m = re.fullmatch(r"to=([^\s<]{1,64})(.+)", tail, re.DOTALL)+ if m and ATEM_MESSAGE.startswith(m.group(2)):+ best = max(best, len(tail)) # partial <|message|> after the name+ for prefix in ("t", "to"): # proper prefixes of "to=" (one-chunk delay at most)+ if len(prefix) > best and text.endswith(prefix):+ best = len(prefix)+ return best+++class MuseGlimmerReasoningParser(BaseReasoningParser):+ """Reasoning parser for Muse Glimmer's ATEM channel output.++ Routes ``to=self`` segments to ``reasoning_text`` and ``to=user`` (or+ recipient-less) segments, unwrapped, to ``normal_text``. A tool-recipient+ segment (``to=<tool>``, carrying an ``<atem:function_calls>`` block) is+ preserved verbatim (header included) in ``normal_text`` for the downstream+ ``MuseGlimmerDetector`` -- when the channel ends without its own terminator+ (an abutting ``<|start|>`` header or a headerless switch), a synthetic+ ``<|eom|>`` is appended so the detector always receives a delimited block.+ Segments open with ``<|start|>...`` headers or with a headerless+ ``to=X<|message|>`` switch (the model leaves a channel without ``<|eom|>``;+ vLLM's rule). The parser is initialized from the prompt it continues:+ every request that reaches it is a templated chat generation whose prompt+ ends with ``<|start|>assistant``, so ``header_open=True`` (the default)+ seeds a synthetic ``<|start|>`` and the turn's first bytes go through the+ ordinary full-header machinery -- a real ``<|message|>`` closes the header+ (a junk recipient yields the empty channel the model asked for), and a+ header that never gets its ``<|message|>`` falls to the unfinished-header+ end-of-stream rule below. This replaces a prefix-guessing "start" mode that+ kept misclassifying content shaped like a header (the same mechanism+ ``force_reasoning`` is for the ``<think>`` families, read from the prompt+ instead of re-derived). Text with no ATEM markers at all passes through as+ content.++ Streaming consumes the buffer as it emits (O(n) over the stream; the naive+ re-scan of the full buffer per chunk was quadratic and stalled the serving+ loop on this family's long default CoT). At end-of-stream, held text is+ delivered as content rather than dropped -- a whole reply that merely looks+ like a bare-header prefix, or prose stranded after a literal closer, must+ not vanish. All three methods are overridden; the base ``<think>`` machinery+ is unused.+ """++ def __init__(+ self,+ force_reasoning: bool = False,+ stream_reasoning: bool = True,+ header_open: bool = True,+ ) -> None:+ super().__init__(+ think_start_token="",+ think_end_token="",+ force_reasoning=force_reasoning,+ stream_reasoning=stream_reasoning,+ )+ self._header_open = header_open+ # header_open: the prompt ended inside a channel header (the template's+ # ``<|start|>assistant``); seed the marker so the turn's first bytes are+ # parsed by the same machinery as every later header. The seed is+ # synthetic -- if it is ever ruled a non-header (released past the span+ # bound, or unfinished at end of stream), the marker text itself is NOT+ # delivered: the model never emitted it.+ self._buffer = ATEM_START if header_open else ""+ self._synthetic_open = header_open+ # "body" (streaming a segment body) | "seek" (between segments, looking+ # for the next header).+ self._mode = "seek"+ self._recipient = "user"++ def detect_and_parse(self, text: str) -> ReasoningParseResult:+ # Closers count as markers too: a literal stray <|eot|> must take the same+ # replay path as streaming (which strips it), not pass through verbatim.+ if not any(tok in text for tok in ATEM_ALL_TOKENS):+ return ReasoningParseResult(normal_text=text)+ clone = type(self)(header_open=self._header_open)+ first = clone.parse_streaming_increment(text)+ rest = clone.flush()+ return ReasoningParseResult(+ reasoning_text=(first.reasoning_text + rest.reasoning_text).strip(),+ normal_text=(first.normal_text + rest.normal_text).strip(),+ )++ def parse_streaming_increment(self, new_text: str) -> ReasoningParseResult:+ self._buffer += new_text+ return self._drain(final=False)++ def flush(self) -> ReasoningParseResult:+ return self._drain(final=True)++ @staticmethod+ def _segment_body_end(buf: str):+ """Earliest segment-body boundary in ``buf``: ``(end, kind, payload)`` with+ kind "closer" (payload = the token), "start" (an abutting ``<|start|>``),+ "inline" (payload = the headerless ``to=X<|message|>`` match), or+ ``(len(buf), None, None)`` while the body is still streaming."""+ best = (len(buf), None, None)+ for tok in ATEM_CLOSING_TOKENS:+ pos = buf.find(tok)+ if pos != -1 and pos < best[0]:+ best = (pos, "closer", tok)+ pos = buf.find(ATEM_START)+ if pos != -1 and pos < best[0]:+ best = (pos, "start", None)+ m = ATEM_INLINE_HEADER_RE.search(buf)+ if m is not None and m.start() < best[0]:+ best = (m.start(), "inline", m)+ return best++ def _drain(self, *, final: bool) -> ReasoningParseResult:+ out_reasoning: list[str] = []+ out_content: list[str] = []++ def emit(recipient: str, piece: str) -> None:+ if not piece:+ return+ if recipient == "self":+ out_reasoning.append(piece)+ else: # user bodies unwrapped, tool slices verbatim+ out_content.append(piece)++ def begin_body(recipient: str, header_text: str) -> None:+ self._recipient = recipient+ self._mode = "body"+ if recipient not in ("self", "user"):+ emit(recipient, header_text) # tool slices keep their header++ def emit_seek(piece: str) -> None:+ # Inter-segment text is content (deliver, don't drop); redundant closer+ # tokens in it are protocol debris and are stripped -- the same filter+ # the detector applies on its text path.+ for tok in ATEM_CLOSING_TOKENS:+ piece = piece.replace(tok, "")+ if piece:+ out_content.append(piece)++ while True:+ buf = self._buffer+ if not buf:+ break++ if self._mode == "seek":+ s = buf.find(ATEM_START)+ m = ATEM_INLINE_HEADER_RE.search(buf)+ if m is not None and (s == -1 or m.start() < s):+ emit_seek(buf[: m.start()])+ self._buffer = buf[m.end():]+ begin_body(m.group(1), buf[m.start(): m.end()])+ continue+ if s != -1:+ # While the header-open seed is unconsumed it is the leftmost+ # marker, so every consuming branch below owns clearing the+ # synthetic flag; a synthetic marker ruled a non-header is+ # dropped, never delivered (the model did not emit it).+ synthetic = self._synthetic_open+ msg = buf.find(ATEM_MESSAGE, s + len(ATEM_START))+ if atem_marker_inside(+ buf, s + len(ATEM_START), msg if msg != -1 else len(buf)+ ):+ # A control token inside the candidate: headers never+ # contain markers, so this <|start|> is literal content+ # (and a synthetic seed is simply dropped). Decides+ # immediately -- no waiting for the span bound or EOS.+ emit_seek(buf[:s])+ if not synthetic:+ out_content.append(ATEM_START)+ self._synthetic_open = False+ self._buffer = buf[s + len(ATEM_START):]+ continue+ if msg != -1 and msg - s - len(ATEM_START) > ATEM_HEADER_SPAN:+ # The found <|message|> is too far away to belong to THIS+ # marker (a stray literal <|start|>, junk, then the NEXT+ # segment's real header): the marker is literal content --+ # the span bound applies whether or not a <|message|> is+ # already in the buffer, or one-shot and streaming diverge.+ emit_seek(buf[:s])+ if not synthetic:+ out_content.append(ATEM_START)+ self._synthetic_open = False+ self._buffer = buf[s + len(ATEM_START):]+ continue+ if msg != -1:+ emit_seek(buf[:s])+ header = buf[s + len(ATEM_START): msg]+ rm = ATEM_RECIPIENT_RE.search(header)+ body_start = msg + len(ATEM_MESSAGE)+ self._buffer = buf[body_start:]+ self._synthetic_open = False+ begin_body(rm.group(1) if rm else "user", buf[s:body_start])+ continue+ # A complete <|start|> whose <|message|> hasn't arrived: text+ # before it is content NOW (never re-dropped), the candidate is+ # held -- bounded, with slack for a <|message|> mid-arrival (a+ # protocol-legal long-name header must not be cut at the nominal+ # span). Past the bound it cannot open a header anymore: release+ # the marker as literal content and resume scanning behind it,+ # so a degenerate wire neither stalls nor eats the turn.+ emit_seek(buf[:s])+ self._buffer = buf[s:]+ if len(self._buffer) - len(ATEM_START) > ATEM_HEADER_SPAN + len(ATEM_MESSAGE):+ if not synthetic:+ out_content.append(ATEM_START) # verbatim: deliver, don't drop+ self._synthetic_open = False+ self._buffer = self._buffer[len(ATEM_START):]+ continue+ if final:+ # At end of stream a candidate that never received its+ # <|message|> is NOT a header: deliver the tail, drop only+ # the marker. (A capped discard here kept a drop window+ # whose edge moved between layers; removing it makes the+ # detector's agreement trivial -- both deliver.)+ emit_seek(self._buffer[len(ATEM_START):])+ self._synthetic_open = False+ self._buffer = ""+ break+ # No marker in sight: stream eagerly as content, holding only the+ # tail that could still grow into one. The buffer is CONSUMED here+ # -- the previous hold-until-flush was quadratic (re-scanned from+ # byte 0 per chunk) and stalled the stream on long headerless+ # continuations.+ hold = 0 if final else atem_hold_len(buf)+ piece = buf[: len(buf) - hold] if hold else buf+ emit_seek(piece)+ self._buffer = buf[len(piece):]+ break++ # self._mode == "body"+ recipient = self._recipient+ end, kind, payload = self._segment_body_end(buf)+ if kind is None:+ hold = 0 if final else atem_hold_len(buf)+ piece = buf[: len(buf) - hold] if hold else buf+ emit(recipient, piece)+ self._buffer = buf[len(piece):]+ break+ body = buf[:end]+ if recipient not in ("self", "user"):+ # A closing terminator belongs to the preserved block; a channel+ # left without one (abutting header / headerless switch) gets a+ # synthetic <|eom|> so the detector still sees a delimited block.+ body += payload if kind == "closer" else "<|eom|>"+ emit(recipient, body)+ if kind == "closer":+ self._buffer = buf[end + len(payload):]+ self._mode = "seek"+ elif kind == "start":+ self._buffer = buf[end:]+ self._mode = "seek"+ else: # headerless switch: the next segment is fully known+ self._buffer = buf[payload.end():]+ begin_body(payload.group(1), buf[end: payload.end()])+ return ReasoningParseResult(+ reasoning_text="".join(out_reasoning), normal_text="".join(out_content)+ )++class ReasoningParser:"""Wraps a reasoning detector for streaming and non-streaming use."""@@ -556,6 +880,7 @@ class ReasoningParser:"glm": ThinkReasoningParser,"minimax": ThinkReasoningParser,"minimax_m3": MiniMaxM3ReasoningParser,+ "muse_glimmer": MuseGlimmerReasoningParser,"gemma4": GemmaThoughtReasoningParser,}diff --git a/python/freetoken/tokenizer/effort.py b/python/freetoken/tokenizer/effort.pyindex d63c2443..c81cd7be 100644--- a/python/freetoken/tokenizer/effort.py+++ b/python/freetoken/tokenizer/effort.py@@ -30,11 +30,39 @@ EFFORT_SCALE: dict[str, float] = {KNOWN_REASONING_EFFORTS = tuple(EFFORT_SCALE)-#: OpenAI's effort triple -- the common-denominator vocabulary offered for a-#: template that grades effort without validating it (nothing observable-#: narrows its gears, so offer what every dialect understands).+#: OpenAI's effort triple -- the common-denominator vocabulary every dialect+#: understands.OPENAI_EFFORT_TRIPLE = ("low", "medium", "high")+#: The graded ladder served to a template that grades effort WITHOUT validating+#: it (muse-glimmer's "Reasoning strength"). Such a template interpolates any+#: string, so its probed ``supported`` set is the whole scale -- but the+#: checkpoint's trained levels are the ladder names, and never-advertised+#: dialects (minimal / none / max) must quantize onto them instead of reaching+#: the model verbatim.+GRADED_EFFORT_LADDER = ("low", "medium", "high", "xhigh")+++def effective_efforts(profile: EffortProfile) -> frozenset[str]:+ """The vocabulary a checkpoint is actually served (and advertised) with.++ A NARROWED ``supported`` set is authoritative -- narrowing only ever comes+ from observed rejections. A grader that accepted the whole scale validated+ nothing, so pass-through would interpolate off-vocabulary values+ ("minimal", "max") into the prompt verbatim: it is capped to its dialect's+ ladder instead -- the strength dialect's documented levels top out at xhigh+ (muse-glimmer's model card), a plain effort grader keeps the OpenAI triple+ (the only vocabulary such a template is known to understand)."""+ if not profile.consumes_effort:+ return frozenset()+ if profile.validates or profile.supported != frozenset(KNOWN_REASONING_EFFORTS):+ return profile.supported+ ladder = GRADED_EFFORT_LADDER if profile.strength_dialect else OPENAI_EFFORT_TRIPLE+ vocab = {name for name in ladder if name in profile.supported}+ if profile.default:+ vocab.add(profile.default)+ return frozenset(vocab)+#: Protocol-level thinking toggles, broadcast in every spelling the ecosystem's#: templates read (``enable_thinking`` bool: qwen/glm/gemma/dsv4;#: ``thinking_mode`` string: minimax-m3). Jinja ignores undeclared variables,@@ -53,13 +81,17 @@ class EffortProfile:ever changed its output or raised -- the template ignores the knob, sorequests should not carry it. ``validates`` True means the probe observed arejection: only then is ``supported`` a real vocabulary rather than "this- template interpolates anything".+ template interpolates anything". ``strength_dialect`` True means the+ template reads the ``reasoning_strength`` spelling (muse-glimmer's family,+ whose documented ladder tops out at xhigh) rather than only+ ``reasoning_effort``."""supported: frozenset[str]default: str | Noneconsumes_effort: boolvalidates: bool = False+ strength_dialect: bool = False@dataclass(frozen=True)@@ -96,15 +128,16 @@ def quantize_effort(value: Any, profile: EffortProfile) -> str | None:the template default. With "max" excluded the remaining scale values areunique, so quantization is deterministic across processes."""- if not profile.consumes_effort:+ supported = effective_efforts(profile)+ if not supported:return None- if isinstance(value, str) and value in profile.supported:+ if isinstance(value, str) and value in supported:return valueposition = EFFORT_SCALE.get(value) if isinstance(value, str) else Noneif position is None:return Noneranked = sorted(- (name for name in profile.supported if name != "max"),+ (name for name in supported if name != "max"),key=lambda name: (abs(EFFORT_SCALE[name] - position), -EFFORT_SCALE[name]),)if ranked and abs(EFFORT_SCALE[ranked[0]] - position) <= _MAX_QUANTIZE_DISTANCE:@@ -170,6 +203,16 @@ def probe_effort_profile(# Nothing learnable: sending no effort is the only safe rendering.return EffortProfile(supported=frozenset(), default=None, consumes_effort=False)+ # Which spelling does the template read? A render that moves on the+ # reasoning_strength kwarg alone marks the muse dialect (its ladder differs).+ strength_dialect = False+ try:+ baseline = render({}, None)+ moved = render({"reasoning_strength": "low"}, None)+ strength_dialect = _renderings_differ(moved, baseline)+ except Exception: # noqa: BLE001 -- a rejecting template is not this dialect+ strength_dialect = False+supported = frozenset(name for name in KNOWN_REASONING_EFFORTS if name not in rejected)consumes = bool(rejected or diverged)default = None@@ -182,6 +225,7 @@ def probe_effort_profile(default=default,consumes_effort=consumes,validates=bool(rejected),+ strength_dialect=strength_dialect,)diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.pyindex 8626061e..0636b342 100644--- a/python/freetoken/tokenizer/tokenize.py+++ b/python/freetoken/tokenizer/tokenize.py@@ -60,8 +60,16 @@ class TokenizeManager:# TODO: batch tokenizationfor msg in msgs:prompt = self.render_prompt(msg)+ # A jinja chat template owns every special token (HF's apply_chat_template+ # tokenizes with add_special_tokens=False for the same reason): tokenizers+ # that auto-add bos (muse-glimmer's, llama's) would otherwise double it --+ # the template already rendered one. Raw-string prompts and the dsv4+ # encoder path keep the default.+ templated = isinstance(msg.text, list) and self._dsv4_encoder is Noneinput_ids: torch.Tensor = ( # type: ignore- self.tokenizer.encode(prompt, return_tensors="pt")+ self.tokenizer.encode(+ prompt, return_tensors="pt", add_special_tokens=not templated+ ))results.append(input_ids.view(-1).to(torch.int32))return results@@ -89,6 +97,15 @@ class TokenizeManager:return _apply_dsv4_chat_encoder(self._dsv4_encoder, messages, tools, chat_template_kwargs)+ # Broadcast the effort in every spelling the ecosystem's templates read+ # (muse-glimmer grades ``reasoning_strength``; Jinja ignores undeclared+ # variables) -- the same rule the thinking toggles use. An explicit+ # caller-provided spelling wins over the broadcast.+ if "reasoning_effort" in chat_template_kwargs:+ chat_template_kwargs = dict(chat_template_kwargs)+ chat_template_kwargs.setdefault(+ "reasoning_strength", chat_template_kwargs["reasoning_effort"]+ )if tools is not None:chat_template_kwargs = {**chat_template_kwargs, "tools": tools}prompt = self.tokenizer.apply_chat_template(diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.pyindex d9cad465..5a6a31f2 100644--- a/python/freetoken/utils/hf.py+++ b/python/freetoken/utils/hf.py@@ -129,12 +129,63 @@ def load_generation_sampling(model_path: str) -> dict[str, Any]:return out+class RawConfigShim:+ """Attribute view over a checkpoint's raw ``config.json``.++ Fallback for model types NEWER than the installed transformers (AutoConfig raises+ on an unknown ``model_type`` when the checkpoint ships no ``auto_map``, e.g.+ Muse-Glimmer on transformers < 5.15). FreeToken only reads config fields+ (parse_config) and never instantiates modeling code, so the raw JSON is enough.+ ``*_config`` sub-dicts are wrapped for attribute access, matching+ PretrainedConfig's nested-config behavior; every other value is served raw+ (``rope_parameters`` and friends stay plain dicts, as they do on the real class).+ """++ def __init__(self, data: dict | None = None, **kwargs: Any) -> None:+ self._data = {**(data or {}), **kwargs}++ def __getattr__(self, name: str) -> Any:+ if name == "_name_or_path":+ # PretrainedConfig carries the checkpoint path under this name and+ # DSV4's parse_config reads it (to find inference/config.json), so it+ # must survive the underscore guard below.+ return self.__dict__.get("_data", {}).get("_name_or_path", "")+ if name.startswith("_"):+ raise AttributeError(name)+ try:+ value = self._data[name]+ except KeyError:+ raise AttributeError(name) from None+ if name.endswith("_config") and isinstance(value, dict):+ return RawConfigShim(value)+ return value++ def to_dict(self) -> dict:+ return json.loads(json.dumps(self._data)) # deep copy, callers may mutate+++def _raw_config_json(model_path: str) -> dict:+ if os.path.isdir(model_path):+ path = os.path.join(model_path, "config.json")+ else:+ path = hf_hub_download(repo_id=model_path, filename="config.json")+ with open(path, encoding="utf-8") as f:+ return json.load(f)++@functools.cachedef _load_hf_config(model_path: str) -> Any:# trust_remote_code: checkpoints that ship a custom config class via ``auto_map``# (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields# (parse_config) and never instantiates the checkpoint's modeling code.- return AutoConfig.from_pretrained(model_path, trust_remote_code=True)+ try:+ return AutoConfig.from_pretrained(model_path, trust_remote_code=True)+ except ValueError as exc:+ # Unknown model_type on this transformers version: serve off the raw JSON.+ # Anything else (bad path, malformed JSON) stays fatal.+ if "model type" not in str(exc):+ raise+ return RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)def cached_load_hf_config(model_path: str) -> PretrainedConfig:@@ -148,6 +199,8 @@ def cached_load_hf_config(model_path: str) -> PretrainedConfig:return build_gguf_shim(gguf_src)config = _load_hf_config(model_path)+ if isinstance(config, RawConfigShim):+ return RawConfigShim(config.to_dict())return type(config)(**config.to_dict())diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.pynew file mode 100644index 00000000..4a4457e9--- /dev/null+++ b/tests/models/test_muse_glimmer.py@@ -0,0 +1,462 @@+"""Muse-Glimmer-30B config parsing and serving-mode resolution.++Drives ``muse_glimmer.parse_config`` off a synthetic HF config shaped exactly like+meta-models/Muse-Glimmer-30B's (multimodal wrapper: text tower in ``text_config``,+weights under ``model.language_model.``) and checks every decision the plan pinned:+the [SWA x3, full] layer split with NoPE full layers, the folded qk scale, the two+norm eps values, the logit post-processing scalars, the compressed-tensors NVFP4+quant modes, and the pool/backend resolution.+"""++from __future__ import annotations++import pytest++from freetoken.attention.base import AttnType+from freetoken.models.muse_glimmer.config import parse_config+++class _Cfg:+ """Attribute-access shim over a dict (what AutoConfig hands parse_config)."""++ def __init__(self, data: dict):+ for k, v in data.items():+ setattr(self, k, _Cfg(v) if isinstance(v, dict) and k == "text_config" else v)+++def _hf_config(num_layers: int = 52, quantized: bool = False) -> _Cfg:+ pattern = ["sliding_attention"] * 3 + ["full_attention"]+ thetas = [500000.0] * 3 + [0.0]+ reps = (num_layers + 3) // 4+ data = {+ "architectures": ["MuseGlimmerForConditionalGeneration"],+ "model_type": "muse_glimmer",+ "image_token_id": 200092,+ "text_config": {+ "hidden_size": 6656,+ "intermediate_size": 19968,+ "num_hidden_layers": num_layers,+ "num_attention_heads": 32,+ "num_key_value_heads": 2,+ "head_dim": 128,+ "vocab_size": 202048,+ "max_position_embeddings": 131072,+ "hidden_activation": "silu",+ "rms_norm_eps": 1e-5,+ "post_norm_eps": 1e-8,+ "qk_scale_factor": 3.87,+ "final_logit_softcapping": 20.0,+ "output_multiplier": 0.19611613513818404,+ "sliding_window": 2048,+ "tie_word_embeddings": False,+ "layer_types": (pattern * reps)[:num_layers],+ "layer_rope_theta": (thetas * reps)[:num_layers],+ "rope_parameters": {"rope_theta": 500000.0, "rope_type": "default"},+ "model_type": "muse_glimmer_text",+ },+ }+ if quantized:+ data["quantization_config"] = {+ "quant_method": "compressed-tensors",+ "format": "nvfp4-pack-quantized",+ "config_groups": {+ "group_0": {+ "format": "nvfp4-pack-quantized",+ "targets": ["Linear"],+ "weights": {+ "num_bits": 4,+ "type": "float",+ "group_size": 16,+ "strategy": "tensor_group",+ },+ }+ },+ }+ return _Cfg(data)+++def test_parse_config_full_model():+ cfg = parse_config(_hf_config())++ assert cfg.num_layers == 52+ assert (cfg.num_qo_heads, cfg.num_kv_heads, cfg.head_dim) == (32, 2, 128)+ assert cfg.hidden_size == 6656 and cfg.intermediate_size == 19968+ assert cfg.vocab_size == 202048 and not cfg.tie_word_embeddings+ assert cfg.hidden_act == "silu" and not cfg.is_moe+ assert cfg.rms_norm_eps == 1e-5 and cfg.post_norm_eps == 1e-8+ assert cfg.use_qk_norm+ # q = qk_norm(q) * 3.87 with standard 1/sqrt(d) attention: folded into sm_scale.+ assert cfg.attn_sm_scale == pytest.approx(3.87 * 128**-0.5)+ assert cfg.final_logit_softcapping == 20.0+ assert cfg.output_multiplier == pytest.approx(0.19611613513818404)+ assert cfg.embedding_scale is None # NormedEmbedding, not Gemma's sqrt(hidden) scale+ assert cfg.vision_config is None # served text-only++ groups = {g.name: g for g in cfg.attention_groups}+ assert groups["swa"].layer_ids == tuple(i for i in range(52) if (i + 1) % 4 != 0)+ assert groups["full"].layer_ids == tuple(i for i in range(52) if (i + 1) % 4 == 0)+ assert groups["swa"].sliding_window == 2048+ assert groups["swa"].rotary_config.base == 500000.0+ # NoPE full layers: base 0.0 is the skip-rope marker read by the attention module.+ assert groups["full"].rotary_config.base == 0.0+ assert cfg.rotary_config.base == 500000.0 # top-level config carries the real rope++ specs = {s.name: s for s in cfg.kv_cache_group_specs()}+ assert specs["swa"].attn_type == AttnType.SWA and specs["swa"].sliding_window == 2048+ assert specs["full"].attn_type == AttnType.FULL and specs["full"].sliding_window is None+ assert all((s.num_kv_heads, s.head_dim) == (2, 128) for s in specs.values())++ # BF16 checkpoint: nothing quantized.+ assert cfg.attn_quant == "none" and cfg.dense_quant == "none"+ assert cfg.lm_head_quant == "none"+++def test_parse_config_nvfp4_checkpoint():+ cfg = parse_config(_hf_config(quantized=True))+ # compressed-tensors NVFP4 quantizes every text Linear (attention incl. the gate,+ # and the MLP); lm_head / embeddings / norms stay bf16 (the ignore list).+ assert cfg.attn_quant == "nvfp4" and cfg.dense_quant == "nvfp4"+ assert cfg.lm_head_quant == "none"+++def test_non_nvfp4_4bit_scheme_is_a_clear_error():+ # A compressed-tensors MXFP4 checkpoint (group_size 32, real since LLM Compressor+ # 0.9) must fail with a clear "unsupported scheme" error instead of routing into+ # the NVFP4 loader and dying in a shape assert.+ hf = _hf_config(quantized=True)+ weights = hf.quantization_config["config_groups"]["group_0"]["weights"]+ weights["group_size"] = 32+ weights["strategy"] = "group"+ with pytest.raises(ValueError, match="unsupported compressed-tensors"):+ parse_config(hf)+++def test_per_layer_theta_beats_shared_rope_theta():+ # layer_rope_theta is the source of truth: a hypothetical checkpoint giving the+ # full layers a real theta must not be forced to NoPE.+ hf = _hf_config(num_layers=8)+ hf.text_config.layer_rope_theta = [1e6] * 8+ cfg = parse_config(hf)+ groups = {g.name: g for g in cfg.attention_groups}+ assert groups["full"].rotary_config.base == 1e6+ assert groups["swa"].rotary_config.base == 1e6+++def test_pool_family_and_backend_resolution():+ from freetoken.engine.engine import _required_attn_types, _resolve_auto_attention_backend+ from freetoken.kvcache import resolve_pool_class+ from freetoken.kvcache.hybrid_swa_pool import HybridSWAKVCache++ cfg = parse_config(_hf_config())+ assert resolve_pool_class(cfg) is HybridSWAKVCache+ required = _required_attn_types(cfg)+ assert required == frozenset({AttnType.FULL, AttnType.SWA})+ # SWA restricts serving to the triton backend (the only one in the capability+ # matrix that consumes per-call sliding windows), same as gemma4.+ assert _resolve_auto_attention_backend(required, False) == "triton"+++def test_registry_resolves_architecture():+ from freetoken.models.register import get_model_spec++ spec = get_model_spec("MuseGlimmerForConditionalGeneration")+ assert spec.module == "freetoken.models.muse_glimmer"+ assert spec.model_cls == "MuseGlimmerForCausalLM"+++def test_aot_table_covers_the_checkpoints():+ from freetoken.kernel.aot_models import SUPPORTED_MODELS++ entry = next(m for m in SUPPORTED_MODELS if m.architecture == "MuseGlimmerForConditionalGeneration")+ assert entry.hidden_size == 6656+ assert entry.kv_groups == ((2, 128),)+ assert entry.expert_formats == () # dense+ assert "RedHatAI/Muse-Glimmer-30B-NVFP4" in entry.aliases+++def test_weight_rename_and_fusion():+ import torch++ from freetoken.models.loader import ct_bf16_fuse+ from freetoken.models.muse_glimmer.weight import _FUSIONS, _rename++ # Text tower renamed, vision dropped, lm_head untouched.+ assert _rename("model.language_model.layers.0.self_attn.q_proj.weight") == (+ "model.layers.0.self_attn.q_proj.weight"+ )+ assert _rename("model.language_model.embed_tokens.weight") == "model.embed_tokens.weight"+ assert _rename("lm_head.weight") == "lm_head.weight"+ assert _rename("model.vision_tower.layers.0.attn.q_proj.weight") is None+ assert _rename("model.vision_adapter.fc1.weight") is None+ assert _rename("model.vision_projection.weight") is None++ # q/k/v + the attention gate fuse into qkvg_proj in declaration order.+ buf: dict = {}+ parts = {+ "q_proj": torch.full((4, 2), 0.0),+ "k_proj": torch.full((2, 2), 1.0),+ "v_proj": torch.full((2, 2), 2.0),+ "gate_proj": torch.full((4, 2), 3.0),+ }+ fused = None+ for name, tensor in parts.items():+ out = ct_bf16_fuse(f"model.layers.0.self_attn.{name}", tensor, buf, _FUSIONS)+ assert out is not None+ if out:+ fused = out[0]+ assert fused is not None and not buf+ key, tensor = fused+ assert key == "model.layers.0.self_attn.qkvg_proj.weight"+ assert tensor.shape == (12, 2)+ assert tensor[0, 0] == 0.0 and tensor[4, 0] == 1.0 and tensor[6, 0] == 2.0+ assert tensor[8, 0] == 3.0++ # The MLP's own gate_proj is a different fusion (gate|up), not the attention one.+ buf2: dict = {}+ assert ct_bf16_fuse("model.layers.0.mlp.gate_proj", torch.zeros(3, 2), buf2, _FUSIONS) == []+ (out,) = ct_bf16_fuse("model.layers.0.mlp.up_proj", torch.ones(3, 2), buf2, _FUSIONS)+ key2, tensor2 = out+ assert key2 == "model.layers.0.mlp.gate_up_proj.weight" and tensor2.shape == (6, 2)+++def _write_shards(tmp_path, shards: dict[str, dict]):+ """Write synthetic safetensors shards + an index, real checkpoint naming."""+ import safetensors.torch++ weight_map = {}+ for shard, tensors in shards.items():+ safetensors.torch.save_file(tensors, str(tmp_path / shard))+ for name in tensors:+ weight_map[name] = shard+ (tmp_path / "model.safetensors.index.json").write_text(+ __import__("json").dumps({"weight_map": weight_map})+ )+++def _bf16_checkpoint_tensors(hf) -> dict:+ import torch++ text = hf.text_config+ H, I = text.hidden_size, text.intermediate_size+ q = text.num_attention_heads * text.head_dim+ kv = text.num_key_value_heads * text.head_dim+ p = "model.language_model."+ tensors = {+ p + "embed_tokens.weight": torch.randn(text.vocab_size, H, dtype=torch.bfloat16),+ p + "norm.weight": torch.randn(H, dtype=torch.bfloat16),+ "lm_head.weight": torch.randn(text.vocab_size, H, dtype=torch.bfloat16),+ # vision tensors must be dropped, not loaded+ "model.vision_tower.ln_pre.weight": torch.randn(4, dtype=torch.bfloat16),+ "model.vision_adapter.fc1.weight": torch.randn(4, 4, dtype=torch.bfloat16),+ "model.vision_projection.weight": torch.randn(4, 4, dtype=torch.bfloat16),+ }+ for i in range(text.num_hidden_layers):+ lp = f"{p}layers.{i}."+ tensors |= {+ lp + "self_attn.q_proj.weight": torch.randn(q, H, dtype=torch.bfloat16),+ lp + "self_attn.k_proj.weight": torch.randn(kv, H, dtype=torch.bfloat16),+ lp + "self_attn.v_proj.weight": torch.randn(kv, H, dtype=torch.bfloat16),+ lp + "self_attn.gate_proj.weight": torch.randn(q, H, dtype=torch.bfloat16),+ lp + "self_attn.o_proj.weight": torch.randn(H, q, dtype=torch.bfloat16),+ lp + "mlp.gate_proj.weight": torch.randn(I, H, dtype=torch.bfloat16),+ lp + "mlp.up_proj.weight": torch.randn(I, H, dtype=torch.bfloat16),+ lp + "mlp.down_proj.weight": torch.randn(H, I, dtype=torch.bfloat16),+ lp + "input_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),+ lp + "post_attention_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),+ lp + "pre_feedforward_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),+ lp + "post_feedforward_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),+ }+ return tensors+++def test_iter_weights_bf16_matches_model_state_dict(tmp_path, monkeypatch):+ """The BF16 loader must produce exactly the model's state-dict keys with the+ right shapes (rename + qkvg / gate_up fusion, vision dropped, norms raw)."""+ import torch++ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ from freetoken.models.muse_glimmer.model import MuseGlimmerForCausalLM+ from freetoken.models.muse_glimmer.weight import iter_weights++ hf = _hf_config(num_layers=4)+ tensors = _bf16_checkpoint_tensors(hf)+ _write_shards(tmp_path, {"model-00001-of-00001.safetensors": tensors})+ import freetoken.models.muse_glimmer.weight as w++ monkeypatch.setattr(w, "cached_load_hf_config", lambda _p: hf)++ loaded = dict(+ iter_weights(str(tmp_path), torch.device("cpu"), include_moe_experts=False, include_non_moe=True)+ )+ model = MuseGlimmerForCausalLM(parse_config(hf))+ expected = model.state_dict()+ assert set(loaded) == set(expected)+ for k in expected:+ assert loaded[k].shape == expected[k].shape, k+ # fusion order [q, k, v, gate] against the raw parts+ fused = loaded["model.layers.0.self_attn.qkvg_proj.weight"]+ p = "model.language_model.layers.0.self_attn."+ q_dim = tensors[p + "q_proj.weight"].shape[0]+ kv_dim = tensors[p + "k_proj.weight"].shape[0]+ assert torch.equal(fused[:q_dim], tensors[p + "q_proj.weight"])+ assert torch.equal(fused[q_dim : q_dim + kv_dim], tensors[p + "k_proj.weight"])+ assert torch.equal(fused[-q_dim:], tensors[p + "gate_proj.weight"])+++def test_iter_weights_nvfp4_cross_shard_scales(tmp_path, monkeypatch):+ """compressed-tensors loader: native FP4 parts fused with per-part scales, the+ reciprocal global, and sibling scales resolved through the index even when they+ land in a different shard than their weight_packed (the real checkpoint splits+ layer 49's down_proj across the shard boundary)."""+ import torch++ from freetoken.models.muse_glimmer.weight import iter_weights++ hf = _hf_config(num_layers=1, quantized=True)+ text = hf.text_config+ text.hidden_size, text.intermediate_size = 64, 96+ text.num_attention_heads, text.num_key_value_heads, text.head_dim = 4, 2, 16+ text.vocab_size = 128+ H, I = 64, 96+ q, kv = 4 * 16, 2 * 16+ fp8 = torch.float8_e4m3fn++ def nvfp4(base, out_f, in_f, global_scale, shard_for_scales=None):+ t = {+ base + ".weight_packed": torch.randint(0, 256, (out_f, in_f // 2), dtype=torch.uint8),+ base + ".weight_scale": torch.randn(out_f, in_f // 16).abs().to(fp8),+ base + ".weight_global_scale": torch.tensor([global_scale]),+ base + ".input_global_scale": torch.tensor([1.0]), # W4A4 scale: skipped+ }+ return t++ p = "model.language_model.layers.0."+ shard1 = {}+ for base, (o, i) in {+ p + "self_attn.q_proj": (q, H), p + "self_attn.k_proj": (kv, H),+ p + "self_attn.v_proj": (kv, H), p + "self_attn.gate_proj": (q, H),+ p + "self_attn.o_proj": (q, H)[::-1], p + "mlp.gate_proj": (I, H),+ p + "mlp.up_proj": (I, H),+ }.items():+ shard1 |= nvfp4(base, o, i, 2.0)+ # down_proj straddles the boundary: packed weight in shard 1, scales in shard 2.+ down = nvfp4(p + "mlp.down_proj", H, I, 4.0)+ shard1[p + "mlp.down_proj.weight_packed"] = down[p + "mlp.down_proj.weight_packed"]+ shard2 = {k: v for k, v in down.items() if not k.endswith(".weight_packed")}+ shard2 |= {+ "model.language_model.embed_tokens.weight": torch.randn(128, H, dtype=torch.bfloat16),+ "model.language_model.norm.weight": torch.randn(H, dtype=torch.bfloat16),+ "lm_head.weight": torch.randn(128, H, dtype=torch.bfloat16),+ }+ for name in (+ "input_layernorm", "post_attention_layernorm",+ "pre_feedforward_layernorm", "post_feedforward_layernorm",+ ):+ shard1[p + name + ".weight"] = torch.randn(H, dtype=torch.bfloat16)+ _write_shards(tmp_path, {+ "model-00001-of-00002.safetensors": shard1,+ "model-00002-of-00002.safetensors": shard2,+ })+ import freetoken.models.muse_glimmer.weight as w++ monkeypatch.setattr(w, "cached_load_hf_config", lambda _p: hf)++ loaded = dict(+ iter_weights(str(tmp_path), torch.device("cpu"), include_moe_experts=False, include_non_moe=True)+ )+ qkvg = "model.layers.0.self_attn.qkvg_proj"+ assert loaded[qkvg + ".weight"].shape == (2 * q + 2 * kv, H // 2)+ assert loaded[qkvg + ".weight_scale"].shape == (2 * q + 2 * kv, H // 16)+ assert loaded[qkvg + ".weight_global"].shape == (2 * q + 2 * kv,)+ # global is the reciprocal of the stored quant-side scale+ assert loaded[qkvg + ".weight_global"][0].item() == pytest.approx(0.5)+ dp = "model.layers.0.mlp.down_proj"+ assert loaded[dp + ".weight"].shape == (H, I // 2) # cross-shard scales resolved+ assert loaded[dp + ".weight_global"][0].item() == pytest.approx(0.25)+ # activation scales are never emitted+ assert not any(k.endswith(".input_global_scale") for k in loaded)+++def test_raw_config_shim_serves_unknown_model_type(tmp_path):+ """A checkpoint whose model_type is newer than the installed transformers must+ still parse: cached_load_hf_config falls back to the raw config.json."""+ import json as _json++ from freetoken.utils.hf import RawConfigShim, _load_hf_config, cached_load_hf_config++ raw = {+ "architectures": ["MuseGlimmerForConditionalGeneration"],+ "model_type": "some_model_type_from_the_future",+ "dtype": "bfloat16",+ "text_config": {"hidden_size": 64, "rope_parameters": {"rope_theta": 1e4}},+ }+ (tmp_path / "config.json").write_text(_json.dumps(raw))+ _load_hf_config.cache_clear()+ try:+ cfg = cached_load_hf_config(str(tmp_path))+ finally:+ _load_hf_config.cache_clear()+ assert isinstance(cfg, RawConfigShim)+ assert cfg.architectures == ["MuseGlimmerForConditionalGeneration"]+ assert cfg.text_config.hidden_size == 64+ # sub-config wrapped for attribute access; plain dicts stay dicts+ assert cfg.text_config.rope_parameters.get("rope_theta") == 1e4+ assert getattr(cfg, "missing_field", None) is None+ assert cfg.to_dict()["dtype"] == "bfloat16"+ # _name_or_path survives the underscore guard (DSV4's parse_config reads it+ # to locate inference/config.json) and the cached copy keeps it.+ assert cfg._name_or_path == str(tmp_path)+++def test_model_state_dict_matches_loader_keys():+ import torch++ from freetoken.distributed import set_tp_info, try_get_tp_info++ if try_get_tp_info() is None:+ set_tp_info(rank=0, size=1)+ from freetoken.models.muse_glimmer.model import MuseGlimmerForCausalLM++ cfg = parse_config(_hf_config(num_layers=4))+ model = MuseGlimmerForCausalLM(cfg)+ keys = set(model.state_dict().keys())+ layer0 = {k for k in keys if k.startswith("model.layers.0.")}+ assert layer0 == {+ "model.layers.0.self_attn.qkvg_proj.weight",+ "model.layers.0.self_attn.o_proj.weight",+ "model.layers.0.mlp.gate_up_proj.weight",+ "model.layers.0.mlp.down_proj.weight",+ "model.layers.0.input_layernorm.weight",+ "model.layers.0.post_attention_layernorm.weight",+ "model.layers.0.pre_feedforward_layernorm.weight",+ "model.layers.0.post_feedforward_layernorm.weight",+ }+ # Weightless norms (embed_norm, qk_norm) must not demand checkpoint tensors.+ assert "model.embed_norm.weight" not in keys+ assert not any("qk_norm" in k for k in keys)+ assert {"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight"} <= keys++ # Rope built only where a real theta exists: sliding layers yes, NoPE full no.+ layers = model.model.layers.op_list+ assert layers[0].self_attn.rotary is not None and layers[0].self_attn.is_swa+ assert layers[3].self_attn.rotary is None and not layers[3].self_attn.is_swa+ assert layers[0].self_attn.attn_spec.sliding_window == 2048+ assert layers[3].self_attn.attn_spec.sliding_window is None+ assert layers[0].self_attn.attn_spec.sm_scale == pytest.approx(3.87 * 128**-0.5)++ # NVFP4 build swaps every text Linear for the W4A16 kernels.+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear++ qcfg = parse_config(_hf_config(num_layers=4, quantized=True))+ qmodel = MuseGlimmerForCausalLM(qcfg)+ attn = qmodel.model.layers.op_list[0].self_attn+ mlp = qmodel.model.layers.op_list[0].mlp+ assert isinstance(attn.qkvg_proj, Nvfp4DenseColMerged)+ assert isinstance(attn.o_proj, Nvfp4DenseLinear)+ assert isinstance(mlp.gate_up_proj, Nvfp4DenseColMerged)+ assert isinstance(mlp.down_proj, Nvfp4DenseLinear)+ assert type(qmodel.lm_head).__name__ == "ParallelLMHead" # lm_head stays bf16+ del model, qmodel, torchdiff --git a/tests/server/test_muse_glimmer_parsers.py b/tests/server/test_muse_glimmer_parsers.pynew file mode 100644index 00000000..3a492f2b--- /dev/null+++ b/tests/server/test_muse_glimmer_parsers.py@@ -0,0 +1,1275 @@+"""Muse Glimmer ATEM reasoning parser (to=self / to=user / tool channels) and+tool-call detector (<atem:function_calls> invoke/parameter XML) -- one-shot and+streaming."""++from __future__ import annotations++import json++import pytest++from freetoken.server.function_call_parser import (+ Function,+ FunctionCallParser,+ MuseGlimmerDetector,+ Tool,+)+from freetoken.server.reasoning_parser import (+ ATEM_START,+ MuseGlimmerReasoningParser,+ ReasoningParser,+)+++def _tools():+ return [+ Tool(function=Function(name="weather.get", parameters={+ "type": "object",+ "properties": {+ "city": {"type": "string"},+ "days": {"type": "integer"},+ "units": {"type": "object"},+ },+ })),+ Tool(function=Function(name="fs.write", parameters={+ "type": "object",+ "properties": {"path": {"type": "string"}, "content": {"type": "string"}},+ })),+ ]+++def _atem(name: str, params: dict[str, str]) -> str:+ body = "".join(+ f'<atem:parameter name="{k}">{v}</atem:parameter>\n' for k, v in params.items()+ )+ return (+ f'<atem:function_calls>\n<atem:invoke name="{name}">\n{body}'+ f"</atem:invoke>\n</atem:function_calls>"+ )+++def _tool_channel(name: str, params: dict[str, str], *, closer: str = "<|eot|>") -> str:+ return f"<|start|>assistant to={name}<|message|>{_atem(name, params)}{closer}"+++def _stream(parser, text: str, step: int = 3):+ reasoning, content = [], []+ for i in range(0, len(text), step):+ r = parser.parse_streaming_increment(text[i : i + step])+ reasoning.append(r.reasoning_text)+ content.append(r.normal_text)+ r = parser.flush()+ reasoning.append(r.reasoning_text)+ content.append(r.normal_text)+ return "".join(reasoning), "".join(content)+++def _stream_detect(det, text: str, tools, step: int = 3):+ normal, calls = [], []+ for i in range(0, len(text), step):+ r = det.parse_streaming_increment(text[i : i + step], tools)+ normal.append(r.normal_text)+ calls.extend(r.calls)+ while True:+ r = det.parse_streaming_increment("", tools)+ if not r.normal_text and not r.calls:+ break+ normal.append(r.normal_text)+ calls.extend(r.calls)+ normal.append(det.finish_streaming())+ return "".join(normal), calls+++def _assemble(calls):+ """(name, joined-args-json) per streamed call, in order."""+ out: list[list] = []+ for c in calls:+ if c.name is not None:+ out.append([c.name, ""])+ if c.parameters:+ out[-1][1] += c.parameters+ return [(name, json.loads(args or "{}")) for name, args in out]+++# ---------------------------------------------------------------------------+# Reasoning parser+# ---------------------------------------------------------------------------+def test_reasoning_registered():+ assert ReasoningParser.ReasoningParserEnum["muse_glimmer"] is MuseGlimmerReasoningParser+++def test_reasoning_bare_first_segment_self_then_user():+ text = (+ " to=self<|message|>Let me think about this.<|eom|>"+ "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"+ )+ r = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert r.reasoning_text == "Let me think about this."+ assert r.normal_text == "The answer is 42."+++def test_reasoning_user_only_turn():+ r = MuseGlimmerReasoningParser().detect_and_parse(" to=user<|message|>Hi!<|eot|>")+ assert r.reasoning_text == "" and r.normal_text == "Hi!"+++def test_reasoning_recipientless_header_is_content():+ r = MuseGlimmerReasoningParser().detect_and_parse("assistant<|message|>Plain.<|eot|>")+ assert r.reasoning_text == "" and r.normal_text == "Plain."+++def test_reasoning_plain_text_passthrough():+ # Raw, non-templated output carries no ATEM markers and must pass through.+ text = "Just a plain answer, no channels."+ r = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert r.reasoning_text == "" and r.normal_text == text+ p = MuseGlimmerReasoningParser()+ reasoning, content = _stream(p, text)+ assert reasoning == "" and content == text+++def test_reasoning_tool_channel_preserved_verbatim():+ text = (+ " to=self<|message|>check weather<|eom|>"+ "<|start|>assistant to=weather.get<|message|>"+ + _atem("weather.get", {"city": "Paris"})+ + "<|eot|>"+ )+ r = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert r.reasoning_text == "check weather"+ assert r.normal_text.startswith("<|start|>assistant to=weather.get<|message|>")+ assert "<atem:function_calls>" in r.normal_text and r.normal_text.endswith("<|eot|>")++ # A tool call as the FIRST (bare) segment: the header-open parser completes+ # the bare header with its synthetic <|start|> seed, so the detector always+ # receives a fully delimited channel -- no bare-header guessing downstream.+ bare = " to=weather.get<|message|>" + _atem("weather.get", {"city": "Rome"}) + "<|eot|>"+ r2 = MuseGlimmerReasoningParser().detect_and_parse(bare)+ assert r2.normal_text.startswith("<|start|> to=weather.get<|message|>")+ assert r2.normal_text.endswith("<|eot|>")+++def test_reasoning_streaming_matches_one_shot():+ text = (+ " to=self<|message|>step one\nstep two<|eom|>"+ "<|start|>assistant to=user<|message|>Done: 42.<|eot|>"+ )+ one = MuseGlimmerReasoningParser().detect_and_parse(text)+ reasoning, content = _stream(MuseGlimmerReasoningParser(), text)+ assert reasoning == one.reasoning_text == "step one\nstep two"+ assert content == one.normal_text == "Done: 42."+++def test_reasoning_streaming_holds_partial_markers():+ p = MuseGlimmerReasoningParser()+ out = p.parse_streaming_increment(" to=user<|message|>Hello <|eo")+ assert out.normal_text == "Hello " # the partial closer is held, not leaked+ out = p.parse_streaming_increment("t|>")+ assert out.normal_text == ""+ r = p.flush()+ assert r.normal_text == "" and r.reasoning_text == ""+++def test_reasoning_undecided_header_delivered_at_flush():+ # Held while it could still become a bare header; at end-of-stream it is a+ # (short) reply and must be delivered, not dropped.+ p = MuseGlimmerReasoningParser()+ assert p.parse_streaming_increment(" to=se").normal_text == ""+ r = p.flush()+ assert r.normal_text == " to=se" and r.reasoning_text == ""+++def test_reasoning_multiple_self_segments_concatenate():+ text = (+ " to=self<|message|>alpha<|eom|>"+ "<|start|>assistant to=self<|message|>beta<|eom|>"+ "<|start|>assistant to=user<|message|>done<|eot|>"+ )+ r = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert r.reasoning_text == "alphabeta"+ assert r.normal_text == "done"+++# ---------------------------------------------------------------------------+# Tool-call detector+# ---------------------------------------------------------------------------+def test_detect_and_parse_typed_args():+ det = MuseGlimmerDetector()+ text = "One sec. " + _tool_channel(+ "weather.get", {"city": "Paris", "days": "3", "units": '{"temp": "C"}'}+ )+ assert det.has_tool_call(text)+ res = det.detect_and_parse(text, _tools())+ assert res.normal_text == "One sec."+ assert len(res.calls) == 1+ call = res.calls[0]+ assert call.name == "weather.get"+ args = json.loads(call.parameters)+ assert args == {"city": "Paris", "days": 3, "units": {"temp": "C"}}+++def test_detect_and_parse_multiple_invokes_in_one_block():+ det = MuseGlimmerDetector()+ block = (+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n'+ '<atem:parameter name="city">Paris</atem:parameter>\n'+ "</atem:invoke>\n"+ '<atem:invoke name="weather.get">\n'+ '<atem:parameter name="city">Tokyo</atem:parameter>\n'+ "</atem:invoke>\n"+ "</atem:function_calls><|eot|>"+ )+ res = det.detect_and_parse(block, _tools())+ assert [json.loads(c.parameters)["city"] for c in res.calls] == ["Paris", "Tokyo"]+ assert [c.tool_index for c in res.calls] == [0, 1]+++def test_string_values_kept_verbatim():+ # The template's contract: "spaces for string values are not stripped".+ det = MuseGlimmerDetector()+ content = "line1\nline2\n"+ text = _tool_channel("fs.write", {"path": " /tmp/x ", "content": content})+ res = det.detect_and_parse(text, _tools())+ assert json.loads(res.calls[0].parameters) == {"path": " /tmp/x ", "content": content}++ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())+ assert _assemble(calls) == [("fs.write", {"path": " /tmp/x ", "content": content})]+ assert normal.strip() == ""+++def test_streaming_matches_one_shot_with_channel_markup():+ text = "Checking. " + _tool_channel("weather.get", {"city": "Paris", "days": "3"})+ one = MuseGlimmerDetector().detect_and_parse(text, _tools())++ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, text, _tools())+ assert normal.strip() == one.normal_text.strip() == "Checking."+ assert _assemble(calls) == [+ (c.name, json.loads(c.parameters)) for c in one.calls+ ] == [("weather.get", {"city": "Paris", "days": 3})]+ # ledgers the serving layer reads at stream end+ assert det.prev_tool_call_arr[0]["name"] == "weather.get"+ assert det.prev_tool_call_arr[0]["arguments"] == {"city": "Paris", "days": 3}+++def test_bare_block_outside_tool_channel_is_text_not_execution():+ # Echo-becomes-execution guard (vLLM's rule): ATEM markup is executed only+ # inside a tool-recipient channel. A block in plain text / a to=user body --+ # e.g. the system prompt's own ATEM example echoed back -- renders as text.+ text = "Look: " + _atem("weather.get", {"city": "Oslo"})+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())+ assert calls == []+ assert normal == text+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())+ assert res.calls == [] and res.normal_text == text++ quoted = (+ " to=user<|message|>Call tools like this:\n"+ + _atem("weather.get", {"city": "Echo"})+ + "\nGot it?<|eot|>"+ )+ normal, calls = _stream_detect(MuseGlimmerDetector(), quoted, _tools())+ assert calls == []+ assert "Call tools like this:" in normal and "Got it?" in normal+ assert "<atem:function_calls>" in normal # the quote stays visible text+++def test_streaming_bare_first_segment_tool_channel():+ # No reasoning parser upstream: generation starts with the header continuation.+ text = " to=weather.get<|message|>" + _atem("weather.get", {"city": "Lima"}) + "<|eot|>"+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())+ assert normal.strip() == ""+ assert _assemble(calls) == [("weather.get", {"city": "Lima"})]+++def test_streaming_self_channel_swallowed_without_reasoning_parser():+ text = (+ " to=self<|message|>secret chain of thought<|eom|>"+ "<|start|>assistant to=user<|message|>Visible.<|eot|>"+ )+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())+ assert calls == []+ assert "secret" not in normal+ assert normal.strip() == "Visible."+++def test_streaming_two_tool_channels_sequential_indices():+ text = (+ _tool_channel("weather.get", {"city": "Paris"}, closer="<|eom|>")+ + _tool_channel("weather.get", {"city": "Rome"})+ )+ _, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())+ assembled = _assemble(calls)+ assert assembled == [+ ("weather.get", {"city": "Paris"}),+ ("weather.get", {"city": "Rome"}),+ ]+ named = [c.tool_index for c in calls if c.name is not None]+ assert named == [0, 1]+++def test_streaming_trailing_text_defers_until_after_call():+ det = MuseGlimmerDetector()+ chunk = "Pre. " + _tool_channel("weather.get", {"city": "Paris"}) + " Post."+ r1 = det.parse_streaming_increment(chunk, _tools())+ assert r1.normal_text == "Pre. "+ assert [c.name for c in r1.calls if c.name] == ["weather.get"]+ r2 = det.parse_streaming_increment("", _tools())+ assert r2.normal_text == " Post." and r2.calls == []+++def test_streaming_plain_text_never_held():+ det = MuseGlimmerDetector()+ r = det.parse_streaming_increment("Just a normal answer, no tools.", _tools())+ assert r.normal_text == "Just a normal answer, no tools."+ assert r.calls == []+++def test_truncated_call_suppressed_and_recovered():+ # Generation cut mid-invoke (max_tokens). The call already started streaming+ # (its Start was emitted at the invoke open), so the serving layer closes it+ # from unstreamed_arguments (the detector's partial-parse ledger), and+ # finish_streaming must not leak the raw markup.+ parser = FunctionCallParser(_tools(), "muse_glimmer")+ det = parser.detector+ truncated = (+ "Ordering. <|start|>assistant to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n'+ '<atem:parameter name="city">Paris</atem:parameter>\n'+ '<atem:parameter name="days">4'+ )+ normal, calls = "", []+ for i in range(0, len(truncated), 11):+ r = det.parse_streaming_increment(truncated[i : i + 11], _tools())+ normal += r.normal_text+ calls.extend(r.calls)+ assert normal == "Ordering. "+ assert [c.name for c in calls if c.name] == ["weather.get"]+ # the completed parameter survives in the ledger; the truncated one is dropped+ assert json.loads(parser.unstreamed_arguments(0)) == {"city": "Paris"}+ assert det.finish_streaming() == ""+++def test_unknown_tool_follows_forwarding_policy():+ import freetoken.server.function_call_parser as fcp++ text = _tool_channel("nope.call", {"a": "1"})+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())+ assert [c.name for c in res.calls] == ["nope.call"] # default: forwarded++ orig = fcp.FORWARD_UNKNOWN_TOOLS+ try:+ fcp.FORWARD_UNKNOWN_TOOLS = False+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())+ assert res.calls == []+ finally:+ fcp.FORWARD_UNKNOWN_TOOLS = orig+++def test_empty_parameters_invoke():+ det = MuseGlimmerDetector()+ block = (+ "<|start|>assistant to=weather.get<|message|>"+ '<atem:function_calls>\n<atem:invoke name="weather.get">\n'+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ )+ res = det.detect_and_parse(block, _tools())+ assert json.loads(res.calls[0].parameters) == {}+ _, calls = _stream_detect(MuseGlimmerDetector(), block, _tools())+ assert _assemble(calls) == [("weather.get", {})]+++# ---------------------------------------------------------------------------+# Review regressions (PR #4)+# ---------------------------------------------------------------------------+def test_channel_close_mid_invoke_does_not_corrupt_next_call():+ """HIGH-1: a channel closer arriving mid-invoke (literal <|eom|> inside a+ parameter value, or a truncated invoke) must finalize the open call --+ completed params kept, JSON closed, ordinal advanced -- so the NEXT+ channel's call never merges into it."""+ text = (+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"+ '<atem:invoke name="fs.write">\n'+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'+ '<atem:parameter name="content">stop token is <|eom|>'+ "<|start|>assistant to=weather.get<|message|>"+ + _atem("weather.get", {"city": "Paris"})+ + "<|eot|>"+ )+ for step in (3, 7, len(text)): # chunked and single-chunk+ det = MuseGlimmerDetector()+ _, calls = _stream_detect(det, text, _tools(), step=step)+ assembled = _assemble(calls)+ assert [a[0] for a in assembled] == ["fs.write", "weather.get"], assembled+ # MED-2: the truncated call's streamed fragments still concatenate to+ # VALID JSON; the completed parameter survives, the cut one is closed.+ first_args = assembled[0][1]+ assert first_args["path"] == "/tmp/t"+ assert assembled[1][1] == {"city": "Paris"}+ named = [c.tool_index for c in calls if c.name is not None]+ assert named == [0, 1] # distinct ordinals: no merge+ assert det.prev_tool_call_arr[0]["arguments"]["path"] == "/tmp/t"+ assert det.prev_tool_call_arr[1]["arguments"] == {"city": "Paris"}+++def test_headerless_channel_switch_executes_the_call():+ """HIGH-2: the model may leave to=self without <|eom|>, writing+ to=<tool><|message|> directly; a complete match is a segment boundary+ (vLLM's rule) in both parsers."""+ text = (+ " to=self<|message|>I should check the weather. to=weather.get<|message|>"+ + _atem("weather.get", {"city": "Paris"})+ + "<|eot|>"+ )+ # reasoning parser: the switch ends reasoning and preserves the tool slice+ one = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert one.reasoning_text == "I should check the weather."+ assert one.normal_text.startswith("to=weather.get<|message|>")+ reasoning, content = _stream(MuseGlimmerReasoningParser(), text)+ assert reasoning.rstrip() == "I should check the weather."+ assert content.startswith("to=weather.get<|message|>")++ # detector on the preserved slice+ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, content, _tools())+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]+ assert normal.strip() == ""++ # and end-to-end without a reasoning parser upstream+ det2 = MuseGlimmerDetector()+ normal2, calls2 = _stream_detect(det2, text, _tools())+ assert _assemble(calls2) == [("weather.get", {"city": "Paris"})]+ assert "I should check" not in normal2 # self body swallowed+++def test_headerless_switch_to_user_streams_content():+ text = " to=self<|message|>quick thought to=user<|message|>Here you go.<|eot|>"+ r = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert r.reasoning_text == "quick thought"+ assert r.normal_text == "Here you go."+ reasoning, content = _stream(MuseGlimmerReasoningParser(), text)+ assert reasoning.rstrip() == "quick thought"+ assert content == "Here you go."+++def test_doubled_tool_name_collapses_to_registered_head():+ """MED-1: the template renders a bare-name tool's namespace as name.*, so the+ model emits get_weather.get_weather; collapse iff the head is registered."""+ tools = [+ Tool(function=Function(name="get_weather", parameters={+ "type": "object", "properties": {"city": {"type": "string"}},+ }))+ ]+ text = (+ "<|start|>assistant to=get_weather.get_weather<|message|>"+ + _atem("get_weather.get_weather", {"city": "Paris"})+ + "<|eot|>"+ )+ det = MuseGlimmerDetector()+ _, calls = _stream_detect(det, text, tools)+ assert _assemble(calls) == [("get_weather", {"city": "Paris"})]+ res = MuseGlimmerDetector().detect_and_parse(text, tools)+ assert [c.name for c in res.calls] == ["get_weather"]++ # a genuinely namespaced name ("weather.get") is never collapsed, and a+ # doubled form that IS registered stays as-is+ res2 = MuseGlimmerDetector().detect_and_parse(+ _tool_channel("weather.get", {"city": "Rome"}), _tools()+ )+ assert [c.name for c in res2.calls] == ["weather.get"]+ tools_doubled = tools + [+ Tool(function=Function(name="get_weather.get_weather", parameters={}))+ ]+ res3 = MuseGlimmerDetector().detect_and_parse(text, tools_doubled)+ assert [c.name for c in res3.calls] == ["get_weather.get_weather"]+++def test_text_after_tool_channel_still_streams():+ """MED-3 companion: after a tool channel closes, following text streams; the+ detector must not stay latched in tool mode."""+ text = _tool_channel("weather.get", {"city": "Paris"}) + " All done."+ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, text, _tools())+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]+ assert normal.strip() == "All done."+++def test_one_shot_classifies_channels_without_reasoning_parser():+ """MED-5: with --reasoning-parser off, non-streaming parsing must still run+ the channel classification: to=self never leaks, to=user is kept."""+ text = (+ " to=self<|message|>hidden chain of thought<|eom|>"+ "<|start|>assistant to=user<|message|>Let me check.<|eom|>"+ "<|start|>assistant to=weather.get<|message|>"+ + _atem("weather.get", {"city": "Paris"})+ + "<|eot|>"+ )+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())+ assert "hidden chain of thought" not in res.normal_text+ assert res.normal_text == "Let me check."+ assert [c.name for c in res.calls] == ["weather.get"]+ assert json.loads(res.calls[0].parameters) == {"city": "Paris"}+++def test_content_starting_with_header_lookalikes_streams():+ """MED-4: committing to "header" requires a full to=/assistant ++ <|message|> match; lookalike content must stream as text."""+ for text in (+ "assistant is a role name, not a header.",+ "to=whom it may concern: hello.",+ ):+ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, text, _tools())+ assert calls == []+ assert normal == text+ p = MuseGlimmerReasoningParser()+ reasoning, content = _stream(p, text)+ assert reasoning == "" and content == text+++def test_tool_slice_holds_partial_markers_while_streaming():+ """LOW-1: the verbatim tool-channel slice must hold back a partial marker;+ emitted content never shrinks when "<|start|" turns out to open the NEXT+ segment."""+ p = MuseGlimmerReasoningParser()+ block = _tool_channel("weather.get", {"city": "Paris"}, closer="")+ emitted = []+ out = p.parse_streaming_increment(block + "<|star")+ emitted.append(out.normal_text)+ assert not "".join(emitted).endswith("<|star") # partial opener held back+ out = p.parse_streaming_increment("t|>assistant to=user<|message|>done<|eot|>")+ emitted.append(out.reasoning_text or "")+ content = "".join(e for e in emitted if e) + out.normal_text + "".join(p.flush().normal_text)+ assert "<|star" + "t|>" not in content.replace("<|start|>", "") # no split debris+++def test_truncated_tool_channel_warns(caplog):+ """LOW-5: a truncated tool channel is logged, not silently dropped."""+ import logging++ text = (+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par<|eot|>'+ )+ det = MuseGlimmerDetector()+ with caplog.at_level(logging.WARNING):+ _stream_detect(det, text, _tools())+ assert any("mid-invoke" in r.message for r in caplog.records)+++# ---------------------------------------------------------------------------+# Review regressions (PR #4, round 2)+# ---------------------------------------------------------------------------+def _pipe(text, tools, step=3):+ """The serving pipeline shape: reasoning parser first, its content feeds the+ detector; returns (reasoning, detector_normal, assembled_calls)."""+ rp = MuseGlimmerReasoningParser()+ det = MuseGlimmerDetector()+ reasoning, normal, calls = "", [], []++ def feed(piece):+ if not piece:+ return+ r = det.parse_streaming_increment(piece, tools)+ normal.append(r.normal_text)+ calls.extend(r.calls)++ for i in range(0, len(text), step):+ r = rp.parse_streaming_increment(text[i : i + step])+ reasoning += r.reasoning_text+ feed(r.normal_text)+ r = rp.flush()+ reasoning += r.reasoning_text+ feed(r.normal_text)+ while True:+ r = det.parse_streaming_increment("", tools)+ if not r.normal_text and not r.calls:+ break+ normal.append(r.normal_text)+ calls.extend(r.calls)+ calls.extend(det.finalize_streaming())+ normal.append(det.finish_streaming())+ return reasoning, "".join(normal), calls+++ "switch",+ [+ "<|start|>assistant to=user<|message|>", # abutting header, no <|eom|>+ "to=user<|message|>", # headerless switch+ ],+)+def test_tool_channel_without_terminator_keeps_rest_of_turn(switch):+ """R2 HIGH-1: a tool channel ending via an abutting header (no closer) must+ reach the detector as a delimited block -- the rest of the turn survives."""+ text = (+ " to=weather.get<|message|>"+ + _atem("weather.get", {"city": "Paris"})+ + switch+ + "visible answer<|eot|>"+ )+ reasoning, normal, calls = _pipe(text, _tools())+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]+ assert normal.strip() == "visible answer"+ assert reasoning == ""+++def test_eos_mid_invoke_finalizes_with_valid_fragments():+ """R2 HIGH-2: max_tokens mid-invoke -- finalize_streaming emits the closing+ fragments so the client's concatenated argument JSON ends valid, and the+ parse ledger serves unstreamed_arguments (even for an empty-dict call)."""+ parser = FunctionCallParser(_tools(), "muse_glimmer")+ det = parser.detector+ truncated = (+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par'+ )+ frags = ""+ for i in range(0, len(truncated), 7):+ r = det.parse_streaming_increment(truncated[i : i + 7], _tools())+ frags += "".join(c.parameters or "" for c in r.calls)+ closing = parser.finalize_stream()+ frags += "".join(c.parameters or "" for c in closing)+ assert json.loads(frags) == {"city": "Par"} # valid, truncated value closed+ assert json.loads(parser.unstreamed_arguments(0)) == {"city": "Par"}+ assert det.finish_streaming() == ""++ # empty-dict ledger entries serve too ({} used to be falsy-skipped)+ parser2 = FunctionCallParser(_tools(), "muse_glimmer")+ det2 = parser2.detector+ trunc2 = (+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n'+ )+ for i in range(0, len(trunc2), 7):+ det2.parse_streaming_increment(trunc2[i : i + 7], _tools())+ frags2 = "".join(c.parameters or "" for c in parser2.finalize_stream())+ assert json.loads(frags2) == {}+ assert parser2.unstreamed_arguments(0) == "{}"+++def test_one_shot_keeps_second_invoke_block_in_one_channel():+ """R2 HIGH-3: two <atem:function_calls> blocks inside ONE channel -- the+ one-shot path (a streaming replay over an undrained buffer) must parse both,+ not discard the second as markup debris at the channel close."""+ text = (+ "<|start|>assistant to=weather.get<|message|>"+ + _atem("weather.get", {"city": "Paris"})+ + "\n"+ + _atem("weather.get", {"city": "Rome"})+ + "<|eot|>"+ )+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())+ assert [json.loads(c.parameters)["city"] for c in res.calls] == ["Paris", "Rome"]+ assert [c.tool_index for c in res.calls] == [0, 1]+ # streaming agrees+ _, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())+ assert [a[1]["city"] for a in _assemble(calls)] == ["Paris", "Rome"]+++def test_reasoning_strength_kwarg_does_not_swallow_the_thinking_toggle():+ """R2 HIGH-4 (post effort-unification): a muse-style reasoning_strength kwarg+ riding along must not disable the broadcast thinking toggle."""+ from freetoken.server.model_meta import effort_toggle_kwargs++ out = effort_toggle_kwargs("high", {"reasoning_strength": "high"})+ assert out.get("enable_thinking") is True # the toggle broadcast still maps+ assert out.get("reasoning_strength") == "high" # the explicit kwarg rides along+ assert out.get("reasoning_effort") == "high"+++def test_reasoning_streaming_buffer_stays_trimmed():+ """R2 MED-5: streaming must consume the buffer as it emits (the full-buffer+ re-scan per chunk was quadratic and stalled the serving loop on long CoT)."""+ p = MuseGlimmerReasoningParser()+ p.parse_streaming_increment(" to=self<|message|>")+ for _ in range(2000):+ p.parse_streaming_increment("think " * 4)+ assert len(p._buffer) < 256 # consumed, not accumulated+++def test_flush_delivers_stray_closer_tail():+ """R2 MED-6(b): prose stranded after a literal closer is delivered at+ end-of-stream instead of dropped."""+ text = "The token <|eot|> ends a turn."+ p = MuseGlimmerReasoningParser()+ content = ""+ for i in range(0, len(text), 5):+ content += p.parse_streaming_increment(text[i : i + 5]).normal_text+ content += p.flush().normal_text+ assert "The token " in content and " ends a turn." in content+++def test_flush_delivers_header_lookalike_reply():+ """R2 MED-6(a): a complete reply that merely looks like a bare-header prefix+ is content at end-of-stream, in both parsers."""+ for reply in ("assistant", "[email protected]"):+ p = MuseGlimmerReasoningParser()+ out = p.parse_streaming_increment(reply)+ got = out.normal_text + p.flush().normal_text+ assert got == reply+ det = MuseGlimmerDetector()+ r = det.parse_streaming_increment(reply, _tools())+ assert (r.normal_text + det.finish_streaming()) == reply+++def test_channel_prose_drop_warns(caplog):+ """R2 small: discarded tool-channel prose leaves a log line."""+ import logging++ text = (+ "<|start|>assistant to=weather.get<|message|>calling now: "+ + _atem("weather.get", {"city": "Paris"})+ + "<|eot|>"+ )+ det = MuseGlimmerDetector()+ with caplog.at_level(logging.WARNING):+ _, calls = _stream_detect(det, text, _tools())+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]+ prose_warnings = [r for r in caplog.records if "inside a tool channel" in r.message]+ assert len(prose_warnings) == 1 # accumulated: ONE warning, not one per fragment+++# ---------------------------------------------------------------------------+# Review regressions (PR #4, round 3)+# ---------------------------------------------------------------------------+def test_seek_mode_streams_eagerly_and_stays_trimmed():+ """R3 MED-1: text after a closer (a headerless continuation, e.g. a+ repetition loop) must stream live with a bounded buffer, not accumulate+ until flush."""+ p = MuseGlimmerReasoningParser()+ p.parse_streaming_increment(" to=user<|message|>answer<|eom|>")+ streamed = ""+ for _ in range(500):+ streamed += p.parse_streaming_increment("loop ").normal_text+ assert len(p._buffer) < 256 # consumed, not held until EOS+ assert len(streamed) > 2000 # and the client actually saw it live+ assert "loop loop" in streamed+++def test_seek_mode_eos_around_incomplete_header():+ """R3 MED-2: at EOS, text before a complete <|start|> whose <|message|>+ never arrived is delivered; the discard is capped at a plausible header+ span instead of eating the rest of the turn."""+ # prose before the marker survives+ p = MuseGlimmerReasoningParser()+ content = p.parse_streaming_increment(+ " to=user<|message|>answer<|eom|>more text <|start|>assistant"+ ).normal_text+ content += p.flush().normal_text+ assert "more text " in content++ # a huge tail after the marker cannot be a header: it is not discarded+ p2 = MuseGlimmerReasoningParser()+ tail = "x" * 400+ c2 = p2.parse_streaming_increment(+ " to=user<|message|>The token <|start|> opens a segment. " + tail+ ).normal_text+ c2 += p2.flush().normal_text+ assert "The token " in c2 and tail in c2+++def test_truncated_channel_markup_residue_dropped_reply_kept(caplog):+ """R3 MED-4 (as re-scoped by R4 HIGH-1): when a literal channel marker cuts an+ invoke exactly at a value's end, the broken channel's CLOSING-MARKUP residue+ is dropped by shape -- and the turn's real reply always survives."""+ import logging++ text = (+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"+ '<atem:invoke name="fs.write">\n'+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'+ '<atem:parameter name="content">send to=user<|message|></atem:parameter>\n'+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ "<|start|>assistant to=user<|message|>Done.<|eot|>"+ )+ with caplog.at_level(logging.WARNING):+ for step in (4, len(text)):+ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, text, _tools(), step=step)+ assembled = _assemble(calls)+ assert assembled[0][0] == "fs.write"+ assert assembled[0][1]["path"] == "/tmp/t" # completed param survives+ assert "</atem:" not in normal and "<atem:" not in normal # no markup leak+ assert normal.strip() == "Done." # the real content after the channel survives+ assert any("mid-invoke" in r.message for r in caplog.records)+++def test_truncated_channel_prose_residue_trade_pinned():+ """R5 test-integrity: the ORIGINAL R3 wire (prose before the closing tags),+ pinning the deliberate leak-over-loss trade. At string level the cut value's+ tail is indistinguishable from a reply, so it LEAKS -- grandparent parity is+ the floor -- and what the machinery guarantees instead is that the turn's+ real reply is never lost."""+ text = (+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"+ '<atem:invoke name="fs.write">\n'+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'+ '<atem:parameter name="content">send to=user<|message|> please</atem:parameter>\n'+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ "<|start|>assistant to=user<|message|>Done.<|eot|>"+ )+ expected = (+ "please</atem:parameter>\n</atem:invoke>\n</atem:function_calls>Done."+ )+ for step in (4, len(text)):+ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, text, _tools(), step=step)+ assembled = _assemble(calls)+ assert assembled[0][0] == "fs.write" and assembled[0][1]["path"] == "/tmp/t"+ assert "Done." in normal # the reply is never lost+ assert normal == expected, (step, normal) # the leak is pinned, not accidental+++def test_reply_right_after_midinvoke_switch_is_never_swallowed():+ """R4 HIGH-1 companion: an inline switch fired mid-invoke followed directly by+ a real reply (no markup residue) -- the reply must flow immediately, not be+ held hostage for a boundary that never comes."""+ text = (+ " to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par'+ "to=user<|message|>The weather tool is on it.<|eot|>"+ )+ for step in (1, 3, 64):+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools(), step=step)+ assert normal.strip() == "The weather tool is on it."+ assert [c.name for c in calls if c.name] == ["weather.get"]+++def test_redundant_closer_stripped_from_seek_delivery():+ """R3 small: a doubled closer from a degenerate decode must not leak the+ second token into content (the detector already filters; the reasoning+ parser's seek path now does too)."""+ text = " to=user<|message|>The answer is 42.<|eot|><|eot|>"+ p = MuseGlimmerReasoningParser()+ content = ""+ for i in range(0, len(text), 5):+ content += p.parse_streaming_increment(text[i : i + 5]).normal_text+ content += p.flush().normal_text+ assert content == "The answer is 42."+ one = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert one.normal_text == "The answer is 42."+++def test_stray_closer_one_shot_matches_streaming():+ """R3 small: the one-shot passthrough shortcut must route closer-containing+ text through the replay, matching the streamed result."""+ text = "The token <|eot|> ends a turn."+ one = MuseGlimmerReasoningParser().detect_and_parse(text)+ p = MuseGlimmerReasoningParser()+ streamed = ""+ for i in range(0, len(text), 5):+ streamed += p.parse_streaming_increment(text[i : i + 5]).normal_text+ streamed += p.flush().normal_text+ assert one.normal_text == streamed.strip()+ assert "<|eot|>" not in one.normal_text+++def test_mixed_quant_groups_rejected_in_any_order():+ """R3 small: a {nvfp4, mxfp4} mixed checkpoint must raise regardless of the+ groups' key order (the first-group short-circuit accepted one order)."""+ import tests.models.test_muse_glimmer as m++ for order in (("group_0", "group_1"), ("group_1", "group_0")):+ hf = m._hf_config(quantized=True)+ groups = hf.quantization_config["config_groups"]+ nvfp4 = groups["group_0"]+ mxfp4 = {"weights": {"num_bits": 4, "type": "float", "group_size": 32, "strategy": "group"}}+ hf.quantization_config["config_groups"] = {+ order[0]: nvfp4 if order[0] == "group_0" else mxfp4,+ order[1]: mxfp4 if order[1] == "group_1" else nvfp4,+ }+ from freetoken.models.muse_glimmer.config import parse_config as pc++ with pytest.raises(ValueError, match="unsupported compressed-tensors"):+ pc(hf)+++# ---------------------------------------------------------------------------+# Review regressions (PR #4, round 4)+# ---------------------------------------------------------------------------+def test_pipeline_delivers_reply_after_truncated_invoke():+ """R4 HIGH-1, PRODUCTION ORDER (reasoning parser first, its content feeds the+ detector -- the shape standalone-detector tests missed twice): a truncated+ invoke must never swallow the rest of the turn's reply."""+ # reviewer wire 1: literal marker cuts the value, closing tags follow+ wire1 = (+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"+ '<atem:invoke name="fs.write">\n'+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'+ '<atem:parameter name="content">send to=user<|message|></atem:parameter>\n'+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ "<|start|>assistant to=user<|message|>Done.<|eot|>"+ )+ # reviewer wire 2: invoke left unclosed before an abutting user header+ wire2 = (+ " to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par'+ "<|start|>assistant to=user<|message|>The weather tool is on it.<|eot|>"+ )+ for wire, reply, first_call in (+ (wire1, "Done.", "fs.write"),+ (wire2, "The weather tool is on it.", "weather.get"),+ ):+ for step in (1, 3, 64):+ _, normal, calls = _pipe(wire, _tools(), step=step)+ assert reply in normal, (wire[:40], step, normal)+ assert "<atem:" not in normal and "</atem:" not in normal+ named = [c.name for c in calls if c.name]+ assert named and named[0] == first_call+++def test_stray_start_before_a_real_header_does_not_form_a_giant_header():+ """R4 MED-2: the span bound applies even when a <|message|> is already in the+ buffer -- a literal <|start|> + junk + the NEXT segment's header must not be+ parsed as one giant header (which also let a to= inside the junk hijack the+ recipient), and streaming must agree with one-shot."""+ junk = "P" * 300+ text = (+ " to=user<|message|>ok<|eom|><|start|>" + junk+ + "<|start|>assistant to=user<|message|>more<|eot|>"+ )+ p = MuseGlimmerReasoningParser()+ streamed_r, streamed_c = "", ""+ for i in range(0, len(text), 5):+ r = p.parse_streaming_increment(text[i : i + 5])+ streamed_r += r.reasoning_text+ streamed_c += r.normal_text+ fl = p.flush()+ streamed_r += fl.reasoning_text+ streamed_c += fl.normal_text+ one = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert "ok" in streamed_c and "more" in streamed_c+ assert junk in streamed_c # delivered, not silently swallowed+ assert one.normal_text == streamed_c.strip()++ # a to=self inside the junk must not hijack the following user reply+ hijack = (+ " to=user<|message|>first.<|eom|><|start|>x to=self y" + "Q" * 250+ + "<|start|>assistant to=user<|message|>the actual answer<|eot|>"+ )+ one = MuseGlimmerReasoningParser().detect_and_parse(hijack)+ assert "the actual answer" in one.normal_text+ assert "the actual answer" not in one.reasoning_text++[email protected]("name_len", [64, 100, 110])+def test_protocol_legal_long_tool_names_survive_streaming(name_len):+ """R4 MED-3: headers stay parseable up to the span bound even when their+ <|message|> is still mid-arrival; the tool call AND the trailing reply+ survive at every step size, matching one-shot."""+ name = "t" * name_len+ tools = [Tool(function=Function(name=name, parameters={+ "type": "object", "properties": {"city": {"type": "string"}},+ }))]+ text = (+ f" to=self<|message|>go<|eom|><|start|>assistant to={name}<|message|>"+ + _atem(name, {"city": "Paris"})+ + "<|eot|><|start|>assistant to=user<|message|>done<|eot|>"+ )+ one_r = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert "done" in one_r.normal_text+ for step in (1, 5, 64):+ reasoning, normal, calls = _pipe(text, tools, step=step)+ assert reasoning.strip() == "go"+ assert normal.strip() == "done", (name_len, step, normal)+ named = [c.name for c in calls if c.name]+ assert named and named[0].startswith("t" * 64)+++def test_eos_start_candidate_delivered_across_layers():+ """R4 item-4 / R5 root-cause fix: at EOS a <|start|> that never received its+ <|message|> is NOT a header -- BOTH layers deliver the tail and drop only+ the marker, so the raw-bytes vs closer-stripped span mismatch between the+ layers has nothing left to disagree about."""+ # R5 MED-2 PoC (ordinary terminators only): the pipeline must deliver+ # everything the parser alone delivers.+ prose = "p" * 119+ wire = " to=user<|message|>The wire format is: <|start|>" + prose + "<|eot|><|end_of_text|>"+ parser_only = ""+ p = MuseGlimmerReasoningParser()+ for i in range(0, len(wire), 7):+ parser_only += p.parse_streaming_increment(wire[i : i + 7]).normal_text+ parser_only += p.flush().normal_text+ assert prose in parser_only+ for step in (1, 7, 4096):+ _, piped, _ = _pipe(wire, _tools(), step=step)+ assert "The wire format is: " in piped+ assert prose in piped, (step, piped) # pipeline == parser, chars not swallowed++ # R4 reviewer wire: the released marker's tail survives the pipeline too+ wire2 = " to=user<|message|>answer<|eot|>prose <|start|>" + "x" * 50 + "<|eom|>" * 15+ for step in (1, 7):+ _, piped2, _ = _pipe(wire2, _tools(), step=step)+ assert "answer" in piped2 and "prose " in piped2+ assert "x" * 50 in piped2, (step, piped2)++[email protected]("junk_len", [103, 115, 127, 140])+def test_reply_after_stray_start_junk_segment_survives(junk_len):+ """R5 HIGH-1: a real to=user reply following a stray <|start|>+junk segment+ must never be dropped -- previously the released marker + junk + reply sat+ under the detector's hold threshold and died in finish_streaming."""+ wire = (+ " to=user<|message|>ok<|eom|><|start|>" + "J" * junk_len+ + "<|start|>assistant to=user<|message|>Here you go.<|eot|>"+ )+ for step in (1, 7, 4096):+ _, piped, _ = _pipe(wire, _tools(), step=step)+ assert "Here you go." in piped, (junk_len, step, piped)+ assert "ok" in piped+ # one-shot agrees+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)+ one = MuseGlimmerDetector().detect_and_parse(rp.normal_text, _tools())+ assert "Here you go." in one.normal_text+++def test_detector_giant_header_bound_pinned():+ """R5 test-integrity: the detector's own msg-too-far bound. A stray+ <|start|> + junk ahead of a real tool channel in ONE detector buffer must+ not merge into a giant header. NOTE which assertion is load-bearing: with+ the bound deleted the tool call still executes correctly -- what breaks is+ the ~300 chars of user content silently vanishing into the giant header, so+ it is the junk/content assertion below that pins the bound, not the calls+ assertion."""+ junk = "J" * 300+ text = (+ "ok<|eom|><|start|>" + junk+ + _tool_channel("weather.get", {"city": "P"})+ + "<|start|>assistant to=user<|message|>done<|eot|>"+ )+ for step in (7, len(text)):+ det = MuseGlimmerDetector()+ normal, calls = _stream_detect(det, text, _tools(), step=step)+ assert _assemble(calls) == [("weather.get", {"city": "P"})], (step,)+ assert "done" in normal and junk in normal+++# ---------------------------------------------------------------------------+# Review regressions (PR #4, round 6): turn-start state is read from the+# prompt (header-open), not guessed from the first bytes. The bare-header+# lookalike family -- shapes the old prefix-guessing blanked to an empty turn.+# ---------------------------------------------------------------------------+def test_junk_recipient_past_cap_yields_empty_channel_not_empty_turn():+ """``to=`` + 70 chars + <|message|>: the recipient (capped at 64) is a junk+ tool name, so the channel body is not user-visible -- but the turn is NOT+ blanked: a following real reply flows."""+ wire = (+ "to=" + "j" * 70 + "<|message|>hidden<|eot|>"+ "<|start|>assistant to=user<|message|>Real reply.<|eot|>"+ )+ for step in (1, 7, 4096):+ _, piped, calls = _pipe(wire, _tools(), step=step)+ assert "Real reply." in piped, (step, piped)+ assert "hidden" not in piped # the junk channel is what the model said+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)+ one = MuseGlimmerDetector().detect_and_parse(rp.normal_text, _tools())+ assert "Real reply." in one.normal_text and "hidden" not in one.normal_text+++def test_glued_assistant_recipient_header_parses():+ """``assistantto=x<|message|>``: the recipient regex still finds to=x, so+ the segment routes as a (junk) tool channel and the next reply flows."""+ wire = (+ "assistantto=x<|message|>hello<|eot|>"+ "<|start|>assistant to=user<|message|>Hi.<|eot|>"+ )+ for step in (1, 7, 4096):+ _, piped, _ = _pipe(wire, _tools(), step=step)+ assert "Hi." in piped, (step, piped)+ assert "hello" not in piped+++def test_deep_leading_whitespace_header_delivers_reply():+ """>8 leading newlines before ``to=user<|message|>``: the old bare-header+ regex allowed at most 8 whitespace chars while the undecided hold lstripped+ unbounded, blanking the turn; the full-header machinery has no such split."""+ wire = "\n" * 9 + "to=user<|message|>Hello!<|eot|>"+ for step in (1, 7, 4096):+ _, piped, _ = _pipe(wire, _tools(), step=step)+ assert "Hello!" in piped, (step, piped)+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)+ assert "Hello!" in rp.normal_text+++def test_prose_before_first_full_header_not_swallowed_into_seed():+ """The header-open seed must not merge turn-start prose with the FIRST real+ header into one giant header: a control token inside a header candidate+ voids the candidate (headers never contain markers). Without the parser's+ marker-inside rule, 'Pre. ' is swallowed into the seed's header -- here the+ header routes to=self, so the swallowed prefix vanishes entirely (a tool+ channel would carry it verbatim and mask the loss downstream)."""+ wire = (+ "Pre. <|start|>assistant to=self<|message|>think<|eom|>"+ "<|start|>assistant to=user<|message|>Post.<|eot|>"+ )+ for step in (1, 7, 4096):+ reasoning, piped, _ = _pipe(wire, _tools(), step=step)+ assert "Pre. " in piped and "Post." in piped, (step, piped)+ assert "think" in reasoning and "Pre" not in reasoning+ r = MuseGlimmerReasoningParser().detect_and_parse(wire)+ assert "Pre." in r.normal_text and "think" in r.reasoning_text+ # and with a tool channel: the call still parses, the prose still flows+ wire2 = (+ "Pre. " + _tool_channel("weather.get", {"city": "P"})+ + "<|start|>assistant to=user<|message|>Post.<|eot|>"+ )+ for step in (1, 7, 4096):+ _, piped2, calls2 = _pipe(wire2, _tools(), step=step)+ assert _assemble(calls2) == [("weather.get", {"city": "P"})], (step,)+ assert "Pre. " in piped2 and "Post." in piped2, (step, piped2)+++def test_headerless_turn_delivers_and_seed_never_leaks():+ """Degenerate output that never emits <|message|>: the header-open seed is+ ruled a non-header and only the model's own bytes are delivered -- the+ synthetic <|start|> must not leak into content at any length."""+ for text in ("Just prose, no protocol.", "x" * 200):+ for step in (1, 7, 4096):+ _, piped, _ = _pipe(text, _tools(), step=step)+ assert piped == text, (len(text), step, piped)+ assert ATEM_START not in piped+ rp = MuseGlimmerReasoningParser().detect_and_parse(text)+ assert rp.normal_text == text+++_CORPUS_REPLY = "All good."+# Historical degenerate shapes x chunkings: streaming, one-shot and the production+# pipeline must agree on the reply and the calls (the regression class that bit+# this PR twice was "standalone green, pipeline broken").+_WIRE_CORPUS = [+ # clean turn+ " to=self<|message|>think<|eom|><|start|>assistant to=user<|message|>All good.<|eot|>",+ # tool channel then reply+ " to=self<|message|>t<|eom|>" + _tool_channel("weather.get", {"city": "P"}, closer="<|eom|>")+ + "<|start|>assistant to=user<|message|>All good.<|eot|>",+ # headerless switches everywhere+ " to=self<|message|>t to=weather.get<|message|>"+ + _atem("weather.get", {"city": "P"}) + "to=user<|message|>All good.<|eot|>",+ # truncated invoke then abutting user header+ " to=weather.get<|message|><atem:function_calls>\n"+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">P'+ "<|start|>assistant to=user<|message|>All good.<|eot|>",+ # stray literal closer inside the reply (redundant <|eom|> mid-sentence)+ " to=user<|message|>All <|eom|>good.<|eot|>",+ # plain clean reply+ " to=user<|message|>All good.<|eot|>",+ # bare first segment is the reply+ " to=user<|message|>All good.<|eom|>",+ # stray <|start|>+junk segment ahead of the real reply (R5 HIGH-1 shape)+ " to=user<|message|>prelude<|eom|><|start|>" + "J" * 115+ + "<|start|>assistant to=user<|message|>All good.<|eot|>",+ # stream dies inside an unfinished header candidate (R5 EOS-family shape)+ " to=user<|message|>All good.<|eot|><|start|>" + "p" * 119 + "<|end_of_text|>",+ # truncated invoke leaving real closing tags before the reply (exercises+ # the </atem: leak clause below)+ " to=fs.write<|message|><atem:function_calls>\n"+ '<atem:invoke name="fs.write">\n'+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'+ '<atem:parameter name="content">send to=user<|message|></atem:parameter>\n'+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ "<|start|>assistant to=user<|message|>All good.<|eot|>",+]++[email protected]("wire_idx", range(len(_WIRE_CORPUS)))[email protected]("step", [1, 7, 4096])+def test_wire_corpus_pipeline_consistency(wire_idx, step):+ wire = _WIRE_CORPUS[wire_idx]+ reasoning, normal, calls = _pipe(wire, _tools(), step=step)+ assert _CORPUS_REPLY in normal, (wire_idx, step, normal)+ assert "<atem:" not in normal and "</atem:" not in normal+ assert "<|" not in normal.replace("<|start|>", "")+ # one-shot detector agrees on calls AND arguments for the same classified content+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)+ one = MuseGlimmerDetector().detect_and_parse(rp.normal_text, _tools())+ assert [(c.name, json.loads(c.parameters)) for c in one.calls] == _assemble(calls)+ # the one-shot DETECTOR's content must carry the reply itself (the parser's+ # content always does, so an or-check here would never fail)+ assert _CORPUS_REPLY in one.normal_text, (wire_idx, one.normal_text)+++# ---------------------------------------------------------------------------+# Wiring: auto-selection and thinking gears+# ---------------------------------------------------------------------------+def test_auto_selection_picks_muse_glimmer():+ from unittest.mock import patch++ from freetoken.server.args import parse_args++ class _Config:+ def to_dict(self):+ return {+ "architectures": ["MuseGlimmerForConditionalGeneration"],+ "model_type": "muse_glimmer",+ "text_config": {"model_type": "muse_glimmer_text"},+ }++ with patch("freetoken.utils.cached_load_hf_config", lambda _p: _Config()):+ args, _ = parse_args(["--model", "/models/anon"])+ assert args.tool_call_parser == "muse_glimmer"+ assert args.reasoning_parser == "muse_glimmer"+++class _MuseLikeTokenizer:+ """The muse template's effort surface: grades ``reasoning_strength`` (default+ high), validates nothing, ignores the thinking-toggle spellings."""++ chat_template = "muse" # non-empty: the dsv4 encoder probe must not engage+ name_or_path = "muse-like"++ def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True, **kwargs):+ strength = kwargs.get("reasoning_strength") or "high"+ return f"<|begin_of_text|>system: Reasoning strength: {strength}. user: ping"++ def encode(self, prompt, return_tensors=None, add_special_tokens=True):+ import torch++ assert add_special_tokens is False # the template owns the bos+ return torch.tensor([[1, 2, 3]], dtype=torch.long)+++def test_effort_broadcast_reaches_reasoning_strength():+ """The render layer broadcasts reasoning_effort in every spelling the+ ecosystem's templates read (the thinking-toggle rule); muse's template picks+ up ``reasoning_strength``. An explicit caller spelling wins."""+ from freetoken.tokenizer.tokenize import TokenizeManager++ manager = TokenizeManager(_MuseLikeTokenizer())+ prompt = manager._render([{"role": "user", "content": "hi"}], None, {"reasoning_effort": "low"})+ assert "Reasoning strength: low." in prompt+ prompt = manager._render(+ [{"role": "user", "content": "hi"}],+ None,+ {"reasoning_effort": "low", "reasoning_strength": "xhigh"},+ )+ assert "Reasoning strength: xhigh." in prompt+++def test_probed_muse_gears_and_effort_vocabulary():+ """The checkpoint-probing pipeline derives muse's gears through the broadcast:+ the template grades effort without validating it, so it is served the graded+ ladder (the model card's trained levels, xhigh included) with the template's+ own default (high); never-advertised dialects quantize onto the ladder+ instead of reaching the model verbatim."""+ from freetoken.server.model_meta import derive_think_gears+ from freetoken.tokenizer.effort import effective_efforts, quantize_effort+ from freetoken.tokenizer.tokenize import TokenizeManager++ manager = TokenizeManager(_MuseLikeTokenizer())+ profile = manager.thinking_profile()+ assert profile.efforts.consumes_effort and not profile.efforts.validates+ assert profile.efforts.default == "high"+ assert not profile.toggleable # the template reads no on/off spelling++ gears, default, kwargs = derive_think_gears(profile, parser_configured=True)+ assert gears == ("low", "medium", "high", "xhigh") and default == "high"+ assert kwargs["low"] == {"reasoning_effort": "low"}+ assert effective_efforts(profile.efforts) == frozenset({"low", "medium", "high", "xhigh"})+ # the native vocabulary passes through untouched...+ assert quantize_effort("xhigh", profile.efforts) == "xhigh"+ # ...while off-ladder dialects quantize instead of interpolating verbatim+ assert quantize_effort("minimal", profile.efforts) == "low"+ assert quantize_effort("max", profile.efforts) == "xhigh"+ assert quantize_effort("none", profile.efforts) is None # template default appliesdiff --git a/tests/server/test_streaming_model_matrix.py b/tests/server/test_streaming_model_matrix.pyindex a0c94a5d..43922021 100644--- a/tests/server/test_streaming_model_matrix.py+++ b/tests/server/test_streaming_model_matrix.py@@ -93,6 +93,12 @@ CALL_BLOCKS = {"<|start|>assistant<|channel|>commentary to=functions.read "'<|constrain|>json<|message|>{"filePath": "/tmp/test_calc.py"}<|end|>'),+ "muse_glimmer": (+ "<|start|>assistant to=read<|message|><atem:function_calls>\n"+ '<atem:invoke name="read">\n'+ "<atem:parameter name=\"filePath\">/tmp/test_calc.py</atem:parameter>\n"+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ ),}# Substrings that must never leak into user-visible content.@@ -107,6 +113,7 @@ MARKUP_MARKERS = {"mistral": ["[TOOL_CALLS]"],"llama3": ["<|python_tag|>"],"gpt_oss": ["<|channel|>", "to=functions."],+ "muse_glimmer": ["<atem:", "<|message|>", "<|eot|>"],}# (tool_call_parser, reasoning_parser, think_open, think_close) per reasoning-capable@@ -123,6 +130,7 @@ REASONING_FAMILIES = {"minimax-m3": ("minimax_m3", "minimax_m3", "<mm:think>", "</mm:think>"),"gemma4": ("gemma4", "gemma4", "<|channel>thought\n", "<channel|>"),"gpt-oss": ("gpt_oss", "gpt_oss", None, None), # harmony channels; custom fixture+ "muse-glimmer": ("muse_glimmer", "muse_glimmer", None, None), # ATEM channels; custom fixture}@@ -274,6 +282,15 @@ def _reasoning_fixture(name):f"<|message|>{json.dumps(READ_ARGS)}<|call|>")return tool, reasoning, text+ if name == "muse-glimmer":+ # Generation resumes after the template's <|start|>assistant, so the first+ # segment is a bare header continuation.+ text = (+ f" to=self<|message|>{THINKING}<|eom|>"+ f"<|start|>assistant to=user<|message|>{ANSWER}<|eom|>"+ + CALL_BLOCKS[tool]+ )+ return tool, reasoning, texttext = f"{think_open}{THINKING}{think_close}{ANSWER} {CALL_BLOCKS[tool]}"return tool, reasoning, text@@ -420,6 +437,12 @@ ARGS_STREAMING_BLOCKS = {"<|start|>assistant<|channel|>commentary to=functions.read <|constrain|>json"f"<|message|>{json.dumps({'filePath': LONG_VALUE})}<|call|>"),+ "muse_glimmer": (+ "<|start|>assistant to=read<|message|><atem:function_calls>\n"+ '<atem:invoke name="read">\n'+ f"<atem:parameter name=\"filePath\">{LONG_VALUE}</atem:parameter>\n"+ "</atem:invoke>\n</atem:function_calls><|eot|>"+ ),}@@ -642,7 +665,7 @@ def test_empty_arguments_call_emitted_exactly_once(tool, block):@pytest.mark.parametrize("family",["qwen25", "qwen3_coder", "glm47", "gemma4", "minimax", "minimax_m3",- "deepseekv32", "gpt_oss"],+ "deepseekv32", "gpt_oss", "muse_glimmer"],)def test_call_then_trailing_text_in_one_chunk_keeps_order(family):# Worst case: the whole generation arrives as ONE chunk. Text after the calldiff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.pyindex 4b64e04c..65fa5f2d 100644--- a/tests/tokenizer/test_tokenize.py+++ b/tests/tokenizer/test_tokenize.py@@ -18,9 +18,12 @@ class FakeTokenizer:self.chat_template_kwargs = kwargsreturn "rendered prompt"- def encode(self, prompt, return_tensors=None):+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):assert prompt == "rendered prompt"assert return_tensors == "pt"+ # The template rendered every special token already; encode must not+ # add another bos on top (the muse-glimmer/llama double-bos bug).+ assert add_special_tokens is Falsereturn torch.tensor([[1, 2, 3]], dtype=torch.long)@@ -78,9 +81,11 @@ class FakeDsv4Tokenizer:self.name_or_path = str(model_path)self.prompt = None- def encode(self, prompt, return_tensors=None):+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):self.prompt = promptassert return_tensors == "pt"+ # dsv4's own encoder path keeps the default special-token behavior.+ assert add_special_tokens is Truereturn torch.tensor([[4, 5, 6]], dtype=torch.long)@@ -197,7 +202,7 @@ class Qwen38LikeTokenizer:return f"prompt effort={effort}"return "prompt effort=off"- def encode(self, prompt, return_tensors=None):+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):return torch.tensor([[7, 8]], dtype=torch.long)