Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

Support muse glimmer (#4)

Shuo Yang6223cda1b0f1
diff --git a/docs/models.md b/docs/models.md
index fd4e0a7e..e4850a12 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -15,6 +15,7 @@ for them; other checkpoints of the same architectures work too.
| gpt-oss | [openai/gpt-oss-120b](https://huggingface.co/openai/gpt-oss-120b), [openai/gpt-oss-20b](https://huggingface.co/openai/gpt-oss-20b) |
| Gemma-4 | [google/gemma-4-26B-A4B-it](https://huggingface.co/google/gemma-4-26B-A4B-it), [nvidia/Gemma-4-26B-A4B-NVFP4](https://huggingface.co/nvidia/Gemma-4-26B-A4B-NVFP4), [google/gemma-4-12B-it](https://huggingface.co/google/gemma-4-12B-it), [nvidia/Gemma-4-31B-IT-NVFP4](https://huggingface.co/nvidia/Gemma-4-31B-IT-NVFP4) .. |
| MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) |
+| Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |
## MoE backends
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 3d195a43..a00154d0 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -316,6 +316,13 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
hidden_size=5376,
kv_groups=((4, 512), (16, 256)),
),
+ AotModel(
+ name="meta-models/Muse-Glimmer-30B",
+ architecture="MuseGlimmerForConditionalGeneration",
+ hidden_size=6656,
+ kv_groups=((2, 128),), # sliding and full (NoPE) layers share the same geometry
+ aliases=("RedHatAI/Muse-Glimmer-30B-NVFP4",),
+ ),
AotModel(
name="meta-llama/Llama-3.1-8B-Instruct",
architecture="LlamaForCausalLM",
diff --git a/python/freetoken/models/config.py b/python/freetoken/models/config.py
index 3dadbdec..f6105e1f 100644
--- a/python/freetoken/models/config.py
+++ b/python/freetoken/models/config.py
@@ -34,6 +34,46 @@ def detect_expert_quant(hf_config: Any) -> str:
return "nvfp4" if "fp4" in str(algo).lower() else str(algo).lower()
+def detect_compressed_tensors_nvfp4(hf_config: Any) -> bool:
+ """Detect a compressed-tensors NVFP4 (W4A16) checkpoint (dense Qwen3.6-27B,
+ Muse-Glimmer-30B-NVFP4, ...).
+
+ llm-compressor stores ``quant_method == "compressed-tensors"`` with a ``config_groups``
+ map whose ``weights`` are ``num_bits=4, type="float", group_size=16,
+ strategy="tensor_group"`` -- the NVFP4 layout (``weight_packed`` uint8 +
+ ``weight_scale`` fp8 block + scalar ``weight_global_scale``). All ``targets:
+ ["Linear"]`` are NVFP4 except the per-module ``ignore`` list. A 4-bit float scheme
+ with a DIFFERENT geometry (MXFP4: group_size 32, real since LLM Compressor 0.9)
+ raises instead of routing into the NVFP4 loader and dying in a shape assert."""
+ quant = getattr(hf_config, "quantization_config", None)
+ if quant is None:
+ return False
+ get = quant.get if isinstance(quant, dict) else (lambda k, d=None: getattr(quant, k, d))
+ if str(get("quant_method") or "").lower() != "compressed-tensors":
+ return False
+ groups = get("config_groups") or {}
+ # Verdicts are collected across ALL groups before returning: an early return on
+ # the first NVFP4 group would accept a mixed {nvfp4, mxfp4} checkpoint (and the
+ # error would depend on the groups' key order).
+ saw_nvfp4 = False
+ for g in (groups.values() if isinstance(groups, dict) else []):
+ w = (g or {}).get("weights") or {}
+ if int(w.get("num_bits", 0) or 0) != 4 or str(w.get("type", "")).lower() != "float":
+ continue
+ # vLLM's _is_nvfp4_format gates on the same two fields.
+ group_size = int(w.get("group_size", 0) or 0)
+ strategy = str(w.get("strategy", "")).lower()
+ if group_size == 16 and strategy == "tensor_group":
+ saw_nvfp4 = True
+ continue
+ raise ValueError(
+ "unsupported compressed-tensors 4-bit float scheme "
+ f"(group_size={group_size}, strategy={strategy!r}); FreeToken serves "
+ "NVFP4 (group_size=16, strategy=tensor_group) only"
+ )
+ return saw_nvfp4
+
+
@dataclass(frozen=True)
class RotaryConfig:
head_dim: int
@@ -221,6 +261,12 @@ class ModelConfig:
attn_sm_scale: float | None = None # None -> 1/sqrt(head_dim)
final_logit_softcapping: float | None = None
embedding_scale: float | None = None
+ # Second RMSNorm eps for models whose post-sublayer norms use a different eps than the
+ # pre-sublayer ones (muse_glimmer: post_attention/post_feedforward at 1e-8 vs 1e-5).
+ post_norm_eps: float | None = None
+ # Scalar the raw lm_head logits are multiplied by BEFORE final_logit_softcapping
+ # (muse_glimmer: 1/sqrt(hidden/256)); None leaves the logits unscaled.
+ output_multiplier: float | None = None
vision_config: Any | None = None
image_token_id: int | None = None
attention_groups: Tuple[AttentionGroupConfig, ...] = ()
diff --git a/python/freetoken/models/loader.py b/python/freetoken/models/loader.py
index 5d4571eb..49419364 100644
--- a/python/freetoken/models/loader.py
+++ b/python/freetoken/models/loader.py
@@ -133,6 +133,125 @@ def iter_merged_tensors(
)
+# ---------------------------------------------------------------------------------
+# compressed-tensors NVFP4 (llm-compressor) dense-weight helpers, shared by the
+# models that serve such checkpoints natively (qwen3_5_moe, muse_glimmer). Storage:
+# ``weight_packed`` (uint8 [O, IN//2]) + ``weight_scale`` (fp8-e4m3 block [O, IN//16])
+# + a scalar ``weight_global_scale``. The stored global is the *quant-side* scale, so
+# the dequant/native global is its reciprocal (vLLM inverts it identically).
+# ---------------------------------------------------------------------------------
+
+# Quant scales consumed with their ``weight_packed`` (or unused: the input scales are
+# for W4A4 activation quant, which FreeToken does not run).
+CT_SCALE_SUFFIXES = (
+ ".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale",
+)
+
+
+class ShardReader:
+ """Serves tensors by name across safetensors shards (handles opened lazily).
+
+ The quant scales of a ``weight_packed`` can land in a DIFFERENT shard than the
+ packed weight itself (Muse-Glimmer-30B-NVFP4 splits layer 49's down_proj across
+ the shard boundary), so sibling lookups must go through the index's weight_map
+ rather than the shard the packed weight came from."""
+
+ def __init__(self, model_path: str, device: torch.device):
+ folder = download_hf_weight(model_path)
+ index = os.path.join(folder, "model.safetensors.index.json")
+ if os.path.exists(index):
+ with open(index, encoding="utf-8") as f:
+ weight_map = json.load(f)["weight_map"]
+ self._map = {
+ name: os.path.join(folder, shard) for name, shard in weight_map.items()
+ }
+ else: # single-file checkpoint
+ import safetensors
+
+ self._map = {}
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device="cpu") as f:
+ for name in f.keys():
+ self._map[name] = file
+ self._device = str(device)
+ self._handles: dict[str, object] = {}
+
+ def files(self) -> list[str]:
+ return sorted(set(self._map.values()))
+
+ def names_in(self, file: str) -> list[str]:
+ return [name for name, shard in self._map.items() if shard == file]
+
+ def get_tensor(self, name: str) -> torch.Tensor:
+ import safetensors
+
+ file = self._map[name]
+ h = self._handles.get(file)
+ if h is None:
+ h = safetensors.safe_open(file, framework="pt", device=self._device).__enter__()
+ self._handles[file] = h
+ return h.get_tensor(name)
+
+ def close(self) -> None:
+ for h in self._handles.values():
+ try:
+ h.__exit__(None, None, None)
+ except Exception: # noqa: BLE001 -- best-effort handle cleanup
+ pass
+ self._handles.clear()
+
+
+def nvfp4_parts_ct(f, raw_base: str) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16],
+ per-output-row global fp16 [O])`` for the W4A16 kernels."""
+ w = f.get_tensor(raw_base + ".weight_packed")
+ s = f.get_tensor(raw_base + ".weight_scale")
+ wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)
+ g = (1.0 / wg).to(torch.float16).expand(w.shape[0]).contiguous()
+ return w, s, g
+
+
+def ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict, groups: dict[str, tuple[str, ...]]):
+ """Buffer a native NVFP4 fusion part ``(w, s, g)``; emit the concatenated native parts
+ (``.weight``/``.weight_scale``/``.weight_global``, output-dim concat with each part
+ keeping its own scales, so the fused FP4 weight is exact) once complete, ``[]`` while
+ incomplete, ``None`` if ``base`` is not a fusion part of any group in ``groups``."""
+ for fused_suffix, parts in groups.items():
+ for idx, part in enumerate(parts):
+ if base.endswith(part):
+ key = base[: -len(part)] + fused_suffix
+ slots = buf.setdefault(key, {})
+ slots[idx] = parts_tuple
+ if len(slots) < len(parts):
+ return []
+ del buf[key]
+ ws = [slots[i][0] for i in range(len(parts))]
+ ss = [slots[i][1] for i in range(len(parts))]
+ gs = [slots[i][2] for i in range(len(parts))]
+ return [
+ (key + ".weight", torch.cat(ws, dim=0)),
+ (key + ".weight_scale", torch.cat(ss, dim=0)),
+ (key + ".weight_global", torch.cat(gs, dim=0)),
+ ]
+ return None
+
+
+def ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict[str, tuple[str, ...]]):
+ """Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]``
+ while incomplete, ``None`` if ``base`` is not a part of any group in ``groups``."""
+ for fused_suffix, parts in groups.items():
+ for idx, part in enumerate(parts):
+ if base.endswith(part):
+ key = base[: -len(part)] + fused_suffix
+ slots = buf.setdefault(key, {})
+ slots[idx] = tensor
+ if len(slots) < len(parts):
+ return []
+ del buf[key]
+ return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))]
+ return None
+
+
def _expert_stack_info(key: str, expert_pattern: re.Pattern[str]) -> tuple[str, int] | None:
match = expert_pattern.match(key)
if match is None:
diff --git a/python/freetoken/models/muse_glimmer/__init__.py b/python/freetoken/models/muse_glimmer/__init__.py
new file mode 100644
index 00000000..307d2866
--- /dev/null
+++ b/python/freetoken/models/muse_glimmer/__init__.py
@@ -0,0 +1,11 @@
+from .attention import MuseGlimmerAttention
+from .config import parse_config
+from .model import MuseGlimmerForCausalLM
+from .weight import iter_weights
+
+__all__ = [
+ "MuseGlimmerAttention",
+ "MuseGlimmerForCausalLM",
+ "parse_config",
+ "iter_weights",
+]
diff --git a/python/freetoken/models/muse_glimmer/attention.py b/python/freetoken/models/muse_glimmer/attention.py
new file mode 100644
index 00000000..29764dcf
--- /dev/null
+++ b/python/freetoken/models/muse_glimmer/attention.py
@@ -0,0 +1,100 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import torch
+from freetoken.attention import AttentionSpec
+from freetoken.core import get_global_ctx
+from freetoken.layers import BaseOP, GemmaRMSNorm
+from freetoken.layers.rotary import get_rope
+from freetoken.models.config import SWAAttentionGroupConfig
+from freetoken.utils import nvtx_annotate
+
+from freetoken.models.quant_linear import make_col_merged, make_replicated
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class MuseGlimmerAttention(BaseOP):
+ """Gated GQA for one sliding-window or full-attention (NoPE) layer.
+
+ q, k, v, gate = split(qkvg_proj(x))
+ q = qk_norm(q); k = qk_norm(k) # weightless per-head RMSNorm
+ q, k = rope(q, k) # sliding layers only; full layers are NoPE
+ attn = paged_attention(q, k, v) # sm_scale carries qk_scale_factor/sqrt(d)
+ out = o_proj(attn * sigmoid(gate))
+
+ The reference multiplies q by ``qk_scale_factor`` before rope; rope is a rotation, so
+ the factor commutes and is folded into ``AttentionSpec.sm_scale`` instead (parse_config
+ sets ``attn_sm_scale = qk_scale_factor / sqrt(head_dim)``). The gate is computed from
+ the same normed layer input as q/k/v, so it rides the fused projection.
+ """
+
+ def __init__(self, config: ModelConfig, layer_id: int):
+ self.layer_id = layer_id
+ group = config.attention_group_for_layer(layer_id)
+ self.is_swa = isinstance(group, SWAAttentionGroupConfig)
+ head_dim = group.head_dim
+ self.head_dim = head_dim
+ self.num_q = config.num_qo_heads
+ self.num_kv = group.num_kv_heads
+ self.qo_attn_dim = self.num_q * head_dim
+ self.kv_attn_dim = self.num_kv * head_dim
+
+ self._qkvg_split = [self.qo_attn_dim, self.kv_attn_dim, self.kv_attn_dim, self.qo_attn_dim]
+ self.qkvg_proj = make_col_merged(
+ config, config.hidden_size, self._qkvg_split, has_bias=False
+ )
+ self.o_proj = make_replicated(config, self.qo_attn_dim, config.hidden_size, has_bias=False)
+ # Weightless: the checkpoint carries no q/k norm weights; the runtime ones vector
+ # is intentionally not part of state_dict.
+ self.qk_norm = GemmaRMSNorm(head_dim, eps=config.rms_norm_eps, with_scale=False)
+ self.attn_spec = AttentionSpec(
+ sliding_window=group.sliding_window if self.is_swa else None,
+ sm_scale=config.attn_sm_scale,
+ )
+ # base == 0.0 marks the NoPE full-attention layers; RotaryEmbedding must never
+ # see it (0 ** x in the inv_freq computation).
+ rotary_config = group.rotary_config
+ self.rotary = (
+ get_rope(
+ head_dim=head_dim,
+ rotary_dim=rotary_config.rotary_dim,
+ max_position=rotary_config.max_position,
+ base=rotary_config.base,
+ )
+ if rotary_config.base > 0.0
+ else None
+ )
+
+ @nvtx_annotate("MHA")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ ctx = get_global_ctx()
+ T = x.shape[0]
+
+ qkvg = self.qkvg_proj.forward(x)
+ q, k, v, gate = torch.split(qkvg, self._qkvg_split, dim=-1)
+ del qkvg
+ q = self.qk_norm.forward(q.contiguous().view(T, self.num_q, self.head_dim))
+ k = self.qk_norm.forward(k.contiguous().view(T, self.num_kv, self.head_dim))
+ v = v.contiguous() # the split view has the qkvg row stride; the KV store needs contiguous
+
+ q = q.reshape(T, self.qo_attn_dim)
+ k = k.reshape(T, self.kv_attn_dim)
+ if self.rotary is not None:
+ q, k = self.rotary.forward(ctx.batch.positions, q, k)
+
+ o = ctx.attn_backend.forward(
+ q.view(T, self.num_q, self.head_dim),
+ k,
+ v,
+ self.layer_id,
+ ctx.batch,
+ attn_spec=self.attn_spec,
+ )
+ gated = o.reshape(T, self.qo_attn_dim) * torch.sigmoid(gate)
+ return self.o_proj.forward(gated)
+
+
+__all__ = ["MuseGlimmerAttention"]
diff --git a/python/freetoken/models/muse_glimmer/config.py b/python/freetoken/models/muse_glimmer/config.py
new file mode 100644
index 00000000..4cc756cb
--- /dev/null
+++ b/python/freetoken/models/muse_glimmer/config.py
@@ -0,0 +1,136 @@
+from __future__ import annotations
+
+from typing import Any
+
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ ModelConfig,
+ RotaryConfig,
+ SWAAttentionGroupConfig,
+ detect_compressed_tensors_nvfp4,
+)
+
+_SWA_TYPE = "sliding_attention"
+_FULL_TYPE = "full_attention"
+
+
+def _text_config(hf_config: Any) -> Any:
+ """Muse Glimmer ships as a multimodal wrapper (MuseGlimmerForConditionalGeneration):
+ the text tower lives in ``text_config`` and the weights carry a ``language_model.``
+ prefix. Served text-only -- the ~1.8B ViT perception encoder is never built, so
+ ``ModelConfig.vision_config`` stays None and the loader drops the vision tensors."""
+ text = getattr(hf_config, "text_config", None)
+ return text if text is not None else hf_config
+
+
+def _group_rope_theta(text: Any, layer_ids: tuple[int, ...], default: float) -> float:
+ """Rope theta of one layer group. ``layer_rope_theta`` is the per-layer source of
+ truth (0 marks the NoPE full-attention layers); absent, fall back to ``default``
+ (the shared ``rope_parameters.rope_theta``)."""
+ per_layer = getattr(text, "layer_rope_theta", None)
+ if not per_layer:
+ return default
+ thetas = {float(per_layer[i]) for i in layer_ids}
+ assert len(thetas) == 1, f"mixed rope thetas within one attention group: {thetas}"
+ return thetas.pop()
+
+
+def parse_config(hf_config: Any) -> ModelConfig:
+ text = _text_config(hf_config)
+
+ head_dim = getattr(text, "head_dim", None) or text.hidden_size // text.num_attention_heads
+ num_kv_heads = getattr(text, "num_key_value_heads", text.num_attention_heads)
+ max_position = text.max_position_embeddings
+
+ rope_params = getattr(text, "rope_parameters", None) or {}
+ base_theta = float(rope_params.get("rope_theta", getattr(text, "rope_theta", 500000.0)))
+
+ layer_types = list(text.layer_types)
+ swa_ids = tuple(i for i, t in enumerate(layer_types) if t == _SWA_TYPE)
+ full_ids = tuple(i for i, t in enumerate(layer_types) if t == _FULL_TYPE)
+ assert len(swa_ids) + len(full_ids) == len(layer_types), (
+ f"unknown layer types in {set(layer_types)}"
+ )
+
+ swa_rotary = RotaryConfig(
+ head_dim=head_dim,
+ rotary_dim=head_dim,
+ max_position=max_position,
+ base=_group_rope_theta(text, swa_ids, base_theta),
+ scaling=None,
+ )
+ # Full-attention layers are NoPE (layer_rope_theta == 0): base 0.0 is the marker the
+ # attention module reads to skip rope entirely. It must never reach RotaryEmbedding
+ # (0 ** x); MuseGlimmerAttention guards on it.
+ full_rotary = RotaryConfig(
+ head_dim=head_dim,
+ rotary_dim=head_dim,
+ max_position=max_position,
+ base=_group_rope_theta(text, full_ids, 0.0),
+ scaling=None,
+ )
+
+ # The reference applies a weightless per-head RMSNorm to q and k, then multiplies q by
+ # qk_scale_factor and runs standard 1/sqrt(head_dim) attention. q is not cached, so the
+ # factor folds exactly into the softmax scale.
+ qk_scale_factor = float(getattr(text, "qk_scale_factor", 1.0))
+ attn_sm_scale = qk_scale_factor * head_dim**-0.5
+
+ # RedHatAI/Muse-Glimmer-30B-NVFP4: every text-tower Linear (q/k/v/o + the attention
+ # gate + the MLP) is compressed-tensors NVFP4 (W4A16); lm_head, embeddings, norms and
+ # the (unserved) vision tower stay bf16.
+ nvfp4 = detect_compressed_tensors_nvfp4(hf_config)
+ quant = "nvfp4" if nvfp4 else "none"
+
+ return ModelConfig(
+ num_layers=text.num_hidden_layers,
+ num_qo_heads=text.num_attention_heads,
+ num_kv_heads=num_kv_heads,
+ head_dim=head_dim,
+ hidden_size=text.hidden_size,
+ vocab_size=text.vocab_size,
+ intermediate_size=text.intermediate_size,
+ hidden_act=getattr(text, "hidden_activation", "silu"),
+ rms_norm_eps=text.rms_norm_eps,
+ post_norm_eps=float(getattr(text, "post_norm_eps", text.rms_norm_eps)),
+ tie_word_embeddings=bool(getattr(text, "tie_word_embeddings", False)),
+ rotary_config=swa_rotary,
+ num_experts=0,
+ num_experts_per_tok=0,
+ moe_intermediate_size=0,
+ norm_topk_prob=False,
+ model_type=getattr(hf_config, "model_type", "muse_glimmer"),
+ architectures=(
+ getattr(hf_config, "architectures", None)
+ or ["MuseGlimmerForConditionalGeneration"]
+ ),
+ use_qk_norm=True,
+ attn_sm_scale=attn_sm_scale,
+ final_logit_softcapping=getattr(text, "final_logit_softcapping", None),
+ output_multiplier=getattr(text, "output_multiplier", None),
+ vision_config=None, # served text-only
+ image_token_id=getattr(hf_config, "image_token_id", None),
+ attn_quant=quant,
+ dense_quant=quant,
+ lm_head_quant="none",
+ attention_groups=(
+ FullAttentionGroupConfig(
+ name="full",
+ layer_ids=full_ids,
+ num_kv_heads=num_kv_heads,
+ head_dim=head_dim,
+ rotary_config=full_rotary,
+ ),
+ SWAAttentionGroupConfig(
+ name="swa",
+ layer_ids=swa_ids,
+ num_kv_heads=num_kv_heads,
+ head_dim=head_dim,
+ rotary_config=swa_rotary,
+ sliding_window=text.sliding_window,
+ ),
+ ),
+ )
+
+
+__all__ = ["parse_config"]
diff --git a/python/freetoken/models/muse_glimmer/model.py b/python/freetoken/models/muse_glimmer/model.py
new file mode 100644
index 00000000..ba632c0b
--- /dev/null
+++ b/python/freetoken/models/muse_glimmer/model.py
@@ -0,0 +1,134 @@
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+import torch
+from freetoken.core import get_global_ctx
+from freetoken.layers import (
+ BaseOP,
+ GemmaPlusOneRMSNorm,
+ GemmaPlusOneRMSNormFused,
+ GemmaRMSNorm,
+ OPList,
+ ParallelLMHead,
+ RMSNorm,
+ VocabParallelEmbedding,
+ silu_and_mul,
+)
+from freetoken.models.blocks import BaseLLMModel
+from freetoken.utils import nvtx_annotate
+
+from .attention import MuseGlimmerAttention
+from freetoken.models.quant_linear import make_col_merged, make_replicated
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+
+
+class MuseGlimmerMLP(BaseOP):
+ """SwiGLU MLP (gate|up fused), NVFP4 (W4A16) on the quantized checkpoint else bf16."""
+
+ def __init__(self, config: ModelConfig):
+ self.gate_up_proj = make_col_merged(
+ config,
+ config.hidden_size,
+ [config.intermediate_size, config.intermediate_size],
+ has_bias=False,
+ )
+ self.down_proj = make_replicated(
+ config, config.intermediate_size, config.hidden_size, has_bias=False
+ )
+
+ @nvtx_annotate("MLP")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return self.down_proj.forward(silu_and_mul(self.gate_up_proj.forward(x)))
+
+
+class MuseGlimmerDecoderLayer(BaseOP):
+ """Sandwich block with centered (1+w) norms:
+
+ h = post_attention_layernorm(attn(input_layernorm(x))); x = x + h
+ h = post_feedforward_layernorm(mlp(pre_feedforward_layernorm(x))); x = x + h
+
+ The pre-norms use ``rms_norm_eps``, the post-norms the tighter ``post_norm_eps``
+ (1e-8). The (1+w) scaling stays a runtime fp32 add (GemmaPlusOneRMSNorm) -- baking
+ +1 into the bf16 weight would round away exactly the precision the centered form
+ exists to keep.
+ """
+
+ def __init__(self, config: ModelConfig, layer_id: int):
+ self._layer_id = layer_id
+ self.self_attn = MuseGlimmerAttention(config, layer_id)
+ self.mlp = MuseGlimmerMLP(config)
+ H = config.hidden_size
+ post_eps = config.post_norm_eps if config.post_norm_eps is not None else config.rms_norm_eps
+ self.input_layernorm = GemmaPlusOneRMSNorm(H, eps=config.rms_norm_eps)
+ self.post_attention_layernorm = GemmaPlusOneRMSNorm(H, eps=post_eps)
+ self.pre_feedforward_layernorm = GemmaPlusOneRMSNormFused(H, eps=config.rms_norm_eps)
+ self.post_feedforward_layernorm = GemmaPlusOneRMSNorm(H, eps=post_eps)
+
+ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ residual = x
+ h = self.input_layernorm.forward(x)
+ h = self.self_attn.forward(h)
+ h = self.post_attention_layernorm.forward(h)
+ # Fused add + norm: residual becomes x + h, pre_ff its norm.
+ pre_ff, residual = self.pre_feedforward_layernorm.forward(h, residual)
+ h = self.mlp.forward(pre_ff)
+ h = self.post_feedforward_layernorm.forward(h)
+ return residual + h
+
+
+class MuseGlimmerModel(BaseOP):
+ def __init__(self, config: ModelConfig):
+ self.embed_tokens = VocabParallelEmbedding(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ )
+ # NormedEmbedding: a weightless RMSNorm on top of the embeddings. Kept separate
+ # from the embedding matrix (the reference cannot fold it either -- the DFlash
+ # drafter embeds without the norm); no weight, so it never appears in state_dict.
+ self.embed_norm = GemmaRMSNorm(
+ config.hidden_size, eps=config.rms_norm_eps, with_scale=False
+ )
+ self.layers = OPList(
+ [MuseGlimmerDecoderLayer(config, layer_id) for layer_id in range(config.num_layers)]
+ )
+ # Final norm scales by the raw checkpoint weight (plain RMSNorm, not the
+ # centered (1+w) form the decoder norms use).
+ self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
+
+ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+ x = self.embed_norm.forward(self.embed_tokens.forward(input_ids))
+ for layer in self.layers.op_list:
+ x = layer.forward(x)
+ return self.norm.forward(x)
+
+
+class MuseGlimmerForCausalLM(BaseLLMModel):
+ def __init__(self, config: ModelConfig):
+ self.model = MuseGlimmerModel(config)
+ self.lm_head = ParallelLMHead(
+ num_embeddings=config.vocab_size,
+ embedding_dim=config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ )
+ self._output_multiplier = config.output_multiplier
+ self._final_logit_softcapping = config.final_logit_softcapping
+ super().__init__()
+
+ def forward(self) -> torch.Tensor:
+ output = self.model.forward(get_global_ctx().batch.input_ids)
+ logits = self.lm_head.forward(output)
+ # Gemma-style logit post-processing, with a pre-scale: T * tanh(logits * mult / T).
+ if self._output_multiplier is not None:
+ logits = logits * self._output_multiplier
+ if self._final_logit_softcapping is not None:
+ cap = self._final_logit_softcapping
+ logits = torch.tanh(logits / cap) * cap
+ return logits
+
+
+__all__ = ["MuseGlimmerForCausalLM"]
diff --git a/python/freetoken/models/muse_glimmer/weight.py b/python/freetoken/models/muse_glimmer/weight.py
new file mode 100644
index 00000000..8a613225
--- /dev/null
+++ b/python/freetoken/models/muse_glimmer/weight.py
@@ -0,0 +1,138 @@
+from __future__ import annotations
+
+from typing import Iterator
+
+import safetensors
+import torch
+from freetoken.distributed import get_tp_info
+from freetoken.models.config import detect_compressed_tensors_nvfp4
+from freetoken.models.loader import (
+ CT_SCALE_SUFFIXES,
+ ShardReader,
+ ct_bf16_fuse,
+ ct_nvfp4_fuse,
+ iter_weight_files,
+ nvfp4_parts_ct,
+)
+from freetoken.utils import cached_load_hf_config
+from tqdm import tqdm
+
+# Vision stack of the multimodal wrapper -- served text-only, always dropped.
+_VISION_PREFIXES = (
+ "model.vision_tower.",
+ "model.vision_adapter.",
+ "model.vision_projection.",
+ "vision_tower.",
+ "vision_adapter.",
+ "vision_projection.",
+)
+
+# Fused projections, concatenated on the output dim in this exact order to match the
+# model's merged-linear splits. The attention gate rides the q/k/v fusion (it is computed
+# from the same layer input); ``.self_attn.gate_proj`` and the SwiGLU ``.mlp.gate_proj``
+# are disambiguated by the full suffix.
+_FUSIONS: dict[str, tuple[str, ...]] = {
+ ".self_attn.qkvg_proj": (
+ ".self_attn.q_proj", ".self_attn.k_proj", ".self_attn.v_proj", ".self_attn.gate_proj",
+ ),
+ ".mlp.gate_up_proj": (".mlp.gate_proj", ".mlp.up_proj"),
+}
+
+
+def _rename(raw_name: str) -> str | None:
+ """HF key -> FreeToken state-dict key, or None to skip (vision stack)."""
+ if raw_name.startswith(_VISION_PREFIXES):
+ return None
+ if raw_name.startswith("model.language_model."):
+ return "model." + raw_name[len("model.language_model.") :]
+ if raw_name.startswith("language_model."):
+ return "model." + raw_name[len("language_model.") :]
+ return raw_name # lm_head.weight
+
+
+def iter_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ if not include_non_moe:
+ return # dense model: there is no experts-only pass
+ if get_tp_info().size > 1:
+ raise NotImplementedError("muse_glimmer weight loading currently supports TP=1 only")
+
+ if detect_compressed_tensors_nvfp4(cached_load_hf_config(model_path)):
+ yield from _iter_weights_compressed_tensors(model_path, device)
+ return
+
+ fuse_buf: dict = {}
+ for file in tqdm(
+ iter_weight_files(model_path),
+ desc="Loading weights",
+ disable=not get_tp_info().is_primary(),
+ ):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
+ for raw_name in f.keys():
+ name = _rename(raw_name)
+ if name is None:
+ continue
+ tensor = f.get_tensor(raw_name)
+ if name.endswith(".weight"):
+ emit = ct_bf16_fuse(name[: -len(".weight")], tensor, fuse_buf, _FUSIONS)
+ if emit is not None:
+ yield from emit
+ continue
+ # All norms pass through raw: the decoder's centered (1+w) norms apply the
+ # +1 at runtime (GemmaPlusOneRMSNorm), the final norm / qk norms need none.
+ yield name, tensor
+
+ assert not fuse_buf, f"Incomplete projection fusions: {list(fuse_buf.keys())}"
+
+
+def _iter_weights_compressed_tensors(
+ model_path: str, device: torch.device
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """Dense pass for the compressed-tensors NVFP4 checkpoint.
+
+ Every text Linear (q/k/v/o, the attention gate, the MLP) is kept native NVFP4
+ (W4A16): ``.weight`` (uint8 packed) + ``.weight_scale`` (fp8 block) + ``.weight_global``
+ (fp16 per-row, the reciprocal of the stored quant-side global). q/k/v/gate fuse into
+ ``qkvg_proj`` and gate/up into ``gate_up_proj`` on the output dim, each part keeping
+ its own scales, so the fused FP4 weights are exact. Embeddings, norms and lm_head are
+ bf16 (the checkpoint's ignore list). Scale lookups go through the shard-map reader:
+ they can land in a different shard than their weight_packed."""
+ nvfp4_buf: dict = {}
+ reader = ShardReader(model_path, device)
+ try:
+ for file in tqdm(
+ reader.files(),
+ desc="Loading compressed-tensors weights",
+ disable=not get_tp_info().is_primary(),
+ ):
+ for raw_name in reader.names_in(file):
+ if raw_name.endswith(CT_SCALE_SUFFIXES):
+ continue # consumed with their weight_packed
+ name = _rename(raw_name)
+ if name is None:
+ continue
+ if raw_name.endswith(".weight_packed"):
+ base = name[: -len(".weight_packed")]
+ parts = nvfp4_parts_ct(reader, raw_name[: -len(".weight_packed")])
+ emit = ct_nvfp4_fuse(base, parts, nvfp4_buf, _FUSIONS)
+ if emit is not None:
+ yield from emit
+ else: # standalone: o_proj, down_proj
+ w, s, g = parts
+ yield base + ".weight", w
+ yield base + ".weight_scale", s
+ yield base + ".weight_global", g
+ continue
+ yield name, reader.get_tensor(raw_name)
+ finally:
+ reader.close()
+
+ assert not nvfp4_buf, f"Incomplete NVFP4 fusions: {list(nvfp4_buf.keys())}"
+
+
+__all__ = ["iter_weights"]
diff --git a/python/freetoken/models/quant_linear.py b/python/freetoken/models/quant_linear.py
new file mode 100644
index 00000000..6ba67a9e
--- /dev/null
+++ b/python/freetoken/models/quant_linear.py
@@ -0,0 +1,78 @@
+"""Quant-aware dense-linear factories, shared by the models that serve quantized
+dense projections (qwen3_5_moe, muse_glimmer).
+
+Maps the model's quant config (``expert_quant`` for the dense MLP / shared-expert path,
+``attn_quant`` for attention + GatedDeltaNet projections) to the right ``BaseOP`` linear:
+block-FP8, per-tensor-FP8 and NVFP4 implementations live under ``freetoken.kernel.triton``;
+the bf16 fallback is the framework's TP-aware ``freetoken.layers``. Only the *dispatch*
+(config -> layer class) lives here.
+"""
+
+from __future__ import annotations
+
+
+def make_col_merged_quant(expert_quant: str, attn_quant: str, in_f: int,
+ output_sizes: list[int], has_bias: bool = False):
+ """Column-merged linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""
+ if expert_quant == "fp8_block":
+ from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged
+
+ return Fp8BlockColMerged(in_f, output_sizes, has_bias)
+ if attn_quant == "fp8_pertensor":
+ from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
+
+ return Fp8PerTensorColMerged(in_f, output_sizes, has_bias)
+ if attn_quant == "nvfp4": # compressed-tensors W4A16 attention (q/k/v fused)
+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged
+
+ return Nvfp4DenseColMerged(in_f, output_sizes, has_bias)
+ from freetoken.layers import LinearColParallelMerged
+
+ return LinearColParallelMerged(in_f, output_sizes, has_bias=has_bias)
+
+
+def make_replicated_quant(expert_quant: str, attn_quant: str, in_f: int, out_f: int,
+ has_bias: bool = False):
+ """Replicated linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""
+ if expert_quant == "fp8_block":
+ from freetoken.kernel.triton.fp8_block_linear import Fp8BlockLinear
+
+ return Fp8BlockLinear(in_f, out_f, has_bias)
+ if attn_quant == "fp8_pertensor":
+ from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear
+
+ return Fp8PerTensorLinear(in_f, out_f, has_bias)
+ if attn_quant == "nvfp4": # compressed-tensors W4A16 attention o_proj / GDN out_proj
+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseLinear
+
+ return Nvfp4DenseLinear(in_f, out_f, has_bias)
+ from freetoken.layers import LinearReplicated
+
+ return LinearReplicated(in_f, out_f, has_bias=has_bias)
+
+
+def make_replicated(config, in_f: int, out_f: int, has_bias: bool = False):
+ """Config-driven replicated linear: ``Fp8BlockLinear`` under block-fp8, ``Fp8PerTensorLinear``
+ under per-tensor-fp8 attention, ``Nvfp4DenseLinear`` under nvfp4, else ``LinearReplicated``."""
+ return make_replicated_quant(
+ getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),
+ in_f, out_f, has_bias,
+ )
+
+
+def make_col_merged(config, in_f: int, output_sizes: list[int], has_bias: bool = False):
+ """Config-driven column-merged linear: ``Fp8BlockColMerged`` under block-fp8,
+ ``Fp8PerTensorColMerged`` under per-tensor-fp8 attention, ``Nvfp4DenseColMerged`` under
+ nvfp4, else ``LinearColParallelMerged``."""
+ return make_col_merged_quant(
+ getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),
+ in_f, output_sizes, has_bias,
+ )
+
+
+__all__ = [
+ "make_col_merged_quant",
+ "make_replicated_quant",
+ "make_replicated",
+ "make_col_merged",
+]
diff --git a/python/freetoken/models/qwen3_5_moe/config.py b/python/freetoken/models/qwen3_5_moe/config.py
index 74bef262..dc16cfff 100644
--- a/python/freetoken/models/qwen3_5_moe/config.py
+++ b/python/freetoken/models/qwen3_5_moe/config.py
@@ -7,6 +7,7 @@ from freetoken.models.config import (
LinearGatedDeltaGroupConfig,
ModelConfig,
RotaryConfig,
+ detect_compressed_tensors_nvfp4,
)
@@ -62,25 +63,9 @@ def _expert_quant(hf_config: Any) -> str:
return "none"
-def _compressed_tensors_nvfp4(hf_config: Any) -> bool:
- """Detect a compressed-tensors NVFP4 (W4A16) checkpoint (e.g. dense Qwen3.6-27B).
-
- llm-compressor stores ``quant_method == "compressed-tensors"`` with a ``config_groups``
- map whose ``weights`` are ``num_bits=4, type="float", group_size=16`` -- the NVFP4 layout
- (``weight_packed`` uint8 + ``weight_scale`` fp8 block + scalar ``weight_global_scale``).
- All ``targets: ["Linear"]`` are NVFP4 except the per-module ``ignore`` list (lm_head,
- GDN in_proj_*, vision, mtp)."""
- get = _quant_accessor(hf_config)
- if get is None:
- return False
- if str(get("quant_method") or "").lower() != "compressed-tensors":
- return False
- groups = get("config_groups") or {}
- for g in (groups.values() if isinstance(groups, dict) else []):
- w = (g or {}).get("weights") or {}
- if int(w.get("num_bits", 0) or 0) == 4 and str(w.get("type", "")).lower() == "float":
- return True
- return False
+# Detection now lives in models/config.py (shared with muse_glimmer); weight.py imports
+# it under this name.
+_compressed_tensors_nvfp4 = detect_compressed_tensors_nvfp4
def _lm_head_quant(hf_config: Any) -> str:
diff --git a/python/freetoken/models/qwen3_5_moe/quant_linear.py b/python/freetoken/models/qwen3_5_moe/quant_linear.py
index e57a5393..cd0443b6 100644
--- a/python/freetoken/models/qwen3_5_moe/quant_linear.py
+++ b/python/freetoken/models/qwen3_5_moe/quant_linear.py
@@ -1,73 +1,15 @@
-"""Quant-aware dense-linear factories for Qwen3.5/3.6.
+"""Re-export of the shared quant-aware dense-linear factories (models/quant_linear.py).
-Maps the model's quant config (``expert_quant`` for the dense MLP / shared-expert path,
-``attn_quant`` for attention + GatedDeltaNet projections) to the right ``BaseOP`` linear:
-block-FP8, per-tensor-FP8 and NVFP4 implementations live under ``freetoken.kernel.triton``
-(shared across models); the bf16 fallback is the framework's TP-aware ``freetoken.layers``.
-Only the *dispatch* (config -> layer class) is model glue and stays here.
+The dispatch used to live here; muse_glimmer needing the same NVFP4/bf16 pair promoted
+it to a model-agnostic home. Kept as a module so existing imports stay valid.
"""
-from __future__ import annotations
-
-
-def make_col_merged_quant(expert_quant: str, attn_quant: str, in_f: int,
- output_sizes: list[int], has_bias: bool = False):
- """Column-merged linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""
- if expert_quant == "fp8_block":
- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockColMerged
-
- return Fp8BlockColMerged(in_f, output_sizes, has_bias)
- if attn_quant == "fp8_pertensor":
- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorColMerged
-
- return Fp8PerTensorColMerged(in_f, output_sizes, has_bias)
- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention (q/k/v fused)
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged
-
- return Nvfp4DenseColMerged(in_f, output_sizes, has_bias)
- from freetoken.layers import LinearColParallelMerged
-
- return LinearColParallelMerged(in_f, output_sizes, has_bias=has_bias)
-
-
-def make_replicated_quant(expert_quant: str, attn_quant: str, in_f: int, out_f: int,
- has_bias: bool = False):
- """Replicated linear for a dense projection: block-fp8 / per-tensor-fp8 / nvfp4 / bf16."""
- if expert_quant == "fp8_block":
- from freetoken.kernel.triton.fp8_block_linear import Fp8BlockLinear
-
- return Fp8BlockLinear(in_f, out_f, has_bias)
- if attn_quant == "fp8_pertensor":
- from freetoken.kernel.triton.fp8_pertensor_linear import Fp8PerTensorLinear
-
- return Fp8PerTensorLinear(in_f, out_f, has_bias)
- if attn_quant == "nvfp4": # compressed-tensors W4A16 attention o_proj / GDN out_proj
- from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseLinear
-
- return Nvfp4DenseLinear(in_f, out_f, has_bias)
- from freetoken.layers import LinearReplicated
-
- return LinearReplicated(in_f, out_f, has_bias=has_bias)
-
-
-def make_replicated(config, in_f: int, out_f: int, has_bias: bool = False):
- """Config-driven replicated linear: ``Fp8BlockLinear`` under block-fp8, ``Fp8PerTensorLinear``
- under per-tensor-fp8 attention, ``Nvfp4DenseLinear`` under nvfp4, else ``LinearReplicated``."""
- return make_replicated_quant(
- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),
- in_f, out_f, has_bias,
- )
-
-
-def make_col_merged(config, in_f: int, output_sizes: list[int], has_bias: bool = False):
- """Config-driven column-merged linear: ``Fp8BlockColMerged`` under block-fp8,
- ``Fp8PerTensorColMerged`` under per-tensor-fp8 attention, ``Nvfp4DenseColMerged`` under
- nvfp4, else ``LinearColParallelMerged``."""
- return make_col_merged_quant(
- getattr(config, "expert_quant", "none"), getattr(config, "attn_quant", "none"),
- in_f, output_sizes, has_bias,
- )
-
+from freetoken.models.quant_linear import (
+ make_col_merged,
+ make_col_merged_quant,
+ make_replicated,
+ make_replicated_quant,
+)
__all__ = [
"make_col_merged_quant",
diff --git a/python/freetoken/models/qwen3_5_moe/weight.py b/python/freetoken/models/qwen3_5_moe/weight.py
index 79622c32..926df7c8 100644
--- a/python/freetoken/models/qwen3_5_moe/weight.py
+++ b/python/freetoken/models/qwen3_5_moe/weight.py
@@ -9,7 +9,15 @@ import safetensors
import torch
from freetoken.distributed import get_tp_info
from freetoken.kernel.triton.nvfp4_dequant import dequant_nvfp4
-from freetoken.models.loader import drop_page_cache, iter_weight_files
+from freetoken.models.loader import (
+ CT_SCALE_SUFFIXES,
+ ShardReader,
+ ct_bf16_fuse,
+ ct_nvfp4_fuse,
+ drop_page_cache,
+ iter_weight_files,
+ nvfp4_parts_ct,
+)
from freetoken.models.nvfp4_banks import (
Nvfp4ExpertSourceSpec,
load_nvfp4_expert_source_banks,
@@ -525,59 +533,15 @@ _CT_BF16_FUSE: dict[str, tuple[str, ...]] = {
".linear_attn.in_proj_b", ".linear_attn.in_proj_a",
),
}
-# Activation/quant scales consumed with weight_packed (or unused: this is W4A16, the input
-# scales are for W4A4 which FreeToken does not run).
-_CT_SCALE_SUFFIXES = (".weight_scale", ".weight_global_scale", ".input_global_scale", ".input_scale")
-
-
-def _nvfp4_parts_ct(f, raw_base: str):
- """compressed-tensors NVFP4 -> ``(packed uint8 [O, IN//2], block scale fp8 [O, IN//16],
- per-output-row global fp16 [O])`` for the W4A16 kernels. The native/dequant global is
- ``1/weight_global_scale`` (compressed-tensors stores the quant-side global)."""
- w = f.get_tensor(raw_base + ".weight_packed")
- s = f.get_tensor(raw_base + ".weight_scale")
- wg = f.get_tensor(raw_base + ".weight_global_scale").reshape(1).to(torch.float32)
- g = (1.0 / wg).to(torch.float16).expand(w.shape[0]).contiguous()
- return w, s, g
+# The scale suffixes and parts/fuse machinery are shared with muse_glimmer and live
+# in models/loader.py.
+_CT_SCALE_SUFFIXES = CT_SCALE_SUFFIXES
+_nvfp4_parts_ct = nvfp4_parts_ct
+_ct_bf16_fuse = ct_bf16_fuse
def _ct_nvfp4_fuse(base: str, parts_tuple: tuple, buf: dict):
- """Buffer a native NVFP4 fusion part ``(w, s, g)``; emit the concatenated native parts once
- complete, ``[]`` while incomplete, ``None`` if ``base`` is not a fusion part."""
- for fused_suffix, parts in _CT_NVFP4_FUSE.items():
- for idx, part in enumerate(parts):
- if base.endswith(part):
- key = base[: -len(part)] + fused_suffix
- slots = buf.setdefault(key, {})
- slots[idx] = parts_tuple
- if len(slots) < len(parts):
- return []
- del buf[key]
- ws = [slots[i][0] for i in range(len(parts))]
- ss = [slots[i][1] for i in range(len(parts))]
- gs = [slots[i][2] for i in range(len(parts))]
- return [
- (key + ".weight", torch.cat(ws, dim=0)),
- (key + ".weight_scale", torch.cat(ss, dim=0)),
- (key + ".weight_global", torch.cat(gs, dim=0)),
- ]
- return None
-
-
-def _ct_bf16_fuse(base: str, tensor: torch.Tensor, buf: dict, groups: dict):
- """Buffer a bf16 fusion part; emit the concatenated ``.weight`` once complete, ``[]`` while
- incomplete, ``None`` if ``base`` is not a part of any group in ``groups``."""
- for fused_suffix, parts in groups.items():
- for idx, part in enumerate(parts):
- if base.endswith(part):
- key = base[: -len(part)] + fused_suffix
- slots = buf.setdefault(key, {})
- slots[idx] = tensor
- if len(slots) < len(parts):
- return []
- del buf[key]
- return [(key + ".weight", torch.cat([slots[i] for i in range(len(parts))], dim=0))]
- return None
+ return ct_nvfp4_fuse(base, parts_tuple, buf, _CT_NVFP4_FUSE)
def _iter_weights_compressed_tensors(
@@ -612,13 +576,17 @@ def _iter_weights_compressed_tensors(
tensor = tensor + 1.0 # (1 + weight) baked into the stored norm weight
yield name, tensor
- for file in tqdm(
- iter_weight_files(model_path),
- desc="Loading compressed-tensors weights",
- disable=not tp_info.is_primary(),
- ):
- with safetensors.safe_open(file, framework="pt", device=str(device)) as f:
- for raw_name in f.keys():
+ # Scale lookups go through the shard-map reader: a weight_packed's quant scales
+ # can land in a different shard than the packed weight (the Muse-Glimmer layer-49
+ # shape; nothing prevents an llm-compressor Qwen export from splitting the same way).
+ reader = ShardReader(model_path, device)
+ try:
+ for file in tqdm(
+ reader.files(),
+ desc="Loading compressed-tensors weights",
+ disable=not tp_info.is_primary(),
+ ):
+ for raw_name in reader.names_in(file):
if raw_name.startswith(("mtp.", "model.visual.", "visual.")):
continue
if raw_name.endswith(_CT_SCALE_SUFFIXES):
@@ -631,7 +599,7 @@ def _iter_weights_compressed_tensors(
if raw_name.endswith(".weight_packed"): # NVFP4 projection
base = name[: -len(".weight_packed")]
raw_base = raw_name[: -len(".weight_packed")]
- w, s, g = _nvfp4_parts_ct(f, raw_base)
+ w, s, g = _nvfp4_parts_ct(reader, raw_base)
# GDN in_proj_* compute in bf16 (model contract) but some checkpoints
# (e.g. sakamakismile/Qwen3.6-27B-NVFP4) quantize them too: dequant to
# bf16 here and let the bf16 fusion assemble ``in_proj`` as usual.
@@ -659,11 +627,13 @@ def _iter_weights_compressed_tensors(
continue
if name.endswith(".weight"):
- yield from _emit_bf16_weight(name, f.get_tensor(raw_name))
+ yield from _emit_bf16_weight(name, reader.get_tensor(raw_name))
continue
# A_log / dt_bias (kept fp32 by the model; the load downcast exempts them).
- yield name, f.get_tensor(raw_name)
+ yield name, reader.get_tensor(raw_name)
+ finally:
+ reader.close()
assert not nvfp4_buf, f"Incomplete NVFP4 fusions: {list(nvfp4_buf.keys())}"
assert not bf16_buf, f"Incomplete bf16 fusions: {list(bf16_buf.keys())}"
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 1e015187..0c033ca0 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -65,6 +65,15 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"freetoken.models.qwen3_5_moe",
"Qwen3_5MoEForCausalLM",
),
+ # Muse-Glimmer-30B (model_type muse_glimmer): multimodal wrapper config (text tower in
+ # text_config, weights under model.language_model.); served text-only. Dense gated GQA
+ # with a [SWA x3, full] pattern -- full layers are NoPE -- weightless qk norms, centered
+ # (1+w) sandwich norms and softcapped logits; the NVFP4 release is compressed-tensors
+ # W4A16 on every text Linear.
+ "MuseGlimmerForConditionalGeneration": ModelSpec(
+ "freetoken.models.muse_glimmer",
+ "MuseGlimmerForCausalLM",
+ ),
"MistralForCausalLM": ModelSpec(
"freetoken.models.mistral",
"MistralForCausalLM",
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index 7bc3593c..b2857b75 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -134,6 +134,8 @@ def parse_args(
return "minimax_m3"
if "minimax" in marker:
return "minimax"
+ if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker:
+ return "muse_glimmer"
if "gemma4" in marker:
return "gemma4"
if (
@@ -187,6 +189,8 @@ def parse_args(
return "minimax_m3"
if "minimax" in marker:
return "minimax"
+ if "muse_glimmer" in marker or "muse-glimmer" in marker or "museglimmer" in marker:
+ return "muse_glimmer"
if "gemma4" in marker:
return "gemma4"
return None
@@ -416,6 +420,7 @@ def parse_args(
"glm47",
"minimax",
"minimax_m3",
+ "muse_glimmer",
"gpt_oss",
"gpt-oss",
],
@@ -428,13 +433,13 @@ def parse_args(
default="auto",
choices=[
"auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm",
- "minimax", "minimax_m3", "gemma4",
+ "minimax", "minimax_m3", "muse_glimmer", "gemma4",
],
help=(
"Reasoning parser that splits chain-of-thought into reasoning_content "
"for OpenAI responses. 'auto' selects per model family (gpt-oss Harmony, "
- "<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, gemma thought, "
- "dsv4); 'off' disables it."
+ "<think> for qwen3/glm/minimax, <mm:think> for minimax-m3, ATEM to=self "
+ "channels for muse-glimmer, gemma thought, dsv4); 'off' disables it."
),
)
diff --git a/python/freetoken/server/function_call_parser.py b/python/freetoken/server/function_call_parser.py
index 0c22b8e0..0804dcc3 100644
--- a/python/freetoken/server/function_call_parser.py
+++ b/python/freetoken/server/function_call_parser.py
@@ -27,6 +27,16 @@ from partial_json_parser.core.options import Allow
from pydantic import BaseModel
from .api_models import Function, Tool
+from .reasoning_parser import (
+ ATEM_CLOSING_TOKENS,
+ ATEM_HEADER_SPAN,
+ ATEM_INLINE_HEADER_RE,
+ ATEM_MESSAGE,
+ ATEM_RECIPIENT_RE,
+ ATEM_START,
+ atem_hold_len,
+ atem_marker_inside,
+)
try:
import orjson
@@ -68,6 +78,7 @@ TOOLS_TAG_LIST = [
"<|DSML|function_calls>",
"<|DSML|tool_calls>",
"<|DSML|invoke",
+ "<atem:function_calls>",
]
@@ -680,6 +691,11 @@ class InvokeParamStreamMixin:
return self._convert_param_value(raw, key, self._ps_param_config, "")
return _parse_loose_json_value(raw)
+ def _ps_canonical_name(self, name: str) -> str:
+ """Hook: normalize an invoke's function name before validation (identity by
+ default; muse_glimmer collapses template-doubled ``name.name`` recipients)."""
+ return name
+
def _ps_trim_leading(self, text: str) -> str:
if self._ps_trim_single:
return text[1:] if text[:1] and text[:1] in self._ps_trim else text
@@ -754,7 +770,7 @@ class InvokeParamStreamMixin:
m = self._ps_invoke_open_re.search(buf, inv)
if m is None:
break # invoke tag still streaming
- func_name = m.group(1).strip()
+ func_name = self._ps_canonical_name(m.group(1).strip())
if self.current_tool_id == -1:
self.current_tool_id = 0
while len(self.prev_tool_call_arr) <= self.current_tool_id:
@@ -3042,6 +3058,460 @@ class GptOssDetector(BaseFormatDetector):
+class MuseGlimmerDetector(InvokeParamStreamMixin, BaseFormatDetector):
+ """Detector for Muse Glimmer's ATEM tool-call protocol.
+
+ Wire format (inside a ``<|start|>assistant to=<tool><|message|> ... <|eot|>``
+ channel; the ``MuseGlimmerReasoningParser`` upstream preserves such blocks
+ verbatim and unwraps everything else):
+
+ ```
+ <atem:function_calls>
+ <atem:invoke name="tool.fn">
+ <atem:parameter name="param">value</atem:parameter>
+ </atem:invoke>
+ </atem:function_calls>
+ ```
+
+ The invoke/parameter body is the InvokeParamStreamMixin grammar; per the
+ template's own contract values are parsed with regexes (not strict XML) and
+ string values are NOT whitespace-trimmed (``_ps_trim = ""``). A channel layer
+ around the mixin swallows the ``<|start|>...<|message|>`` headers and the
+ ``<|eot|>/<|eom|>`` terminators, streams ``to=user`` bodies as text, and drops
+ ``to=self`` bodies (they only reach this detector when no reasoning parser
+ runs upstream). Channels also open with a mid-stream headerless
+ ``to=X<|message|>`` switch (the model leaves a channel without ``<|eom|>``).
+
+ ``header_open`` initializes the detector from the prompt it continues:
+ True when the detector receives the raw turn bytes directly (no muse
+ reasoning parser stacked above), whose templated prompt ends with
+ ``<|start|>assistant`` -- a synthetic ``<|start|>`` is seeded so the bare
+ first header goes through the ordinary full-header machinery. False (the
+ default) downstream of ``MuseGlimmerReasoningParser``, which delivers tool
+ slices with their full headers and everything else already classified.
+
+ ATEM markup is EXECUTED only inside a tool-recipient channel (vLLM's rule).
+ A block quoted in a ``to=user`` body -- or the system prompt's own ATEM
+ example echoed back -- renders as plain text instead of becoming a real
+ call.
+ """
+
+ # <atem:function_calls> is not a unique tool opener under the channel-scoped
+ # execution rule (a to=user body may quote it), so no checkpoint anchor.
+ toolcall_opener = None
+ _ps_trim = "" # "spaces for string values are not stripped" (chat-template contract)
+ _ps_missing_type = "string"
+ accepts_header_open = True # FunctionCallParser passes the turn-start state
+
+ def __init__(self, header_open: bool = False):
+ super().__init__()
+ self.bot_token = "<atem:function_calls>"
+ self.eot_token = "</atem:function_calls>"
+ self.tool_call_separator = "\n"
+
+ # InvokeParamStreamMixin grammar
+ self._ps_outer_open = "<atem:function_calls>"
+ self._ps_outer_close = "</atem:function_calls>"
+ self._ps_invoke_open_prefix = "<atem:invoke"
+ self._ps_invoke_open_re = re.compile(r'<atem:invoke name="([^"]*)">')
+ self._ps_invoke_close = "</atem:invoke>"
+ self._ps_param_open_prefix = "<atem:parameter"
+ self._ps_param_open_re = re.compile(r'<atem:parameter name="([^"]*)">')
+ self._ps_param_close = "</atem:parameter>"
+ self._ps_reset()
+
+ # Channel layer state: "text" (to=user / outside channels) | "skip"
+ # (non-user, non-tool channel body) | "tool" (tool channel body,
+ # delegated to the mixin).
+ self._ch_mode = "text"
+ self._header_open = header_open
+ # The seed is synthetic: if it is ever ruled a non-header, the marker
+ # text itself is not delivered -- the model never emitted it.
+ self._buffer = ATEM_START if header_open else ""
+ self._synthetic_open = header_open
+ # A channel boundary fired mid-invoke: until the NEXT boundary, incoming
+ # text is that broken channel's residue (raw ATEM markup), not content.
+ self._truncated_channel = False
+ # Dropped tool-channel prose, accumulated for ONE warning per channel
+ # (a per-fragment warning logs once per generated character).
+ self._dropped_prose: List[str] = []
+
+ def has_tool_call(self, text: str) -> bool:
+ return self.bot_token in text
+
+ def block_close_tokens(self) -> tuple:
+ return (self.eot_token,) + ATEM_CLOSING_TOKENS
+
+ def _ps_canonical_name(self, name: str) -> str:
+ """The template renders a bare-name tool's recipient namespace as
+ ``name.*``, so the model emits ``get_weather.get_weather``: collapse the
+ doubled form to the registered head (never when the doubled form itself
+ is a registered tool)."""
+ head, dot, tail = name.partition(".")
+ indices = getattr(self, "_tool_indices", {})
+ if dot and head == tail and head in indices and name not in indices:
+ return head
+ return name
+
+ # ------------------------------------------------------------------
+ # streaming
+ # ------------------------------------------------------------------
+ def _enter_recipient(self, recipient: str) -> None:
+ if recipient == "self":
+ self._ch_mode = "skip"
+ elif recipient == "user":
+ self._ch_mode = "text"
+ else:
+ self._ch_mode = "tool"
+
+ @staticmethod
+ def _channel_boundary(buf: str):
+ """Earliest channel boundary in ``buf``: ``(pos, kind, payload)`` with kind
+ "closer" (payload = token), "start", or "inline" (payload = the headerless
+ ``to=X<|message|>`` match); ``(-1, None, None)`` when none is present."""
+ best = (len(buf) + 1, None, None)
+ for tok in ATEM_CLOSING_TOKENS:
+ pos = buf.find(tok)
+ if pos != -1 and pos < best[0]:
+ best = (pos, "closer", tok)
+ pos = buf.find(ATEM_START)
+ if pos != -1 and pos < best[0]:
+ best = (pos, "start", None)
+ m = ATEM_INLINE_HEADER_RE.search(buf)
+ if m is not None and m.start() < best[0]:
+ best = (m.start(), "inline", m)
+ if best[1] is None:
+ return -1, None, None
+ return best
+
+ def _finalize_truncated_invoke(self, reason: str = "tool channel closed") -> List[ToolCallItem]:
+ """The channel (or the whole stream) ended while the ATEM machinery was
+ mid-flight. If an invoke already streamed its Start, close its argument
+ JSON so the streamed fragments stay valid (never emit broken arguments),
+ ledger the completed parameters, and advance the ordinal so the next
+ call cannot merge into this one. Anything less than an open invoke is
+ markup debris: warn and reset."""
+ self._warn_dropped_prose()
+ mode = getattr(self, "_ps_mode", "idle")
+ if mode == "idle":
+ return []
+ self._truncated_channel = True # what follows is the broken channel's residue
+ calls: List[ToolCallItem] = []
+ if mode in ("invoke", "pstr", "pbuf"):
+ ledger = self.streamed_args_for_tool[self.current_tool_id]
+ if mode == "pstr":
+ frag = '"}' # the ledger ends inside an open string value
+ elif ledger:
+ frag = "}" # ends after a complete "key": value pair
+ else:
+ frag = "{}" # invoke opened, no parameter emitted yet
+ self.streamed_args_for_tool[self.current_tool_id] += frag
+ calls.append(
+ ToolCallItem(tool_index=self.current_tool_id, name=None, parameters=frag)
+ )
+ try:
+ parsed = json.loads(self.streamed_args_for_tool[self.current_tool_id])
+ except (json.JSONDecodeError, ValueError):
+ parsed = {}
+ self.prev_tool_call_arr[self.current_tool_id]["arguments"] = (
+ parsed if isinstance(parsed, dict) else {}
+ )
+ self.streamed_args_for_tool[self.current_tool_id] = ""
+ self.current_tool_id += 1
+ while len(self.streamed_args_for_tool) <= self.current_tool_id:
+ self.streamed_args_for_tool.append("")
+ logger.warning("muse_glimmer: %s mid-invoke; arguments truncated", reason)
+ else: # block / invoke_skip / pskip: no open call, just partial markup
+ logger.warning("muse_glimmer: %s mid-block; partial tool markup dropped", reason)
+ self._ps_reset()
+ return calls
+
+ def finalize_streaming(self) -> List[ToolCallItem]:
+ """End-of-stream hook (FunctionCallParser.finalize_stream): the generation
+ ran out (max_tokens) while an invoke was still streaming -- close it the
+ same way a channel boundary would, so the client's concatenated argument
+ fragments end as valid JSON instead of an unterminated string."""
+ return self._finalize_truncated_invoke("generation ended")
+
+ # The truncated channel's trailing-markup shapes: what a broken-off invoke
+ # leaves behind when the model closes the tags it opened.
+ _CLOSING_MARKUP = ("</atem:parameter>", "</atem:invoke>", "</atem:function_calls>")
+
+ @classmethod
+ def _channel_residue_end(cls, buf: str) -> tuple[int, bool]:
+ """Length of the leading run of ATEM closing markup (closing tags plus
+ whitespace) in ``buf``, and whether the scan DECIDED. False means the
+ buffer ends inside the run or a partial tag: hold for more input.
+
+ Known cost of shape-based dropping: a reply that literally BEGINS with a
+ complete closing tag ("</atem:parameter> is the closing tag") loses that
+ leading tag. Partial-tag-shaped prose ("</a", "</atem") is held and
+ released intact, so no ordinary word can be eaten -- only a verbatim
+ full tag right after a truncated invoke, which is indistinguishable from
+ the residue this filter exists to drop."""
+ i = 0
+ while i < len(buf):
+ if buf[i] in " \t\r\n":
+ i += 1
+ continue
+ matched = False
+ for tag in cls._CLOSING_MARKUP:
+ if buf.startswith(tag, i):
+ i += len(tag)
+ matched = True
+ break
+ if matched:
+ continue
+ rest = buf[i:]
+ if any(tag.startswith(rest) for tag in cls._CLOSING_MARKUP):
+ return i, False # a partial closing tag at the end: hold
+ return i, True # first non-markup character: the residue ended
+ return i, False # consumed the whole buffer; the run may continue
+
+ def _drop_channel_prose(self, text: str) -> None:
+ """Text inside a tool channel that is not ATEM markup is discarded (only
+ tool-recipient markup is executed; the template puts nothing else there).
+ Accumulated and logged ONCE per channel by ``_warn_dropped_prose`` --
+ token-by-token serving would otherwise warn per generated character."""
+ if text and text.strip():
+ self._dropped_prose.append(text)
+
+ def _warn_dropped_prose(self) -> None:
+ if not self._dropped_prose:
+ return
+ dropped = "".join(self._dropped_prose)
+ self._dropped_prose = []
+ logger.warning(
+ "muse_glimmer: dropped %d chars of non-markup text inside a tool channel: %.120r",
+ len(dropped),
+ dropped,
+ )
+
+ def _run_mixin(self, atem_part: str, remainder: str, tools: List[Tool]) -> StreamingParseResult:
+ """Feed ``atem_part`` through the invoke/parameter machinery; whatever the
+ mixin leaves unconsumed stays buffered ahead of ``remainder``."""
+ self._buffer = atem_part
+ result = InvokeParamStreamMixin.parse_streaming_increment(self, "", tools)
+ self._buffer += remainder
+ return result
+
+ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> StreamingParseResult:
+ self._buffer += new_text
+ if not hasattr(self, "_tool_indices"):
+ self._tool_indices = self._get_tool_indices(tools)
+ normal_parts: List[str] = []
+ calls: List[ToolCallItem] = []
+ while True:
+ buf = self._buffer
+ if not buf:
+ break
+
+ if self._ch_mode == "text":
+ if calls:
+ break # text after a call defers to the next step (wire order)
+ if self._truncated_channel:
+ # After a mid-invoke channel break, drop the broken channel's
+ # trailing markup BY SHAPE (closing tags + whitespace) and
+ # clear at the first non-markup character -- a real reply
+ # following the break must flow, not be swallowed until some
+ # later boundary that the production pipeline never delivers.
+ consumed, decided = self._channel_residue_end(buf)
+ if consumed:
+ self._buffer = buf[consumed:]
+ if not decided:
+ break
+ self._truncated_channel = False
+ self._warn_dropped_prose()
+ continue
+ pos, kind, payload = self._channel_boundary(buf)
+ if pos == -1:
+ hold = atem_hold_len(buf)
+ release = buf[: len(buf) - hold] if hold else buf
+ if release:
+ normal_parts.append(release)
+ self._buffer = buf[len(release):]
+ break
+ if pos > 0:
+ normal_parts.append(buf[:pos])
+ self._buffer = buf[pos:]
+ continue
+ if kind == "closer":
+ self._buffer = buf[len(payload):] # stray terminator: drop
+ continue
+ if kind == "inline":
+ self._enter_recipient(payload.group(1))
+ self._buffer = buf[payload.end():]
+ continue
+ # kind == "start": parse the channel header. While the header-open
+ # seed is unconsumed it is the leftmost marker, so each consuming
+ # branch owns the synthetic flag: a synthetic marker ruled a
+ # non-header is dropped, never delivered (the model never emitted it).
+ synthetic = self._synthetic_open
+ msg = buf.find(ATEM_MESSAGE)
+ if atem_marker_inside(
+ buf, len(ATEM_START), msg if msg != -1 else len(buf)
+ ):
+ # A control token inside the candidate: headers never contain
+ # markers, so this <|start|> is literal text (a synthetic
+ # seed is simply dropped) -- mirrors the reasoning parser.
+ if not synthetic:
+ normal_parts.append(ATEM_START)
+ self._synthetic_open = False
+ self._buffer = buf[len(ATEM_START):]
+ continue
+ if msg == -1:
+ # +len(<|message|>) slack: a protocol-legal header whose marker
+ # is still mid-arrival must not be cut at the nominal span.
+ if len(buf) > len(ATEM_START) + ATEM_HEADER_SPAN + len(ATEM_MESSAGE):
+ # Past a plausible header span the marker cannot open a
+ # header anymore: release it as literal text and move on
+ # (mirrors the reasoning parser's bound).
+ if not synthetic:
+ normal_parts.append(ATEM_START)
+ self._synthetic_open = False
+ self._buffer = buf[len(ATEM_START):]
+ continue
+ break # header still streaming
+ if msg - len(ATEM_START) > ATEM_HEADER_SPAN:
+ # The found <|message|> is too far away to belong to THIS
+ # marker (a stray literal <|start|> followed by junk, then the
+ # NEXT segment's real header): the marker is literal text.
+ if not synthetic:
+ normal_parts.append(ATEM_START)
+ self._synthetic_open = False
+ self._buffer = buf[len(ATEM_START):]
+ continue
+ m = ATEM_RECIPIENT_RE.search(buf[len(ATEM_START):msg])
+ self._enter_recipient(m.group(1) if m else "user")
+ self._synthetic_open = False
+ self._buffer = buf[msg + len(ATEM_MESSAGE):]
+ continue
+
+ pos, kind, payload = self._channel_boundary(buf)
+
+ if self._ch_mode == "skip":
+ if pos == -1:
+ hold = atem_hold_len(buf)
+ self._buffer = buf[len(buf) - hold:] if hold else ""
+ break
+ # Drop the skipped body; transition exactly like text mode would.
+ self._buffer = buf[pos:]
+ self._ch_mode = "text"
+ continue
+
+ # self._ch_mode == "tool": the body up to the channel boundary belongs
+ # to the ATEM machinery; text the mixin releases inside the channel is
+ # not executed (only tool-recipient markup is) and is dropped.
+ if pos == -1:
+ hold = atem_hold_len(buf)
+ atem_part = buf[: len(buf) - hold] if hold else buf
+ if not atem_part:
+ break
+ result = self._run_mixin(atem_part, buf[len(atem_part):], tools)
+ self._drop_channel_prose(result.normal_text)
+ calls.extend(result.calls)
+ if self._buffer == buf:
+ break # no progress: the mixin is holding a partial marker
+ continue
+ result = self._run_mixin(buf[:pos], buf[pos:], tools)
+ self._drop_channel_prose(result.normal_text)
+ calls.extend(result.calls)
+ if self._buffer != buf:
+ # The mixin advanced (it stops after each call's close, wire-order):
+ # more blocks may still precede the boundary -- a second invoke block
+ # in the same undrained buffer must parse, not vanish as debris.
+ continue
+ # Inert: nothing before the boundary the machinery can still consume.
+ # Finalize a mid-flight invoke (its Start already streamed) instead of
+ # letting the next channel's call merge into it, then transition; any
+ # residue ahead of the boundary is markup debris and drops with it.
+ calls.extend(self._finalize_truncated_invoke())
+ if kind == "closer":
+ self._buffer = buf[pos + len(payload):]
+ self._ch_mode = "text"
+ # NOTE: the truncation mark (when finalize set it) survives the
+ # closer on purpose: on the production pipeline the broken
+ # channel's markup residue arrives AFTER the reasoning parser's
+ # synthetic terminator. Text mode clears the mark by SHAPE -- at
+ # the first non-markup character -- so a real reply flows
+ # immediately while trailing closing tags are dropped.
+ elif kind == "inline":
+ self._enter_recipient(payload.group(1))
+ self._buffer = buf[payload.end():]
+ else: # abutting <|start|>: reprocess it in text mode
+ self._buffer = buf[pos:]
+ self._ch_mode = "text"
+ continue
+ return StreamingParseResult(normal_text="".join(normal_parts), calls=calls)
+
+ def finish_streaming(self) -> str:
+ self._warn_dropped_prose()
+ residual, self._buffer = self._buffer, ATEM_START if self._header_open else ""
+ ch_mode, self._ch_mode = self._ch_mode, "text"
+ synthetic, self._synthetic_open = self._synthetic_open, self._header_open
+ truncated, self._truncated_channel = self._truncated_channel, False
+ ps_mode = getattr(self, "_ps_mode", "idle")
+ self._ps_reset()
+ if ch_mode != "text" or ps_mode != "idle" or truncated:
+ return ""
+ if synthetic:
+ residual = residual[len(ATEM_START):] # the seed: never model output
+ # At end of stream a <|start|> that never received its <|message|> is NOT
+ # a header: deliver the text, drop only the marker(s). Any capped discard
+ # here diverged from the layer above -- its span runs on raw bytes while
+ # this layer sees closer-stripped text, so the two edges can never agree;
+ # removing the drop window entirely makes the agreement trivial.
+ residual = residual.replace(ATEM_START, "")
+ for tok in ATEM_CLOSING_TOKENS:
+ residual = residual.replace(tok, "")
+ if self.prev_tool_call_arr and residual.strip() == "":
+ return ""
+ return residual
+
+ # ------------------------------------------------------------------
+ # one-shot
+ # ------------------------------------------------------------------
+ def detect_and_parse(self, text: str, tools: List[Tool]) -> StreamingParseResult:
+ """One-shot parse by replaying the streaming machinery on a fresh clone,
+ so both paths share one definition of channel classification (a raw
+ ``to=self`` body never leaks into content), execution scoping and value
+ semantics. Streamed fragments are reassembled into complete calls; a
+ call truncated by end-of-input closes from the parse ledger, exactly as
+ the serving layer would."""
+ clone = type(self)(header_open=self._header_open)
+ raw_calls: List[ToolCallItem] = []
+ normal_parts: List[str] = []
+ result = clone.parse_streaming_increment(text, tools)
+ normal_parts.append(result.normal_text)
+ raw_calls.extend(result.calls)
+ while True: # drain wire-order deferrals (one call boundary per step)
+ result = clone.parse_streaming_increment("", tools)
+ if not result.normal_text and not result.calls:
+ break
+ normal_parts.append(result.normal_text)
+ raw_calls.extend(result.calls)
+ raw_calls.extend(clone._finalize_truncated_invoke("input ended")) # mid-invoke
+ normal_parts.append(clone.finish_streaming())
+
+ order: List[int] = []
+ by_index: Dict[int, Dict[str, str]] = {}
+ for item in raw_calls:
+ if item.name is not None and item.tool_index not in by_index:
+ by_index[item.tool_index] = {"name": item.name, "params": ""}
+ order.append(item.tool_index)
+ if item.parameters and item.tool_index in by_index:
+ by_index[item.tool_index]["params"] += item.parameters
+ calls = [
+ ToolCallItem(
+ tool_index=ordinal,
+ name=by_index[idx]["name"],
+ parameters=by_index[idx]["params"] or "{}",
+ )
+ for ordinal, idx in enumerate(order)
+ ]
+ return StreamingParseResult(normal_text="".join(normal_parts).strip(), calls=calls)
+
+
class FunctionCallParser:
"""
Parser for function/tool calls in model outputs.
@@ -3061,18 +3531,24 @@ class FunctionCallParser:
"minimax": MiniMaxDetector,
"minimax_m3": MiniMaxM3Detector,
"mistral": MistralDetector,
+ "muse_glimmer": MuseGlimmerDetector,
"qwen": Qwen25Detector,
"qwen25": Qwen25Detector,
"qwen3_coder": Qwen3CoderDetector,
}
- def __init__(self, tools: List[Tool], tool_call_parser: str):
+ def __init__(self, tools: List[Tool], tool_call_parser: str, turn_starts_open: bool = False):
detector: Type[BaseFormatDetector] = None
detector_class = self.ToolCallParserEnum.get(tool_call_parser)
- if detector_class:
- detector = detector_class()
- else:
+ if detector_class is None:
raise ValueError(f"Unsupported tool_call_parser: {tool_call_parser}")
+ if getattr(detector_class, "accepts_header_open", False):
+ # Turn-start parse state read from the prompt: the templated chat
+ # prompt ends inside a channel header (``<|start|>assistant``), so a
+ # detector that receives the raw turn bytes starts header-open.
+ detector = detector_class(header_open=turn_starts_open)
+ else:
+ detector = detector_class()
self.detector = detector
self.tools = _coerce_tools(tools)
@@ -3211,10 +3687,21 @@ class FunctionCallParser:
arr = self.detector.prev_tool_call_arr
if 0 <= tool_ordinal < len(arr):
args = arr[tool_ordinal].get("arguments")
- if args:
+ # `is not None`, not truthiness: {} is a legitimate ledger entry (an
+ # empty-argument call) and must serialize instead of falling through.
+ if args is not None:
return json.dumps(args, ensure_ascii=False)
return None
+ def finalize_stream(self) -> List[ToolCallItem]:
+ """End-of-stream finalize: argument fragments that CLOSE a call cut off
+ mid-arguments, from detectors that can (muse_glimmer closes the streamed
+ JSON so the fragments the client concatenated stay valid). The serving
+ layer routes these before it closes the open call; [] for detectors
+ without the hook."""
+ finalize = getattr(self.detector, "finalize_streaming", None)
+ return finalize() if finalize is not None else []
+
SUPPORTED_TOOL_CALL_PARSERS = list(FunctionCallParser.ToolCallParserEnum.keys())
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index 1fc2b02b..d6689680 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -388,6 +388,18 @@ def _leaked_special_tokens(state: Any) -> list[str]:
return DSV4_SPECIAL_TOKENS if getattr(state.config, "reasoning_parser", None) == "deepseekv32" else []
+def _make_tool_parser(spec: GenSpec, state: Any) -> FunctionCallParser:
+ """Build the tool-call parser with its turn-start state read from the prompt:
+ a muse detector receives the raw turn bytes (opened by the template's
+ ``<|start|>assistant``) only when its reasoning parser is not stacked above,
+ which otherwise delivers tool slices with full headers."""
+ return FunctionCallParser(
+ spec.parser_tools or [],
+ getattr(state.config, "tool_call_parser", "llama3"),
+ turn_starts_open=getattr(state.config, "reasoning_parser", None) != "muse_glimmer",
+ )
+
+
def _parse_tool_response(
text: str,
spec: GenSpec,
@@ -397,7 +409,7 @@ def _parse_tool_response(
return None
if not any(tag in text for tag in TOOLS_TAG_LIST):
return None
- parser = FunctionCallParser(spec.parser_tools or [], getattr(state.config, "tool_call_parser", "llama3"))
+ parser = _make_tool_parser(spec, state)
result = parser.parse_non_stream(text)
if not result.calls:
return None
@@ -548,9 +560,7 @@ async def _generate_events_impl(uid: int, spec: GenSpec, state: Any) -> AsyncIte
tool_parser: FunctionCallParser | None = None
if parse_tools:
try:
- candidate = FunctionCallParser(
- spec.parser_tools or [], getattr(state.config, "tool_call_parser", "llama3")
- )
+ candidate = _make_tool_parser(spec, state)
except ValueError:
candidate = None # unsupported parser name: keep the buffered path's behavior
if candidate is not None and candidate.supports_streaming():
@@ -678,9 +688,15 @@ async def _generate_events_impl(uid: int, spec: GenSpec, state: Any) -> AsyncIte
finish_reason = engine_finish_reason or "stop"
if tool_parser is not None:
- # End-of-stream drain: close a call whose end marker never arrived (truncated
- # generation), best-effort recover a call cut off inside an unterminated tag
- # block, then release text still held back for tag disambiguation.
+ # End-of-stream drain: let the detector finalize a call cut off mid-arguments
+ # (closing fragments keep the client's concatenated JSON valid), close a call
+ # whose end marker never arrived (truncated generation), best-effort recover a
+ # call cut off inside an unterminated tag block, then release text still held
+ # back for tag disambiguation.
+ for frag in tool_parser.finalize_stream():
+ if open_call is not None and frag.parameters:
+ open_call["params"] += frag.parameters
+ yield ToolCallArgsDelta(tool_index=open_call["ordinal"], fragment=frag.parameters)
done = _close_open_call()
if done is not None:
yield done
diff --git a/python/freetoken/server/model_meta.py b/python/freetoken/server/model_meta.py
index 0045cd7f..8ce8009a 100644
--- a/python/freetoken/server/model_meta.py
+++ b/python/freetoken/server/model_meta.py
@@ -13,11 +13,11 @@ from typing import Any, Tuple
from freetoken.tokenizer.effort import (
EFFORT_SCALE,
- OPENAI_EFFORT_TRIPLE,
THINKING_ADAPTIVE_KWARGS,
THINKING_OFF_KWARGS,
THINKING_ON_KWARGS,
ThinkingProfile,
+ effective_efforts,
)
@@ -38,8 +38,8 @@ def derive_think_gears(
controls -- the checkpoint owns this knowledge; nothing here is keyed by
model family. ``None`` when there is nothing controllable to offer.
- A template that grades effort without validating it gets the OpenAI triple
- (the only vocabulary such a template is known to understand); an always-on
+ A template that grades effort without validating it gets the graded ladder
+ (its trained levels; never-advertised dialects quantize onto it); an always-on
model with a reasoning parser but no observable knob shows a single "on"
gear so clients can still label the state."""
efforts = profile.efforts
@@ -53,11 +53,10 @@ def derive_think_gears(
kwargs["adaptive"] = dict(THINKING_ADAPTIVE_KWARGS)
if efforts.consumes_effort:
- names = (
- [n for n in efforts.supported if n in EFFORT_SCALE]
- if efforts.validates
- else list(OPENAI_EFFORT_TRIPLE)
- )
+ # The served vocabulary: a validating template's own probed set; a
+ # non-validating grader gets the graded ladder (incl. xhigh -- muse's
+ # model card recommends it for agentic use) rather than the OpenAI triple.
+ names = [n for n in effective_efforts(efforts) if n in EFFORT_SCALE]
for name in sorted(names, key=lambda n: EFFORT_SCALE[n]):
gears.append(name)
gear_kwargs = dict(THINKING_ON_KWARGS) if profile.toggleable else {}
diff --git a/python/freetoken/server/openai_api.py b/python/freetoken/server/openai_api.py
index 3fb74eec..b4becd26 100644
--- a/python/freetoken/server/openai_api.py
+++ b/python/freetoken/server/openai_api.py
@@ -663,9 +663,12 @@ async def _effort_fields(state: Any) -> tuple[list[str] | None, str | None]:
profile = await asyncio.to_thread(manager.effort_profile)
except Exception: # noqa: BLE001 -- metadata only; the generation path reports real faults
return None, None
- if not profile.consumes_effort:
+ from freetoken.tokenizer.effort import effective_efforts
+
+ served = effective_efforts(profile)
+ if not served:
return None, None
- ordered = sorted(profile.supported, key=lambda name: -EFFORT_SCALE.get(name, 0.0))
+ ordered = sorted(served, key=lambda name: -EFFORT_SCALE.get(name, 0.0))
return ordered, profile.default
diff --git a/python/freetoken/server/reasoning_parser.py b/python/freetoken/server/reasoning_parser.py
index 2d5e3d29..6788675c 100644
--- a/python/freetoken/server/reasoning_parser.py
+++ b/python/freetoken/server/reasoning_parser.py
@@ -18,6 +18,7 @@ preserves the block for the tool-call parser.
from __future__ import annotations
+import re
from dataclasses import dataclass
from typing import Dict, List, Optional, Tuple, Type
@@ -546,6 +547,329 @@ class GemmaThoughtReasoningParser(BaseReasoningParser):
)
+# Muse Glimmer ATEM protocol control tokens. Generation starts right after the
+# template's ``<|start|>assistant``, so the FIRST segment arrives as a bare header
+# continuation (`` to=self<|message|>...``) -- the parser is constructed
+# header-open and seeds a synthetic ``<|start|>`` so those bytes go through the
+# ordinary full-header machinery instead of a duplicate guessing path. Later
+# segments open with a full ``<|start|>assistant to=X<|message|>`` -- or, when
+# the model leaves a channel without emitting ``<|eom|>`` first, with a bare
+# ``to=X<|message|>`` (a complete match of that shape is a segment boundary;
+# vLLM applies the same rule). ``<|eom|>`` separates segments within one turn;
+# ``<|eot|>`` / ``<|end_of_text|>`` are the stops.
+ATEM_START = "<|start|>"
+ATEM_MESSAGE = "<|message|>"
+ATEM_CLOSING_TOKENS = ("<|eot|>", "<|eom|>", "<|end_of_text|>")
+ATEM_ALL_TOKENS = (ATEM_START, ATEM_MESSAGE) + ATEM_CLOSING_TOKENS
+# Recipient names cap at 64 chars in EVERY header shape (inline switches and the
+# bare stream-start header use the same bound below), so streaming and one-shot
+# agree on which recipients are valid and >64 degrades identically everywhere.
+ATEM_RECIPIENT_RE = re.compile(r"to=([^\s<]{1,64})")
+# A complete headerless channel switch: ``to=<name>`` abutting ``<|message|>``.
+# The name length is capped so the streaming hold-back below can bound how much
+# tail it withholds while a potential switch is still arriving.
+ATEM_INLINE_HEADER_RE = re.compile(r"to=([^\s<]{1,64})<\|message\|>")
+# Lookback window for the streaming hold-back's inline-switch scan: a headerless
+# ``to=<name><|message|>`` switch can occupy at most this many trailing bytes.
+_ATEM_HOLD_WINDOW = 96
+# Longest span after "<|start|>" a real header can occupy before its <|message|>
+# arrives ("assistant to=<name>" + slack). Past this, the marker cannot open a
+# header anymore and is released as literal content instead of held forever.
+ATEM_HEADER_SPAN = 128
+
+
+def atem_marker_inside(text: str, start: int, end: int) -> bool:
+ """True when a complete ATEM control token lies fully inside ``text[start:end]``.
+ A channel header can never contain one, so a header candidate with a marker
+ before its ``<|message|>`` is not a header."""
+ for tok in ATEM_ALL_TOKENS:
+ if tok == ATEM_MESSAGE:
+ continue # the terminator being sought, not a header byte
+ if text.find(tok, start, end) != -1:
+ return True
+ return False
+
+
+def _longest_atem_partial_suffix(text: str) -> int:
+ """Length of the longest suffix of ``text`` that is a *proper* prefix of any ATEM
+ control token (the Harmony hold-back, for Muse Glimmer's marker set)."""
+ best = 0
+ for tok in ATEM_ALL_TOKENS:
+ for k in range(min(len(tok) - 1, len(text)), best, -1):
+ if text.endswith(tok[:k]):
+ best = k
+ break
+ return best
+
+
+def atem_hold_len(text: str) -> int:
+ """Streaming hold-back: the longest suffix of ``text`` that could still grow into
+ an ATEM marker OR a headerless ``to=<name><|message|>`` channel switch. Withheld
+ text is re-examined on the next chunk, so an emitted body never has to shrink
+ when the boundary completes."""
+ best = _longest_atem_partial_suffix(text)
+ window = text[-_ATEM_HOLD_WINDOW:]
+ i = window.rfind("to=")
+ if i != -1:
+ tail = window[i:]
+ if re.fullmatch(r"to=[^\s<]{0,64}", tail):
+ best = max(best, len(tail)) # name (or "to=") still streaming
+ else:
+ m = re.fullmatch(r"to=([^\s<]{1,64})(.+)", tail, re.DOTALL)
+ if m and ATEM_MESSAGE.startswith(m.group(2)):
+ best = max(best, len(tail)) # partial <|message|> after the name
+ for prefix in ("t", "to"): # proper prefixes of "to=" (one-chunk delay at most)
+ if len(prefix) > best and text.endswith(prefix):
+ best = len(prefix)
+ return best
+
+
+class MuseGlimmerReasoningParser(BaseReasoningParser):
+ """Reasoning parser for Muse Glimmer's ATEM channel output.
+
+ Routes ``to=self`` segments to ``reasoning_text`` and ``to=user`` (or
+ recipient-less) segments, unwrapped, to ``normal_text``. A tool-recipient
+ segment (``to=<tool>``, carrying an ``<atem:function_calls>`` block) is
+ preserved verbatim (header included) in ``normal_text`` for the downstream
+ ``MuseGlimmerDetector`` -- when the channel ends without its own terminator
+ (an abutting ``<|start|>`` header or a headerless switch), a synthetic
+ ``<|eom|>`` is appended so the detector always receives a delimited block.
+ Segments open with ``<|start|>...`` headers or with a headerless
+ ``to=X<|message|>`` switch (the model leaves a channel without ``<|eom|>``;
+ vLLM's rule). The parser is initialized from the prompt it continues:
+ every request that reaches it is a templated chat generation whose prompt
+ ends with ``<|start|>assistant``, so ``header_open=True`` (the default)
+ seeds a synthetic ``<|start|>`` and the turn's first bytes go through the
+ ordinary full-header machinery -- a real ``<|message|>`` closes the header
+ (a junk recipient yields the empty channel the model asked for), and a
+ header that never gets its ``<|message|>`` falls to the unfinished-header
+ end-of-stream rule below. This replaces a prefix-guessing "start" mode that
+ kept misclassifying content shaped like a header (the same mechanism
+ ``force_reasoning`` is for the ``<think>`` families, read from the prompt
+ instead of re-derived). Text with no ATEM markers at all passes through as
+ content.
+
+ Streaming consumes the buffer as it emits (O(n) over the stream; the naive
+ re-scan of the full buffer per chunk was quadratic and stalled the serving
+ loop on this family's long default CoT). At end-of-stream, held text is
+ delivered as content rather than dropped -- a whole reply that merely looks
+ like a bare-header prefix, or prose stranded after a literal closer, must
+ not vanish. All three methods are overridden; the base ``<think>`` machinery
+ is unused.
+ """
+
+ def __init__(
+ self,
+ force_reasoning: bool = False,
+ stream_reasoning: bool = True,
+ header_open: bool = True,
+ ) -> None:
+ super().__init__(
+ think_start_token="",
+ think_end_token="",
+ force_reasoning=force_reasoning,
+ stream_reasoning=stream_reasoning,
+ )
+ self._header_open = header_open
+ # header_open: the prompt ended inside a channel header (the template's
+ # ``<|start|>assistant``); seed the marker so the turn's first bytes are
+ # parsed by the same machinery as every later header. The seed is
+ # synthetic -- if it is ever ruled a non-header (released past the span
+ # bound, or unfinished at end of stream), the marker text itself is NOT
+ # delivered: the model never emitted it.
+ self._buffer = ATEM_START if header_open else ""
+ self._synthetic_open = header_open
+ # "body" (streaming a segment body) | "seek" (between segments, looking
+ # for the next header).
+ self._mode = "seek"
+ self._recipient = "user"
+
+ def detect_and_parse(self, text: str) -> ReasoningParseResult:
+ # Closers count as markers too: a literal stray <|eot|> must take the same
+ # replay path as streaming (which strips it), not pass through verbatim.
+ if not any(tok in text for tok in ATEM_ALL_TOKENS):
+ return ReasoningParseResult(normal_text=text)
+ clone = type(self)(header_open=self._header_open)
+ first = clone.parse_streaming_increment(text)
+ rest = clone.flush()
+ return ReasoningParseResult(
+ reasoning_text=(first.reasoning_text + rest.reasoning_text).strip(),
+ normal_text=(first.normal_text + rest.normal_text).strip(),
+ )
+
+ def parse_streaming_increment(self, new_text: str) -> ReasoningParseResult:
+ self._buffer += new_text
+ return self._drain(final=False)
+
+ def flush(self) -> ReasoningParseResult:
+ return self._drain(final=True)
+
+ @staticmethod
+ def _segment_body_end(buf: str):
+ """Earliest segment-body boundary in ``buf``: ``(end, kind, payload)`` with
+ kind "closer" (payload = the token), "start" (an abutting ``<|start|>``),
+ "inline" (payload = the headerless ``to=X<|message|>`` match), or
+ ``(len(buf), None, None)`` while the body is still streaming."""
+ best = (len(buf), None, None)
+ for tok in ATEM_CLOSING_TOKENS:
+ pos = buf.find(tok)
+ if pos != -1 and pos < best[0]:
+ best = (pos, "closer", tok)
+ pos = buf.find(ATEM_START)
+ if pos != -1 and pos < best[0]:
+ best = (pos, "start", None)
+ m = ATEM_INLINE_HEADER_RE.search(buf)
+ if m is not None and m.start() < best[0]:
+ best = (m.start(), "inline", m)
+ return best
+
+ def _drain(self, *, final: bool) -> ReasoningParseResult:
+ out_reasoning: list[str] = []
+ out_content: list[str] = []
+
+ def emit(recipient: str, piece: str) -> None:
+ if not piece:
+ return
+ if recipient == "self":
+ out_reasoning.append(piece)
+ else: # user bodies unwrapped, tool slices verbatim
+ out_content.append(piece)
+
+ def begin_body(recipient: str, header_text: str) -> None:
+ self._recipient = recipient
+ self._mode = "body"
+ if recipient not in ("self", "user"):
+ emit(recipient, header_text) # tool slices keep their header
+
+ def emit_seek(piece: str) -> None:
+ # Inter-segment text is content (deliver, don't drop); redundant closer
+ # tokens in it are protocol debris and are stripped -- the same filter
+ # the detector applies on its text path.
+ for tok in ATEM_CLOSING_TOKENS:
+ piece = piece.replace(tok, "")
+ if piece:
+ out_content.append(piece)
+
+ while True:
+ buf = self._buffer
+ if not buf:
+ break
+
+ if self._mode == "seek":
+ s = buf.find(ATEM_START)
+ m = ATEM_INLINE_HEADER_RE.search(buf)
+ if m is not None and (s == -1 or m.start() < s):
+ emit_seek(buf[: m.start()])
+ self._buffer = buf[m.end():]
+ begin_body(m.group(1), buf[m.start(): m.end()])
+ continue
+ if s != -1:
+ # While the header-open seed is unconsumed it is the leftmost
+ # marker, so every consuming branch below owns clearing the
+ # synthetic flag; a synthetic marker ruled a non-header is
+ # dropped, never delivered (the model did not emit it).
+ synthetic = self._synthetic_open
+ msg = buf.find(ATEM_MESSAGE, s + len(ATEM_START))
+ if atem_marker_inside(
+ buf, s + len(ATEM_START), msg if msg != -1 else len(buf)
+ ):
+ # A control token inside the candidate: headers never
+ # contain markers, so this <|start|> is literal content
+ # (and a synthetic seed is simply dropped). Decides
+ # immediately -- no waiting for the span bound or EOS.
+ emit_seek(buf[:s])
+ if not synthetic:
+ out_content.append(ATEM_START)
+ self._synthetic_open = False
+ self._buffer = buf[s + len(ATEM_START):]
+ continue
+ if msg != -1 and msg - s - len(ATEM_START) > ATEM_HEADER_SPAN:
+ # The found <|message|> is too far away to belong to THIS
+ # marker (a stray literal <|start|>, junk, then the NEXT
+ # segment's real header): the marker is literal content --
+ # the span bound applies whether or not a <|message|> is
+ # already in the buffer, or one-shot and streaming diverge.
+ emit_seek(buf[:s])
+ if not synthetic:
+ out_content.append(ATEM_START)
+ self._synthetic_open = False
+ self._buffer = buf[s + len(ATEM_START):]
+ continue
+ if msg != -1:
+ emit_seek(buf[:s])
+ header = buf[s + len(ATEM_START): msg]
+ rm = ATEM_RECIPIENT_RE.search(header)
+ body_start = msg + len(ATEM_MESSAGE)
+ self._buffer = buf[body_start:]
+ self._synthetic_open = False
+ begin_body(rm.group(1) if rm else "user", buf[s:body_start])
+ continue
+ # A complete <|start|> whose <|message|> hasn't arrived: text
+ # before it is content NOW (never re-dropped), the candidate is
+ # held -- bounded, with slack for a <|message|> mid-arrival (a
+ # protocol-legal long-name header must not be cut at the nominal
+ # span). Past the bound it cannot open a header anymore: release
+ # the marker as literal content and resume scanning behind it,
+ # so a degenerate wire neither stalls nor eats the turn.
+ emit_seek(buf[:s])
+ self._buffer = buf[s:]
+ if len(self._buffer) - len(ATEM_START) > ATEM_HEADER_SPAN + len(ATEM_MESSAGE):
+ if not synthetic:
+ out_content.append(ATEM_START) # verbatim: deliver, don't drop
+ self._synthetic_open = False
+ self._buffer = self._buffer[len(ATEM_START):]
+ continue
+ if final:
+ # At end of stream a candidate that never received its
+ # <|message|> is NOT a header: deliver the tail, drop only
+ # the marker. (A capped discard here kept a drop window
+ # whose edge moved between layers; removing it makes the
+ # detector's agreement trivial -- both deliver.)
+ emit_seek(self._buffer[len(ATEM_START):])
+ self._synthetic_open = False
+ self._buffer = ""
+ break
+ # No marker in sight: stream eagerly as content, holding only the
+ # tail that could still grow into one. The buffer is CONSUMED here
+ # -- the previous hold-until-flush was quadratic (re-scanned from
+ # byte 0 per chunk) and stalled the stream on long headerless
+ # continuations.
+ hold = 0 if final else atem_hold_len(buf)
+ piece = buf[: len(buf) - hold] if hold else buf
+ emit_seek(piece)
+ self._buffer = buf[len(piece):]
+ break
+
+ # self._mode == "body"
+ recipient = self._recipient
+ end, kind, payload = self._segment_body_end(buf)
+ if kind is None:
+ hold = 0 if final else atem_hold_len(buf)
+ piece = buf[: len(buf) - hold] if hold else buf
+ emit(recipient, piece)
+ self._buffer = buf[len(piece):]
+ break
+ body = buf[:end]
+ if recipient not in ("self", "user"):
+ # A closing terminator belongs to the preserved block; a channel
+ # left without one (abutting header / headerless switch) gets a
+ # synthetic <|eom|> so the detector still sees a delimited block.
+ body += payload if kind == "closer" else "<|eom|>"
+ emit(recipient, body)
+ if kind == "closer":
+ self._buffer = buf[end + len(payload):]
+ self._mode = "seek"
+ elif kind == "start":
+ self._buffer = buf[end:]
+ self._mode = "seek"
+ else: # headerless switch: the next segment is fully known
+ self._buffer = buf[payload.end():]
+ begin_body(payload.group(1), buf[end: payload.end()])
+ return ReasoningParseResult(
+ reasoning_text="".join(out_reasoning), normal_text="".join(out_content)
+ )
+
+
class ReasoningParser:
"""Wraps a reasoning detector for streaming and non-streaming use."""
@@ -556,6 +880,7 @@ class ReasoningParser:
"glm": ThinkReasoningParser,
"minimax": ThinkReasoningParser,
"minimax_m3": MiniMaxM3ReasoningParser,
+ "muse_glimmer": MuseGlimmerReasoningParser,
"gemma4": GemmaThoughtReasoningParser,
}
diff --git a/python/freetoken/tokenizer/effort.py b/python/freetoken/tokenizer/effort.py
index d63c2443..c81cd7be 100644
--- a/python/freetoken/tokenizer/effort.py
+++ b/python/freetoken/tokenizer/effort.py
@@ -30,11 +30,39 @@ EFFORT_SCALE: dict[str, float] = {
KNOWN_REASONING_EFFORTS = tuple(EFFORT_SCALE)
-#: OpenAI's effort triple -- the common-denominator vocabulary offered for a
-#: template that grades effort without validating it (nothing observable
-#: narrows its gears, so offer what every dialect understands).
+#: OpenAI's effort triple -- the common-denominator vocabulary every dialect
+#: understands.
OPENAI_EFFORT_TRIPLE = ("low", "medium", "high")
+#: The graded ladder served to a template that grades effort WITHOUT validating
+#: it (muse-glimmer's "Reasoning strength"). Such a template interpolates any
+#: string, so its probed ``supported`` set is the whole scale -- but the
+#: checkpoint's trained levels are the ladder names, and never-advertised
+#: dialects (minimal / none / max) must quantize onto them instead of reaching
+#: the model verbatim.
+GRADED_EFFORT_LADDER = ("low", "medium", "high", "xhigh")
+
+
+def effective_efforts(profile: EffortProfile) -> frozenset[str]:
+ """The vocabulary a checkpoint is actually served (and advertised) with.
+
+ A NARROWED ``supported`` set is authoritative -- narrowing only ever comes
+ from observed rejections. A grader that accepted the whole scale validated
+ nothing, so pass-through would interpolate off-vocabulary values
+ ("minimal", "max") into the prompt verbatim: it is capped to its dialect's
+ ladder instead -- the strength dialect's documented levels top out at xhigh
+ (muse-glimmer's model card), a plain effort grader keeps the OpenAI triple
+ (the only vocabulary such a template is known to understand)."""
+ if not profile.consumes_effort:
+ return frozenset()
+ if profile.validates or profile.supported != frozenset(KNOWN_REASONING_EFFORTS):
+ return profile.supported
+ ladder = GRADED_EFFORT_LADDER if profile.strength_dialect else OPENAI_EFFORT_TRIPLE
+ vocab = {name for name in ladder if name in profile.supported}
+ if profile.default:
+ vocab.add(profile.default)
+ return frozenset(vocab)
+
#: Protocol-level thinking toggles, broadcast in every spelling the ecosystem's
#: templates read (``enable_thinking`` bool: qwen/glm/gemma/dsv4;
#: ``thinking_mode`` string: minimax-m3). Jinja ignores undeclared variables,
@@ -53,13 +81,17 @@ class EffortProfile:
ever changed its output or raised -- the template ignores the knob, so
requests should not carry it. ``validates`` True means the probe observed a
rejection: only then is ``supported`` a real vocabulary rather than "this
- template interpolates anything".
+ template interpolates anything". ``strength_dialect`` True means the
+ template reads the ``reasoning_strength`` spelling (muse-glimmer's family,
+ whose documented ladder tops out at xhigh) rather than only
+ ``reasoning_effort``.
"""
supported: frozenset[str]
default: str | None
consumes_effort: bool
validates: bool = False
+ strength_dialect: bool = False
@dataclass(frozen=True)
@@ -96,15 +128,16 @@ def quantize_effort(value: Any, profile: EffortProfile) -> str | None:
the template default. With "max" excluded the remaining scale values are
unique, so quantization is deterministic across processes.
"""
- if not profile.consumes_effort:
+ supported = effective_efforts(profile)
+ if not supported:
return None
- if isinstance(value, str) and value in profile.supported:
+ if isinstance(value, str) and value in supported:
return value
position = EFFORT_SCALE.get(value) if isinstance(value, str) else None
if position is None:
return None
ranked = sorted(
- (name for name in profile.supported if name != "max"),
+ (name for name in supported if name != "max"),
key=lambda name: (abs(EFFORT_SCALE[name] - position), -EFFORT_SCALE[name]),
)
if ranked and abs(EFFORT_SCALE[ranked[0]] - position) <= _MAX_QUANTIZE_DISTANCE:
@@ -170,6 +203,16 @@ def probe_effort_profile(
# Nothing learnable: sending no effort is the only safe rendering.
return EffortProfile(supported=frozenset(), default=None, consumes_effort=False)
+ # Which spelling does the template read? A render that moves on the
+ # reasoning_strength kwarg alone marks the muse dialect (its ladder differs).
+ strength_dialect = False
+ try:
+ baseline = render({}, None)
+ moved = render({"reasoning_strength": "low"}, None)
+ strength_dialect = _renderings_differ(moved, baseline)
+ except Exception: # noqa: BLE001 -- a rejecting template is not this dialect
+ strength_dialect = False
+
supported = frozenset(name for name in KNOWN_REASONING_EFFORTS if name not in rejected)
consumes = bool(rejected or diverged)
default = None
@@ -182,6 +225,7 @@ def probe_effort_profile(
default=default,
consumes_effort=consumes,
validates=bool(rejected),
+ strength_dialect=strength_dialect,
)
diff --git a/python/freetoken/tokenizer/tokenize.py b/python/freetoken/tokenizer/tokenize.py
index 8626061e..0636b342 100644
--- a/python/freetoken/tokenizer/tokenize.py
+++ b/python/freetoken/tokenizer/tokenize.py
@@ -60,8 +60,16 @@ class TokenizeManager:
# TODO: batch tokenization
for msg in msgs:
prompt = self.render_prompt(msg)
+ # A jinja chat template owns every special token (HF's apply_chat_template
+ # tokenizes with add_special_tokens=False for the same reason): tokenizers
+ # that auto-add bos (muse-glimmer's, llama's) would otherwise double it --
+ # the template already rendered one. Raw-string prompts and the dsv4
+ # encoder path keep the default.
+ templated = isinstance(msg.text, list) and self._dsv4_encoder is None
input_ids: torch.Tensor = ( # type: ignore
- self.tokenizer.encode(prompt, return_tensors="pt")
+ self.tokenizer.encode(
+ prompt, return_tensors="pt", add_special_tokens=not templated
+ )
)
results.append(input_ids.view(-1).to(torch.int32))
return results
@@ -89,6 +97,15 @@ class TokenizeManager:
return _apply_dsv4_chat_encoder(
self._dsv4_encoder, messages, tools, chat_template_kwargs
)
+ # Broadcast the effort in every spelling the ecosystem's templates read
+ # (muse-glimmer grades ``reasoning_strength``; Jinja ignores undeclared
+ # variables) -- the same rule the thinking toggles use. An explicit
+ # caller-provided spelling wins over the broadcast.
+ if "reasoning_effort" in chat_template_kwargs:
+ chat_template_kwargs = dict(chat_template_kwargs)
+ chat_template_kwargs.setdefault(
+ "reasoning_strength", chat_template_kwargs["reasoning_effort"]
+ )
if tools is not None:
chat_template_kwargs = {**chat_template_kwargs, "tools": tools}
prompt = self.tokenizer.apply_chat_template(
diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.py
index d9cad465..5a6a31f2 100644
--- a/python/freetoken/utils/hf.py
+++ b/python/freetoken/utils/hf.py
@@ -129,12 +129,63 @@ def load_generation_sampling(model_path: str) -> dict[str, Any]:
return out
+class RawConfigShim:
+ """Attribute view over a checkpoint's raw ``config.json``.
+
+ Fallback for model types NEWER than the installed transformers (AutoConfig raises
+ on an unknown ``model_type`` when the checkpoint ships no ``auto_map``, e.g.
+ Muse-Glimmer on transformers < 5.15). FreeToken only reads config fields
+ (parse_config) and never instantiates modeling code, so the raw JSON is enough.
+ ``*_config`` sub-dicts are wrapped for attribute access, matching
+ PretrainedConfig's nested-config behavior; every other value is served raw
+ (``rope_parameters`` and friends stay plain dicts, as they do on the real class).
+ """
+
+ def __init__(self, data: dict | None = None, **kwargs: Any) -> None:
+ self._data = {**(data or {}), **kwargs}
+
+ def __getattr__(self, name: str) -> Any:
+ if name == "_name_or_path":
+ # PretrainedConfig carries the checkpoint path under this name and
+ # DSV4's parse_config reads it (to find inference/config.json), so it
+ # must survive the underscore guard below.
+ return self.__dict__.get("_data", {}).get("_name_or_path", "")
+ if name.startswith("_"):
+ raise AttributeError(name)
+ try:
+ value = self._data[name]
+ except KeyError:
+ raise AttributeError(name) from None
+ if name.endswith("_config") and isinstance(value, dict):
+ return RawConfigShim(value)
+ return value
+
+ def to_dict(self) -> dict:
+ return json.loads(json.dumps(self._data)) # deep copy, callers may mutate
+
+
+def _raw_config_json(model_path: str) -> dict:
+ if os.path.isdir(model_path):
+ path = os.path.join(model_path, "config.json")
+ else:
+ path = hf_hub_download(repo_id=model_path, filename="config.json")
+ with open(path, encoding="utf-8") as f:
+ return json.load(f)
+
+
@functools.cache
def _load_hf_config(model_path: str) -> Any:
# trust_remote_code: checkpoints that ship a custom config class via ``auto_map``
# (e.g. MiniMax-M2) refuse to load without it. FreeToken only reads config fields
# (parse_config) and never instantiates the checkpoint's modeling code.
- return AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+ try:
+ return AutoConfig.from_pretrained(model_path, trust_remote_code=True)
+ except ValueError as exc:
+ # Unknown model_type on this transformers version: serve off the raw JSON.
+ # Anything else (bad path, malformed JSON) stays fatal.
+ if "model type" not in str(exc):
+ raise
+ return RawConfigShim(_raw_config_json(model_path), _name_or_path=model_path)
def cached_load_hf_config(model_path: str) -> PretrainedConfig:
@@ -148,6 +199,8 @@ def cached_load_hf_config(model_path: str) -> PretrainedConfig:
return build_gguf_shim(gguf_src)
config = _load_hf_config(model_path)
+ if isinstance(config, RawConfigShim):
+ return RawConfigShim(config.to_dict())
return type(config)(**config.to_dict())
diff --git a/tests/models/test_muse_glimmer.py b/tests/models/test_muse_glimmer.py
new file mode 100644
index 00000000..4a4457e9
--- /dev/null
+++ b/tests/models/test_muse_glimmer.py
@@ -0,0 +1,462 @@
+"""Muse-Glimmer-30B config parsing and serving-mode resolution.
+
+Drives ``muse_glimmer.parse_config`` off a synthetic HF config shaped exactly like
+meta-models/Muse-Glimmer-30B's (multimodal wrapper: text tower in ``text_config``,
+weights under ``model.language_model.``) and checks every decision the plan pinned:
+the [SWA x3, full] layer split with NoPE full layers, the folded qk scale, the two
+norm eps values, the logit post-processing scalars, the compressed-tensors NVFP4
+quant modes, and the pool/backend resolution.
+"""
+
+from __future__ import annotations
+
+import pytest
+
+from freetoken.attention.base import AttnType
+from freetoken.models.muse_glimmer.config import parse_config
+
+
+class _Cfg:
+ """Attribute-access shim over a dict (what AutoConfig hands parse_config)."""
+
+ def __init__(self, data: dict):
+ for k, v in data.items():
+ setattr(self, k, _Cfg(v) if isinstance(v, dict) and k == "text_config" else v)
+
+
+def _hf_config(num_layers: int = 52, quantized: bool = False) -> _Cfg:
+ pattern = ["sliding_attention"] * 3 + ["full_attention"]
+ thetas = [500000.0] * 3 + [0.0]
+ reps = (num_layers + 3) // 4
+ data = {
+ "architectures": ["MuseGlimmerForConditionalGeneration"],
+ "model_type": "muse_glimmer",
+ "image_token_id": 200092,
+ "text_config": {
+ "hidden_size": 6656,
+ "intermediate_size": 19968,
+ "num_hidden_layers": num_layers,
+ "num_attention_heads": 32,
+ "num_key_value_heads": 2,
+ "head_dim": 128,
+ "vocab_size": 202048,
+ "max_position_embeddings": 131072,
+ "hidden_activation": "silu",
+ "rms_norm_eps": 1e-5,
+ "post_norm_eps": 1e-8,
+ "qk_scale_factor": 3.87,
+ "final_logit_softcapping": 20.0,
+ "output_multiplier": 0.19611613513818404,
+ "sliding_window": 2048,
+ "tie_word_embeddings": False,
+ "layer_types": (pattern * reps)[:num_layers],
+ "layer_rope_theta": (thetas * reps)[:num_layers],
+ "rope_parameters": {"rope_theta": 500000.0, "rope_type": "default"},
+ "model_type": "muse_glimmer_text",
+ },
+ }
+ if quantized:
+ data["quantization_config"] = {
+ "quant_method": "compressed-tensors",
+ "format": "nvfp4-pack-quantized",
+ "config_groups": {
+ "group_0": {
+ "format": "nvfp4-pack-quantized",
+ "targets": ["Linear"],
+ "weights": {
+ "num_bits": 4,
+ "type": "float",
+ "group_size": 16,
+ "strategy": "tensor_group",
+ },
+ }
+ },
+ }
+ return _Cfg(data)
+
+
+def test_parse_config_full_model():
+ cfg = parse_config(_hf_config())
+
+ assert cfg.num_layers == 52
+ assert (cfg.num_qo_heads, cfg.num_kv_heads, cfg.head_dim) == (32, 2, 128)
+ assert cfg.hidden_size == 6656 and cfg.intermediate_size == 19968
+ assert cfg.vocab_size == 202048 and not cfg.tie_word_embeddings
+ assert cfg.hidden_act == "silu" and not cfg.is_moe
+ assert cfg.rms_norm_eps == 1e-5 and cfg.post_norm_eps == 1e-8
+ assert cfg.use_qk_norm
+ # q = qk_norm(q) * 3.87 with standard 1/sqrt(d) attention: folded into sm_scale.
+ assert cfg.attn_sm_scale == pytest.approx(3.87 * 128**-0.5)
+ assert cfg.final_logit_softcapping == 20.0
+ assert cfg.output_multiplier == pytest.approx(0.19611613513818404)
+ assert cfg.embedding_scale is None # NormedEmbedding, not Gemma's sqrt(hidden) scale
+ assert cfg.vision_config is None # served text-only
+
+ groups = {g.name: g for g in cfg.attention_groups}
+ assert groups["swa"].layer_ids == tuple(i for i in range(52) if (i + 1) % 4 != 0)
+ assert groups["full"].layer_ids == tuple(i for i in range(52) if (i + 1) % 4 == 0)
+ assert groups["swa"].sliding_window == 2048
+ assert groups["swa"].rotary_config.base == 500000.0
+ # NoPE full layers: base 0.0 is the skip-rope marker read by the attention module.
+ assert groups["full"].rotary_config.base == 0.0
+ assert cfg.rotary_config.base == 500000.0 # top-level config carries the real rope
+
+ specs = {s.name: s for s in cfg.kv_cache_group_specs()}
+ assert specs["swa"].attn_type == AttnType.SWA and specs["swa"].sliding_window == 2048
+ assert specs["full"].attn_type == AttnType.FULL and specs["full"].sliding_window is None
+ assert all((s.num_kv_heads, s.head_dim) == (2, 128) for s in specs.values())
+
+ # BF16 checkpoint: nothing quantized.
+ assert cfg.attn_quant == "none" and cfg.dense_quant == "none"
+ assert cfg.lm_head_quant == "none"
+
+
+def test_parse_config_nvfp4_checkpoint():
+ cfg = parse_config(_hf_config(quantized=True))
+ # compressed-tensors NVFP4 quantizes every text Linear (attention incl. the gate,
+ # and the MLP); lm_head / embeddings / norms stay bf16 (the ignore list).
+ assert cfg.attn_quant == "nvfp4" and cfg.dense_quant == "nvfp4"
+ assert cfg.lm_head_quant == "none"
+
+
+def test_non_nvfp4_4bit_scheme_is_a_clear_error():
+ # A compressed-tensors MXFP4 checkpoint (group_size 32, real since LLM Compressor
+ # 0.9) must fail with a clear "unsupported scheme" error instead of routing into
+ # the NVFP4 loader and dying in a shape assert.
+ hf = _hf_config(quantized=True)
+ weights = hf.quantization_config["config_groups"]["group_0"]["weights"]
+ weights["group_size"] = 32
+ weights["strategy"] = "group"
+ with pytest.raises(ValueError, match="unsupported compressed-tensors"):
+ parse_config(hf)
+
+
+def test_per_layer_theta_beats_shared_rope_theta():
+ # layer_rope_theta is the source of truth: a hypothetical checkpoint giving the
+ # full layers a real theta must not be forced to NoPE.
+ hf = _hf_config(num_layers=8)
+ hf.text_config.layer_rope_theta = [1e6] * 8
+ cfg = parse_config(hf)
+ groups = {g.name: g for g in cfg.attention_groups}
+ assert groups["full"].rotary_config.base == 1e6
+ assert groups["swa"].rotary_config.base == 1e6
+
+
+def test_pool_family_and_backend_resolution():
+ from freetoken.engine.engine import _required_attn_types, _resolve_auto_attention_backend
+ from freetoken.kvcache import resolve_pool_class
+ from freetoken.kvcache.hybrid_swa_pool import HybridSWAKVCache
+
+ cfg = parse_config(_hf_config())
+ assert resolve_pool_class(cfg) is HybridSWAKVCache
+ required = _required_attn_types(cfg)
+ assert required == frozenset({AttnType.FULL, AttnType.SWA})
+ # SWA restricts serving to the triton backend (the only one in the capability
+ # matrix that consumes per-call sliding windows), same as gemma4.
+ assert _resolve_auto_attention_backend(required, False) == "triton"
+
+
+def test_registry_resolves_architecture():
+ from freetoken.models.register import get_model_spec
+
+ spec = get_model_spec("MuseGlimmerForConditionalGeneration")
+ assert spec.module == "freetoken.models.muse_glimmer"
+ assert spec.model_cls == "MuseGlimmerForCausalLM"
+
+
+def test_aot_table_covers_the_checkpoints():
+ from freetoken.kernel.aot_models import SUPPORTED_MODELS
+
+ entry = next(m for m in SUPPORTED_MODELS if m.architecture == "MuseGlimmerForConditionalGeneration")
+ assert entry.hidden_size == 6656
+ assert entry.kv_groups == ((2, 128),)
+ assert entry.expert_formats == () # dense
+ assert "RedHatAI/Muse-Glimmer-30B-NVFP4" in entry.aliases
+
+
+def test_weight_rename_and_fusion():
+ import torch
+
+ from freetoken.models.loader import ct_bf16_fuse
+ from freetoken.models.muse_glimmer.weight import _FUSIONS, _rename
+
+ # Text tower renamed, vision dropped, lm_head untouched.
+ assert _rename("model.language_model.layers.0.self_attn.q_proj.weight") == (
+ "model.layers.0.self_attn.q_proj.weight"
+ )
+ assert _rename("model.language_model.embed_tokens.weight") == "model.embed_tokens.weight"
+ assert _rename("lm_head.weight") == "lm_head.weight"
+ assert _rename("model.vision_tower.layers.0.attn.q_proj.weight") is None
+ assert _rename("model.vision_adapter.fc1.weight") is None
+ assert _rename("model.vision_projection.weight") is None
+
+ # q/k/v + the attention gate fuse into qkvg_proj in declaration order.
+ buf: dict = {}
+ parts = {
+ "q_proj": torch.full((4, 2), 0.0),
+ "k_proj": torch.full((2, 2), 1.0),
+ "v_proj": torch.full((2, 2), 2.0),
+ "gate_proj": torch.full((4, 2), 3.0),
+ }
+ fused = None
+ for name, tensor in parts.items():
+ out = ct_bf16_fuse(f"model.layers.0.self_attn.{name}", tensor, buf, _FUSIONS)
+ assert out is not None
+ if out:
+ fused = out[0]
+ assert fused is not None and not buf
+ key, tensor = fused
+ assert key == "model.layers.0.self_attn.qkvg_proj.weight"
+ assert tensor.shape == (12, 2)
+ assert tensor[0, 0] == 0.0 and tensor[4, 0] == 1.0 and tensor[6, 0] == 2.0
+ assert tensor[8, 0] == 3.0
+
+ # The MLP's own gate_proj is a different fusion (gate|up), not the attention one.
+ buf2: dict = {}
+ assert ct_bf16_fuse("model.layers.0.mlp.gate_proj", torch.zeros(3, 2), buf2, _FUSIONS) == []
+ (out,) = ct_bf16_fuse("model.layers.0.mlp.up_proj", torch.ones(3, 2), buf2, _FUSIONS)
+ key2, tensor2 = out
+ assert key2 == "model.layers.0.mlp.gate_up_proj.weight" and tensor2.shape == (6, 2)
+
+
+def _write_shards(tmp_path, shards: dict[str, dict]):
+ """Write synthetic safetensors shards + an index, real checkpoint naming."""
+ import safetensors.torch
+
+ weight_map = {}
+ for shard, tensors in shards.items():
+ safetensors.torch.save_file(tensors, str(tmp_path / shard))
+ for name in tensors:
+ weight_map[name] = shard
+ (tmp_path / "model.safetensors.index.json").write_text(
+ __import__("json").dumps({"weight_map": weight_map})
+ )
+
+
+def _bf16_checkpoint_tensors(hf) -> dict:
+ import torch
+
+ text = hf.text_config
+ H, I = text.hidden_size, text.intermediate_size
+ q = text.num_attention_heads * text.head_dim
+ kv = text.num_key_value_heads * text.head_dim
+ p = "model.language_model."
+ tensors = {
+ p + "embed_tokens.weight": torch.randn(text.vocab_size, H, dtype=torch.bfloat16),
+ p + "norm.weight": torch.randn(H, dtype=torch.bfloat16),
+ "lm_head.weight": torch.randn(text.vocab_size, H, dtype=torch.bfloat16),
+ # vision tensors must be dropped, not loaded
+ "model.vision_tower.ln_pre.weight": torch.randn(4, dtype=torch.bfloat16),
+ "model.vision_adapter.fc1.weight": torch.randn(4, 4, dtype=torch.bfloat16),
+ "model.vision_projection.weight": torch.randn(4, 4, dtype=torch.bfloat16),
+ }
+ for i in range(text.num_hidden_layers):
+ lp = f"{p}layers.{i}."
+ tensors |= {
+ lp + "self_attn.q_proj.weight": torch.randn(q, H, dtype=torch.bfloat16),
+ lp + "self_attn.k_proj.weight": torch.randn(kv, H, dtype=torch.bfloat16),
+ lp + "self_attn.v_proj.weight": torch.randn(kv, H, dtype=torch.bfloat16),
+ lp + "self_attn.gate_proj.weight": torch.randn(q, H, dtype=torch.bfloat16),
+ lp + "self_attn.o_proj.weight": torch.randn(H, q, dtype=torch.bfloat16),
+ lp + "mlp.gate_proj.weight": torch.randn(I, H, dtype=torch.bfloat16),
+ lp + "mlp.up_proj.weight": torch.randn(I, H, dtype=torch.bfloat16),
+ lp + "mlp.down_proj.weight": torch.randn(H, I, dtype=torch.bfloat16),
+ lp + "input_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),
+ lp + "post_attention_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),
+ lp + "pre_feedforward_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),
+ lp + "post_feedforward_layernorm.weight": torch.randn(H, dtype=torch.bfloat16),
+ }
+ return tensors
+
+
+def test_iter_weights_bf16_matches_model_state_dict(tmp_path, monkeypatch):
+ """The BF16 loader must produce exactly the model's state-dict keys with the
+ right shapes (rename + qkvg / gate_up fusion, vision dropped, norms raw)."""
+ import torch
+
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+ from freetoken.models.muse_glimmer.model import MuseGlimmerForCausalLM
+ from freetoken.models.muse_glimmer.weight import iter_weights
+
+ hf = _hf_config(num_layers=4)
+ tensors = _bf16_checkpoint_tensors(hf)
+ _write_shards(tmp_path, {"model-00001-of-00001.safetensors": tensors})
+ import freetoken.models.muse_glimmer.weight as w
+
+ monkeypatch.setattr(w, "cached_load_hf_config", lambda _p: hf)
+
+ loaded = dict(
+ iter_weights(str(tmp_path), torch.device("cpu"), include_moe_experts=False, include_non_moe=True)
+ )
+ model = MuseGlimmerForCausalLM(parse_config(hf))
+ expected = model.state_dict()
+ assert set(loaded) == set(expected)
+ for k in expected:
+ assert loaded[k].shape == expected[k].shape, k
+ # fusion order [q, k, v, gate] against the raw parts
+ fused = loaded["model.layers.0.self_attn.qkvg_proj.weight"]
+ p = "model.language_model.layers.0.self_attn."
+ q_dim = tensors[p + "q_proj.weight"].shape[0]
+ kv_dim = tensors[p + "k_proj.weight"].shape[0]
+ assert torch.equal(fused[:q_dim], tensors[p + "q_proj.weight"])
+ assert torch.equal(fused[q_dim : q_dim + kv_dim], tensors[p + "k_proj.weight"])
+ assert torch.equal(fused[-q_dim:], tensors[p + "gate_proj.weight"])
+
+
+def test_iter_weights_nvfp4_cross_shard_scales(tmp_path, monkeypatch):
+ """compressed-tensors loader: native FP4 parts fused with per-part scales, the
+ reciprocal global, and sibling scales resolved through the index even when they
+ land in a different shard than their weight_packed (the real checkpoint splits
+ layer 49's down_proj across the shard boundary)."""
+ import torch
+
+ from freetoken.models.muse_glimmer.weight import iter_weights
+
+ hf = _hf_config(num_layers=1, quantized=True)
+ text = hf.text_config
+ text.hidden_size, text.intermediate_size = 64, 96
+ text.num_attention_heads, text.num_key_value_heads, text.head_dim = 4, 2, 16
+ text.vocab_size = 128
+ H, I = 64, 96
+ q, kv = 4 * 16, 2 * 16
+ fp8 = torch.float8_e4m3fn
+
+ def nvfp4(base, out_f, in_f, global_scale, shard_for_scales=None):
+ t = {
+ base + ".weight_packed": torch.randint(0, 256, (out_f, in_f // 2), dtype=torch.uint8),
+ base + ".weight_scale": torch.randn(out_f, in_f // 16).abs().to(fp8),
+ base + ".weight_global_scale": torch.tensor([global_scale]),
+ base + ".input_global_scale": torch.tensor([1.0]), # W4A4 scale: skipped
+ }
+ return t
+
+ p = "model.language_model.layers.0."
+ shard1 = {}
+ for base, (o, i) in {
+ p + "self_attn.q_proj": (q, H), p + "self_attn.k_proj": (kv, H),
+ p + "self_attn.v_proj": (kv, H), p + "self_attn.gate_proj": (q, H),
+ p + "self_attn.o_proj": (q, H)[::-1], p + "mlp.gate_proj": (I, H),
+ p + "mlp.up_proj": (I, H),
+ }.items():
+ shard1 |= nvfp4(base, o, i, 2.0)
+ # down_proj straddles the boundary: packed weight in shard 1, scales in shard 2.
+ down = nvfp4(p + "mlp.down_proj", H, I, 4.0)
+ shard1[p + "mlp.down_proj.weight_packed"] = down[p + "mlp.down_proj.weight_packed"]
+ shard2 = {k: v for k, v in down.items() if not k.endswith(".weight_packed")}
+ shard2 |= {
+ "model.language_model.embed_tokens.weight": torch.randn(128, H, dtype=torch.bfloat16),
+ "model.language_model.norm.weight": torch.randn(H, dtype=torch.bfloat16),
+ "lm_head.weight": torch.randn(128, H, dtype=torch.bfloat16),
+ }
+ for name in (
+ "input_layernorm", "post_attention_layernorm",
+ "pre_feedforward_layernorm", "post_feedforward_layernorm",
+ ):
+ shard1[p + name + ".weight"] = torch.randn(H, dtype=torch.bfloat16)
+ _write_shards(tmp_path, {
+ "model-00001-of-00002.safetensors": shard1,
+ "model-00002-of-00002.safetensors": shard2,
+ })
+ import freetoken.models.muse_glimmer.weight as w
+
+ monkeypatch.setattr(w, "cached_load_hf_config", lambda _p: hf)
+
+ loaded = dict(
+ iter_weights(str(tmp_path), torch.device("cpu"), include_moe_experts=False, include_non_moe=True)
+ )
+ qkvg = "model.layers.0.self_attn.qkvg_proj"
+ assert loaded[qkvg + ".weight"].shape == (2 * q + 2 * kv, H // 2)
+ assert loaded[qkvg + ".weight_scale"].shape == (2 * q + 2 * kv, H // 16)
+ assert loaded[qkvg + ".weight_global"].shape == (2 * q + 2 * kv,)
+ # global is the reciprocal of the stored quant-side scale
+ assert loaded[qkvg + ".weight_global"][0].item() == pytest.approx(0.5)
+ dp = "model.layers.0.mlp.down_proj"
+ assert loaded[dp + ".weight"].shape == (H, I // 2) # cross-shard scales resolved
+ assert loaded[dp + ".weight_global"][0].item() == pytest.approx(0.25)
+ # activation scales are never emitted
+ assert not any(k.endswith(".input_global_scale") for k in loaded)
+
+
+def test_raw_config_shim_serves_unknown_model_type(tmp_path):
+ """A checkpoint whose model_type is newer than the installed transformers must
+ still parse: cached_load_hf_config falls back to the raw config.json."""
+ import json as _json
+
+ from freetoken.utils.hf import RawConfigShim, _load_hf_config, cached_load_hf_config
+
+ raw = {
+ "architectures": ["MuseGlimmerForConditionalGeneration"],
+ "model_type": "some_model_type_from_the_future",
+ "dtype": "bfloat16",
+ "text_config": {"hidden_size": 64, "rope_parameters": {"rope_theta": 1e4}},
+ }
+ (tmp_path / "config.json").write_text(_json.dumps(raw))
+ _load_hf_config.cache_clear()
+ try:
+ cfg = cached_load_hf_config(str(tmp_path))
+ finally:
+ _load_hf_config.cache_clear()
+ assert isinstance(cfg, RawConfigShim)
+ assert cfg.architectures == ["MuseGlimmerForConditionalGeneration"]
+ assert cfg.text_config.hidden_size == 64
+ # sub-config wrapped for attribute access; plain dicts stay dicts
+ assert cfg.text_config.rope_parameters.get("rope_theta") == 1e4
+ assert getattr(cfg, "missing_field", None) is None
+ assert cfg.to_dict()["dtype"] == "bfloat16"
+ # _name_or_path survives the underscore guard (DSV4's parse_config reads it
+ # to locate inference/config.json) and the cached copy keeps it.
+ assert cfg._name_or_path == str(tmp_path)
+
+
+def test_model_state_dict_matches_loader_keys():
+ import torch
+
+ from freetoken.distributed import set_tp_info, try_get_tp_info
+
+ if try_get_tp_info() is None:
+ set_tp_info(rank=0, size=1)
+ from freetoken.models.muse_glimmer.model import MuseGlimmerForCausalLM
+
+ cfg = parse_config(_hf_config(num_layers=4))
+ model = MuseGlimmerForCausalLM(cfg)
+ keys = set(model.state_dict().keys())
+ layer0 = {k for k in keys if k.startswith("model.layers.0.")}
+ assert layer0 == {
+ "model.layers.0.self_attn.qkvg_proj.weight",
+ "model.layers.0.self_attn.o_proj.weight",
+ "model.layers.0.mlp.gate_up_proj.weight",
+ "model.layers.0.mlp.down_proj.weight",
+ "model.layers.0.input_layernorm.weight",
+ "model.layers.0.post_attention_layernorm.weight",
+ "model.layers.0.pre_feedforward_layernorm.weight",
+ "model.layers.0.post_feedforward_layernorm.weight",
+ }
+ # Weightless norms (embed_norm, qk_norm) must not demand checkpoint tensors.
+ assert "model.embed_norm.weight" not in keys
+ assert not any("qk_norm" in k for k in keys)
+ assert {"model.embed_tokens.weight", "model.norm.weight", "lm_head.weight"} <= keys
+
+ # Rope built only where a real theta exists: sliding layers yes, NoPE full no.
+ layers = model.model.layers.op_list
+ assert layers[0].self_attn.rotary is not None and layers[0].self_attn.is_swa
+ assert layers[3].self_attn.rotary is None and not layers[3].self_attn.is_swa
+ assert layers[0].self_attn.attn_spec.sliding_window == 2048
+ assert layers[3].self_attn.attn_spec.sliding_window is None
+ assert layers[0].self_attn.attn_spec.sm_scale == pytest.approx(3.87 * 128**-0.5)
+
+ # NVFP4 build swaps every text Linear for the W4A16 kernels.
+ from freetoken.kernel.triton.nvfp4_linear import Nvfp4DenseColMerged, Nvfp4DenseLinear
+
+ qcfg = parse_config(_hf_config(num_layers=4, quantized=True))
+ qmodel = MuseGlimmerForCausalLM(qcfg)
+ attn = qmodel.model.layers.op_list[0].self_attn
+ mlp = qmodel.model.layers.op_list[0].mlp
+ assert isinstance(attn.qkvg_proj, Nvfp4DenseColMerged)
+ assert isinstance(attn.o_proj, Nvfp4DenseLinear)
+ assert isinstance(mlp.gate_up_proj, Nvfp4DenseColMerged)
+ assert isinstance(mlp.down_proj, Nvfp4DenseLinear)
+ assert type(qmodel.lm_head).__name__ == "ParallelLMHead" # lm_head stays bf16
+ del model, qmodel, torch
diff --git a/tests/server/test_muse_glimmer_parsers.py b/tests/server/test_muse_glimmer_parsers.py
new file mode 100644
index 00000000..3a492f2b
--- /dev/null
+++ b/tests/server/test_muse_glimmer_parsers.py
@@ -0,0 +1,1275 @@
+"""Muse Glimmer ATEM reasoning parser (to=self / to=user / tool channels) and
+tool-call detector (<atem:function_calls> invoke/parameter XML) -- one-shot and
+streaming."""
+
+from __future__ import annotations
+
+import json
+
+import pytest
+
+from freetoken.server.function_call_parser import (
+ Function,
+ FunctionCallParser,
+ MuseGlimmerDetector,
+ Tool,
+)
+from freetoken.server.reasoning_parser import (
+ ATEM_START,
+ MuseGlimmerReasoningParser,
+ ReasoningParser,
+)
+
+
+def _tools():
+ return [
+ Tool(function=Function(name="weather.get", parameters={
+ "type": "object",
+ "properties": {
+ "city": {"type": "string"},
+ "days": {"type": "integer"},
+ "units": {"type": "object"},
+ },
+ })),
+ Tool(function=Function(name="fs.write", parameters={
+ "type": "object",
+ "properties": {"path": {"type": "string"}, "content": {"type": "string"}},
+ })),
+ ]
+
+
+def _atem(name: str, params: dict[str, str]) -> str:
+ body = "".join(
+ f'<atem:parameter name="{k}">{v}</atem:parameter>\n' for k, v in params.items()
+ )
+ return (
+ f'<atem:function_calls>\n<atem:invoke name="{name}">\n{body}'
+ f"</atem:invoke>\n</atem:function_calls>"
+ )
+
+
+def _tool_channel(name: str, params: dict[str, str], *, closer: str = "<|eot|>") -> str:
+ return f"<|start|>assistant to={name}<|message|>{_atem(name, params)}{closer}"
+
+
+def _stream(parser, text: str, step: int = 3):
+ reasoning, content = [], []
+ for i in range(0, len(text), step):
+ r = parser.parse_streaming_increment(text[i : i + step])
+ reasoning.append(r.reasoning_text)
+ content.append(r.normal_text)
+ r = parser.flush()
+ reasoning.append(r.reasoning_text)
+ content.append(r.normal_text)
+ return "".join(reasoning), "".join(content)
+
+
+def _stream_detect(det, text: str, tools, step: int = 3):
+ normal, calls = [], []
+ for i in range(0, len(text), step):
+ r = det.parse_streaming_increment(text[i : i + step], tools)
+ normal.append(r.normal_text)
+ calls.extend(r.calls)
+ while True:
+ r = det.parse_streaming_increment("", tools)
+ if not r.normal_text and not r.calls:
+ break
+ normal.append(r.normal_text)
+ calls.extend(r.calls)
+ normal.append(det.finish_streaming())
+ return "".join(normal), calls
+
+
+def _assemble(calls):
+ """(name, joined-args-json) per streamed call, in order."""
+ out: list[list] = []
+ for c in calls:
+ if c.name is not None:
+ out.append([c.name, ""])
+ if c.parameters:
+ out[-1][1] += c.parameters
+ return [(name, json.loads(args or "{}")) for name, args in out]
+
+
+# ---------------------------------------------------------------------------
+# Reasoning parser
+# ---------------------------------------------------------------------------
+def test_reasoning_registered():
+ assert ReasoningParser.ReasoningParserEnum["muse_glimmer"] is MuseGlimmerReasoningParser
+
+
+def test_reasoning_bare_first_segment_self_then_user():
+ text = (
+ " to=self<|message|>Let me think about this.<|eom|>"
+ "<|start|>assistant to=user<|message|>The answer is 42.<|eot|>"
+ )
+ r = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert r.reasoning_text == "Let me think about this."
+ assert r.normal_text == "The answer is 42."
+
+
+def test_reasoning_user_only_turn():
+ r = MuseGlimmerReasoningParser().detect_and_parse(" to=user<|message|>Hi!<|eot|>")
+ assert r.reasoning_text == "" and r.normal_text == "Hi!"
+
+
+def test_reasoning_recipientless_header_is_content():
+ r = MuseGlimmerReasoningParser().detect_and_parse("assistant<|message|>Plain.<|eot|>")
+ assert r.reasoning_text == "" and r.normal_text == "Plain."
+
+
+def test_reasoning_plain_text_passthrough():
+ # Raw, non-templated output carries no ATEM markers and must pass through.
+ text = "Just a plain answer, no channels."
+ r = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert r.reasoning_text == "" and r.normal_text == text
+ p = MuseGlimmerReasoningParser()
+ reasoning, content = _stream(p, text)
+ assert reasoning == "" and content == text
+
+
+def test_reasoning_tool_channel_preserved_verbatim():
+ text = (
+ " to=self<|message|>check weather<|eom|>"
+ "<|start|>assistant to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "Paris"})
+ + "<|eot|>"
+ )
+ r = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert r.reasoning_text == "check weather"
+ assert r.normal_text.startswith("<|start|>assistant to=weather.get<|message|>")
+ assert "<atem:function_calls>" in r.normal_text and r.normal_text.endswith("<|eot|>")
+
+ # A tool call as the FIRST (bare) segment: the header-open parser completes
+ # the bare header with its synthetic <|start|> seed, so the detector always
+ # receives a fully delimited channel -- no bare-header guessing downstream.
+ bare = " to=weather.get<|message|>" + _atem("weather.get", {"city": "Rome"}) + "<|eot|>"
+ r2 = MuseGlimmerReasoningParser().detect_and_parse(bare)
+ assert r2.normal_text.startswith("<|start|> to=weather.get<|message|>")
+ assert r2.normal_text.endswith("<|eot|>")
+
+
+def test_reasoning_streaming_matches_one_shot():
+ text = (
+ " to=self<|message|>step one\nstep two<|eom|>"
+ "<|start|>assistant to=user<|message|>Done: 42.<|eot|>"
+ )
+ one = MuseGlimmerReasoningParser().detect_and_parse(text)
+ reasoning, content = _stream(MuseGlimmerReasoningParser(), text)
+ assert reasoning == one.reasoning_text == "step one\nstep two"
+ assert content == one.normal_text == "Done: 42."
+
+
+def test_reasoning_streaming_holds_partial_markers():
+ p = MuseGlimmerReasoningParser()
+ out = p.parse_streaming_increment(" to=user<|message|>Hello <|eo")
+ assert out.normal_text == "Hello " # the partial closer is held, not leaked
+ out = p.parse_streaming_increment("t|>")
+ assert out.normal_text == ""
+ r = p.flush()
+ assert r.normal_text == "" and r.reasoning_text == ""
+
+
+def test_reasoning_undecided_header_delivered_at_flush():
+ # Held while it could still become a bare header; at end-of-stream it is a
+ # (short) reply and must be delivered, not dropped.
+ p = MuseGlimmerReasoningParser()
+ assert p.parse_streaming_increment(" to=se").normal_text == ""
+ r = p.flush()
+ assert r.normal_text == " to=se" and r.reasoning_text == ""
+
+
+def test_reasoning_multiple_self_segments_concatenate():
+ text = (
+ " to=self<|message|>alpha<|eom|>"
+ "<|start|>assistant to=self<|message|>beta<|eom|>"
+ "<|start|>assistant to=user<|message|>done<|eot|>"
+ )
+ r = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert r.reasoning_text == "alphabeta"
+ assert r.normal_text == "done"
+
+
+# ---------------------------------------------------------------------------
+# Tool-call detector
+# ---------------------------------------------------------------------------
+def test_detect_and_parse_typed_args():
+ det = MuseGlimmerDetector()
+ text = "One sec. " + _tool_channel(
+ "weather.get", {"city": "Paris", "days": "3", "units": '{"temp": "C"}'}
+ )
+ assert det.has_tool_call(text)
+ res = det.detect_and_parse(text, _tools())
+ assert res.normal_text == "One sec."
+ assert len(res.calls) == 1
+ call = res.calls[0]
+ assert call.name == "weather.get"
+ args = json.loads(call.parameters)
+ assert args == {"city": "Paris", "days": 3, "units": {"temp": "C"}}
+
+
+def test_detect_and_parse_multiple_invokes_in_one_block():
+ det = MuseGlimmerDetector()
+ block = (
+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n'
+ '<atem:parameter name="city">Paris</atem:parameter>\n'
+ "</atem:invoke>\n"
+ '<atem:invoke name="weather.get">\n'
+ '<atem:parameter name="city">Tokyo</atem:parameter>\n'
+ "</atem:invoke>\n"
+ "</atem:function_calls><|eot|>"
+ )
+ res = det.detect_and_parse(block, _tools())
+ assert [json.loads(c.parameters)["city"] for c in res.calls] == ["Paris", "Tokyo"]
+ assert [c.tool_index for c in res.calls] == [0, 1]
+
+
+def test_string_values_kept_verbatim():
+ # The template's contract: "spaces for string values are not stripped".
+ det = MuseGlimmerDetector()
+ content = "line1\nline2\n"
+ text = _tool_channel("fs.write", {"path": " /tmp/x ", "content": content})
+ res = det.detect_and_parse(text, _tools())
+ assert json.loads(res.calls[0].parameters) == {"path": " /tmp/x ", "content": content}
+
+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())
+ assert _assemble(calls) == [("fs.write", {"path": " /tmp/x ", "content": content})]
+ assert normal.strip() == ""
+
+
+def test_streaming_matches_one_shot_with_channel_markup():
+ text = "Checking. " + _tool_channel("weather.get", {"city": "Paris", "days": "3"})
+ one = MuseGlimmerDetector().detect_and_parse(text, _tools())
+
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, text, _tools())
+ assert normal.strip() == one.normal_text.strip() == "Checking."
+ assert _assemble(calls) == [
+ (c.name, json.loads(c.parameters)) for c in one.calls
+ ] == [("weather.get", {"city": "Paris", "days": 3})]
+ # ledgers the serving layer reads at stream end
+ assert det.prev_tool_call_arr[0]["name"] == "weather.get"
+ assert det.prev_tool_call_arr[0]["arguments"] == {"city": "Paris", "days": 3}
+
+
+def test_bare_block_outside_tool_channel_is_text_not_execution():
+ # Echo-becomes-execution guard (vLLM's rule): ATEM markup is executed only
+ # inside a tool-recipient channel. A block in plain text / a to=user body --
+ # e.g. the system prompt's own ATEM example echoed back -- renders as text.
+ text = "Look: " + _atem("weather.get", {"city": "Oslo"})
+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())
+ assert calls == []
+ assert normal == text
+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())
+ assert res.calls == [] and res.normal_text == text
+
+ quoted = (
+ " to=user<|message|>Call tools like this:\n"
+ + _atem("weather.get", {"city": "Echo"})
+ + "\nGot it?<|eot|>"
+ )
+ normal, calls = _stream_detect(MuseGlimmerDetector(), quoted, _tools())
+ assert calls == []
+ assert "Call tools like this:" in normal and "Got it?" in normal
+ assert "<atem:function_calls>" in normal # the quote stays visible text
+
+
+def test_streaming_bare_first_segment_tool_channel():
+ # No reasoning parser upstream: generation starts with the header continuation.
+ text = " to=weather.get<|message|>" + _atem("weather.get", {"city": "Lima"}) + "<|eot|>"
+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())
+ assert normal.strip() == ""
+ assert _assemble(calls) == [("weather.get", {"city": "Lima"})]
+
+
+def test_streaming_self_channel_swallowed_without_reasoning_parser():
+ text = (
+ " to=self<|message|>secret chain of thought<|eom|>"
+ "<|start|>assistant to=user<|message|>Visible.<|eot|>"
+ )
+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())
+ assert calls == []
+ assert "secret" not in normal
+ assert normal.strip() == "Visible."
+
+
+def test_streaming_two_tool_channels_sequential_indices():
+ text = (
+ _tool_channel("weather.get", {"city": "Paris"}, closer="<|eom|>")
+ + _tool_channel("weather.get", {"city": "Rome"})
+ )
+ _, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())
+ assembled = _assemble(calls)
+ assert assembled == [
+ ("weather.get", {"city": "Paris"}),
+ ("weather.get", {"city": "Rome"}),
+ ]
+ named = [c.tool_index for c in calls if c.name is not None]
+ assert named == [0, 1]
+
+
+def test_streaming_trailing_text_defers_until_after_call():
+ det = MuseGlimmerDetector()
+ chunk = "Pre. " + _tool_channel("weather.get", {"city": "Paris"}) + " Post."
+ r1 = det.parse_streaming_increment(chunk, _tools())
+ assert r1.normal_text == "Pre. "
+ assert [c.name for c in r1.calls if c.name] == ["weather.get"]
+ r2 = det.parse_streaming_increment("", _tools())
+ assert r2.normal_text == " Post." and r2.calls == []
+
+
+def test_streaming_plain_text_never_held():
+ det = MuseGlimmerDetector()
+ r = det.parse_streaming_increment("Just a normal answer, no tools.", _tools())
+ assert r.normal_text == "Just a normal answer, no tools."
+ assert r.calls == []
+
+
+def test_truncated_call_suppressed_and_recovered():
+ # Generation cut mid-invoke (max_tokens). The call already started streaming
+ # (its Start was emitted at the invoke open), so the serving layer closes it
+ # from unstreamed_arguments (the detector's partial-parse ledger), and
+ # finish_streaming must not leak the raw markup.
+ parser = FunctionCallParser(_tools(), "muse_glimmer")
+ det = parser.detector
+ truncated = (
+ "Ordering. <|start|>assistant to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n'
+ '<atem:parameter name="city">Paris</atem:parameter>\n'
+ '<atem:parameter name="days">4'
+ )
+ normal, calls = "", []
+ for i in range(0, len(truncated), 11):
+ r = det.parse_streaming_increment(truncated[i : i + 11], _tools())
+ normal += r.normal_text
+ calls.extend(r.calls)
+ assert normal == "Ordering. "
+ assert [c.name for c in calls if c.name] == ["weather.get"]
+ # the completed parameter survives in the ledger; the truncated one is dropped
+ assert json.loads(parser.unstreamed_arguments(0)) == {"city": "Paris"}
+ assert det.finish_streaming() == ""
+
+
+def test_unknown_tool_follows_forwarding_policy():
+ import freetoken.server.function_call_parser as fcp
+
+ text = _tool_channel("nope.call", {"a": "1"})
+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())
+ assert [c.name for c in res.calls] == ["nope.call"] # default: forwarded
+
+ orig = fcp.FORWARD_UNKNOWN_TOOLS
+ try:
+ fcp.FORWARD_UNKNOWN_TOOLS = False
+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())
+ assert res.calls == []
+ finally:
+ fcp.FORWARD_UNKNOWN_TOOLS = orig
+
+
+def test_empty_parameters_invoke():
+ det = MuseGlimmerDetector()
+ block = (
+ "<|start|>assistant to=weather.get<|message|>"
+ '<atem:function_calls>\n<atem:invoke name="weather.get">\n'
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ )
+ res = det.detect_and_parse(block, _tools())
+ assert json.loads(res.calls[0].parameters) == {}
+ _, calls = _stream_detect(MuseGlimmerDetector(), block, _tools())
+ assert _assemble(calls) == [("weather.get", {})]
+
+
+# ---------------------------------------------------------------------------
+# Review regressions (PR #4)
+# ---------------------------------------------------------------------------
+def test_channel_close_mid_invoke_does_not_corrupt_next_call():
+ """HIGH-1: a channel closer arriving mid-invoke (literal <|eom|> inside a
+ parameter value, or a truncated invoke) must finalize the open call --
+ completed params kept, JSON closed, ordinal advanced -- so the NEXT
+ channel's call never merges into it."""
+ text = (
+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"
+ '<atem:invoke name="fs.write">\n'
+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'
+ '<atem:parameter name="content">stop token is <|eom|>'
+ "<|start|>assistant to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "Paris"})
+ + "<|eot|>"
+ )
+ for step in (3, 7, len(text)): # chunked and single-chunk
+ det = MuseGlimmerDetector()
+ _, calls = _stream_detect(det, text, _tools(), step=step)
+ assembled = _assemble(calls)
+ assert [a[0] for a in assembled] == ["fs.write", "weather.get"], assembled
+ # MED-2: the truncated call's streamed fragments still concatenate to
+ # VALID JSON; the completed parameter survives, the cut one is closed.
+ first_args = assembled[0][1]
+ assert first_args["path"] == "/tmp/t"
+ assert assembled[1][1] == {"city": "Paris"}
+ named = [c.tool_index for c in calls if c.name is not None]
+ assert named == [0, 1] # distinct ordinals: no merge
+ assert det.prev_tool_call_arr[0]["arguments"]["path"] == "/tmp/t"
+ assert det.prev_tool_call_arr[1]["arguments"] == {"city": "Paris"}
+
+
+def test_headerless_channel_switch_executes_the_call():
+ """HIGH-2: the model may leave to=self without <|eom|>, writing
+ to=<tool><|message|> directly; a complete match is a segment boundary
+ (vLLM's rule) in both parsers."""
+ text = (
+ " to=self<|message|>I should check the weather. to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "Paris"})
+ + "<|eot|>"
+ )
+ # reasoning parser: the switch ends reasoning and preserves the tool slice
+ one = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert one.reasoning_text == "I should check the weather."
+ assert one.normal_text.startswith("to=weather.get<|message|>")
+ reasoning, content = _stream(MuseGlimmerReasoningParser(), text)
+ assert reasoning.rstrip() == "I should check the weather."
+ assert content.startswith("to=weather.get<|message|>")
+
+ # detector on the preserved slice
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, content, _tools())
+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]
+ assert normal.strip() == ""
+
+ # and end-to-end without a reasoning parser upstream
+ det2 = MuseGlimmerDetector()
+ normal2, calls2 = _stream_detect(det2, text, _tools())
+ assert _assemble(calls2) == [("weather.get", {"city": "Paris"})]
+ assert "I should check" not in normal2 # self body swallowed
+
+
+def test_headerless_switch_to_user_streams_content():
+ text = " to=self<|message|>quick thought to=user<|message|>Here you go.<|eot|>"
+ r = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert r.reasoning_text == "quick thought"
+ assert r.normal_text == "Here you go."
+ reasoning, content = _stream(MuseGlimmerReasoningParser(), text)
+ assert reasoning.rstrip() == "quick thought"
+ assert content == "Here you go."
+
+
+def test_doubled_tool_name_collapses_to_registered_head():
+ """MED-1: the template renders a bare-name tool's namespace as name.*, so the
+ model emits get_weather.get_weather; collapse iff the head is registered."""
+ tools = [
+ Tool(function=Function(name="get_weather", parameters={
+ "type": "object", "properties": {"city": {"type": "string"}},
+ }))
+ ]
+ text = (
+ "<|start|>assistant to=get_weather.get_weather<|message|>"
+ + _atem("get_weather.get_weather", {"city": "Paris"})
+ + "<|eot|>"
+ )
+ det = MuseGlimmerDetector()
+ _, calls = _stream_detect(det, text, tools)
+ assert _assemble(calls) == [("get_weather", {"city": "Paris"})]
+ res = MuseGlimmerDetector().detect_and_parse(text, tools)
+ assert [c.name for c in res.calls] == ["get_weather"]
+
+ # a genuinely namespaced name ("weather.get") is never collapsed, and a
+ # doubled form that IS registered stays as-is
+ res2 = MuseGlimmerDetector().detect_and_parse(
+ _tool_channel("weather.get", {"city": "Rome"}), _tools()
+ )
+ assert [c.name for c in res2.calls] == ["weather.get"]
+ tools_doubled = tools + [
+ Tool(function=Function(name="get_weather.get_weather", parameters={}))
+ ]
+ res3 = MuseGlimmerDetector().detect_and_parse(text, tools_doubled)
+ assert [c.name for c in res3.calls] == ["get_weather.get_weather"]
+
+
+def test_text_after_tool_channel_still_streams():
+ """MED-3 companion: after a tool channel closes, following text streams; the
+ detector must not stay latched in tool mode."""
+ text = _tool_channel("weather.get", {"city": "Paris"}) + " All done."
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, text, _tools())
+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]
+ assert normal.strip() == "All done."
+
+
+def test_one_shot_classifies_channels_without_reasoning_parser():
+ """MED-5: with --reasoning-parser off, non-streaming parsing must still run
+ the channel classification: to=self never leaks, to=user is kept."""
+ text = (
+ " to=self<|message|>hidden chain of thought<|eom|>"
+ "<|start|>assistant to=user<|message|>Let me check.<|eom|>"
+ "<|start|>assistant to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "Paris"})
+ + "<|eot|>"
+ )
+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())
+ assert "hidden chain of thought" not in res.normal_text
+ assert res.normal_text == "Let me check."
+ assert [c.name for c in res.calls] == ["weather.get"]
+ assert json.loads(res.calls[0].parameters) == {"city": "Paris"}
+
+
+def test_content_starting_with_header_lookalikes_streams():
+ """MED-4: committing to "header" requires a full to=/assistant +
+ <|message|> match; lookalike content must stream as text."""
+ for text in (
+ "assistant is a role name, not a header.",
+ "to=whom it may concern: hello.",
+ ):
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, text, _tools())
+ assert calls == []
+ assert normal == text
+ p = MuseGlimmerReasoningParser()
+ reasoning, content = _stream(p, text)
+ assert reasoning == "" and content == text
+
+
+def test_tool_slice_holds_partial_markers_while_streaming():
+ """LOW-1: the verbatim tool-channel slice must hold back a partial marker;
+ emitted content never shrinks when "<|start|" turns out to open the NEXT
+ segment."""
+ p = MuseGlimmerReasoningParser()
+ block = _tool_channel("weather.get", {"city": "Paris"}, closer="")
+ emitted = []
+ out = p.parse_streaming_increment(block + "<|star")
+ emitted.append(out.normal_text)
+ assert not "".join(emitted).endswith("<|star") # partial opener held back
+ out = p.parse_streaming_increment("t|>assistant to=user<|message|>done<|eot|>")
+ emitted.append(out.reasoning_text or "")
+ content = "".join(e for e in emitted if e) + out.normal_text + "".join(p.flush().normal_text)
+ assert "<|star" + "t|>" not in content.replace("<|start|>", "") # no split debris
+
+
+def test_truncated_tool_channel_warns(caplog):
+ """LOW-5: a truncated tool channel is logged, not silently dropped."""
+ import logging
+
+ text = (
+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par<|eot|>'
+ )
+ det = MuseGlimmerDetector()
+ with caplog.at_level(logging.WARNING):
+ _stream_detect(det, text, _tools())
+ assert any("mid-invoke" in r.message for r in caplog.records)
+
+
+# ---------------------------------------------------------------------------
+# Review regressions (PR #4, round 2)
+# ---------------------------------------------------------------------------
+def _pipe(text, tools, step=3):
+ """The serving pipeline shape: reasoning parser first, its content feeds the
+ detector; returns (reasoning, detector_normal, assembled_calls)."""
+ rp = MuseGlimmerReasoningParser()
+ det = MuseGlimmerDetector()
+ reasoning, normal, calls = "", [], []
+
+ def feed(piece):
+ if not piece:
+ return
+ r = det.parse_streaming_increment(piece, tools)
+ normal.append(r.normal_text)
+ calls.extend(r.calls)
+
+ for i in range(0, len(text), step):
+ r = rp.parse_streaming_increment(text[i : i + step])
+ reasoning += r.reasoning_text
+ feed(r.normal_text)
+ r = rp.flush()
+ reasoning += r.reasoning_text
+ feed(r.normal_text)
+ while True:
+ r = det.parse_streaming_increment("", tools)
+ if not r.normal_text and not r.calls:
+ break
+ normal.append(r.normal_text)
+ calls.extend(r.calls)
+ calls.extend(det.finalize_streaming())
+ normal.append(det.finish_streaming())
+ return reasoning, "".join(normal), calls
+
+
+ "switch",
+ [
+ "<|start|>assistant to=user<|message|>", # abutting header, no <|eom|>
+ "to=user<|message|>", # headerless switch
+ ],
+)
+def test_tool_channel_without_terminator_keeps_rest_of_turn(switch):
+ """R2 HIGH-1: a tool channel ending via an abutting header (no closer) must
+ reach the detector as a delimited block -- the rest of the turn survives."""
+ text = (
+ " to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "Paris"})
+ + switch
+ + "visible answer<|eot|>"
+ )
+ reasoning, normal, calls = _pipe(text, _tools())
+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]
+ assert normal.strip() == "visible answer"
+ assert reasoning == ""
+
+
+def test_eos_mid_invoke_finalizes_with_valid_fragments():
+ """R2 HIGH-2: max_tokens mid-invoke -- finalize_streaming emits the closing
+ fragments so the client's concatenated argument JSON ends valid, and the
+ parse ledger serves unstreamed_arguments (even for an empty-dict call)."""
+ parser = FunctionCallParser(_tools(), "muse_glimmer")
+ det = parser.detector
+ truncated = (
+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par'
+ )
+ frags = ""
+ for i in range(0, len(truncated), 7):
+ r = det.parse_streaming_increment(truncated[i : i + 7], _tools())
+ frags += "".join(c.parameters or "" for c in r.calls)
+ closing = parser.finalize_stream()
+ frags += "".join(c.parameters or "" for c in closing)
+ assert json.loads(frags) == {"city": "Par"} # valid, truncated value closed
+ assert json.loads(parser.unstreamed_arguments(0)) == {"city": "Par"}
+ assert det.finish_streaming() == ""
+
+ # empty-dict ledger entries serve too ({} used to be falsy-skipped)
+ parser2 = FunctionCallParser(_tools(), "muse_glimmer")
+ det2 = parser2.detector
+ trunc2 = (
+ "<|start|>assistant to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n'
+ )
+ for i in range(0, len(trunc2), 7):
+ det2.parse_streaming_increment(trunc2[i : i + 7], _tools())
+ frags2 = "".join(c.parameters or "" for c in parser2.finalize_stream())
+ assert json.loads(frags2) == {}
+ assert parser2.unstreamed_arguments(0) == "{}"
+
+
+def test_one_shot_keeps_second_invoke_block_in_one_channel():
+ """R2 HIGH-3: two <atem:function_calls> blocks inside ONE channel -- the
+ one-shot path (a streaming replay over an undrained buffer) must parse both,
+ not discard the second as markup debris at the channel close."""
+ text = (
+ "<|start|>assistant to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "Paris"})
+ + "\n"
+ + _atem("weather.get", {"city": "Rome"})
+ + "<|eot|>"
+ )
+ res = MuseGlimmerDetector().detect_and_parse(text, _tools())
+ assert [json.loads(c.parameters)["city"] for c in res.calls] == ["Paris", "Rome"]
+ assert [c.tool_index for c in res.calls] == [0, 1]
+ # streaming agrees
+ _, calls = _stream_detect(MuseGlimmerDetector(), text, _tools())
+ assert [a[1]["city"] for a in _assemble(calls)] == ["Paris", "Rome"]
+
+
+def test_reasoning_strength_kwarg_does_not_swallow_the_thinking_toggle():
+ """R2 HIGH-4 (post effort-unification): a muse-style reasoning_strength kwarg
+ riding along must not disable the broadcast thinking toggle."""
+ from freetoken.server.model_meta import effort_toggle_kwargs
+
+ out = effort_toggle_kwargs("high", {"reasoning_strength": "high"})
+ assert out.get("enable_thinking") is True # the toggle broadcast still maps
+ assert out.get("reasoning_strength") == "high" # the explicit kwarg rides along
+ assert out.get("reasoning_effort") == "high"
+
+
+def test_reasoning_streaming_buffer_stays_trimmed():
+ """R2 MED-5: streaming must consume the buffer as it emits (the full-buffer
+ re-scan per chunk was quadratic and stalled the serving loop on long CoT)."""
+ p = MuseGlimmerReasoningParser()
+ p.parse_streaming_increment(" to=self<|message|>")
+ for _ in range(2000):
+ p.parse_streaming_increment("think " * 4)
+ assert len(p._buffer) < 256 # consumed, not accumulated
+
+
+def test_flush_delivers_stray_closer_tail():
+ """R2 MED-6(b): prose stranded after a literal closer is delivered at
+ end-of-stream instead of dropped."""
+ text = "The token <|eot|> ends a turn."
+ p = MuseGlimmerReasoningParser()
+ content = ""
+ for i in range(0, len(text), 5):
+ content += p.parse_streaming_increment(text[i : i + 5]).normal_text
+ content += p.flush().normal_text
+ assert "The token " in content and " ends a turn." in content
+
+
+def test_flush_delivers_header_lookalike_reply():
+ """R2 MED-6(a): a complete reply that merely looks like a bare-header prefix
+ is content at end-of-stream, in both parsers."""
+ for reply in ("assistant", "[email protected]"):
+ p = MuseGlimmerReasoningParser()
+ out = p.parse_streaming_increment(reply)
+ got = out.normal_text + p.flush().normal_text
+ assert got == reply
+ det = MuseGlimmerDetector()
+ r = det.parse_streaming_increment(reply, _tools())
+ assert (r.normal_text + det.finish_streaming()) == reply
+
+
+def test_channel_prose_drop_warns(caplog):
+ """R2 small: discarded tool-channel prose leaves a log line."""
+ import logging
+
+ text = (
+ "<|start|>assistant to=weather.get<|message|>calling now: "
+ + _atem("weather.get", {"city": "Paris"})
+ + "<|eot|>"
+ )
+ det = MuseGlimmerDetector()
+ with caplog.at_level(logging.WARNING):
+ _, calls = _stream_detect(det, text, _tools())
+ assert _assemble(calls) == [("weather.get", {"city": "Paris"})]
+ prose_warnings = [r for r in caplog.records if "inside a tool channel" in r.message]
+ assert len(prose_warnings) == 1 # accumulated: ONE warning, not one per fragment
+
+
+# ---------------------------------------------------------------------------
+# Review regressions (PR #4, round 3)
+# ---------------------------------------------------------------------------
+def test_seek_mode_streams_eagerly_and_stays_trimmed():
+ """R3 MED-1: text after a closer (a headerless continuation, e.g. a
+ repetition loop) must stream live with a bounded buffer, not accumulate
+ until flush."""
+ p = MuseGlimmerReasoningParser()
+ p.parse_streaming_increment(" to=user<|message|>answer<|eom|>")
+ streamed = ""
+ for _ in range(500):
+ streamed += p.parse_streaming_increment("loop ").normal_text
+ assert len(p._buffer) < 256 # consumed, not held until EOS
+ assert len(streamed) > 2000 # and the client actually saw it live
+ assert "loop loop" in streamed
+
+
+def test_seek_mode_eos_around_incomplete_header():
+ """R3 MED-2: at EOS, text before a complete <|start|> whose <|message|>
+ never arrived is delivered; the discard is capped at a plausible header
+ span instead of eating the rest of the turn."""
+ # prose before the marker survives
+ p = MuseGlimmerReasoningParser()
+ content = p.parse_streaming_increment(
+ " to=user<|message|>answer<|eom|>more text <|start|>assistant"
+ ).normal_text
+ content += p.flush().normal_text
+ assert "more text " in content
+
+ # a huge tail after the marker cannot be a header: it is not discarded
+ p2 = MuseGlimmerReasoningParser()
+ tail = "x" * 400
+ c2 = p2.parse_streaming_increment(
+ " to=user<|message|>The token <|start|> opens a segment. " + tail
+ ).normal_text
+ c2 += p2.flush().normal_text
+ assert "The token " in c2 and tail in c2
+
+
+def test_truncated_channel_markup_residue_dropped_reply_kept(caplog):
+ """R3 MED-4 (as re-scoped by R4 HIGH-1): when a literal channel marker cuts an
+ invoke exactly at a value's end, the broken channel's CLOSING-MARKUP residue
+ is dropped by shape -- and the turn's real reply always survives."""
+ import logging
+
+ text = (
+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"
+ '<atem:invoke name="fs.write">\n'
+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'
+ '<atem:parameter name="content">send to=user<|message|></atem:parameter>\n'
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ "<|start|>assistant to=user<|message|>Done.<|eot|>"
+ )
+ with caplog.at_level(logging.WARNING):
+ for step in (4, len(text)):
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, text, _tools(), step=step)
+ assembled = _assemble(calls)
+ assert assembled[0][0] == "fs.write"
+ assert assembled[0][1]["path"] == "/tmp/t" # completed param survives
+ assert "</atem:" not in normal and "<atem:" not in normal # no markup leak
+ assert normal.strip() == "Done." # the real content after the channel survives
+ assert any("mid-invoke" in r.message for r in caplog.records)
+
+
+def test_truncated_channel_prose_residue_trade_pinned():
+ """R5 test-integrity: the ORIGINAL R3 wire (prose before the closing tags),
+ pinning the deliberate leak-over-loss trade. At string level the cut value's
+ tail is indistinguishable from a reply, so it LEAKS -- grandparent parity is
+ the floor -- and what the machinery guarantees instead is that the turn's
+ real reply is never lost."""
+ text = (
+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"
+ '<atem:invoke name="fs.write">\n'
+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'
+ '<atem:parameter name="content">send to=user<|message|> please</atem:parameter>\n'
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ "<|start|>assistant to=user<|message|>Done.<|eot|>"
+ )
+ expected = (
+ "please</atem:parameter>\n</atem:invoke>\n</atem:function_calls>Done."
+ )
+ for step in (4, len(text)):
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, text, _tools(), step=step)
+ assembled = _assemble(calls)
+ assert assembled[0][0] == "fs.write" and assembled[0][1]["path"] == "/tmp/t"
+ assert "Done." in normal # the reply is never lost
+ assert normal == expected, (step, normal) # the leak is pinned, not accidental
+
+
+def test_reply_right_after_midinvoke_switch_is_never_swallowed():
+ """R4 HIGH-1 companion: an inline switch fired mid-invoke followed directly by
+ a real reply (no markup residue) -- the reply must flow immediately, not be
+ held hostage for a boundary that never comes."""
+ text = (
+ " to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par'
+ "to=user<|message|>The weather tool is on it.<|eot|>"
+ )
+ for step in (1, 3, 64):
+ normal, calls = _stream_detect(MuseGlimmerDetector(), text, _tools(), step=step)
+ assert normal.strip() == "The weather tool is on it."
+ assert [c.name for c in calls if c.name] == ["weather.get"]
+
+
+def test_redundant_closer_stripped_from_seek_delivery():
+ """R3 small: a doubled closer from a degenerate decode must not leak the
+ second token into content (the detector already filters; the reasoning
+ parser's seek path now does too)."""
+ text = " to=user<|message|>The answer is 42.<|eot|><|eot|>"
+ p = MuseGlimmerReasoningParser()
+ content = ""
+ for i in range(0, len(text), 5):
+ content += p.parse_streaming_increment(text[i : i + 5]).normal_text
+ content += p.flush().normal_text
+ assert content == "The answer is 42."
+ one = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert one.normal_text == "The answer is 42."
+
+
+def test_stray_closer_one_shot_matches_streaming():
+ """R3 small: the one-shot passthrough shortcut must route closer-containing
+ text through the replay, matching the streamed result."""
+ text = "The token <|eot|> ends a turn."
+ one = MuseGlimmerReasoningParser().detect_and_parse(text)
+ p = MuseGlimmerReasoningParser()
+ streamed = ""
+ for i in range(0, len(text), 5):
+ streamed += p.parse_streaming_increment(text[i : i + 5]).normal_text
+ streamed += p.flush().normal_text
+ assert one.normal_text == streamed.strip()
+ assert "<|eot|>" not in one.normal_text
+
+
+def test_mixed_quant_groups_rejected_in_any_order():
+ """R3 small: a {nvfp4, mxfp4} mixed checkpoint must raise regardless of the
+ groups' key order (the first-group short-circuit accepted one order)."""
+ import tests.models.test_muse_glimmer as m
+
+ for order in (("group_0", "group_1"), ("group_1", "group_0")):
+ hf = m._hf_config(quantized=True)
+ groups = hf.quantization_config["config_groups"]
+ nvfp4 = groups["group_0"]
+ mxfp4 = {"weights": {"num_bits": 4, "type": "float", "group_size": 32, "strategy": "group"}}
+ hf.quantization_config["config_groups"] = {
+ order[0]: nvfp4 if order[0] == "group_0" else mxfp4,
+ order[1]: mxfp4 if order[1] == "group_1" else nvfp4,
+ }
+ from freetoken.models.muse_glimmer.config import parse_config as pc
+
+ with pytest.raises(ValueError, match="unsupported compressed-tensors"):
+ pc(hf)
+
+
+# ---------------------------------------------------------------------------
+# Review regressions (PR #4, round 4)
+# ---------------------------------------------------------------------------
+def test_pipeline_delivers_reply_after_truncated_invoke():
+ """R4 HIGH-1, PRODUCTION ORDER (reasoning parser first, its content feeds the
+ detector -- the shape standalone-detector tests missed twice): a truncated
+ invoke must never swallow the rest of the turn's reply."""
+ # reviewer wire 1: literal marker cuts the value, closing tags follow
+ wire1 = (
+ "<|start|>assistant to=fs.write<|message|><atem:function_calls>\n"
+ '<atem:invoke name="fs.write">\n'
+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'
+ '<atem:parameter name="content">send to=user<|message|></atem:parameter>\n'
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ "<|start|>assistant to=user<|message|>Done.<|eot|>"
+ )
+ # reviewer wire 2: invoke left unclosed before an abutting user header
+ wire2 = (
+ " to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">Par'
+ "<|start|>assistant to=user<|message|>The weather tool is on it.<|eot|>"
+ )
+ for wire, reply, first_call in (
+ (wire1, "Done.", "fs.write"),
+ (wire2, "The weather tool is on it.", "weather.get"),
+ ):
+ for step in (1, 3, 64):
+ _, normal, calls = _pipe(wire, _tools(), step=step)
+ assert reply in normal, (wire[:40], step, normal)
+ assert "<atem:" not in normal and "</atem:" not in normal
+ named = [c.name for c in calls if c.name]
+ assert named and named[0] == first_call
+
+
+def test_stray_start_before_a_real_header_does_not_form_a_giant_header():
+ """R4 MED-2: the span bound applies even when a <|message|> is already in the
+ buffer -- a literal <|start|> + junk + the NEXT segment's header must not be
+ parsed as one giant header (which also let a to= inside the junk hijack the
+ recipient), and streaming must agree with one-shot."""
+ junk = "P" * 300
+ text = (
+ " to=user<|message|>ok<|eom|><|start|>" + junk
+ + "<|start|>assistant to=user<|message|>more<|eot|>"
+ )
+ p = MuseGlimmerReasoningParser()
+ streamed_r, streamed_c = "", ""
+ for i in range(0, len(text), 5):
+ r = p.parse_streaming_increment(text[i : i + 5])
+ streamed_r += r.reasoning_text
+ streamed_c += r.normal_text
+ fl = p.flush()
+ streamed_r += fl.reasoning_text
+ streamed_c += fl.normal_text
+ one = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert "ok" in streamed_c and "more" in streamed_c
+ assert junk in streamed_c # delivered, not silently swallowed
+ assert one.normal_text == streamed_c.strip()
+
+ # a to=self inside the junk must not hijack the following user reply
+ hijack = (
+ " to=user<|message|>first.<|eom|><|start|>x to=self y" + "Q" * 250
+ + "<|start|>assistant to=user<|message|>the actual answer<|eot|>"
+ )
+ one = MuseGlimmerReasoningParser().detect_and_parse(hijack)
+ assert "the actual answer" in one.normal_text
+ assert "the actual answer" not in one.reasoning_text
+
+
[email protected]("name_len", [64, 100, 110])
+def test_protocol_legal_long_tool_names_survive_streaming(name_len):
+ """R4 MED-3: headers stay parseable up to the span bound even when their
+ <|message|> is still mid-arrival; the tool call AND the trailing reply
+ survive at every step size, matching one-shot."""
+ name = "t" * name_len
+ tools = [Tool(function=Function(name=name, parameters={
+ "type": "object", "properties": {"city": {"type": "string"}},
+ }))]
+ text = (
+ f" to=self<|message|>go<|eom|><|start|>assistant to={name}<|message|>"
+ + _atem(name, {"city": "Paris"})
+ + "<|eot|><|start|>assistant to=user<|message|>done<|eot|>"
+ )
+ one_r = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert "done" in one_r.normal_text
+ for step in (1, 5, 64):
+ reasoning, normal, calls = _pipe(text, tools, step=step)
+ assert reasoning.strip() == "go"
+ assert normal.strip() == "done", (name_len, step, normal)
+ named = [c.name for c in calls if c.name]
+ assert named and named[0].startswith("t" * 64)
+
+
+def test_eos_start_candidate_delivered_across_layers():
+ """R4 item-4 / R5 root-cause fix: at EOS a <|start|> that never received its
+ <|message|> is NOT a header -- BOTH layers deliver the tail and drop only
+ the marker, so the raw-bytes vs closer-stripped span mismatch between the
+ layers has nothing left to disagree about."""
+ # R5 MED-2 PoC (ordinary terminators only): the pipeline must deliver
+ # everything the parser alone delivers.
+ prose = "p" * 119
+ wire = " to=user<|message|>The wire format is: <|start|>" + prose + "<|eot|><|end_of_text|>"
+ parser_only = ""
+ p = MuseGlimmerReasoningParser()
+ for i in range(0, len(wire), 7):
+ parser_only += p.parse_streaming_increment(wire[i : i + 7]).normal_text
+ parser_only += p.flush().normal_text
+ assert prose in parser_only
+ for step in (1, 7, 4096):
+ _, piped, _ = _pipe(wire, _tools(), step=step)
+ assert "The wire format is: " in piped
+ assert prose in piped, (step, piped) # pipeline == parser, chars not swallowed
+
+ # R4 reviewer wire: the released marker's tail survives the pipeline too
+ wire2 = " to=user<|message|>answer<|eot|>prose <|start|>" + "x" * 50 + "<|eom|>" * 15
+ for step in (1, 7):
+ _, piped2, _ = _pipe(wire2, _tools(), step=step)
+ assert "answer" in piped2 and "prose " in piped2
+ assert "x" * 50 in piped2, (step, piped2)
+
+
[email protected]("junk_len", [103, 115, 127, 140])
+def test_reply_after_stray_start_junk_segment_survives(junk_len):
+ """R5 HIGH-1: a real to=user reply following a stray <|start|>+junk segment
+ must never be dropped -- previously the released marker + junk + reply sat
+ under the detector's hold threshold and died in finish_streaming."""
+ wire = (
+ " to=user<|message|>ok<|eom|><|start|>" + "J" * junk_len
+ + "<|start|>assistant to=user<|message|>Here you go.<|eot|>"
+ )
+ for step in (1, 7, 4096):
+ _, piped, _ = _pipe(wire, _tools(), step=step)
+ assert "Here you go." in piped, (junk_len, step, piped)
+ assert "ok" in piped
+ # one-shot agrees
+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)
+ one = MuseGlimmerDetector().detect_and_parse(rp.normal_text, _tools())
+ assert "Here you go." in one.normal_text
+
+
+def test_detector_giant_header_bound_pinned():
+ """R5 test-integrity: the detector's own msg-too-far bound. A stray
+ <|start|> + junk ahead of a real tool channel in ONE detector buffer must
+ not merge into a giant header. NOTE which assertion is load-bearing: with
+ the bound deleted the tool call still executes correctly -- what breaks is
+ the ~300 chars of user content silently vanishing into the giant header, so
+ it is the junk/content assertion below that pins the bound, not the calls
+ assertion."""
+ junk = "J" * 300
+ text = (
+ "ok<|eom|><|start|>" + junk
+ + _tool_channel("weather.get", {"city": "P"})
+ + "<|start|>assistant to=user<|message|>done<|eot|>"
+ )
+ for step in (7, len(text)):
+ det = MuseGlimmerDetector()
+ normal, calls = _stream_detect(det, text, _tools(), step=step)
+ assert _assemble(calls) == [("weather.get", {"city": "P"})], (step,)
+ assert "done" in normal and junk in normal
+
+
+# ---------------------------------------------------------------------------
+# Review regressions (PR #4, round 6): turn-start state is read from the
+# prompt (header-open), not guessed from the first bytes. The bare-header
+# lookalike family -- shapes the old prefix-guessing blanked to an empty turn.
+# ---------------------------------------------------------------------------
+def test_junk_recipient_past_cap_yields_empty_channel_not_empty_turn():
+ """``to=`` + 70 chars + <|message|>: the recipient (capped at 64) is a junk
+ tool name, so the channel body is not user-visible -- but the turn is NOT
+ blanked: a following real reply flows."""
+ wire = (
+ "to=" + "j" * 70 + "<|message|>hidden<|eot|>"
+ "<|start|>assistant to=user<|message|>Real reply.<|eot|>"
+ )
+ for step in (1, 7, 4096):
+ _, piped, calls = _pipe(wire, _tools(), step=step)
+ assert "Real reply." in piped, (step, piped)
+ assert "hidden" not in piped # the junk channel is what the model said
+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)
+ one = MuseGlimmerDetector().detect_and_parse(rp.normal_text, _tools())
+ assert "Real reply." in one.normal_text and "hidden" not in one.normal_text
+
+
+def test_glued_assistant_recipient_header_parses():
+ """``assistantto=x<|message|>``: the recipient regex still finds to=x, so
+ the segment routes as a (junk) tool channel and the next reply flows."""
+ wire = (
+ "assistantto=x<|message|>hello<|eot|>"
+ "<|start|>assistant to=user<|message|>Hi.<|eot|>"
+ )
+ for step in (1, 7, 4096):
+ _, piped, _ = _pipe(wire, _tools(), step=step)
+ assert "Hi." in piped, (step, piped)
+ assert "hello" not in piped
+
+
+def test_deep_leading_whitespace_header_delivers_reply():
+ """>8 leading newlines before ``to=user<|message|>``: the old bare-header
+ regex allowed at most 8 whitespace chars while the undecided hold lstripped
+ unbounded, blanking the turn; the full-header machinery has no such split."""
+ wire = "\n" * 9 + "to=user<|message|>Hello!<|eot|>"
+ for step in (1, 7, 4096):
+ _, piped, _ = _pipe(wire, _tools(), step=step)
+ assert "Hello!" in piped, (step, piped)
+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)
+ assert "Hello!" in rp.normal_text
+
+
+def test_prose_before_first_full_header_not_swallowed_into_seed():
+ """The header-open seed must not merge turn-start prose with the FIRST real
+ header into one giant header: a control token inside a header candidate
+ voids the candidate (headers never contain markers). Without the parser's
+ marker-inside rule, 'Pre. ' is swallowed into the seed's header -- here the
+ header routes to=self, so the swallowed prefix vanishes entirely (a tool
+ channel would carry it verbatim and mask the loss downstream)."""
+ wire = (
+ "Pre. <|start|>assistant to=self<|message|>think<|eom|>"
+ "<|start|>assistant to=user<|message|>Post.<|eot|>"
+ )
+ for step in (1, 7, 4096):
+ reasoning, piped, _ = _pipe(wire, _tools(), step=step)
+ assert "Pre. " in piped and "Post." in piped, (step, piped)
+ assert "think" in reasoning and "Pre" not in reasoning
+ r = MuseGlimmerReasoningParser().detect_and_parse(wire)
+ assert "Pre." in r.normal_text and "think" in r.reasoning_text
+ # and with a tool channel: the call still parses, the prose still flows
+ wire2 = (
+ "Pre. " + _tool_channel("weather.get", {"city": "P"})
+ + "<|start|>assistant to=user<|message|>Post.<|eot|>"
+ )
+ for step in (1, 7, 4096):
+ _, piped2, calls2 = _pipe(wire2, _tools(), step=step)
+ assert _assemble(calls2) == [("weather.get", {"city": "P"})], (step,)
+ assert "Pre. " in piped2 and "Post." in piped2, (step, piped2)
+
+
+def test_headerless_turn_delivers_and_seed_never_leaks():
+ """Degenerate output that never emits <|message|>: the header-open seed is
+ ruled a non-header and only the model's own bytes are delivered -- the
+ synthetic <|start|> must not leak into content at any length."""
+ for text in ("Just prose, no protocol.", "x" * 200):
+ for step in (1, 7, 4096):
+ _, piped, _ = _pipe(text, _tools(), step=step)
+ assert piped == text, (len(text), step, piped)
+ assert ATEM_START not in piped
+ rp = MuseGlimmerReasoningParser().detect_and_parse(text)
+ assert rp.normal_text == text
+
+
+_CORPUS_REPLY = "All good."
+# Historical degenerate shapes x chunkings: streaming, one-shot and the production
+# pipeline must agree on the reply and the calls (the regression class that bit
+# this PR twice was "standalone green, pipeline broken").
+_WIRE_CORPUS = [
+ # clean turn
+ " to=self<|message|>think<|eom|><|start|>assistant to=user<|message|>All good.<|eot|>",
+ # tool channel then reply
+ " to=self<|message|>t<|eom|>" + _tool_channel("weather.get", {"city": "P"}, closer="<|eom|>")
+ + "<|start|>assistant to=user<|message|>All good.<|eot|>",
+ # headerless switches everywhere
+ " to=self<|message|>t to=weather.get<|message|>"
+ + _atem("weather.get", {"city": "P"}) + "to=user<|message|>All good.<|eot|>",
+ # truncated invoke then abutting user header
+ " to=weather.get<|message|><atem:function_calls>\n"
+ '<atem:invoke name="weather.get">\n<atem:parameter name="city">P'
+ "<|start|>assistant to=user<|message|>All good.<|eot|>",
+ # stray literal closer inside the reply (redundant <|eom|> mid-sentence)
+ " to=user<|message|>All <|eom|>good.<|eot|>",
+ # plain clean reply
+ " to=user<|message|>All good.<|eot|>",
+ # bare first segment is the reply
+ " to=user<|message|>All good.<|eom|>",
+ # stray <|start|>+junk segment ahead of the real reply (R5 HIGH-1 shape)
+ " to=user<|message|>prelude<|eom|><|start|>" + "J" * 115
+ + "<|start|>assistant to=user<|message|>All good.<|eot|>",
+ # stream dies inside an unfinished header candidate (R5 EOS-family shape)
+ " to=user<|message|>All good.<|eot|><|start|>" + "p" * 119 + "<|end_of_text|>",
+ # truncated invoke leaving real closing tags before the reply (exercises
+ # the </atem: leak clause below)
+ " to=fs.write<|message|><atem:function_calls>\n"
+ '<atem:invoke name="fs.write">\n'
+ '<atem:parameter name="path">/tmp/t</atem:parameter>\n'
+ '<atem:parameter name="content">send to=user<|message|></atem:parameter>\n'
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ "<|start|>assistant to=user<|message|>All good.<|eot|>",
+]
+
+
[email protected]("wire_idx", range(len(_WIRE_CORPUS)))
[email protected]("step", [1, 7, 4096])
+def test_wire_corpus_pipeline_consistency(wire_idx, step):
+ wire = _WIRE_CORPUS[wire_idx]
+ reasoning, normal, calls = _pipe(wire, _tools(), step=step)
+ assert _CORPUS_REPLY in normal, (wire_idx, step, normal)
+ assert "<atem:" not in normal and "</atem:" not in normal
+ assert "<|" not in normal.replace("<|start|>", "")
+ # one-shot detector agrees on calls AND arguments for the same classified content
+ rp = MuseGlimmerReasoningParser().detect_and_parse(wire)
+ one = MuseGlimmerDetector().detect_and_parse(rp.normal_text, _tools())
+ assert [(c.name, json.loads(c.parameters)) for c in one.calls] == _assemble(calls)
+ # the one-shot DETECTOR's content must carry the reply itself (the parser's
+ # content always does, so an or-check here would never fail)
+ assert _CORPUS_REPLY in one.normal_text, (wire_idx, one.normal_text)
+
+
+# ---------------------------------------------------------------------------
+# Wiring: auto-selection and thinking gears
+# ---------------------------------------------------------------------------
+def test_auto_selection_picks_muse_glimmer():
+ from unittest.mock import patch
+
+ from freetoken.server.args import parse_args
+
+ class _Config:
+ def to_dict(self):
+ return {
+ "architectures": ["MuseGlimmerForConditionalGeneration"],
+ "model_type": "muse_glimmer",
+ "text_config": {"model_type": "muse_glimmer_text"},
+ }
+
+ with patch("freetoken.utils.cached_load_hf_config", lambda _p: _Config()):
+ args, _ = parse_args(["--model", "/models/anon"])
+ assert args.tool_call_parser == "muse_glimmer"
+ assert args.reasoning_parser == "muse_glimmer"
+
+
+class _MuseLikeTokenizer:
+ """The muse template's effort surface: grades ``reasoning_strength`` (default
+ high), validates nothing, ignores the thinking-toggle spellings."""
+
+ chat_template = "muse" # non-empty: the dsv4 encoder probe must not engage
+ name_or_path = "muse-like"
+
+ def apply_chat_template(self, messages, tokenize=False, add_generation_prompt=True, **kwargs):
+ strength = kwargs.get("reasoning_strength") or "high"
+ return f"<|begin_of_text|>system: Reasoning strength: {strength}. user: ping"
+
+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):
+ import torch
+
+ assert add_special_tokens is False # the template owns the bos
+ return torch.tensor([[1, 2, 3]], dtype=torch.long)
+
+
+def test_effort_broadcast_reaches_reasoning_strength():
+ """The render layer broadcasts reasoning_effort in every spelling the
+ ecosystem's templates read (the thinking-toggle rule); muse's template picks
+ up ``reasoning_strength``. An explicit caller spelling wins."""
+ from freetoken.tokenizer.tokenize import TokenizeManager
+
+ manager = TokenizeManager(_MuseLikeTokenizer())
+ prompt = manager._render([{"role": "user", "content": "hi"}], None, {"reasoning_effort": "low"})
+ assert "Reasoning strength: low." in prompt
+ prompt = manager._render(
+ [{"role": "user", "content": "hi"}],
+ None,
+ {"reasoning_effort": "low", "reasoning_strength": "xhigh"},
+ )
+ assert "Reasoning strength: xhigh." in prompt
+
+
+def test_probed_muse_gears_and_effort_vocabulary():
+ """The checkpoint-probing pipeline derives muse's gears through the broadcast:
+ the template grades effort without validating it, so it is served the graded
+ ladder (the model card's trained levels, xhigh included) with the template's
+ own default (high); never-advertised dialects quantize onto the ladder
+ instead of reaching the model verbatim."""
+ from freetoken.server.model_meta import derive_think_gears
+ from freetoken.tokenizer.effort import effective_efforts, quantize_effort
+ from freetoken.tokenizer.tokenize import TokenizeManager
+
+ manager = TokenizeManager(_MuseLikeTokenizer())
+ profile = manager.thinking_profile()
+ assert profile.efforts.consumes_effort and not profile.efforts.validates
+ assert profile.efforts.default == "high"
+ assert not profile.toggleable # the template reads no on/off spelling
+
+ gears, default, kwargs = derive_think_gears(profile, parser_configured=True)
+ assert gears == ("low", "medium", "high", "xhigh") and default == "high"
+ assert kwargs["low"] == {"reasoning_effort": "low"}
+ assert effective_efforts(profile.efforts) == frozenset({"low", "medium", "high", "xhigh"})
+ # the native vocabulary passes through untouched...
+ assert quantize_effort("xhigh", profile.efforts) == "xhigh"
+ # ...while off-ladder dialects quantize instead of interpolating verbatim
+ assert quantize_effort("minimal", profile.efforts) == "low"
+ assert quantize_effort("max", profile.efforts) == "xhigh"
+ assert quantize_effort("none", profile.efforts) is None # template default applies
diff --git a/tests/server/test_streaming_model_matrix.py b/tests/server/test_streaming_model_matrix.py
index a0c94a5d..43922021 100644
--- a/tests/server/test_streaming_model_matrix.py
+++ b/tests/server/test_streaming_model_matrix.py
@@ -93,6 +93,12 @@ CALL_BLOCKS = {
"<|start|>assistant<|channel|>commentary to=functions.read "
'<|constrain|>json<|message|>{"filePath": "/tmp/test_calc.py"}<|end|>'
),
+ "muse_glimmer": (
+ "<|start|>assistant to=read<|message|><atem:function_calls>\n"
+ '<atem:invoke name="read">\n'
+ "<atem:parameter name=\"filePath\">/tmp/test_calc.py</atem:parameter>\n"
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ ),
}
# Substrings that must never leak into user-visible content.
@@ -107,6 +113,7 @@ MARKUP_MARKERS = {
"mistral": ["[TOOL_CALLS]"],
"llama3": ["<|python_tag|>"],
"gpt_oss": ["<|channel|>", "to=functions."],
+ "muse_glimmer": ["<atem:", "<|message|>", "<|eot|>"],
}
# (tool_call_parser, reasoning_parser, think_open, think_close) per reasoning-capable
@@ -123,6 +130,7 @@ REASONING_FAMILIES = {
"minimax-m3": ("minimax_m3", "minimax_m3", "<mm:think>", "</mm:think>"),
"gemma4": ("gemma4", "gemma4", "<|channel>thought\n", "<channel|>"),
"gpt-oss": ("gpt_oss", "gpt_oss", None, None), # harmony channels; custom fixture
+ "muse-glimmer": ("muse_glimmer", "muse_glimmer", None, None), # ATEM channels; custom fixture
}
@@ -274,6 +282,15 @@ def _reasoning_fixture(name):
f"<|message|>{json.dumps(READ_ARGS)}<|call|>"
)
return tool, reasoning, text
+ if name == "muse-glimmer":
+ # Generation resumes after the template's <|start|>assistant, so the first
+ # segment is a bare header continuation.
+ text = (
+ f" to=self<|message|>{THINKING}<|eom|>"
+ f"<|start|>assistant to=user<|message|>{ANSWER}<|eom|>"
+ + CALL_BLOCKS[tool]
+ )
+ return tool, reasoning, text
text = f"{think_open}{THINKING}{think_close}{ANSWER} {CALL_BLOCKS[tool]}"
return tool, reasoning, text
@@ -420,6 +437,12 @@ ARGS_STREAMING_BLOCKS = {
"<|start|>assistant<|channel|>commentary to=functions.read <|constrain|>json"
f"<|message|>{json.dumps({'filePath': LONG_VALUE})}<|call|>"
),
+ "muse_glimmer": (
+ "<|start|>assistant to=read<|message|><atem:function_calls>\n"
+ '<atem:invoke name="read">\n'
+ f"<atem:parameter name=\"filePath\">{LONG_VALUE}</atem:parameter>\n"
+ "</atem:invoke>\n</atem:function_calls><|eot|>"
+ ),
}
@@ -642,7 +665,7 @@ def test_empty_arguments_call_emitted_exactly_once(tool, block):
@pytest.mark.parametrize(
"family",
["qwen25", "qwen3_coder", "glm47", "gemma4", "minimax", "minimax_m3",
- "deepseekv32", "gpt_oss"],
+ "deepseekv32", "gpt_oss", "muse_glimmer"],
)
def test_call_then_trailing_text_in_one_chunk_keeps_order(family):
# Worst case: the whole generation arrives as ONE chunk. Text after the call
diff --git a/tests/tokenizer/test_tokenize.py b/tests/tokenizer/test_tokenize.py
index 4b64e04c..65fa5f2d 100644
--- a/tests/tokenizer/test_tokenize.py
+++ b/tests/tokenizer/test_tokenize.py
@@ -18,9 +18,12 @@ class FakeTokenizer:
self.chat_template_kwargs = kwargs
return "rendered prompt"
- def encode(self, prompt, return_tensors=None):
+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):
assert prompt == "rendered prompt"
assert return_tensors == "pt"
+ # The template rendered every special token already; encode must not
+ # add another bos on top (the muse-glimmer/llama double-bos bug).
+ assert add_special_tokens is False
return torch.tensor([[1, 2, 3]], dtype=torch.long)
@@ -78,9 +81,11 @@ class FakeDsv4Tokenizer:
self.name_or_path = str(model_path)
self.prompt = None
- def encode(self, prompt, return_tensors=None):
+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):
self.prompt = prompt
assert return_tensors == "pt"
+ # dsv4's own encoder path keeps the default special-token behavior.
+ assert add_special_tokens is True
return torch.tensor([[4, 5, 6]], dtype=torch.long)
@@ -197,7 +202,7 @@ class Qwen38LikeTokenizer:
return f"prompt effort={effort}"
return "prompt effort=off"
- def encode(self, prompt, return_tensors=None):
+ def encode(self, prompt, return_tensors=None, add_special_tokens=True):
return torch.tensor([[7, 8]], dtype=torch.long)