Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(flint): add nvfp4 modelopt and gguf support (#797)

Jesús Jódar Piernas2d930855450d
diff --git a/docs/models.md b/docs/models.md
index 914e813b..9807df80 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -20,6 +20,7 @@ for them; other checkpoints of the same architectures work too.
| MiniMax-M2.5 | [nvidia/MiniMax-M2.5-NVFP4](https://huggingface.co/nvidia/MiniMax-M2.5-NVFP4) |
| MiniMax-M3 | [nvidia/MiniMax-M3-NVFP4](https://huggingface.co/nvidia/MiniMax-M3-NVFP4) |
| Muse-Glimmer | [meta-models/Muse-Glimmer-30B](https://huggingface.co/meta-models/Muse-Glimmer-30B), [RedHatAI/Muse-Glimmer-30B-NVFP4](https://huggingface.co/RedHatAI/Muse-Glimmer-30B-NVFP4) |
+| Flint | [FutureMLS-Lab/flint-nvfp4-coldgptq](https://huggingface.co/FutureMLS-Lab/flint-nvfp4-coldgptq), [RecursiveSI/flint-nvfp4-gguf](https://huggingface.co/RecursiveSI/flint-nvfp4-gguf) |
### Image input
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index bf7e75b6..bc7b4370 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -347,6 +347,17 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
moe_intermediate_size=2304,
expert_formats=("ds_fp4",),
),
+ AotModel(
+ name="RecursiveSI/flint",
+ architecture="FlintForCausalLM",
+ hidden_size=2048,
+ kv_groups=((4, 128),),
+ top_k=6,
+ moe_intermediate_size=512,
+ expert_formats=("bf16", *_NVFP4_FORMATS),
+ aliases=("FutureMLS-Lab/flint-nvfp4-coldgptq",),
+ arch_aliases=("FlintGGUFForCausalLM",),
+ ),
# ---- dense checkpoints (store/index only, no expert banks) ----
AotModel(
name="Qwen/Qwen3.6-27B",
diff --git a/python/freetoken/models/flint/__init__.py b/python/freetoken/models/flint/__init__.py
new file mode 100644
index 00000000..7fa05030
--- /dev/null
+++ b/python/freetoken/models/flint/__init__.py
@@ -0,0 +1,9 @@
+from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
+from .model import FlintForCausalLM
+from .weight import iter_weights, nvfp4_expert_spec
+
+__all__ = [
+ "FlintForCausalLM", "parse_config", "iter_weights", "nvfp4_expert_spec",
+ "iter_gguf_weights", "parse_gguf_config", "to_gguf",
+]
diff --git a/python/freetoken/models/flint/attention.py b/python/freetoken/models/flint/attention.py
new file mode 100644
index 00000000..6a720cc7
--- /dev/null
+++ b/python/freetoken/models/flint/attention.py
@@ -0,0 +1,57 @@
+from __future__ import annotations
+
+import torch
+
+from freetoken.attention import AttentionSpec
+from freetoken.core import get_global_ctx
+from freetoken.distributed import get_tp_info
+from freetoken.layers import BaseOP, LinearOProj, LinearQKVMerged, RMSNorm
+from freetoken.layers.rotary import get_rope
+from freetoken.models.config import ModelConfig, SWAAttentionGroupConfig
+from freetoken.utils import div_even, nvtx_annotate
+
+
+class FlintAttention(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.layer_id = layer_id
+ group = config.attention_group_for_layer(layer_id)
+ self.num_qo_heads = div_even(config.num_qo_heads, get_tp_info().size)
+ self.num_kv_heads = div_even(config.num_kv_heads, get_tp_info().size, allow_replicate=True)
+ self.head_dim = config.head_dim
+ self.q_dim = self.num_qo_heads * self.head_dim
+ self.kv_dim = self.num_kv_heads * self.head_dim
+ self.qkv_proj = LinearQKVMerged(
+ config.hidden_size, self.head_dim, config.num_qo_heads, config.num_kv_heads,
+ has_bias=False, quant_config=config.quant, prefix=f"{prefix}.qkv_proj",
+ )
+ self.o_proj = LinearOProj(
+ config.num_qo_heads * self.head_dim, config.hidden_size,
+ has_bias=False, quant_config=config.quant, prefix=f"{prefix}.o_proj",
+ )
+ self.q_norm = RMSNorm(self.head_dim, config.rms_norm_eps)
+ self.k_norm = RMSNorm(self.head_dim, config.rms_norm_eps)
+ self.softmax_offset = torch.empty(self.num_qo_heads, dtype=torch.float32)
+ self.sliding_window = group.sliding_window if isinstance(group, SWAAttentionGroupConfig) else None
+ rope = group.rotary_config
+ self.rotary = get_rope(
+ head_dim=self.head_dim, rotary_dim=rope.rotary_dim,
+ max_position=rope.max_position, base=rope.base, rope_scaling=None,
+ ) if rope.rotary_dim else None
+
+ @nvtx_annotate("MHA")
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ ctx = get_global_ctx()
+ qkv = self.qkv_proj.forward(x)
+ q, k, v = qkv.split((self.q_dim, self.kv_dim, self.kv_dim), dim=-1)
+ # Flatten heads for FlashInfer's 2D CUDA norm; normalizing the concatenated
+ # heads would silently change both the attention and the MoE routes.
+ q = self.q_norm.forward(q.reshape(-1, self.head_dim)).reshape(-1, self.q_dim)
+ k = self.k_norm.forward(k.reshape(-1, self.head_dim)).reshape(-1, self.kv_dim)
+ if self.rotary is not None:
+ q, k = self.rotary.forward(ctx.batch.get_attn_positions(), q, k)
+ out = ctx.attn_backend.forward(
+ q.view(-1, self.num_qo_heads, self.head_dim), k.contiguous(), v.contiguous(),
+ self.layer_id, ctx.batch,
+ attn_spec=AttentionSpec(sliding_window=self.sliding_window, sinks=self.softmax_offset),
+ )
+ return self.o_proj.forward(out.reshape(-1, self.q_dim))
diff --git a/python/freetoken/models/flint/config.py b/python/freetoken/models/flint/config.py
new file mode 100644
index 00000000..916e8829
--- /dev/null
+++ b/python/freetoken/models/flint/config.py
@@ -0,0 +1,66 @@
+from __future__ import annotations
+
+from dataclasses import replace
+from typing import Any
+
+from freetoken.models.config import (
+ FullAttentionGroupConfig,
+ ModelConfig,
+ RotaryConfig,
+ SWAAttentionGroupConfig,
+ detect_expert_quant,
+)
+
+
+def parse_config(hf: Any) -> ModelConfig:
+ layers = int(hf.num_hidden_layers)
+ types = tuple(hf.layer_types)
+ if len(types) != layers or set(types) - {"full_attention", "sliding_attention"}:
+ raise ValueError("Flint requires one full_attention/sliding_attention type per layer")
+ full = tuple(i for i, kind in enumerate(types) if kind == "full_attention")
+ swa = tuple(i for i, kind in enumerate(types) if kind == "sliding_attention")
+ if set(hf.no_rope_layer_indices) != set(full):
+ raise ValueError("Flint supports RoPE on sliding layers and NoPE on full layers only")
+ dense = tuple(hf.mlp_only_layers)
+ if dense != tuple(range(len(dense))) or len(dense) >= layers:
+ raise ValueError("Flint dense MLP layers must form a leading contiguous prefix")
+ if int(getattr(hf, "decoder_sparse_step", 1)) != 1:
+ raise ValueError("Flint requires an MoE block in every layer after the dense prefix")
+ if getattr(hf, "rope_scaling", None):
+ raise NotImplementedError("Flint RoPE scaling is not supported")
+ if hf.hidden_act != "silu" or hf.router_score_function != "sigmoid":
+ raise ValueError("Flint requires SwiGLU experts and sigmoid routing")
+ if hf.softmax_type != "learnable" or bool(getattr(hf, "attention_bias", False)):
+ raise ValueError("Flint requires learned attention sinks and bias-free projections")
+ window = int(hf.sliding_window)
+ if window <= 0 or not full or not swa:
+ raise ValueError("Flint requires full and sliding attention with a positive window")
+ rope = RotaryConfig(
+ head_dim=int(hf.head_dim), rotary_dim=int(hf.head_dim),
+ max_position=int(hf.max_position_embeddings), base=float(hf.rope_theta), scaling=None,
+ )
+ return ModelConfig(
+ num_layers=layers, num_qo_heads=int(hf.num_attention_heads),
+ num_kv_heads=int(hf.num_key_value_heads), head_dim=int(hf.head_dim),
+ hidden_size=int(hf.hidden_size), vocab_size=int(hf.vocab_size),
+ intermediate_size=int(hf.intermediate_size), hidden_act="silu",
+ rms_norm_eps=float(hf.rms_norm_eps),
+ tie_word_embeddings=bool(hf.tie_word_embeddings), rotary_config=rope,
+ num_experts=int(hf.num_experts), num_experts_per_tok=int(hf.num_experts_per_tok),
+ moe_intermediate_size=int(hf.moe_intermediate_size),
+ norm_topk_prob=bool(hf.norm_topk_prob),
+ model_type="flint", architectures=list(hf.architectures), moe_enabled=True,
+ expert_quant=detect_expert_quant(hf), first_k_dense_replace=len(dense),
+ shared_expert_intermediate_size=int(hf.shared_expert_intermediate_size),
+ use_qk_norm=True,
+ attention_groups=(
+ SWAAttentionGroupConfig(
+ name="swa", layer_ids=swa, num_kv_heads=int(hf.num_key_value_heads),
+ head_dim=int(hf.head_dim), rotary_config=rope, sliding_window=window,
+ ),
+ FullAttentionGroupConfig(
+ name="full", layer_ids=full, num_kv_heads=int(hf.num_key_value_heads),
+ head_dim=int(hf.head_dim), rotary_config=replace(rope, rotary_dim=0),
+ ),
+ ),
+ )
diff --git a/python/freetoken/models/flint/gguf.py b/python/freetoken/models/flint/gguf.py
new file mode 100644
index 00000000..3cc38a0e
--- /dev/null
+++ b/python/freetoken/models/flint/gguf.py
@@ -0,0 +1,118 @@
+from __future__ import annotations
+
+import re
+from dataclasses import replace
+from types import SimpleNamespace
+from typing import Iterator
+
+import torch
+
+from freetoken.models.gguf.reader import gguf_tensor_index
+from freetoken.models.gguf.config import GgufConfigShim
+from freetoken.models.config import ModelConfig
+from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix
+from freetoken.utils import cached_load_hf_config
+
+from .config import parse_config
+
+_LAYER = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+_LEAVES = {
+ "self_attn.q_proj": "attn_q", "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v", "self_attn.o_proj": "attn_output",
+ "mlp.gate": "ffn_gate_inp",
+ **{f"mlp.{p}_proj": f"ffn_{p}" for p in ("gate", "up", "down")},
+ **{f"mlp.shared_expert.{p}_proj": f"ffn_{p}_shexp" for p in ("gate", "up", "down")},
+ **{f"mlp.experts.0.{p}_proj": f"ffn_{p}_exps" for p in ("gate", "up", "down")},
+}
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ top = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+ if name in top:
+ return (top[name],)
+ match = _LAYER.match(name)
+ if match is None:
+ return ()
+ leaf = _LEAVES.get(match[2])
+ return (f"blk.{match[1]}.{leaf}",) if leaf else ()
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ md = shim.metadata
+ def kv(name):
+ return md[f"flint.{name}"]
+ if md.get("flint.rope.scaling.type", "none") != "none":
+ raise NotImplementedError("Flint GGUF RoPE scaling is not supported")
+ if int(kv("expert_gating_func")) != 2 or float(kv("expert_weights_scale")) != 1.0:
+ raise NotImplementedError("Flint GGUF requires unscaled sigmoid routing")
+ check_scales(shim.model_path)
+ swa = tuple(kv("attention.sliding_window_pattern"))
+ rope = tuple(kv("attention.rope_pattern"))
+ if len(rope) != int(kv("block_count")):
+ raise ValueError("Flint GGUF requires one RoPE flag per layer")
+ return SimpleNamespace(
+ architectures=list(shim.architectures), model_type="flint",
+ num_hidden_layers=int(kv("block_count")), hidden_size=int(kv("embedding_length")),
+ intermediate_size=int(kv("feed_forward_length")),
+ num_attention_heads=int(kv("attention.head_count")),
+ num_key_value_heads=int(kv("attention.head_count_kv")),
+ head_dim=int(kv("attention.key_length")), vocab_size=shim.vocab_size,
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ rope_theta=float(kv("rope.freq_base")),
+ layer_types=["sliding_attention" if v else "full_attention" for v in swa],
+ no_rope_layer_indices=[i for i, v in enumerate(rope) if not v],
+ mlp_only_layers=list(range(int(kv("leading_dense_block_count")))),
+ num_experts=int(kv("expert_count")), num_experts_per_tok=int(kv("expert_used_count")),
+ moe_intermediate_size=int(kv("expert_feed_forward_length")),
+ shared_expert_intermediate_size=int(kv("expert_shared_feed_forward_length")),
+ norm_topk_prob=bool(kv("expert_weights_norm")),
+ sliding_window=int(kv("attention.sliding_window")),
+ hidden_act="silu", router_score_function="sigmoid", softmax_type="learnable",
+ tie_word_embeddings=shim.tie_word_embeddings,
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ return replace(parse_config(_hf_config(shim)), expert_quant="gguf")
+
+
+def iter_gguf_weights(
+ model_path: str, device: torch.device, *, include_moe_experts: bool, include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ if not include_non_moe:
+ return
+ hf = _hf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+ def t(name):
+ return index[name if name in index else f"{name}.weight"]
+ def linear(module, names):
+ infos = [t(n) for n in names]
+ yield from fused(module, infos, [matrix(v) for v in infos], int(infos[0].ne[0]))
+ yield from linear("model.embed_tokens", ["token_embd"])
+ if not hf.tie_word_embeddings:
+ yield from linear("lm_head", ["output"])
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+ for layer in range(hf.num_hidden_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))
+ sa = f"{pre}.self_attn"
+ yield from linear(f"{sa}.qkv_proj", [f"{blk}.attn_{p}" for p in ("q", "k", "v")])
+ yield from linear(f"{sa}.o_proj", [f"{blk}.attn_output"])
+ for p in ("q", "k"):
+ yield f"{sa}.{p}_norm.weight", float_tensor(t(f"{blk}.attn_{p}_norm"))
+ yield f"{sa}.softmax_offset", float_tensor(t(f"{blk}.attn_sinks")).float()
+ mlp = f"{pre}.mlp"
+ if layer in hf.mlp_only_layers:
+ yield from linear(f"{mlp}.gate_up_proj", [f"{blk}.ffn_{p}" for p in ("gate", "up")])
+ yield from linear(f"{mlp}.down_proj", [f"{blk}.ffn_down"])
+ else:
+ yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
+ bias = index.get(f"{blk}.exp_probs_b.bias")
+ yield f"{mlp}.gate.expert_bias", (
+ float_tensor(bias).float() if bias is not None
+ else torch.zeros(hf.num_experts, device=device, dtype=torch.float32)
+ )
+ yield from linear(f"{mlp}.shared_expert.gate_up_proj", [f"{blk}.ffn_{p}_shexp" for p in ("gate", "up")])
+ yield from linear(f"{mlp}.shared_expert.down_proj", [f"{blk}.ffn_down_shexp"])
diff --git a/python/freetoken/models/flint/model.py b/python/freetoken/models/flint/model.py
new file mode 100644
index 00000000..3f0b4208
--- /dev/null
+++ b/python/freetoken/models/flint/model.py
@@ -0,0 +1,65 @@
+from __future__ import annotations
+
+import torch
+
+from freetoken.core import get_global_ctx
+from freetoken.layers import BaseOP, OPList, ParallelLMHead, RMSNormFused, VocabParallelEmbedding
+from freetoken.models.blocks import BaseLLMModel, GatedMLP
+from freetoken.models.config import ModelConfig
+from freetoken.utils import nvtx_annotate
+
+from .attention import FlintAttention
+from .moe import FlintMoE
+
+
+class FlintDecoderLayer(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self._layer_id = layer_id
+ self.self_attn = FlintAttention(config, layer_id, prefix=f"{prefix}.self_attn")
+ self.mlp = (
+ GatedMLP(config, quant_config=config.quant, prefix=f"{prefix}.mlp")
+ if layer_id < config.first_k_dense_replace
+ else FlintMoE(config, layer_id, prefix=f"{prefix}.mlp")
+ )
+ self.input_layernorm = RMSNormFused(config.hidden_size, config.rms_norm_eps)
+ self.post_attention_layernorm = RMSNormFused(config.hidden_size, config.rms_norm_eps)
+
+ @nvtx_annotate("Layer_{}", layer_id_field="_layer_id")
+ def forward(self, x: torch.Tensor, residual: torch.Tensor | None) -> tuple[torch.Tensor, torch.Tensor]:
+ x, residual = self.input_layernorm.forward(x, residual)
+ x = self.self_attn.forward(x)
+ x, residual = self.post_attention_layernorm.forward(x, residual)
+ return self.mlp.forward(x), residual
+
+
+class FlintModel(BaseOP):
+ def __init__(self, config: ModelConfig, *, prefix: str = "model"):
+ self.embed_tokens = VocabParallelEmbedding(
+ config.vocab_size, config.hidden_size, quant_config=config.quant, prefix=f"{prefix}.embed_tokens",
+ )
+ self.layers = OPList([
+ FlintDecoderLayer(config, i, prefix=f"{prefix}.layers.{i}") for i in range(config.num_layers)
+ ])
+ self.norm = RMSNormFused(config.hidden_size, config.rms_norm_eps)
+
+ def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
+ x, residual = self.embed_tokens.forward(input_ids), None
+ for layer in self.layers.op_list:
+ x, residual = layer.forward(x, residual)
+ return self.norm.forward(x, residual)[0]
+
+
+class FlintForCausalLM(BaseLLMModel):
+ def __init__(self, config: ModelConfig):
+ self.model = FlintModel(config)
+ self.lm_head = ParallelLMHead(
+ config.vocab_size, config.hidden_size,
+ tie_word_embeddings=config.tie_word_embeddings,
+ tied_embedding=self.model.embed_tokens if config.tie_word_embeddings else None,
+ quant_config=config.quant, prefix="lm_head",
+ )
+ super().__init__()
+
+ def forward(self) -> torch.Tensor:
+ output = self.model.forward(get_global_ctx().batch.input_ids)
+ return self.lm_head.forward(output)
diff --git a/python/freetoken/models/flint/moe.py b/python/freetoken/models/flint/moe.py
new file mode 100644
index 00000000..4f7bf7bf
--- /dev/null
+++ b/python/freetoken/models/flint/moe.py
@@ -0,0 +1,37 @@
+from __future__ import annotations
+
+from dataclasses import replace
+
+import torch
+
+from freetoken.kernel.triton.moe_router import fused_topk_sigmoid
+from freetoken.layers import BaseOP, LinearReplicated, make_moe_layer
+from freetoken.models.blocks import GatedMLP
+from freetoken.models.config import ModelConfig
+
+
+class FlintMoE(BaseOP):
+ def __init__(self, config: ModelConfig, layer_id: int, *, prefix: str = ""):
+ self.top_k = config.num_experts_per_tok
+ self.renormalize = config.norm_topk_prob
+ self.gate = LinearReplicated(config.hidden_size, config.num_experts, has_bias=False)
+ self.gate.expert_bias = torch.zeros(config.num_experts, dtype=torch.float32)
+ self.experts = make_moe_layer(
+ config, layer_id=layer_id - config.first_k_dense_replace,
+ quant_config=config.quant, prefix=f"{prefix}.experts",
+ )
+ shared_config = replace(config, intermediate_size=config.shared_expert_intermediate_size)
+ self.shared_expert = GatedMLP(shared_config, quant_config=config.quant, prefix=f"{prefix}.shared_expert")
+
+ def _route(self, x: torch.Tensor) -> tuple[torch.Tensor, torch.Tensor]:
+ # The reference projects in model dtype, then computes sigmoid in fp32.
+ return fused_topk_sigmoid(
+ self.gate.forward(x).float(), self.gate.expert_bias,
+ self.top_k, self.renormalize,
+ )
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ weights, ids = self._route(x)
+ # Some routed kernels overwrite their input; the shared branch needs it first.
+ shared = self.shared_expert.forward(x)
+ return self.experts.routed_forward(x, weights, ids) + shared
diff --git a/python/freetoken/models/flint/weight.py b/python/freetoken/models/flint/weight.py
new file mode 100644
index 00000000..d867274e
--- /dev/null
+++ b/python/freetoken/models/flint/weight.py
@@ -0,0 +1,77 @@
+from __future__ import annotations
+
+import re
+from typing import Iterator
+
+import safetensors
+import torch
+
+from freetoken.distributed import get_tp_info
+from freetoken.models.loader import (
+ MergeRule, drop_page_cache, iter_merged_tensors, iter_stacked_experts,
+ iter_weight_files, safetensors_weight_map, shard_tensor,
+)
+from freetoken.models.nvfp4_banks import Nvfp4ExpertSourceSpec
+from freetoken.utils import cached_load_hf_config, download_hf_weight
+
+from .config import parse_config
+
+_EXPERT = re.compile(r"^(?P<prefix>model\.layers\.\d+\.mlp\.experts)\.(?P<idx>\d+)\.(?P<name>.+)$")
+_MERGE = {
+ ".q_proj": MergeRule(".qkv_proj", "q", ("q", "k", "v")),
+ ".k_proj": MergeRule(".qkv_proj", "k", ("q", "k", "v")),
+ ".v_proj": MergeRule(".qkv_proj", "v", ("q", "k", "v")),
+ ".gate_proj": MergeRule(".gate_up_proj", "gate", ("gate", "up")),
+ ".up_proj": MergeRule(".gate_up_proj", "up", ("gate", "up")),
+}
+_NVFP4 = Nvfp4ExpertSourceSpec(
+ key_pattern=re.compile(
+ r"^model\.layers\.(?P<layer>\d+)\.mlp\.experts\.(?P<expert>\d+)\."
+ r"(?P<proj>gate_proj|up_proj|down_proj)\.(?P<kind>weight|weight_scale|weight_scale_2)$"
+ ),
+ proj_to_role={"gate_proj": "gate", "up_proj": "up", "down_proj": "down"},
+ layer_to_bank=lambda layer, config: (
+ layer - config.first_k_dense_replace if config.first_k_dense_replace <= layer < config.num_layers else None
+ ),
+ desc="Flint NVFP4 experts",
+)
+
+
+def nvfp4_expert_spec(model_path: str, config) -> Nvfp4ExpertSourceSpec:
+ return _NVFP4
+
+
+def iter_weights(
+ model_path: str, device: torch.device, *, include_moe_experts: bool, include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ model_path = download_hf_weight(model_path)
+ config = parse_config(cached_load_hf_config(model_path))
+ if include_moe_experts and config.expert_quant != "none":
+ raise ValueError("Flint NVFP4 uses the offload, cpu or hybrid MoE strategy")
+ tp = get_tp_info()
+ names = safetensors_weight_map(model_path)
+
+ def tensors():
+ for file in iter_weight_files(model_path):
+ with safetensors.safe_open(file, framework="pt", device=str(device)) as reader:
+ for name in reader.keys():
+ expert = _EXPERT.match(name) is not None
+ if (expert and not include_moe_experts) or (not expert and not include_non_moe):
+ continue
+ raw = reader.get_tensor(name)
+ if name.endswith(".softmax_offset"):
+ raw = raw.chunk(tp.size)[tp.rank].float()
+ elif name.endswith(".expert_bias"):
+ raw = raw.float()
+ else:
+ raw = shard_tensor(name, raw, rank=tp.rank, world_size=tp.size, num_kv_heads=config.num_kv_heads)
+ yield name, raw
+ drop_page_cache(file)
+ if include_non_moe:
+ for layer in range(config.first_k_dense_replace, config.num_layers):
+ name = f"model.layers.{layer}.mlp.gate.expert_bias"
+ if name not in names:
+ yield name, torch.zeros(config.num_experts, dtype=torch.float32, device=device)
+
+ merged = iter_merged_tensors(tensors(), _MERGE, model_name="flint")
+ yield from iter_stacked_experts(merged, num_experts=config.num_experts, model_name="flint", expert_pattern=_EXPERT)
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 61198ead..e718809f 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -20,6 +20,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
# GGUF ``general.architecture`` -> FreeToken registry key (a GGUF-specific spec that
# reuses the model classes but a GGUF parse_config / iter_weights).
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
+ "flint": "FlintGGUFForCausalLM",
"deepseek4": "DeepseekV4GGUFForCausalLM",
"gemma4": "Gemma4GGUFForCausalLM",
"glm4moe": "Glm4MoeGGUFForCausalLM",
diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py
index a9c77d79..c78428dc 100644
--- a/python/freetoken/models/gguf/tokenizer.py
+++ b/python/freetoken/models/gguf/tokenizer.py
@@ -10,7 +10,8 @@ import json
from .reader import load_gguf_metadata
-# end-of-generation token names from llama.cpp (llama_vocab::impl::load in src/llama-vocab.cpp): a GGUF stores one eos id and no stop list
+# End-of-generation names for GGUFs without an explicit eog_token_ids array.
+# See llama.cpp llama_vocab::impl::load in src/llama-vocab.cpp.
_EOG_NAMES = frozenset((
"<|eot_id|>", "<|im_end|>", "<|end|>", "<|return|>", "<|call|>", "<|flush|>", "<|calls|>", "<end_of_turn>",
"<|endoftext|>", "</s>", "<|eom_id|>", "<EOT>", "_<EOT>", "[EOT]", "[EOS]", "<|end_of_text|>", "<end_of_utterance>",
@@ -33,6 +34,15 @@ _JOYAI_LLM_SPLITS = (
r'''[!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+|[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+''',
)
+# RecursiveSI/flint's tokenizer.json applies these splits in sequence.
+_FLINT_SPLITS = (
+ r"\p{N}{1,3}",
+ "[一-龥぀-ゟ゠-ヿ]+",
+ r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+",
+)
+
+_CUSTOM_SPLITS = {"joyai-llm": _JOYAI_LLM_SPLITS, "flint": _FLINT_SPLITS}
+
def _add_bos(meta: dict) -> bool:
if "tokenizer.ggml.add_bos_token" in meta:
@@ -105,11 +115,11 @@ def load_gguf_tokenizer(model_path: str):
from tokenizers import normalizers
fast.normalizer = normalizers.NFC()
- elif pre == "joyai-llm":
+ elif pre in _CUSTOM_SPLITS:
from tokenizers import Regex, pre_tokenizers
fast.pre_tokenizer = pre_tokenizers.Sequence(
- [pre_tokenizers.Split(Regex(r), behavior="isolated") for r in _JOYAI_LLM_SPLITS]
+ [pre_tokenizers.Split(Regex(r), behavior="isolated") for r in _CUSTOM_SPLITS[pre]]
+ [pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)]
)
if pre in _IGNORE_MERGES_PRE:
@@ -129,6 +139,7 @@ def gguf_eos_token_ids(model_path: str) -> set[int]:
tokens = meta["tokenizer.ggml.tokens"]
ids = {i for i, t in enumerate(tokens) if t in _EOG_NAMES}
ids |= {int(meta[f"tokenizer.ggml.{k}_token_id"]) for k in ("eos", "eot", "eom") if f"tokenizer.ggml.{k}_token_id" in meta}
+ ids |= {int(i) for i in meta.get("tokenizer.ggml.eog_token_ids", ())}
names = {tokens[i]: i for i in ids if i < len(tokens)}
# llama.cpp's exceptions: <|end|> only ends a message when <|call|> and <|return|> end the turn (gpt-oss), and gemma4's </s> is a normal token
if "<|end|>" in names and names.keys() & {"<|call|>", "<|calls|>"} and names.keys() & {"<|return|>", "<|flush|>"}:
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index d78a8548..efea3d30 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -95,6 +95,17 @@ _DSV4_PROCESSOR = "freetoken.mm.processors.deepseek_v4:DSV4MMProcessor"
_DSV4_ENCODERS = (EncoderSpec("vision", "vision_n_layers", ("image",)),)
_MODEL_REGISTRY: dict[str, ModelSpec] = {
+ "FlintGGUFForCausalLM": ModelSpec(
+ "freetoken.models.flint", "FlintForCausalLM",
+ parse_config="parse_gguf_config", iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,
+ unquantized_modules=("*.mlp.gate",),
+ ),
+ "FlintForCausalLM": ModelSpec(
+ "freetoken.models.flint", "FlintForCausalLM",
+ packed_modules_mapping=_DENSE_PACKED + _EXPERTS_PACKED,
+ unquantized_modules=("*.mlp.gate",),
+ ),
"LlamaForCausalLM": ModelSpec(
"freetoken.models.llama",
"LlamaForCausalLM",
diff --git a/python/freetoken/server/args.py b/python/freetoken/server/args.py
index b3eea9f8..f0bd0523 100644
--- a/python/freetoken/server/args.py
+++ b/python/freetoken/server/args.py
@@ -206,6 +206,8 @@ def parse_args(
" ".join(str(v) for v in text_cfg.get("architectures", []) or []),
]
marker = " ".join(candidates).lower()
+ if "flint" in marker:
+ return "glm47"
if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker:
return "gpt_oss"
# M3 first: its marker also contains the bare "minimax" substring, but the
@@ -257,6 +259,8 @@ def parse_args(
" ".join(str(v) for v in text_cfg.get("architectures", []) or []),
]
marker = " ".join(candidates).lower()
+ if "flint" in marker:
+ return "flint"
if "gpt_oss" in marker or "gpt-oss" in marker or "gptoss" in marker:
return "gpt_oss"
if "deepseek" in marker and any(
@@ -638,7 +642,7 @@ def parse_args(
default="auto",
choices=[
"auto", "off", "deepseekv32", "gpt_oss", "qwen3", "glm",
- "minimax", "minimax_m3", "muse_glimmer", "gemma4",
+ "minimax", "minimax_m3", "muse_glimmer", "gemma4", "flint",
],
help=(
"Reasoning parser that splits chain-of-thought into reasoning_content "
diff --git a/python/freetoken/server/generation.py b/python/freetoken/server/generation.py
index 2e342ff8..ef90402a 100644
--- a/python/freetoken/server/generation.py
+++ b/python/freetoken/server/generation.py
@@ -410,6 +410,9 @@ def _make_reasoning_parser(spec: GenSpec, state: Any) -> ReasoningParser | None:
or bool(ctk.get("enable_thinking"))
or bool(ctk.get("thinking"))
)
+ elif parser_name == "flint":
+ # Flint opens its own optional think block, including when tools are present.
+ force_reasoning = False
elif parser_name == "minimax_m3":
# M3's template pre-opens <mm:think> only in thinking_mode "enabled" (the
# model then emits just the closing tag); "adaptive" (default) leaves the
diff --git a/python/freetoken/server/reasoning_parser.py b/python/freetoken/server/reasoning_parser.py
index 6788675c..411f9bfe 100644
--- a/python/freetoken/server/reasoning_parser.py
+++ b/python/freetoken/server/reasoning_parser.py
@@ -874,6 +874,7 @@ class ReasoningParser:
"""Wraps a reasoning detector for streaming and non-streaming use."""
ReasoningParserEnum: Dict[str, Type[BaseReasoningParser]] = {
+ "flint": ThinkReasoningParser,
"deepseekv32": DeepSeekV32ReasoningParser,
"gpt_oss": GptOssHarmonyReasoningParser,
"qwen3": ThinkReasoningParser,
diff --git a/tests/README.md b/tests/README.md
index 8b2ec885..3cd4cc90 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -68,7 +68,8 @@ checkpoint is set:
| `FREETOKEN_MINIMAX_M3_MODEL` | `tokenizer/test_mm_tokenize.py` — local MiniMax-M3 checkpoint (start/end expansion, token budget) |
| `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size |
| `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) |
-| `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |
+| `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — local checkpoint directory or GGUF file; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |
+| `FREETOKEN_REBUILD_EXTRA_ARGS` | `e2e/test_cache_rebuild.py` — extra server flags, e.g. `--moe-strategy hybrid --moe-cache-size 1024 --max-running-requests 1` for offloaded MoE |
| `FREETOKEN_GEMMA4_GGUF_GLOB` | `models/test_gemma4_gguf_rope.py` — glob matching a local gemma-4 GGUF file |
| `FREETOKEN_TEST_MODELS_ROOT` | `models/test_quant_config.py` — directory whose checkpoints the stored-tensor scheme scan checks (default `/mnt/nvme/models`) |
| `FREETOKEN_DSV4_VL_MODEL` | `tokenizer/test_mm_tokenize.py` - local DeepSeek-V4 vision checkpoint |
diff --git a/tests/e2e/test_cache_rebuild.py b/tests/e2e/test_cache_rebuild.py
index b17fc28b..8e6632a1 100644
--- a/tests/e2e/test_cache_rebuild.py
+++ b/tests/e2e/test_cache_rebuild.py
@@ -16,12 +16,14 @@ Gated behind ``needs_weights``:
FREETOKEN_REBUILD_TEST_MODEL small local model dir (falls back to FREETOKEN_TEST_MODEL)
FREETOKEN_REBUILD_MIN_FREE_GIB free-GPU-memory gate (default 20)
FREETOKEN_REBUILD_BOOT_TIMEOUT seconds to wait for "serving" (default 300)
+ FREETOKEN_REBUILD_EXTRA_ARGS additional ft serve flags (e.g. offloaded MoE)
"""
from __future__ import annotations
import json
import os
+import shlex
import socket
import subprocess
import sys
@@ -92,7 +94,7 @@ def _wait_until_serving(base: str, proc: subprocess.Popen, deadline: float) -> N
def _generate(base: str) -> str:
- """A short non-thinking completion — the point is that the engine still runs, not what it says."""
+ """A short completion; reasoning models may spend the whole budget in their thought channel."""
code, body = _post(
base,
"/v1/chat/completions",
@@ -105,7 +107,8 @@ def _generate(base: str) -> str:
},
)
assert code == 200, body
- return str(body["choices"][0]["message"]["content"])
+ message = body["choices"][0]["message"]
+ return message.get("content") or message.get("reasoning_content") or ""
@pytest.mark.skipif(not torch.cuda.is_available(), reason="cache rebuild e2e needs CUDA")
@@ -113,7 +116,7 @@ def test_cache_rebuild_resizes_a_live_engine_and_keeps_serving(tmp_path):
model_dir = _model_dir()
if model_dir is None:
pytest.skip("set FREETOKEN_REBUILD_TEST_MODEL to a small local model directory")
- if not model_dir.is_dir():
+ if not model_dir.is_dir() and not model_dir.is_file():
pytest.skip(f"model is not downloaded: {model_dir}")
min_free = float(os.environ.get("FREETOKEN_REBUILD_MIN_FREE_GIB", "20"))
@@ -133,6 +136,7 @@ def test_cache_rebuild_resizes_a_live_engine_and_keeps_serving(tmp_path):
"--port", str(port),
"--num-pages", str(BOOT_PAGES),
"--enable-cache-report",
+ *shlex.split(os.environ.get("FREETOKEN_REBUILD_EXTRA_ARGS", "")),
],
stdout=log,
stderr=subprocess.STDOUT,
diff --git a/tests/models/test_flint.py b/tests/models/test_flint.py
new file mode 100644
index 00000000..73d91312
--- /dev/null
+++ b/tests/models/test_flint.py
@@ -0,0 +1,130 @@
+"""Flint contracts checked against independent tensor math and checkpoint layouts."""
+from __future__ import annotations
+
+import json
+import os
+from types import SimpleNamespace
+from unittest.mock import patch
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from freetoken.models.flint.config import parse_config
+from freetoken.models.flint.moe import FlintMoE
+
+
[email protected](autouse=True)
+def _single_rank(monkeypatch):
+ from freetoken.distributed import info
+ monkeypatch.setattr(info, "_TP_INFO", info.DistributedInfo(rank=0, size=1))
+
+
+def _hf(**overrides):
+ return SimpleNamespace(**(dict(
+ architectures=["FlintForCausalLM"], model_type="flint", num_hidden_layers=3,
+ hidden_size=64, head_dim=128, num_attention_heads=4, num_key_value_heads=2,
+ intermediate_size=128, moe_intermediate_size=64,
+ shared_expert_intermediate_size=128, vocab_size=128, max_position_embeddings=2048,
+ layer_types=["full_attention", "sliding_attention", "full_attention"],
+ no_rope_layer_indices=[0, 2], mlp_only_layers=[0],
+ hidden_act="silu", router_score_function="sigmoid", softmax_type="learnable",
+ norm_topk_prob=True, num_experts=8, num_experts_per_tok=2,
+ rms_norm_eps=1e-6, sliding_window=513, rope_theta=10000., tie_word_embeddings=False,
+ ) | overrides))
+
+
[email protected](not torch.cuda.is_available(), reason="fused CUDA router")
[email protected]("renormalize", [False, True])
+def test_sigmoid_routing_bias_affects_selection_only(renormalize):
+ from freetoken.layers import LinearReplicated
+ torch.manual_seed(42)
+ block = FlintMoE.__new__(FlintMoE)
+ block.top_k, block.renormalize = 6, renormalize
+ block.gate = LinearReplicated(128, 256, has_bias=False)
+ block.gate.weight = torch.randn(256, 128, device="cuda", dtype=torch.bfloat16) * .1
+ block.gate.expert_bias = torch.randn(256, device="cuda") * .05
+ x = torch.randn(32, 128, device="cuda", dtype=torch.bfloat16)
+ weights, ids = block._route(x)
+ logits = F.linear(x, block.gate.weight).float()
+ scores = 1. / (1. + torch.exp(-logits))
+ expected_ids = torch.argsort(scores + block.gate.expert_bias, dim=-1, descending=True)[:, :6]
+ assert [set(row) for row in ids.tolist()] == [set(row) for row in expected_ids.tolist()]
+ expected = scores.gather(1, ids.long())
+ if renormalize:
+ expected = expected / expected.sum(1, keepdim=True)
+ torch.testing.assert_close(weights, expected, rtol=1e-6, atol=1e-7)
+ assert ids.dtype == torch.int32
+
+
+def test_weight_reader_merges_cross_shard_projections_and_preserves_sinks(tmp_path):
+ from safetensors.torch import save_file
+ from freetoken.models.flint.weight import iter_weights
+ hf = _hf()
+ (tmp_path / "config.json").write_text(json.dumps(vars(hf)))
+ q = torch.randn(512, 64, dtype=torch.bfloat16)
+ k = torch.randn(256, 64, dtype=torch.bfloat16)
+ v = torch.randn(256, 64, dtype=torch.bfloat16)
+ pre = "model.layers.0.self_attn"
+ sink = torch.tensor([.125, .25, .5, 1.])
+ save_file({f"{pre}.q_proj.weight": q, f"{pre}.softmax_offset": sink}, tmp_path / "a.safetensors")
+ save_file({f"{pre}.k_proj.weight": k, f"{pre}.v_proj.weight": v}, tmp_path / "b.safetensors")
+ weights = dict(iter_weights(str(tmp_path), torch.device("cpu"), include_moe_experts=False, include_non_moe=True))
+ torch.testing.assert_close(weights[f"{pre}.qkv_proj.weight"], torch.cat([q, k, v]))
+ torch.testing.assert_close(weights[f"{pre}.softmax_offset"], sink)
+ for i in (1, 2):
+ assert not weights[f"model.layers.{i}.mlp.gate.expert_bias"].any()
+
+
+def test_gguf_explicit_end_of_generation_ids_are_not_lost():
+ from freetoken.models.gguf.tokenizer import gguf_eos_token_ids
+ tokens = ["x"] * 19
+ for i, token in [(1, "<|end_of_text|>"), (3, "<|user|>"), (5, "<|observation|>"), (18, "<|developer|>")]:
+ tokens[i] = token
+ meta = {"tokenizer.ggml.tokens": tokens, "tokenizer.ggml.eos_token_id": 1,
+ "tokenizer.ggml.eog_token_ids": [1, 3, 5, 18]}
+ with patch("freetoken.models.gguf.tokenizer.load_gguf_metadata", return_value=meta):
+ assert gguf_eos_token_ids("test.gguf") == {1, 3, 5, 18}
+
+
+def test_official_gguf_matches_hf_tokenizer_and_config():
+ from transformers import AutoTokenizer
+ from freetoken.models.gguf.config import build_gguf_shim
+ from freetoken.models.gguf.tokenizer import load_gguf_tokenizer, gguf_eos_token_ids
+ from freetoken.models.flint.gguf import parse_gguf_config
+ from freetoken.utils import cached_load_hf_config
+ gguf_path = os.environ.get("FREETOKEN_FLINT_GGUF")
+ hf_path = os.environ.get("FREETOKEN_FLINT_MODEL")
+ if not gguf_path or not hf_path:
+ pytest.skip("set FREETOKEN_FLINT_GGUF and FREETOKEN_FLINT_MODEL")
+ hf = AutoTokenizer.from_pretrained(hf_path, local_files_only=True)
+ gg = load_gguf_tokenizer(gguf_path)
+ texts = [
+ "HelloWorld camelCase XMLHTTPRequest don't we're I'LL",
+ "1234567890 3.1415926535 -42 1e-10",
+ "漢字かなカナ 中文测试 café e\u0301 👩‍💻",
+ "def f(x):\n\treturn {'value': x + 1}\n\n# EOF ",
+ "<|user|>hello<|assistant|><think>check</think>42<|end_of_text|>",
+ ]
+ for text in texts:
+ assert gg.encode(text, add_special_tokens=False) == hf.encode(text, add_special_tokens=False), text
+ tools = [{"type": "function", "function": {"name": "read", "parameters": {
+ "type": "object", "properties": {"path": {"type": "string"}}, "required": ["path"],
+ }}}]
+ messages = [
+ {"role": "user", "content": "Read the file."},
+ {"role": "assistant", "content": "", "reasoning_content": "Inspect it first.",
+ "tool_calls": [{"type": "function", "function": {"name": "read", "arguments": {"path": "/tmp/a.py"}}}]},
+ {"role": "tool", "content": "print(42)"},
+ ]
+ assert gg.apply_chat_template(messages, tools=tools, add_generation_prompt=True) == hf.apply_chat_template(
+ messages, tools=tools, add_generation_prompt=True,
+ )
+ assert gguf_eos_token_ids(gguf_path) == {150001, 150003, 150005, 150018}
+ cfg_hf = parse_config(cached_load_hf_config(hf_path))
+ cfg_gg = parse_gguf_config(build_gguf_shim(gguf_path))
+ for name in ("head_dim", "num_qo_heads", "num_kv_heads", "num_layers", "hidden_size",
+ "num_experts", "num_experts_per_tok", "moe_intermediate_size",
+ "shared_expert_intermediate_size", "first_k_dense_replace", "attention_groups"):
+ assert getattr(cfg_hf, name) == getattr(cfg_gg, name), name
diff --git a/tests/server/test_parser_auto_selection.py b/tests/server/test_parser_auto_selection.py
index 6bdc6579..49b0f1a6 100644
--- a/tests/server/test_parser_auto_selection.py
+++ b/tests/server/test_parser_auto_selection.py
@@ -103,3 +103,8 @@ def test_an_explicit_choice_beats_inference():
pinned, _ = parse_args(["--model", ANON_PATH, "--reasoning-parser", "qwen3"])
assert off.reasoning_parser is None
assert pinned.reasoning_parser == "qwen3"
+
+
+def test_flint_uses_optional_reasoning_and_xml_tool_calls():
+ assert _inferred("FlintForCausalLM") == ("glm47", "flint")
+ assert _inferred("FlintGGUFForCausalLM") == ("glm47", "flint")
diff --git a/tests/server/test_reasoning_parser_all_models.py b/tests/server/test_reasoning_parser_all_models.py
index 5d115dc1..47e0f094 100644
--- a/tests/server/test_reasoning_parser_all_models.py
+++ b/tests/server/test_reasoning_parser_all_models.py
@@ -1,6 +1,11 @@
from __future__ import annotations
+from types import SimpleNamespace
+
+import pytest
+
from freetoken.server.reasoning_parser import ReasoningParser
+from freetoken.server.generation import GenSpec, _make_reasoning_parser
def _stream(parser: ReasoningParser, chunks):
@@ -15,6 +20,21 @@ def _stream(parser: ReasoningParser, chunks):
return reasoning, content
[email protected]("kwargs", [{}, {"enable_thinking": True}, {"enable_thinking": False}])
[email protected]("tools", [None, [{"type": "function", "function": {"name": "read"}}]])
+def test_flint_streaming_opens_reasoning_only_on_a_think_tag(kwargs, tools):
+ spec = GenSpec(messages=[], sampling_params=SimpleNamespace(), chat_template_kwargs=kwargs, template_tools=tools)
+ state = SimpleNamespace(config=SimpleNamespace(reasoning_parser="flint"))
+ thought = "The literal <tool_call> here is a quotation."
+ text = f"<think>{thought}</think>42"
+ for size in (1, 3, 7, len(text)):
+ parser = _make_reasoning_parser(spec, state)
+ assert _stream(parser, [text[i:i+size] for i in range(0, len(text), size)]) == (thought, "42")
+ parser = _make_reasoning_parser(spec, state)
+ assert _stream(parser, ["A direct ", "answer."]) == ("", "A direct answer.")
+ assert _make_reasoning_parser(spec, state).parse_non_stream("A direct answer.") == ("", "A direct answer.")
+
+
# --------------------------------------------------------------- gpt-oss harmony
ANALYSIS_FINAL = (
"<|channel|>analysis<|message|>The user says hi. Greet back.<|end|>"
diff --git a/tests/server/test_streaming_model_matrix.py b/tests/server/test_streaming_model_matrix.py
index eb6e3a2e..1ee79379 100644
--- a/tests/server/test_streaming_model_matrix.py
+++ b/tests/server/test_streaming_model_matrix.py
@@ -126,6 +126,7 @@ MARKUP_MARKERS = {
# family, mirroring args.py auto-inference. think_open == "" when the chat template
# opens the block implicitly (model emits only the closing marker).
REASONING_FAMILIES = {
+ "flint": ("glm47", "flint", "<think>", "</think>"),
"dsv4": ("deepseekv32", "deepseekv32", "", "</think>"),
"dsv4.1": ("deepseekv41", "deepseekv32", "", "</think>"),
"qwen3.5": ("qwen3_coder", "qwen3", "", "</think>"),