FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(glm5_next): serve image input on GLM-5.3-Flash (#479)
Xiaoze Fan3faef36b8149
diff --git a/docs/models.md b/docs/models.mdindex 7f7f7b6d..2a1600aa 100644--- a/docs/models.md+++ b/docs/models.md@@ -29,6 +29,7 @@ These families accept image input by default; pass `--text-model-only` to skip t| Qwen3.6 (both variants, every listed weight format), Qwen3.8-Flash-Next, Qwen3-VL | one token per 32x32 pixels of the resized image, dynamic resolution | pixel areas in `size.shortest_edge` / `longest_edge`; checkpoint defaults 64 to 16384 tokens | `{"size": {"longest_edge": 1048576}}` || Gemma-4 26B-A4B, 31B (`gemma4`: ViT tower, streamed under `--mm-encoder-weights host`) | one of the soft-token budgets 70 / 140 / 280 / 560 / 1120, every image scaled to its budget as far as the aspect ratio allows | the maximum picks the largest budget within it, below 70 is refused at start-up; the minimum has no effect | `{"max_soft_tokens": 1120}` || Gemma-4 12B (`gemma4_unified`: linear patch embedder, resident under either placement) | same budgets, one 48x48 super-patch per soft token | same as the tower releases | same |+| GLM-5.3-Flash (`glm5_next`: ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution on a canvas zero-padded to a 28-multiple | token counts, passed through as the processor's `min_image_tokens` / `max_image_tokens`; checkpoint defaults 16 to 8000 tokens | `{"max_image_tokens": 2048}` |## MoE strategies@@ -53,4 +54,3 @@ These families accept image input by default; pass `--text-model-only` to skip t- DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — theauthoritative model args are read from there.- Qwen3.8-Flash-Next keeps a 47.7 GiB PLE n-gram table pinned in host RAM.-- Multimodal checkpoints are served text-only.diff --git a/pyproject.toml b/pyproject.tomlindex 228108f0..5511c1f2 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -58,7 +58,7 @@ dependencies = ["torch>=2.11,<2.12","torchvision>=0.26,<0.27","tqdm>=4.66,<5",- "transformers>=5.15,<6",+ "transformers>=5.16,<5.17","triton==3.6.0; platform_system == 'Linux'","uvicorn>=0.30,<1",]diff --git a/python/freetoken/mm/processors/glm5_next.py b/python/freetoken/mm/processors/glm5_next.pynew file mode 100644index 00000000..b739492a--- /dev/null+++ b/python/freetoken/mm/processors/glm5_next.py@@ -0,0 +1,76 @@+"""GLM-5.3-Flash image processor: patch-grid ViT with a 2x2 merger, a token-count budget and 1-D rope."""++from __future__ import annotations++import struct+from typing import Any++import torch++from freetoken.message import MMItem+from freetoken.mm import mm_pad_value+from freetoken.mm.config import MultimodalConfig+from freetoken.mm.processor import MMProcessor, PromptReplacement, content_hash+++class Glm5NextMMProcessor(MMProcessor):+ def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:+ super().__init__(model_path, mm)+ vc = hf_config.vision_config+ self.image_token_id = hf_config.image_token_id+ # the template renders the begin/end wrapper tokens itself; only <|image|> expands+ self.placeholder = [self.image_token_id]+ self.merge = vc.spatial_merge_size+ self.patch_dim = vc.in_channels * vc.temporal_patch_size * vc.patch_size**2++ def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:+ kwargs: dict[str, Any] = {"return_tensors": "pt"}+ # the image processor takes the budget as token counts+ if mm.image_min_tokens is not None:+ kwargs["min_image_tokens"] = mm.image_min_tokens+ if mm.image_max_tokens is not None:+ kwargs["max_image_tokens"] = mm.image_max_tokens+ return {**kwargs, **mm.processor_kwargs}++ def process(self, images: list[Any]) -> list[MMItem]:+ processor = self._image_processor()+ kwargs = self.get_mm_processor_kwargs(self.mm)+ items: list[MMItem] = []+ for pil in images:+ out = processor(images=pil, **kwargs)+ pixel_values = out["pixel_values"].to(torch.bfloat16)+ grid = out["image_grid_thw"][0].tolist()+ t, h, w = int(grid[0]), int(grid[1]), int(grid[2])+ if t != 1:+ raise ValueError("video/temporal input is not supported")+ # hash the bf16 wire buffer: half the bytes, and the exact tensor the cache and radix key agree on+ item_hash = content_hash(pixel_values, struct.pack("<3i", t, h, w))+ items.append(+ MMItem(+ modality="image",+ hash=item_hash,+ pad_value=mm_pad_value(item_hash),+ offsets=[],+ feature=pixel_values,+ model_specific_data={"grid_thw": [t, h, w]},+ )+ )+ return items++ def prompt_replacement(self, item: MMItem) -> PromptReplacement:+ t, h, w = item.grid_thw+ return PromptReplacement([self.image_token_id] * ((t * h * w) // (self.merge * self.merge)))++ def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:+ merge = self.merge+ return [MMItem(+ modality="image",+ hash=0,+ pad_value=0,+ offsets=[[0, 1]],+ feature=torch.zeros(merge * merge, self.patch_dim, dtype=dtype, device=device),+ model_specific_data={"grid_thw": [1, merge, merge]},+ )]+++__all__ = ["Glm5NextMMProcessor"]diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.pyindex 7e949097..9417286c 100644--- a/python/freetoken/models/glm5_next/__init__.py+++ b/python/freetoken/models/glm5_next/__init__.py@@ -1,11 +1,16 @@-from .config import parse_config-from .model import Glm5NextForCausalLM+from .config import VisionConfig, parse_config, parse_vision_config+from .model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration+from .vision import Glm5NextVisionModelfrom .weight import iter_expert_pieces, iter_weights, nvfp4_expert_spec__all__ = ["nvfp4_expert_spec","Glm5NextForCausalLM",+ "Glm5NextForConditionalGeneration",+ "Glm5NextVisionModel",+ "VisionConfig","parse_config",+ "parse_vision_config","iter_weights","iter_expert_pieces",]diff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.pyindex 34958466..b1a50097 100644--- a/python/freetoken/models/glm5_next/config.py+++ b/python/freetoken/models/glm5_next/config.py@@ -23,6 +23,7 @@ The checkpoint is served in the precision its quantization_config declares.from __future__ import annotationsimport os+from dataclasses import dataclassfrom typing import Anyfrom freetoken.models.config import (@@ -36,6 +37,46 @@ from freetoken.models.config import (from .args import load_args+@dataclass(frozen=True)+class VisionConfig:+ hidden_size: int+ depth: int+ num_heads: int+ intermediate_size: int+ projection_intermediate_size: int+ out_hidden_size: int+ in_channels: int+ patch_size: int+ temporal_patch_size: int+ spatial_merge_size: int+ rms_norm_eps: float+ swiglu_limit: float+ attention_bias: bool+++def parse_vision_config(hf_config: Any) -> VisionConfig | None:+ """None when the config carries no vision section, which is how a text-only engine asks for no tower."""+ vc = getattr(hf_config, "vision_config", None)+ if vc is None:+ return None+ if vc.hidden_act != "silu":+ raise NotImplementedError(f"glm5_next vision tower activation {vc.hidden_act!r}; only silu is implemented")+ return VisionConfig(+ hidden_size=vc.hidden_size,+ depth=vc.depth,+ num_heads=vc.num_heads,+ intermediate_size=vc.intermediate_size,+ projection_intermediate_size=vc.projection_intermediate_size,+ out_hidden_size=vc.out_hidden_size,+ in_channels=vc.in_channels,+ patch_size=vc.patch_size,+ temporal_patch_size=vc.temporal_patch_size,+ spatial_merge_size=vc.spatial_merge_size,+ rms_norm_eps=vc.rms_norm_eps,+ swiglu_limit=vc.swiglu_limit,+ attention_bias=bool(vc.attention_bias),+ )+def _dsa_on(args, dsa_layer_ids) -> bool:"""DSA serving switch, resolved ONCE into the attention-group spec (the pool@@ -204,10 +245,10 @@ def parse_config(hf_config: Any) -> ModelConfig:attn_sm_scale=args.qk_head_dim**-0.5,has_attn_bias=bool(getattr(text, "attention_bias", False)),swiglu_limit=args.swiglu_limit,- vision_config=None, # text-only milestone; model.visual.* weights are dropped+ vision_config=parse_vision_config(hf_config),image_token_id=getattr(hf_config, "image_token_id", None),glm5_args=args,)-__all__ = ["parse_config"]+__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.pyindex 5ec3c66a..f10b7360 100644--- a/python/freetoken/models/glm5_next/mlp.py+++ b/python/freetoken/models/glm5_next/mlp.py@@ -1,4 +1,4 @@-"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts.+"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers, shared experts and vision tower.Same shape as glm_moe_dsa's GlmDsaGatedMLP (every projection built from the QuantConfig), but the activationis the GLM-5.3 clamped SwiGLU (``swiglu_limit``):@@ -22,12 +22,13 @@ class Glm5NextGatedMLP(BaseOP):intermediate_size: int,swiglu_limit: float | None = None,*,+ has_bias: bool = False,quant_config=None,prefix: str = "",):- self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")- self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")- self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=has_bias, quant_config=quant_config, prefix=f"{prefix}.gate_proj")+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=has_bias, quant_config=quant_config, prefix=f"{prefix}.up_proj")+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=has_bias, quant_config=quant_config, prefix=f"{prefix}.down_proj")self.swiglu_limit = swiglu_limit@nvtx_annotate("MLP")diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.pyindex ae287215..d2bfa684 100644--- a/python/freetoken/models/glm5_next/model.py+++ b/python/freetoken/models/glm5_next/model.py@@ -29,15 +29,17 @@ from freetoken.layers import (VocabParallelEmbedding,)from freetoken.layers.mhc import hc_contract, hc_expand, mhc_fused_post_pre, mhc_post, mhc_pre-from freetoken.models.blocks import BaseLLMModel+from freetoken.models.blocks import BaseLLMModel, embed_input_idsfrom freetoken.utils import nvtx_annotatefrom .attention import Glm5NextAttentionfrom .kda import Glm5NextKDAfrom .mlp import Glm5NextGatedMLPfrom .moe import Glm5NextSparseBlock+from .vision import Glm5NextVisionModelif TYPE_CHECKING:+ from freetoken.message import MMItemfrom freetoken.models.config import ModelConfig@@ -147,7 +149,7 @@ class Glm5NextModel(BaseOP):self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)def forward(self, input_ids: torch.Tensor) -> torch.Tensor:- x = self.embed_tokens.forward(input_ids)+ x = embed_input_ids(self.embed_tokens, input_ids, get_global_ctx().batch)residual = post = comb = Nonefor layer in self.layers.op_list:x, residual, post, comb = layer.forward(x, residual, post, comb)@@ -181,4 +183,17 @@ class Glm5NextForCausalLM(BaseLLMModel):return self.lm_head.forward(output)-__all__ = ["Glm5NextForCausalLM"]+class Glm5NextForConditionalGeneration(Glm5NextForCausalLM):+ def __init__(self, config: ModelConfig):+ super().__init__(config)+ if config.is_multimodal:+ self.visual = Glm5NextVisionModel(config.vision_config)++ def place_encoder_weights(self, mode: str) -> None:+ self.visual.place_weights(mode)++ def encode(self, item: MMItem) -> torch.Tensor:+ return self.visual.forward(item.feature, [item.grid_thw])+++__all__ = ["Glm5NextForCausalLM", "Glm5NextForConditionalGeneration"]diff --git a/python/freetoken/models/glm5_next/vision.py b/python/freetoken/models/glm5_next/vision.pynew file mode 100644index 00000000..bd59296e--- /dev/null+++ b/python/freetoken/models/glm5_next/vision.py@@ -0,0 +1,151 @@+"""GLM-5.3-Flash vision tower: Conv3d patch embedding, pre-RMSNorm blocks with q/k norms and 2-D rope, a 2x2 Conv2d downsample and a clamped-SwiGLU merger into the text width."""++from __future__ import annotations++from typing import TYPE_CHECKING, List++import torch+import torch.nn.functional as F+from freetoken.layers import BaseOP, LayerNorm, LinearReplicated, OPList, RMSNorm+from freetoken.models.weight_stream import BlockWeightStreamer++from .mlp import Glm5NextGatedMLP++if TYPE_CHECKING:+ from .config import VisionConfig+++class _ConvParams(BaseOP):+ def __init__(self, out_ch: int, shape: tuple[int, ...]):+ self.weight = torch.empty(out_ch, *shape)+ self.bias = torch.empty(out_ch)++ forward = None+++class Glm5NextVisionPatchEmbed(BaseOP):+ """Kept as a real Conv3d (kernel == stride): an equivalent unfolded matmul rounds differently."""++ def __init__(self, vc: VisionConfig):+ self._shape = (vc.in_channels, vc.temporal_patch_size, vc.patch_size, vc.patch_size)+ self.proj = _ConvParams(vc.hidden_size, self._shape)++ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:+ x = pixel_values.view(-1, *self._shape).to(self.proj.weight.dtype)+ return F.conv3d(x, self.proj.weight, self.proj.bias, stride=self._shape[1:]).view(x.shape[0], -1)+++class Glm5NextVisionAttention(BaseOP):+ def __init__(self, vc: VisionConfig):+ self.num_heads = vc.num_heads+ self.head_dim = vc.hidden_size // vc.num_heads+ self.qkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=vc.attention_bias)+ self.proj = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=vc.attention_bias)+ self.q_norm = RMSNorm(self.head_dim, eps=vc.rms_norm_eps)+ self.k_norm = RMSNorm(self.head_dim, eps=vc.rms_norm_eps)++ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]) -> torch.Tensor:+ from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace++ S, H, D = x.shape[0], self.num_heads, self.head_dim+ q, k, v = self.qkv.forward(x).view(S, 3, H * D).unbind(1)+ q = self.q_norm.forward(q.reshape(S * H, D)).view(S, H * D)+ k = self.k_norm.forward(k.reshape(S * H, D)).view(S, H * D)+ apply_rope_with_cos_sin_cache_inplace(positions, q, k, D, cache, is_neox=True)+ q, k, v = (t.view(S, H, D).transpose(0, 1).unsqueeze(0) for t in (q, k, v))+ # bidirectional attention within each image; images never attend across+ outs = [+ F.scaled_dot_product_attention(qs, ks, vs)+ for qs, ks, vs in zip(torch.split(q, lengths, dim=2), torch.split(k, lengths, dim=2), torch.split(v, lengths, dim=2))+ ]+ o = outs[0] if len(outs) == 1 else torch.cat(outs, dim=2)+ return self.proj.forward(o[0].transpose(0, 1).reshape(S, H * D))+++class Glm5NextVisionBlock(BaseOP):+ def __init__(self, vc: VisionConfig):+ self.norm1 = RMSNorm(vc.hidden_size, eps=vc.rms_norm_eps)+ self.norm2 = RMSNorm(vc.hidden_size, eps=vc.rms_norm_eps)+ self.attn = Glm5NextVisionAttention(vc)+ self.mlp = Glm5NextGatedMLP(vc.hidden_size, vc.intermediate_size, vc.swiglu_limit, has_bias=vc.attention_bias)++ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]) -> torch.Tensor:+ x = x + self.attn.forward(self.norm1.forward(x), cache, positions, lengths)+ return x + self.mlp.forward(self.norm2.forward(x))+++class Glm5NextVisionPatchMerger(Glm5NextGatedMLP):+ """proj -> LayerNorm -> GELU in front of the clamped-SwiGLU MLP; the checkpoint stores both halves under one module."""++ def __init__(self, vc: VisionConfig):+ super().__init__(vc.out_hidden_size, vc.projection_intermediate_size, vc.swiglu_limit)+ self.proj = LinearReplicated(vc.out_hidden_size, vc.out_hidden_size, has_bias=False)+ # the reference builds a torch LayerNorm with the default eps+ self.post_projection_norm = LayerNorm(vc.out_hidden_size, eps=1e-5)++ def forward(self, x: torch.Tensor) -> torch.Tensor:+ return super().forward(F.gelu(self.post_projection_norm.forward(self.proj.forward(x))))+++class Glm5NextVisionModel(BaseOP):+ """Pixels -> soft tokens [sum(h*w) / merge^2, out_hidden] in the text width, one grid_thw row per image."""++ def __init__(self, vc: VisionConfig):+ m = vc.spatial_merge_size+ self.patch_embed = Glm5NextVisionPatchEmbed(vc)+ self.blocks = OPList([Glm5NextVisionBlock(vc) for _ in range(vc.depth)])+ self.post_layernorm = RMSNorm(vc.hidden_size, eps=vc.rms_norm_eps)+ self.downsample = _ConvParams(vc.out_hidden_size, (vc.hidden_size, m, m))+ self.merger = Glm5NextVisionPatchMerger(vc)+ self._vc = vc+ self._inv_freq: torch.Tensor | None = None+ self._streamer: BlockWeightStreamer | None = None++ def place_weights(self, mode: str) -> None:+ """gpu: every tensor resident; host: block tensors in pinned banks streamed two blocks at a time (patch embedding, downsample and merger stay resident)."""+ if mode == "host" and self._streamer is None:+ self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)+ elif mode == "gpu" and self._streamer is not None:+ self._streamer.unstream()+ self._streamer = None+ elif mode not in ("gpu", "host"):+ raise ValueError(f"unknown vision weight placement {mode!r}")++ def _blocks(self):+ if self._streamer is None:+ return enumerate(self.blocks.op_list)+ return self._streamer.blocks(self.blocks.op_list)++ def _rope_table(self, grid: torch.Tensor, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:+ from transformers.vision_utils import get_vision_position_ids++ if self._inv_freq is None or self._inv_freq.device != device:+ # fp32 like the reference; head_dim/2 angles, one quarter of the head each for the row and the column index, pairs (i, i + head_dim/2) share an angle+ half = (self._vc.hidden_size // self._vc.num_heads) // 2+ self._inv_freq = 1.0 / (10000.0 ** (torch.arange(0, half, 2, dtype=torch.float32, device=device) / half))+ pos_ids = get_vision_position_ids(grid, self._vc.spatial_merge_size)+ freqs = (pos_ids.unsqueeze(-1) * self._inv_freq).flatten(1)+ # per-token rope rows [cos | sin] consumed by the NeoX kernel via positions=arange+ cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1).contiguous()+ return cache, torch.arange(pos_ids.shape[0], dtype=torch.int32, device=device)++ @torch.inference_mode()+ def forward(self, pixel_values: torch.Tensor, grid_thw: List[List[int]]) -> torch.Tensor:+ from transformers.vision_utils import get_vision_cu_seqlens++ vc = self._vc+ device = self.patch_embed.proj.weight.device+ grid = torch.tensor(grid_thw, dtype=torch.long, device=device)+ x = self.patch_embed.forward(pixel_values.to(device))+ cache, positions = self._rope_table(grid, device)+ lengths = torch.diff(get_vision_cu_seqlens(grid)).tolist()+ for _, blk in self._blocks():+ x = blk.forward(x, cache, positions, lengths)+ x = self.post_layernorm.forward(x)+ m = vc.spatial_merge_size+ x = x.view(-1, m, m, vc.hidden_size).permute(0, 3, 1, 2)+ x = F.conv2d(x, self.downsample.weight, self.downsample.bias, stride=m).view(-1, vc.out_hidden_size)+ return self.merger.forward(x)+++__all__ = ["Glm5NextVisionModel"]diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.pyindex 03e2a7a5..60d97e90 100644--- a/python/freetoken/models/glm5_next/weight.py+++ b/python/freetoken/models/glm5_next/weight.py@@ -7,8 +7,9 @@ supported: text-only key layouts, TP > 1, resident routed experts.Routed experts go to the offload cache from their NVFP4 or block-fp8 pieces; everyother projection loads as stored (bf16, or fp8 codes with their block scales) with keys-renamed ``model.language_model.X`` -> ``model.X``. ``model.visual.*`` and the trailing-MTP layer are never read.+renamed ``model.language_model.X`` -> ``model.X``. The vision tower loads as stored under+``model.visual.X`` -> ``visual.X`` when an encoder is built; the trailing MTP layer is+never read.Load-time fusions (must mirror the module split orders):@@ -157,12 +158,19 @@ def _iter_dsa_layer(reader, layer: int) -> Iterator[tuple[str, torch.Tensor]]:yield f"{dst}.indexer.{part}", reader.get(f"{src}.indexer.{part}").to(dtype)+def _iter_vision(reader, weight_map: dict) -> Iterator[tuple[str, torch.Tensor]]:+ for name in weight_map:+ if name.startswith("model.visual."):+ yield "visual." + name[len("model.visual.") :], reader.get(name).to(torch.bfloat16)++def iter_weights(model_path: str,device: torch.device,*,include_moe_experts: bool,include_non_moe: bool,+ include_vision: bool = True,) -> Iterator[tuple[str, torch.Tensor]]:assert not include_moe_experts, ("GLM-5.3 routed experts only serve from the offload cache; they are loaded from their expert pieces."@@ -224,6 +232,8 @@ def iter_weights().to(torch.bfloat16)yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16)yield "lm_head.weight", reader.get("lm_head.weight").to(torch.bfloat16)+ if include_vision:+ yield from _iter_vision(reader, weight_map)finally:reader.close()diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 62eade18..d84d2ca6 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -65,6 +65,8 @@ _QWEN4_EXP_PACKED = _QWEN3_5_PACKED + (_GLM5_NEXT_PACKED = _EXPERTS_PACKED + (("in_proj", ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")),)+_GLM5_NEXT_PROCESSOR = "freetoken.mm.processors.glm5_next:Glm5NextMMProcessor"+_GLM5_NEXT_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)# Gemma 4 keeps HF's flat layer children (mlp / experts / router) under one feed_forward block._GEMMA4_SEGMENTS = (("feed_forward.shared_mlp", "mlp"),@@ -275,12 +277,15 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {# GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers)# + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams,# 288-expert sigmoid/noaux_tc MoE; natively-multimodal wrapper config (text tower- # in text_config, weights under model.language_model.), served text-only.+ # in text_config, weights under model.language_model.); the patch-grid vision tower+ # under model.visual. serves image input."Glm5NextForConditionalGeneration": ModelSpec("freetoken.models.glm5_next",- "Glm5NextForCausalLM",+ "Glm5NextForConditionalGeneration",checkpoint_roots=_LANGUAGE_MODEL_ROOT,packed_modules_mapping=_GLM5_NEXT_PACKED,+ mm_processor=_GLM5_NEXT_PROCESSOR,+ encoders=_GLM5_NEXT_ENCODERS,),# Text-only sibling (the text_config's own architectures entry)."Glm5NextForCausalLM": ModelSpec(diff --git a/tests/README.md b/tests/README.mdindex cb594663..9b1d7fdc 100644--- a/tests/README.md+++ b/tests/README.md@@ -52,6 +52,7 @@ checkpoint is set:| `FREETOKEN_QWEN3VL_MODEL` | `tokenizer/test_mm_tokenize.py` — local Qwen3-VL checkpoint (same expansion, DeepStack family) || `FREETOKEN_GEMMA4_MODEL` | `models/test_gemma4_vision.py`, `tokenizer/test_mm_tokenize.py` — local Gemma-4 `gemma4` checkpoint (tower parity, boi/eoi expansion; the tokenizer test also takes the 12B `gemma4_unified` checkpoint) || `FREETOKEN_GEMMA4_UNIFIED_MODEL` | `models/test_gemma4_vision.py` — local gemma-4-12B `gemma4_unified` checkpoint (embedder parity) |+| `FREETOKEN_GLM53_MODEL` | `tokenizer/test_mm_tokenize.py` — local GLM-5.3-Flash checkpoint (image-token expansion) || `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size || `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) || `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |diff --git a/tests/mm/test_processor.py b/tests/mm/test_processor.pyindex 6f5fe703..86668972 100644--- a/tests/mm/test_processor.py+++ b/tests/mm/test_processor.py@@ -13,6 +13,7 @@ from freetoken.mm import MM_PAD_SHIFT_VALUEfrom freetoken.mm.config import MultimodalConfigfrom freetoken.mm.processor import MMProcessor, PromptReplacement, image_positionsfrom freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessor+from freetoken.mm.processors.glm5_next import Glm5NextMMProcessorfrom freetoken.mm.processors.qwen_vl import QwenVLMMProcessorPLACEHOLDER = 7@@ -275,6 +276,42 @@ def test_gemma_unified_items_are_one_row_per_soft_token():assert dummy.feature.shape == (1, 6912) and dummy.num_tokens == 1 and dummy.position_ids.tolist() == [[0, 0]]+def _glm_config():+ return SimpleNamespace(+ architectures=["Glm5NextForConditionalGeneration"],+ image_token_id=154854,+ vision_config=SimpleNamespace(spatial_merge_size=2, patch_size=14, temporal_patch_size=2, in_channels=3),+ text_config=SimpleNamespace(vocab_size=154880),+ )+++def test_glm_token_budget_reaches_the_image_processor_and_rope_stays_one_dimensional():+ calls = []++ class _FakeImageProcessor:+ def __call__(self, images, **kwargs):+ calls.append(kwargs)+ return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}++ mm = MultimodalConfig(image_min_tokens=32, image_max_tokens=1024, processor_kwargs={"do_convert_rgb": False})+ proc = Glm5NextMMProcessor(_glm_config(), "/nonexistent", mm)+ proc._image_processor = lambda: _FakeImageProcessor()+ (item,) = proc.process([object()])+ # the budget is already in tokens; the extra kwarg rides along+ assert calls == [{"return_tensors": "pt", "min_image_tokens": 32, "max_image_tokens": 1024, "do_convert_rgb": False}]+ assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16+ # the template renders the begin/end wrapper tokens itself; only <|image|> expands+ assert proc.placeholder == [154854] and proc.prompt_replacement(item).full == [154854] * 4+ assert proc.positions(10, [item]) is None+ proc = Glm5NextMMProcessor(_glm_config(), "/nonexistent", MultimodalConfig())+ proc._image_processor = lambda: _FakeImageProcessor()+ proc.process([object()])+ assert calls[-1] == {"return_tensors": "pt"}+ (dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))+ dummy.validate()+ assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1++def test_registry_resolves_by_architecture(monkeypatch):import freetoken.utilsfrom freetoken.mm.processor import get_mm_processor@@ -283,6 +320,7 @@ def test_registry_resolves_by_architecture(monkeypatch):"/qwen": _hf_config(),"/gemma": _gemma_config(),"/gemma-unified": _gemma_unified_config(),+ "/glm": _glm_config(),"/text-only": _hf_config(vision=False),"/unknown-vlm": _hf_config(arch="SomeOtherForConditionalGeneration"),}@@ -291,6 +329,7 @@ def test_registry_resolves_by_architecture(monkeypatch):assert get_mm_processor("/qwen", MultimodalConfig(disabled_encoders=frozenset({"vision"}))) is None # --mm-disable visionassert isinstance(get_mm_processor("/gemma"), Gemma4MMProcessor)assert isinstance(get_mm_processor("/gemma-unified"), Gemma4UnifiedMMProcessor)+ assert isinstance(get_mm_processor("/glm"), Glm5NextMMProcessor)assert get_mm_processor("/text-only") is Noneassert get_mm_processor("/unknown-vlm") is Noneassert get_mm_processor("/missing") is None # a config that fails to load means no visiondiff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.pyindex 54060e16..ad56f7e1 100644--- a/tests/models/test_glm5_next_config.py+++ b/tests/models/test_glm5_next_config.py@@ -101,7 +101,12 @@ def _hf_config(quantization_config: dict | None = None) -> RawConfigShim:"architectures": ["Glm5NextForConditionalGeneration"],"model_type": "glm5_next","text_config": _text_config(),- "vision_config": {"model_type": "glm5_next_vision", "depth": 24},+ "vision_config": {+ "model_type": "glm5_next_vision", "depth": 24, "hidden_size": 1024, "num_heads": 16, "intermediate_size": 4096,+ "projection_intermediate_size": 10240, "out_hidden_size": 4096, "in_channels": 3, "patch_size": 14,+ "temporal_patch_size": 2, "spatial_merge_size": 2, "rms_norm_eps": 1e-5, "swiglu_limit": 10.0,+ "attention_bias": True, "hidden_act": "silu",+ },"image_token_id": 154854,}if quantization_config is not None:@@ -204,8 +209,19 @@ def test_moe_and_scalars():assert cfg.is_moe# checkpoint-faithful: the resident projections follow the quantization_config, never a switchassert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3- # Text-only serving: the vision tower is never built.- assert cfg.vision_config is None+++def test_vision_section_parses_into_the_tower_config():+ cfg = parse_config(_hf_config())+ # the merger lands in the text width, so the soft tokens scatter into the embeddings unprojected+ assert cfg.vision_config.out_hidden_size == cfg.hidden_size+ # a text-only engine hands the parser a config without the section+ assert parse_config(RawConfigShim({**_hf_config().to_dict(), "vision_config": None})).vision_config is None+ # the tower implements the clamped SwiGLU only+ data = _hf_config().to_dict()+ data["vision_config"]["hidden_act"] = "gelu"+ with pytest.raises(NotImplementedError, match="silu"):+ parse_config(RawConfigShim(data))def test_args_alias_folding_and_nope():@@ -233,7 +249,7 @@ def test_registry_resolves_glm5_next():spec = get_model_spec("Glm5NextForConditionalGeneration")assert spec.module == "freetoken.models.glm5_next"- assert spec.model_cls == "Glm5NextForCausalLM"+ assert spec.model_cls == "Glm5NextForConditionalGeneration"assert get_model_spec("Glm5NextForCausalLM").module == spec.modulediff --git a/tests/models/test_glm5_next_vision.py b/tests/models/test_glm5_next_vision.pynew file mode 100644index 00000000..ccc8549b--- /dev/null+++ b/tests/models/test_glm5_next_vision.py@@ -0,0 +1,56 @@+"""GLM-5.3-Flash vision tower: shapes and host streaming on a tiny random tower."""++from __future__ import annotations++import pytest+import torch++from freetoken.models.glm5_next import Glm5NextVisionModel, VisionConfig++pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")+++def _tiny_vc():+ return VisionConfig(+ hidden_size=64, depth=2, num_heads=4, intermediate_size=128, projection_intermediate_size=96, out_hidden_size=48,+ in_channels=3, patch_size=4, temporal_patch_size=2, spatial_merge_size=2, rms_norm_eps=1e-5, swiglu_limit=10.0,+ attention_bias=True,+ )+++def _build(vc):+ torch.set_default_dtype(torch.bfloat16)+ try:+ with torch.device("cuda"):+ tower = Glm5NextVisionModel(vc)+ finally:+ torch.set_default_dtype(torch.float32)+ for p in tower.state_dict().values():+ p.normal_(0, 0.02)+ return tower+++def test_tiny_tower_merges_four_patches_per_token():+ tower = _build(_tiny_vc())+ feature = torch.randn(16 + 64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")+ out = tower.forward(feature, [[1, 4, 4], [1, 8, 8]])+ assert out.shape == (4 + 16, 48) and out.dtype == torch.bfloat16+ keys = tower.state_dict()+ assert keys["blocks.0.attn.qkv.bias"].shape == (192,) and keys["merger.post_projection_norm.bias"].shape == (48,)+ assert "merger.gate_proj.bias" not in keys and keys["downsample.weight"].shape == (48, 64, 2, 2)+++def test_host_streamed_weights_compute_the_same_output():+ tower = _build(_tiny_vc())+ feature = torch.randn(64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")+ resident = tower.forward(feature, [[1, 8, 8]])+ keys = tower.state_dict()+ tower.place_weights("host")+ assert tower.state_dict().keys() == keys.keys()+ assert tower.state_dict()["blocks.0.attn.qkv.weight"].device.type == "cpu"+ assert tower.state_dict()["merger.proj.weight"].device.type == "cuda"+ for _ in range(2): # a second forward reuses the staging buffers behind the release events+ assert torch.equal(tower.forward(feature, [[1, 8, 8]]), resident)+ tower.place_weights("gpu")+ assert tower.state_dict()["blocks.0.attn.qkv.weight"].device.type == "cuda"+ assert torch.equal(tower.forward(feature, [[1, 8, 8]]), resident)diff --git a/tests/tokenizer/test_mm_tokenize.py b/tests/tokenizer/test_mm_tokenize.pyindex ea3bcfa9..b3866f7d 100644--- a/tests/tokenizer/test_mm_tokenize.py+++ b/tests/tokenizer/test_mm_tokenize.py@@ -1,4 +1,4 @@-"""Image tokenization against real Qwen VL checkpoints (skipped when absent)."""+"""Image tokenization against real Qwen VL, Gemma-4 and GLM-5.3 checkpoints (skipped when absent)."""from __future__ import annotations@@ -17,12 +17,13 @@ MODELS = []GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")+GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")pytestmark = [pytest.mark.needs_weights,pytest.mark.skipif(- not MODELS and not os.path.exists(os.path.join(GEMMA, "config.json")),- reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL checkpoint",+ not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM)),+ reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL checkpoint",),]@@ -140,3 +141,25 @@ def test_gemma_image_max_tokens_picks_a_smaller_soft_token_budget():budgeted = TokenizeManager(tokenizer, get_mm_processor(GEMMA, MultimodalConfig(image_max_tokens=100))).tokenize([_msg([_png(640, 400)])])[0]# the processor scales the image to the budget as far as its aspect ratio allows: 10x6 pooled cells for 640x400assert budgeted.mm_items[0].num_soft_tokens == 60 < default.mm_items[0].num_soft_tokens++[email protected](not os.path.exists(os.path.join(GLM, "config.json")), reason="FREETOKEN_GLM53_MODEL not set")+def test_glm_expands_the_image_token_between_the_template_wrappers():+ from freetoken.mm.config import MultimodalConfig+ from freetoken.mm.processor import get_mm_processor+ from freetoken.tokenizer.tokenize import TokenizeManager+ from freetoken.utils import cached_load_hf_config+ from freetoken.utils.hf import load_tokenizer++ hf = cached_load_hf_config(GLM)+ tokenizer = load_tokenizer(GLM)+ r = TokenizeManager(tokenizer, get_mm_processor(GLM)).tokenize([_msg([_png(640, 400)])])[0]+ item = r.mm_items[0]+ ((start, end),) = item.offsets+ ids = r.input_ids+ # 640x400 is zero-padded to a 644x420 canvas: 46x30 patches, one token per 2x2 of them+ assert item.grid_thw == [1, 30, 46] and end - start == item.num_tokens == 345+ assert ids[start - 1].item() == hf.image_start_token_id and ids[end].item() == hf.image_end_token_id+ assert bool((ids[start:end] == item.pad_value).all()) and r.mrope_positions is None+ budgeted = TokenizeManager(tokenizer, get_mm_processor(GLM, MultimodalConfig(image_max_tokens=100))).tokenize([_msg([_png(640, 400)])])[0]+ assert budgeted.mm_items[0].num_tokens <= 100