Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(glm5_next): serve image input on GLM-5.3-Flash (#479)

Xiaoze Fan3faef36b8149
diff --git a/docs/models.md b/docs/models.md
index 7f7f7b6d..2a1600aa 100644
--- a/docs/models.md
+++ b/docs/models.md
@@ -29,6 +29,7 @@ These families accept image input by default; pass `--text-model-only` to skip t
| Qwen3.6 (both variants, every listed weight format), Qwen3.8-Flash-Next, Qwen3-VL | one token per 32x32 pixels of the resized image, dynamic resolution | pixel areas in `size.shortest_edge` / `longest_edge`; checkpoint defaults 64 to 16384 tokens | `{"size": {"longest_edge": 1048576}}` |
| Gemma-4 26B-A4B, 31B (`gemma4`: ViT tower, streamed under `--mm-encoder-weights host`) | one of the soft-token budgets 70 / 140 / 280 / 560 / 1120, every image scaled to its budget as far as the aspect ratio allows | the maximum picks the largest budget within it, below 70 is refused at start-up; the minimum has no effect | `{"max_soft_tokens": 1120}` |
| Gemma-4 12B (`gemma4_unified`: linear patch embedder, resident under either placement) | same budgets, one 48x48 super-patch per soft token | same as the tower releases | same |
+| GLM-5.3-Flash (`glm5_next`: ViT tower, streamed under `--mm-encoder-weights host`) | one token per 28x28 pixels of the resized image, dynamic resolution on a canvas zero-padded to a 28-multiple | token counts, passed through as the processor's `min_image_tokens` / `max_image_tokens`; checkpoint defaults 16 to 8000 tokens | `{"max_image_tokens": 2048}` |
## MoE strategies
@@ -53,4 +54,3 @@ These families accept image input by default; pass `--text-model-only` to skip t
- DeepSeek-V4 checkpoints must keep the `inference/config.json` subdir — the
authoritative model args are read from there.
- Qwen3.8-Flash-Next keeps a 47.7 GiB PLE n-gram table pinned in host RAM.
-- Multimodal checkpoints are served text-only.
diff --git a/pyproject.toml b/pyproject.toml
index 228108f0..5511c1f2 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -58,7 +58,7 @@ dependencies = [
"torch>=2.11,<2.12",
"torchvision>=0.26,<0.27",
"tqdm>=4.66,<5",
- "transformers>=5.15,<6",
+ "transformers>=5.16,<5.17",
"triton==3.6.0; platform_system == 'Linux'",
"uvicorn>=0.30,<1",
]
diff --git a/python/freetoken/mm/processors/glm5_next.py b/python/freetoken/mm/processors/glm5_next.py
new file mode 100644
index 00000000..b739492a
--- /dev/null
+++ b/python/freetoken/mm/processors/glm5_next.py
@@ -0,0 +1,76 @@
+"""GLM-5.3-Flash image processor: patch-grid ViT with a 2x2 merger, a token-count budget and 1-D rope."""
+
+from __future__ import annotations
+
+import struct
+from typing import Any
+
+import torch
+
+from freetoken.message import MMItem
+from freetoken.mm import mm_pad_value
+from freetoken.mm.config import MultimodalConfig
+from freetoken.mm.processor import MMProcessor, PromptReplacement, content_hash
+
+
+class Glm5NextMMProcessor(MMProcessor):
+ def __init__(self, hf_config: Any, model_path: str, mm: MultimodalConfig) -> None:
+ super().__init__(model_path, mm)
+ vc = hf_config.vision_config
+ self.image_token_id = hf_config.image_token_id
+ # the template renders the begin/end wrapper tokens itself; only <|image|> expands
+ self.placeholder = [self.image_token_id]
+ self.merge = vc.spatial_merge_size
+ self.patch_dim = vc.in_channels * vc.temporal_patch_size * vc.patch_size**2
+
+ def get_mm_processor_kwargs(self, mm: MultimodalConfig) -> dict[str, Any]:
+ kwargs: dict[str, Any] = {"return_tensors": "pt"}
+ # the image processor takes the budget as token counts
+ if mm.image_min_tokens is not None:
+ kwargs["min_image_tokens"] = mm.image_min_tokens
+ if mm.image_max_tokens is not None:
+ kwargs["max_image_tokens"] = mm.image_max_tokens
+ return {**kwargs, **mm.processor_kwargs}
+
+ def process(self, images: list[Any]) -> list[MMItem]:
+ processor = self._image_processor()
+ kwargs = self.get_mm_processor_kwargs(self.mm)
+ items: list[MMItem] = []
+ for pil in images:
+ out = processor(images=pil, **kwargs)
+ pixel_values = out["pixel_values"].to(torch.bfloat16)
+ grid = out["image_grid_thw"][0].tolist()
+ t, h, w = int(grid[0]), int(grid[1]), int(grid[2])
+ if t != 1:
+ raise ValueError("video/temporal input is not supported")
+ # hash the bf16 wire buffer: half the bytes, and the exact tensor the cache and radix key agree on
+ item_hash = content_hash(pixel_values, struct.pack("<3i", t, h, w))
+ items.append(
+ MMItem(
+ modality="image",
+ hash=item_hash,
+ pad_value=mm_pad_value(item_hash),
+ offsets=[],
+ feature=pixel_values,
+ model_specific_data={"grid_thw": [t, h, w]},
+ )
+ )
+ return items
+
+ def prompt_replacement(self, item: MMItem) -> PromptReplacement:
+ t, h, w = item.grid_thw
+ return PromptReplacement([self.image_token_id] * ((t * h * w) // (self.merge * self.merge)))
+
+ def dummy_items(self, dtype: torch.dtype, device: torch.device) -> list[MMItem]:
+ merge = self.merge
+ return [MMItem(
+ modality="image",
+ hash=0,
+ pad_value=0,
+ offsets=[[0, 1]],
+ feature=torch.zeros(merge * merge, self.patch_dim, dtype=dtype, device=device),
+ model_specific_data={"grid_thw": [1, merge, merge]},
+ )]
+
+
+__all__ = ["Glm5NextMMProcessor"]
diff --git a/python/freetoken/models/glm5_next/__init__.py b/python/freetoken/models/glm5_next/__init__.py
index 7e949097..9417286c 100644
--- a/python/freetoken/models/glm5_next/__init__.py
+++ b/python/freetoken/models/glm5_next/__init__.py
@@ -1,11 +1,16 @@
-from .config import parse_config
-from .model import Glm5NextForCausalLM
+from .config import VisionConfig, parse_config, parse_vision_config
+from .model import Glm5NextForCausalLM, Glm5NextForConditionalGeneration
+from .vision import Glm5NextVisionModel
from .weight import iter_expert_pieces, iter_weights, nvfp4_expert_spec
__all__ = [
"nvfp4_expert_spec",
"Glm5NextForCausalLM",
+ "Glm5NextForConditionalGeneration",
+ "Glm5NextVisionModel",
+ "VisionConfig",
"parse_config",
+ "parse_vision_config",
"iter_weights",
"iter_expert_pieces",
]
diff --git a/python/freetoken/models/glm5_next/config.py b/python/freetoken/models/glm5_next/config.py
index 34958466..b1a50097 100644
--- a/python/freetoken/models/glm5_next/config.py
+++ b/python/freetoken/models/glm5_next/config.py
@@ -23,6 +23,7 @@ The checkpoint is served in the precision its quantization_config declares.
from __future__ import annotations
import os
+from dataclasses import dataclass
from typing import Any
from freetoken.models.config import (
@@ -36,6 +37,46 @@ from freetoken.models.config import (
from .args import load_args
+@dataclass(frozen=True)
+class VisionConfig:
+ hidden_size: int
+ depth: int
+ num_heads: int
+ intermediate_size: int
+ projection_intermediate_size: int
+ out_hidden_size: int
+ in_channels: int
+ patch_size: int
+ temporal_patch_size: int
+ spatial_merge_size: int
+ rms_norm_eps: float
+ swiglu_limit: float
+ attention_bias: bool
+
+
+def parse_vision_config(hf_config: Any) -> VisionConfig | None:
+ """None when the config carries no vision section, which is how a text-only engine asks for no tower."""
+ vc = getattr(hf_config, "vision_config", None)
+ if vc is None:
+ return None
+ if vc.hidden_act != "silu":
+ raise NotImplementedError(f"glm5_next vision tower activation {vc.hidden_act!r}; only silu is implemented")
+ return VisionConfig(
+ hidden_size=vc.hidden_size,
+ depth=vc.depth,
+ num_heads=vc.num_heads,
+ intermediate_size=vc.intermediate_size,
+ projection_intermediate_size=vc.projection_intermediate_size,
+ out_hidden_size=vc.out_hidden_size,
+ in_channels=vc.in_channels,
+ patch_size=vc.patch_size,
+ temporal_patch_size=vc.temporal_patch_size,
+ spatial_merge_size=vc.spatial_merge_size,
+ rms_norm_eps=vc.rms_norm_eps,
+ swiglu_limit=vc.swiglu_limit,
+ attention_bias=bool(vc.attention_bias),
+ )
+
def _dsa_on(args, dsa_layer_ids) -> bool:
"""DSA serving switch, resolved ONCE into the attention-group spec (the pool
@@ -204,10 +245,10 @@ def parse_config(hf_config: Any) -> ModelConfig:
attn_sm_scale=args.qk_head_dim**-0.5,
has_attn_bias=bool(getattr(text, "attention_bias", False)),
swiglu_limit=args.swiglu_limit,
- vision_config=None, # text-only milestone; model.visual.* weights are dropped
+ vision_config=parse_vision_config(hf_config),
image_token_id=getattr(hf_config, "image_token_id", None),
glm5_args=args,
)
-__all__ = ["parse_config"]
+__all__ = ["VisionConfig", "parse_config", "parse_vision_config"]
diff --git a/python/freetoken/models/glm5_next/mlp.py b/python/freetoken/models/glm5_next/mlp.py
index 5ec3c66a..f10b7360 100644
--- a/python/freetoken/models/glm5_next/mlp.py
+++ b/python/freetoken/models/glm5_next/mlp.py
@@ -1,4 +1,4 @@
-"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers and shared experts.
+"""Clamped-SwiGLU MLP for GLM-5.3-Flash's leading dense layers, shared experts and vision tower.
Same shape as glm_moe_dsa's GlmDsaGatedMLP (every projection built from the QuantConfig), but the activation
is the GLM-5.3 clamped SwiGLU (``swiglu_limit``):
@@ -22,12 +22,13 @@ class Glm5NextGatedMLP(BaseOP):
intermediate_size: int,
swiglu_limit: float | None = None,
*,
+ has_bias: bool = False,
quant_config=None,
prefix: str = "",
):
- self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.gate_proj")
- self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.up_proj")
- self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=False, quant_config=quant_config, prefix=f"{prefix}.down_proj")
+ self.gate_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=has_bias, quant_config=quant_config, prefix=f"{prefix}.gate_proj")
+ self.up_proj = LinearReplicated(hidden_size, intermediate_size, has_bias=has_bias, quant_config=quant_config, prefix=f"{prefix}.up_proj")
+ self.down_proj = LinearReplicated(intermediate_size, hidden_size, has_bias=has_bias, quant_config=quant_config, prefix=f"{prefix}.down_proj")
self.swiglu_limit = swiglu_limit
@nvtx_annotate("MLP")
diff --git a/python/freetoken/models/glm5_next/model.py b/python/freetoken/models/glm5_next/model.py
index ae287215..d2bfa684 100644
--- a/python/freetoken/models/glm5_next/model.py
+++ b/python/freetoken/models/glm5_next/model.py
@@ -29,15 +29,17 @@ from freetoken.layers import (
VocabParallelEmbedding,
)
from freetoken.layers.mhc import hc_contract, hc_expand, mhc_fused_post_pre, mhc_post, mhc_pre
-from freetoken.models.blocks import BaseLLMModel
+from freetoken.models.blocks import BaseLLMModel, embed_input_ids
from freetoken.utils import nvtx_annotate
from .attention import Glm5NextAttention
from .kda import Glm5NextKDA
from .mlp import Glm5NextGatedMLP
from .moe import Glm5NextSparseBlock
+from .vision import Glm5NextVisionModel
if TYPE_CHECKING:
+ from freetoken.message import MMItem
from freetoken.models.config import ModelConfig
@@ -147,7 +149,7 @@ class Glm5NextModel(BaseOP):
self.norm = RMSNorm(size=config.hidden_size, eps=config.rms_norm_eps)
def forward(self, input_ids: torch.Tensor) -> torch.Tensor:
- x = self.embed_tokens.forward(input_ids)
+ x = embed_input_ids(self.embed_tokens, input_ids, get_global_ctx().batch)
residual = post = comb = None
for layer in self.layers.op_list:
x, residual, post, comb = layer.forward(x, residual, post, comb)
@@ -181,4 +183,17 @@ class Glm5NextForCausalLM(BaseLLMModel):
return self.lm_head.forward(output)
-__all__ = ["Glm5NextForCausalLM"]
+class Glm5NextForConditionalGeneration(Glm5NextForCausalLM):
+ def __init__(self, config: ModelConfig):
+ super().__init__(config)
+ if config.is_multimodal:
+ self.visual = Glm5NextVisionModel(config.vision_config)
+
+ def place_encoder_weights(self, mode: str) -> None:
+ self.visual.place_weights(mode)
+
+ def encode(self, item: MMItem) -> torch.Tensor:
+ return self.visual.forward(item.feature, [item.grid_thw])
+
+
+__all__ = ["Glm5NextForCausalLM", "Glm5NextForConditionalGeneration"]
diff --git a/python/freetoken/models/glm5_next/vision.py b/python/freetoken/models/glm5_next/vision.py
new file mode 100644
index 00000000..bd59296e
--- /dev/null
+++ b/python/freetoken/models/glm5_next/vision.py
@@ -0,0 +1,151 @@
+"""GLM-5.3-Flash vision tower: Conv3d patch embedding, pre-RMSNorm blocks with q/k norms and 2-D rope, a 2x2 Conv2d downsample and a clamped-SwiGLU merger into the text width."""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, List
+
+import torch
+import torch.nn.functional as F
+from freetoken.layers import BaseOP, LayerNorm, LinearReplicated, OPList, RMSNorm
+from freetoken.models.weight_stream import BlockWeightStreamer
+
+from .mlp import Glm5NextGatedMLP
+
+if TYPE_CHECKING:
+ from .config import VisionConfig
+
+
+class _ConvParams(BaseOP):
+ def __init__(self, out_ch: int, shape: tuple[int, ...]):
+ self.weight = torch.empty(out_ch, *shape)
+ self.bias = torch.empty(out_ch)
+
+ forward = None
+
+
+class Glm5NextVisionPatchEmbed(BaseOP):
+ """Kept as a real Conv3d (kernel == stride): an equivalent unfolded matmul rounds differently."""
+
+ def __init__(self, vc: VisionConfig):
+ self._shape = (vc.in_channels, vc.temporal_patch_size, vc.patch_size, vc.patch_size)
+ self.proj = _ConvParams(vc.hidden_size, self._shape)
+
+ def forward(self, pixel_values: torch.Tensor) -> torch.Tensor:
+ x = pixel_values.view(-1, *self._shape).to(self.proj.weight.dtype)
+ return F.conv3d(x, self.proj.weight, self.proj.bias, stride=self._shape[1:]).view(x.shape[0], -1)
+
+
+class Glm5NextVisionAttention(BaseOP):
+ def __init__(self, vc: VisionConfig):
+ self.num_heads = vc.num_heads
+ self.head_dim = vc.hidden_size // vc.num_heads
+ self.qkv = LinearReplicated(vc.hidden_size, 3 * vc.hidden_size, has_bias=vc.attention_bias)
+ self.proj = LinearReplicated(vc.hidden_size, vc.hidden_size, has_bias=vc.attention_bias)
+ self.q_norm = RMSNorm(self.head_dim, eps=vc.rms_norm_eps)
+ self.k_norm = RMSNorm(self.head_dim, eps=vc.rms_norm_eps)
+
+ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]) -> torch.Tensor:
+ from freetoken.kernel.triton.rope import apply_rope_with_cos_sin_cache_inplace
+
+ S, H, D = x.shape[0], self.num_heads, self.head_dim
+ q, k, v = self.qkv.forward(x).view(S, 3, H * D).unbind(1)
+ q = self.q_norm.forward(q.reshape(S * H, D)).view(S, H * D)
+ k = self.k_norm.forward(k.reshape(S * H, D)).view(S, H * D)
+ apply_rope_with_cos_sin_cache_inplace(positions, q, k, D, cache, is_neox=True)
+ q, k, v = (t.view(S, H, D).transpose(0, 1).unsqueeze(0) for t in (q, k, v))
+ # bidirectional attention within each image; images never attend across
+ outs = [
+ F.scaled_dot_product_attention(qs, ks, vs)
+ for qs, ks, vs in zip(torch.split(q, lengths, dim=2), torch.split(k, lengths, dim=2), torch.split(v, lengths, dim=2))
+ ]
+ o = outs[0] if len(outs) == 1 else torch.cat(outs, dim=2)
+ return self.proj.forward(o[0].transpose(0, 1).reshape(S, H * D))
+
+
+class Glm5NextVisionBlock(BaseOP):
+ def __init__(self, vc: VisionConfig):
+ self.norm1 = RMSNorm(vc.hidden_size, eps=vc.rms_norm_eps)
+ self.norm2 = RMSNorm(vc.hidden_size, eps=vc.rms_norm_eps)
+ self.attn = Glm5NextVisionAttention(vc)
+ self.mlp = Glm5NextGatedMLP(vc.hidden_size, vc.intermediate_size, vc.swiglu_limit, has_bias=vc.attention_bias)
+
+ def forward(self, x: torch.Tensor, cache: torch.Tensor, positions: torch.Tensor, lengths: List[int]) -> torch.Tensor:
+ x = x + self.attn.forward(self.norm1.forward(x), cache, positions, lengths)
+ return x + self.mlp.forward(self.norm2.forward(x))
+
+
+class Glm5NextVisionPatchMerger(Glm5NextGatedMLP):
+ """proj -> LayerNorm -> GELU in front of the clamped-SwiGLU MLP; the checkpoint stores both halves under one module."""
+
+ def __init__(self, vc: VisionConfig):
+ super().__init__(vc.out_hidden_size, vc.projection_intermediate_size, vc.swiglu_limit)
+ self.proj = LinearReplicated(vc.out_hidden_size, vc.out_hidden_size, has_bias=False)
+ # the reference builds a torch LayerNorm with the default eps
+ self.post_projection_norm = LayerNorm(vc.out_hidden_size, eps=1e-5)
+
+ def forward(self, x: torch.Tensor) -> torch.Tensor:
+ return super().forward(F.gelu(self.post_projection_norm.forward(self.proj.forward(x))))
+
+
+class Glm5NextVisionModel(BaseOP):
+ """Pixels -> soft tokens [sum(h*w) / merge^2, out_hidden] in the text width, one grid_thw row per image."""
+
+ def __init__(self, vc: VisionConfig):
+ m = vc.spatial_merge_size
+ self.patch_embed = Glm5NextVisionPatchEmbed(vc)
+ self.blocks = OPList([Glm5NextVisionBlock(vc) for _ in range(vc.depth)])
+ self.post_layernorm = RMSNorm(vc.hidden_size, eps=vc.rms_norm_eps)
+ self.downsample = _ConvParams(vc.out_hidden_size, (vc.hidden_size, m, m))
+ self.merger = Glm5NextVisionPatchMerger(vc)
+ self._vc = vc
+ self._inv_freq: torch.Tensor | None = None
+ self._streamer: BlockWeightStreamer | None = None
+
+ def place_weights(self, mode: str) -> None:
+ """gpu: every tensor resident; host: block tensors in pinned banks streamed two blocks at a time (patch embedding, downsample and merger stay resident)."""
+ if mode == "host" and self._streamer is None:
+ self._streamer = BlockWeightStreamer(self.blocks.op_list, self.patch_embed.proj.weight.device)
+ elif mode == "gpu" and self._streamer is not None:
+ self._streamer.unstream()
+ self._streamer = None
+ elif mode not in ("gpu", "host"):
+ raise ValueError(f"unknown vision weight placement {mode!r}")
+
+ def _blocks(self):
+ if self._streamer is None:
+ return enumerate(self.blocks.op_list)
+ return self._streamer.blocks(self.blocks.op_list)
+
+ def _rope_table(self, grid: torch.Tensor, device: torch.device) -> tuple[torch.Tensor, torch.Tensor]:
+ from transformers.vision_utils import get_vision_position_ids
+
+ if self._inv_freq is None or self._inv_freq.device != device:
+ # fp32 like the reference; head_dim/2 angles, one quarter of the head each for the row and the column index, pairs (i, i + head_dim/2) share an angle
+ half = (self._vc.hidden_size // self._vc.num_heads) // 2
+ self._inv_freq = 1.0 / (10000.0 ** (torch.arange(0, half, 2, dtype=torch.float32, device=device) / half))
+ pos_ids = get_vision_position_ids(grid, self._vc.spatial_merge_size)
+ freqs = (pos_ids.unsqueeze(-1) * self._inv_freq).flatten(1)
+ # per-token rope rows [cos | sin] consumed by the NeoX kernel via positions=arange
+ cache = torch.cat((freqs.cos(), freqs.sin()), dim=-1).contiguous()
+ return cache, torch.arange(pos_ids.shape[0], dtype=torch.int32, device=device)
+
+ @torch.inference_mode()
+ def forward(self, pixel_values: torch.Tensor, grid_thw: List[List[int]]) -> torch.Tensor:
+ from transformers.vision_utils import get_vision_cu_seqlens
+
+ vc = self._vc
+ device = self.patch_embed.proj.weight.device
+ grid = torch.tensor(grid_thw, dtype=torch.long, device=device)
+ x = self.patch_embed.forward(pixel_values.to(device))
+ cache, positions = self._rope_table(grid, device)
+ lengths = torch.diff(get_vision_cu_seqlens(grid)).tolist()
+ for _, blk in self._blocks():
+ x = blk.forward(x, cache, positions, lengths)
+ x = self.post_layernorm.forward(x)
+ m = vc.spatial_merge_size
+ x = x.view(-1, m, m, vc.hidden_size).permute(0, 3, 1, 2)
+ x = F.conv2d(x, self.downsample.weight, self.downsample.bias, stride=m).view(-1, vc.out_hidden_size)
+ return self.merger.forward(x)
+
+
+__all__ = ["Glm5NextVisionModel"]
diff --git a/python/freetoken/models/glm5_next/weight.py b/python/freetoken/models/glm5_next/weight.py
index 03e2a7a5..60d97e90 100644
--- a/python/freetoken/models/glm5_next/weight.py
+++ b/python/freetoken/models/glm5_next/weight.py
@@ -7,8 +7,9 @@ supported: text-only key layouts, TP > 1, resident routed experts.
Routed experts go to the offload cache from their NVFP4 or block-fp8 pieces; every
other projection loads as stored (bf16, or fp8 codes with their block scales) with keys
-renamed ``model.language_model.X`` -> ``model.X``. ``model.visual.*`` and the trailing
-MTP layer are never read.
+renamed ``model.language_model.X`` -> ``model.X``. The vision tower loads as stored under
+``model.visual.X`` -> ``visual.X`` when an encoder is built; the trailing MTP layer is
+never read.
Load-time fusions (must mirror the module split orders):
@@ -157,12 +158,19 @@ def _iter_dsa_layer(reader, layer: int) -> Iterator[tuple[str, torch.Tensor]]:
yield f"{dst}.indexer.{part}", reader.get(f"{src}.indexer.{part}").to(dtype)
+def _iter_vision(reader, weight_map: dict) -> Iterator[tuple[str, torch.Tensor]]:
+ for name in weight_map:
+ if name.startswith("model.visual."):
+ yield "visual." + name[len("model.visual.") :], reader.get(name).to(torch.bfloat16)
+
+
def iter_weights(
model_path: str,
device: torch.device,
*,
include_moe_experts: bool,
include_non_moe: bool,
+ include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
assert not include_moe_experts, (
"GLM-5.3 routed experts only serve from the offload cache; they are loaded from their expert pieces."
@@ -224,6 +232,8 @@ def iter_weights(
).to(torch.bfloat16)
yield f"{_MODEL}.norm.weight", reader.get(f"{_CKPT}.norm.weight").to(torch.bfloat16)
yield "lm_head.weight", reader.get("lm_head.weight").to(torch.bfloat16)
+ if include_vision:
+ yield from _iter_vision(reader, weight_map)
finally:
reader.close()
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 62eade18..d84d2ca6 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -65,6 +65,8 @@ _QWEN4_EXP_PACKED = _QWEN3_5_PACKED + (
_GLM5_NEXT_PACKED = _EXPERTS_PACKED + (
("in_proj", ("q_proj", "k_proj", "v_proj", "b_proj", "f_a_proj", "g_a_proj")),
)
+_GLM5_NEXT_PROCESSOR = "freetoken.mm.processors.glm5_next:Glm5NextMMProcessor"
+_GLM5_NEXT_ENCODERS = (EncoderSpec("vision", "vision_config", ("image",)),)
# Gemma 4 keeps HF's flat layer children (mlp / experts / router) under one feed_forward block.
_GEMMA4_SEGMENTS = (
("feed_forward.shared_mlp", "mlp"),
@@ -275,12 +277,15 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
# GLM-5.3-Flash (model_type glm5_next): hybrid KDA linear attention (34/45 layers)
# + NoPE-MLA/DSA with a kpool-compressed indexer (11/45), mHC x4 residual streams,
# 288-expert sigmoid/noaux_tc MoE; natively-multimodal wrapper config (text tower
- # in text_config, weights under model.language_model.), served text-only.
+ # in text_config, weights under model.language_model.); the patch-grid vision tower
+ # under model.visual. serves image input.
"Glm5NextForConditionalGeneration": ModelSpec(
"freetoken.models.glm5_next",
- "Glm5NextForCausalLM",
+ "Glm5NextForConditionalGeneration",
checkpoint_roots=_LANGUAGE_MODEL_ROOT,
packed_modules_mapping=_GLM5_NEXT_PACKED,
+ mm_processor=_GLM5_NEXT_PROCESSOR,
+ encoders=_GLM5_NEXT_ENCODERS,
),
# Text-only sibling (the text_config's own architectures entry).
"Glm5NextForCausalLM": ModelSpec(
diff --git a/tests/README.md b/tests/README.md
index cb594663..9b1d7fdc 100644
--- a/tests/README.md
+++ b/tests/README.md
@@ -52,6 +52,7 @@ checkpoint is set:
| `FREETOKEN_QWEN3VL_MODEL` | `tokenizer/test_mm_tokenize.py` — local Qwen3-VL checkpoint (same expansion, DeepStack family) |
| `FREETOKEN_GEMMA4_MODEL` | `models/test_gemma4_vision.py`, `tokenizer/test_mm_tokenize.py` — local Gemma-4 `gemma4` checkpoint (tower parity, boi/eoi expansion; the tokenizer test also takes the 12B `gemma4_unified` checkpoint) |
| `FREETOKEN_GEMMA4_UNIFIED_MODEL` | `models/test_gemma4_vision.py` — local gemma-4-12B `gemma4_unified` checkpoint (embedder parity) |
+| `FREETOKEN_GLM53_MODEL` | `tokenizer/test_mm_tokenize.py` — local GLM-5.3-Flash checkpoint (image-token expansion) |
| `FREETOKEN_TEST_MOE_CACHE_SIZE` | `e2e/test_aime.py` — >0 switches to the offload MoE backend with this cache size |
| `FREETOKEN_TEST_MEM_RATIO` | `e2e/test_aime.py` — offload-mode memory_ratio (default `0.9`) |
| `FREETOKEN_REBUILD_TEST_MODEL` | `e2e/test_cache_rebuild.py` — a SMALL local model dir; boots a real server (falls back to `FREETOKEN_TEST_MODEL`) |
diff --git a/tests/mm/test_processor.py b/tests/mm/test_processor.py
index 6f5fe703..86668972 100644
--- a/tests/mm/test_processor.py
+++ b/tests/mm/test_processor.py
@@ -13,6 +13,7 @@ from freetoken.mm import MM_PAD_SHIFT_VALUE
from freetoken.mm.config import MultimodalConfig
from freetoken.mm.processor import MMProcessor, PromptReplacement, image_positions
from freetoken.mm.processors.gemma4 import Gemma4MMProcessor, Gemma4UnifiedMMProcessor
+from freetoken.mm.processors.glm5_next import Glm5NextMMProcessor
from freetoken.mm.processors.qwen_vl import QwenVLMMProcessor
PLACEHOLDER = 7
@@ -275,6 +276,42 @@ def test_gemma_unified_items_are_one_row_per_soft_token():
assert dummy.feature.shape == (1, 6912) and dummy.num_tokens == 1 and dummy.position_ids.tolist() == [[0, 0]]
+def _glm_config():
+ return SimpleNamespace(
+ architectures=["Glm5NextForConditionalGeneration"],
+ image_token_id=154854,
+ vision_config=SimpleNamespace(spatial_merge_size=2, patch_size=14, temporal_patch_size=2, in_channels=3),
+ text_config=SimpleNamespace(vocab_size=154880),
+ )
+
+
+def test_glm_token_budget_reaches_the_image_processor_and_rope_stays_one_dimensional():
+ calls = []
+
+ class _FakeImageProcessor:
+ def __call__(self, images, **kwargs):
+ calls.append(kwargs)
+ return {"pixel_values": torch.zeros(16, 1176), "image_grid_thw": torch.tensor([[1, 4, 4]])}
+
+ mm = MultimodalConfig(image_min_tokens=32, image_max_tokens=1024, processor_kwargs={"do_convert_rgb": False})
+ proc = Glm5NextMMProcessor(_glm_config(), "/nonexistent", mm)
+ proc._image_processor = lambda: _FakeImageProcessor()
+ (item,) = proc.process([object()])
+ # the budget is already in tokens; the extra kwarg rides along
+ assert calls == [{"return_tensors": "pt", "min_image_tokens": 32, "max_image_tokens": 1024, "do_convert_rgb": False}]
+ assert item.grid_thw == [1, 4, 4] and item.feature.dtype == torch.bfloat16
+ # the template renders the begin/end wrapper tokens itself; only <|image|> expands
+ assert proc.placeholder == [154854] and proc.prompt_replacement(item).full == [154854] * 4
+ assert proc.positions(10, [item]) is None
+ proc = Glm5NextMMProcessor(_glm_config(), "/nonexistent", MultimodalConfig())
+ proc._image_processor = lambda: _FakeImageProcessor()
+ proc.process([object()])
+ assert calls[-1] == {"return_tensors": "pt"}
+ (dummy,) = proc.dummy_items(torch.bfloat16, torch.device("cpu"))
+ dummy.validate()
+ assert dummy.feature.shape == (4, 1176) and dummy.num_tokens == 1
+
+
def test_registry_resolves_by_architecture(monkeypatch):
import freetoken.utils
from freetoken.mm.processor import get_mm_processor
@@ -283,6 +320,7 @@ def test_registry_resolves_by_architecture(monkeypatch):
"/qwen": _hf_config(),
"/gemma": _gemma_config(),
"/gemma-unified": _gemma_unified_config(),
+ "/glm": _glm_config(),
"/text-only": _hf_config(vision=False),
"/unknown-vlm": _hf_config(arch="SomeOtherForConditionalGeneration"),
}
@@ -291,6 +329,7 @@ def test_registry_resolves_by_architecture(monkeypatch):
assert get_mm_processor("/qwen", MultimodalConfig(disabled_encoders=frozenset({"vision"}))) is None # --mm-disable vision
assert isinstance(get_mm_processor("/gemma"), Gemma4MMProcessor)
assert isinstance(get_mm_processor("/gemma-unified"), Gemma4UnifiedMMProcessor)
+ assert isinstance(get_mm_processor("/glm"), Glm5NextMMProcessor)
assert get_mm_processor("/text-only") is None
assert get_mm_processor("/unknown-vlm") is None
assert get_mm_processor("/missing") is None # a config that fails to load means no vision
diff --git a/tests/models/test_glm5_next_config.py b/tests/models/test_glm5_next_config.py
index 54060e16..ad56f7e1 100644
--- a/tests/models/test_glm5_next_config.py
+++ b/tests/models/test_glm5_next_config.py
@@ -101,7 +101,12 @@ def _hf_config(quantization_config: dict | None = None) -> RawConfigShim:
"architectures": ["Glm5NextForConditionalGeneration"],
"model_type": "glm5_next",
"text_config": _text_config(),
- "vision_config": {"model_type": "glm5_next_vision", "depth": 24},
+ "vision_config": {
+ "model_type": "glm5_next_vision", "depth": 24, "hidden_size": 1024, "num_heads": 16, "intermediate_size": 4096,
+ "projection_intermediate_size": 10240, "out_hidden_size": 4096, "in_channels": 3, "patch_size": 14,
+ "temporal_patch_size": 2, "spatial_merge_size": 2, "rms_norm_eps": 1e-5, "swiglu_limit": 10.0,
+ "attention_bias": True, "hidden_act": "silu",
+ },
"image_token_id": 154854,
}
if quantization_config is not None:
@@ -204,8 +209,19 @@ def test_moe_and_scalars():
assert cfg.is_moe
# checkpoint-faithful: the resident projections follow the quantization_config, never a switch
assert (cfg.attn_quant, cfg.dense_quant, cfg.lm_head_quant) == ("none",) * 3
- # Text-only serving: the vision tower is never built.
- assert cfg.vision_config is None
+
+
+def test_vision_section_parses_into_the_tower_config():
+ cfg = parse_config(_hf_config())
+ # the merger lands in the text width, so the soft tokens scatter into the embeddings unprojected
+ assert cfg.vision_config.out_hidden_size == cfg.hidden_size
+ # a text-only engine hands the parser a config without the section
+ assert parse_config(RawConfigShim({**_hf_config().to_dict(), "vision_config": None})).vision_config is None
+ # the tower implements the clamped SwiGLU only
+ data = _hf_config().to_dict()
+ data["vision_config"]["hidden_act"] = "gelu"
+ with pytest.raises(NotImplementedError, match="silu"):
+ parse_config(RawConfigShim(data))
def test_args_alias_folding_and_nope():
@@ -233,7 +249,7 @@ def test_registry_resolves_glm5_next():
spec = get_model_spec("Glm5NextForConditionalGeneration")
assert spec.module == "freetoken.models.glm5_next"
- assert spec.model_cls == "Glm5NextForCausalLM"
+ assert spec.model_cls == "Glm5NextForConditionalGeneration"
assert get_model_spec("Glm5NextForCausalLM").module == spec.module
diff --git a/tests/models/test_glm5_next_vision.py b/tests/models/test_glm5_next_vision.py
new file mode 100644
index 00000000..ccc8549b
--- /dev/null
+++ b/tests/models/test_glm5_next_vision.py
@@ -0,0 +1,56 @@
+"""GLM-5.3-Flash vision tower: shapes and host streaming on a tiny random tower."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+
+from freetoken.models.glm5_next import Glm5NextVisionModel, VisionConfig
+
+pytestmark = pytest.mark.skipif(not torch.cuda.is_available(), reason="needs CUDA")
+
+
+def _tiny_vc():
+ return VisionConfig(
+ hidden_size=64, depth=2, num_heads=4, intermediate_size=128, projection_intermediate_size=96, out_hidden_size=48,
+ in_channels=3, patch_size=4, temporal_patch_size=2, spatial_merge_size=2, rms_norm_eps=1e-5, swiglu_limit=10.0,
+ attention_bias=True,
+ )
+
+
+def _build(vc):
+ torch.set_default_dtype(torch.bfloat16)
+ try:
+ with torch.device("cuda"):
+ tower = Glm5NextVisionModel(vc)
+ finally:
+ torch.set_default_dtype(torch.float32)
+ for p in tower.state_dict().values():
+ p.normal_(0, 0.02)
+ return tower
+
+
+def test_tiny_tower_merges_four_patches_per_token():
+ tower = _build(_tiny_vc())
+ feature = torch.randn(16 + 64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")
+ out = tower.forward(feature, [[1, 4, 4], [1, 8, 8]])
+ assert out.shape == (4 + 16, 48) and out.dtype == torch.bfloat16
+ keys = tower.state_dict()
+ assert keys["blocks.0.attn.qkv.bias"].shape == (192,) and keys["merger.post_projection_norm.bias"].shape == (48,)
+ assert "merger.gate_proj.bias" not in keys and keys["downsample.weight"].shape == (48, 64, 2, 2)
+
+
+def test_host_streamed_weights_compute_the_same_output():
+ tower = _build(_tiny_vc())
+ feature = torch.randn(64, 3 * 2 * 4 * 4, dtype=torch.bfloat16, device="cuda")
+ resident = tower.forward(feature, [[1, 8, 8]])
+ keys = tower.state_dict()
+ tower.place_weights("host")
+ assert tower.state_dict().keys() == keys.keys()
+ assert tower.state_dict()["blocks.0.attn.qkv.weight"].device.type == "cpu"
+ assert tower.state_dict()["merger.proj.weight"].device.type == "cuda"
+ for _ in range(2): # a second forward reuses the staging buffers behind the release events
+ assert torch.equal(tower.forward(feature, [[1, 8, 8]]), resident)
+ tower.place_weights("gpu")
+ assert tower.state_dict()["blocks.0.attn.qkv.weight"].device.type == "cuda"
+ assert torch.equal(tower.forward(feature, [[1, 8, 8]]), resident)
diff --git a/tests/tokenizer/test_mm_tokenize.py b/tests/tokenizer/test_mm_tokenize.py
index ea3bcfa9..b3866f7d 100644
--- a/tests/tokenizer/test_mm_tokenize.py
+++ b/tests/tokenizer/test_mm_tokenize.py
@@ -1,4 +1,4 @@
-"""Image tokenization against real Qwen VL checkpoints (skipped when absent)."""
+"""Image tokenization against real Qwen VL, Gemma-4 and GLM-5.3 checkpoints (skipped when absent)."""
from __future__ import annotations
@@ -17,12 +17,13 @@ MODELS = [
]
GEMMA = os.environ.get("FREETOKEN_GEMMA4_MODEL", "")
+GLM = os.environ.get("FREETOKEN_GLM53_MODEL", "")
pytestmark = [
pytest.mark.needs_weights,
pytest.mark.skipif(
- not MODELS and not os.path.exists(os.path.join(GEMMA, "config.json")),
- reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL checkpoint",
+ not MODELS and not any(os.path.exists(os.path.join(p, "config.json")) for p in (GEMMA, GLM)),
+ reason="no FREETOKEN_QWEN36_MODEL / FREETOKEN_QWEN3VL_MODEL / FREETOKEN_GEMMA4_MODEL / FREETOKEN_GLM53_MODEL checkpoint",
),
]
@@ -140,3 +141,25 @@ def test_gemma_image_max_tokens_picks_a_smaller_soft_token_budget():
budgeted = TokenizeManager(tokenizer, get_mm_processor(GEMMA, MultimodalConfig(image_max_tokens=100))).tokenize([_msg([_png(640, 400)])])[0]
# the processor scales the image to the budget as far as its aspect ratio allows: 10x6 pooled cells for 640x400
assert budgeted.mm_items[0].num_soft_tokens == 60 < default.mm_items[0].num_soft_tokens
+
+
[email protected](not os.path.exists(os.path.join(GLM, "config.json")), reason="FREETOKEN_GLM53_MODEL not set")
+def test_glm_expands_the_image_token_between_the_template_wrappers():
+ from freetoken.mm.config import MultimodalConfig
+ from freetoken.mm.processor import get_mm_processor
+ from freetoken.tokenizer.tokenize import TokenizeManager
+ from freetoken.utils import cached_load_hf_config
+ from freetoken.utils.hf import load_tokenizer
+
+ hf = cached_load_hf_config(GLM)
+ tokenizer = load_tokenizer(GLM)
+ r = TokenizeManager(tokenizer, get_mm_processor(GLM)).tokenize([_msg([_png(640, 400)])])[0]
+ item = r.mm_items[0]
+ ((start, end),) = item.offsets
+ ids = r.input_ids
+ # 640x400 is zero-padded to a 644x420 canvas: 46x30 patches, one token per 2x2 of them
+ assert item.grid_thw == [1, 30, 46] and end - start == item.num_tokens == 345
+ assert ids[start - 1].item() == hf.image_start_token_id and ids[end].item() == hf.image_end_token_id
+ assert bool((ids[start:end] == item.pad_value).all()) and r.mrope_positions is None
+ budgeted = TokenizeManager(tokenizer, get_mm_processor(GLM, MultimodalConfig(image_max_tokens=100))).tokenize([_msg([_png(640, 400)])])[0]
+ assert budgeted.mm_items[0].num_tokens <= 100