FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""GLM-5.3-Flash from a llama.cpp ``glm5-next`` GGUF (unsloth ``GLM-5.3-Flash-GGUF``): config from the GGUF metadata, tensor names, and the converter's transforms undone.
The GGUF geometry is identical to the HF checkpoint (34 KDA linear-attention layers,
11 NoPE-MLA/DSA layers with a kpool-compressed indexer on a 3+1 hybrid block layout,
mHC x4 residual streams, 288-expert sigmoid/noaux_tc MoE with a shared expert and
clamped SwiGLU), so ``parse_gguf_config`` derives a text-config-shaped namespace from
the GGUF KV metadata and reuses ``glm5_next.config.parse_config`` wholesale.
A ``glm5v`` mmproj beside the text file adds the vision tower: its metadata becomes
the HF ``vision_config`` keys and the image processor config, and its tensors load
under ``visual.`` as the HF checkpoints do. The llama.cpp converter splits the Conv3d
patch embedding into two temporal halves (``v.patch_embd.weight`` and ``.weight.1``),
stacked back at load; every other tower tensor is stored in HF layout.
The llama.cpp converter (unsloth's glm5-next writer; tensor names identical across
the ``glm5-next`` and the older ``glm5next`` arch spellings) stores KDA's ``A_log``
pre-transformed as ``-exp(A_log)`` -- inverted at load -- and splits the MLA
``kv_b_proj`` into per-head 3-D ``attn_k_b`` / ``attn_v_b`` tensors (k stored
transposed, [heads, kv_lora, nope] next to [heads, v, kv_lora]); the halves may be
quantized (dynamic quants store them Q8_0), so they dequantize and fuse into the
dense layout the module's MLA absorption reads. The trailing MTP block (``blk.N.nextn.*``, ``nextn_predict_layers == 1``) is
dropped exactly like the HF loader drops the never-read MTP layer; routed experts
serve from the GGUF expert banks (``freetoken.models.gguf.experts``).
"""
from __future__ import annotations
import re
from dataclasses import replace
from types import SimpleNamespace
from typing import TYPE_CHECKING, Any, Iterator
import numpy as np
import torch
from freetoken.layers.quantization.ggml_types import is_quantized
from freetoken.models.gguf.mmproj import Mmproj, find_mmproj, log_mmproj, mmproj_tensor
from freetoken.models.gguf.reader import (
GgufTensorInfo,
gguf_tensor_index,
gguf_tensor_names,
load_gguf_metadata,
)
from freetoken.models.gguf.tensors import check_scales, fused, matrix
from .config import parse_config
if TYPE_CHECKING:
from freetoken.models.config import ModelConfig
from freetoken.models.gguf.config import GgufConfigShim
# HF config fields the GGUF metadata does not carry (GLM-5.3-Flash's config.json values).
_FAMILY_DEFAULTS = dict(
hidden_act="silu",
attention_bias=False,
# NoPE means no rope on the main attention; the indexer's interleaved rope is a
# served-model fact the GGUF KV predates, and getting it wrong scores the indexer
# keys incorrectly (DSA retrieval degrades to garbage).
indexer_rope_interleave=True,
)
# checkpoint module leaf (glm5_next HF naming, language-model rooted) -> GGUF tensors
_LAYER_LEAVES: dict[str, tuple[str, ...]] = {
# KDA input projections (the six leaves of the fused in_proj)
"self_attn.q_proj": ("attn_q",),
"self_attn.k_proj": ("attn_k",),
"self_attn.v_proj": ("attn_v",),
"self_attn.b_proj": ("ssm_beta",),
"self_attn.f_a_proj": ("ssm_f_a",),
"self_attn.g_a_proj": ("ssm_g_a",),
"self_attn.f_b_proj": ("ssm_f_b",),
"self_attn.g_b_proj": ("ssm_g_b",),
# both layer kinds name their output projection the same
"self_attn.o_proj": ("attn_output",),
# DSA (MLA) projections
"self_attn.q_a_proj": ("attn_q_a",),
"self_attn.q_b_proj": ("attn_q_b",),
"self_attn.kv_a_proj_with_mqa": ("attn_kv_a_mqa",),
# two tensors, dequantized and fused at load: the module keeps its dense bf16 kv_b
# (exempted in the registry's unquantized_modules, so quantized halves are fine too)
"self_attn.kv_b_proj": ("attn_k_b", "attn_v_b"),
# kpool indexer
"self_attn.indexer.wq_b": ("indexer.attn_q_b",),
"self_attn.indexer.wk": ("indexer.attn_k",),
"self_attn.indexer.weights_proj": ("indexer.proj",),
# dense MLP of the leading blocks
"mlp.gate_proj": ("ffn_gate",),
"mlp.up_proj": ("ffn_up",),
"mlp.down_proj": ("ffn_down",),
# shared expert of the MoE blocks
"mlp.shared_experts.gate_proj": ("ffn_gate_shexp",),
"mlp.shared_experts.up_proj": ("ffn_up_shexp",),
"mlp.shared_experts.down_proj": ("ffn_down_shexp",),
# routed experts (GGUF expert banks)
"mlp.experts.0.gate_proj": ("ffn_gate_exps",),
"mlp.experts.0.up_proj": ("ffn_up_exps",),
"mlp.experts.0.down_proj": ("ffn_down_exps",),
}
_TOP_LEAVES: dict[str, tuple[str, ...]] = {
# checkpoint-space names: the language-model root is already applied when to_gguf runs
"model.language_model.embed_tokens": ("token_embd",),
"lm_head": ("output",),
}
_LAYER_RE = re.compile(r"^model\.language_model\.layers\.(\d+)\.(.+)$")
def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
"""Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
if name in _TOP_LEAVES:
return _TOP_LEAVES[name]
m = _LAYER_RE.match(name)
if m is None:
return ()
return tuple(f"blk.{m.group(1)}.{leaf}" for leaf in _LAYER_LEAVES.get(m.group(2), ()))
def _layer_types(shim: "GgufConfigShim", num_layers: int) -> tuple[str, ...]:
"""Per-decoder-layer kinds: DSA where the MLA projections sit, KDA elsewhere, from the
per-block ``attention.head_count_kv`` list (0 on the KDA blocks, 1 on the MLA ones).
Every public glm5-next GGUF carries the list (the generic scalar spelling cannot express
the hybrid; llama.cpp broadcasts it and then fails tensor creation), so anything else is
refused rather than guessed at."""
from .args import DSA_LAYER, KDA_LAYER
per_block = shim.metadata.get(f"{shim.model_type}.attention.head_count_kv")
if not isinstance(per_block, list) or len(per_block) < num_layers:
raise ValueError(
f"{shim.model_path}: glm5-next GGUF lacks the per-block attention.head_count_kv list "
"the KDA/MLA hybrid layout needs (reconvert with a current llama.cpp)"
)
return tuple(DSA_LAYER if int(kv) else KDA_LAYER for kv in per_block[:num_layers])
def parse_gguf_config(shim: "GgufConfigShim") -> ModelConfig:
md = shim.metadata
prefix = shim.model_type
check_scales(shim.model_path)
def kv(key: str, default=None):
return md.get(f"{prefix}.{key}", default)
def need(key: str):
val = kv(key)
if val is None:
raise KeyError(f"missing GGUF metadata key {prefix}.{key}")
return val
# block_count also counts the MTP block a GGUF carries after the decoder
n_mtp = int(kv("nextn_predict_layers", 0) or 0)
num_layers = int(need("block_count")) - n_mtp
layer_types = _layer_types(shim, num_layers)
first_dense = int(need("leading_dense_block_count"))
heads = int(need("attention.head_count"))
qk_nope, v_head = int(need("attention.key_length_mla")), int(need("attention.value_length_mla"))
# NoPE: the main attention carries no rotary dims (rope survives only in the indexer)
if int(kv("rope.dimension_count", 0) or 0) != 0:
raise NotImplementedError("glm5-next GGUF: rotary dims on the main attention are not served")
# KDA head count: one q/k/v head set of kda.head_dim each; the conv channels or the
# beta rows count them, falling back to the shared head_count (the two coincide on
# every release) when the tensor table is gone (metadata-only GGUF).
kda_head_dim = int(need("kda.head_dim"))
index = gguf_tensor_index(shim.model_path)
conv_q = next((i for i in index.values() if i.name.endswith("ssm_conv1d_q.weight")), None)
kda_heads = conv_q.ne[2] // kda_head_dim if conv_q is not None else heads
kpool = int(kv("attention.indexer.kpool", 1) or 1)
# the compressor tensors exist exactly when the indexer pools; a metadata-only GGUF
# keeps the fact only through kpool > 1
names = gguf_tensor_names(shim.model_path)
kpool_compress = any(n.endswith("indexer_compressor_gate.weight") for n in names) if names else kpool > 1
indexer_types_kv = kv("attention.indexer.types")
if indexer_types_kv is not None and not all(indexer_types_kv):
# llama.cpp reads False as "no full indexer on this layer", which the model cannot express
raise NotImplementedError(f"{shim.model_path}: glm5-next GGUF with layers that lack a full indexer is not served")
indexer_types = ("full",) * num_layers
# the clamp is per-layer in the GGUF (and the shared expert carries its own array); the model
# takes one value, so anything non-uniform is refused instead of silently taking the first
clamp = need("swiglu_clamp_exp")
clamp_vals = list(clamp) if isinstance(clamp, list) else [clamp]
clamp_shexp = kv("swiglu_clamp_shexp")
shexp_vals = list(clamp_shexp) if isinstance(clamp_shexp, list) else [clamp_shexp] if clamp_shexp is not None else []
if len(set(map(float, clamp_vals + shexp_vals))) != 1:
raise NotImplementedError(f"{shim.model_path}: non-uniform swiglu_clamp_exp / swiglu_clamp_shexp is not served")
if int(kv("expert_gating_func", 2) or 2) != 2:
raise NotImplementedError(f"{shim.model_path}: expert_gating_func other than sigmoid (2) is not served")
text = SimpleNamespace(
**_FAMILY_DEFAULTS,
num_hidden_layers=num_layers,
layer_types=tuple(layer_types),
mlp_layer_types=("dense",) * first_dense + ("sparse",) * (num_layers - first_dense),
first_k_dense_replace=first_dense,
hidden_size=int(need("embedding_length")),
num_attention_heads=heads,
vocab_size=int(shim.vocab_size),
intermediate_size=int(need("feed_forward_length")),
rms_norm_eps=float(need("attention.layer_norm_rms_epsilon")),
max_position_embeddings=int(need("context_length")),
q_lora_rank=int(need("attention.q_lora_rank")),
kv_lora_rank=int(need("attention.kv_lora_rank")),
qk_nope_head_dim=qk_nope,
qk_rope_head_dim=0,
v_head_dim=v_head,
mla_use_nope=True,
index_n_heads=int(need("attention.indexer.head_count")),
index_head_dim=int(need("attention.indexer.key_length")),
index_topk=int(need("attention.indexer.top_k")),
indexer_types=indexer_types,
index_kpool=kpool,
index_kpool_compress=kpool_compress,
# old converters predate the key; both released checkpoints set it true
index_kpool_always_select_tail=bool(kv("attention.indexer.kpool_select_tail", True)),
linear_attn_config={
"num_heads": kda_heads,
"head_dim": kda_head_dim,
"short_conv_kernel_size": int(need("ssm.conv_kernel")),
"gate_lower_bound": float(need("kda.gate_lower_bound")),
},
mhc=True,
hc_mult=int(need("hyper_connection.count")),
hc_sinkhorn_iters=int(need("hyper_connection.sinkhorn_iterations")),
hc_eps=float(need("hyper_connection.epsilon")),
n_routed_experts=int(need("expert_count")),
n_shared_experts=int(kv("expert_shared_count", 1) or 1),
num_experts_per_tok=int(need("expert_used_count")),
moe_intermediate_size=int(need("expert_feed_forward_length")),
norm_topk_prob=bool(need("expert_weights_norm")),
routed_scaling_factor=float(need("expert_weights_scale")),
n_group=int(kv("expert_group_count", 1) or 1),
topk_group=int(kv("expert_group_used_count", 1) or 1),
tie_word_embeddings=bool(shim.tie_word_embeddings),
swiglu_limit=float(clamp_vals[0]),
)
hf_like = SimpleNamespace(
text_config=text,
model_type="glm5_next",
architectures=list(shim.architectures),
vision_config=getattr(shim, "vision_config", None),
image_token_id=getattr(shim, "image_token_id", None),
)
config = parse_config(hf_like)
return replace(config, expert_quant="gguf")
# ----------------------------------------------------------------------------------------------
# Vision tower (a glm5v mmproj)
# ----------------------------------------------------------------------------------------------
_PROJECTOR = "glm5v"
_IMAGE_TOKEN = "<|image|>"
_PATCH = ("v.patch_embd.weight", "v.patch_embd.weight.1")
# tower / merger tensors in HF layout already; only the patch embedding is split
_TOP = {
"v.patch_embd.bias": "patch_embed.proj.bias",
"v.post_ln.weight": "post_layernorm.weight",
"mm.patch_merger.weight": "downsample.weight",
"mm.patch_merger.bias": "downsample.bias",
"mm.model.fc.weight": "merger.proj.weight",
"mm.post_norm.weight": "merger.post_projection_norm.weight",
"mm.post_norm.bias": "merger.post_projection_norm.bias",
"mm.gate.weight": "merger.gate_proj.weight",
"mm.up.weight": "merger.up_proj.weight",
"mm.down.weight": "merger.down_proj.weight",
}
_BLOCK = {
"attn_qkv": "attn.qkv",
"attn_out": "attn.proj",
"attn_q_norm": "attn.q_norm",
"attn_k_norm": "attn.k_norm",
"ffn_gate": "mlp.gate_proj",
"ffn_up": "mlp.up_proj",
"ffn_down": "mlp.down_proj",
"ln1": "norm1",
"ln2": "norm2",
}
_BLOCK_RE = re.compile(r"^v\.blk\.(\d+)\.(\w+)\.(weight|bias)$")
def gguf_mm_config(text_md: dict[str, Any], mmproj: Mmproj, vocab_size: int, *, model_path: str = "") -> tuple[dict[str, Any], dict[str, Any]]:
"""The HF config.json keys and the preprocessor_config.json of the tower in ``mmproj``, for the text GGUF whose metadata is ``text_md``."""
if mmproj.projector != _PROJECTOR:
raise ValueError(f"mmproj {mmproj.path} is a {mmproj.projector!r} tower; GLM-5.3-Flash needs {_PROJECTOR!r}")
md, index = load_gguf_metadata(mmproj.path), gguf_tensor_index(mmproj.path)
# the converter writes use_silu=True only for the silu tower; a gelu tower carries
# use_gelu or neither key, and the model has no gelu epilogue
if md.get("clip.use_silu") is not True:
raise NotImplementedError(f"mmproj {mmproj.path}: only the silu tower is served")
def kv(key: str):
return md[f"clip.vision.{key}"]
arch = text_md["general.architecture"]
text_hidden = int(text_md[f"{arch}.embedding_length"])
if int(kv("projection_dim")) != text_hidden:
raise ValueError(f"mmproj {mmproj.path} projects to {int(kv('projection_dim'))} but the text model is {text_hidden} wide")
tokens = text_md["tokenizer.ggml.tokens"]
if _IMAGE_TOKEN not in tokens:
raise ValueError(f"the text GGUF tokenizer has no {_IMAGE_TOKEN} token")
# the converter only splits a temporal-2 patch embedding, so a glm5v file always
# carries both halves; one without the second is not this tower
if _PATCH[1] not in index:
raise ValueError(f"mmproj {mmproj.path}: the second patch-embedding half {_PATCH[1]!r} is missing")
temporal = 2
patch, merge = int(kv("patch_size")), int(kv("spatial_merge_size"))
# llama.cpp stores the canvas budget in single-frame pixels; the processor counts
# soft tokens, one per (patch * merge) square of pixels per frame
per_token = patch * patch * merge * merge
budget = {}
for key in ("image_min_pixels", "image_max_pixels"):
pixels = int(kv(key))
if pixels % per_token:
raise ValueError(f"mmproj {mmproj.path}: {key} {pixels} is not a whole number of {per_token}-pixel tokens")
budget[key] = pixels // per_token
vision_config = {
"hidden_act": "silu",
"hidden_size": int(kv("embedding_length")),
"depth": int(kv("block_count")),
"num_heads": int(kv("attention.head_count")),
"intermediate_size": int(kv("feed_forward_length")),
# ggml order [in, out]: the gate's out features are the merger's intermediate width
"projection_intermediate_size": int(index["mm.gate.weight"].ne[1]),
"out_hidden_size": text_hidden,
# ggml order [kw, kh, in, out]
"in_channels": int(index[_PATCH[0]].ne[2]),
"patch_size": patch,
"temporal_patch_size": temporal,
"spatial_merge_size": merge,
"rms_norm_eps": float(kv("attention.layer_norm_epsilon")),
"swiglu_limit": float(kv("swiglu_clamp")),
"attention_bias": "v.blk.0.attn_qkv.bias" in index,
}
hf_fields = {
"vision_config": vision_config,
"image_token_id": tokens.index(_IMAGE_TOKEN),
}
preprocessor = {
"image_processor_type": "Glm5NextImageProcessor",
"patch_size": patch,
"temporal_patch_size": temporal,
"merge_size": merge,
"image_mean": [float(v) for v in kv("image_mean")],
"image_std": [float(v) for v in kv("image_std")],
"min_image_tokens": budget["image_min_pixels"],
"max_image_tokens": budget["image_max_pixels"],
}
return hf_fields, preprocessor
def _vision_name(gguf_name: str) -> str:
if gguf_name in _TOP:
return _TOP[gguf_name]
if (m := _BLOCK_RE.match(gguf_name)) and m[2] in _BLOCK:
return f"blocks.{m[1]}.{_BLOCK[m[2]]}.{m[3]}"
raise ValueError(f"unexpected tensor {gguf_name!r} in a {_PROJECTOR} mmproj")
def iter_mmproj_weights(mmproj: Mmproj) -> Iterator[tuple[str, torch.Tensor]]:
"""The tower state dict under ``visual.``, named as the HF checkpoints load."""
index = gguf_tensor_index(mmproj.path)
slices = [mmproj_tensor(index[n]) for n in _PATCH]
yield "visual.patch_embed.proj.weight", torch.stack(slices, dim=2)
for gguf_name, info in index.items():
if gguf_name not in _PATCH:
yield f"visual.{_vision_name(gguf_name)}", mmproj_tensor(info)
# ----------------------------------------------------------------------------------------------
# Weights
# ----------------------------------------------------------------------------------------------
def _float(info: GgufTensorInfo) -> torch.Tensor:
"""A float tensor in torch order, dequantizing packed ggml rows: dynamic quants quantize
tensors the module consumes as floats (the kv_b halves, the mHC mixers, the compressor gate).
Trailing unit dims in ``ne`` are dropped -- some writers ship the conv kernels 4-D
(llama.cpp's glm5-next loader tries that shape first) and a trailing 1 carries no elements."""
ne = info.ne
while len(ne) > 1 and ne[-1] == 1:
ne = ne[:-1]
if not is_quantized(info.ggml_type):
from freetoken.models.gguf.tensors import float_tensor
return float_tensor(info).reshape(tuple(reversed(ne)))
from freetoken.third_party.gguf import quants
from freetoken.third_party.gguf.constants import GGMLQuantizationType
flat = quants.dequantize(np.array(info.data()), GGMLQuantizationType(info.ggml_type))
return torch.from_numpy(flat).view(*reversed(ne))
def _kv_b(kb: GgufTensorInfo, vb: GgufTensorInfo, num_heads: int, kv_lora: int) -> torch.Tensor:
"""Per-head k [kv_lora, nope] (stored transposed) + v [v, kv_lora] -> the module's fused dense [heads * (nope + v), kv_lora]."""
k = _float(kb).transpose(1, 2) # [heads, nope, kv_lora]
fused_w = torch.cat([k, _float(vb)], dim=1)
assert fused_w.shape[0] == num_heads, (fused_w.shape, num_heads)
return fused_w.reshape(num_heads * fused_w.shape[1], kv_lora).to(torch.bfloat16).contiguous()
def iter_gguf_weights(
model_path: str,
device: torch.device,
*,
include_moe_experts: bool,
include_non_moe: bool,
include_vision: bool = True,
) -> Iterator[tuple[str, torch.Tensor]]:
"""The dense state dict from the GGUF; the routed experts load as offload banks (``freetoken.models.gguf.experts``)."""
if not include_non_moe:
return
from freetoken.utils import cached_load_hf_config
config = parse_gguf_config(cached_load_hf_config(model_path))
vision = include_vision and config.is_multimodal
log_mmproj(model_path, vision)
args = config.glm5_args
index = gguf_tensor_index(model_path)
hidden = config.hidden_size
def t(name: str) -> GgufTensorInfo:
return index[name if name in index else f"{name}.weight"]
def bf16(name: str) -> torch.Tensor:
return _float(t(name)).to(torch.bfloat16)
yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
yield "model.norm.weight", bf16("output_norm")
for layer in range(config.num_layers):
blk, pre = f"blk.{layer}", f"model.layers.{layer}"
yield f"{pre}.input_layernorm.weight", bf16(f"{blk}.attn_norm")
yield f"{pre}.post_attention_layernorm.weight", bf16(f"{blk}.ffn_norm")
# mHC mixing tensors, fp32 on every layer
for sub in ("attn", "ffn"):
for part in ("fn", "base", "scale"):
yield f"{pre}.hc_{sub}_{part}", _float(t(f"{blk}.hc_{sub}_{part}"))
if args.is_kda_layer(layer):
sa = f"{pre}.self_attn"
p, h, d = args.linear_num_heads * args.linear_head_dim, args.linear_num_heads, args.linear_head_dim
group = [t(f"{blk}.{n}") for n in ("attn_q", "attn_k", "attn_v", "ssm_beta", "ssm_f_a", "ssm_g_a")]
yield from fused(f"{sa}.in_proj", group, [matrix(i) for i in group], hidden)
# the GGUF conv kernels already carry the [channels, 1, kernel] singleton dim
conv = torch.cat([_float(t(f"{blk}.ssm_conv1d_{q}")) for q in ("q", "k", "v")], dim=0)
yield f"{sa}.conv1d.weight", conv.to(torch.bfloat16)
for proj, src, k in (("f_b_proj", "ssm_f_b", d), ("g_b_proj", "ssm_g_b", d), ("o_proj", "attn_output", p)):
info = t(f"{blk}.{src}")
yield from fused(f"{sa}.{proj}", [info], [matrix(info)], k)
# the converter stores A_log pre-transformed as -exp(A_log); the kernels want the raw log
yield f"{sa}.A_log", torch.log(-_float(t(f"{blk}.ssm_a")))
yield f"{sa}.dt_bias", _float(t(f"{blk}.ssm_dt.bias"))
yield f"{sa}.o_norm.weight", bf16(f"{blk}.ssm_norm")
else:
sa = f"{pre}.self_attn"
for proj, src, k in (
("q_a_proj", "attn_q_a", hidden),
("q_b_proj", "attn_q_b", args.q_lora_rank),
("kv_a_proj_with_mqa", "attn_kv_a_mqa", hidden),
("o_proj", "attn_output", args.num_heads * args.v_head_dim),
):
info = t(f"{blk}.{src}")
yield from fused(f"{sa}.{proj}", [info], [matrix(info)], k)
yield f"{sa}.kv_b_proj.weight", _kv_b(
t(f"{blk}.attn_k_b"), t(f"{blk}.attn_v_b"), args.num_heads, args.kv_lora_rank
)
yield f"{sa}.q_a_layernorm.weight", bf16(f"{blk}.attn_q_a_norm")
yield f"{sa}.kv_a_layernorm.weight", bf16(f"{blk}.attn_kv_a_norm")
# wq_b serves its checkpoint quant (packed); wk / weights_proj are dense modules
# (glm_moe_dsa's indexer reasoning), dequantized at load when the file quantized them
yield from fused(f"{sa}.indexer.wq_b", [t(f"{blk}.indexer.attn_q_b")], [matrix(t(f"{blk}.indexer.attn_q_b"))], args.q_lora_rank)
yield f"{sa}.indexer.wk.weight", _float(t(f"{blk}.indexer.attn_k")).to(torch.bfloat16)
yield f"{sa}.indexer.weights_proj.weight", _float(t(f"{blk}.indexer.proj")).to(torch.bfloat16)
yield f"{sa}.indexer.k_norm.weight", bf16(f"{blk}.indexer.k_norm")
yield f"{sa}.indexer.k_norm.bias", bf16(f"{blk}.indexer.k_norm.bias")
yield f"{sa}.indexer.index_kpool_compress_gate", bf16(f"{blk}.indexer_compressor_gate")
yield f"{sa}.indexer.index_kpool_compress_ape", _float(t(f"{blk}.indexer_compressor_ape"))
mlp = f"{pre}.mlp"
if layer < config.first_k_dense_replace:
for proj, src, k in (
("gate_proj", "ffn_gate", hidden),
("up_proj", "ffn_up", hidden),
("down_proj", "ffn_down", config.intermediate_size),
):
info = t(f"{blk}.{src}")
yield from fused(f"{mlp}.{proj}", [info], [matrix(info)], k)
else:
yield f"{mlp}.gate.weight", bf16(f"{blk}.ffn_gate_inp")
yield f"{mlp}.e_score_correction_bias", _float(t(f"{blk}.exp_probs_b.bias"))
shared_i = config.moe_intermediate_size * max(1, config.n_shared_experts)
for proj, src, k in (
("gate_proj", "ffn_gate_shexp", hidden),
("up_proj", "ffn_up_shexp", hidden),
("down_proj", "ffn_down_shexp", shared_i),
):
info = t(f"{blk}.{src}")
yield from fused(f"{mlp}.shared_experts.{proj}", [info], [matrix(info)], k)
if vision:
yield from iter_mmproj_weights(find_mmproj(model_path))
__all__ = ["gguf_mm_config", "iter_gguf_weights", "iter_mmproj_weights", "parse_gguf_config", "to_gguf"]