FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
from __future__ import annotations
import re
from dataclasses import replace
from types import SimpleNamespace
from typing import Iterator
import torch
from freetoken.models.gguf.reader import gguf_tensor_index
from freetoken.models.gguf.config import GgufConfigShim
from freetoken.models.config import ModelConfig
from freetoken.models.gguf.tensors import check_scales, float_tensor, fused, matrix
from freetoken.utils import cached_load_hf_config
from .config import parse_config
_LAYER = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
_LEAVES = {
"self_attn.q_proj": "attn_q", "self_attn.k_proj": "attn_k",
"self_attn.v_proj": "attn_v", "self_attn.o_proj": "attn_output",
"mlp.gate": "ffn_gate_inp",
**{f"mlp.{p}_proj": f"ffn_{p}" for p in ("gate", "up", "down")},
**{f"mlp.shared_expert.{p}_proj": f"ffn_{p}_shexp" for p in ("gate", "up", "down")},
**{f"mlp.experts.0.{p}_proj": f"ffn_{p}_exps" for p in ("gate", "up", "down")},
}
def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
top = {"model.embed_tokens": "token_embd", "lm_head": "output"}
if name in top:
return (top[name],)
match = _LAYER.match(name)
if match is None:
return ()
leaf = _LEAVES.get(match[2])
return (f"blk.{match[1]}.{leaf}",) if leaf else ()
def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
md = shim.metadata
def kv(name):
return md[f"flint.{name}"]
if md.get("flint.rope.scaling.type", "none") != "none":
raise NotImplementedError("Flint GGUF RoPE scaling is not supported")
if int(kv("expert_gating_func")) != 2 or float(kv("expert_weights_scale")) != 1.0:
raise NotImplementedError("Flint GGUF requires unscaled sigmoid routing")
check_scales(shim.model_path)
swa = tuple(kv("attention.sliding_window_pattern"))
rope = tuple(kv("attention.rope_pattern"))
if len(rope) != int(kv("block_count")):
raise ValueError("Flint GGUF requires one RoPE flag per layer")
return SimpleNamespace(
architectures=list(shim.architectures), model_type="flint",
num_hidden_layers=int(kv("block_count")), hidden_size=int(kv("embedding_length")),
intermediate_size=int(kv("feed_forward_length")),
num_attention_heads=int(kv("attention.head_count")),
num_key_value_heads=int(kv("attention.head_count_kv")),
head_dim=int(kv("attention.key_length")), vocab_size=shim.vocab_size,
max_position_embeddings=int(kv("context_length")),
rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
rope_theta=float(kv("rope.freq_base")),
layer_types=["sliding_attention" if v else "full_attention" for v in swa],
no_rope_layer_indices=[i for i, v in enumerate(rope) if not v],
mlp_only_layers=list(range(int(kv("leading_dense_block_count")))),
num_experts=int(kv("expert_count")), num_experts_per_tok=int(kv("expert_used_count")),
moe_intermediate_size=int(kv("expert_feed_forward_length")),
shared_expert_intermediate_size=int(kv("expert_shared_feed_forward_length")),
norm_topk_prob=bool(kv("expert_weights_norm")),
sliding_window=int(kv("attention.sliding_window")),
hidden_act="silu", router_score_function="sigmoid", softmax_type="learnable",
tie_word_embeddings=shim.tie_word_embeddings,
)
def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
return replace(parse_config(_hf_config(shim)), expert_quant="gguf")
def iter_gguf_weights(
model_path: str, device: torch.device, *, include_moe_experts: bool, include_non_moe: bool,
) -> Iterator[tuple[str, torch.Tensor]]:
if not include_non_moe:
return
hf = _hf_config(cached_load_hf_config(model_path))
index = gguf_tensor_index(model_path)
def t(name):
return index[name if name in index else f"{name}.weight"]
def linear(module, names):
infos = [t(n) for n in names]
yield from fused(module, infos, [matrix(v) for v in infos], int(infos[0].ne[0]))
yield from linear("model.embed_tokens", ["token_embd"])
if not hf.tie_word_embeddings:
yield from linear("lm_head", ["output"])
yield "model.norm.weight", float_tensor(t("output_norm"))
for layer in range(hf.num_hidden_layers):
blk, pre = f"blk.{layer}", f"model.layers.{layer}"
yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))
sa = f"{pre}.self_attn"
yield from linear(f"{sa}.qkv_proj", [f"{blk}.attn_{p}" for p in ("q", "k", "v")])
yield from linear(f"{sa}.o_proj", [f"{blk}.attn_output"])
for p in ("q", "k"):
yield f"{sa}.{p}_norm.weight", float_tensor(t(f"{blk}.attn_{p}_norm"))
yield f"{sa}.softmax_offset", float_tensor(t(f"{blk}.attn_sinks")).float()
mlp = f"{pre}.mlp"
if layer in hf.mlp_only_layers:
yield from linear(f"{mlp}.gate_up_proj", [f"{blk}.ffn_{p}" for p in ("gate", "up")])
yield from linear(f"{mlp}.down_proj", [f"{blk}.ffn_down"])
else:
yield f"{mlp}.gate.weight", float_tensor(t(f"{blk}.ffn_gate_inp"))
bias = index.get(f"{blk}.exp_probs_b.bias")
yield f"{mlp}.gate.expert_bias", (
float_tensor(bias).float() if bias is not None
else torch.zeros(hf.num_experts, device=device, dtype=torch.float32)
)
yield from linear(f"{mlp}.shared_expert.gate_up_proj", [f"{blk}.ffn_{p}_shexp" for p in ("gate", "up")])
yield from linear(f"{mlp}.shared_expert.down_proj", [f"{blk}.ffn_down_shexp"])