FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
feat(gguf): serve Llama and Mistral llama-arch GGUF checkpoints (#722)
Xiaoze Fan21d9578bde22
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.pyindex 8c5d32ba..040236cb 100644--- a/python/freetoken/kernel/aot_models.py+++ b/python/freetoken/kernel/aot_models.py@@ -375,6 +375,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (architecture="LlamaForCausalLM",hidden_size=4096,kv_groups=((8, 128),),+ arch_aliases=("LlamaGGUFForCausalLM",),),AotModel(name="Qwen/Qwen2-7B",diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.pyindex 518acfe4..9f860e66 100644--- a/python/freetoken/models/gguf/config.py+++ b/python/freetoken/models/gguf/config.py@@ -18,6 +18,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names# reuses the model classes but a GGUF parse_config / iter_weights).GGUF_ARCH_TO_REGISTRY: dict[str, str] = {"gemma4": "Gemma4GGUFForCausalLM",+ "llama": "LlamaGGUFForCausalLM","qwen2": "Qwen2GGUFForCausalLM","qwen3": "Qwen3GGUFForCausalLM","qwen3moe": "Qwen3MoeGGUFForCausalLM",diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.pyindex af715c65..d6c53fa4 100644--- a/python/freetoken/models/gguf/tokenizer.py+++ b/python/freetoken/models/gguf/tokenizer.py@@ -6,6 +6,8 @@ So we run only its tokenizer half: ``get_gguf_tokenizer`` and ``convert_gguf_tokfrom __future__ import annotations+import json+from .reader import load_gguf_metadata# end-of-generation token names from llama.cpp (llama_vocab::impl::load in src/llama-vocab.cpp): a GGUF stores one eos id and no stop list@@ -15,6 +17,57 @@ _EOG_NAMES = frozenset(("<eos>", "<turn|>", "<|tool_response>", "<\uff5cend\u2581of\u2581sentence\uff5c>", "[e~[", "<|ifm|im_end|>",))+# BPE pre-tokenizers that llama.cpp adds a bos for when the file has no add_bos_token key (llama_vocab::impl::load)+_BOS_PRE = frozenset((+ "llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon",+))+++def _add_bos(meta: dict) -> bool:+ if "tokenizer.ggml.add_bos_token" in meta:+ return bool(meta["tokenizer.ggml.add_bos_token"])+ model = meta.get("tokenizer.ggml.model")+ return model == "llama" or (model == "gpt2" and meta.get("tokenizer.ggml.pre") in _BOS_PRE)+++def _fix_added_tokens(fast, meta: dict):+ """Register the control and user-defined tokens that transformers skipped at their file ids, and drop added tokens past the file's vocabulary."""+ from tokenizers import Tokenizer+ from transformers.convert_slow_tokenizer import bytes_to_unicode++ tokens = meta["tokenizer.ggml.tokens"]+ spelling = bytes_to_unicode()+ alphabet = set(spelling.values())+ # transformers' byte-level vocabulary spells a token with characters outside the byte alphabet in bytes and then skips it (DeepSeek's full-width-bar tokens, MiniMax's tokens with spaces)+ skipped = [+ (i, text, kind == 3)+ for i, (text, kind) in enumerate(zip(tokens, meta.get("tokenizer.ggml.token_type") or ()))+ if kind in (3, 4) and meta.get("tokenizer.ggml.model") == "gpt2" and not set(text) <= alphabet and fast.token_to_id(text) != i+ ]+ # its GPT2Tokenizer also adds a default <|endoftext|> when the vocabulary lacks one+ if not skipped and fast.get_vocab_size(with_added_tokens=True) <= len(tokens):+ return fast+ # a piece the merges use or build stays a plain BPE token, since renaming it breaks the merges (unsloth's MiniMax-M3 marks U+FFFD as control)+ pieces = {p for m in meta.get("tokenizer.ggml.merges") or () for p in (*m.split(" "), m.replace(" ", ""))}+ state = json.loads(fast.to_str())+ vocab = state["model"]["vocab"]+ added = [t for t in state["added_tokens"] if t["id"] < len(tokens)]+ for i, text, special in skipped:+ spelled = "".join(spelling[b] for b in text.encode("utf-8"))+ if spelled in pieces or vocab.get(spelled) != i:+ continue+ # renamed so the added token keeps the file id instead of taking a new one+ vocab[text] = vocab.pop(spelled)+ added.append({"id": i, "content": text, "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": special})+ state["added_tokens"] = sorted(added, key=lambda t: t["id"])+ return Tokenizer.from_str(json.dumps(state))+++def _known_specials(special: dict, fast) -> dict:+ """Drop a bos/eos/unk/pad the vocabulary lacks, which PreTrainedTokenizerFast would add past it (unsloth's MiniMax-M3 unk is U+FFFD stored decoded)."""+ vocab = fast.get_vocab(with_added_tokens=True)+ return {k: v for k, v in special.items() if k == "chat_template" or v is None or v in vocab}+def load_gguf_tokenizer(model_path: str):from transformers import PreTrainedTokenizerFast@@ -23,11 +76,12 @@ def load_gguf_tokenizer(model_path: str):arch, tok_dict, special = get_gguf_tokenizer(model_path)fast, _ = convert_gguf_tokenizer(arch, tok_dict)meta = load_gguf_metadata(model_path)+ fast = _fix_added_tokens(fast, meta)# special: bos/eos/unk/pad as the file names them (None when absent) and the chat templatereturn PreTrainedTokenizerFast(tokenizer_object=fast,- **special,- add_bos_token=bool(meta.get("tokenizer.ggml.add_bos_token")),+ **_known_specials(special, fast),+ add_bos_token=_add_bos(meta),add_eos_token=bool(meta.get("tokenizer.ggml.add_eos_token")),)diff --git a/python/freetoken/models/llama/__init__.py b/python/freetoken/models/llama/__init__.pyindex 9ffad0e9..31ffa447 100644--- a/python/freetoken/models/llama/__init__.py+++ b/python/freetoken/models/llama/__init__.py@@ -1,5 +1,6 @@from .config import parse_config+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguffrom .model import LlamaForCausalLMfrom .weight import iter_weights-__all__ = ["LlamaForCausalLM", "parse_config", "iter_weights"]+__all__ = ["LlamaForCausalLM", "parse_config", "iter_weights", "iter_gguf_weights", "parse_gguf_config", "to_gguf"]diff --git a/python/freetoken/models/llama/gguf.py b/python/freetoken/models/llama/gguf.pynew file mode 100644index 00000000..09d02627--- /dev/null+++ b/python/freetoken/models/llama/gguf.py@@ -0,0 +1,191 @@+"""Llama and Mistral from a llama.cpp ``llama`` GGUF: config from the GGUF metadata, tensors by name.++llama.cpp runs interleaved rope on this arch, so its converter (conversion/llama.py) permutes the rows of each q and k head; the loader undoes that, and the model keeps HF's NeoX rope.+The RMSNorm weights are the HF ones.+"""++from __future__ import annotations++import re+from types import SimpleNamespace+from typing import TYPE_CHECKING, Iterator++import torch++from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index+from freetoken.models.gguf.tensors import float_tensor, fused, matrix++from .config import parse_config++if TYPE_CHECKING:+ from freetoken.models.config import ModelConfig+ from freetoken.models.gguf.config import GgufConfigShim++_ARCH = "llama"+_LAYER_LEAVES = {+ "self_attn.q_proj": "attn_q",+ "self_attn.k_proj": "attn_k",+ "self_attn.v_proj": "attn_v",+ "self_attn.o_proj": "attn_output",+ "mlp.gate_proj": "ffn_gate",+ "mlp.up_proj": "ffn_up",+ "mlp.down_proj": "ffn_down",+}+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")+++def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""+ if name in _TOP_LEAVES:+ return (_TOP_LEAVES[name],)+ m = _LAYER_RE.match(name)+ if m is None:+ return ()+ blk, leaf = f"blk.{m.group(1)}", m.group(2)+ if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:+ return (f"{blk}.attn_qkv",)+ tensor = _LAYER_LEAVES.get(leaf)+ return (f"{blk}.{tensor}",) if tensor is not None else ()+++def _check_supported(shim: GgufConfigShim) -> None:+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")+ names = sorted(gguf_tensor_index(shim.model_path))+ scaled = [n for n in names if n.endswith((".scale", ".input_scale"))]+ if scaled:+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")+ # the llama arch also carries Mixtral experts and optional biases, which the llama model does not have+ extra = [n for n in names if n.endswith(".bias") or ".ffn_gate_inp." in n]+ if extra:+ raise NotImplementedError(f"{shim.model_path}: llama GGUF tensors with biases or MoE experts are not supported (e.g. {extra[0]})")+++def _rope_scaling(shim: GgufConfigShim, base: float, head_dim: int) -> dict | None:+ """The HF rope scaling of the file: none, or Llama 3's, which the converter stores only as the ``rope_freqs`` divisors."""+ md = shim.metadata+ scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")+ # llama.cpp scales linearly by rope.scaling.factor (or the older rope.scale_linear) when the type key is absent+ factor = float(md.get(f"{_ARCH}.rope.scaling.factor", md.get(f"{_ARCH}.rope.scale_linear", 0.0)))+ attn_factor = float(md.get(f"{_ARCH}.rope.scaling.attn_factor", 1.0))+ if scaling not in ("none", "linear") or factor not in (0.0, 1.0) or attn_factor != 1.0:+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} (factor {factor}, attn_factor {attn_factor}) is not supported")+ index = gguf_tensor_index(shim.model_path)+ if not index:+ raise NotImplementedError(f"{shim.model_path}: a metadata-only llama GGUF has no rope_freqs tensor to tell Llama 3 rope from plain rope")+ info = index.get("rope_freqs.weight")+ if info is None:+ return None+ from freetoken.layers.rotary import _rope_frequencies++ divisors = float_tensor(info).float()+ # Llama 3.1-3.3 differ only in factor+ llama3 = {+ "rope_type": "llama3",+ "factor": float(divisors.max()),+ "low_freq_factor": 1.0,+ "high_freq_factor": 4.0,+ "original_max_position_embeddings": 8192,+ }+ plain, _ = _rope_frequencies(head_dim, head_dim, base, None)+ scaled, _ = _rope_frequencies(head_dim, head_dim, base, llama3)+ if not torch.allclose(plain / scaled, divisors, rtol=1e-5):+ raise NotImplementedError(f"{shim.model_path}: rope_freqs does not match Llama 3 rope scaling")+ return llama3+++def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""+ md = shim.metadata++ def kv(key: str):+ return md[f"{_ARCH}.{key}"]++ _check_supported(shim)+ hidden, heads = int(kv("embedding_length")), int(kv("attention.head_count"))+ # Mistral-7B files carry no key_length+ head_dim = int(md.get(f"{_ARCH}.attention.key_length", hidden // heads))+ q = gguf_tensor_index(shim.model_path).get("blk.0.attn_q.weight")+ # early Mistral-Nemo conversions lack key_length, so hidden / heads gives the wrong head_dim; llama.cpp rejects them too+ if q is not None and q.ne[1] != heads * head_dim:+ raise NotImplementedError(f"{shim.model_path}: attn_q has {q.ne[1]} rows, not head_count x head_dim = {heads * head_dim}")+ if int(md.get(f"{_ARCH}.rope.dimension_count", head_dim)) != head_dim:+ raise NotImplementedError(f"{shim.model_path}: partial rope is not supported")+ base = float(md.get(f"{_ARCH}.rope.freq_base", 10000.0))+ return SimpleNamespace(+ model_type="llama",+ architectures=list(shim.architectures),+ hidden_act="silu",+ hidden_size=hidden,+ num_hidden_layers=int(kv("block_count")),+ num_attention_heads=heads,+ num_key_value_heads=int(md.get(f"{_ARCH}.attention.head_count_kv", heads)),+ head_dim=head_dim,+ rope_theta=base,+ rope_scaling=_rope_scaling(shim, base, head_dim),+ max_position_embeddings=int(kv("context_length")),+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),+ vocab_size=int(shim.vocab_size),+ tie_word_embeddings=bool(shim.tie_word_embeddings),+ intermediate_size=int(kv("feed_forward_length")),+ )+++def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:+ return parse_config(_hf_config(shim))+++def _unpermute(rows: torch.Tensor, heads: int) -> torch.Tensor:+ """Undo the converter's rope permute of one q or k matrix (``LlamaModel.permute``); it moves whole rows, so packed ggml rows work too."""+ return rows.reshape(heads, rows.shape[0] // heads // 2, 2, *rows.shape[1:]).swapaxes(1, 2).reshape(rows.shape)+++def iter_gguf_weights(+ model_path: str,+ device: torch.device,+ *,+ include_moe_experts: bool,+ include_non_moe: bool,+) -> Iterator[tuple[str, torch.Tensor]]:+ """The state dict from the GGUF."""+ if not include_non_moe:+ return+ from freetoken.utils import cached_load_hf_config++ hf = _hf_config(cached_load_hf_config(model_path))+ index = gguf_tensor_index(model_path)+ hidden, heads, kv_heads = hf.hidden_size, hf.num_attention_heads, hf.num_key_value_heads+ q_rows, kv_rows = heads * hf.head_dim, kv_heads * hf.head_dim++ def t(name: str) -> GgufTensorInfo:+ return index[name if name in index else f"{name}.weight"]++ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)+ if not hf.tie_word_embeddings:+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)+ yield "model.norm.weight", float_tensor(t("output_norm"))++ for layer in range(hf.num_hidden_layers):+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))+ sa = f"{pre}.self_attn"+ # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix+ if f"{blk}.attn_qkv.weight" in index:+ qkv = [t(f"{blk}.attn_qkv")] * 3+ q, k, v = matrix(qkv[0]).split([q_rows, kv_rows, kv_rows])+ else:+ qkv = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]+ q, k, v = (matrix(p) for p in qkv)+ yield from fused(f"{sa}.qkv_proj", qkv, [_unpermute(q, heads), _unpermute(k, kv_heads), v], hidden)+ out = t(f"{blk}.attn_output")+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))+ mlp = f"{pre}.mlp"+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))+ yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)+ yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))+++__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]diff --git a/python/freetoken/models/llama/model.py b/python/freetoken/models/llama/model.pyindex eb7ff28f..107d9e0a 100644--- a/python/freetoken/models/llama/model.py+++ b/python/freetoken/models/llama/model.py@@ -48,6 +48,8 @@ class LlamaModel(BaseOP):self.embed_tokens = VocabParallelEmbedding(num_embeddings=config.vocab_size,embedding_dim=config.hidden_size,+ quant_config=config.quant,+ prefix=f"{prefix}.embed_tokens",)self.layers = OPList([diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.pyindex 263b5f93..057e2ae6 100644--- a/python/freetoken/models/register.py+++ b/python/freetoken/models/register.py@@ -95,6 +95,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {"LlamaForCausalLM",packed_modules_mapping=_DENSE_PACKED,),+ # llama.cpp llama GGUF (Llama, Mistral): the llama model classes with every tensor in its packed ggml type+ "LlamaGGUFForCausalLM": ModelSpec(+ "freetoken.models.llama",+ "LlamaForCausalLM",+ parse_config="parse_gguf_config",+ iter_weights="iter_gguf_weights",+ packed_modules_mapping=_DENSE_PACKED,+ ),"Qwen2ForCausalLM": ModelSpec("freetoken.models.qwen2","Qwen2ForCausalLM",