Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

feat(gguf): serve Llama and Mistral llama-arch GGUF checkpoints (#722)

Xiaoze Fan21d9578bde22
diff --git a/python/freetoken/kernel/aot_models.py b/python/freetoken/kernel/aot_models.py
index 8c5d32ba..040236cb 100644
--- a/python/freetoken/kernel/aot_models.py
+++ b/python/freetoken/kernel/aot_models.py
@@ -375,6 +375,7 @@ SUPPORTED_MODELS: tuple[AotModel, ...] = (
architecture="LlamaForCausalLM",
hidden_size=4096,
kv_groups=((8, 128),),
+ arch_aliases=("LlamaGGUFForCausalLM",),
),
AotModel(
name="Qwen/Qwen2-7B",
diff --git a/python/freetoken/models/gguf/config.py b/python/freetoken/models/gguf/config.py
index 518acfe4..9f860e66 100644
--- a/python/freetoken/models/gguf/config.py
+++ b/python/freetoken/models/gguf/config.py
@@ -18,6 +18,7 @@ from .reader import gguf_architecture, load_gguf_metadata, gguf_tensor_names
# reuses the model classes but a GGUF parse_config / iter_weights).
GGUF_ARCH_TO_REGISTRY: dict[str, str] = {
"gemma4": "Gemma4GGUFForCausalLM",
+ "llama": "LlamaGGUFForCausalLM",
"qwen2": "Qwen2GGUFForCausalLM",
"qwen3": "Qwen3GGUFForCausalLM",
"qwen3moe": "Qwen3MoeGGUFForCausalLM",
diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py
index af715c65..d6c53fa4 100644
--- a/python/freetoken/models/gguf/tokenizer.py
+++ b/python/freetoken/models/gguf/tokenizer.py
@@ -6,6 +6,8 @@ So we run only its tokenizer half: ``get_gguf_tokenizer`` and ``convert_gguf_tok
from __future__ import annotations
+import json
+
from .reader import load_gguf_metadata
# end-of-generation token names from llama.cpp (llama_vocab::impl::load in src/llama-vocab.cpp): a GGUF stores one eos id and no stop list
@@ -15,6 +17,57 @@ _EOG_NAMES = frozenset((
"<eos>", "<turn|>", "<|tool_response>", "<\uff5cend\u2581of\u2581sentence\uff5c>", "[e~[", "<|ifm|im_end|>",
))
+# BPE pre-tokenizers that llama.cpp adds a bos for when the file has no add_bos_token key (llama_vocab::impl::load)
+_BOS_PRE = frozenset((
+ "llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon",
+))
+
+
+def _add_bos(meta: dict) -> bool:
+ if "tokenizer.ggml.add_bos_token" in meta:
+ return bool(meta["tokenizer.ggml.add_bos_token"])
+ model = meta.get("tokenizer.ggml.model")
+ return model == "llama" or (model == "gpt2" and meta.get("tokenizer.ggml.pre") in _BOS_PRE)
+
+
+def _fix_added_tokens(fast, meta: dict):
+ """Register the control and user-defined tokens that transformers skipped at their file ids, and drop added tokens past the file's vocabulary."""
+ from tokenizers import Tokenizer
+ from transformers.convert_slow_tokenizer import bytes_to_unicode
+
+ tokens = meta["tokenizer.ggml.tokens"]
+ spelling = bytes_to_unicode()
+ alphabet = set(spelling.values())
+ # transformers' byte-level vocabulary spells a token with characters outside the byte alphabet in bytes and then skips it (DeepSeek's full-width-bar tokens, MiniMax's tokens with spaces)
+ skipped = [
+ (i, text, kind == 3)
+ for i, (text, kind) in enumerate(zip(tokens, meta.get("tokenizer.ggml.token_type") or ()))
+ if kind in (3, 4) and meta.get("tokenizer.ggml.model") == "gpt2" and not set(text) <= alphabet and fast.token_to_id(text) != i
+ ]
+ # its GPT2Tokenizer also adds a default <|endoftext|> when the vocabulary lacks one
+ if not skipped and fast.get_vocab_size(with_added_tokens=True) <= len(tokens):
+ return fast
+ # a piece the merges use or build stays a plain BPE token, since renaming it breaks the merges (unsloth's MiniMax-M3 marks U+FFFD as control)
+ pieces = {p for m in meta.get("tokenizer.ggml.merges") or () for p in (*m.split(" "), m.replace(" ", ""))}
+ state = json.loads(fast.to_str())
+ vocab = state["model"]["vocab"]
+ added = [t for t in state["added_tokens"] if t["id"] < len(tokens)]
+ for i, text, special in skipped:
+ spelled = "".join(spelling[b] for b in text.encode("utf-8"))
+ if spelled in pieces or vocab.get(spelled) != i:
+ continue
+ # renamed so the added token keeps the file id instead of taking a new one
+ vocab[text] = vocab.pop(spelled)
+ added.append({"id": i, "content": text, "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": special})
+ state["added_tokens"] = sorted(added, key=lambda t: t["id"])
+ return Tokenizer.from_str(json.dumps(state))
+
+
+def _known_specials(special: dict, fast) -> dict:
+ """Drop a bos/eos/unk/pad the vocabulary lacks, which PreTrainedTokenizerFast would add past it (unsloth's MiniMax-M3 unk is U+FFFD stored decoded)."""
+ vocab = fast.get_vocab(with_added_tokens=True)
+ return {k: v for k, v in special.items() if k == "chat_template" or v is None or v in vocab}
+
def load_gguf_tokenizer(model_path: str):
from transformers import PreTrainedTokenizerFast
@@ -23,11 +76,12 @@ def load_gguf_tokenizer(model_path: str):
arch, tok_dict, special = get_gguf_tokenizer(model_path)
fast, _ = convert_gguf_tokenizer(arch, tok_dict)
meta = load_gguf_metadata(model_path)
+ fast = _fix_added_tokens(fast, meta)
# special: bos/eos/unk/pad as the file names them (None when absent) and the chat template
return PreTrainedTokenizerFast(
tokenizer_object=fast,
- **special,
- add_bos_token=bool(meta.get("tokenizer.ggml.add_bos_token")),
+ **_known_specials(special, fast),
+ add_bos_token=_add_bos(meta),
add_eos_token=bool(meta.get("tokenizer.ggml.add_eos_token")),
)
diff --git a/python/freetoken/models/llama/__init__.py b/python/freetoken/models/llama/__init__.py
index 9ffad0e9..31ffa447 100644
--- a/python/freetoken/models/llama/__init__.py
+++ b/python/freetoken/models/llama/__init__.py
@@ -1,5 +1,6 @@
from .config import parse_config
+from .gguf import iter_gguf_weights, parse_gguf_config, to_gguf
from .model import LlamaForCausalLM
from .weight import iter_weights
-__all__ = ["LlamaForCausalLM", "parse_config", "iter_weights"]
+__all__ = ["LlamaForCausalLM", "parse_config", "iter_weights", "iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/llama/gguf.py b/python/freetoken/models/llama/gguf.py
new file mode 100644
index 00000000..09d02627
--- /dev/null
+++ b/python/freetoken/models/llama/gguf.py
@@ -0,0 +1,191 @@
+"""Llama and Mistral from a llama.cpp ``llama`` GGUF: config from the GGUF metadata, tensors by name.
+
+llama.cpp runs interleaved rope on this arch, so its converter (conversion/llama.py) permutes the rows of each q and k head; the loader undoes that, and the model keeps HF's NeoX rope.
+The RMSNorm weights are the HF ones.
+"""
+
+from __future__ import annotations
+
+import re
+from types import SimpleNamespace
+from typing import TYPE_CHECKING, Iterator
+
+import torch
+
+from freetoken.models.gguf.reader import GgufTensorInfo, gguf_tensor_index
+from freetoken.models.gguf.tensors import float_tensor, fused, matrix
+
+from .config import parse_config
+
+if TYPE_CHECKING:
+ from freetoken.models.config import ModelConfig
+ from freetoken.models.gguf.config import GgufConfigShim
+
+_ARCH = "llama"
+_LAYER_LEAVES = {
+ "self_attn.q_proj": "attn_q",
+ "self_attn.k_proj": "attn_k",
+ "self_attn.v_proj": "attn_v",
+ "self_attn.o_proj": "attn_output",
+ "mlp.gate_proj": "ffn_gate",
+ "mlp.up_proj": "ffn_up",
+ "mlp.down_proj": "ffn_down",
+}
+_QKV_LEAVES = ("self_attn.q_proj", "self_attn.k_proj", "self_attn.v_proj")
+_TOP_LEAVES = {"model.embed_tokens": "token_embd", "lm_head": "output"}
+_LAYER_RE = re.compile(r"^model\.layers\.(\d+)\.(.+)$")
+
+
+def to_gguf(name: str, tensors: dict[str, int]) -> tuple[str, ...]:
+ """Checkpoint module name -> the GGUF tensors (without ``.weight``) that store it; empty for modules the GGUF has no matrix for."""
+ if name in _TOP_LEAVES:
+ return (_TOP_LEAVES[name],)
+ m = _LAYER_RE.match(name)
+ if m is None:
+ return ()
+ blk, leaf = f"blk.{m.group(1)}", m.group(2)
+ if leaf in _QKV_LEAVES and f"{blk}.attn_qkv.weight" in tensors:
+ return (f"{blk}.attn_qkv",)
+ tensor = _LAYER_LEAVES.get(leaf)
+ return (f"{blk}.{tensor}",) if tensor is not None else ()
+
+
+def _check_supported(shim: GgufConfigShim) -> None:
+ if any(k.startswith("prism.hadamard.") for k in shim.metadata):
+ raise NotImplementedError(f"{shim.model_path}: Hadamard-rotated (prism) GGUF weights are not supported")
+ names = sorted(gguf_tensor_index(shim.model_path))
+ scaled = [n for n in names if n.endswith((".scale", ".input_scale"))]
+ if scaled:
+ raise NotImplementedError(f"{shim.model_path}: GGUF tensors with per-tensor scales are not supported (e.g. {scaled[0]})")
+ # the llama arch also carries Mixtral experts and optional biases, which the llama model does not have
+ extra = [n for n in names if n.endswith(".bias") or ".ffn_gate_inp." in n]
+ if extra:
+ raise NotImplementedError(f"{shim.model_path}: llama GGUF tensors with biases or MoE experts are not supported (e.g. {extra[0]})")
+
+
+def _rope_scaling(shim: GgufConfigShim, base: float, head_dim: int) -> dict | None:
+ """The HF rope scaling of the file: none, or Llama 3's, which the converter stores only as the ``rope_freqs`` divisors."""
+ md = shim.metadata
+ scaling = md.get(f"{_ARCH}.rope.scaling.type", "none")
+ # llama.cpp scales linearly by rope.scaling.factor (or the older rope.scale_linear) when the type key is absent
+ factor = float(md.get(f"{_ARCH}.rope.scaling.factor", md.get(f"{_ARCH}.rope.scale_linear", 0.0)))
+ attn_factor = float(md.get(f"{_ARCH}.rope.scaling.attn_factor", 1.0))
+ if scaling not in ("none", "linear") or factor not in (0.0, 1.0) or attn_factor != 1.0:
+ raise NotImplementedError(f"{shim.model_path}: rope scaling {scaling!r} (factor {factor}, attn_factor {attn_factor}) is not supported")
+ index = gguf_tensor_index(shim.model_path)
+ if not index:
+ raise NotImplementedError(f"{shim.model_path}: a metadata-only llama GGUF has no rope_freqs tensor to tell Llama 3 rope from plain rope")
+ info = index.get("rope_freqs.weight")
+ if info is None:
+ return None
+ from freetoken.layers.rotary import _rope_frequencies
+
+ divisors = float_tensor(info).float()
+ # Llama 3.1-3.3 differ only in factor
+ llama3 = {
+ "rope_type": "llama3",
+ "factor": float(divisors.max()),
+ "low_freq_factor": 1.0,
+ "high_freq_factor": 4.0,
+ "original_max_position_embeddings": 8192,
+ }
+ plain, _ = _rope_frequencies(head_dim, head_dim, base, None)
+ scaled, _ = _rope_frequencies(head_dim, head_dim, base, llama3)
+ if not torch.allclose(plain / scaled, divisors, rtol=1e-5):
+ raise NotImplementedError(f"{shim.model_path}: rope_freqs does not match Llama 3 rope scaling")
+ return llama3
+
+
+def _hf_config(shim: GgufConfigShim) -> SimpleNamespace:
+ """The HF config fields ``parse_config`` reads, from the GGUF metadata."""
+ md = shim.metadata
+
+ def kv(key: str):
+ return md[f"{_ARCH}.{key}"]
+
+ _check_supported(shim)
+ hidden, heads = int(kv("embedding_length")), int(kv("attention.head_count"))
+ # Mistral-7B files carry no key_length
+ head_dim = int(md.get(f"{_ARCH}.attention.key_length", hidden // heads))
+ q = gguf_tensor_index(shim.model_path).get("blk.0.attn_q.weight")
+ # early Mistral-Nemo conversions lack key_length, so hidden / heads gives the wrong head_dim; llama.cpp rejects them too
+ if q is not None and q.ne[1] != heads * head_dim:
+ raise NotImplementedError(f"{shim.model_path}: attn_q has {q.ne[1]} rows, not head_count x head_dim = {heads * head_dim}")
+ if int(md.get(f"{_ARCH}.rope.dimension_count", head_dim)) != head_dim:
+ raise NotImplementedError(f"{shim.model_path}: partial rope is not supported")
+ base = float(md.get(f"{_ARCH}.rope.freq_base", 10000.0))
+ return SimpleNamespace(
+ model_type="llama",
+ architectures=list(shim.architectures),
+ hidden_act="silu",
+ hidden_size=hidden,
+ num_hidden_layers=int(kv("block_count")),
+ num_attention_heads=heads,
+ num_key_value_heads=int(md.get(f"{_ARCH}.attention.head_count_kv", heads)),
+ head_dim=head_dim,
+ rope_theta=base,
+ rope_scaling=_rope_scaling(shim, base, head_dim),
+ max_position_embeddings=int(kv("context_length")),
+ rms_norm_eps=float(kv("attention.layer_norm_rms_epsilon")),
+ vocab_size=int(shim.vocab_size),
+ tie_word_embeddings=bool(shim.tie_word_embeddings),
+ intermediate_size=int(kv("feed_forward_length")),
+ )
+
+
+def parse_gguf_config(shim: GgufConfigShim) -> ModelConfig:
+ return parse_config(_hf_config(shim))
+
+
+def _unpermute(rows: torch.Tensor, heads: int) -> torch.Tensor:
+ """Undo the converter's rope permute of one q or k matrix (``LlamaModel.permute``); it moves whole rows, so packed ggml rows work too."""
+ return rows.reshape(heads, rows.shape[0] // heads // 2, 2, *rows.shape[1:]).swapaxes(1, 2).reshape(rows.shape)
+
+
+def iter_gguf_weights(
+ model_path: str,
+ device: torch.device,
+ *,
+ include_moe_experts: bool,
+ include_non_moe: bool,
+) -> Iterator[tuple[str, torch.Tensor]]:
+ """The state dict from the GGUF."""
+ if not include_non_moe:
+ return
+ from freetoken.utils import cached_load_hf_config
+
+ hf = _hf_config(cached_load_hf_config(model_path))
+ index = gguf_tensor_index(model_path)
+ hidden, heads, kv_heads = hf.hidden_size, hf.num_attention_heads, hf.num_key_value_heads
+ q_rows, kv_rows = heads * hf.head_dim, kv_heads * hf.head_dim
+
+ def t(name: str) -> GgufTensorInfo:
+ return index[name if name in index else f"{name}.weight"]
+
+ yield from fused("model.embed_tokens", [t("token_embd")], [matrix(t("token_embd"))], hidden)
+ if not hf.tie_word_embeddings:
+ yield from fused("lm_head", [t("output")], [matrix(t("output"))], hidden)
+ yield "model.norm.weight", float_tensor(t("output_norm"))
+
+ for layer in range(hf.num_hidden_layers):
+ blk, pre = f"blk.{layer}", f"model.layers.{layer}"
+ yield f"{pre}.input_layernorm.weight", float_tensor(t(f"{blk}.attn_norm"))
+ yield f"{pre}.post_attention_layernorm.weight", float_tensor(t(f"{blk}.ffn_norm"))
+ sa = f"{pre}.self_attn"
+ # convert_hf_to_gguf.py --fuse-qkv stores q, k and v as one attn_qkv matrix
+ if f"{blk}.attn_qkv.weight" in index:
+ qkv = [t(f"{blk}.attn_qkv")] * 3
+ q, k, v = matrix(qkv[0]).split([q_rows, kv_rows, kv_rows])
+ else:
+ qkv = [t(f"{blk}.attn_{p}") for p in ("q", "k", "v")]
+ q, k, v = (matrix(p) for p in qkv)
+ yield from fused(f"{sa}.qkv_proj", qkv, [_unpermute(q, heads), _unpermute(k, kv_heads), v], hidden)
+ out = t(f"{blk}.attn_output")
+ yield from fused(f"{sa}.o_proj", [out], [matrix(out)], int(out.ne[0]))
+ mlp = f"{pre}.mlp"
+ gate, up, down = (t(f"{blk}.ffn_{p}") for p in ("gate", "up", "down"))
+ yield from fused(f"{mlp}.gate_up_proj", [gate, up], [matrix(gate), matrix(up)], hidden)
+ yield from fused(f"{mlp}.down_proj", [down], [matrix(down)], int(down.ne[0]))
+
+
+__all__ = ["iter_gguf_weights", "parse_gguf_config", "to_gguf"]
diff --git a/python/freetoken/models/llama/model.py b/python/freetoken/models/llama/model.py
index eb7ff28f..107d9e0a 100644
--- a/python/freetoken/models/llama/model.py
+++ b/python/freetoken/models/llama/model.py
@@ -48,6 +48,8 @@ class LlamaModel(BaseOP):
self.embed_tokens = VocabParallelEmbedding(
num_embeddings=config.vocab_size,
embedding_dim=config.hidden_size,
+ quant_config=config.quant,
+ prefix=f"{prefix}.embed_tokens",
)
self.layers = OPList(
[
diff --git a/python/freetoken/models/register.py b/python/freetoken/models/register.py
index 263b5f93..057e2ae6 100644
--- a/python/freetoken/models/register.py
+++ b/python/freetoken/models/register.py
@@ -95,6 +95,14 @@ _MODEL_REGISTRY: dict[str, ModelSpec] = {
"LlamaForCausalLM",
packed_modules_mapping=_DENSE_PACKED,
),
+ # llama.cpp llama GGUF (Llama, Mistral): the llama model classes with every tensor in its packed ggml type
+ "LlamaGGUFForCausalLM": ModelSpec(
+ "freetoken.models.llama",
+ "LlamaForCausalLM",
+ parse_config="parse_gguf_config",
+ iter_weights="iter_gguf_weights",
+ packed_modules_mapping=_DENSE_PACKED,
+ ),
"Qwen2ForCausalLM": ModelSpec(
"freetoken.models.qwen2",
"Qwen2ForCausalLM",