FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
"""Build a HF fast tokenizer from a GGUF file's embedded tokenizer metadata.
transformers' ``AutoTokenizer.from_pretrained(gguf_file=...)`` first builds the HF model config from the GGUF, which needs the PyPI ``gguf`` package and an architecture transformers knows (``qwen4exp`` is not one).
So we run only its tokenizer half: ``get_gguf_tokenizer`` and ``convert_gguf_tokenizer``.
"""
from __future__ import annotations
import json
from .reader import load_gguf_metadata
# End-of-generation names for GGUFs without an explicit eog_token_ids array.
# See llama.cpp llama_vocab::impl::load in src/llama-vocab.cpp.
_EOG_NAMES = frozenset((
"<|eot_id|>", "<|im_end|>", "<|end|>", "<|return|>", "<|call|>", "<|flush|>", "<|calls|>", "<end_of_turn>",
"<|endoftext|>", "</s>", "<|eom_id|>", "<EOT>", "_<EOT>", "[EOT]", "[EOS]", "<|end_of_text|>", "<end_of_utterance>",
"<eos>", "<turn|>", "<|tool_response>", "<\uff5cend\u2581of\u2581sentence\uff5c>", "[e~[", "<|ifm|im_end|>",
))
# BPE pre-tokenizers that llama.cpp adds a bos for when the file has no add_bos_token key (llama_vocab::impl::load)
_BOS_PRE = frozenset((
"llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon",
))
# Llama 3 and GLM pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it
_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "glm4", "glm5"))
# DeepSeek-V4's pre-tokenizer (llama.cpp spells it "joyai-llm"; transformers maps no split
# for it): the three splits of the model's own tokenizer.json, in order
_JOYAI_LLM_SPLITS = (
r"\p{N}{1,3}",
"[一-龥-ゟ゠-ヿ]+",
r'''[!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+|[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+''',
)
# RecursiveSI/flint's tokenizer.json applies these splits in sequence.
_FLINT_SPLITS = (
r"\p{N}{1,3}",
"[一-龥-ゟ゠-ヿ]+",
r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+",
)
_CUSTOM_SPLITS = {"joyai-llm": _JOYAI_LLM_SPLITS, "flint": _FLINT_SPLITS}
def _add_bos(meta: dict) -> bool:
if "tokenizer.ggml.add_bos_token" in meta:
return bool(meta["tokenizer.ggml.add_bos_token"])
model = meta.get("tokenizer.ggml.model")
return model == "llama" or (model == "gpt2" and meta.get("tokenizer.ggml.pre") in _BOS_PRE)
def _fix_added_tokens(fast, meta: dict):
"""Register the control and user-defined tokens that transformers skipped at their file ids, and drop added tokens past the file's vocabulary."""
from tokenizers import Tokenizer
from transformers.convert_slow_tokenizer import bytes_to_unicode
tokens = meta["tokenizer.ggml.tokens"]
spelling = bytes_to_unicode()
alphabet = set(spelling.values())
# transformers' byte-level vocabulary spells a token with characters outside the byte alphabet in bytes and then skips it (DeepSeek's full-width-bar tokens, MiniMax's tokens with spaces)
skipped = [
(i, text, kind == 3)
for i, (text, kind) in enumerate(zip(tokens, meta.get("tokenizer.ggml.token_type") or ()))
if kind in (3, 4) and meta.get("tokenizer.ggml.model") == "gpt2" and not set(text) <= alphabet and fast.token_to_id(text) != i
]
# its GPT2Tokenizer also adds a default <|endoftext|> when the vocabulary lacks one
if not skipped and fast.get_vocab_size(with_added_tokens=True) <= len(tokens):
return fast
# a piece the merges use or build stays a plain BPE token, since renaming it breaks the merges (unsloth's MiniMax-M3 marks U+FFFD as control)
pieces = {p for m in meta.get("tokenizer.ggml.merges") or () for p in (*m.split(" "), m.replace(" ", ""))}
state = json.loads(fast.to_str())
vocab = state["model"]["vocab"]
added = [t for t in state["added_tokens"] if t["id"] < len(tokens)]
for i, text, special in skipped:
spelled = "".join(spelling[b] for b in text.encode("utf-8"))
if spelled in pieces or vocab.get(spelled) != i:
continue
# renamed so the added token keeps the file id instead of taking a new one
vocab[text] = vocab.pop(spelled)
added.append({"id": i, "content": text, "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": special})
state["added_tokens"] = sorted(added, key=lambda t: t["id"])
return Tokenizer.from_str(json.dumps(state))
def _known_specials(special: dict, fast) -> dict:
"""Drop a bos/eos/unk/pad the vocabulary lacks, which PreTrainedTokenizerFast would add past it (unsloth's MiniMax-M3 unk is U+FFFD stored decoded)."""
vocab = fast.get_vocab(with_added_tokens=True)
return {k: v for k, v in special.items() if k == "chat_template" or v is None or v in vocab}
def load_gguf_tokenizer(model_path: str):
from transformers import PreTrainedTokenizerFast
from transformers.integrations.gguf import convert_gguf_tokenizer, get_gguf_tokenizer
arch, tok_dict, special = get_gguf_tokenizer(model_path)
fast, _ = convert_gguf_tokenizer(arch, tok_dict)
meta = load_gguf_metadata(model_path)
fast = _fix_added_tokens(fast, meta)
pre = meta.get("tokenizer.ggml.pre") if meta.get("tokenizer.ggml.model") == "gpt2" else None
if pre in ("glm4", "glm5"):
# llama.cpp spells the same GLM pre-tokenizer "glm4" and "glm5" (llama-vocab.cpp maps both
# to CHATGLM4 with ignore_merges); transformers maps no split for either
from tokenizers import Regex, pre_tokenizers
from transformers.integrations.gguf.gguf_tokenizer_mapping import GGUF_PRE_TOKENIZER_SPLITS
fast.pre_tokenizer = pre_tokenizers.Sequence([
pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),
pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
])
elif pre in ("qwen2", "qwen35"):
# every Qwen release's tokenizer.json applies NFC (Unicode Normalization Form C) before the
# byte-level BPE, but a GGUF carries no normalizer; without it decomposed text differs
from tokenizers import normalizers
fast.normalizer = normalizers.NFC()
elif pre in _CUSTOM_SPLITS:
from tokenizers import Regex, pre_tokenizers
fast.pre_tokenizer = pre_tokenizers.Sequence(
[pre_tokenizers.Split(Regex(r), behavior="isolated") for r in _CUSTOM_SPLITS[pre]]
+ [pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)]
)
if pre in _IGNORE_MERGES_PRE:
fast.model.ignore_merges = True
# special: bos/eos/unk/pad as the file names them (None when absent) and the chat template
return PreTrainedTokenizerFast(
tokenizer_object=fast,
**_known_specials(special, fast),
add_bos_token=_add_bos(meta),
add_eos_token=bool(meta.get("tokenizer.ggml.add_eos_token")),
)
def gguf_eos_token_ids(model_path: str) -> set[int]:
"""Stop ids for GGUF generation, picked the way llama.cpp picks its end-of-generation tokens."""
meta = load_gguf_metadata(model_path)
tokens = meta["tokenizer.ggml.tokens"]
ids = {i for i, t in enumerate(tokens) if t in _EOG_NAMES}
ids |= {int(meta[f"tokenizer.ggml.{k}_token_id"]) for k in ("eos", "eot", "eom") if f"tokenizer.ggml.{k}_token_id" in meta}
ids |= {int(i) for i in meta.get("tokenizer.ggml.eog_token_ids", ())}
names = {tokens[i]: i for i in ids if i < len(tokens)}
# llama.cpp's exceptions: <|end|> only ends a message when <|call|> and <|return|> end the turn (gpt-oss), and gemma4's </s> is a normal token
if "<|end|>" in names and names.keys() & {"<|call|>", "<|calls|>"} and names.keys() & {"<|return|>", "<|flush|>"}:
ids.discard(names["<|end|>"])
if "</s>" in names and names.keys() & {"<|tool_response>", "<|plamo:eos|>"}:
ids.discard(names["</s>"])
return ids
__all__ = ["load_gguf_tokenizer", "gguf_eos_token_ids"]