Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

8.0 KB
Raw Blame
"""Build a HF fast tokenizer from a GGUF file's embedded tokenizer metadata.

transformers' ``AutoTokenizer.from_pretrained(gguf_file=...)`` first builds the HF model config from the GGUF, which needs the PyPI ``gguf`` package and an architecture transformers knows (``qwen4exp`` is not one).
So we run only its tokenizer half: ``get_gguf_tokenizer`` and ``convert_gguf_tokenizer``.
"""

from __future__ import annotations

import json

from .reader import load_gguf_metadata

# End-of-generation names for GGUFs without an explicit eog_token_ids array.
# See llama.cpp llama_vocab::impl::load in src/llama-vocab.cpp.
_EOG_NAMES = frozenset((
    "<|eot_id|>", "<|im_end|>", "<|end|>", "<|return|>", "<|call|>", "<|flush|>", "<|calls|>", "<end_of_turn>",
    "<|endoftext|>", "</s>", "<|eom_id|>", "<EOT>", "_<EOT>", "[EOT]", "[EOS]", "<|end_of_text|>", "<end_of_utterance>",
    "<eos>", "<turn|>", "<|tool_response>", "<\uff5cend\u2581of\u2581sentence\uff5c>", "[e~[", "<|ifm|im_end|>",
))

# BPE pre-tokenizers that llama.cpp adds a bos for when the file has no add_bos_token key (llama_vocab::impl::load)
_BOS_PRE = frozenset((
    "llama3", "llama-v3", "llama-bpe", "falcon3", "falcon-h1", "pixtral", "midm-2.0", "lfm2", "jina-v5-nano", "tekken", "chameleon",
))

# Llama 3 and GLM pre-tokenizers, whose HF tokenizer and llama.cpp take a word that is one vocabulary token as is instead of merging it
_IGNORE_MERGES_PRE = frozenset(("llama3", "llama-v3", "llama-bpe", "glm4", "glm5"))

# DeepSeek-V4's pre-tokenizer (llama.cpp spells it "joyai-llm"; transformers maps no split
# for it): the three splits of the model's own tokenizer.json, in order
_JOYAI_LLM_SPLITS = (
    r"\p{N}{1,3}",
    "[一-龥぀-ゟ゠-ヿ]+",
    r'''[!"#$%&'()*+,\-./:;<=>?@\[\\\]^_`{|}~][A-Za-z]+|[^\r\n\p{L}\p{P}\p{S}]?[\p{L}\p{M}]+| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+''',
)

# RecursiveSI/flint's tokenizer.json applies these splits in sequence.
_FLINT_SPLITS = (
    r"\p{N}{1,3}",
    "[一-龥぀-ゟ゠-ヿ]+",
    r"[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]*[\p{Ll}\p{Lm}\p{Lo}\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]+[\p{Ll}\p{Lm}\p{Lo}\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?| ?[\p{P}\p{S}]+[\r\n]*|\s*[\r\n]+|\s+(?!\S)|\s+",
)

_CUSTOM_SPLITS = {"joyai-llm": _JOYAI_LLM_SPLITS, "flint": _FLINT_SPLITS}


def _add_bos(meta: dict) -> bool:
    if "tokenizer.ggml.add_bos_token" in meta:
        return bool(meta["tokenizer.ggml.add_bos_token"])
    model = meta.get("tokenizer.ggml.model")
    return model == "llama" or (model == "gpt2" and meta.get("tokenizer.ggml.pre") in _BOS_PRE)


def _fix_added_tokens(fast, meta: dict):
    """Register the control and user-defined tokens that transformers skipped at their file ids, and drop added tokens past the file's vocabulary."""
    from tokenizers import Tokenizer
    from transformers.convert_slow_tokenizer import bytes_to_unicode

    tokens = meta["tokenizer.ggml.tokens"]
    spelling = bytes_to_unicode()
    alphabet = set(spelling.values())
    # transformers' byte-level vocabulary spells a token with characters outside the byte alphabet in bytes and then skips it (DeepSeek's full-width-bar tokens, MiniMax's tokens with spaces)
    skipped = [
        (i, text, kind == 3)
        for i, (text, kind) in enumerate(zip(tokens, meta.get("tokenizer.ggml.token_type") or ()))
        if kind in (3, 4) and meta.get("tokenizer.ggml.model") == "gpt2" and not set(text) <= alphabet and fast.token_to_id(text) != i
    ]
    # its GPT2Tokenizer also adds a default <|endoftext|> when the vocabulary lacks one
    if not skipped and fast.get_vocab_size(with_added_tokens=True) <= len(tokens):
        return fast
    # a piece the merges use or build stays a plain BPE token, since renaming it breaks the merges (unsloth's MiniMax-M3 marks U+FFFD as control)
    pieces = {p for m in meta.get("tokenizer.ggml.merges") or () for p in (*m.split(" "), m.replace(" ", ""))}
    state = json.loads(fast.to_str())
    vocab = state["model"]["vocab"]
    added = [t for t in state["added_tokens"] if t["id"] < len(tokens)]
    for i, text, special in skipped:
        spelled = "".join(spelling[b] for b in text.encode("utf-8"))
        if spelled in pieces or vocab.get(spelled) != i:
            continue
        # renamed so the added token keeps the file id instead of taking a new one
        vocab[text] = vocab.pop(spelled)
        added.append({"id": i, "content": text, "single_word": False, "lstrip": False, "rstrip": False, "normalized": False, "special": special})
    state["added_tokens"] = sorted(added, key=lambda t: t["id"])
    return Tokenizer.from_str(json.dumps(state))


def _known_specials(special: dict, fast) -> dict:
    """Drop a bos/eos/unk/pad the vocabulary lacks, which PreTrainedTokenizerFast would add past it (unsloth's MiniMax-M3 unk is U+FFFD stored decoded)."""
    vocab = fast.get_vocab(with_added_tokens=True)
    return {k: v for k, v in special.items() if k == "chat_template" or v is None or v in vocab}


def load_gguf_tokenizer(model_path: str):
    from transformers import PreTrainedTokenizerFast
    from transformers.integrations.gguf import convert_gguf_tokenizer, get_gguf_tokenizer

    arch, tok_dict, special = get_gguf_tokenizer(model_path)
    fast, _ = convert_gguf_tokenizer(arch, tok_dict)
    meta = load_gguf_metadata(model_path)
    fast = _fix_added_tokens(fast, meta)
    pre = meta.get("tokenizer.ggml.pre") if meta.get("tokenizer.ggml.model") == "gpt2" else None
    if pre in ("glm4", "glm5"):
        # llama.cpp spells the same GLM pre-tokenizer "glm4" and "glm5" (llama-vocab.cpp maps both
        # to CHATGLM4 with ignore_merges); transformers maps no split for either
        from tokenizers import Regex, pre_tokenizers
        from transformers.integrations.gguf.gguf_tokenizer_mapping import GGUF_PRE_TOKENIZER_SPLITS

        fast.pre_tokenizer = pre_tokenizers.Sequence([
            pre_tokenizers.Split(Regex(GGUF_PRE_TOKENIZER_SPLITS["llama3"]), behavior="isolated"),
            pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False),
        ])
    elif pre in ("qwen2", "qwen35"):
        # every Qwen release's tokenizer.json applies NFC (Unicode Normalization Form C) before the
        # byte-level BPE, but a GGUF carries no normalizer; without it decomposed text differs
        from tokenizers import normalizers

        fast.normalizer = normalizers.NFC()
    elif pre in _CUSTOM_SPLITS:
        from tokenizers import Regex, pre_tokenizers

        fast.pre_tokenizer = pre_tokenizers.Sequence(
            [pre_tokenizers.Split(Regex(r), behavior="isolated") for r in _CUSTOM_SPLITS[pre]]
            + [pre_tokenizers.ByteLevel(add_prefix_space=False, use_regex=False)]
        )
    if pre in _IGNORE_MERGES_PRE:
        fast.model.ignore_merges = True
    # special: bos/eos/unk/pad as the file names them (None when absent) and the chat template
    return PreTrainedTokenizerFast(
        tokenizer_object=fast,
        **_known_specials(special, fast),
        add_bos_token=_add_bos(meta),
        add_eos_token=bool(meta.get("tokenizer.ggml.add_eos_token")),
    )


def gguf_eos_token_ids(model_path: str) -> set[int]:
    """Stop ids for GGUF generation, picked the way llama.cpp picks its end-of-generation tokens."""
    meta = load_gguf_metadata(model_path)
    tokens = meta["tokenizer.ggml.tokens"]
    ids = {i for i, t in enumerate(tokens) if t in _EOG_NAMES}
    ids |= {int(meta[f"tokenizer.ggml.{k}_token_id"]) for k in ("eos", "eot", "eom") if f"tokenizer.ggml.{k}_token_id" in meta}
    ids |= {int(i) for i in meta.get("tokenizer.ggml.eog_token_ids", ())}
    names = {tokens[i]: i for i in ids if i < len(tokens)}
    # llama.cpp's exceptions: <|end|> only ends a message when <|call|> and <|return|> end the turn (gpt-oss), and gemma4's </s> is a normal token
    if "<|end|>" in names and names.keys() & {"<|call|>", "<|calls|>"} and names.keys() & {"<|return|>", "<|flush|>"}:
        ids.discard(names["<|end|>"])
    if "</s>" in names and names.keys() & {"<|tool_response>", "<|plamo:eos|>"}:
        ids.discard(names["</s>"])
    return ids


__all__ = ["load_gguf_tokenizer", "gguf_eos_token_ids"]