Skip to content

FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.

build(deps): upgrade transformers to 5.18 (#698)

Xiaoze Fanb6193e86f7fd
diff --git a/pyproject.toml b/pyproject.toml
index ab1dfd07..384b905f 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,7 +57,7 @@ dependencies = [
"torch>=2.14,<2.15",
"torchvision>=0.29,<0.30",
"tqdm>=4.66,<5",
- "transformers>=5.16,<5.17",
+ "transformers>=5.18,<5.19",
# CUDA torch 2.14 resolves Triton 3.8; AMD's ROCm 7.14 image supplies its
# gfx1201-enabled Triton 3.7 build. Keep both supported without replacing the
# runtime-specific wheel selected by the PyTorch distribution.
diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.py
index e6a5d4ac..af715c65 100644
--- a/python/freetoken/models/gguf/tokenizer.py
+++ b/python/freetoken/models/gguf/tokenizer.py
@@ -1,74 +1,49 @@
"""Build a HF fast tokenizer from a GGUF file's embedded tokenizer metadata.
-transformers' ``AutoTokenizer.from_pretrained(gguf_file=...)`` first builds the HF
-config, which the gemma4 strict dataclass rejects (per-layer ``num_key_value_heads``
-array). So we call the GGUF->fast tokenizer converter directly on the
-``tokenizer.ggml.*`` metadata, bypassing config entirely.
+transformers' ``AutoTokenizer.from_pretrained(gguf_file=...)`` first builds the HF model config from the GGUF, which needs the PyPI ``gguf`` package and an architecture transformers knows (``qwen4exp`` is not one).
+So we run only its tokenizer half: ``get_gguf_tokenizer`` and ``convert_gguf_tokenizer``.
"""
from __future__ import annotations
-from typing import Any
+from .reader import load_gguf_metadata
-from .reader import gguf_architecture, load_gguf_metadata
-
-# GGUF architecture -> transformers GGUF tokenizer-converter key.
-_TOKENIZER_ARCH = {"gemma4": "gemma4_text", "qwen4exp": "qwen3"}
+# end-of-generation token names from llama.cpp (llama_vocab::impl::load in src/llama-vocab.cpp): a GGUF stores one eos id and no stop list
+_EOG_NAMES = frozenset((
+ "<|eot_id|>", "<|im_end|>", "<|end|>", "<|return|>", "<|call|>", "<|flush|>", "<|calls|>", "<end_of_turn>",
+ "<|endoftext|>", "</s>", "<|eom_id|>", "<EOT>", "_<EOT>", "[EOT]", "[EOS]", "<|end_of_text|>", "<end_of_utterance>",
+ "<eos>", "<turn|>", "<|tool_response>", "<\uff5cend\u2581of\u2581sentence\uff5c>", "[e~[", "<|ifm|im_end|>",
+))
def load_gguf_tokenizer(model_path: str):
from transformers import PreTrainedTokenizerFast
- from transformers.integrations.ggml import convert_gguf_tokenizer
+ from transformers.integrations.gguf import convert_gguf_tokenizer, get_gguf_tokenizer
+ arch, tok_dict, special = get_gguf_tokenizer(model_path)
+ fast, _ = convert_gguf_tokenizer(arch, tok_dict)
meta = load_gguf_metadata(model_path)
- arch = gguf_architecture(model_path)
- conv_arch = _TOKENIZER_ARCH.get(arch, arch)
- tok_dict: dict[str, Any] = {
- k[len("tokenizer.ggml.") :]: v
- for k, v in meta.items()
- if k.startswith("tokenizer.ggml.")
- }
- fast, _extra = convert_gguf_tokenizer(conv_arch, tok_dict)
-
- tokens = tok_dict["tokens"]
-
- def tok_for(id_key: str, default: str) -> str:
- tid = meta.get(f"tokenizer.ggml.{id_key}")
- return tokens[int(tid)] if tid is not None and int(tid) < len(tokens) else default
-
- # gemma4 chat turns end with <turn|>; prefer it as eos so chat generation halts
- # (the formal <eos> is also a stop id, see gguf_eos_token_ids).
- turn_end = "<turn|>" if "<turn|>" in tokens else None
- tokenizer = PreTrainedTokenizerFast(
+ # special: bos/eos/unk/pad as the file names them (None when absent) and the chat template
+ return PreTrainedTokenizerFast(
tokenizer_object=fast,
- bos_token=tok_for("bos_token_id", "<bos>"),
- eos_token=turn_end or tok_for("eos_token_id", "<eos>"),
- unk_token=tok_for("unknown_token_id", "<unk>"),
- pad_token=tok_for("padding_token_id", "<pad>"),
+ **special,
+ add_bos_token=bool(meta.get("tokenizer.ggml.add_bos_token")),
+ add_eos_token=bool(meta.get("tokenizer.ggml.add_eos_token")),
)
- chat_template = meta.get("tokenizer.chat_template")
- if chat_template:
- tokenizer.chat_template = chat_template
- return tokenizer
-def gguf_eos_token_ids(model_path: str, tokenizer) -> set[int]:
- """Stop ids for GGUF generation: the formal <eos> plus the chat turn end <turn|>."""
+def gguf_eos_token_ids(model_path: str) -> set[int]:
+ """Stop ids for GGUF generation, picked the way llama.cpp picks its end-of-generation tokens."""
meta = load_gguf_metadata(model_path)
tokens = meta["tokenizer.ggml.tokens"]
- ids: set[int] = set()
- if tokenizer.eos_token_id is not None:
- ids.add(int(tokenizer.eos_token_id))
- eid = meta.get("tokenizer.ggml.eos_token_id")
- if eid is not None:
- ids.add(int(eid))
- # Look the stop tokens up in the vocab directly (convert_tokens_to_ids would map an
- # absent name to <unk>, wrongly adding it as a stop id).
- for name in ("<eos>", "<turn|>"):
- try:
- ids.add(tokens.index(name))
- except ValueError:
- pass
+ ids = {i for i, t in enumerate(tokens) if t in _EOG_NAMES}
+ ids |= {int(meta[f"tokenizer.ggml.{k}_token_id"]) for k in ("eos", "eot", "eom") if f"tokenizer.ggml.{k}_token_id" in meta}
+ names = {tokens[i]: i for i in ids if i < len(tokens)}
+ # llama.cpp's exceptions: <|end|> only ends a message when <|call|> and <|return|> end the turn (gpt-oss), and gemma4's </s> is a normal token
+ if "<|end|>" in names and names.keys() & {"<|call|>", "<|calls|>"} and names.keys() & {"<|return|>", "<|flush|>"}:
+ ids.discard(names["<|end|>"])
+ if "</s>" in names and names.keys() & {"<|tool_response>", "<|plamo:eos|>"}:
+ ids.discard(names["</s>"])
return ids
diff --git a/python/freetoken/models/glm5_next/args.py b/python/freetoken/models/glm5_next/args.py
index c25fef48..c771467b 100644
--- a/python/freetoken/models/glm5_next/args.py
+++ b/python/freetoken/models/glm5_next/args.py
@@ -24,16 +24,16 @@ from __future__ import annotations
from dataclasses import dataclass
from typing import Any, Tuple
-# Layer-type strings used by the checkpoint's ``layer_types`` field.
+# Layer-type strings after transformers' legacy remap, which renames the checkpoint's deepseek_sparse_attention.
KDA_LAYER = "linear_attention"
-DSA_LAYER = "deepseek_sparse_attention"
+DSA_LAYER = "indexed_attention"
@dataclass(frozen=True)
class Glm5NextArgs:
hidden_size: int
num_heads: int
- # ---- MLA (the 11 "deepseek_sparse_attention" layers) ----
+ # ---- MLA (the 11 "indexed_attention" layers) ----
q_lora_rank: int
kv_lora_rank: int
qk_nope_head_dim: int
@@ -119,8 +119,10 @@ def load_args(hf_config: Any) -> Glm5NextArgs:
are accepted as fallbacks."""
text = _get(hf_config, "text_config", hf_config)
+ from transformers.configuration_utils import remap_legacy_layer_types
+
num_layers = int(text.num_hidden_layers)
- layer_types = tuple(_get(text, "layer_types", ()) or ())
+ layer_types = tuple(remap_legacy_layer_types(list(_get(text, "layer_types", ()) or ())))
if not layer_types:
raise ValueError("glm5_next config is missing layer_types")
if len(layer_types) != num_layers:
diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.py
index b1e5f9aa..2bc96f67 100644
--- a/python/freetoken/models/qwen4_exp/config.py
+++ b/python/freetoken/models/qwen4_exp/config.py
@@ -101,9 +101,9 @@ def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]:
def _layer_types(text: Any) -> list[str]:
layer_types = getattr(text, "layer_types", None)
if layer_types is not None:
- # HF Qwen4ExpTextConfig rewrites full_attention to qwen_sparse_attention in __post_init__.
+ # HF Qwen4ExpTextConfig rewrites full_attention to indexed_attention in __post_init__.
return [
- "full_attention" if t == "qwen_sparse_attention" else t for t in layer_types
+ "full_attention" if t == "indexed_attention" else t for t in layer_types
]
# Fall back to full_attention_interval: every Nth layer (1-indexed) is full.
interval = int(getattr(text, "full_attention_interval", 4))
diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.py
index 489585e8..5a7b7e69 100644
--- a/python/freetoken/utils/hf.py
+++ b/python/freetoken/utils/hf.py
@@ -62,7 +62,7 @@ def load_eos_token_ids(
if (gguf_src := gguf_config_source(model_path)) is not None:
from freetoken.models.gguf.tokenizer import gguf_eos_token_ids
- return frozenset(gguf_eos_token_ids(gguf_src, tokenizer))
+ return frozenset(gguf_eos_token_ids(gguf_src))
ids: set[int] = set()
if tokenizer.eos_token_id is not None:
diff --git a/tests/attention/test_dsa_kpool.py b/tests/attention/test_dsa_kpool.py
index fb856e17..ec2bbb1f 100644
--- a/tests/attention/test_dsa_kpool.py
+++ b/tests/attention/test_dsa_kpool.py
@@ -41,7 +41,7 @@ def _args(num_layers=1):
index_kpool_always_select_tail=True,
linear_num_heads=0, linear_head_dim=0, linear_conv_kernel_dim=4,
linear_lower_bound=-5.0,
- layer_types=("deepseek_sparse_attention",) * num_layers,
+ layer_types=("indexed_attention",) * num_layers,
mlp_layer_types=("dense",) * num_layers,
mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,
mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,
diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.py
index 0dceabad..7a82894d 100644
--- a/tests/models/test_glm_dsa.py
+++ b/tests/models/test_glm_dsa.py
@@ -41,7 +41,9 @@ def _hf_indexer(seq: int, topk: int):
freqs = torch.outer(pos.float(), inv)
emb = torch.cat((freqs, freqs), dim=-1)
cos, sin = emb.cos()[None].to(torch.bfloat16), emb.sin()[None].to(torch.bfloat16)
- ref_topk = idx(x, q_resid, (cos, sin), None, pos[None]) # [1, S, topk]
+ # bool causal mask, True = visible: since transformers 5.18 the indexer no longer builds one when given None
+ causal = (pos[None, :] <= pos[:, None])[None]
+ ref_topk = idx(x, q_resid, (cos, sin), causal, pos[None]) # [1, S, topk]
return idx, x, q_resid, pos, ref_topk