FreeToken brings datacenter-scale model serving to your desktop. Run massive models locally, fast and efficiently.
build(deps): upgrade transformers to 5.18 (#698)
Xiaoze Fanb6193e86f7fd
diff --git a/pyproject.toml b/pyproject.tomlindex ab1dfd07..384b905f 100644--- a/pyproject.toml+++ b/pyproject.toml@@ -57,7 +57,7 @@ dependencies = ["torch>=2.14,<2.15","torchvision>=0.29,<0.30","tqdm>=4.66,<5",- "transformers>=5.16,<5.17",+ "transformers>=5.18,<5.19",# CUDA torch 2.14 resolves Triton 3.8; AMD's ROCm 7.14 image supplies its# gfx1201-enabled Triton 3.7 build. Keep both supported without replacing the# runtime-specific wheel selected by the PyTorch distribution.diff --git a/python/freetoken/models/gguf/tokenizer.py b/python/freetoken/models/gguf/tokenizer.pyindex e6a5d4ac..af715c65 100644--- a/python/freetoken/models/gguf/tokenizer.py+++ b/python/freetoken/models/gguf/tokenizer.py@@ -1,74 +1,49 @@"""Build a HF fast tokenizer from a GGUF file's embedded tokenizer metadata.-transformers' ``AutoTokenizer.from_pretrained(gguf_file=...)`` first builds the HF-config, which the gemma4 strict dataclass rejects (per-layer ``num_key_value_heads``-array). So we call the GGUF->fast tokenizer converter directly on the-``tokenizer.ggml.*`` metadata, bypassing config entirely.+transformers' ``AutoTokenizer.from_pretrained(gguf_file=...)`` first builds the HF model config from the GGUF, which needs the PyPI ``gguf`` package and an architecture transformers knows (``qwen4exp`` is not one).+So we run only its tokenizer half: ``get_gguf_tokenizer`` and ``convert_gguf_tokenizer``."""from __future__ import annotations-from typing import Any+from .reader import load_gguf_metadata-from .reader import gguf_architecture, load_gguf_metadata--# GGUF architecture -> transformers GGUF tokenizer-converter key.-_TOKENIZER_ARCH = {"gemma4": "gemma4_text", "qwen4exp": "qwen3"}+# end-of-generation token names from llama.cpp (llama_vocab::impl::load in src/llama-vocab.cpp): a GGUF stores one eos id and no stop list+_EOG_NAMES = frozenset((+ "<|eot_id|>", "<|im_end|>", "<|end|>", "<|return|>", "<|call|>", "<|flush|>", "<|calls|>", "<end_of_turn>",+ "<|endoftext|>", "</s>", "<|eom_id|>", "<EOT>", "_<EOT>", "[EOT]", "[EOS]", "<|end_of_text|>", "<end_of_utterance>",+ "<eos>", "<turn|>", "<|tool_response>", "<\uff5cend\u2581of\u2581sentence\uff5c>", "[e~[", "<|ifm|im_end|>",+))def load_gguf_tokenizer(model_path: str):from transformers import PreTrainedTokenizerFast- from transformers.integrations.ggml import convert_gguf_tokenizer+ from transformers.integrations.gguf import convert_gguf_tokenizer, get_gguf_tokenizer+ arch, tok_dict, special = get_gguf_tokenizer(model_path)+ fast, _ = convert_gguf_tokenizer(arch, tok_dict)meta = load_gguf_metadata(model_path)- arch = gguf_architecture(model_path)- conv_arch = _TOKENIZER_ARCH.get(arch, arch)- tok_dict: dict[str, Any] = {- k[len("tokenizer.ggml.") :]: v- for k, v in meta.items()- if k.startswith("tokenizer.ggml.")- }- fast, _extra = convert_gguf_tokenizer(conv_arch, tok_dict)-- tokens = tok_dict["tokens"]-- def tok_for(id_key: str, default: str) -> str:- tid = meta.get(f"tokenizer.ggml.{id_key}")- return tokens[int(tid)] if tid is not None and int(tid) < len(tokens) else default-- # gemma4 chat turns end with <turn|>; prefer it as eos so chat generation halts- # (the formal <eos> is also a stop id, see gguf_eos_token_ids).- turn_end = "<turn|>" if "<turn|>" in tokens else None- tokenizer = PreTrainedTokenizerFast(+ # special: bos/eos/unk/pad as the file names them (None when absent) and the chat template+ return PreTrainedTokenizerFast(tokenizer_object=fast,- bos_token=tok_for("bos_token_id", "<bos>"),- eos_token=turn_end or tok_for("eos_token_id", "<eos>"),- unk_token=tok_for("unknown_token_id", "<unk>"),- pad_token=tok_for("padding_token_id", "<pad>"),+ **special,+ add_bos_token=bool(meta.get("tokenizer.ggml.add_bos_token")),+ add_eos_token=bool(meta.get("tokenizer.ggml.add_eos_token")),)- chat_template = meta.get("tokenizer.chat_template")- if chat_template:- tokenizer.chat_template = chat_template- return tokenizer-def gguf_eos_token_ids(model_path: str, tokenizer) -> set[int]:- """Stop ids for GGUF generation: the formal <eos> plus the chat turn end <turn|>."""+def gguf_eos_token_ids(model_path: str) -> set[int]:+ """Stop ids for GGUF generation, picked the way llama.cpp picks its end-of-generation tokens."""meta = load_gguf_metadata(model_path)tokens = meta["tokenizer.ggml.tokens"]- ids: set[int] = set()- if tokenizer.eos_token_id is not None:- ids.add(int(tokenizer.eos_token_id))- eid = meta.get("tokenizer.ggml.eos_token_id")- if eid is not None:- ids.add(int(eid))- # Look the stop tokens up in the vocab directly (convert_tokens_to_ids would map an- # absent name to <unk>, wrongly adding it as a stop id).- for name in ("<eos>", "<turn|>"):- try:- ids.add(tokens.index(name))- except ValueError:- pass+ ids = {i for i, t in enumerate(tokens) if t in _EOG_NAMES}+ ids |= {int(meta[f"tokenizer.ggml.{k}_token_id"]) for k in ("eos", "eot", "eom") if f"tokenizer.ggml.{k}_token_id" in meta}+ names = {tokens[i]: i for i in ids if i < len(tokens)}+ # llama.cpp's exceptions: <|end|> only ends a message when <|call|> and <|return|> end the turn (gpt-oss), and gemma4's </s> is a normal token+ if "<|end|>" in names and names.keys() & {"<|call|>", "<|calls|>"} and names.keys() & {"<|return|>", "<|flush|>"}:+ ids.discard(names["<|end|>"])+ if "</s>" in names and names.keys() & {"<|tool_response>", "<|plamo:eos|>"}:+ ids.discard(names["</s>"])return idsdiff --git a/python/freetoken/models/glm5_next/args.py b/python/freetoken/models/glm5_next/args.pyindex c25fef48..c771467b 100644--- a/python/freetoken/models/glm5_next/args.py+++ b/python/freetoken/models/glm5_next/args.py@@ -24,16 +24,16 @@ from __future__ import annotationsfrom dataclasses import dataclassfrom typing import Any, Tuple-# Layer-type strings used by the checkpoint's ``layer_types`` field.+# Layer-type strings after transformers' legacy remap, which renames the checkpoint's deepseek_sparse_attention.KDA_LAYER = "linear_attention"-DSA_LAYER = "deepseek_sparse_attention"+DSA_LAYER = "indexed_attention"@dataclass(frozen=True)class Glm5NextArgs:hidden_size: intnum_heads: int- # ---- MLA (the 11 "deepseek_sparse_attention" layers) ----+ # ---- MLA (the 11 "indexed_attention" layers) ----q_lora_rank: intkv_lora_rank: intqk_nope_head_dim: int@@ -119,8 +119,10 @@ def load_args(hf_config: Any) -> Glm5NextArgs:are accepted as fallbacks."""text = _get(hf_config, "text_config", hf_config)+ from transformers.configuration_utils import remap_legacy_layer_types+num_layers = int(text.num_hidden_layers)- layer_types = tuple(_get(text, "layer_types", ()) or ())+ layer_types = tuple(remap_legacy_layer_types(list(_get(text, "layer_types", ()) or ())))if not layer_types:raise ValueError("glm5_next config is missing layer_types")if len(layer_types) != num_layers:diff --git a/python/freetoken/models/qwen4_exp/config.py b/python/freetoken/models/qwen4_exp/config.pyindex b1e5f9aa..2bc96f67 100644--- a/python/freetoken/models/qwen4_exp/config.py+++ b/python/freetoken/models/qwen4_exp/config.py@@ -101,9 +101,9 @@ def ple_slot_states(args: Qwen4ExpArgs) -> Tuple[SlotStateSpec, ...]:def _layer_types(text: Any) -> list[str]:layer_types = getattr(text, "layer_types", None)if layer_types is not None:- # HF Qwen4ExpTextConfig rewrites full_attention to qwen_sparse_attention in __post_init__.+ # HF Qwen4ExpTextConfig rewrites full_attention to indexed_attention in __post_init__.return [- "full_attention" if t == "qwen_sparse_attention" else t for t in layer_types+ "full_attention" if t == "indexed_attention" else t for t in layer_types]# Fall back to full_attention_interval: every Nth layer (1-indexed) is full.interval = int(getattr(text, "full_attention_interval", 4))diff --git a/python/freetoken/utils/hf.py b/python/freetoken/utils/hf.pyindex 489585e8..5a7b7e69 100644--- a/python/freetoken/utils/hf.py+++ b/python/freetoken/utils/hf.py@@ -62,7 +62,7 @@ def load_eos_token_ids(if (gguf_src := gguf_config_source(model_path)) is not None:from freetoken.models.gguf.tokenizer import gguf_eos_token_ids- return frozenset(gguf_eos_token_ids(gguf_src, tokenizer))+ return frozenset(gguf_eos_token_ids(gguf_src))ids: set[int] = set()if tokenizer.eos_token_id is not None:diff --git a/tests/attention/test_dsa_kpool.py b/tests/attention/test_dsa_kpool.pyindex fb856e17..ec2bbb1f 100644--- a/tests/attention/test_dsa_kpool.py+++ b/tests/attention/test_dsa_kpool.py@@ -41,7 +41,7 @@ def _args(num_layers=1):index_kpool_always_select_tail=True,linear_num_heads=0, linear_head_dim=0, linear_conv_kernel_dim=4,linear_lower_bound=-5.0,- layer_types=("deepseek_sparse_attention",) * num_layers,+ layer_types=("indexed_attention",) * num_layers,mlp_layer_types=("dense",) * num_layers,mhc=False, mhc_num_residual_streams=1, hc_eps=1e-6,mhc_sinkhorn_iterations=0, mhc_tau=0.05, mhc_post_mult_value=2.0,diff --git a/tests/models/test_glm_dsa.py b/tests/models/test_glm_dsa.pyindex 0dceabad..7a82894d 100644--- a/tests/models/test_glm_dsa.py+++ b/tests/models/test_glm_dsa.py@@ -41,7 +41,9 @@ def _hf_indexer(seq: int, topk: int):freqs = torch.outer(pos.float(), inv)emb = torch.cat((freqs, freqs), dim=-1)cos, sin = emb.cos()[None].to(torch.bfloat16), emb.sin()[None].to(torch.bfloat16)- ref_topk = idx(x, q_resid, (cos, sin), None, pos[None]) # [1, S, topk]+ # bool causal mask, True = visible: since transformers 5.18 the indexer no longer builds one when given None+ causal = (pos[None, :] <= pos[:, None])[None]+ ref_topk = idx(x, q_resid, (cos, sin), causal, pos[None]) # [1, S, topk]return idx, x, q_resid, pos, ref_topk