Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
fix(tokenizer): enable add_bos_token for Gemma 4 base models (#10312)
Sourav Rajvi30529a605178
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.pyindex b9094102bcc..4a0d74b6a1a 100644--- a/studio/backend/core/inference/inference.py+++ b/studio/backend/core/inference/inference.py@@ -259,6 +259,25 @@ class _GenerationThreadError(RuntimeError):"""Generation worker failures that should propagate through stream routes."""+def _prompt_already_has_bos(tokenizer, prompt):+ """Did the rendered chat template emit BOS itself?++ Most do, so the tokenizer must not add a second. Some do not (zephyr, tinyllama-chat), and+ suppressing special tokens there drops BOS entirely.+ """+ tok = getattr(tokenizer, "tokenizer", tokenizer)+ bos_token_id = getattr(tok, "bos_token_id", None)+ if bos_token_id is None:+ return False+ try:+ ids = tok(prompt, add_special_tokens = False)["input_ids"]+ except Exception:+ return False+ while isinstance(ids, (list, tuple)) and ids and isinstance(ids[0], (list, tuple)):+ ids = ids[0]+ return bool(len(ids)) and ids[0] == bos_token_id++class InferenceBackend:"""Unified inference backend supporting text, vision, and LoRA models"""@@ -1190,6 +1209,7 @@ class InferenceBackend:else:template_messages = messagesreasoning_channel_markers_resolved = False+ add_special_tokens = Truetry:if not (hasattr(tokenizer, "chat_template") and tokenizer.chat_template):raise ValueError(@@ -1234,6 +1254,8 @@ class InferenceBackend:formatted_prompt = render_result.promptreasoning_channel_markers = render_result.reasoning_channel_markersreasoning_channel_markers_resolved = True+ # Suppress the tokenizer's BOS only when the template already emitted one.+ add_special_tokens = not _prompt_already_has_bos(tokenizer, formatted_prompt)logger.debug(f"Formatted prompt: {formatted_prompt[:200]}...")except Exception as e:@@ -1260,6 +1282,7 @@ class InferenceBackend:reasoning_channel_markers = reasoning_channel_markers,reasoning_channel_markers_resolved = reasoning_channel_markers_resolved,continued = bool(continue_final_message and trailing_assistant_text(template_messages)),+ add_special_tokens = add_special_tokens,)def _generate_vision_response(@@ -1459,7 +1482,11 @@ class InferenceBackend:formatted_prompt = self.format_chat_prompt(messages, system_prompt, continue_final_message = continue_final_message)- inputs = raw_tokenizer(formatted_prompt, return_tensors = "pt").to(model.device)+ inputs = raw_tokenizer(+ formatted_prompt,+ return_tensors = "pt",+ add_special_tokens = not _prompt_already_has_bos(raw_tokenizer, formatted_prompt),+ ).to(model.device)prompt_text = formatted_prompt# Stream with TextIteratorStreamer + background thread@@ -1936,9 +1963,13 @@ class InferenceBackend:reasoning_channel_markers = None,reasoning_channel_markers_resolved: bool = False,continued: bool = False,+ add_special_tokens: bool = True,) -> Generator[str, None, None]:"""Generate a streaming text response (text models only).+ Rendered chat prompts pass add_special_tokens=False; raw prompts keep+ the tokenizer defaults, including BOS insertion for base models.+_adapter_state: if not None, the background thread toggles adaptersbefore model.generate(), under _generation_lock.``presence_penalty`` matches the GGUF sampling path via a logits processor (0 disables it).@@ -1958,7 +1989,9 @@ class InferenceBackend:tokenizer = getattr(tokenizer, "tokenizer", tokenizer)try:- inputs = tokenizer(prompt, return_tensors = "pt").to(model.device)+ inputs = tokenizer(+ prompt, return_tensors = "pt", add_special_tokens = add_special_tokens+ ).to(model.device)import threadingdiff --git a/studio/backend/tests/test_chat_prompt_bos.py b/studio/backend/tests/test_chat_prompt_bos.pynew file mode 100644index 00000000000..6c95cb0ddfd--- /dev/null+++ b/studio/backend/tests/test_chat_prompt_bos.py@@ -0,0 +1,173 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0++"""A rendered chat prompt must carry exactly one BOS.++Most templates emit it, so letting the tokenizer add another doubles it; zephyr and+tinyllama-chat emit none, so suppressing specials unconditionally drops it. The deciding+fact is whether the rendered text already starts with BOS.+"""++from __future__ import annotations++import importlib+import importlib.machinery+import sys+import types+from pathlib import Path+from unittest.mock import MagicMock++import pytest++_BACKEND = Path(__file__).resolve().parent.parent+if str(_BACKEND) not in sys.path:+ sys.path.insert(0, str(_BACKEND))++_STUBBED: list[str] = []+++def _stub_if_missing(+ name,+ attrs = (),+ named_spec = False,+):+ """Register a stub for a dep this job does not install. A real install is left alone.++ Same helper and reason as test_vision_client_tools.py: core.inference.inference imports+ unsloth and trl at module scope, which studio-backend-ci.yml does not install, so+ unstubbed this file fails COLLECTION and takes the whole job down.++ ``named_spec`` gives the stub a real ModuleSpec, which only torchao needs: transformers+ probes it with find_spec, which raises ValueError on ``__spec__ = None``.+ """+ if name in sys.modules:+ return+ try:+ importlib.import_module(name)+ return+ except Exception: # noqa: BLE001 - unusable here either way, so stub it+ pass+ _STUBBED.append(name)+ module = types.ModuleType(name)+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None+ module.__version__ = "0.0.0"+ module.__getattr__ = lambda _attr: MagicMock()+ for attr in attrs:+ setattr(module, attr, MagicMock())+ sys.modules[name] = module+ parent, _, child = name.rpartition(".")+ if parent and parent in sys.modules:+ setattr(sys.modules[parent], child, module)+++# Fires only where torchao is installed but unusable against the local torch, in which+# case transformers.quantizers imports it and poisons transformers for every later module.+for _torchao in (+ "torchao",+ "torchao.prototype",+ "torchao.prototype.safetensors",+ "torchao.prototype.safetensors.safetensors_support",+ "torchao.prototype.safetensors.safetensors_utils",+ "torchao.quantization",+ "torchao.dtypes",+ "torchao.float8",+ "torchao.utils",+):+ _stub_if_missing(_torchao, named_spec = True)++_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))+_stub_if_missing("unsloth_zoo")+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))++from core.inference.inference import _prompt_already_has_bos # noqa: E402++# Drop the stubs now the name is bound; one left behind outlives this module.+for _name in reversed(_STUBBED):+ sys.modules.pop(_name, None)+++def _tokenizer(auto_adds_bos: bool):+ tokenizers = pytest.importorskip("tokenizers")+ from transformers import PreTrainedTokenizerFast++ vocab = {"<s>": 0, "</s>": 1, "hello": 2, "world": 3, "user": 4}+ backend = tokenizers.Tokenizer(tokenizers.models.WordLevel(vocab, unk_token = None))+ backend.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend, bos_token = "<s>", eos_token = "</s>")+ if auto_adds_bos:+ tokenizer.add_bos_token = True+ return tokenizer+++def _encode(tokenizer, prompt):+ add_special_tokens = not _prompt_already_has_bos(tokenizer, prompt)+ return tokenizer(prompt, add_special_tokens = add_special_tokens)["input_ids"]+++def _leading_bos(ids, bos_token_id):+ count = 0+ for token_id in ids:+ if token_id != bos_token_id:+ break+ count += 1+ return count++[email protected]("auto_adds_bos", [True, False])[email protected]("template_emits_bos", [True, False])+def test_rendered_prompt_carries_exactly_one_bos(template_emits_bos, auto_adds_bos):+ tokenizer = _tokenizer(auto_adds_bos)+ if not (template_emits_bos or auto_adds_bos):+ pytest.skip("nothing supplies BOS, so there is none to count")+ prompt = ("<s> " if template_emits_bos else "") + "user hello world"++ assert _leading_bos(_encode(tokenizer, prompt), tokenizer.bos_token_id) == 1+++def test_template_that_omits_bos_keeps_the_tokenizer_one():+ # Blanket add_special_tokens = False regressed exactly this shape.+ tokenizer = _tokenizer(auto_adds_bos = True)+ prompt = "user hello world"++ assert _prompt_already_has_bos(tokenizer, prompt) is False+ assert _encode(tokenizer, prompt)[0] == tokenizer.bos_token_id+++def test_template_that_emits_bos_is_not_doubled():+ tokenizer = _tokenizer(auto_adds_bos = True)+ prompt = "<s> user hello world"++ assert _prompt_already_has_bos(tokenizer, prompt) is True+ assert _leading_bos(_encode(tokenizer, prompt), tokenizer.bos_token_id) == 1+++def test_tokenizer_without_a_bos_token_is_left_alone():+ tokenizer = _tokenizer(auto_adds_bos = False)+ tokenizer.bos_token = None++ assert _prompt_already_has_bos(tokenizer, "user hello") is False+++def test_a_tokenizer_that_raises_is_treated_as_having_no_bos():+ class _Raises:+ bos_token_id = 0++ def __call__(self, *args, **kwargs):+ raise RuntimeError("needs an image")++ assert _prompt_already_has_bos(_Raises(), "user hello") is False+++def test_a_processor_is_probed_through_its_inner_tokenizer():+ import types++ tokenizer = _tokenizer(auto_adds_bos = True)+ processor = types.SimpleNamespace(tokenizer = tokenizer)++ assert _prompt_already_has_bos(processor, "<s> user hello") is True+ assert _prompt_already_has_bos(processor, "user hello") is False+++def test_empty_prompt_does_not_index_out_of_range():+ assert _prompt_already_has_bos(_tokenizer(auto_adds_bos = True), "") is Falsediff --git a/tests/python/test_gemma4_base_bos_token.py b/tests/python/test_gemma4_base_bos_token.pynew file mode 100644index 00000000000..eef48df1162--- /dev/null+++ b/tests/python/test_gemma4_base_bos_token.py@@ -0,0 +1,458 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""Gemma 4 base tokenizers must prepend <bos> at load time.++Every google/gemma-4-* base repo prepends <bos>; no unsloth base mirror does. The delta is in+tokenizer.json's post_processor, not tokenizer_config.json's add_bos_token key, which google+omits on E4B, 31B and 26B-A4B while still prepending. Without the runtime fix, generation+repeats degenerate text. See unslothai/unsloth#7903.++Detection keys off the loaded tokenizer / config, not the Hub repo name, so+local folders and extra quant suffixes still get the fix.+"""++import types+from unittest.mock import patch++import pytest++import unsloth.tokenizer_utils as tu+++class _Tok:+ def __init__(+ self,+ add_bos_token = False,+ bos_token_id = 2,+ processor_class = None,+ chat_template = None,+ eos_token = "<eos>",+ init_kwargs = None,+ ):+ self.add_bos_token = add_bos_token+ self.bos_token_id = bos_token_id+ self.processor_class = processor_class+ self.chat_template = chat_template+ self.eos_token = eos_token+ if init_kwargs is not None:+ self.init_kwargs = init_kwargs+++class _Proc:+ def __init__(+ self,+ tokenizer,+ processor_class = "Gemma4Processor",+ chat_template = None,+ ):+ self.tokenizer = tokenizer+ self.processor_class = processor_class+ self.chat_template = chat_template+++def _gemma4_base(**kwargs):+ kwargs.setdefault("processor_class", "Gemma4Processor")+ return _Tok(**kwargs)+++def test_gemma4_from_processor_class():+ assert tu._is_gemma4_tokenizer(_gemma4_base()) is True+++def test_gemma4_from_init_kwargs():+ tok = _Tok(init_kwargs = {"processor_class": "Gemma4Processor"})+ assert tu._is_gemma4_tokenizer(tok) is True+++def test_gemma4_from_processor_wrapper():+ proc = _Proc(_Tok())+ assert tu._is_gemma4_tokenizer(proc) is True+++def test_gemma3_processor_is_not_gemma4():+ tok = _Tok(processor_class = "Gemma3Processor")+ assert tu._is_gemma4_tokenizer(tok) is False+ assert tu._needs_gemma4_base_bos(tok) is False+++def test_plain_tokenizer_is_not_gemma4():+ assert tu._is_gemma4_tokenizer(_Tok()) is False+++def test_gemma4_config_model_type():+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)+ assert tu._is_gemma4_config(config) is True+ assert tu._needs_gemma4_base_bos(_Tok(), config = config) is True+++def test_gemma4_config_nested_text_config():+ config = types.SimpleNamespace(+ model_type = "gemma4",+ text_config = types.SimpleNamespace(model_type = "gemma4_text"),+ )+ assert tu._is_gemma4_config(config) is True+++def test_name_alone_does_not_trigger_fix():+ tok = _Tok()+ # Repo / folder names are ignored: a generic tokenizer must not flip BOS.+ fixed = tu._fix_gemma4_base_bos_token(tok)+ assert fixed.add_bos_token is False+++def test_fix_sets_flag_for_quant_and_local_shapes():+ tok = _gemma4_base(add_bos_token = False)+ fixed = tu._fix_gemma4_base_bos_token(tok)+ assert fixed.add_bos_token is True+++def test_fix_sets_flag_on_wrapped_processor():+ inner = _Tok(add_bos_token = False)+ proc = _Proc(inner)+ tu._fix_gemma4_base_bos_token(proc)+ assert inner.add_bos_token is True+++def test_fix_skips_chat_template_that_emits_bos():+ tok = _gemma4_base(+ add_bos_token = False,+ chat_template = "{{- bos_token -}}{{ messages }}",+ )+ fixed = tu._fix_gemma4_base_bos_token(tok)+ assert fixed.add_bos_token is False+++def test_fix_skips_turn_eos_instruct():+ tok = _gemma4_base(add_bos_token = False, eos_token = "<turn|>")+ fixed = tu._fix_gemma4_base_bos_token(tok)+ assert fixed.add_bos_token is False+++def test_fix_honors_fix_tokenizer_false():+ tok = _gemma4_base(add_bos_token = False)+ fixed = tu._apply_post_load_tokenizer_fixes(tok, fix_tokenizer = False)+ assert fixed.add_bos_token is False+++def test_load_correct_tokenizer_enables_bos_for_gemma4_base():+ def from_pretrained(model_name, **kwargs):+ return _gemma4_base(add_bos_token = False)++ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):+ result = tu._load_correct_tokenizer("/models/gemma-4-31B-bnb-4bit", fix_tokenizer = True)++ assert result.add_bos_token is True+++def test_load_correct_tokenizer_skips_instruct():+ def from_pretrained(model_name, **kwargs):+ return _gemma4_base(+ add_bos_token = False,+ chat_template = "{{- bos_token -}}",+ eos_token = "<turn|>",+ )++ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):+ result = tu._load_correct_tokenizer("unsloth/gemma-4-E2B-it", fix_tokenizer = True)++ assert result.add_bos_token is False+++def test_load_correct_tokenizer_uses_model_config_when_tokenizer_is_generic():+ # Stripped local tokenizers have no processor_class, but config.model_type is still gemma4.+ def from_pretrained(model_name, **kwargs):+ return _Tok(add_bos_token = False)++ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)+ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):+ result = tu._load_correct_tokenizer(+ "/models/local-gemma4-bnb-4bit",+ fix_tokenizer = True,+ config = config,+ )++ assert result.add_bos_token is True+++def test_fastmodel_processor_path_heals_from_config():+ # FastModel loads Gemma4Processor, then heals after the processor is final.+ inner = _Tok(add_bos_token = False)+ processor = types.SimpleNamespace(+ tokenizer = inner,+ image_processor = object(),+ chat_template = None,+ )+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)++ fixed = tu._apply_post_load_tokenizer_fixes(processor, fix_tokenizer = True, config = config)++ assert fixed is processor+ assert inner.add_bos_token is True+++def test_fastmodel_processor_path_skips_instruct_template():+ inner = _Tok(add_bos_token = False, chat_template = "{{- bos_token -}}")+ processor = types.SimpleNamespace(+ tokenizer = inner,+ image_processor = object(),+ chat_template = "{{- bos_token -}}{{ messages }}",+ )+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)++ tu._apply_post_load_tokenizer_fixes(processor, fix_tokenizer = True, config = config)+ assert inner.add_bos_token is False+++def test_gemma4_e2b_hub_tokenizer_prepends_bos():+ pytest.importorskip("transformers")+ from transformers import AutoTokenizer++ tok = tu.load_correct_tokenizer("unsloth/gemma-4-E2B", fix_tokenizer = True)+ assert tok.add_bos_token is True+ ids = tok("This book is largely concerned with Hobbits,")["input_ids"]+ assert ids[0] == tok.bos_token_id++ # Control: raw Hub tokenizer still omits BOS without the fix.+ raw = AutoTokenizer.from_pretrained("unsloth/gemma-4-E2B", trust_remote_code = True)+ raw_ids = raw("This book is largely concerned with Hobbits,")["input_ids"]+ assert raw_ids[0] != raw.bos_token_id+++def test_chat_template_bos_is_preserved_when_tokenizer_auto_adds():+ tok = _gemma4_base(+ add_bos_token = True,+ chat_template = "{{ bos_token }}{% for m in messages %}{{ m }}{% endfor %}",+ )+ tu._fix_gemma4_base_bos_token(tok)+ assert tok.add_bos_token is True+ assert tok.chat_template.startswith("{{ bos_token }}")++[email protected]("prefix", ["", " \n"])+def test_real_tokenizer_chat_bos_survives_save_reload(tmp_path, prefix):+ from tokenizers import Tokenizer, models, pre_tokenizers, processors+ from transformers import PreTrainedTokenizerFast++ backend = Tokenizer(+ models.WordLevel({"[UNK]": 0, "[PAD]": 1, "<bos>": 2, "Hello": 3}, unk_token = "[UNK]")+ )+ backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()+ backend.post_processor = processors.TemplateProcessing(+ single = "<bos> $A", special_tokens = [("<bos>", 2)]+ )+ tok = PreTrainedTokenizerFast(tokenizer_object = backend, bos_token = "<bos>", unk_token = "[UNK]")+ tok.chat_template = prefix + "{{ bos_token }}Hello"+ config = types.SimpleNamespace(model_type = "gemma4")+ tu._fix_gemma4_base_bos_token(tok, config = config)+ tok.save_pretrained(tmp_path)+ tok = PreTrainedTokenizerFast.from_pretrained(tmp_path)+ tu._fix_gemma4_base_bos_token(tok, config = config)+ messages = [{"role": "user", "content": "Hello"}]+ encoded = tok.apply_chat_template(messages, tokenize = True)+ ids = encoded["input_ids"] if hasattr(encoded, "keys") else encoded+ assert ids == [2, 3]+ rendered = tok.apply_chat_template(messages, tokenize = False)+ assert tok(rendered, add_special_tokens = False)["input_ids"] == [2, 3]+ assert tok("Hello")["input_ids"] == [2, 3]+++def test_export_helper_strips_dict_chat_template_without_crash():+ tok = _gemma4_base(+ add_bos_token = True,+ chat_template = {+ "default": "{{ bos_token }}{% for m in messages %}{{ m }}{% endfor %}",+ "tool_use": "{% for m in messages %}{{ m }}{% endfor %}",+ },+ )+ tu._dedupe_bos_chat_template(tok)+ assert "{{ bos_token }}" not in tok.chat_template["default"]+ assert tok.chat_template["tool_use"].startswith("{% for m in messages %}")+++def test_instruct_template_is_not_stripped_when_tokenizer_does_not_add_bos():+ tok = _gemma4_base(+ add_bos_token = False,+ chat_template = "{{- bos_token -}}{{ messages }}",+ eos_token = "<turn|>",+ )+ tu._fix_gemma4_base_bos_token(tok)+ assert tok.add_bos_token is False+ assert "bos_token" in tok.chat_template+++# Real backends: the fakes above accept any attribute, so they cannot tell a repair from a no-op.+++def _build_fast_tokenizer():+ tokenizers = pytest.importorskip("tokenizers")+ from transformers import PreTrainedTokenizerFast++ backend = tokenizers.Tokenizer(+ tokenizers.models.WordLevel(+ {"<bos>": 0, "<eos>": 1, "hello": 2, "world": 3}, unk_token = None+ )+ )+ backend.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()+ return PreTrainedTokenizerFast(tokenizer_object = backend, bos_token = "<bos>", eos_token = "<eos>")+++def _backend_honors_add_bos_token():+ # Pre-5.x fast tokenizers store add_bos_token without changing what they emit. Gemma 4 needs+ # transformers >= 5.5.0 anyway (loader.py SUPPORTS_GEMMA4), so record it rather than fail.+ try:+ tokenizer = _build_fast_tokenizer()+ except Exception:+ return False+ tokenizer.add_bos_token = True+ return tokenizer("hello world")["input_ids"][0] == tokenizer.bos_token_id+++requires_working_add_bos_token = pytest.mark.skipif(+ not _backend_honors_add_bos_token(),+ reason = "this transformers treats add_bos_token as an inert attribute",+)+++def _real_tokenizer(add_bos = False):+ tokenizer = _build_fast_tokenizer()+ # Gemma 4 is identified by its processor, not by this toy vocabulary.+ tokenizer.processor_class = "Gemma4Processor"+ if add_bos:+ tokenizer.add_bos_token = True+ return tokenizer+++def _ids(+ tokenizer,+ text = "hello world",+ **kwargs,+):+ return tokenizer(text, **kwargs)["input_ids"]+++@requires_working_add_bos_token+def test_real_backend_gains_exactly_one_bos():+ tok = _real_tokenizer()+ assert _ids(tok)[0] != tok.bos_token_id+ tu._fix_gemma4_base_bos_token(tok)+ ids = _ids(tok)+ assert ids[0] == tok.bos_token_id and ids[1] != tok.bos_token_id+++@requires_working_add_bos_token+def test_real_backend_repair_is_idempotent():+ tok = _real_tokenizer()+ tu._fix_gemma4_base_bos_token(tok)+ once = _ids(tok)+ tu._fix_gemma4_base_bos_token(tok)+ assert _ids(tok) == once+++@requires_working_add_bos_token+def test_real_backend_add_special_tokens_false_never_gains_bos():+ tok = _real_tokenizer()+ tu._fix_gemma4_base_bos_token(tok)+ assert tok.bos_token_id not in _ids(tok, add_special_tokens = False)+++@requires_working_add_bos_token+def test_real_backend_already_correct_tokenizer_is_left_alone():+ # google base mirrors report add_bos_token = False and still prepend, so keying on the+ # attribute would rebuild a post_processor that already works.+ tok = _real_tokenizer(add_bos = True)+ before = str(tok._tokenizer.post_processor)+ ids_before = _ids(tok)+ tu._fix_gemma4_base_bos_token(tok)+ assert _ids(tok) == ids_before+ assert str(tok._tokenizer.post_processor) == before+++@requires_working_add_bos_token+def test_real_backend_without_bos_token_does_not_claim_success():+ tok = _real_tokenizer()+ tok.bos_token = None+ tu._fix_gemma4_base_bos_token(tok)+ assert not getattr(tok, "add_bos_token", False) or _ids(tok)[0] == tok.bos_token_id+++ "model_type, expected",+ [+ ("gemma4", True),+ ("gemma4_text", True),+ ("gemma-4", True),+ ("gemma3", False),+ ("gemma3n", False),+ ("gemma3n_text", False),+ ("gemma2", False),+ ("llama", False),+ # A future Gemma 4.5 is a different model with its own BOS policy.+ ("gemma_45", False),+ ("gemma-4.5", False),+ # Substring matching would catch unsloth's own diffusion_gemma4.+ ("diffusion_gemma4", False),+ ],+)+def test_config_model_type_detection_is_anchored(model_type, expected):+ config = types.SimpleNamespace(model_type = model_type, text_config = None)+ assert tu._is_gemma4_config(config) is expected+++ "architecture, expected",+ [+ ("Gemma4ForConditionalGeneration", True),+ ("DiffusionGemma4ForConditionalGeneration", False),+ ("Gemma3ForConditionalGeneration", False),+ ],+)+def test_config_architectures_detection_is_anchored(architecture, expected):+ config = types.SimpleNamespace(+ model_type = "unknown", text_config = None, architectures = [architecture]+ )+ assert tu._is_gemma4_config(config) is expected+++@requires_working_add_bos_token+def test_bos_token_inside_a_jinja_comment_does_not_suppress_the_fix():+ tok = _real_tokenizer()+ tok.chat_template = "{# bos_token is handled elsewhere #}{{ messages }}"+ tu._fix_gemma4_base_bos_token(tok)+ assert _ids(tok)[0] == tok.bos_token_id+++@requires_working_add_bos_token+def test_bos_token_emitted_by_the_template_still_suppresses_the_fix():+ tok = _real_tokenizer()+ tok.chat_template = "{{- bos_token -}}{{ messages }}"+ tu._fix_gemma4_base_bos_token(tok)+ assert _ids(tok)[0] != tok.bos_token_id+++def test_processor_chat_template_is_deduped_too():+ # ProcessorMixin.save_pretrained writes the processor's own chat_template.jinja, so leaving+ # that copy alone exports a second BOS on a VLM.+ emits_bos = "{{ bos_token }}{% for m in messages %}{{ m.content }}{% endfor %}"+ inner = _Tok(add_bos_token = True, chat_template = emits_bos)+ inner.bos_token_id = None # force the attribute fallback in _tokenizer_auto_adds_bos+ processor = types.SimpleNamespace(tokenizer = inner, chat_template = emits_bos)++ tu._dedupe_bos_chat_template(processor)++ assert "bos_token" not in processor.chat_template+ assert "bos_token" not in processor.tokenizer.chat_template+++def test_dedupe_is_a_noop_when_the_tokenizer_does_not_add_bos():+ emits_bos = "{{ bos_token }}hello"+ inner = _Tok(add_bos_token = False, chat_template = emits_bos)+ inner.bos_token_id = None+ processor = types.SimpleNamespace(tokenizer = inner, chat_template = emits_bos)++ tu._dedupe_bos_chat_template(processor)++ assert processor.chat_template == emits_bos+ assert processor.tokenizer.chat_template == emits_bosdiff --git a/unsloth/models/llama.py b/unsloth/models/llama.pyindex dc3993f2927..4e8f0e1b78d 100644--- a/unsloth/models/llama.py+++ b/unsloth/models/llama.py@@ -2794,6 +2794,7 @@ class FastLlamaModel:trust_remote_code = trust_remote_code,fix_tokenizer = fix_tokenizer,revision = tokenizer_revision,+ config = getattr(model, "config", None),**_tokenizer_cache_kwargs,)diff --git a/unsloth/models/loader.py b/unsloth/models/loader.pyindex 0313cc75c2b..0092c0ae128 100644--- a/unsloth/models/loader.py+++ b/unsloth/models/loader.py@@ -513,7 +513,7 @@ class FastLanguageModel(FastLlamaModel):device_map = device_map,device_map_planner_kwargs = device_map_planner_kwargs,rope_scaling = rope_scaling, # [TODO] No effect- fix_tokenizer = fix_tokenizer, # [TODO] No effect+ fix_tokenizer = fix_tokenizer,trust_remote_code = trust_remote_code,use_gradient_checkpointing = use_gradient_checkpointing,resize_model_vocab = resize_model_vocab, # [TODO] No effect@@ -940,7 +940,7 @@ class FastLanguageModel(FastLlamaModel):device_map = device_map,device_map_planner_kwargs = device_map_planner_kwargs,rope_scaling = rope_scaling, # [TODO] No effect- fix_tokenizer = fix_tokenizer, # [TODO] No effect+ fix_tokenizer = fix_tokenizer,trust_remote_code = trust_remote_code,use_gradient_checkpointing = use_gradient_checkpointing,resize_model_vocab = resize_model_vocab, # [TODO] No effect@@ -1231,7 +1231,7 @@ class FastModel(FastBaseModel):# Planner hints for device_map = "unsloth"; see resolve_unsloth_device_map.device_map_planner_kwargs = None,rope_scaling = None, # [TODO] No effect- fix_tokenizer = True, # [TODO] No effect+ fix_tokenizer = True,trust_remote_code = False,use_gradient_checkpointing = "unsloth",resize_model_vocab = None, # [TODO] No effect@@ -2015,6 +2015,7 @@ class FastModel(FastBaseModel):token = token,device_map = device_map,device_map_planner_kwargs = device_map_planner_kwargs,+ fix_tokenizer = fix_tokenizer,trust_remote_code = trust_remote_code,revision = model_revision,tokenizer_revision = _revision_for_tokenizer_repo(diff --git a/unsloth/models/vision.py b/unsloth/models/vision.pyindex 32056863c45..c2d8968bb73 100644--- a/unsloth/models/vision.py+++ b/unsloth/models/vision.py@@ -1124,6 +1124,7 @@ class FastBaseModel:# True when auto_config came from the caller. It cannot be inferred here: FastModel pops config# out of kwargs before this sees them, so it looks exactly like one we resolved ourselves.auto_config_from_caller = False,+ fix_tokenizer = True,**kwargs,):user_config = kwargs.pop("config", None)@@ -2065,6 +2066,15 @@ class FastBaseModel:"or set HF_HUB_OFFLINE=1 to force local loading. ""Otherwise please check that the model has a tokenizer.") from _last_resort_err+ # FastModel never calls load_correct_tokenizer; heal Gemma 4 base BOS from+ # the finalized processor / model config (unslothai/unsloth#7903).+ from ..tokenizer_utils import _apply_post_load_tokenizer_fixes++ tokenizer = _apply_post_load_tokenizer_fixes(+ tokenizer,+ fix_tokenizer = fix_tokenizer,+ config = auto_config if auto_config is not None else getattr(model, "config", None),+ )patch_saving_functions(tokenizer, vision = True)# Fix gradient accumulation; see #4982.diff --git a/unsloth/save.py b/unsloth/save.pyindex 2272006dd6f..c0545cc1a36 100644--- a/unsloth/save.py+++ b/unsloth/save.py@@ -2877,31 +2877,24 @@ def upload_to_huggingface(def fix_tokenizer_bos_token(tokenizer):+ from .tokenizer_utils import (+ _chat_template_emits_bos,+ _dedupe_bos_chat_template,+ _tokenizer_auto_adds_bos,+ )+fix_bos_token = Falsechat_template = getattr(tokenizer, "chat_template", None)- if tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None):- if chat_template is not None and (- tokenizer.bos_token in chat_template- or "{bos_token}" in chat_template.replace(" ", "")- or "{bos_token+" in chat_template.replace(" ", "")- ):+ # A processor cannot be called without an image, so ask its inner tokenizer instead.+ if _tokenizer_auto_adds_bos(getattr(tokenizer, "tokenizer", tokenizer)):+ if _chat_template_emits_bos(tokenizer):fix_bos_token = Truelogger.warning("Unsloth: ##### The current model auto adds a BOS token.\n""Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily.")-- new_chat_template = re.sub(- r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template- )- new_chat_template = re.sub(- r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}",- "",- new_chat_template,- )-- tokenizer.chat_template = new_chat_template+ _dedupe_bos_chat_template(tokenizer)return fix_bos_token, chat_template@@ -4635,10 +4628,13 @@ def unsloth_save_pretrained_gguf(# preflight sized, so it measured the disk these files land on.gguf_directory = _gguf_output_directory(save_directory)- if is_processor:- fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer.tokenizer)- else:- fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)+ # save_pretrained writes the processor's own chat_template.jinja, so dedupe and restore both.+ from .tokenizer_utils import _tokenizer_objects++ old_chat_templates = [+ (obj, getattr(obj, "chat_template", None)) for obj in _tokenizer_objects(tokenizer)+ ]+ fix_bos_token, _ = fix_tokenizer_bos_token(tokenizer)# Resolve the imatrix (download, validate, rename *.gguf_file) up front, so a bad path or an unavailable# upstream fails before the expensive merge and never reaches the IQ-quant gate.@@ -4704,12 +4700,13 @@ def unsloth_save_pretrained_gguf(f"{_offloaded_parameter_hint(self)}") from e+ if fix_bos_token:+ for _template_owner, _old_template in old_chat_templates:+ _template_owner.chat_template = _old_template+if is_processor:tokenizer = tokenizer.tokenizer- if fix_bos_token:- tokenizer.chat_template = old_chat_template-for _ in range(3):import gcgc.collect()diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex bf8b4e5753f..9b43470535c 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -68,6 +68,178 @@ IGNORED_TOKENIZER_NAMES = frozenset()os.environ["UNSLOTH_IGNORED_TOKENIZER_NAMES"] = "\n".join(IGNORED_TOKENIZER_NAMES)+# gemma-4 base mirrors do not prepend <bos> (post_processor.single = [A], google's is [<bos>, A]).+# Not keyed on add_bos_token (google omits it on E4B/31B/26B-A4B and still prepends) nor on repo+# name. -it is skipped: its chat_template emits BOS. unslothai/unsloth#7903+_GEMMA4_INSTRUCT_EOS = "<turn|>"++# Anchored: a future gemma-4.5 / gemma_45 has its own BOS policy, and DiffusionGemma4... is not gemma 4.+_GEMMA4_NAME_RE = re.compile(r"^gemma[\s_-]*4(?![\d.])", re.IGNORECASE)++# A bos_token inside a Jinja comment renders to nothing, so it is not an emission.+_JINJA_COMMENT_RE = re.compile(r"\{#.*?#\}", re.DOTALL)+++def _tokenizer_objects(tokenizer):+ """Yield the processor and its inner tokenizer once each."""+ seen = []+ for obj in (tokenizer, getattr(tokenizer, "tokenizer", None)):+ # Identity, not equality: a wrapper defining __eq__ would compare fields.+ if obj is None or any(obj is s for s in seen):+ continue+ seen.append(obj)+ yield obj+++def _is_gemma4_config(config):+ if config is None:+ return False+ for cfg in (config, getattr(config, "text_config", None)):+ if cfg is None:+ continue+ model_type = getattr(cfg, "model_type", None)+ if model_type is not None and _GEMMA4_NAME_RE.match(str(model_type)):+ return True+ if any(_GEMMA4_NAME_RE.match(str(a)) for a in (getattr(cfg, "architectures", None) or [])):+ return True+ return False+++def _is_gemma4_tokenizer(tokenizer):+ """processor_class = Gemma4Processor, usually only in init_kwargs.++ The class is GemmaTokenizer, so a class-name check never fires.+ """+ for obj in _tokenizer_objects(tokenizer):+ processor_class = getattr(obj, "processor_class", None)+ if processor_class is None:+ processor_class = (getattr(obj, "init_kwargs", None) or {}).get("processor_class")+ if str(processor_class or "").strip().lower() == "gemma4processor":+ return True+ return False+++def _chat_template_emits_bos(tokenizer):+ for obj in _tokenizer_objects(tokenizer):+ template = getattr(obj, "chat_template", None)+ if not template:+ continue+ chunks = template.values() if isinstance(template, dict) else (template,)+ for chunk in chunks:+ text = _JINJA_COMMENT_RE.sub("", chunk if isinstance(chunk, str) else str(chunk))+ if "bos_token" in text or "<bos>" in text:+ return True+ return False+++def _tokenizer_auto_adds_bos(tokenizer):+ """Does this tokenizer already emit <bos>?++ Not add_bos_token: google/gemma-4-E2B reports False and still prepends.+ """+ bos_token_id = getattr(tokenizer, "bos_token_id", None)+ if bos_token_id is None:+ return bool(getattr(tokenizer, "add_bos_token", False))+ try:+ input_ids = tokenizer("A")["input_ids"]+ except Exception:+ return bool(getattr(tokenizer, "add_bos_token", False))+ # Processors return a batched nested list.+ while (+ isinstance(input_ids, (list, tuple))+ and input_ids+ and isinstance(input_ids[0], (list, tuple))+ ):+ input_ids = input_ids[0]+ return bool(input_ids) and input_ids[0] == bos_token_id+++def _strip_bos_from_chat_template_text(chat_template):+ if not isinstance(chat_template, str) or not chat_template:+ return chat_template+ stripped = re.sub(r"\{[\s\-]*\{[\s\-]*bos\_token[\s\-]*\}[\s\-]*\}", "", chat_template, count = 1)+ return re.sub(r"\{[\s\-]*\{[\s\-]*bos\_token[\s\-]*\+[\s\-]*", "", stripped, count = 1)+++def _dedupe_bos_chat_template(tokenizer):+ """Drop template-emitted BOS when the tokenizer already prepends one.++ A processor keeps its own copy and save_pretrained writes that one, so both must lose it.+ Only the inner tokenizer can be asked: calling a processor needs an image.+ """+ if not _tokenizer_auto_adds_bos(getattr(tokenizer, "tokenizer", tokenizer)):+ return+ for obj in _tokenizer_objects(tokenizer):+ template = getattr(obj, "chat_template", None)+ if template is None:+ continue+ if isinstance(template, dict):+ obj.chat_template = {+ k: _strip_bos_from_chat_template_text(v) if isinstance(v, str) else v+ for k, v in template.items()+ }+ elif isinstance(template, str):+ obj.chat_template = _strip_bos_from_chat_template_text(template)+++def _is_gemma4_instruct_tokenizer(tokenizer):+ """-it emits BOS from its chat template, so flipping the flag would double it."""+ if _chat_template_emits_bos(tokenizer):+ return True+ return any(+ getattr(o, "eos_token", None) == _GEMMA4_INSTRUCT_EOS for o in _tokenizer_objects(tokenizer)+ )+++def _needs_gemma4_base_bos(tokenizer, config = None):+ if tokenizer is None or _is_gemma4_instruct_tokenizer(tokenizer):+ return False+ return _is_gemma4_tokenizer(tokenizer) or _is_gemma4_config(config)+++def _enable_add_bos_token(tokenizer):+ """Make the tokenizer prepend <bos>, and warn if that could not be done.++ Nothing is recorded for save_pretrained: transformers 5.x drops add_bos_token from+ tokenizer_config.json, so the setter's post_processor rewrite is what persists.+ """+ for obj in _tokenizer_objects(tokenizer):+ # Already correct: keep its post_processor rather than rebuilding one.+ if _tokenizer_auto_adds_bos(obj):+ continue+ try:+ obj.add_bos_token = True+ except AttributeError:+ continue # Read-only property, or rejects the attribute. Not our tokenizer.+ except ValueError as error:+ # Raised when bos_token is None; swallowing it would fake success.+ logger.warning(f"Unsloth: Could not enable add_bos_token for Gemma 4: {error}")+ continue+ # Pre-5.x fast tokenizers accept the attribute without changing what they emit.+ if not _tokenizer_auto_adds_bos(obj):+ logger.warning(+ "Unsloth: add_bos_token was set for this Gemma 4 base tokenizer but it still "+ "does not prepend <bos>. See unslothai/unsloth#7903."+ )+++def _fix_gemma4_base_bos_token(tokenizer, config = None):+ if tokenizer is None or not _needs_gemma4_base_bos(tokenizer, config = config):+ return tokenizer+ _enable_add_bos_token(tokenizer)+ return tokenizer+++def _apply_post_load_tokenizer_fixes(+ tokenizer,+ fix_tokenizer = True,+ config = None,+):+ if not fix_tokenizer:+ return tokenizer+ return _fix_gemma4_base_bos_token(tokenizer, config = config)++# A KAGGLE_* variable is not a Kaggle kernel: the Kaggle CLI reads KAGGLE_USERNAME / KAGGLE_KEY on# ordinary machines, and redirecting their tokenizer cache to /tmp because of it was wrong.from .disk_utils import (@@ -550,6 +722,7 @@ def _load_correct_tokenizer(cache_dir = "huggingface_tokenizers_cache",fix_tokenizer = True,revision = None,+ config = None,):if IS_COLAB_ENVIRONMENT:cache_dir = cache_dir@@ -596,13 +769,13 @@ def _load_correct_tokenizer()if not fix_tokenizer or tokenizer_name.lower() in IGNORED_TOKENIZER_NAMES:- return fast_tokenizer+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)# Ignore Mistral ones - they're a bit weird to handle!elif "mistral" in tokenizer_name.lower():- return fast_tokenizer+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)# Ignore Phi-4 ones as wellelif "phi-4" in tokenizer_name.lower():- return fast_tokenizer+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)elif slow_tokenizer is not None:if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token@@ -611,13 +784,17 @@ def _load_correct_tokenizer(# Confirm whether slow and fast are equivalent.if assert_same_tokenization(slow_tokenizer, fast_tokenizer):- return fast_tokenizer+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)else:logger.warning(f"Unsloth: Will load {tokenizer_name} as a legacy tokenizer.")- return convert_to_fast_tokenizer(slow_tokenizer)+ return _apply_post_load_tokenizer_fixes(+ convert_to_fast_tokenizer(slow_tokenizer),+ fix_tokenizer,+ config,+ )passelse:- return fast_tokenizer+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)def _fix_pad_token(tokenizer):@@ -649,6 +826,7 @@ def load_correct_tokenizer(cache_dir = "huggingface_tokenizers_cache",fix_tokenizer = True,revision = None,+ config = None,):tokenizer = _load_correct_tokenizer(tokenizer_name = tokenizer_name,@@ -659,6 +837,7 @@ def load_correct_tokenizer(cache_dir = cache_dir,fix_tokenizer = fix_tokenizer,revision = revision,+ config = config,)if fix_tokenizer: