Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

fix(tokenizer): enable add_bos_token for Gemma 4 base models (#10312)

Sourav Rajvi30529a605178
diff --git a/studio/backend/core/inference/inference.py b/studio/backend/core/inference/inference.py
index b9094102bcc..4a0d74b6a1a 100644
--- a/studio/backend/core/inference/inference.py
+++ b/studio/backend/core/inference/inference.py
@@ -259,6 +259,25 @@ class _GenerationThreadError(RuntimeError):
"""Generation worker failures that should propagate through stream routes."""
+def _prompt_already_has_bos(tokenizer, prompt):
+ """Did the rendered chat template emit BOS itself?
+
+ Most do, so the tokenizer must not add a second. Some do not (zephyr, tinyllama-chat), and
+ suppressing special tokens there drops BOS entirely.
+ """
+ tok = getattr(tokenizer, "tokenizer", tokenizer)
+ bos_token_id = getattr(tok, "bos_token_id", None)
+ if bos_token_id is None:
+ return False
+ try:
+ ids = tok(prompt, add_special_tokens = False)["input_ids"]
+ except Exception:
+ return False
+ while isinstance(ids, (list, tuple)) and ids and isinstance(ids[0], (list, tuple)):
+ ids = ids[0]
+ return bool(len(ids)) and ids[0] == bos_token_id
+
+
class InferenceBackend:
"""Unified inference backend supporting text, vision, and LoRA models"""
@@ -1190,6 +1209,7 @@ class InferenceBackend:
else:
template_messages = messages
reasoning_channel_markers_resolved = False
+ add_special_tokens = True
try:
if not (hasattr(tokenizer, "chat_template") and tokenizer.chat_template):
raise ValueError(
@@ -1234,6 +1254,8 @@ class InferenceBackend:
formatted_prompt = render_result.prompt
reasoning_channel_markers = render_result.reasoning_channel_markers
reasoning_channel_markers_resolved = True
+ # Suppress the tokenizer's BOS only when the template already emitted one.
+ add_special_tokens = not _prompt_already_has_bos(tokenizer, formatted_prompt)
logger.debug(f"Formatted prompt: {formatted_prompt[:200]}...")
except Exception as e:
@@ -1260,6 +1282,7 @@ class InferenceBackend:
reasoning_channel_markers = reasoning_channel_markers,
reasoning_channel_markers_resolved = reasoning_channel_markers_resolved,
continued = bool(continue_final_message and trailing_assistant_text(template_messages)),
+ add_special_tokens = add_special_tokens,
)
def _generate_vision_response(
@@ -1459,7 +1482,11 @@ class InferenceBackend:
formatted_prompt = self.format_chat_prompt(
messages, system_prompt, continue_final_message = continue_final_message
)
- inputs = raw_tokenizer(formatted_prompt, return_tensors = "pt").to(model.device)
+ inputs = raw_tokenizer(
+ formatted_prompt,
+ return_tensors = "pt",
+ add_special_tokens = not _prompt_already_has_bos(raw_tokenizer, formatted_prompt),
+ ).to(model.device)
prompt_text = formatted_prompt
# Stream with TextIteratorStreamer + background thread
@@ -1936,9 +1963,13 @@ class InferenceBackend:
reasoning_channel_markers = None,
reasoning_channel_markers_resolved: bool = False,
continued: bool = False,
+ add_special_tokens: bool = True,
) -> Generator[str, None, None]:
"""Generate a streaming text response (text models only).
+ Rendered chat prompts pass add_special_tokens=False; raw prompts keep
+ the tokenizer defaults, including BOS insertion for base models.
+
_adapter_state: if not None, the background thread toggles adapters
before model.generate(), under _generation_lock.
``presence_penalty`` matches the GGUF sampling path via a logits processor (0 disables it).
@@ -1958,7 +1989,9 @@ class InferenceBackend:
tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
try:
- inputs = tokenizer(prompt, return_tensors = "pt").to(model.device)
+ inputs = tokenizer(
+ prompt, return_tensors = "pt", add_special_tokens = add_special_tokens
+ ).to(model.device)
import threading
diff --git a/studio/backend/tests/test_chat_prompt_bos.py b/studio/backend/tests/test_chat_prompt_bos.py
new file mode 100644
index 00000000000..6c95cb0ddfd
--- /dev/null
+++ b/studio/backend/tests/test_chat_prompt_bos.py
@@ -0,0 +1,173 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. See /studio/LICENSE.AGPL-3.0
+
+"""A rendered chat prompt must carry exactly one BOS.
+
+Most templates emit it, so letting the tokenizer add another doubles it; zephyr and
+tinyllama-chat emit none, so suppressing specials unconditionally drops it. The deciding
+fact is whether the rendered text already starts with BOS.
+"""
+
+from __future__ import annotations
+
+import importlib
+import importlib.machinery
+import sys
+import types
+from pathlib import Path
+from unittest.mock import MagicMock
+
+import pytest
+
+_BACKEND = Path(__file__).resolve().parent.parent
+if str(_BACKEND) not in sys.path:
+ sys.path.insert(0, str(_BACKEND))
+
+_STUBBED: list[str] = []
+
+
+def _stub_if_missing(
+ name,
+ attrs = (),
+ named_spec = False,
+):
+ """Register a stub for a dep this job does not install. A real install is left alone.
+
+ Same helper and reason as test_vision_client_tools.py: core.inference.inference imports
+ unsloth and trl at module scope, which studio-backend-ci.yml does not install, so
+ unstubbed this file fails COLLECTION and takes the whole job down.
+
+ ``named_spec`` gives the stub a real ModuleSpec, which only torchao needs: transformers
+ probes it with find_spec, which raises ValueError on ``__spec__ = None``.
+ """
+ if name in sys.modules:
+ return
+ try:
+ importlib.import_module(name)
+ return
+ except Exception: # noqa: BLE001 - unusable here either way, so stub it
+ pass
+ _STUBBED.append(name)
+ module = types.ModuleType(name)
+ module.__spec__ = importlib.machinery.ModuleSpec(name, None) if named_spec else None
+ module.__version__ = "0.0.0"
+ module.__getattr__ = lambda _attr: MagicMock()
+ for attr in attrs:
+ setattr(module, attr, MagicMock())
+ sys.modules[name] = module
+ parent, _, child = name.rpartition(".")
+ if parent and parent in sys.modules:
+ setattr(sys.modules[parent], child, module)
+
+
+# Fires only where torchao is installed but unusable against the local torch, in which
+# case transformers.quantizers imports it and poisons transformers for every later module.
+for _torchao in (
+ "torchao",
+ "torchao.prototype",
+ "torchao.prototype.safetensors",
+ "torchao.prototype.safetensors.safetensors_support",
+ "torchao.prototype.safetensors.safetensors_utils",
+ "torchao.quantization",
+ "torchao.dtypes",
+ "torchao.float8",
+ "torchao.utils",
+):
+ _stub_if_missing(_torchao, named_spec = True)
+
+_stub_if_missing("unsloth", ("FastLanguageModel", "FastVisionModel", "is_bfloat16_supported"))
+_stub_if_missing("unsloth.chat_templates", ("get_chat_template",))
+_stub_if_missing("unsloth_zoo")
+_stub_if_missing("trl", ("SFTTrainer", "SFTConfig"))
+
+from core.inference.inference import _prompt_already_has_bos # noqa: E402
+
+# Drop the stubs now the name is bound; one left behind outlives this module.
+for _name in reversed(_STUBBED):
+ sys.modules.pop(_name, None)
+
+
+def _tokenizer(auto_adds_bos: bool):
+ tokenizers = pytest.importorskip("tokenizers")
+ from transformers import PreTrainedTokenizerFast
+
+ vocab = {"<s>": 0, "</s>": 1, "hello": 2, "world": 3, "user": 4}
+ backend = tokenizers.Tokenizer(tokenizers.models.WordLevel(vocab, unk_token = None))
+ backend.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()
+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend, bos_token = "<s>", eos_token = "</s>")
+ if auto_adds_bos:
+ tokenizer.add_bos_token = True
+ return tokenizer
+
+
+def _encode(tokenizer, prompt):
+ add_special_tokens = not _prompt_already_has_bos(tokenizer, prompt)
+ return tokenizer(prompt, add_special_tokens = add_special_tokens)["input_ids"]
+
+
+def _leading_bos(ids, bos_token_id):
+ count = 0
+ for token_id in ids:
+ if token_id != bos_token_id:
+ break
+ count += 1
+ return count
+
+
[email protected]("auto_adds_bos", [True, False])
[email protected]("template_emits_bos", [True, False])
+def test_rendered_prompt_carries_exactly_one_bos(template_emits_bos, auto_adds_bos):
+ tokenizer = _tokenizer(auto_adds_bos)
+ if not (template_emits_bos or auto_adds_bos):
+ pytest.skip("nothing supplies BOS, so there is none to count")
+ prompt = ("<s> " if template_emits_bos else "") + "user hello world"
+
+ assert _leading_bos(_encode(tokenizer, prompt), tokenizer.bos_token_id) == 1
+
+
+def test_template_that_omits_bos_keeps_the_tokenizer_one():
+ # Blanket add_special_tokens = False regressed exactly this shape.
+ tokenizer = _tokenizer(auto_adds_bos = True)
+ prompt = "user hello world"
+
+ assert _prompt_already_has_bos(tokenizer, prompt) is False
+ assert _encode(tokenizer, prompt)[0] == tokenizer.bos_token_id
+
+
+def test_template_that_emits_bos_is_not_doubled():
+ tokenizer = _tokenizer(auto_adds_bos = True)
+ prompt = "<s> user hello world"
+
+ assert _prompt_already_has_bos(tokenizer, prompt) is True
+ assert _leading_bos(_encode(tokenizer, prompt), tokenizer.bos_token_id) == 1
+
+
+def test_tokenizer_without_a_bos_token_is_left_alone():
+ tokenizer = _tokenizer(auto_adds_bos = False)
+ tokenizer.bos_token = None
+
+ assert _prompt_already_has_bos(tokenizer, "user hello") is False
+
+
+def test_a_tokenizer_that_raises_is_treated_as_having_no_bos():
+ class _Raises:
+ bos_token_id = 0
+
+ def __call__(self, *args, **kwargs):
+ raise RuntimeError("needs an image")
+
+ assert _prompt_already_has_bos(_Raises(), "user hello") is False
+
+
+def test_a_processor_is_probed_through_its_inner_tokenizer():
+ import types
+
+ tokenizer = _tokenizer(auto_adds_bos = True)
+ processor = types.SimpleNamespace(tokenizer = tokenizer)
+
+ assert _prompt_already_has_bos(processor, "<s> user hello") is True
+ assert _prompt_already_has_bos(processor, "user hello") is False
+
+
+def test_empty_prompt_does_not_index_out_of_range():
+ assert _prompt_already_has_bos(_tokenizer(auto_adds_bos = True), "") is False
diff --git a/tests/python/test_gemma4_base_bos_token.py b/tests/python/test_gemma4_base_bos_token.py
new file mode 100644
index 00000000000..eef48df1162
--- /dev/null
+++ b/tests/python/test_gemma4_base_bos_token.py
@@ -0,0 +1,458 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""Gemma 4 base tokenizers must prepend <bos> at load time.
+
+Every google/gemma-4-* base repo prepends <bos>; no unsloth base mirror does. The delta is in
+tokenizer.json's post_processor, not tokenizer_config.json's add_bos_token key, which google
+omits on E4B, 31B and 26B-A4B while still prepending. Without the runtime fix, generation
+repeats degenerate text. See unslothai/unsloth#7903.
+
+Detection keys off the loaded tokenizer / config, not the Hub repo name, so
+local folders and extra quant suffixes still get the fix.
+"""
+
+import types
+from unittest.mock import patch
+
+import pytest
+
+import unsloth.tokenizer_utils as tu
+
+
+class _Tok:
+ def __init__(
+ self,
+ add_bos_token = False,
+ bos_token_id = 2,
+ processor_class = None,
+ chat_template = None,
+ eos_token = "<eos>",
+ init_kwargs = None,
+ ):
+ self.add_bos_token = add_bos_token
+ self.bos_token_id = bos_token_id
+ self.processor_class = processor_class
+ self.chat_template = chat_template
+ self.eos_token = eos_token
+ if init_kwargs is not None:
+ self.init_kwargs = init_kwargs
+
+
+class _Proc:
+ def __init__(
+ self,
+ tokenizer,
+ processor_class = "Gemma4Processor",
+ chat_template = None,
+ ):
+ self.tokenizer = tokenizer
+ self.processor_class = processor_class
+ self.chat_template = chat_template
+
+
+def _gemma4_base(**kwargs):
+ kwargs.setdefault("processor_class", "Gemma4Processor")
+ return _Tok(**kwargs)
+
+
+def test_gemma4_from_processor_class():
+ assert tu._is_gemma4_tokenizer(_gemma4_base()) is True
+
+
+def test_gemma4_from_init_kwargs():
+ tok = _Tok(init_kwargs = {"processor_class": "Gemma4Processor"})
+ assert tu._is_gemma4_tokenizer(tok) is True
+
+
+def test_gemma4_from_processor_wrapper():
+ proc = _Proc(_Tok())
+ assert tu._is_gemma4_tokenizer(proc) is True
+
+
+def test_gemma3_processor_is_not_gemma4():
+ tok = _Tok(processor_class = "Gemma3Processor")
+ assert tu._is_gemma4_tokenizer(tok) is False
+ assert tu._needs_gemma4_base_bos(tok) is False
+
+
+def test_plain_tokenizer_is_not_gemma4():
+ assert tu._is_gemma4_tokenizer(_Tok()) is False
+
+
+def test_gemma4_config_model_type():
+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)
+ assert tu._is_gemma4_config(config) is True
+ assert tu._needs_gemma4_base_bos(_Tok(), config = config) is True
+
+
+def test_gemma4_config_nested_text_config():
+ config = types.SimpleNamespace(
+ model_type = "gemma4",
+ text_config = types.SimpleNamespace(model_type = "gemma4_text"),
+ )
+ assert tu._is_gemma4_config(config) is True
+
+
+def test_name_alone_does_not_trigger_fix():
+ tok = _Tok()
+ # Repo / folder names are ignored: a generic tokenizer must not flip BOS.
+ fixed = tu._fix_gemma4_base_bos_token(tok)
+ assert fixed.add_bos_token is False
+
+
+def test_fix_sets_flag_for_quant_and_local_shapes():
+ tok = _gemma4_base(add_bos_token = False)
+ fixed = tu._fix_gemma4_base_bos_token(tok)
+ assert fixed.add_bos_token is True
+
+
+def test_fix_sets_flag_on_wrapped_processor():
+ inner = _Tok(add_bos_token = False)
+ proc = _Proc(inner)
+ tu._fix_gemma4_base_bos_token(proc)
+ assert inner.add_bos_token is True
+
+
+def test_fix_skips_chat_template_that_emits_bos():
+ tok = _gemma4_base(
+ add_bos_token = False,
+ chat_template = "{{- bos_token -}}{{ messages }}",
+ )
+ fixed = tu._fix_gemma4_base_bos_token(tok)
+ assert fixed.add_bos_token is False
+
+
+def test_fix_skips_turn_eos_instruct():
+ tok = _gemma4_base(add_bos_token = False, eos_token = "<turn|>")
+ fixed = tu._fix_gemma4_base_bos_token(tok)
+ assert fixed.add_bos_token is False
+
+
+def test_fix_honors_fix_tokenizer_false():
+ tok = _gemma4_base(add_bos_token = False)
+ fixed = tu._apply_post_load_tokenizer_fixes(tok, fix_tokenizer = False)
+ assert fixed.add_bos_token is False
+
+
+def test_load_correct_tokenizer_enables_bos_for_gemma4_base():
+ def from_pretrained(model_name, **kwargs):
+ return _gemma4_base(add_bos_token = False)
+
+ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):
+ result = tu._load_correct_tokenizer("/models/gemma-4-31B-bnb-4bit", fix_tokenizer = True)
+
+ assert result.add_bos_token is True
+
+
+def test_load_correct_tokenizer_skips_instruct():
+ def from_pretrained(model_name, **kwargs):
+ return _gemma4_base(
+ add_bos_token = False,
+ chat_template = "{{- bos_token -}}",
+ eos_token = "<turn|>",
+ )
+
+ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):
+ result = tu._load_correct_tokenizer("unsloth/gemma-4-E2B-it", fix_tokenizer = True)
+
+ assert result.add_bos_token is False
+
+
+def test_load_correct_tokenizer_uses_model_config_when_tokenizer_is_generic():
+ # Stripped local tokenizers have no processor_class, but config.model_type is still gemma4.
+ def from_pretrained(model_name, **kwargs):
+ return _Tok(add_bos_token = False)
+
+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)
+ with patch.object(tu, "AutoTokenizer", types.SimpleNamespace(from_pretrained = from_pretrained)):
+ result = tu._load_correct_tokenizer(
+ "/models/local-gemma4-bnb-4bit",
+ fix_tokenizer = True,
+ config = config,
+ )
+
+ assert result.add_bos_token is True
+
+
+def test_fastmodel_processor_path_heals_from_config():
+ # FastModel loads Gemma4Processor, then heals after the processor is final.
+ inner = _Tok(add_bos_token = False)
+ processor = types.SimpleNamespace(
+ tokenizer = inner,
+ image_processor = object(),
+ chat_template = None,
+ )
+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)
+
+ fixed = tu._apply_post_load_tokenizer_fixes(processor, fix_tokenizer = True, config = config)
+
+ assert fixed is processor
+ assert inner.add_bos_token is True
+
+
+def test_fastmodel_processor_path_skips_instruct_template():
+ inner = _Tok(add_bos_token = False, chat_template = "{{- bos_token -}}")
+ processor = types.SimpleNamespace(
+ tokenizer = inner,
+ image_processor = object(),
+ chat_template = "{{- bos_token -}}{{ messages }}",
+ )
+ config = types.SimpleNamespace(model_type = "gemma4", text_config = None)
+
+ tu._apply_post_load_tokenizer_fixes(processor, fix_tokenizer = True, config = config)
+ assert inner.add_bos_token is False
+
+
+def test_gemma4_e2b_hub_tokenizer_prepends_bos():
+ pytest.importorskip("transformers")
+ from transformers import AutoTokenizer
+
+ tok = tu.load_correct_tokenizer("unsloth/gemma-4-E2B", fix_tokenizer = True)
+ assert tok.add_bos_token is True
+ ids = tok("This book is largely concerned with Hobbits,")["input_ids"]
+ assert ids[0] == tok.bos_token_id
+
+ # Control: raw Hub tokenizer still omits BOS without the fix.
+ raw = AutoTokenizer.from_pretrained("unsloth/gemma-4-E2B", trust_remote_code = True)
+ raw_ids = raw("This book is largely concerned with Hobbits,")["input_ids"]
+ assert raw_ids[0] != raw.bos_token_id
+
+
+def test_chat_template_bos_is_preserved_when_tokenizer_auto_adds():
+ tok = _gemma4_base(
+ add_bos_token = True,
+ chat_template = "{{ bos_token }}{% for m in messages %}{{ m }}{% endfor %}",
+ )
+ tu._fix_gemma4_base_bos_token(tok)
+ assert tok.add_bos_token is True
+ assert tok.chat_template.startswith("{{ bos_token }}")
+
+
[email protected]("prefix", ["", " \n"])
+def test_real_tokenizer_chat_bos_survives_save_reload(tmp_path, prefix):
+ from tokenizers import Tokenizer, models, pre_tokenizers, processors
+ from transformers import PreTrainedTokenizerFast
+
+ backend = Tokenizer(
+ models.WordLevel({"[UNK]": 0, "[PAD]": 1, "<bos>": 2, "Hello": 3}, unk_token = "[UNK]")
+ )
+ backend.pre_tokenizer = pre_tokenizers.WhitespaceSplit()
+ backend.post_processor = processors.TemplateProcessing(
+ single = "<bos> $A", special_tokens = [("<bos>", 2)]
+ )
+ tok = PreTrainedTokenizerFast(tokenizer_object = backend, bos_token = "<bos>", unk_token = "[UNK]")
+ tok.chat_template = prefix + "{{ bos_token }}Hello"
+ config = types.SimpleNamespace(model_type = "gemma4")
+ tu._fix_gemma4_base_bos_token(tok, config = config)
+ tok.save_pretrained(tmp_path)
+ tok = PreTrainedTokenizerFast.from_pretrained(tmp_path)
+ tu._fix_gemma4_base_bos_token(tok, config = config)
+ messages = [{"role": "user", "content": "Hello"}]
+ encoded = tok.apply_chat_template(messages, tokenize = True)
+ ids = encoded["input_ids"] if hasattr(encoded, "keys") else encoded
+ assert ids == [2, 3]
+ rendered = tok.apply_chat_template(messages, tokenize = False)
+ assert tok(rendered, add_special_tokens = False)["input_ids"] == [2, 3]
+ assert tok("Hello")["input_ids"] == [2, 3]
+
+
+def test_export_helper_strips_dict_chat_template_without_crash():
+ tok = _gemma4_base(
+ add_bos_token = True,
+ chat_template = {
+ "default": "{{ bos_token }}{% for m in messages %}{{ m }}{% endfor %}",
+ "tool_use": "{% for m in messages %}{{ m }}{% endfor %}",
+ },
+ )
+ tu._dedupe_bos_chat_template(tok)
+ assert "{{ bos_token }}" not in tok.chat_template["default"]
+ assert tok.chat_template["tool_use"].startswith("{% for m in messages %}")
+
+
+def test_instruct_template_is_not_stripped_when_tokenizer_does_not_add_bos():
+ tok = _gemma4_base(
+ add_bos_token = False,
+ chat_template = "{{- bos_token -}}{{ messages }}",
+ eos_token = "<turn|>",
+ )
+ tu._fix_gemma4_base_bos_token(tok)
+ assert tok.add_bos_token is False
+ assert "bos_token" in tok.chat_template
+
+
+# Real backends: the fakes above accept any attribute, so they cannot tell a repair from a no-op.
+
+
+def _build_fast_tokenizer():
+ tokenizers = pytest.importorskip("tokenizers")
+ from transformers import PreTrainedTokenizerFast
+
+ backend = tokenizers.Tokenizer(
+ tokenizers.models.WordLevel(
+ {"<bos>": 0, "<eos>": 1, "hello": 2, "world": 3}, unk_token = None
+ )
+ )
+ backend.pre_tokenizer = tokenizers.pre_tokenizers.Whitespace()
+ return PreTrainedTokenizerFast(tokenizer_object = backend, bos_token = "<bos>", eos_token = "<eos>")
+
+
+def _backend_honors_add_bos_token():
+ # Pre-5.x fast tokenizers store add_bos_token without changing what they emit. Gemma 4 needs
+ # transformers >= 5.5.0 anyway (loader.py SUPPORTS_GEMMA4), so record it rather than fail.
+ try:
+ tokenizer = _build_fast_tokenizer()
+ except Exception:
+ return False
+ tokenizer.add_bos_token = True
+ return tokenizer("hello world")["input_ids"][0] == tokenizer.bos_token_id
+
+
+requires_working_add_bos_token = pytest.mark.skipif(
+ not _backend_honors_add_bos_token(),
+ reason = "this transformers treats add_bos_token as an inert attribute",
+)
+
+
+def _real_tokenizer(add_bos = False):
+ tokenizer = _build_fast_tokenizer()
+ # Gemma 4 is identified by its processor, not by this toy vocabulary.
+ tokenizer.processor_class = "Gemma4Processor"
+ if add_bos:
+ tokenizer.add_bos_token = True
+ return tokenizer
+
+
+def _ids(
+ tokenizer,
+ text = "hello world",
+ **kwargs,
+):
+ return tokenizer(text, **kwargs)["input_ids"]
+
+
+@requires_working_add_bos_token
+def test_real_backend_gains_exactly_one_bos():
+ tok = _real_tokenizer()
+ assert _ids(tok)[0] != tok.bos_token_id
+ tu._fix_gemma4_base_bos_token(tok)
+ ids = _ids(tok)
+ assert ids[0] == tok.bos_token_id and ids[1] != tok.bos_token_id
+
+
+@requires_working_add_bos_token
+def test_real_backend_repair_is_idempotent():
+ tok = _real_tokenizer()
+ tu._fix_gemma4_base_bos_token(tok)
+ once = _ids(tok)
+ tu._fix_gemma4_base_bos_token(tok)
+ assert _ids(tok) == once
+
+
+@requires_working_add_bos_token
+def test_real_backend_add_special_tokens_false_never_gains_bos():
+ tok = _real_tokenizer()
+ tu._fix_gemma4_base_bos_token(tok)
+ assert tok.bos_token_id not in _ids(tok, add_special_tokens = False)
+
+
+@requires_working_add_bos_token
+def test_real_backend_already_correct_tokenizer_is_left_alone():
+ # google base mirrors report add_bos_token = False and still prepend, so keying on the
+ # attribute would rebuild a post_processor that already works.
+ tok = _real_tokenizer(add_bos = True)
+ before = str(tok._tokenizer.post_processor)
+ ids_before = _ids(tok)
+ tu._fix_gemma4_base_bos_token(tok)
+ assert _ids(tok) == ids_before
+ assert str(tok._tokenizer.post_processor) == before
+
+
+@requires_working_add_bos_token
+def test_real_backend_without_bos_token_does_not_claim_success():
+ tok = _real_tokenizer()
+ tok.bos_token = None
+ tu._fix_gemma4_base_bos_token(tok)
+ assert not getattr(tok, "add_bos_token", False) or _ids(tok)[0] == tok.bos_token_id
+
+
+ "model_type, expected",
+ [
+ ("gemma4", True),
+ ("gemma4_text", True),
+ ("gemma-4", True),
+ ("gemma3", False),
+ ("gemma3n", False),
+ ("gemma3n_text", False),
+ ("gemma2", False),
+ ("llama", False),
+ # A future Gemma 4.5 is a different model with its own BOS policy.
+ ("gemma_45", False),
+ ("gemma-4.5", False),
+ # Substring matching would catch unsloth's own diffusion_gemma4.
+ ("diffusion_gemma4", False),
+ ],
+)
+def test_config_model_type_detection_is_anchored(model_type, expected):
+ config = types.SimpleNamespace(model_type = model_type, text_config = None)
+ assert tu._is_gemma4_config(config) is expected
+
+
+ "architecture, expected",
+ [
+ ("Gemma4ForConditionalGeneration", True),
+ ("DiffusionGemma4ForConditionalGeneration", False),
+ ("Gemma3ForConditionalGeneration", False),
+ ],
+)
+def test_config_architectures_detection_is_anchored(architecture, expected):
+ config = types.SimpleNamespace(
+ model_type = "unknown", text_config = None, architectures = [architecture]
+ )
+ assert tu._is_gemma4_config(config) is expected
+
+
+@requires_working_add_bos_token
+def test_bos_token_inside_a_jinja_comment_does_not_suppress_the_fix():
+ tok = _real_tokenizer()
+ tok.chat_template = "{# bos_token is handled elsewhere #}{{ messages }}"
+ tu._fix_gemma4_base_bos_token(tok)
+ assert _ids(tok)[0] == tok.bos_token_id
+
+
+@requires_working_add_bos_token
+def test_bos_token_emitted_by_the_template_still_suppresses_the_fix():
+ tok = _real_tokenizer()
+ tok.chat_template = "{{- bos_token -}}{{ messages }}"
+ tu._fix_gemma4_base_bos_token(tok)
+ assert _ids(tok)[0] != tok.bos_token_id
+
+
+def test_processor_chat_template_is_deduped_too():
+ # ProcessorMixin.save_pretrained writes the processor's own chat_template.jinja, so leaving
+ # that copy alone exports a second BOS on a VLM.
+ emits_bos = "{{ bos_token }}{% for m in messages %}{{ m.content }}{% endfor %}"
+ inner = _Tok(add_bos_token = True, chat_template = emits_bos)
+ inner.bos_token_id = None # force the attribute fallback in _tokenizer_auto_adds_bos
+ processor = types.SimpleNamespace(tokenizer = inner, chat_template = emits_bos)
+
+ tu._dedupe_bos_chat_template(processor)
+
+ assert "bos_token" not in processor.chat_template
+ assert "bos_token" not in processor.tokenizer.chat_template
+
+
+def test_dedupe_is_a_noop_when_the_tokenizer_does_not_add_bos():
+ emits_bos = "{{ bos_token }}hello"
+ inner = _Tok(add_bos_token = False, chat_template = emits_bos)
+ inner.bos_token_id = None
+ processor = types.SimpleNamespace(tokenizer = inner, chat_template = emits_bos)
+
+ tu._dedupe_bos_chat_template(processor)
+
+ assert processor.chat_template == emits_bos
+ assert processor.tokenizer.chat_template == emits_bos
diff --git a/unsloth/models/llama.py b/unsloth/models/llama.py
index dc3993f2927..4e8f0e1b78d 100644
--- a/unsloth/models/llama.py
+++ b/unsloth/models/llama.py
@@ -2794,6 +2794,7 @@ class FastLlamaModel:
trust_remote_code = trust_remote_code,
fix_tokenizer = fix_tokenizer,
revision = tokenizer_revision,
+ config = getattr(model, "config", None),
**_tokenizer_cache_kwargs,
)
diff --git a/unsloth/models/loader.py b/unsloth/models/loader.py
index 0313cc75c2b..0092c0ae128 100644
--- a/unsloth/models/loader.py
+++ b/unsloth/models/loader.py
@@ -513,7 +513,7 @@ class FastLanguageModel(FastLlamaModel):
device_map = device_map,
device_map_planner_kwargs = device_map_planner_kwargs,
rope_scaling = rope_scaling, # [TODO] No effect
- fix_tokenizer = fix_tokenizer, # [TODO] No effect
+ fix_tokenizer = fix_tokenizer,
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
resize_model_vocab = resize_model_vocab, # [TODO] No effect
@@ -940,7 +940,7 @@ class FastLanguageModel(FastLlamaModel):
device_map = device_map,
device_map_planner_kwargs = device_map_planner_kwargs,
rope_scaling = rope_scaling, # [TODO] No effect
- fix_tokenizer = fix_tokenizer, # [TODO] No effect
+ fix_tokenizer = fix_tokenizer,
trust_remote_code = trust_remote_code,
use_gradient_checkpointing = use_gradient_checkpointing,
resize_model_vocab = resize_model_vocab, # [TODO] No effect
@@ -1231,7 +1231,7 @@ class FastModel(FastBaseModel):
# Planner hints for device_map = "unsloth"; see resolve_unsloth_device_map.
device_map_planner_kwargs = None,
rope_scaling = None, # [TODO] No effect
- fix_tokenizer = True, # [TODO] No effect
+ fix_tokenizer = True,
trust_remote_code = False,
use_gradient_checkpointing = "unsloth",
resize_model_vocab = None, # [TODO] No effect
@@ -2015,6 +2015,7 @@ class FastModel(FastBaseModel):
token = token,
device_map = device_map,
device_map_planner_kwargs = device_map_planner_kwargs,
+ fix_tokenizer = fix_tokenizer,
trust_remote_code = trust_remote_code,
revision = model_revision,
tokenizer_revision = _revision_for_tokenizer_repo(
diff --git a/unsloth/models/vision.py b/unsloth/models/vision.py
index 32056863c45..c2d8968bb73 100644
--- a/unsloth/models/vision.py
+++ b/unsloth/models/vision.py
@@ -1124,6 +1124,7 @@ class FastBaseModel:
# True when auto_config came from the caller. It cannot be inferred here: FastModel pops config
# out of kwargs before this sees them, so it looks exactly like one we resolved ourselves.
auto_config_from_caller = False,
+ fix_tokenizer = True,
**kwargs,
):
user_config = kwargs.pop("config", None)
@@ -2065,6 +2066,15 @@ class FastBaseModel:
"or set HF_HUB_OFFLINE=1 to force local loading. "
"Otherwise please check that the model has a tokenizer."
) from _last_resort_err
+ # FastModel never calls load_correct_tokenizer; heal Gemma 4 base BOS from
+ # the finalized processor / model config (unslothai/unsloth#7903).
+ from ..tokenizer_utils import _apply_post_load_tokenizer_fixes
+
+ tokenizer = _apply_post_load_tokenizer_fixes(
+ tokenizer,
+ fix_tokenizer = fix_tokenizer,
+ config = auto_config if auto_config is not None else getattr(model, "config", None),
+ )
patch_saving_functions(tokenizer, vision = True)
# Fix gradient accumulation; see #4982.
diff --git a/unsloth/save.py b/unsloth/save.py
index 2272006dd6f..c0545cc1a36 100644
--- a/unsloth/save.py
+++ b/unsloth/save.py
@@ -2877,31 +2877,24 @@ def upload_to_huggingface(
def fix_tokenizer_bos_token(tokenizer):
+ from .tokenizer_utils import (
+ _chat_template_emits_bos,
+ _dedupe_bos_chat_template,
+ _tokenizer_auto_adds_bos,
+ )
+
fix_bos_token = False
chat_template = getattr(tokenizer, "chat_template", None)
- if tokenizer("A").input_ids[0] == getattr(tokenizer, "bos_token_id", None):
- if chat_template is not None and (
- tokenizer.bos_token in chat_template
- or "{bos_token}" in chat_template.replace(" ", "")
- or "{bos_token+" in chat_template.replace(" ", "")
- ):
+ # A processor cannot be called without an image, so ask its inner tokenizer instead.
+ if _tokenizer_auto_adds_bos(getattr(tokenizer, "tokenizer", tokenizer)):
+ if _chat_template_emits_bos(tokenizer):
fix_bos_token = True
logger.warning(
"Unsloth: ##### The current model auto adds a BOS token.\n"
"Unsloth: ##### Your chat template has a BOS token. We shall remove it temporarily."
)
-
- new_chat_template = re.sub(
- r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\}[\s]{0,}\}", "", chat_template
- )
- new_chat_template = re.sub(
- r"\{[\s]{0,}\{[\s]{0,}bos\_token[\s]{0,}\+[\s]{0,}",
- "",
- new_chat_template,
- )
-
- tokenizer.chat_template = new_chat_template
+ _dedupe_bos_chat_template(tokenizer)
return fix_bos_token, chat_template
@@ -4635,10 +4628,13 @@ def unsloth_save_pretrained_gguf(
# preflight sized, so it measured the disk these files land on.
gguf_directory = _gguf_output_directory(save_directory)
- if is_processor:
- fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer.tokenizer)
- else:
- fix_bos_token, old_chat_template = fix_tokenizer_bos_token(tokenizer)
+ # save_pretrained writes the processor's own chat_template.jinja, so dedupe and restore both.
+ from .tokenizer_utils import _tokenizer_objects
+
+ old_chat_templates = [
+ (obj, getattr(obj, "chat_template", None)) for obj in _tokenizer_objects(tokenizer)
+ ]
+ fix_bos_token, _ = fix_tokenizer_bos_token(tokenizer)
# Resolve the imatrix (download, validate, rename *.gguf_file) up front, so a bad path or an unavailable
# upstream fails before the expensive merge and never reaches the IQ-quant gate.
@@ -4704,12 +4700,13 @@ def unsloth_save_pretrained_gguf(
f"{_offloaded_parameter_hint(self)}"
) from e
+ if fix_bos_token:
+ for _template_owner, _old_template in old_chat_templates:
+ _template_owner.chat_template = _old_template
+
if is_processor:
tokenizer = tokenizer.tokenizer
- if fix_bos_token:
- tokenizer.chat_template = old_chat_template
-
for _ in range(3):
import gc
gc.collect()
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index bf8b4e5753f..9b43470535c 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -68,6 +68,178 @@ IGNORED_TOKENIZER_NAMES = frozenset(
)
os.environ["UNSLOTH_IGNORED_TOKENIZER_NAMES"] = "\n".join(IGNORED_TOKENIZER_NAMES)
+# gemma-4 base mirrors do not prepend <bos> (post_processor.single = [A], google's is [<bos>, A]).
+# Not keyed on add_bos_token (google omits it on E4B/31B/26B-A4B and still prepends) nor on repo
+# name. -it is skipped: its chat_template emits BOS. unslothai/unsloth#7903
+_GEMMA4_INSTRUCT_EOS = "<turn|>"
+
+# Anchored: a future gemma-4.5 / gemma_45 has its own BOS policy, and DiffusionGemma4... is not gemma 4.
+_GEMMA4_NAME_RE = re.compile(r"^gemma[\s_-]*4(?![\d.])", re.IGNORECASE)
+
+# A bos_token inside a Jinja comment renders to nothing, so it is not an emission.
+_JINJA_COMMENT_RE = re.compile(r"\{#.*?#\}", re.DOTALL)
+
+
+def _tokenizer_objects(tokenizer):
+ """Yield the processor and its inner tokenizer once each."""
+ seen = []
+ for obj in (tokenizer, getattr(tokenizer, "tokenizer", None)):
+ # Identity, not equality: a wrapper defining __eq__ would compare fields.
+ if obj is None or any(obj is s for s in seen):
+ continue
+ seen.append(obj)
+ yield obj
+
+
+def _is_gemma4_config(config):
+ if config is None:
+ return False
+ for cfg in (config, getattr(config, "text_config", None)):
+ if cfg is None:
+ continue
+ model_type = getattr(cfg, "model_type", None)
+ if model_type is not None and _GEMMA4_NAME_RE.match(str(model_type)):
+ return True
+ if any(_GEMMA4_NAME_RE.match(str(a)) for a in (getattr(cfg, "architectures", None) or [])):
+ return True
+ return False
+
+
+def _is_gemma4_tokenizer(tokenizer):
+ """processor_class = Gemma4Processor, usually only in init_kwargs.
+
+ The class is GemmaTokenizer, so a class-name check never fires.
+ """
+ for obj in _tokenizer_objects(tokenizer):
+ processor_class = getattr(obj, "processor_class", None)
+ if processor_class is None:
+ processor_class = (getattr(obj, "init_kwargs", None) or {}).get("processor_class")
+ if str(processor_class or "").strip().lower() == "gemma4processor":
+ return True
+ return False
+
+
+def _chat_template_emits_bos(tokenizer):
+ for obj in _tokenizer_objects(tokenizer):
+ template = getattr(obj, "chat_template", None)
+ if not template:
+ continue
+ chunks = template.values() if isinstance(template, dict) else (template,)
+ for chunk in chunks:
+ text = _JINJA_COMMENT_RE.sub("", chunk if isinstance(chunk, str) else str(chunk))
+ if "bos_token" in text or "<bos>" in text:
+ return True
+ return False
+
+
+def _tokenizer_auto_adds_bos(tokenizer):
+ """Does this tokenizer already emit <bos>?
+
+ Not add_bos_token: google/gemma-4-E2B reports False and still prepends.
+ """
+ bos_token_id = getattr(tokenizer, "bos_token_id", None)
+ if bos_token_id is None:
+ return bool(getattr(tokenizer, "add_bos_token", False))
+ try:
+ input_ids = tokenizer("A")["input_ids"]
+ except Exception:
+ return bool(getattr(tokenizer, "add_bos_token", False))
+ # Processors return a batched nested list.
+ while (
+ isinstance(input_ids, (list, tuple))
+ and input_ids
+ and isinstance(input_ids[0], (list, tuple))
+ ):
+ input_ids = input_ids[0]
+ return bool(input_ids) and input_ids[0] == bos_token_id
+
+
+def _strip_bos_from_chat_template_text(chat_template):
+ if not isinstance(chat_template, str) or not chat_template:
+ return chat_template
+ stripped = re.sub(r"\{[\s\-]*\{[\s\-]*bos\_token[\s\-]*\}[\s\-]*\}", "", chat_template, count = 1)
+ return re.sub(r"\{[\s\-]*\{[\s\-]*bos\_token[\s\-]*\+[\s\-]*", "", stripped, count = 1)
+
+
+def _dedupe_bos_chat_template(tokenizer):
+ """Drop template-emitted BOS when the tokenizer already prepends one.
+
+ A processor keeps its own copy and save_pretrained writes that one, so both must lose it.
+ Only the inner tokenizer can be asked: calling a processor needs an image.
+ """
+ if not _tokenizer_auto_adds_bos(getattr(tokenizer, "tokenizer", tokenizer)):
+ return
+ for obj in _tokenizer_objects(tokenizer):
+ template = getattr(obj, "chat_template", None)
+ if template is None:
+ continue
+ if isinstance(template, dict):
+ obj.chat_template = {
+ k: _strip_bos_from_chat_template_text(v) if isinstance(v, str) else v
+ for k, v in template.items()
+ }
+ elif isinstance(template, str):
+ obj.chat_template = _strip_bos_from_chat_template_text(template)
+
+
+def _is_gemma4_instruct_tokenizer(tokenizer):
+ """-it emits BOS from its chat template, so flipping the flag would double it."""
+ if _chat_template_emits_bos(tokenizer):
+ return True
+ return any(
+ getattr(o, "eos_token", None) == _GEMMA4_INSTRUCT_EOS for o in _tokenizer_objects(tokenizer)
+ )
+
+
+def _needs_gemma4_base_bos(tokenizer, config = None):
+ if tokenizer is None or _is_gemma4_instruct_tokenizer(tokenizer):
+ return False
+ return _is_gemma4_tokenizer(tokenizer) or _is_gemma4_config(config)
+
+
+def _enable_add_bos_token(tokenizer):
+ """Make the tokenizer prepend <bos>, and warn if that could not be done.
+
+ Nothing is recorded for save_pretrained: transformers 5.x drops add_bos_token from
+ tokenizer_config.json, so the setter's post_processor rewrite is what persists.
+ """
+ for obj in _tokenizer_objects(tokenizer):
+ # Already correct: keep its post_processor rather than rebuilding one.
+ if _tokenizer_auto_adds_bos(obj):
+ continue
+ try:
+ obj.add_bos_token = True
+ except AttributeError:
+ continue # Read-only property, or rejects the attribute. Not our tokenizer.
+ except ValueError as error:
+ # Raised when bos_token is None; swallowing it would fake success.
+ logger.warning(f"Unsloth: Could not enable add_bos_token for Gemma 4: {error}")
+ continue
+ # Pre-5.x fast tokenizers accept the attribute without changing what they emit.
+ if not _tokenizer_auto_adds_bos(obj):
+ logger.warning(
+ "Unsloth: add_bos_token was set for this Gemma 4 base tokenizer but it still "
+ "does not prepend <bos>. See unslothai/unsloth#7903."
+ )
+
+
+def _fix_gemma4_base_bos_token(tokenizer, config = None):
+ if tokenizer is None or not _needs_gemma4_base_bos(tokenizer, config = config):
+ return tokenizer
+ _enable_add_bos_token(tokenizer)
+ return tokenizer
+
+
+def _apply_post_load_tokenizer_fixes(
+ tokenizer,
+ fix_tokenizer = True,
+ config = None,
+):
+ if not fix_tokenizer:
+ return tokenizer
+ return _fix_gemma4_base_bos_token(tokenizer, config = config)
+
+
# A KAGGLE_* variable is not a Kaggle kernel: the Kaggle CLI reads KAGGLE_USERNAME / KAGGLE_KEY on
# ordinary machines, and redirecting their tokenizer cache to /tmp because of it was wrong.
from .disk_utils import (
@@ -550,6 +722,7 @@ def _load_correct_tokenizer(
cache_dir = "huggingface_tokenizers_cache",
fix_tokenizer = True,
revision = None,
+ config = None,
):
if IS_COLAB_ENVIRONMENT:
cache_dir = cache_dir
@@ -596,13 +769,13 @@ def _load_correct_tokenizer(
)
if not fix_tokenizer or tokenizer_name.lower() in IGNORED_TOKENIZER_NAMES:
- return fast_tokenizer
+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)
# Ignore Mistral ones - they're a bit weird to handle!
elif "mistral" in tokenizer_name.lower():
- return fast_tokenizer
+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)
# Ignore Phi-4 ones as well
elif "phi-4" in tokenizer_name.lower():
- return fast_tokenizer
+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)
elif slow_tokenizer is not None:
if hasattr(fast_tokenizer, "add_bos_token") and hasattr(slow_tokenizer, "add_bos_token"):
fast_tokenizer.add_bos_token = slow_tokenizer.add_bos_token
@@ -611,13 +784,17 @@ def _load_correct_tokenizer(
# Confirm whether slow and fast are equivalent.
if assert_same_tokenization(slow_tokenizer, fast_tokenizer):
- return fast_tokenizer
+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)
else:
logger.warning(f"Unsloth: Will load {tokenizer_name} as a legacy tokenizer.")
- return convert_to_fast_tokenizer(slow_tokenizer)
+ return _apply_post_load_tokenizer_fixes(
+ convert_to_fast_tokenizer(slow_tokenizer),
+ fix_tokenizer,
+ config,
+ )
pass
else:
- return fast_tokenizer
+ return _apply_post_load_tokenizer_fixes(fast_tokenizer, fix_tokenizer, config)
def _fix_pad_token(tokenizer):
@@ -649,6 +826,7 @@ def load_correct_tokenizer(
cache_dir = "huggingface_tokenizers_cache",
fix_tokenizer = True,
revision = None,
+ config = None,
):
tokenizer = _load_correct_tokenizer(
tokenizer_name = tokenizer_name,
@@ -659,6 +837,7 @@ def load_correct_tokenizer(
cache_dir = cache_dir,
fix_tokenizer = fix_tokenizer,
revision = revision,
+ config = config,
)
if fix_tokenizer: