Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Repair chat templates that always append the generation prompt, also on FastModel loads (#12139)
Daniel Hane417798d3fec
diff --git a/tests/test_chat_template_trailing_generation_prompt.py b/tests/test_chat_template_trailing_generation_prompt.pynew file mode 100644index 00000000000..f9e918c418d--- /dev/null+++ b/tests/test_chat_template_trailing_generation_prompt.py@@ -0,0 +1,91 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++import pytest++tokenizers = pytest.importorskip("tokenizers")+from transformers import PreTrainedTokenizerFast++from unsloth.tokenizer_utils import _fix_chat_template, _apply_post_load_tokenizer_fixes++# Tail of baidu/ERNIE-4.5-21B-A3B-Thinking's template, message loop simplified.+ERNIE_LIKE = (+ "{{- '<|im_start|>system\\n<global_setting>\\nthink_mode=True\\n</global_setting><|im_end|>\\n\\n' }}"+ "{%- for message in messages %}"+ "{%- if message.role == 'user' %}{{- '<|im_start|>user\\n' + message.content + '<|im_end|>\\n\\n' }}"+ "{%- elif message.role == 'assistant' %}{{- '<|im_start|>assistant\\n<response>\\n' + message.content + '\\n</response>\\n<|im_end|>\\n\\n' }}"+ "{%- endif %}"+ '{%- endfor %}\n {{- "<|im_start|>assistant\\n<think>\\n"}}'+)+HEADER = "<|im_start|>assistant\n<think>\n"+CONVO = [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello"}]+++def _tokenizer(template):+ from tokenizers import Tokenizer, models, pre_tokenizers++ tk = Tokenizer(models.WordLevel({"[UNK]": 0, "a": 1}, unk_token = "[UNK]"))+ tk.pre_tokenizer = pre_tokenizers.Whitespace()+ tok = PreTrainedTokenizerFast(tokenizer_object = tk, unk_token = "[UNK]", eos_token = "[UNK]")+ tok.chat_template = template+ tok.name_or_path = "baidu/ERNIE-4.5-21B-A3B-Thinking"+ return tok+++def test_trailing_expression_after_whitespace_is_wrapped():+ fixed = _fix_chat_template(ERNIE_LIKE)+ assert fixed != ERNIE_LIKE+ tok = _tokenizer(fixed)+ no = tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False)+ yes = tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = True)+ assert no.endswith("</response>\n<|im_end|>\n\n") and not no.endswith(HEADER)+ assert yes == no + HEADER+++def test_original_render_with_prompt_is_unchanged():+ orig = _tokenizer(ERNIE_LIKE).apply_chat_template(+ CONVO, tokenize = False, add_generation_prompt = True+ )+ fixed = _tokenizer(_fix_chat_template(ERNIE_LIKE)).apply_chat_template(+ CONVO, tokenize = False, add_generation_prompt = True+ )+ assert orig == fixed+++def test_fastmodel_post_load_fixes_repair_the_template():+ tok = _tokenizer(ERNIE_LIKE)+ assert tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False).endswith(+ HEADER+ )+ tok = _apply_post_load_tokenizer_fixes(tok, fix_tokenizer = True, config = None)+ assert not tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False).endswith(+ HEADER+ )+++def test_fix_tokenizer_false_leaves_template_alone():+ tok = _apply_post_load_tokenizer_fixes(_tokenizer(ERNIE_LIKE), fix_tokenizer = False, config = None)+ assert tok.chat_template == ERNIE_LIKE+++def test_template_that_honours_the_flag_is_untouched():+ good = ERNIE_LIKE.replace(+ '{%- endfor %}\n {{- "<|im_start|>assistant\\n<think>\\n"}}',+ "{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}",+ )+ tok = _apply_post_load_tokenizer_fixes(_tokenizer(good), fix_tokenizer = True, config = None)+ assert tok.chat_template == good++[email protected]("sep", ["", "\n"])+def test_trailing_eos_footer_is_not_wrapped(sep):+ footer = (+ "{%- for message in messages %}{{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}"+ "{%- endfor %}" + sep + "{{ eos_token }}"+ )+ assert _fix_chat_template(footer) == footer+ tok = _apply_post_load_tokenizer_fixes(_tokenizer(footer), fix_tokenizer = True, config = None)+ assert tok.chat_template == footer+ assert tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False).endswith(+ "[UNK]"+ )diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.pyindex 36820cc4b4e..1ea3406c42c 100644--- a/unsloth/tokenizer_utils.py+++ b/unsloth/tokenizer_utils.py@@ -342,7 +342,32 @@ def _apply_post_load_tokenizer_fixes()if not fix_tokenizer:return tokenizer- return _fix_gemma4_base_bos_token(tokenizer, config = config)+ tokenizer = _fix_gemma4_base_bos_token(tokenizer, config = config)+ return _fix_post_load_chat_template(tokenizer)+++def _fix_post_load_chat_template(tokenizer):+ # FastModel twin of load_correct_tokenizer's template repair, same exclusions.+ text_tokenizer = getattr(tokenizer, "tokenizer", tokenizer)+ old = getattr(text_tokenizer, "chat_template", None)+ if not isinstance(old, str) or not old:+ return tokenizer+ name = str(getattr(text_tokenizer, "name_or_path", "")).lower()+ if any(s in name for s in ("mistral", "qwen3guard")):+ return tokenizer+ if "[/INST]" in old and "[INST]" in old and "bos_token" in old and "eos_token" in old:+ return tokenizer+ try:+ new = _fix_chat_template_for_tokenizer(text_tokenizer, old)+ except RuntimeError:+ raise # UNSLOTH_STRICT_CHAT_TEMPLATE=1+ except Exception:+ return tokenizer+ if isinstance(new, str) and new != old:+ text_tokenizer.chat_template = new+ if tokenizer is not text_tokenizer and getattr(tokenizer, "chat_template", None) == old:+ tokenizer.chat_template = new+ return tokenizer# A KAGGLE_* variable is not a Kaggle kernel: the Kaggle CLI reads KAGGLE_USERNAME / KAGGLE_KEY on ordinary machines, and redirecting their tokenizer cache to /tmp because of it was wrong.@@ -1158,13 +1183,17 @@ def _fix_chat_template(chat_template, is_sharegpt = False):open_tag = lambda body: "{%" + dash_l + " " + body + " " + dash_r + "%}"# Case 1: the template ends with a single trailing {{ expr }} that is the generation prefix, so wrap it in an {% if add_generation_prompt %} block.+ # Surrounding whitespace allowed (ERNIE-4.5: `{%- endfor %}\n {{- "..." }}`); it moves inside the block.+ trailing = after_endfor.strip()if ("{%" + dash_l + " if" not in after_endforand "{%" + dash_l + " set " not in after_endfor- and after_endfor.startswith("{{")- and after_endfor.endswith("}}")- and after_endfor.count("{{") == 1- and after_endfor.count("}}") == 1+ and trailing.startswith("{{")+ and trailing.endswith("}}")+ and trailing.count("{{") == 1+ and trailing.count("}}") == 1+ # An EOS footer closes the last turn; wrapping it would strip EOS from every training text.+ and "eos_token" not in trailing):wrapped = open_tag("if add_generation_prompt") + after_endfor + open_tag("endif")return chat_template[: end["end"]] + wrapped