Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Repair chat templates that always append the generation prompt, also on FastModel loads (#12139)

Daniel Hane417798d3fec
diff --git a/tests/test_chat_template_trailing_generation_prompt.py b/tests/test_chat_template_trailing_generation_prompt.py
new file mode 100644
index 00000000000..f9e918c418d
--- /dev/null
+++ b/tests/test_chat_template_trailing_generation_prompt.py
@@ -0,0 +1,91 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+import pytest
+
+tokenizers = pytest.importorskip("tokenizers")
+from transformers import PreTrainedTokenizerFast
+
+from unsloth.tokenizer_utils import _fix_chat_template, _apply_post_load_tokenizer_fixes
+
+# Tail of baidu/ERNIE-4.5-21B-A3B-Thinking's template, message loop simplified.
+ERNIE_LIKE = (
+ "{{- '<|im_start|>system\\n<global_setting>\\nthink_mode=True\\n</global_setting><|im_end|>\\n\\n' }}"
+ "{%- for message in messages %}"
+ "{%- if message.role == 'user' %}{{- '<|im_start|>user\\n' + message.content + '<|im_end|>\\n\\n' }}"
+ "{%- elif message.role == 'assistant' %}{{- '<|im_start|>assistant\\n<response>\\n' + message.content + '\\n</response>\\n<|im_end|>\\n\\n' }}"
+ "{%- endif %}"
+ '{%- endfor %}\n {{- "<|im_start|>assistant\\n<think>\\n"}}'
+)
+HEADER = "<|im_start|>assistant\n<think>\n"
+CONVO = [{"role": "user", "content": "Hi"}, {"role": "assistant", "content": "Hello"}]
+
+
+def _tokenizer(template):
+ from tokenizers import Tokenizer, models, pre_tokenizers
+
+ tk = Tokenizer(models.WordLevel({"[UNK]": 0, "a": 1}, unk_token = "[UNK]"))
+ tk.pre_tokenizer = pre_tokenizers.Whitespace()
+ tok = PreTrainedTokenizerFast(tokenizer_object = tk, unk_token = "[UNK]", eos_token = "[UNK]")
+ tok.chat_template = template
+ tok.name_or_path = "baidu/ERNIE-4.5-21B-A3B-Thinking"
+ return tok
+
+
+def test_trailing_expression_after_whitespace_is_wrapped():
+ fixed = _fix_chat_template(ERNIE_LIKE)
+ assert fixed != ERNIE_LIKE
+ tok = _tokenizer(fixed)
+ no = tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False)
+ yes = tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = True)
+ assert no.endswith("</response>\n<|im_end|>\n\n") and not no.endswith(HEADER)
+ assert yes == no + HEADER
+
+
+def test_original_render_with_prompt_is_unchanged():
+ orig = _tokenizer(ERNIE_LIKE).apply_chat_template(
+ CONVO, tokenize = False, add_generation_prompt = True
+ )
+ fixed = _tokenizer(_fix_chat_template(ERNIE_LIKE)).apply_chat_template(
+ CONVO, tokenize = False, add_generation_prompt = True
+ )
+ assert orig == fixed
+
+
+def test_fastmodel_post_load_fixes_repair_the_template():
+ tok = _tokenizer(ERNIE_LIKE)
+ assert tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False).endswith(
+ HEADER
+ )
+ tok = _apply_post_load_tokenizer_fixes(tok, fix_tokenizer = True, config = None)
+ assert not tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False).endswith(
+ HEADER
+ )
+
+
+def test_fix_tokenizer_false_leaves_template_alone():
+ tok = _apply_post_load_tokenizer_fixes(_tokenizer(ERNIE_LIKE), fix_tokenizer = False, config = None)
+ assert tok.chat_template == ERNIE_LIKE
+
+
+def test_template_that_honours_the_flag_is_untouched():
+ good = ERNIE_LIKE.replace(
+ '{%- endfor %}\n {{- "<|im_start|>assistant\\n<think>\\n"}}',
+ "{%- endfor %}{%- if add_generation_prompt %}{{- '<|im_start|>assistant\\n' }}{%- endif %}",
+ )
+ tok = _apply_post_load_tokenizer_fixes(_tokenizer(good), fix_tokenizer = True, config = None)
+ assert tok.chat_template == good
+
+
[email protected]("sep", ["", "\n"])
+def test_trailing_eos_footer_is_not_wrapped(sep):
+ footer = (
+ "{%- for message in messages %}{{- '<|im_start|>' + message.role + '\\n' + message.content + '<|im_end|>\\n' }}"
+ "{%- endfor %}" + sep + "{{ eos_token }}"
+ )
+ assert _fix_chat_template(footer) == footer
+ tok = _apply_post_load_tokenizer_fixes(_tokenizer(footer), fix_tokenizer = True, config = None)
+ assert tok.chat_template == footer
+ assert tok.apply_chat_template(CONVO, tokenize = False, add_generation_prompt = False).endswith(
+ "[UNK]"
+ )
diff --git a/unsloth/tokenizer_utils.py b/unsloth/tokenizer_utils.py
index 36820cc4b4e..1ea3406c42c 100644
--- a/unsloth/tokenizer_utils.py
+++ b/unsloth/tokenizer_utils.py
@@ -342,7 +342,32 @@ def _apply_post_load_tokenizer_fixes(
)
if not fix_tokenizer:
return tokenizer
- return _fix_gemma4_base_bos_token(tokenizer, config = config)
+ tokenizer = _fix_gemma4_base_bos_token(tokenizer, config = config)
+ return _fix_post_load_chat_template(tokenizer)
+
+
+def _fix_post_load_chat_template(tokenizer):
+ # FastModel twin of load_correct_tokenizer's template repair, same exclusions.
+ text_tokenizer = getattr(tokenizer, "tokenizer", tokenizer)
+ old = getattr(text_tokenizer, "chat_template", None)
+ if not isinstance(old, str) or not old:
+ return tokenizer
+ name = str(getattr(text_tokenizer, "name_or_path", "")).lower()
+ if any(s in name for s in ("mistral", "qwen3guard")):
+ return tokenizer
+ if "[/INST]" in old and "[INST]" in old and "bos_token" in old and "eos_token" in old:
+ return tokenizer
+ try:
+ new = _fix_chat_template_for_tokenizer(text_tokenizer, old)
+ except RuntimeError:
+ raise # UNSLOTH_STRICT_CHAT_TEMPLATE=1
+ except Exception:
+ return tokenizer
+ if isinstance(new, str) and new != old:
+ text_tokenizer.chat_template = new
+ if tokenizer is not text_tokenizer and getattr(tokenizer, "chat_template", None) == old:
+ tokenizer.chat_template = new
+ return tokenizer
# A KAGGLE_* variable is not a Kaggle kernel: the Kaggle CLI reads KAGGLE_USERNAME / KAGGLE_KEY on ordinary machines, and redirecting their tokenizer cache to /tmp because of it was wrong.
@@ -1158,13 +1183,17 @@ def _fix_chat_template(chat_template, is_sharegpt = False):
open_tag = lambda body: "{%" + dash_l + " " + body + " " + dash_r + "%}"
# Case 1: the template ends with a single trailing {{ expr }} that is the generation prefix, so wrap it in an {% if add_generation_prompt %} block.
+ # Surrounding whitespace allowed (ERNIE-4.5: `{%- endfor %}\n {{- "..." }}`); it moves inside the block.
+ trailing = after_endfor.strip()
if (
"{%" + dash_l + " if" not in after_endfor
and "{%" + dash_l + " set " not in after_endfor
- and after_endfor.startswith("{{")
- and after_endfor.endswith("}}")
- and after_endfor.count("{{") == 1
- and after_endfor.count("}}") == 1
+ and trailing.startswith("{{")
+ and trailing.endswith("}}")
+ and trailing.count("{{") == 1
+ and trailing.count("}}") == 1
+ # An EOS footer closes the last turn; wrapping it would strip EOS from every training text.
+ and "eos_token" not in trailing
):
wrapped = open_tag("if add_generation_prompt") + after_endfor + open_tag("endif")
return chat_template[: end["end"]] + wrapped