Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
Gemma-4 26B/31B: train with the empty thought channel on non-thinking turns (#12867)
Daniel Han3182f568d4e2
diff --git a/tests/test_gemma4_empty_thought.py b/tests/test_gemma4_empty_thought.pynew file mode 100644index 00000000000..c2fb2b965ea--- /dev/null+++ b/tests/test_gemma4_empty_thought.py@@ -0,0 +1,240 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++"""Gemma-4 26B / 31B empty thought channel on non-thinking model turns; E2B / E4B unchanged.++unsloth import needs a GPU, so template code is pulled out via ast.+"""++import ast+import os++import pytest+from jinja2.exceptions import TemplateError+from jinja2.sandbox import ImmutableSandboxedEnvironment++CHAT_TEMPLATES_PATH = os.path.join(+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),+ "unsloth",+ "chat_templates.py",+)+_SOURCE = open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read()+_TREE = ast.parse(_SOURCE)++EMPTY = "<|channel>thought\n<channel|>"+++def _load():+ wanted_assign = {+ "gemma4_template",+ "gemma4_thinking_template",+ "_gemma4_model_turn",+ "gemma4_empty_thought_template",+ "GEMMA4_TEMPLATE_NAMES",+ "gemma4_empty_thought_ollama",+ }+ namespace = {}+ for node in _TREE.body:+ if isinstance(node, ast.Assign) and any(+ getattr(target, "id", None) in wanted_assign for target in node.targets+ ):+ exec(compile(ast.Module([node], []), CHAT_TEMPLATES_PATH, "exec"), namespace)+ elif isinstance(node, ast.FunctionDef) and node.name == "_gemma4_wants_empty_thought":+ exec(compile(ast.Module([node], []), CHAT_TEMPLATES_PATH, "exec"), namespace)+ return namespace+++NS = _load()+++def _render(template, messages, **kwargs):+ env = ImmutableSandboxedEnvironment(trim_blocks = True, lstrip_blocks = True)+ env.globals["raise_exception"] = lambda msg: (_ for _ in ()).throw(TemplateError(msg))+ ctx = {"messages": messages, "add_generation_prompt": False, "bos_token": "<bos>"}+ ctx.update(kwargs)+ return env.from_string("{{ bos_token }}" + template).render(**ctx)+++class _Holder:+ def __init__(self, chat_template):+ self.chat_template = chat_template+++# Tails of google/gemma-4-26B-A4B-it and google/gemma-4-E2B-it chat_template.jinja (verbatim)+GOOGLE_26B_TAIL = (+ "{%- if add_generation_prompt -%}\n"+ " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n"+ " {{- '<|turn>model\\n' -}}\n"+ " {%- if not enable_thinking -%}\n"+ " {{- '<|channel>thought\\n<channel|>' -}}\n"+ " {%- endif -%}\n"+ " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n"+ " {{- '<|channel>thought\\n' -}}\n"+ " {%- endif -%}\n"+ "{%- endif -%}\n"+)+GOOGLE_E2B_TAIL = (+ "{%- if add_generation_prompt -%}\n"+ " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n"+ " {{- '<|turn>model\\n' -}}\n"+ " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n"+ " {{- '<|channel>thought\\n' -}}\n"+ " {%- endif -%}\n"+ "{%- endif -%}\n"+)++CONVO = [+ {"role": "system", "content": "Be brief."},+ {"role": "user", "content": "Hi"},+ {"role": "assistant", "content": "Hello!"},+ {"role": "user", "content": "2+2?"},+ {"role": "assistant", "content": "4"},+]+++def test_detects_26b_31b_template():+ assert NS["_gemma4_wants_empty_thought"](_Holder(GOOGLE_26B_TAIL))+++def test_ignores_e2b_template():+ assert not NS["_gemma4_wants_empty_thought"](_Holder(GOOGLE_E2B_TAIL))+++def test_ignores_missing_or_non_string_templates():+ assert not NS["_gemma4_wants_empty_thought"](None, _Holder(None), _Holder(123))+++def test_ignores_unsloth_thinking_template():+ # Re-calling get_chat_template on an E2B tokenizer already carrying it must not switch.+ tmpl = "{{ bos_token }}" + NS["gemma4_thinking_template"]+ assert not NS["_gemma4_wants_empty_thought"](_Holder(tmpl))+++def test_detects_own_output_on_recall():+ tmpl = "{{ bos_token }}" + NS["gemma4_empty_thought_template"]+ assert NS["_gemma4_wants_empty_thought"](_Holder(tmpl))+++def test_processor_dict_template():+ assert NS["_gemma4_wants_empty_thought"](_Holder({"default": GOOGLE_26B_TAIL}))+++def test_multi_turn_with_system_prompt():+ out = _render(NS["gemma4_empty_thought_template"], CONVO)+ assert out == (+ "<bos><|turn>system\nBe brief.<turn|>\n"+ "<|turn>user\nHi<turn|>\n"+ f"<|turn>model\n{EMPTY}Hello!<turn|>\n"+ "<|turn>user\n2+2?<turn|>\n"+ f"<|turn>model\n{EMPTY}4<turn|>\n"+ )+++def test_generation_prompt_matches_google_26b():+ msgs = [{"role": "user", "content": "Hi"}]+ out = _render(NS["gemma4_empty_thought_template"], msgs, add_generation_prompt = True)+ assert out == f"<bos><|turn>user\nHi<turn|>\n<|turn>model\n{EMPTY}"+ out = _render(+ NS["gemma4_empty_thought_template"], msgs, add_generation_prompt = True, enable_thinking = True+ )+ assert out == "<bos><|turn>system\n<|think|>\n<turn|>\n<|turn>user\nHi<turn|>\n<|turn>model\n"+++def test_training_text_is_prefix_consistent_with_generation_prompt():+ full = _render(NS["gemma4_empty_thought_template"], CONVO[:3])+ prompt = _render(NS["gemma4_empty_thought_template"], CONVO[:2], add_generation_prompt = True)+ assert full == prompt + "Hello!<turn|>\n"+++def test_list_content():+ msgs = [+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "What?"}]},+ {"role": "assistant", "content": [{"type": "text", "text": "A cat."}]},+ ]+ out = _render(NS["gemma4_empty_thought_template"], msgs)+ assert out == f"<bos><|turn>user\n<|image|>What?<turn|>\n<|turn>model\n{EMPTY}A cat.<turn|>\n"++[email protected]("key", ["reasoning_content", "reasoning"])+def test_final_turn_reasoning_becomes_the_thought_channel(key):+ msgs = [+ {"role": "user", "content": "2+2?"},+ {"role": "assistant", "content": "4", key: "2+2=4"},+ ]+ out = _render(NS["gemma4_empty_thought_template"], msgs)+ assert (+ out+ == "<bos><|turn>user\n2+2?<turn|>\n<|turn>model\n<|channel>thought\n2+2=4\n<channel|>4<turn|>\n"+ )+++def test_reasoning_before_the_last_user_turn_is_dropped_like_the_model_template():+ msgs = [+ {"role": "user", "content": "2+2?"},+ {"role": "assistant", "content": "4", "reasoning_content": "2+2=4"},+ {"role": "user", "content": "3+3?"},+ {"role": "assistant", "content": "6"},+ ]+ out = _render(NS["gemma4_empty_thought_template"], msgs)+ assert "2+2=4" not in out+ assert out.count(EMPTY) == 2+++ "content",+ [+ "<|channel>thought\n2+2=4<channel|>4",+ [{"type": "text", "text": "<|channel>thought\nx<channel|>4"}],+ ],+)+def test_inline_thoughts_are_stripped_and_get_the_empty_channel(content):+ msgs = [{"role": "user", "content": "2+2?"}, {"role": "assistant", "content": content}]+ out = _render(NS["gemma4_empty_thought_template"], msgs)+ assert out == f"<bos><|turn>user\n2+2?<turn|>\n<|turn>model\n{EMPTY}4<turn|>\n"+++def test_ollama_generation_prompt_has_the_empty_channel():+ ollama = NS["gemma4_empty_thought_ollama"]+ assert "{{ if $last }}<|turn>model\n" + EMPTY + "{{ end }}" in ollama+++def test_ollama_history_assistant_turns_have_the_empty_channel():+ ollama = NS["gemma4_empty_thought_ollama"]+ assert '{{- if eq .Role "assistant" }}<|turn>model\n' + EMPTY + "{{ .Content }}" in ollama+ assert "<|turn>assistant" not in ollama+ assert ollama.count(EMPTY) == 2+++def test_enable_thinking_is_unchanged():+ new = _render(NS["gemma4_empty_thought_template"], CONVO, enable_thinking = True)+ old = _render(NS["gemma4_thinking_template"], CONVO, enable_thinking = True)+ assert new == old+ assert EMPTY not in new+++def test_only_model_turns_change():+ new = _render(NS["gemma4_empty_thought_template"], CONVO)+ old = _render(NS["gemma4_thinking_template"], CONVO)+ assert new.replace(EMPTY, "") == old+++def test_roles_must_alternate_still_raises():+ with pytest.raises(TemplateError):+ _render(NS["gemma4_empty_thought_template"], [{"role": "assistant", "content": "x"}])+++def test_get_chat_template_switches_only_gemma4_names():+ body = next(+ node+ for node in _TREE.body+ if isinstance(node, ast.FunctionDef) and node.name == "get_chat_template"+ )+ src = ast.get_source_segment(_SOURCE, body)+ assert "_gemma4_wants_empty_thought(_processor, old_tokenizer)" in src+ assert "type_chat_template in GEMMA4_TEMPLATE_NAMES" in src+ assert set(NS["GEMMA4_TEMPLATE_NAMES"]) == {+ "gemma-4",+ "gemma4",+ "gemma-4-thinking",+ "gemma4-thinking",+ }diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.pyindex f09f925986f..3c2ed422417 100644--- a/unsloth/chat_templates.py+++ b/unsloth/chat_templates.py@@ -1070,6 +1070,61 @@ DEFAULT_SYSTEM_MESSAGE["gemma-4-thinking"] = NoneCHAT_TEMPLATES["gemma4-thinking"] = (gemma4_thinking_template, gemma4_template_eos_token, False, gemma4_ollama,)DEFAULT_SYSTEM_MESSAGE["gemma4-thinking"] = None+# 26B-A4B / 31B generate after an empty thought channel their template omits from history; final-turn reasoning fills it.+_gemma4_model_turn = "{{ '<|turn>' + role + '\n' }}\n"+gemma4_empty_thought_template = gemma4_thinking_template.replace(+ _gemma4_model_turn,+ _gemma4_model_turn + \+""" {%- if role == "model" -%}+ {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}+ {%- if thinking_text and loop.index0 > ns_turn.last_user_idx -%}+ {{ '<|channel>thought\n' + thinking_text + '\n<channel|>' }}+ {%- elif not thinking -%}+ {{ '<|channel>thought\n<channel|>' }}+ {%- endif -%}+ {%- endif -%}+""",+ 1,+).replace(+ "{%- for message in loop_messages -%}",+ "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n"+ "{%- for m in loop_messages -%}{%- if m['role'] == 'user' -%}{%- set ns_turn.last_user_idx = loop.index0 -%}{%- endif -%}{%- endfor -%}\n"+ "{%- for message in loop_messages -%}",+ 1,+)+assert gemma4_empty_thought_template != gemma4_thinking_template+# Ollama re-renders history each request, so every assistant turn gets the channel too.+gemma4_empty_thought_ollama = '''+FROM {__FILE_LOCATION__}+TEMPLATE """{{- range $i, $_ := .Messages }}+{{- $last := eq (len (slice $.Messages $i)) 1 }}+{{- if eq .Role "assistant" }}<|turn>model+<|channel>thought+<channel|>{{ .Content }}{{ if not $last }}<turn|>+{{ end }}+{{- else }}<|turn>{{ .Role }}+{{ .Content }}<turn|>+{{ if $last }}<|turn>model+<|channel>thought+<channel|>{{ end }}+{{- end }}+{{- end }}"""+'''+GEMMA4_TEMPLATE_NAMES = ("gemma-4", "gemma4", "gemma-4-thinking", "gemma4-thinking",)+++def _gemma4_wants_empty_thought(*holders):+ # Content based: the model's own template primes the empty channel (E2B / E4B do not).+ for holder in holders:+ template = getattr(holder, "chat_template", None)+ if isinstance(template, dict): template = template.get("default")+ if not isinstance(template, str): continue+ # Unsloth's gemma-4-thinking primes it for every size+ if template.endswith(gemma4_thinking_template): continue+ if "<|channel>thought\\n<channel|>" in template or "<|channel>thought\n<channel|>" in template:+ return True+ return False+# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))# =========================================== GPT-OSSgptoss_template = \@@ -1955,6 +2010,15 @@ def get_chat_template(chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]+ if type_chat_template in GEMMA4_TEMPLATE_NAMES and \+ _gemma4_wants_empty_thought(_processor, old_tokenizer):+ logger.warning_once(+ "Unsloth: This Gemma-4 model expects an empty thought channel on non-thinking turns. "\+ "Adding <|channel>thought\\n<channel|> to assistant turns without thinking content."+ )+ chat_template = gemma4_empty_thought_template+ ollama_modelfile = gemma4_empty_thought_ollama+# The template can veto the eos mapping, but it must not force it back on: map_eos_token = False is# an explicit choice by the caller.if not yes_map_eos_token and map_eos_token: map_eos_token = False