Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

Gemma-4 26B/31B: train with the empty thought channel on non-thinking turns (#12867)

Daniel Han3182f568d4e2
diff --git a/tests/test_gemma4_empty_thought.py b/tests/test_gemma4_empty_thought.py
new file mode 100644
index 00000000000..c2fb2b965ea
--- /dev/null
+++ b/tests/test_gemma4_empty_thought.py
@@ -0,0 +1,240 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+"""Gemma-4 26B / 31B empty thought channel on non-thinking model turns; E2B / E4B unchanged.
+
+unsloth import needs a GPU, so template code is pulled out via ast.
+"""
+
+import ast
+import os
+
+import pytest
+from jinja2.exceptions import TemplateError
+from jinja2.sandbox import ImmutableSandboxedEnvironment
+
+CHAT_TEMPLATES_PATH = os.path.join(
+ os.path.dirname(os.path.dirname(os.path.abspath(__file__))),
+ "unsloth",
+ "chat_templates.py",
+)
+_SOURCE = open(CHAT_TEMPLATES_PATH, encoding = "utf-8").read()
+_TREE = ast.parse(_SOURCE)
+
+EMPTY = "<|channel>thought\n<channel|>"
+
+
+def _load():
+ wanted_assign = {
+ "gemma4_template",
+ "gemma4_thinking_template",
+ "_gemma4_model_turn",
+ "gemma4_empty_thought_template",
+ "GEMMA4_TEMPLATE_NAMES",
+ "gemma4_empty_thought_ollama",
+ }
+ namespace = {}
+ for node in _TREE.body:
+ if isinstance(node, ast.Assign) and any(
+ getattr(target, "id", None) in wanted_assign for target in node.targets
+ ):
+ exec(compile(ast.Module([node], []), CHAT_TEMPLATES_PATH, "exec"), namespace)
+ elif isinstance(node, ast.FunctionDef) and node.name == "_gemma4_wants_empty_thought":
+ exec(compile(ast.Module([node], []), CHAT_TEMPLATES_PATH, "exec"), namespace)
+ return namespace
+
+
+NS = _load()
+
+
+def _render(template, messages, **kwargs):
+ env = ImmutableSandboxedEnvironment(trim_blocks = True, lstrip_blocks = True)
+ env.globals["raise_exception"] = lambda msg: (_ for _ in ()).throw(TemplateError(msg))
+ ctx = {"messages": messages, "add_generation_prompt": False, "bos_token": "<bos>"}
+ ctx.update(kwargs)
+ return env.from_string("{{ bos_token }}" + template).render(**ctx)
+
+
+class _Holder:
+ def __init__(self, chat_template):
+ self.chat_template = chat_template
+
+
+# Tails of google/gemma-4-26B-A4B-it and google/gemma-4-E2B-it chat_template.jinja (verbatim)
+GOOGLE_26B_TAIL = (
+ "{%- if add_generation_prompt -%}\n"
+ " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n"
+ " {{- '<|turn>model\\n' -}}\n"
+ " {%- if not enable_thinking -%}\n"
+ " {{- '<|channel>thought\\n<channel|>' -}}\n"
+ " {%- endif -%}\n"
+ " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n"
+ " {{- '<|channel>thought\\n' -}}\n"
+ " {%- endif -%}\n"
+ "{%- endif -%}\n"
+)
+GOOGLE_E2B_TAIL = (
+ "{%- if add_generation_prompt -%}\n"
+ " {%- if ns.prev_message_type != 'tool_response' and ns.prev_message_type != 'tool_call' -%}\n"
+ " {{- '<|turn>model\\n' -}}\n"
+ " {%- elif ns.prev_message_type == 'tool_response' and enable_thinking -%}\n"
+ " {{- '<|channel>thought\\n' -}}\n"
+ " {%- endif -%}\n"
+ "{%- endif -%}\n"
+)
+
+CONVO = [
+ {"role": "system", "content": "Be brief."},
+ {"role": "user", "content": "Hi"},
+ {"role": "assistant", "content": "Hello!"},
+ {"role": "user", "content": "2+2?"},
+ {"role": "assistant", "content": "4"},
+]
+
+
+def test_detects_26b_31b_template():
+ assert NS["_gemma4_wants_empty_thought"](_Holder(GOOGLE_26B_TAIL))
+
+
+def test_ignores_e2b_template():
+ assert not NS["_gemma4_wants_empty_thought"](_Holder(GOOGLE_E2B_TAIL))
+
+
+def test_ignores_missing_or_non_string_templates():
+ assert not NS["_gemma4_wants_empty_thought"](None, _Holder(None), _Holder(123))
+
+
+def test_ignores_unsloth_thinking_template():
+ # Re-calling get_chat_template on an E2B tokenizer already carrying it must not switch.
+ tmpl = "{{ bos_token }}" + NS["gemma4_thinking_template"]
+ assert not NS["_gemma4_wants_empty_thought"](_Holder(tmpl))
+
+
+def test_detects_own_output_on_recall():
+ tmpl = "{{ bos_token }}" + NS["gemma4_empty_thought_template"]
+ assert NS["_gemma4_wants_empty_thought"](_Holder(tmpl))
+
+
+def test_processor_dict_template():
+ assert NS["_gemma4_wants_empty_thought"](_Holder({"default": GOOGLE_26B_TAIL}))
+
+
+def test_multi_turn_with_system_prompt():
+ out = _render(NS["gemma4_empty_thought_template"], CONVO)
+ assert out == (
+ "<bos><|turn>system\nBe brief.<turn|>\n"
+ "<|turn>user\nHi<turn|>\n"
+ f"<|turn>model\n{EMPTY}Hello!<turn|>\n"
+ "<|turn>user\n2+2?<turn|>\n"
+ f"<|turn>model\n{EMPTY}4<turn|>\n"
+ )
+
+
+def test_generation_prompt_matches_google_26b():
+ msgs = [{"role": "user", "content": "Hi"}]
+ out = _render(NS["gemma4_empty_thought_template"], msgs, add_generation_prompt = True)
+ assert out == f"<bos><|turn>user\nHi<turn|>\n<|turn>model\n{EMPTY}"
+ out = _render(
+ NS["gemma4_empty_thought_template"], msgs, add_generation_prompt = True, enable_thinking = True
+ )
+ assert out == "<bos><|turn>system\n<|think|>\n<turn|>\n<|turn>user\nHi<turn|>\n<|turn>model\n"
+
+
+def test_training_text_is_prefix_consistent_with_generation_prompt():
+ full = _render(NS["gemma4_empty_thought_template"], CONVO[:3])
+ prompt = _render(NS["gemma4_empty_thought_template"], CONVO[:2], add_generation_prompt = True)
+ assert full == prompt + "Hello!<turn|>\n"
+
+
+def test_list_content():
+ msgs = [
+ {"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "What?"}]},
+ {"role": "assistant", "content": [{"type": "text", "text": "A cat."}]},
+ ]
+ out = _render(NS["gemma4_empty_thought_template"], msgs)
+ assert out == f"<bos><|turn>user\n<|image|>What?<turn|>\n<|turn>model\n{EMPTY}A cat.<turn|>\n"
+
+
[email protected]("key", ["reasoning_content", "reasoning"])
+def test_final_turn_reasoning_becomes_the_thought_channel(key):
+ msgs = [
+ {"role": "user", "content": "2+2?"},
+ {"role": "assistant", "content": "4", key: "2+2=4"},
+ ]
+ out = _render(NS["gemma4_empty_thought_template"], msgs)
+ assert (
+ out
+ == "<bos><|turn>user\n2+2?<turn|>\n<|turn>model\n<|channel>thought\n2+2=4\n<channel|>4<turn|>\n"
+ )
+
+
+def test_reasoning_before_the_last_user_turn_is_dropped_like_the_model_template():
+ msgs = [
+ {"role": "user", "content": "2+2?"},
+ {"role": "assistant", "content": "4", "reasoning_content": "2+2=4"},
+ {"role": "user", "content": "3+3?"},
+ {"role": "assistant", "content": "6"},
+ ]
+ out = _render(NS["gemma4_empty_thought_template"], msgs)
+ assert "2+2=4" not in out
+ assert out.count(EMPTY) == 2
+
+
+ "content",
+ [
+ "<|channel>thought\n2+2=4<channel|>4",
+ [{"type": "text", "text": "<|channel>thought\nx<channel|>4"}],
+ ],
+)
+def test_inline_thoughts_are_stripped_and_get_the_empty_channel(content):
+ msgs = [{"role": "user", "content": "2+2?"}, {"role": "assistant", "content": content}]
+ out = _render(NS["gemma4_empty_thought_template"], msgs)
+ assert out == f"<bos><|turn>user\n2+2?<turn|>\n<|turn>model\n{EMPTY}4<turn|>\n"
+
+
+def test_ollama_generation_prompt_has_the_empty_channel():
+ ollama = NS["gemma4_empty_thought_ollama"]
+ assert "{{ if $last }}<|turn>model\n" + EMPTY + "{{ end }}" in ollama
+
+
+def test_ollama_history_assistant_turns_have_the_empty_channel():
+ ollama = NS["gemma4_empty_thought_ollama"]
+ assert '{{- if eq .Role "assistant" }}<|turn>model\n' + EMPTY + "{{ .Content }}" in ollama
+ assert "<|turn>assistant" not in ollama
+ assert ollama.count(EMPTY) == 2
+
+
+def test_enable_thinking_is_unchanged():
+ new = _render(NS["gemma4_empty_thought_template"], CONVO, enable_thinking = True)
+ old = _render(NS["gemma4_thinking_template"], CONVO, enable_thinking = True)
+ assert new == old
+ assert EMPTY not in new
+
+
+def test_only_model_turns_change():
+ new = _render(NS["gemma4_empty_thought_template"], CONVO)
+ old = _render(NS["gemma4_thinking_template"], CONVO)
+ assert new.replace(EMPTY, "") == old
+
+
+def test_roles_must_alternate_still_raises():
+ with pytest.raises(TemplateError):
+ _render(NS["gemma4_empty_thought_template"], [{"role": "assistant", "content": "x"}])
+
+
+def test_get_chat_template_switches_only_gemma4_names():
+ body = next(
+ node
+ for node in _TREE.body
+ if isinstance(node, ast.FunctionDef) and node.name == "get_chat_template"
+ )
+ src = ast.get_source_segment(_SOURCE, body)
+ assert "_gemma4_wants_empty_thought(_processor, old_tokenizer)" in src
+ assert "type_chat_template in GEMMA4_TEMPLATE_NAMES" in src
+ assert set(NS["GEMMA4_TEMPLATE_NAMES"]) == {
+ "gemma-4",
+ "gemma4",
+ "gemma-4-thinking",
+ "gemma4-thinking",
+ }
diff --git a/unsloth/chat_templates.py b/unsloth/chat_templates.py
index f09f925986f..3c2ed422417 100644
--- a/unsloth/chat_templates.py
+++ b/unsloth/chat_templates.py
@@ -1070,6 +1070,61 @@ DEFAULT_SYSTEM_MESSAGE["gemma-4-thinking"] = None
CHAT_TEMPLATES["gemma4-thinking"] = (gemma4_thinking_template, gemma4_template_eos_token, False, gemma4_ollama,)
DEFAULT_SYSTEM_MESSAGE["gemma4-thinking"] = None
+# 26B-A4B / 31B generate after an empty thought channel their template omits from history; final-turn reasoning fills it.
+_gemma4_model_turn = "{{ '<|turn>' + role + '\n' }}\n"
+gemma4_empty_thought_template = gemma4_thinking_template.replace(
+ _gemma4_model_turn,
+ _gemma4_model_turn + \
+""" {%- if role == "model" -%}
+ {%- set thinking_text = message.get('reasoning') or message.get('reasoning_content') -%}
+ {%- if thinking_text and loop.index0 > ns_turn.last_user_idx -%}
+ {{ '<|channel>thought\n' + thinking_text + '\n<channel|>' }}
+ {%- elif not thinking -%}
+ {{ '<|channel>thought\n<channel|>' }}
+ {%- endif -%}
+ {%- endif -%}
+""",
+ 1,
+).replace(
+ "{%- for message in loop_messages -%}",
+ "{%- set ns_turn = namespace(last_user_idx=-1) -%}\n"
+ "{%- for m in loop_messages -%}{%- if m['role'] == 'user' -%}{%- set ns_turn.last_user_idx = loop.index0 -%}{%- endif -%}{%- endfor -%}\n"
+ "{%- for message in loop_messages -%}",
+ 1,
+)
+assert gemma4_empty_thought_template != gemma4_thinking_template
+# Ollama re-renders history each request, so every assistant turn gets the channel too.
+gemma4_empty_thought_ollama = '''
+FROM {__FILE_LOCATION__}
+TEMPLATE """{{- range $i, $_ := .Messages }}
+{{- $last := eq (len (slice $.Messages $i)) 1 }}
+{{- if eq .Role "assistant" }}<|turn>model
+<|channel>thought
+<channel|>{{ .Content }}{{ if not $last }}<turn|>
+{{ end }}
+{{- else }}<|turn>{{ .Role }}
+{{ .Content }}<turn|>
+{{ if $last }}<|turn>model
+<|channel>thought
+<channel|>{{ end }}
+{{- end }}
+{{- end }}"""
+'''
+GEMMA4_TEMPLATE_NAMES = ("gemma-4", "gemma4", "gemma-4-thinking", "gemma4-thinking",)
+
+
+def _gemma4_wants_empty_thought(*holders):
+ # Content based: the model's own template primes the empty channel (E2B / E4B do not).
+ for holder in holders:
+ template = getattr(holder, "chat_template", None)
+ if isinstance(template, dict): template = template.get("default")
+ if not isinstance(template, str): continue
+ # Unsloth's gemma-4-thinking primes it for every size
+ if template.endswith(gemma4_thinking_template): continue
+ if "<|channel>thought\\n<channel|>" in template or "<|channel>thought\n<channel|>" in template:
+ return True
+ return False
+
# Obtained via print(tokenizer.chat_template.replace("}\n", "####").replace("\n", "\\n").replace("####", "}\n"))
# =========================================== GPT-OSS
gptoss_template = \
@@ -1955,6 +2010,15 @@ def get_chat_template(
chat_template, stop_word, yes_map_eos_token, ollama_modelfile = CHAT_TEMPLATES[chat_template]
+ if type_chat_template in GEMMA4_TEMPLATE_NAMES and \
+ _gemma4_wants_empty_thought(_processor, old_tokenizer):
+ logger.warning_once(
+ "Unsloth: This Gemma-4 model expects an empty thought channel on non-thinking turns. "\
+ "Adding <|channel>thought\\n<channel|> to assistant turns without thinking content."
+ )
+ chat_template = gemma4_empty_thought_template
+ ollama_modelfile = gemma4_empty_thought_ollama
+
# The template can veto the eos mapping, but it must not force it back on: map_eos_token = False is
# an explicit choice by the caller.
if not yes_map_eos_token and map_eos_token: map_eos_token = False