Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

GRPO tool calling: accept Unsloth chat templates in TRL's response schema and training template lookup (#13202)

Daniel Hanaffbc0a826fd
diff --git a/tests/test_trl_tool_chat_templates.py b/tests/test_trl_tool_chat_templates.py
new file mode 100644
index 00000000000..a5e7f06537d
--- /dev/null
+++ b/tests/test_trl_tool_chat_templates.py
@@ -0,0 +1,208 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""TRL GRPO tools= / environment_factory= on Unsloth-edited chat templates.
+
+TRL matches chat templates by exact string, so an unsloth/* repo whose template carries fixes is
+rejected with "Unrecognized chat template". The patch must accept such a template only when it
+renders like a TRL-known one, reuse that family's parser / training template, and leave every
+other outcome of TRL unchanged.
+"""
+
+from __future__ import annotations
+
+import copy
+import importlib.util
+import pathlib
+import sys
+
+import pytest
+
+trl_utils = pytest.importorskip("trl.chat_template_utils")
+if not hasattr(trl_utils, "add_response_schema") or not hasattr(trl_utils, "qwen3_chat_template"):
+ pytest.skip("this TRL has no tool-call response parsing", allow_module_level = True)
+
+from tokenizers import Tokenizer, decoders, pre_tokenizers
+from tokenizers.models import BPE
+from transformers import PreTrainedTokenizerFast
+
+MODULE_PATH = (
+ pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_trl_tool_templates.py"
+)
+NAMES = ("add_response_schema", "get_training_chat_template")
+
+# What unsloth/Qwen3-* ships: same rendering, different source (here a comment and a None guard).
+UNSLOTH_QWEN3 = "{#- Chat template fixes by Unsloth #}\n" + trl_utils.qwen3_chat_template.replace(
+ "{%- if message.content is string %}",
+ "{%- if message.content is defined and message.content is string %}",
+ 1,
+)
+PLAIN = "{% for m in messages %}{{ m['role'] }}: {{ m['content'] }}\n{% endfor %}"
+
+
+def _tokenizer(template):
+ # One token per byte, so token ids keep the text (TRL's prefix check compares ids).
+ alphabet = sorted(pre_tokenizers.ByteLevel.alphabet())
+ backend = Tokenizer(BPE({c: i for i, c in enumerate(alphabet)}, []))
+ backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)
+ backend.decoder = decoders.ByteLevel()
+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend)
+ tokenizer.chat_template = template
+ return tokenizer
+
+
+def _parser(tokenizer):
+ return tuple(getattr(tokenizer, name, None) for name in ("response_template", "response_schema"))
+
+
+def patched():
+ saved = [
+ (module, name, module.__dict__[name])
+ for module in list(sys.modules.values())
+ for name in NAMES
+ if name in getattr(module, "__dict__", {})
+ ]
+ originals = {}
+ for name in NAMES:
+ # conftest may already have imported unsloth, which installs the patch.
+ function = getattr(trl_utils, name)
+ while getattr(function, "_unsloth_tool_template_patched", False):
+ function = function.__wrapped__
+ originals[name] = function
+ spec = importlib.util.spec_from_file_location("_unsloth_trl_tool_templates", MODULE_PATH)
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ assert module.patch_trl_tool_chat_templates()
+ yield module, originals
+ for owner, name, value in saved:
+ setattr(owner, name, value)
+
+
+def test_rejected_unsloth_qwen3_template_gets_the_qwen3_parser(patched):
+ _, originals = patched
+ tokenizer = _tokenizer(UNSLOTH_QWEN3)
+ with pytest.raises(ValueError, match = "Unrecognized chat template"):
+ originals["add_response_schema"](copy.copy(tokenizer))
+
+ reference = originals["add_response_schema"](_tokenizer(trl_utils.qwen3_chat_template))
+ assert trl_utils.add_response_schema(tokenizer) is tokenizer
+ assert _parser(tokenizer) == _parser(reference)
+ assert any(value is not None for value in _parser(tokenizer))
+ assert tokenizer.chat_template == UNSLOTH_QWEN3
+
+
+def test_training_template_follows_the_matching_family(patched):
+ _, originals = patched
+ tokenizer = _tokenizer(UNSLOTH_QWEN3)
+ assert not trl_utils.is_chat_template_prefix_preserving(tokenizer)
+ with pytest.raises(ValueError):
+ originals["get_training_chat_template"](tokenizer)
+
+ expected = originals["get_training_chat_template"](_tokenizer(trl_utils.qwen3_chat_template))
+ assert trl_utils.get_training_chat_template(tokenizer) == expected
+ assert tokenizer.chat_template == UNSLOTH_QWEN3
+
+
+def test_known_templates_and_unrelated_failures_are_untouched(patched):
+ _, originals = patched
+ known = _tokenizer(trl_utils.qwen3_chat_template)
+ expected = _parser(originals["add_response_schema"](_tokenizer(trl_utils.qwen3_chat_template)))
+ assert _parser(trl_utils.add_response_schema(known)) == expected
+
+ # Renders no tool calls, so it matches no family and TRL's own error must surface.
+ with pytest.raises(ValueError, match = "Unrecognized chat template"):
+ trl_utils.add_response_schema(_tokenizer(PLAIN))
+
+
+def test_edited_tool_call_rendering_is_not_matched(patched):
+ edited = UNSLOTH_QWEN3.replace("'<tool_call>\\n{\"name\": \"'", "'<tool_call>{\"name\": \"'")
+ assert edited != UNSLOTH_QWEN3
+ tokenizer = _tokenizer(edited)
+ with pytest.raises(ValueError, match = "Unrecognized chat template"):
+ trl_utils.add_response_schema(tokenizer)
+
+
+def test_legacy_response_schema_path(patched, monkeypatch):
+ if not hasattr(trl_utils, "_SUPPORTS_RESPONSE_TEMPLATE"):
+ pytest.skip("TRL predates response_template")
+ monkeypatch.setattr(trl_utils, "_SUPPORTS_RESPONSE_TEMPLATE", False)
+ tokenizer = _tokenizer(UNSLOTH_QWEN3)
+ trl_utils.add_response_schema(tokenizer)
+ assert tokenizer.response_schema == trl_utils.qwen3_schema
+
+
+def test_patch_is_idempotent_and_rebinds_trainer_imports(patched):
+ module, _ = patched
+ first = trl_utils.add_response_schema
+ assert module.patch_trl_tool_chat_templates()
+ assert trl_utils.add_response_schema is first
+ grpo = pytest.importorskip("trl.trainer.grpo_trainer")
+ for name in NAMES:
+ if name in grpo.__dict__:
+ assert getattr(grpo, name) is getattr(trl_utils, name)
+ assert getattr(trl_utils, name)._unsloth_tool_template_patched
+
+
+def test_template_refusing_parallel_tool_calls_still_matches(patched):
+ # Llama 3.1/3.2 raise on parallel tool calls; an edited copy must still match its family.
+ if "only supports single tool-calls" not in trl_utils.llama3_2_chat_template:
+ pytest.skip("this TRL's Llama 3.2 template accepts parallel tool calls")
+ _, originals = patched
+ edited = "{#- Chat template fixes by Unsloth #}\n" + trl_utils.llama3_2_chat_template
+ tokenizer = _tokenizer(edited)
+ tokenizer.bos_token = "<|begin_of_text|>"
+ with pytest.raises(ValueError, match = "Unrecognized chat template"):
+ originals["add_response_schema"](copy.copy(tokenizer))
+ reference = _tokenizer(trl_utils.llama3_2_chat_template)
+ reference.bos_token = "<|begin_of_text|>"
+ reference = originals["add_response_schema"](reference)
+ trl_utils.add_response_schema(tokenizer)
+ assert _parser(tokenizer) == _parser(reference)
+
+
+def test_template_ignoring_enable_thinking_does_not_borrow_qwen3(patched):
+ # Qwen3-4B-Instruct-2507 lacks qwen3's empty <think> for enable_thinking=False.
+ branch = "{%- if enable_thinking is defined and enable_thinking is false %}"
+ assert branch in UNSLOTH_QWEN3
+ ignores = UNSLOTH_QWEN3.replace(branch, "{%- if false %}", 1)
+ with pytest.raises(ValueError):
+ trl_utils.get_training_chat_template(_tokenizer(ignores))
+ with pytest.raises(ValueError, match = "Unrecognized chat template"):
+ trl_utils.add_response_schema(_tokenizer(ignores))
+
+
+def test_vision_template_keeps_its_own_training_family(patched):
+ if not hasattr(trl_utils, "qwen3_vl_chat_template"):
+ pytest.skip("this TRL has no Qwen3-VL template")
+ _, originals = patched
+ edited = "{#- Chat template fixes by Unsloth #}\n" + trl_utils.qwen3_vl_chat_template
+ try:
+ expected = originals["get_training_chat_template"](_tokenizer(trl_utils.qwen3_vl_chat_template))
+ except ValueError:
+ pytest.skip("this TRL has no Qwen3-VL training template")
+ got = trl_utils.get_training_chat_template(_tokenizer(edited))
+ assert got == (expected or trl_utils.qwen3_vl_chat_template)
+
+
+def test_named_template_sets_are_left_to_trl(patched):
+ tokenizer = _tokenizer(UNSLOTH_QWEN3)
+ tokenizer.chat_template = {"default": UNSLOTH_QWEN3, "tool_use": PLAIN}
+ with pytest.raises(ValueError, match = "Unrecognized chat template"):
+ trl_utils.add_response_schema(tokenizer)
+ # A lone entry is what transformers uses for every call, so it is matched like a plain template.
+ tokenizer.chat_template = {"tool_use": UNSLOTH_QWEN3}
+ trl_utils.add_response_schema(tokenizer)
+ assert any(value is not None for value in _parser(tokenizer))
+
+
+def test_typed_tool_arguments_pick_the_right_training_family(patched):
+ # Qwen3.6 writes JSON true / null, Qwen3.5 Python True / None; integer-only probes miss it.
+ if not (hasattr(trl_utils, "qwen3_6_chat_template") and hasattr(trl_utils, "qwen3_5_think_chat_template")):
+ pytest.skip("this TRL lacks the Qwen3.5 / 3.6 templates")
+ module, _ = patched
+ tokenizer = _tokenizer(PLAIN)
+ for name in ("qwen3_6_chat_template", "qwen3_5_think_chat_template"):
+ edited = "{#- Chat template fixes by Unsloth #}\n" + getattr(trl_utils, name)
+ found = [n for n, _ in module._matching_families(trl_utils, tokenizer, edited, "conversation")]
+ assert found == [name]
diff --git a/unsloth/models/_trl_tool_templates.py b/unsloth/models/_trl_tool_templates.py
new file mode 100644
index 00000000000..d29a305438a
--- /dev/null
+++ b/unsloth/models/_trl_tool_templates.py
@@ -0,0 +1,331 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.
+
+# TRL matches chat templates by exact string (add_response_schema, get_training_chat_template), rejecting
+# unsloth/* fixed templates for GRPO tools=. On rejection, reuse the TRL-known template rendering the same
+# text. The user's chat_template is never modified.
+
+__all__ = [
+ "patch_trl_tool_chat_templates",
+]
+
+import copy
+import functools
+import itertools
+import logging
+import sys
+
+logger = logging.getLogger(__name__)
+
+_RESPONSE_SCHEMA_ERRORS = ("Unrecognized chat template",)
+_TRAINING_TEMPLATE_ERRORS = ("patching is not supported",)
+_PARSER_ATTRIBUTES = ("response_template", "response_schema")
+
+_TOOLS = [
+ {
+ "type": "function",
+ "function": {
+ "name": "multiply",
+ "description": "Multiply two integers.",
+ "parameters": {
+ "type": "object",
+ "properties": {
+ "a": {"type": "integer", "description": "First factor."},
+ "b": {"type": "integer", "description": "Second factor."},
+ },
+ "required": ["a", "b"],
+ },
+ },
+ }
+]
+
+
+def _call(a, b):
+ return {"type": "function", "function": {"name": "multiply", "arguments": {"a": a, "b": b}}}
+
+
+_USER = {"role": "user", "content": "What is 3 times 4?"}
+# Families differ on non-integer args (JSON true / null vs Python True / None).
+_TYPED_CALL = {
+ "type": "function",
+ "function": {
+ "name": "multiply",
+ "arguments": {"a": 3, "b": 4.5, "exact": True, "note": None, "unit": "m"},
+ },
+}
+_TOOL_TURN = {"role": "assistant", "content": "", "tool_calls": [_call(3, 4)]}
+_TOOL_RESULT = {"role": "tool", "name": "multiply", "content": "12"}
+
+# (history, assistant turn the model would generate): the parser only ever sees generated text.
+_COMPLETION_PROBES = (
+ ([_USER], _TOOL_TURN),
+ ([_USER], {"role": "assistant", "content": "Let me compute.", "tool_calls": [_call(3, 4)]}),
+ ([_USER], {"role": "assistant", "content": "", "tool_calls": [_call(3, 4), _call(5, 6)]}),
+ ([_USER], {"role": "assistant", "content": "The answer is 12."}),
+ ([_USER, _TOOL_TURN, _TOOL_RESULT], {"role": "assistant", "content": "The answer is 12."}),
+)
+
+# A training template replaces the user's for every prompt, so whole conversations must match.
+_CONVERSATION_PROBES = (
+ ([_USER], True, True),
+ ([_USER, _TOOL_TURN], False, True),
+ ([_USER, {"role": "assistant", "content": "", "tool_calls": [_TYPED_CALL]}], False, True),
+ ([_USER, _TOOL_TURN, _TOOL_RESULT], True, True),
+ (
+ [_USER, _TOOL_TURN, _TOOL_RESULT, {"role": "assistant", "content": "The answer is 12."}],
+ False,
+ True,
+ ),
+ ([{"role": "system", "content": "Be brief."}, _USER], True, False),
+ (
+ [
+ _USER,
+ {"role": "assistant", "content": "12."},
+ {"role": "user", "content": "And 5 times 6?"},
+ ],
+ True,
+ False,
+ ),
+)
+
+# Text and vision families differ only on images; skipped if the user's template raises on list content.
+_IMAGE_PROBE = (
+ [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "What is shown?"}]}],
+ True,
+ False,
+)
+
+
+def _tokenizer_of(processing_class):
+ from transformers import ProcessorMixin
+ if isinstance(processing_class, ProcessorMixin):
+ return processing_class.tokenizer
+ return processing_class
+
+
+def _template_of(processing_class):
+ template = getattr(processing_class, "chat_template", None)
+ if template is None and processing_class is not None:
+ template = getattr(_tokenizer_of(processing_class), "chat_template", None)
+ if isinstance(template, dict):
+ # A named set (e.g. "tool_use") is picked per call by transformers; one family cannot stand in for it.
+ template = next(iter(template.values())) if len(template) == 1 else None
+ return template if isinstance(template, str) else None
+
+
+# Else a template ignoring enable_thinking=False could borrow one writing an empty <think> into every prompt.
+_TEMPLATE_KWARGS = ({}, {"enable_thinking": False})
+
+
+def _render(tokenizer, template, messages, add_generation_prompt, tools, template_kwargs):
+ return tokenizer.apply_chat_template(
+ messages,
+ tools = _TOOLS if tools else None,
+ tokenize = False,
+ add_generation_prompt = add_generation_prompt,
+ chat_template = template,
+ **template_kwargs,
+ )
+
+
+def _completion_signature(tokenizer, template):
+ signature = []
+ rendered = 0
+ for (history, turn), kwargs in itertools.product(_COMPLETION_PROBES, _TEMPLATE_KWARGS):
+ try:
+ bare = _render(tokenizer, template, history, False, True, kwargs)
+ prompt = _render(tokenizer, template, history, True, True, kwargs)
+ full = _render(tokenizer, template, history + [turn], False, True, kwargs)
+ except Exception as error:
+ # Some templates refuse a probe (Llama 3.1 raises on parallel tool calls): match the refusal too.
+ signature.append(("error", type(error).__name__))
+ continue
+ rendered += 1
+ # Parsers also read what the generation prompt pre-writes (e.g. an opening <think>).
+ signature.append(prompt[len(bare) :] if prompt.startswith(bare) else prompt)
+ signature.append(full[len(prompt) :] if full.startswith(prompt) else full)
+ return tuple(signature) if rendered else None
+
+
+def _conversation_signature(tokenizer, template):
+ try:
+ return tuple(
+ _render(tokenizer, template, messages, add_generation_prompt, tools, kwargs)
+ for (messages, add_generation_prompt, tools), kwargs in itertools.product(
+ _CONVERSATION_PROBES, _TEMPLATE_KWARGS
+ )
+ )
+ except Exception:
+ return None
+
+
+def _image_signature(tokenizer, template):
+ try:
+ return tuple(
+ _render(tokenizer, template, *_IMAGE_PROBE, kwargs) for kwargs in _TEMPLATE_KWARGS
+ )
+ except Exception:
+ return None
+
+
+def _known_templates(trl_module):
+ return sorted(
+ (name, value)
+ for name, value in vars(trl_module).items()
+ if name.endswith("_chat_template")
+ and not name.endswith("_training_chat_template")
+ and isinstance(value, str)
+ )
+
+
+def _template_copy(tokenizer, template):
+ clone = copy.copy(tokenizer)
+ clone.chat_template = template
+ for attribute in _PARSER_ATTRIBUTES:
+ if attribute in vars(clone):
+ setattr(clone, attribute, None)
+ return clone
+
+
+_matches = {}
+
+
+def _matching_families(trl_module, tokenizer, template, kind):
+ # Templates read special tokens (bos_token, eos_token), so they are part of what renders.
+ key = (
+ id(trl_module),
+ kind,
+ template,
+ repr(sorted(getattr(tokenizer, "special_tokens_map", {}).items())),
+ )
+ if key in _matches:
+ return _matches[key]
+ signature_of = _completion_signature if kind == "completion" else _conversation_signature
+ target = signature_of(tokenizer, template)
+ image_target = _image_signature(tokenizer, template) if kind == "conversation" else None
+ found = []
+ if target is not None:
+ for name, known in _known_templates(trl_module):
+ if known == template or signature_of(tokenizer, known) != target:
+ continue
+ if image_target is not None and _image_signature(tokenizer, known) != image_target:
+ continue
+ found.append((name, known))
+ _matches[key] = found
+ return found
+
+
+def _parser_for(original, tokenizer, known):
+ clone = _template_copy(tokenizer, known)
+ try:
+ original(clone)
+ except Exception:
+ return None
+ parser = tuple((attribute, getattr(clone, attribute, None)) for attribute in _PARSER_ATTRIBUTES)
+ return parser if any(value is not None for _, value in parser) else None
+
+
+def _wrap_add_response_schema(original, trl_module):
+ @functools.wraps(original)
+ def add_response_schema(processing_class, *args, **kwargs):
+ try:
+ return original(processing_class, *args, **kwargs)
+ except ValueError as error:
+ if not any(text in str(error) for text in _RESPONSE_SCHEMA_ERRORS):
+ raise
+ tokenizer = _tokenizer_of(processing_class)
+ template = _template_of(processing_class)
+ if template is None:
+ raise
+ parsers = {}
+ for name, known in _matching_families(trl_module, tokenizer, template, "completion"):
+ parser = _parser_for(original, tokenizer, known)
+ if parser is not None:
+ parsers.setdefault(repr(parser), (name, parser))
+ if len(parsers) != 1:
+ raise
+ ((name, parser),) = parsers.values()
+ for attribute, value in parser:
+ if value is not None:
+ setattr(tokenizer, attribute, value)
+ logger.info(
+ f"Unsloth: TRL does not know this chat template; using the tool-call parser of "
+ f"`{name}`, which renders tool calls identically."
+ )
+ return processing_class
+
+ add_response_schema._unsloth_tool_template_patched = True
+ return add_response_schema
+
+
+def _wrap_get_training_chat_template(original, trl_module):
+ @functools.wraps(original)
+ def get_training_chat_template(*args, **kwargs):
+ try:
+ return original(*args, **kwargs)
+ except ValueError as error:
+ if not any(text in str(error) for text in _TRAINING_TEMPLATE_ERRORS):
+ raise
+ processing_class = (
+ args[0] if args else (kwargs.get("processing_class") or kwargs.get("tokenizer"))
+ )
+ template = _template_of(processing_class)
+ if template is None:
+ raise
+ tokenizer = _tokenizer_of(processing_class)
+ results = {}
+ for name, known in _matching_families(trl_module, tokenizer, template, "conversation"):
+ clone = copy.copy(processing_class)
+ clone.chat_template = known
+ try:
+ training = original(clone)
+ except Exception:
+ continue
+ # None: known template qualifies, but the user's copy may lack {% generation %} markers.
+ training = known if training is None else training
+ rendered = _conversation_signature(tokenizer, training)
+ if rendered is not None:
+ results.setdefault(rendered, (name, training))
+ if len(results) != 1:
+ raise
+ ((name, training),) = results.values()
+ logger.info(
+ f"Unsloth: TRL does not know this chat template; using the training template of "
+ f"`{name}`, which renders conversations identically."
+ )
+ return training
+
+ get_training_chat_template._unsloth_tool_template_patched = True
+ return get_training_chat_template
+
+
+def _rebind(name, original, replacement):
+ # TRL trainers import these by name, so rebind every module still holding the original.
+ for module in list(sys.modules.values()):
+ namespace = getattr(module, "__dict__", None)
+ if namespace is not None and namespace.get(name) is original:
+ try:
+ setattr(module, name, replacement)
+ except (AttributeError, TypeError):
+ pass
+
+
+def patch_trl_tool_chat_templates():
+ try:
+ import trl.chat_template_utils as trl_module
+ except Exception:
+ return False
+ patched = False
+ for name, wrap in (
+ ("add_response_schema", _wrap_add_response_schema),
+ ("get_training_chat_template", _wrap_get_training_chat_template),
+ ):
+ original = getattr(trl_module, name, None)
+ if original is None:
+ continue
+ if getattr(original, "_unsloth_tool_template_patched", False):
+ patched = True
+ continue
+ _rebind(name, original, wrap(original, trl_module))
+ patched = True
+ return patched
diff --git a/unsloth/models/rl.py b/unsloth/models/rl.py
index 077c1b8f950..d6edf917415 100644
--- a/unsloth/models/rl.py
+++ b/unsloth/models/rl.py
@@ -4192,6 +4192,12 @@ def PatchFastRL(algorithm = None, FastLanguageModel = None):
return
# Install the disable_gradient_checkpointing noop BEFORE patch_trl_rl_trainers, which imports more trl.* submodules: anything imported after the sys.modules walk keeps the old binding.
patch_trl_disable_gradient_checkpointing()
+ # Also before patch_trl_rl_trainers, so the generated trainers import the wrapped helpers.
+ try:
+ from ._trl_tool_templates import patch_trl_tool_chat_templates
+ patch_trl_tool_chat_templates()
+ except Exception as e:
+ logger.info(f"Unsloth: Could not patch TRL tool-call chat template matching: {e}")
patch_trl_rl_trainers()
patch_trl_openenv()
patch_trl_vllm_generation()