Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
GRPO tool calling: accept Unsloth chat templates in TRL's response schema and training template lookup (#13202)
Daniel Hanaffbc0a826fd
diff --git a/tests/test_trl_tool_chat_templates.py b/tests/test_trl_tool_chat_templates.pynew file mode 100644index 00000000000..a5e7f06537d--- /dev/null+++ b/tests/test_trl_tool_chat_templates.py@@ -0,0 +1,208 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""TRL GRPO tools= / environment_factory= on Unsloth-edited chat templates.++TRL matches chat templates by exact string, so an unsloth/* repo whose template carries fixes is+rejected with "Unrecognized chat template". The patch must accept such a template only when it+renders like a TRL-known one, reuse that family's parser / training template, and leave every+other outcome of TRL unchanged.+"""++from __future__ import annotations++import copy+import importlib.util+import pathlib+import sys++import pytest++trl_utils = pytest.importorskip("trl.chat_template_utils")+if not hasattr(trl_utils, "add_response_schema") or not hasattr(trl_utils, "qwen3_chat_template"):+ pytest.skip("this TRL has no tool-call response parsing", allow_module_level = True)++from tokenizers import Tokenizer, decoders, pre_tokenizers+from tokenizers.models import BPE+from transformers import PreTrainedTokenizerFast++MODULE_PATH = (+ pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "_trl_tool_templates.py"+)+NAMES = ("add_response_schema", "get_training_chat_template")++# What unsloth/Qwen3-* ships: same rendering, different source (here a comment and a None guard).+UNSLOTH_QWEN3 = "{#- Chat template fixes by Unsloth #}\n" + trl_utils.qwen3_chat_template.replace(+ "{%- if message.content is string %}",+ "{%- if message.content is defined and message.content is string %}",+ 1,+)+PLAIN = "{% for m in messages %}{{ m['role'] }}: {{ m['content'] }}\n{% endfor %}"+++def _tokenizer(template):+ # One token per byte, so token ids keep the text (TRL's prefix check compares ids).+ alphabet = sorted(pre_tokenizers.ByteLevel.alphabet())+ backend = Tokenizer(BPE({c: i for i, c in enumerate(alphabet)}, []))+ backend.pre_tokenizer = pre_tokenizers.ByteLevel(add_prefix_space = False)+ backend.decoder = decoders.ByteLevel()+ tokenizer = PreTrainedTokenizerFast(tokenizer_object = backend)+ tokenizer.chat_template = template+ return tokenizer+++def _parser(tokenizer):+ return tuple(getattr(tokenizer, name, None) for name in ("response_template", "response_schema"))+++def patched():+ saved = [+ (module, name, module.__dict__[name])+ for module in list(sys.modules.values())+ for name in NAMES+ if name in getattr(module, "__dict__", {})+ ]+ originals = {}+ for name in NAMES:+ # conftest may already have imported unsloth, which installs the patch.+ function = getattr(trl_utils, name)+ while getattr(function, "_unsloth_tool_template_patched", False):+ function = function.__wrapped__+ originals[name] = function+ spec = importlib.util.spec_from_file_location("_unsloth_trl_tool_templates", MODULE_PATH)+ module = importlib.util.module_from_spec(spec)+ spec.loader.exec_module(module)+ assert module.patch_trl_tool_chat_templates()+ yield module, originals+ for owner, name, value in saved:+ setattr(owner, name, value)+++def test_rejected_unsloth_qwen3_template_gets_the_qwen3_parser(patched):+ _, originals = patched+ tokenizer = _tokenizer(UNSLOTH_QWEN3)+ with pytest.raises(ValueError, match = "Unrecognized chat template"):+ originals["add_response_schema"](copy.copy(tokenizer))++ reference = originals["add_response_schema"](_tokenizer(trl_utils.qwen3_chat_template))+ assert trl_utils.add_response_schema(tokenizer) is tokenizer+ assert _parser(tokenizer) == _parser(reference)+ assert any(value is not None for value in _parser(tokenizer))+ assert tokenizer.chat_template == UNSLOTH_QWEN3+++def test_training_template_follows_the_matching_family(patched):+ _, originals = patched+ tokenizer = _tokenizer(UNSLOTH_QWEN3)+ assert not trl_utils.is_chat_template_prefix_preserving(tokenizer)+ with pytest.raises(ValueError):+ originals["get_training_chat_template"](tokenizer)++ expected = originals["get_training_chat_template"](_tokenizer(trl_utils.qwen3_chat_template))+ assert trl_utils.get_training_chat_template(tokenizer) == expected+ assert tokenizer.chat_template == UNSLOTH_QWEN3+++def test_known_templates_and_unrelated_failures_are_untouched(patched):+ _, originals = patched+ known = _tokenizer(trl_utils.qwen3_chat_template)+ expected = _parser(originals["add_response_schema"](_tokenizer(trl_utils.qwen3_chat_template)))+ assert _parser(trl_utils.add_response_schema(known)) == expected++ # Renders no tool calls, so it matches no family and TRL's own error must surface.+ with pytest.raises(ValueError, match = "Unrecognized chat template"):+ trl_utils.add_response_schema(_tokenizer(PLAIN))+++def test_edited_tool_call_rendering_is_not_matched(patched):+ edited = UNSLOTH_QWEN3.replace("'<tool_call>\\n{\"name\": \"'", "'<tool_call>{\"name\": \"'")+ assert edited != UNSLOTH_QWEN3+ tokenizer = _tokenizer(edited)+ with pytest.raises(ValueError, match = "Unrecognized chat template"):+ trl_utils.add_response_schema(tokenizer)+++def test_legacy_response_schema_path(patched, monkeypatch):+ if not hasattr(trl_utils, "_SUPPORTS_RESPONSE_TEMPLATE"):+ pytest.skip("TRL predates response_template")+ monkeypatch.setattr(trl_utils, "_SUPPORTS_RESPONSE_TEMPLATE", False)+ tokenizer = _tokenizer(UNSLOTH_QWEN3)+ trl_utils.add_response_schema(tokenizer)+ assert tokenizer.response_schema == trl_utils.qwen3_schema+++def test_patch_is_idempotent_and_rebinds_trainer_imports(patched):+ module, _ = patched+ first = trl_utils.add_response_schema+ assert module.patch_trl_tool_chat_templates()+ assert trl_utils.add_response_schema is first+ grpo = pytest.importorskip("trl.trainer.grpo_trainer")+ for name in NAMES:+ if name in grpo.__dict__:+ assert getattr(grpo, name) is getattr(trl_utils, name)+ assert getattr(trl_utils, name)._unsloth_tool_template_patched+++def test_template_refusing_parallel_tool_calls_still_matches(patched):+ # Llama 3.1/3.2 raise on parallel tool calls; an edited copy must still match its family.+ if "only supports single tool-calls" not in trl_utils.llama3_2_chat_template:+ pytest.skip("this TRL's Llama 3.2 template accepts parallel tool calls")+ _, originals = patched+ edited = "{#- Chat template fixes by Unsloth #}\n" + trl_utils.llama3_2_chat_template+ tokenizer = _tokenizer(edited)+ tokenizer.bos_token = "<|begin_of_text|>"+ with pytest.raises(ValueError, match = "Unrecognized chat template"):+ originals["add_response_schema"](copy.copy(tokenizer))+ reference = _tokenizer(trl_utils.llama3_2_chat_template)+ reference.bos_token = "<|begin_of_text|>"+ reference = originals["add_response_schema"](reference)+ trl_utils.add_response_schema(tokenizer)+ assert _parser(tokenizer) == _parser(reference)+++def test_template_ignoring_enable_thinking_does_not_borrow_qwen3(patched):+ # Qwen3-4B-Instruct-2507 lacks qwen3's empty <think> for enable_thinking=False.+ branch = "{%- if enable_thinking is defined and enable_thinking is false %}"+ assert branch in UNSLOTH_QWEN3+ ignores = UNSLOTH_QWEN3.replace(branch, "{%- if false %}", 1)+ with pytest.raises(ValueError):+ trl_utils.get_training_chat_template(_tokenizer(ignores))+ with pytest.raises(ValueError, match = "Unrecognized chat template"):+ trl_utils.add_response_schema(_tokenizer(ignores))+++def test_vision_template_keeps_its_own_training_family(patched):+ if not hasattr(trl_utils, "qwen3_vl_chat_template"):+ pytest.skip("this TRL has no Qwen3-VL template")+ _, originals = patched+ edited = "{#- Chat template fixes by Unsloth #}\n" + trl_utils.qwen3_vl_chat_template+ try:+ expected = originals["get_training_chat_template"](_tokenizer(trl_utils.qwen3_vl_chat_template))+ except ValueError:+ pytest.skip("this TRL has no Qwen3-VL training template")+ got = trl_utils.get_training_chat_template(_tokenizer(edited))+ assert got == (expected or trl_utils.qwen3_vl_chat_template)+++def test_named_template_sets_are_left_to_trl(patched):+ tokenizer = _tokenizer(UNSLOTH_QWEN3)+ tokenizer.chat_template = {"default": UNSLOTH_QWEN3, "tool_use": PLAIN}+ with pytest.raises(ValueError, match = "Unrecognized chat template"):+ trl_utils.add_response_schema(tokenizer)+ # A lone entry is what transformers uses for every call, so it is matched like a plain template.+ tokenizer.chat_template = {"tool_use": UNSLOTH_QWEN3}+ trl_utils.add_response_schema(tokenizer)+ assert any(value is not None for value in _parser(tokenizer))+++def test_typed_tool_arguments_pick_the_right_training_family(patched):+ # Qwen3.6 writes JSON true / null, Qwen3.5 Python True / None; integer-only probes miss it.+ if not (hasattr(trl_utils, "qwen3_6_chat_template") and hasattr(trl_utils, "qwen3_5_think_chat_template")):+ pytest.skip("this TRL lacks the Qwen3.5 / 3.6 templates")+ module, _ = patched+ tokenizer = _tokenizer(PLAIN)+ for name in ("qwen3_6_chat_template", "qwen3_5_think_chat_template"):+ edited = "{#- Chat template fixes by Unsloth #}\n" + getattr(trl_utils, name)+ found = [n for n, _ in module._matching_families(trl_utils, tokenizer, edited, "conversation")]+ assert found == [name]diff --git a/unsloth/models/_trl_tool_templates.py b/unsloth/models/_trl_tool_templates.pynew file mode 100644index 00000000000..d29a305438a--- /dev/null+++ b/unsloth/models/_trl_tool_templates.py@@ -0,0 +1,331 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2023-present Daniel Han-Chen & the Unsloth team. All rights reserved.++# TRL matches chat templates by exact string (add_response_schema, get_training_chat_template), rejecting+# unsloth/* fixed templates for GRPO tools=. On rejection, reuse the TRL-known template rendering the same+# text. The user's chat_template is never modified.++__all__ = [+ "patch_trl_tool_chat_templates",+]++import copy+import functools+import itertools+import logging+import sys++logger = logging.getLogger(__name__)++_RESPONSE_SCHEMA_ERRORS = ("Unrecognized chat template",)+_TRAINING_TEMPLATE_ERRORS = ("patching is not supported",)+_PARSER_ATTRIBUTES = ("response_template", "response_schema")++_TOOLS = [+ {+ "type": "function",+ "function": {+ "name": "multiply",+ "description": "Multiply two integers.",+ "parameters": {+ "type": "object",+ "properties": {+ "a": {"type": "integer", "description": "First factor."},+ "b": {"type": "integer", "description": "Second factor."},+ },+ "required": ["a", "b"],+ },+ },+ }+]+++def _call(a, b):+ return {"type": "function", "function": {"name": "multiply", "arguments": {"a": a, "b": b}}}+++_USER = {"role": "user", "content": "What is 3 times 4?"}+# Families differ on non-integer args (JSON true / null vs Python True / None).+_TYPED_CALL = {+ "type": "function",+ "function": {+ "name": "multiply",+ "arguments": {"a": 3, "b": 4.5, "exact": True, "note": None, "unit": "m"},+ },+}+_TOOL_TURN = {"role": "assistant", "content": "", "tool_calls": [_call(3, 4)]}+_TOOL_RESULT = {"role": "tool", "name": "multiply", "content": "12"}++# (history, assistant turn the model would generate): the parser only ever sees generated text.+_COMPLETION_PROBES = (+ ([_USER], _TOOL_TURN),+ ([_USER], {"role": "assistant", "content": "Let me compute.", "tool_calls": [_call(3, 4)]}),+ ([_USER], {"role": "assistant", "content": "", "tool_calls": [_call(3, 4), _call(5, 6)]}),+ ([_USER], {"role": "assistant", "content": "The answer is 12."}),+ ([_USER, _TOOL_TURN, _TOOL_RESULT], {"role": "assistant", "content": "The answer is 12."}),+)++# A training template replaces the user's for every prompt, so whole conversations must match.+_CONVERSATION_PROBES = (+ ([_USER], True, True),+ ([_USER, _TOOL_TURN], False, True),+ ([_USER, {"role": "assistant", "content": "", "tool_calls": [_TYPED_CALL]}], False, True),+ ([_USER, _TOOL_TURN, _TOOL_RESULT], True, True),+ (+ [_USER, _TOOL_TURN, _TOOL_RESULT, {"role": "assistant", "content": "The answer is 12."}],+ False,+ True,+ ),+ ([{"role": "system", "content": "Be brief."}, _USER], True, False),+ (+ [+ _USER,+ {"role": "assistant", "content": "12."},+ {"role": "user", "content": "And 5 times 6?"},+ ],+ True,+ False,+ ),+)++# Text and vision families differ only on images; skipped if the user's template raises on list content.+_IMAGE_PROBE = (+ [{"role": "user", "content": [{"type": "image"}, {"type": "text", "text": "What is shown?"}]}],+ True,+ False,+)+++def _tokenizer_of(processing_class):+ from transformers import ProcessorMixin+ if isinstance(processing_class, ProcessorMixin):+ return processing_class.tokenizer+ return processing_class+++def _template_of(processing_class):+ template = getattr(processing_class, "chat_template", None)+ if template is None and processing_class is not None:+ template = getattr(_tokenizer_of(processing_class), "chat_template", None)+ if isinstance(template, dict):+ # A named set (e.g. "tool_use") is picked per call by transformers; one family cannot stand in for it.+ template = next(iter(template.values())) if len(template) == 1 else None+ return template if isinstance(template, str) else None+++# Else a template ignoring enable_thinking=False could borrow one writing an empty <think> into every prompt.+_TEMPLATE_KWARGS = ({}, {"enable_thinking": False})+++def _render(tokenizer, template, messages, add_generation_prompt, tools, template_kwargs):+ return tokenizer.apply_chat_template(+ messages,+ tools = _TOOLS if tools else None,+ tokenize = False,+ add_generation_prompt = add_generation_prompt,+ chat_template = template,+ **template_kwargs,+ )+++def _completion_signature(tokenizer, template):+ signature = []+ rendered = 0+ for (history, turn), kwargs in itertools.product(_COMPLETION_PROBES, _TEMPLATE_KWARGS):+ try:+ bare = _render(tokenizer, template, history, False, True, kwargs)+ prompt = _render(tokenizer, template, history, True, True, kwargs)+ full = _render(tokenizer, template, history + [turn], False, True, kwargs)+ except Exception as error:+ # Some templates refuse a probe (Llama 3.1 raises on parallel tool calls): match the refusal too.+ signature.append(("error", type(error).__name__))+ continue+ rendered += 1+ # Parsers also read what the generation prompt pre-writes (e.g. an opening <think>).+ signature.append(prompt[len(bare) :] if prompt.startswith(bare) else prompt)+ signature.append(full[len(prompt) :] if full.startswith(prompt) else full)+ return tuple(signature) if rendered else None+++def _conversation_signature(tokenizer, template):+ try:+ return tuple(+ _render(tokenizer, template, messages, add_generation_prompt, tools, kwargs)+ for (messages, add_generation_prompt, tools), kwargs in itertools.product(+ _CONVERSATION_PROBES, _TEMPLATE_KWARGS+ )+ )+ except Exception:+ return None+++def _image_signature(tokenizer, template):+ try:+ return tuple(+ _render(tokenizer, template, *_IMAGE_PROBE, kwargs) for kwargs in _TEMPLATE_KWARGS+ )+ except Exception:+ return None+++def _known_templates(trl_module):+ return sorted(+ (name, value)+ for name, value in vars(trl_module).items()+ if name.endswith("_chat_template")+ and not name.endswith("_training_chat_template")+ and isinstance(value, str)+ )+++def _template_copy(tokenizer, template):+ clone = copy.copy(tokenizer)+ clone.chat_template = template+ for attribute in _PARSER_ATTRIBUTES:+ if attribute in vars(clone):+ setattr(clone, attribute, None)+ return clone+++_matches = {}+++def _matching_families(trl_module, tokenizer, template, kind):+ # Templates read special tokens (bos_token, eos_token), so they are part of what renders.+ key = (+ id(trl_module),+ kind,+ template,+ repr(sorted(getattr(tokenizer, "special_tokens_map", {}).items())),+ )+ if key in _matches:+ return _matches[key]+ signature_of = _completion_signature if kind == "completion" else _conversation_signature+ target = signature_of(tokenizer, template)+ image_target = _image_signature(tokenizer, template) if kind == "conversation" else None+ found = []+ if target is not None:+ for name, known in _known_templates(trl_module):+ if known == template or signature_of(tokenizer, known) != target:+ continue+ if image_target is not None and _image_signature(tokenizer, known) != image_target:+ continue+ found.append((name, known))+ _matches[key] = found+ return found+++def _parser_for(original, tokenizer, known):+ clone = _template_copy(tokenizer, known)+ try:+ original(clone)+ except Exception:+ return None+ parser = tuple((attribute, getattr(clone, attribute, None)) for attribute in _PARSER_ATTRIBUTES)+ return parser if any(value is not None for _, value in parser) else None+++def _wrap_add_response_schema(original, trl_module):+ @functools.wraps(original)+ def add_response_schema(processing_class, *args, **kwargs):+ try:+ return original(processing_class, *args, **kwargs)+ except ValueError as error:+ if not any(text in str(error) for text in _RESPONSE_SCHEMA_ERRORS):+ raise+ tokenizer = _tokenizer_of(processing_class)+ template = _template_of(processing_class)+ if template is None:+ raise+ parsers = {}+ for name, known in _matching_families(trl_module, tokenizer, template, "completion"):+ parser = _parser_for(original, tokenizer, known)+ if parser is not None:+ parsers.setdefault(repr(parser), (name, parser))+ if len(parsers) != 1:+ raise+ ((name, parser),) = parsers.values()+ for attribute, value in parser:+ if value is not None:+ setattr(tokenizer, attribute, value)+ logger.info(+ f"Unsloth: TRL does not know this chat template; using the tool-call parser of "+ f"`{name}`, which renders tool calls identically."+ )+ return processing_class++ add_response_schema._unsloth_tool_template_patched = True+ return add_response_schema+++def _wrap_get_training_chat_template(original, trl_module):+ @functools.wraps(original)+ def get_training_chat_template(*args, **kwargs):+ try:+ return original(*args, **kwargs)+ except ValueError as error:+ if not any(text in str(error) for text in _TRAINING_TEMPLATE_ERRORS):+ raise+ processing_class = (+ args[0] if args else (kwargs.get("processing_class") or kwargs.get("tokenizer"))+ )+ template = _template_of(processing_class)+ if template is None:+ raise+ tokenizer = _tokenizer_of(processing_class)+ results = {}+ for name, known in _matching_families(trl_module, tokenizer, template, "conversation"):+ clone = copy.copy(processing_class)+ clone.chat_template = known+ try:+ training = original(clone)+ except Exception:+ continue+ # None: known template qualifies, but the user's copy may lack {% generation %} markers.+ training = known if training is None else training+ rendered = _conversation_signature(tokenizer, training)+ if rendered is not None:+ results.setdefault(rendered, (name, training))+ if len(results) != 1:+ raise+ ((name, training),) = results.values()+ logger.info(+ f"Unsloth: TRL does not know this chat template; using the training template of "+ f"`{name}`, which renders conversations identically."+ )+ return training++ get_training_chat_template._unsloth_tool_template_patched = True+ return get_training_chat_template+++def _rebind(name, original, replacement):+ # TRL trainers import these by name, so rebind every module still holding the original.+ for module in list(sys.modules.values()):+ namespace = getattr(module, "__dict__", None)+ if namespace is not None and namespace.get(name) is original:+ try:+ setattr(module, name, replacement)+ except (AttributeError, TypeError):+ pass+++def patch_trl_tool_chat_templates():+ try:+ import trl.chat_template_utils as trl_module+ except Exception:+ return False+ patched = False+ for name, wrap in (+ ("add_response_schema", _wrap_add_response_schema),+ ("get_training_chat_template", _wrap_get_training_chat_template),+ ):+ original = getattr(trl_module, name, None)+ if original is None:+ continue+ if getattr(original, "_unsloth_tool_template_patched", False):+ patched = True+ continue+ _rebind(name, original, wrap(original, trl_module))+ patched = True+ return patcheddiff --git a/unsloth/models/rl.py b/unsloth/models/rl.pyindex 077c1b8f950..d6edf917415 100644--- a/unsloth/models/rl.py+++ b/unsloth/models/rl.py@@ -4192,6 +4192,12 @@ def PatchFastRL(algorithm = None, FastLanguageModel = None):return# Install the disable_gradient_checkpointing noop BEFORE patch_trl_rl_trainers, which imports more trl.* submodules: anything imported after the sys.modules walk keeps the old binding.patch_trl_disable_gradient_checkpointing()+ # Also before patch_trl_rl_trainers, so the generated trainers import the wrapped helpers.+ try:+ from ._trl_tool_templates import patch_trl_tool_chat_templates+ patch_trl_tool_chat_templates()+ except Exception as e:+ logger.info(f"Unsloth: Could not patch TRL tool-call chat template matching: {e}")patch_trl_rl_trainers()patch_trl_openenv()patch_trl_vllm_generation()