Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

assistant_only_loss with Unsloth chat templates: fall back to response markers (#13204)

Daniel Hane7a1a917bd05
diff --git a/tests/test_sft_assistant_mask_fallback.py b/tests/test_sft_assistant_mask_fallback.py
new file mode 100644
index 00000000000..069d3f0a8a9
--- /dev/null
+++ b/tests/test_sft_assistant_mask_fallback.py
@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""assistant_only_loss fallback for chat templates TRL cannot train; ast-loaded, CPU-only."""
+
+from __future__ import annotations
+
+import ast
+import re
+from pathlib import Path
+
+import pytest
+
+SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"
+NAMES = (
+ "_SFT_TRAINING_TEMPLATE",
+ "_SFT_STOP_TOKEN_CHECK",
+ "_zoo_reads_assistant_mask_fallback",
+ "sft_trainer_assistant_mask_fallback",
+)
+
+
+def _load(zoo_reads_flag = True):
+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"))
+ nodes = []
+ for node in tree.body:
+ if isinstance(node, ast.FunctionDef) and node.name in NAMES:
+ nodes.append(node)
+ elif isinstance(node, ast.Assign) and any(
+ isinstance(t, ast.Name) and t.id in NAMES for t in node.targets
+ ):
+ nodes.append(node)
+ namespace = {"re": re}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)
+ namespace["_zoo_reads_assistant_mask_fallback"] = lambda: zoo_reads_flag
+ return namespace["sft_trainer_assistant_mask_fallback"]
+
+
+TRL_SHAPED_INIT = """
+def __init__(self, processing_class, args, get_training_chat_template, is_chat_template_stop_token_trained, warn):
+ if args.assistant_only_loss and not has_generation_markers(processing_class.chat_template):
+ self.chat_template = get_training_chat_template(processing_class)
+ else:
+ self.chat_template = None
+ if args.assistant_only_loss and not is_chat_template_stop_token_trained(
+ processing_class, chat_template=self.chat_template
+ ):
+ warn()
+"""
+
+
+class _Args:
+ assistant_only_loss = True
+ dataset_kwargs = None
+
+
+class _Tok:
+ chat_template = "{{ messages }}"
+
+
+def _run(source, template_fn):
+ namespace = {"has_generation_markers": lambda template: False}
+ exec(source, namespace)
+ trainer = type("T", (), {})()
+ warned = []
+ namespace["__init__"](
+ trainer, _Tok(), _Args(), template_fn, lambda *a, **k: False, lambda: warned.append(1)
+ )
+ return trainer, warned
+
+
+def _unsupported(processing_class):
+ raise ValueError("The chat template is not training-compatible")
+
+
+def test_unsupported_template_falls_back_instead_of_raising():
+ fallback = _load()
+ with pytest.raises(ValueError):
+ _run(TRL_SHAPED_INIT, _unsupported)
+ trainer, warned = _run(fallback("__init__", TRL_SHAPED_INIT), _unsupported)
+ assert trainer.chat_template is None
+ assert trainer._unsloth_assistant_mask_fallback is True
+ assert warned == [] # the stop-token warning describes TRL's masks, not the marker masks
+
+
+def test_supported_template_keeps_trl_training_template():
+ fallback = _load()
+ trainer, warned = _run(fallback("__init__", TRL_SHAPED_INIT), lambda pc: "TRAINING_TEMPLATE")
+ assert trainer.chat_template == "TRAINING_TEMPLATE"
+ assert not hasattr(trainer, "_unsloth_assistant_mask_fallback")
+ assert warned == [1]
+
+
+def test_other_functions_and_idempotent():
+ fallback = _load()
+ assert fallback("compute_loss", TRL_SHAPED_INIT) == TRL_SHAPED_INIT
+ once = fallback("__init__", TRL_SHAPED_INIT)
+ assert fallback("__init__", once) == once
+
+
+def test_applies_to_installed_trl():
+ sft_trainer = pytest.importorskip("trl.trainer.sft_trainer")
+ # The class, not __init__: other tests may leave __init__ wrapped.
+ source = Path(sft_trainer.__file__).read_text(encoding = "utf-8")
+ if "get_training_chat_template(processing_class)" not in source:
+ pytest.skip("installed TRL predates the training-template swap")
+ patched = _load()("__init__", source)
+ assert "_unsloth_assistant_mask_fallback = True" in patched
+ assert "not getattr(self, '_unsloth_assistant_mask_fallback', False)" in patched
+ compile(patched, "<patched SFTTrainer.__init__>", "exec")
+
+
+def test_older_zoo_keeps_trl_error():
+ # Without a Zoo that masks from the flag, TRL's error must stand rather than train on every token.
+ with pytest.raises(ValueError):
+ _run(_load(zoo_reads_flag = False)("__init__", TRL_SHAPED_INIT), _unsupported)
+
+
+def test_vision_dataset_keeps_trl_error():
+ namespace = {"has_generation_markers": lambda template: False}
+ exec(_load()("__init__", TRL_SHAPED_INIT), namespace)
+ trainer = type("T", (), {"_is_vision_dataset": True})()
+ with pytest.raises(ValueError):
+ namespace["__init__"](
+ trainer, _Tok(), _Args(), _unsupported, lambda *a, **k: False, lambda: None
+ )
+
+
+def test_skipped_preparation_keeps_trl_error():
+ namespace = {"has_generation_markers": lambda template: False}
+ exec(_load()("__init__", TRL_SHAPED_INIT), namespace)
+ args = _Args()
+ args.dataset_kwargs = {"skip_prepare_dataset": True}
+ with pytest.raises(ValueError):
+ namespace["__init__"](
+ type("T", (), {})(), _Tok(), args, _unsupported, lambda *a, **k: False, lambda: None
+ )
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 50fcf7bc8b6..133a736bf47 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -1020,6 +1020,56 @@ def sft_trainer_push_to_hub_token(function_name, function):
RL_FUNCTIONS["sft_trainer"].append(sft_trainer_push_to_hub_token)
+# assistant_only_loss: TRL raises for chat templates it does not know (all Unsloth ones); fall back to Zoo's train_on_responses_only masks.
+_SFT_TRAINING_TEMPLATE = re.compile(
+ r"^(?P<indent>[ \t]*)self\.chat_template = get_training_chat_template\(processing_class\)[ \t]*$",
+ flags = re.MULTILINE,
+)
+_SFT_STOP_TOKEN_CHECK = re.compile(
+ r"if args\.assistant_only_loss and not is_chat_template_stop_token_trained\("
+)
+
+
+def _zoo_reads_assistant_mask_fallback():
+ # An older unsloth_zoo never reads the flag, so suppressing TRL's error there would train on every token.
+ try:
+ from unsloth_zoo.dataset_utils import sft_prepare_dataset
+ return "_unsloth_assistant_mask_fallback" in inspect.getsource(sft_prepare_dataset)
+ except Exception:
+ return False
+
+
+def sft_trainer_assistant_mask_fallback(function_name, function):
+ if function_name != "__init__" or "_unsloth_assistant_mask_fallback" in function:
+ return function
+ if _SFT_TRAINING_TEMPLATE.search(function) is None or not _zoo_reads_assistant_mask_fallback():
+ return function
+
+ def _replace(match):
+ i = match.group("indent")
+ return (
+ f"{i}try:\n"
+ f"{i} self.chat_template = get_training_chat_template(processing_class)\n"
+ f"{i}except ValueError:\n"
+ # Zoo's text preparation reads the flag; vision datasets and skip_prepare_dataset never reach it.
+ f"{i} if getattr(self, '_is_vision_dataset', False) or (getattr(args, 'dataset_kwargs', None) or {{}}).get('skip_prepare_dataset'): raise\n"
+ f"{i} self.chat_template = None\n"
+ f"{i} self._unsloth_assistant_mask_fallback = True\n"
+ f"{i} print('Unsloth: TRL has no training chat template for this tokenizer, so assistant_only_loss masks non-assistant tokens with train_on_responses_only markers.')"
+ )
+
+ function = _SFT_TRAINING_TEMPLATE.sub(_replace, function, count = 1)
+ function = _SFT_STOP_TOKEN_CHECK.sub(
+ "if args.assistant_only_loss and not getattr(self, '_unsloth_assistant_mask_fallback', False) and not is_chat_template_stop_token_trained(",
+ function,
+ count = 1,
+ )
+ return function
+
+
+RL_FUNCTIONS["sft_trainer"].append(sft_trainer_assistant_mask_fallback)
+
+
def _unsloth_grpo_autocast(self):
"""Decide the GRPO autocast once and latch it on the trainer. ACCELERATE_MIXED_PRECISION is process wide, so a trainer built later but run first would hand this trainer its precision; args belongs to this trainer."""
if not hasattr(self, "_autocast_enabled"):