Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
assistant_only_loss with Unsloth chat templates: fall back to response markers (#13204)
Daniel Hane7a1a917bd05
diff --git a/tests/test_sft_assistant_mask_fallback.py b/tests/test_sft_assistant_mask_fallback.pynew file mode 100644index 00000000000..069d3f0a8a9--- /dev/null+++ b/tests/test_sft_assistant_mask_fallback.py@@ -0,0 +1,137 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""assistant_only_loss fallback for chat templates TRL cannot train; ast-loaded, CPU-only."""++from __future__ import annotations++import ast+import re+from pathlib import Path++import pytest++SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+NAMES = (+ "_SFT_TRAINING_TEMPLATE",+ "_SFT_STOP_TOKEN_CHECK",+ "_zoo_reads_assistant_mask_fallback",+ "sft_trainer_assistant_mask_fallback",+)+++def _load(zoo_reads_flag = True):+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"))+ nodes = []+ for node in tree.body:+ if isinstance(node, ast.FunctionDef) and node.name in NAMES:+ nodes.append(node)+ elif isinstance(node, ast.Assign) and any(+ isinstance(t, ast.Name) and t.id in NAMES for t in node.targets+ ):+ nodes.append(node)+ namespace = {"re": re}+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)+ namespace["_zoo_reads_assistant_mask_fallback"] = lambda: zoo_reads_flag+ return namespace["sft_trainer_assistant_mask_fallback"]+++TRL_SHAPED_INIT = """+def __init__(self, processing_class, args, get_training_chat_template, is_chat_template_stop_token_trained, warn):+ if args.assistant_only_loss and not has_generation_markers(processing_class.chat_template):+ self.chat_template = get_training_chat_template(processing_class)+ else:+ self.chat_template = None+ if args.assistant_only_loss and not is_chat_template_stop_token_trained(+ processing_class, chat_template=self.chat_template+ ):+ warn()+"""+++class _Args:+ assistant_only_loss = True+ dataset_kwargs = None+++class _Tok:+ chat_template = "{{ messages }}"+++def _run(source, template_fn):+ namespace = {"has_generation_markers": lambda template: False}+ exec(source, namespace)+ trainer = type("T", (), {})()+ warned = []+ namespace["__init__"](+ trainer, _Tok(), _Args(), template_fn, lambda *a, **k: False, lambda: warned.append(1)+ )+ return trainer, warned+++def _unsupported(processing_class):+ raise ValueError("The chat template is not training-compatible")+++def test_unsupported_template_falls_back_instead_of_raising():+ fallback = _load()+ with pytest.raises(ValueError):+ _run(TRL_SHAPED_INIT, _unsupported)+ trainer, warned = _run(fallback("__init__", TRL_SHAPED_INIT), _unsupported)+ assert trainer.chat_template is None+ assert trainer._unsloth_assistant_mask_fallback is True+ assert warned == [] # the stop-token warning describes TRL's masks, not the marker masks+++def test_supported_template_keeps_trl_training_template():+ fallback = _load()+ trainer, warned = _run(fallback("__init__", TRL_SHAPED_INIT), lambda pc: "TRAINING_TEMPLATE")+ assert trainer.chat_template == "TRAINING_TEMPLATE"+ assert not hasattr(trainer, "_unsloth_assistant_mask_fallback")+ assert warned == [1]+++def test_other_functions_and_idempotent():+ fallback = _load()+ assert fallback("compute_loss", TRL_SHAPED_INIT) == TRL_SHAPED_INIT+ once = fallback("__init__", TRL_SHAPED_INIT)+ assert fallback("__init__", once) == once+++def test_applies_to_installed_trl():+ sft_trainer = pytest.importorskip("trl.trainer.sft_trainer")+ # The class, not __init__: other tests may leave __init__ wrapped.+ source = Path(sft_trainer.__file__).read_text(encoding = "utf-8")+ if "get_training_chat_template(processing_class)" not in source:+ pytest.skip("installed TRL predates the training-template swap")+ patched = _load()("__init__", source)+ assert "_unsloth_assistant_mask_fallback = True" in patched+ assert "not getattr(self, '_unsloth_assistant_mask_fallback', False)" in patched+ compile(patched, "<patched SFTTrainer.__init__>", "exec")+++def test_older_zoo_keeps_trl_error():+ # Without a Zoo that masks from the flag, TRL's error must stand rather than train on every token.+ with pytest.raises(ValueError):+ _run(_load(zoo_reads_flag = False)("__init__", TRL_SHAPED_INIT), _unsupported)+++def test_vision_dataset_keeps_trl_error():+ namespace = {"has_generation_markers": lambda template: False}+ exec(_load()("__init__", TRL_SHAPED_INIT), namespace)+ trainer = type("T", (), {"_is_vision_dataset": True})()+ with pytest.raises(ValueError):+ namespace["__init__"](+ trainer, _Tok(), _Args(), _unsupported, lambda *a, **k: False, lambda: None+ )+++def test_skipped_preparation_keeps_trl_error():+ namespace = {"has_generation_markers": lambda template: False}+ exec(_load()("__init__", TRL_SHAPED_INIT), namespace)+ args = _Args()+ args.dataset_kwargs = {"skip_prepare_dataset": True}+ with pytest.raises(ValueError):+ namespace["__init__"](+ type("T", (), {})(), _Tok(), args, _unsupported, lambda *a, **k: False, lambda: None+ )diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 50fcf7bc8b6..133a736bf47 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -1020,6 +1020,56 @@ def sft_trainer_push_to_hub_token(function_name, function):RL_FUNCTIONS["sft_trainer"].append(sft_trainer_push_to_hub_token)+# assistant_only_loss: TRL raises for chat templates it does not know (all Unsloth ones); fall back to Zoo's train_on_responses_only masks.+_SFT_TRAINING_TEMPLATE = re.compile(+ r"^(?P<indent>[ \t]*)self\.chat_template = get_training_chat_template\(processing_class\)[ \t]*$",+ flags = re.MULTILINE,+)+_SFT_STOP_TOKEN_CHECK = re.compile(+ r"if args\.assistant_only_loss and not is_chat_template_stop_token_trained\("+)+++def _zoo_reads_assistant_mask_fallback():+ # An older unsloth_zoo never reads the flag, so suppressing TRL's error there would train on every token.+ try:+ from unsloth_zoo.dataset_utils import sft_prepare_dataset+ return "_unsloth_assistant_mask_fallback" in inspect.getsource(sft_prepare_dataset)+ except Exception:+ return False+++def sft_trainer_assistant_mask_fallback(function_name, function):+ if function_name != "__init__" or "_unsloth_assistant_mask_fallback" in function:+ return function+ if _SFT_TRAINING_TEMPLATE.search(function) is None or not _zoo_reads_assistant_mask_fallback():+ return function++ def _replace(match):+ i = match.group("indent")+ return (+ f"{i}try:\n"+ f"{i} self.chat_template = get_training_chat_template(processing_class)\n"+ f"{i}except ValueError:\n"+ # Zoo's text preparation reads the flag; vision datasets and skip_prepare_dataset never reach it.+ f"{i} if getattr(self, '_is_vision_dataset', False) or (getattr(args, 'dataset_kwargs', None) or {{}}).get('skip_prepare_dataset'): raise\n"+ f"{i} self.chat_template = None\n"+ f"{i} self._unsloth_assistant_mask_fallback = True\n"+ f"{i} print('Unsloth: TRL has no training chat template for this tokenizer, so assistant_only_loss masks non-assistant tokens with train_on_responses_only markers.')"+ )++ function = _SFT_TRAINING_TEMPLATE.sub(_replace, function, count = 1)+ function = _SFT_STOP_TOKEN_CHECK.sub(+ "if args.assistant_only_loss and not getattr(self, '_unsloth_assistant_mask_fallback', False) and not is_chat_template_stop_token_trained(",+ function,+ count = 1,+ )+ return function+++RL_FUNCTIONS["sft_trainer"].append(sft_trainer_assistant_mask_fallback)++def _unsloth_grpo_autocast(self):"""Decide the GRPO autocast once and latch it on the trainer. ACCELERATE_MIXED_PRECISION is process wide, so a trainer built later but run first would hand this trainer its precision; args belongs to this trainer."""if not hasattr(self, "_autocast_enabled"):