Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
SFT: compute loss_type="dft" on TRL's fused LM head instead of silently training NLL (#15213)
Daniel Han2c81e9f9f0c0
diff --git a/tests/test_sft_dft_loss_routing.py b/tests/test_sft_dft_loss_routing.pynew file mode 100644index 00000000000..6efb2fb3f1a--- /dev/null+++ b/tests/test_sft_dft_loss_routing.py@@ -0,0 +1,147 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""SFT loss_type="dft" must not silently train as NLL under the Unsloth compute_loss; ast-loaded, CPU-only."""++from __future__ import annotations++import ast+import importlib.util+import inspect+import textwrap+from pathlib import Path+from types import SimpleNamespace++import pytest+import torch++SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+++def _trainer_class():+ if importlib.util.find_spec("trl") is None:+ pytest.skip("trl not installed")+ import trl.trainer.utils as trl_utils++ if not hasattr(trl_utils, "add_fused_lm_head"):+ pytest.skip("TRL < 1.15 computes dft through compute_loss_func")+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"))+ nodes = [+ n+ for n in tree.body+ if isinstance(n, ast.FunctionDef) and n.name == "sft_trainer_compute_loss"+ ]+ namespace = {"inspect": inspect, "RL_FUNCTIONS": None}+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)+ source = namespace["sft_trainer_compute_loss"](+ "compute_loss", "def compute_loss(self):\n pass\n"+ )++ class Base:+ def compute_loss(+ self,+ model,+ inputs,+ return_outputs = False,+ num_items_in_batch = None,+ ):+ return ("super", num_items_in_batch)++ # Compiled into a class body, as in the generated trainer, so the zero-argument super() resolves.+ generated = {"torch": torch, "Base": Base}+ exec("class Generated(Base):\n" + textwrap.indent(textwrap.dedent(source), " "), generated)++ class Trainer(generated["Generated"]):+ def __init__(+ self,+ loss_type,+ accepts_loss_kwargs = True,+ ):+ self.args = SimpleNamespace(loss_type = loss_type, average_tokens_across_devices = False)+ self.compute_loss_func = None+ self.label_smoother = None+ self.model_accepts_loss_kwargs = accepts_loss_kwargs+ self.aux_loss_enabled = False++ return Trainer+++class _FusedModel:+ def __init__(self):+ self.calls = []+ self.log_probs = torch.tensor([[-0.5, -2.0, 0.0], [-1.0, 0.0, 0.0]], requires_grad = True)+ self.mask = torch.tensor([[True, True, False], [True, False, False]])++ def __call__(self, **kwargs):+ self.calls.append(kwargs)+ if not kwargs.get("fused_lm_head"):+ assert "labels" not in kwargs+ return SimpleNamespace(logits = torch.zeros(2, 3, 5))+ return SimpleNamespace(log_probs = self.log_probs, label_mask = self.mask, aux_loss = None)+++def _dft(lp, denom):+ return (-(lp * lp.exp())).sum().item() / denom+++def test_dft_runs_on_the_fused_head_not_the_nll_loss():+ Trainer = _trainer_class()+ model = _FusedModel()+ loss = Trainer("dft").compute_loss(model, {"input_ids": None}, num_items_in_batch = 3)+ assert len(model.calls) == 1, "dft was routed to the NLL forward, so it trains as plain NLL"+ assert model.calls[0].get("fused_lm_head") is True+ assert model.calls[0]["use_cache"] is False+ assert loss.item() == pytest.approx(_dft(model.log_probs.detach(), 3))+ loss.backward()+ # The DFT weight exp(log_p) is detached: d/dlp of -lp * sg(exp(lp)) is -exp(lp).+ expected = -model.log_probs.detach().exp() / 3+ assert torch.allclose(model.log_probs.grad, expected)+++def test_dft_eval_outputs_carry_logits():+ Trainer = _trainer_class()+ model = _FusedModel()+ loss, outputs = Trainer("dft").compute_loss(+ model, {"input_ids": None, "labels": None}, return_outputs = True, num_items_in_batch = 3+ )+ assert loss.item() == pytest.approx(_dft(model.log_probs.detach(), 3))+ assert outputs.logits.shape == (2, 3, 5)+ assert [bool(c.get("fused_lm_head")) for c in model.calls] == [True, False]+++def test_dft_refuses_a_qat_lm_head():+ Trainer = _trainer_class()+ model = _FusedModel()+ head = torch.nn.Linear(4, 5, bias = False)+ head.weight_fake_quantizer = torch.nn.Identity()+ model.get_output_embeddings = lambda: head+ with pytest.raises(NotImplementedError, match = "QAT"):+ Trainer("dft").compute_loss(model, {"input_ids": None}, num_items_in_batch = 3)+ assert model.calls == []+ model.get_output_embeddings = lambda: torch.nn.Linear(4, 5, bias = False)+ Trainer("dft").compute_loss(model, {"input_ids": None}, num_items_in_batch = 3)+ assert len(model.calls) == 1+++def test_dft_device_average_ignores_tensor_parallel_ranks():+ Trainer = _trainer_class()+ trainer = Trainer("dft")+ trainer.args.average_tokens_across_devices = True+ trainer.accelerator = SimpleNamespace(num_processes = 4)+ trainer.get_tp_size = lambda: 2+ loss = trainer.compute_loss(_FusedModel(), {"input_ids": None}, num_items_in_batch = 3)+ assert loss.item() == pytest.approx(2 * _dft(_FusedModel().log_probs.detach(), 3))+++def test_dft_without_loss_kwargs_is_a_per_micro_batch_mean():+ Trainer = _trainer_class()+ model = _FusedModel()+ loss = Trainer("dft", accepts_loss_kwargs = False).compute_loss(model, {}, num_items_in_batch = 100)+ assert loss.item() == pytest.approx(_dft(model.log_probs.detach(), 3))+++def test_nll_keeps_the_unsloth_path():+ Trainer = _trainer_class()+ model = _FusedModel()+ assert Trainer("nll").compute_loss(model, {}, num_items_in_batch = 7) == ("super", 7)+ assert model.calls == []diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex a36c9961e21..3d5f41e41d0 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -956,6 +956,70 @@ def sft_trainer_compute_loss(function_name, function):return_outputs = False,num_items_in_batch = None,):+ # TRL >= 1.15 computes loss_type="dft" only in the compute_loss this replaces (else it trains as NLL):+ # run it on TRL's fused LM head. Older TRL routes DFT through compute_loss_func.+ if (+ getattr(self.args, "loss_type", None) == "dft"+ and self.compute_loss_func is None+ and getattr(self, "label_smoother", None) is None+ ):+ import trl.trainer.utils as _trl_utils+ if hasattr(_trl_utils, "add_fused_lm_head"):+ # TRL's fused head projects with lm_head.weight directly, skipping QAT fake quantization.+ try:+ head = model.get_output_embeddings()+ except Exception:+ head = None+ try:+ from unsloth.models.llama import lm_head_needs_forward as _needs_forward+ except Exception:+ _needs_forward = lambda module: False+ if head is not None and (+ _needs_forward(head)+ or any(+ getattr(m, "activation_fake_quantizer", None) is not None+ or getattr(m, "weight_fake_quantizer", None) is not None+ for m in head.modules()+ )+ ):+ raise NotImplementedError(+ "Unsloth: loss_type = 'dft' runs on TRL's fused LM head, which skips the QAT fake "+ "quantization of lm_head. Leave lm_head out of QAT or use loss_type = 'nll'."+ )+ inputs = {**inputs, "use_cache": False}+ outputs = model(**inputs, fused_lm_head = True)+ if getattr(outputs, "log_probs", None) is None:+ raise RuntimeError(+ "Unsloth: loss_type = 'dft' needs TRL's fused LM head, but the model returned no "+ "per-token log-probabilities. Use loss_type = 'nll'."+ )+ per_token_loss = -outputs.log_probs * outputs.log_probs.exp().detach()+ # Trainer divides by gradient accumulation itself when the model does not take num_items_in_batch.+ if not getattr(self, "model_accepts_loss_kwargs", True):+ num_items_in_batch = None+ num_tokens = (+ outputs.label_mask.sum() if num_items_in_batch is None else num_items_in_batch+ )+ loss = per_token_loss.sum() / torch.as_tensor(num_tokens).clamp(min = 1)+ if self.args.average_tokens_across_devices and num_items_in_batch is not None:+ # As TRL / Trainer: tensor parallel ranks share one batch, so they do not count.+ tp_size = (+ self.get_tp_size() if callable(getattr(self, "get_tp_size", None)) else 1+ )+ loss = loss * (self.accelerator.num_processes // max(int(tp_size or 1), 1))+ aux_loss = getattr(outputs, "aux_loss", None)+ if getattr(self, "aux_loss_enabled", False) and aux_loss is not None:+ steps = (+ self.current_gradient_accumulation_steps+ if num_items_in_batch is not None+ else 1+ )+ loss = loss + self.router_aux_loss_coef * aux_loss / steps+ if return_outputs:+ # prediction_step / compute_metrics read logits, which the fused head never builds (as TRL does).+ with torch.no_grad():+ outputs = model(**{k: v for k, v in inputs.items() if k != "labels"})+ return (loss, outputs) if return_outputs else lossoutputs = super().compute_loss(model,inputs,