Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

SFT: compute loss_type="dft" on TRL's fused LM head instead of silently training NLL (#15213)

Daniel Han2c81e9f9f0c0
diff --git a/tests/test_sft_dft_loss_routing.py b/tests/test_sft_dft_loss_routing.py
new file mode 100644
index 00000000000..6efb2fb3f1a
--- /dev/null
+++ b/tests/test_sft_dft_loss_routing.py
@@ -0,0 +1,147 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""SFT loss_type="dft" must not silently train as NLL under the Unsloth compute_loss; ast-loaded, CPU-only."""
+
+from __future__ import annotations
+
+import ast
+import importlib.util
+import inspect
+import textwrap
+from pathlib import Path
+from types import SimpleNamespace
+
+import pytest
+import torch
+
+SOURCE_PATH = Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"
+
+
+def _trainer_class():
+ if importlib.util.find_spec("trl") is None:
+ pytest.skip("trl not installed")
+ import trl.trainer.utils as trl_utils
+
+ if not hasattr(trl_utils, "add_fused_lm_head"):
+ pytest.skip("TRL < 1.15 computes dft through compute_loss_func")
+ tree = ast.parse(SOURCE_PATH.read_text(encoding = "utf-8"))
+ nodes = [
+ n
+ for n in tree.body
+ if isinstance(n, ast.FunctionDef) and n.name == "sft_trainer_compute_loss"
+ ]
+ namespace = {"inspect": inspect, "RL_FUNCTIONS": None}
+ exec(compile(ast.Module(body = nodes, type_ignores = []), str(SOURCE_PATH), "exec"), namespace)
+ source = namespace["sft_trainer_compute_loss"](
+ "compute_loss", "def compute_loss(self):\n pass\n"
+ )
+
+ class Base:
+ def compute_loss(
+ self,
+ model,
+ inputs,
+ return_outputs = False,
+ num_items_in_batch = None,
+ ):
+ return ("super", num_items_in_batch)
+
+ # Compiled into a class body, as in the generated trainer, so the zero-argument super() resolves.
+ generated = {"torch": torch, "Base": Base}
+ exec("class Generated(Base):\n" + textwrap.indent(textwrap.dedent(source), " "), generated)
+
+ class Trainer(generated["Generated"]):
+ def __init__(
+ self,
+ loss_type,
+ accepts_loss_kwargs = True,
+ ):
+ self.args = SimpleNamespace(loss_type = loss_type, average_tokens_across_devices = False)
+ self.compute_loss_func = None
+ self.label_smoother = None
+ self.model_accepts_loss_kwargs = accepts_loss_kwargs
+ self.aux_loss_enabled = False
+
+ return Trainer
+
+
+class _FusedModel:
+ def __init__(self):
+ self.calls = []
+ self.log_probs = torch.tensor([[-0.5, -2.0, 0.0], [-1.0, 0.0, 0.0]], requires_grad = True)
+ self.mask = torch.tensor([[True, True, False], [True, False, False]])
+
+ def __call__(self, **kwargs):
+ self.calls.append(kwargs)
+ if not kwargs.get("fused_lm_head"):
+ assert "labels" not in kwargs
+ return SimpleNamespace(logits = torch.zeros(2, 3, 5))
+ return SimpleNamespace(log_probs = self.log_probs, label_mask = self.mask, aux_loss = None)
+
+
+def _dft(lp, denom):
+ return (-(lp * lp.exp())).sum().item() / denom
+
+
+def test_dft_runs_on_the_fused_head_not_the_nll_loss():
+ Trainer = _trainer_class()
+ model = _FusedModel()
+ loss = Trainer("dft").compute_loss(model, {"input_ids": None}, num_items_in_batch = 3)
+ assert len(model.calls) == 1, "dft was routed to the NLL forward, so it trains as plain NLL"
+ assert model.calls[0].get("fused_lm_head") is True
+ assert model.calls[0]["use_cache"] is False
+ assert loss.item() == pytest.approx(_dft(model.log_probs.detach(), 3))
+ loss.backward()
+ # The DFT weight exp(log_p) is detached: d/dlp of -lp * sg(exp(lp)) is -exp(lp).
+ expected = -model.log_probs.detach().exp() / 3
+ assert torch.allclose(model.log_probs.grad, expected)
+
+
+def test_dft_eval_outputs_carry_logits():
+ Trainer = _trainer_class()
+ model = _FusedModel()
+ loss, outputs = Trainer("dft").compute_loss(
+ model, {"input_ids": None, "labels": None}, return_outputs = True, num_items_in_batch = 3
+ )
+ assert loss.item() == pytest.approx(_dft(model.log_probs.detach(), 3))
+ assert outputs.logits.shape == (2, 3, 5)
+ assert [bool(c.get("fused_lm_head")) for c in model.calls] == [True, False]
+
+
+def test_dft_refuses_a_qat_lm_head():
+ Trainer = _trainer_class()
+ model = _FusedModel()
+ head = torch.nn.Linear(4, 5, bias = False)
+ head.weight_fake_quantizer = torch.nn.Identity()
+ model.get_output_embeddings = lambda: head
+ with pytest.raises(NotImplementedError, match = "QAT"):
+ Trainer("dft").compute_loss(model, {"input_ids": None}, num_items_in_batch = 3)
+ assert model.calls == []
+ model.get_output_embeddings = lambda: torch.nn.Linear(4, 5, bias = False)
+ Trainer("dft").compute_loss(model, {"input_ids": None}, num_items_in_batch = 3)
+ assert len(model.calls) == 1
+
+
+def test_dft_device_average_ignores_tensor_parallel_ranks():
+ Trainer = _trainer_class()
+ trainer = Trainer("dft")
+ trainer.args.average_tokens_across_devices = True
+ trainer.accelerator = SimpleNamespace(num_processes = 4)
+ trainer.get_tp_size = lambda: 2
+ loss = trainer.compute_loss(_FusedModel(), {"input_ids": None}, num_items_in_batch = 3)
+ assert loss.item() == pytest.approx(2 * _dft(_FusedModel().log_probs.detach(), 3))
+
+
+def test_dft_without_loss_kwargs_is_a_per_micro_batch_mean():
+ Trainer = _trainer_class()
+ model = _FusedModel()
+ loss = Trainer("dft", accepts_loss_kwargs = False).compute_loss(model, {}, num_items_in_batch = 100)
+ assert loss.item() == pytest.approx(_dft(model.log_probs.detach(), 3))
+
+
+def test_nll_keeps_the_unsloth_path():
+ Trainer = _trainer_class()
+ model = _FusedModel()
+ assert Trainer("nll").compute_loss(model, {}, num_items_in_batch = 7) == ("super", 7)
+ assert model.calls == []
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index a36c9961e21..3d5f41e41d0 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -956,6 +956,70 @@ def sft_trainer_compute_loss(function_name, function):
return_outputs = False,
num_items_in_batch = None,
):
+ # TRL >= 1.15 computes loss_type="dft" only in the compute_loss this replaces (else it trains as NLL):
+ # run it on TRL's fused LM head. Older TRL routes DFT through compute_loss_func.
+ if (
+ getattr(self.args, "loss_type", None) == "dft"
+ and self.compute_loss_func is None
+ and getattr(self, "label_smoother", None) is None
+ ):
+ import trl.trainer.utils as _trl_utils
+ if hasattr(_trl_utils, "add_fused_lm_head"):
+ # TRL's fused head projects with lm_head.weight directly, skipping QAT fake quantization.
+ try:
+ head = model.get_output_embeddings()
+ except Exception:
+ head = None
+ try:
+ from unsloth.models.llama import lm_head_needs_forward as _needs_forward
+ except Exception:
+ _needs_forward = lambda module: False
+ if head is not None and (
+ _needs_forward(head)
+ or any(
+ getattr(m, "activation_fake_quantizer", None) is not None
+ or getattr(m, "weight_fake_quantizer", None) is not None
+ for m in head.modules()
+ )
+ ):
+ raise NotImplementedError(
+ "Unsloth: loss_type = 'dft' runs on TRL's fused LM head, which skips the QAT fake "
+ "quantization of lm_head. Leave lm_head out of QAT or use loss_type = 'nll'."
+ )
+ inputs = {**inputs, "use_cache": False}
+ outputs = model(**inputs, fused_lm_head = True)
+ if getattr(outputs, "log_probs", None) is None:
+ raise RuntimeError(
+ "Unsloth: loss_type = 'dft' needs TRL's fused LM head, but the model returned no "
+ "per-token log-probabilities. Use loss_type = 'nll'."
+ )
+ per_token_loss = -outputs.log_probs * outputs.log_probs.exp().detach()
+ # Trainer divides by gradient accumulation itself when the model does not take num_items_in_batch.
+ if not getattr(self, "model_accepts_loss_kwargs", True):
+ num_items_in_batch = None
+ num_tokens = (
+ outputs.label_mask.sum() if num_items_in_batch is None else num_items_in_batch
+ )
+ loss = per_token_loss.sum() / torch.as_tensor(num_tokens).clamp(min = 1)
+ if self.args.average_tokens_across_devices and num_items_in_batch is not None:
+ # As TRL / Trainer: tensor parallel ranks share one batch, so they do not count.
+ tp_size = (
+ self.get_tp_size() if callable(getattr(self, "get_tp_size", None)) else 1
+ )
+ loss = loss * (self.accelerator.num_processes // max(int(tp_size or 1), 1))
+ aux_loss = getattr(outputs, "aux_loss", None)
+ if getattr(self, "aux_loss_enabled", False) and aux_loss is not None:
+ steps = (
+ self.current_gradient_accumulation_steps
+ if num_items_in_batch is not None
+ else 1
+ )
+ loss = loss + self.router_aux_loss_coef * aux_loss / steps
+ if return_outputs:
+ # prediction_step / compute_metrics read logits, which the fused head never builds (as TRL does).
+ with torch.no_grad():
+ outputs = model(**{k: v for k, v in inputs.items() if k != "labels"})
+ return (loss, outputs) if return_outputs else loss
outputs = super().compute_loss(
model,
inputs,