Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.
Python agentaichatgptdeepseekfine-tuninggemmaimage-generationllamallmllmsopenaipythonqwenreinforcement-learningself-hostedstable-diffusiontext-to-speechttsuiunsloth
DPO: UNSLOTH_FUSED_PREFERENCE_LOSS on TRL 1.x DPOTrainer._compute_loss (#15277)
Daniel Hanb6cde16723ea
diff --git a/tests/test_fused_dpo_trl1.py b/tests/test_fused_dpo_trl1.pynew file mode 100644index 00000000000..2c8567f1464--- /dev/null+++ b/tests/test_fused_dpo_trl1.py@@ -0,0 +1,388 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""UNSLOTH_FUSED_PREFERENCE_LOSS=1 on TRL >= 1.0: ``unsloth_fused_dpo_compute_loss_v1`` must+give TRL's ``DPOTrainer._compute_loss`` loss, gradients and logged metrics (float64, CPU)."""++from __future__ import annotations++import inspect+import os+import re+import types+from collections import defaultdict++import pytest+import torch++pytest.importorskip("transformers")+trl = pytest.importorskip("trl")+++def _pristine_dpo_trainer():+ """TRL's own DPOTrainer, loaded fresh from its file: tests/conftest.py imports unsloth, which+ replaces trl.DPOTrainer with the compiled UnslothDPOTrainer, and that is not the oracle here."""+ import importlib.util+ import sys++ import trl.trainer.dpo_trainer as installed++ name = "trl.trainer._pristine_dpo_trainer"+ if name not in sys.modules:+ spec = importlib.util.spec_from_file_location(name, installed.__file__)+ module = importlib.util.module_from_spec(spec)+ sys.modules[name] = module+ spec.loader.exec_module(module)+ return sys.modules[name].DPOTrainer+++DPOTrainer = _pristine_dpo_trainer()++if not hasattr(DPOTrainer, "_compute_loss") or "completion_mask" not in inspect.getsource(+ DPOTrainer._compute_loss+):+ pytest.skip("TRL < 1.0 DPOTrainer has no _compute_loss", allow_module_level = True)++from tests.test_fused_preference_loss import ( # noqa: E402+ PAD,+ VOCAB,+ HiddenStatesLlama,+ _Accelerator,+ _model,+ dpo_module, # noqa: F401 (fixture)+)+++def _rl_replacements_namespace():+ import pathlib+ import textwrap++ text = (+ pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"+ ).read_text(encoding = "utf-8")+ start = text.index("def _insert_after_docstring(")+ end = text.index('RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_fused_preference_forward)')+ namespace = {"os": os, "re": re, "textwrap": textwrap}+ exec(text[start:end], namespace)+ return namespace+++_RR = _rl_replacements_namespace()+_TRL_SOURCE = inspect.getsource(DPOTrainer._compute_loss)+# What the installed TRL's _compute_loss does; None = the hook is not installed on this TRL.+TRL_SEMANTICS = _RR["_dpo_v1_semantics"](_TRL_SOURCE)+needs_hook = pytest.mark.skipif(TRL_SEMANTICS is None, reason = "the hook does not apply to this TRL")+++def _batch(+ seed,+ n = 3,+ width = 12,+):+ gen = torch.Generator().manual_seed(seed)+ rows = 2 * n+ ids = torch.full((rows, width), PAD, dtype = torch.long)+ attention = torch.zeros((rows, width), dtype = torch.long)+ completion = torch.zeros((rows, width), dtype = torch.long)+ for i in range(rows):+ prompt = int(torch.randint(2, 5, (1,), generator = gen))+ answer = int(torch.randint(1, width - prompt + 1, (1,), generator = gen))+ ids[i, : prompt + answer] = torch.randint(1, VOCAB, (prompt + answer,), generator = gen)+ attention[i, : prompt + answer] = 1+ completion[i, prompt : prompt + answer] = 1+ return {"input_ids": ids, "attention_mask": attention, "completion_mask": completion}+++def _trainer(model, ref_model, **overrides):+ trainer = types.SimpleNamespace()+ trainer.__dict__.update(+ accelerator = _Accelerator(),+ model = model,+ ref_model = ref_model,+ precompute_ref_logps = False,+ aux_loss_enabled = False,+ use_liger_kernel = False,+ is_encoder_decoder = False,+ is_vision_model = False,+ ld_alpha = None,+ f_divergence_type = "reverse_kl",+ f_alpha_divergence_coef = 1.0,+ loss_types = ["sigmoid"],+ loss_weights = [1.0],+ beta = 0.1,+ label_smoothing = 0.0,+ use_weighting = False,+ _metrics = {"train": defaultdict(list), "eval": defaultdict(list)},+ _total_train_tokens = 0,+ )+ if "_tp_size" in _TRL_SOURCE:+ # Only releases that read it set it (1.13+); older ones would raise on it.+ trainer._tp_size = 1+ trainer.args = types.SimpleNamespace(+ gradient_checkpointing_kwargs = None, discopop_tau = 0.05, padding_free = False+ )+ for key, value in overrides.items():+ setattr(trainer, key, value)+ if "loss_weights" not in overrides:+ trainer.loss_weights = [1.0] * len(trainer.loss_types)+ return trainer+++CASES = [+ dict(),+ dict(loss_types = ["ipo"]),+ dict(loss_types = ["hinge"]),+ dict(loss_types = ["robust"], label_smoothing = 0.1),+ dict(loss_types = ["exo_pair"], label_smoothing = 0.1),+ dict(loss_types = ["nca_pair"]),+ dict(loss_types = ["bco_pair"]),+ dict(loss_types = ["sppo_hard"]),+ dict(loss_types = ["aot"]),+ dict(loss_types = ["aot_unpaired"]),+ dict(loss_types = ["apo_zero"]),+ dict(loss_types = ["apo_down"]),+ dict(loss_types = ["discopop"]),+ dict(loss_types = ["sigmoid_norm"]),+ dict(loss_types = ["sigmoid", "sft"], loss_weights = [0.7, 0.3]),+ dict(ld_alpha = 0.5),+ dict(use_weighting = True),+ dict(f_divergence_type = "forward_kl"),+ dict(f_divergence_type = "js_divergence"),+ dict(beta = 2.0),+ dict(softcap = 3.0),+ dict(bias = True),+]+++def _run(trainer, model, batch, fused, module):+ model.zero_grad(set_to_none = True)+ model.train()+ if fused:+ HiddenStatesLlama.hidden_calls = 0+ loss = module.unsloth_fused_dpo_compute_loss_v1(+ trainer, model, dict(batch), False, trl_semantics = TRL_SEMANTICS+ )+ assert loss is not None, "fused path declined a supported case"+ assert HiddenStatesLlama.hidden_calls >= 1+ else:+ loss = DPOTrainer._compute_loss(trainer, model, dict(batch), False)+ loss.backward()+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}+ metrics = {k: list(v) for k, v in trainer._metrics["train"].items()}+ return loss.detach(), metrics, grads+++def _same(+ base,+ head,+ tol = 1e-9,+):+ (loss_b, metrics_b, grads_b), (loss_h, metrics_h, grads_h) = base, head+ assert torch.isfinite(loss_b)+ torch.testing.assert_close(loss_h, loss_b, rtol = tol, atol = tol)+ assert set(metrics_b) == set(metrics_h)+ for key in metrics_b:+ assert metrics_h[key] == pytest.approx(metrics_b[key], rel = 1e-7, abs = 1e-7), key+ assert set(grads_b) == set(grads_h)+ assert any(g.abs().sum() > 0 for g in grads_b.values())+ for name in grads_b:+ torch.testing.assert_close(grads_h[name], grads_b[name], rtol = tol, atol = tol, msg = name)+++def _declined_here(case):+ # Pre-1.10 TRL: the fused path keeps forward_kl / js on TRL's own function.+ return (+ TRL_SEMANTICS["legacy_fdiv"] and case.get("f_divergence_type", "reverse_kl") != "reverse_kl"+ )+++def _loss_type_known(case):+ known = set(re.findall(r'loss_type == "([^"]+)"', _TRL_SOURCE))+ return set(case.get("loss_types", ["sigmoid"])) <= known+++@needs_hook+ "case", CASES, ids = lambda c: "-".join(f"{k}={v}" for k, v in c.items()) or "default"+)[email protected]("reference", ["ref_model", "precomputed"])+def test_compute_loss_matches_trl(dpo_module, monkeypatch, case, reference):+ case = dict(case)+ if not _loss_type_known(case):+ pytest.skip("loss type not in this TRL")+ softcap = case.pop("softcap", 0.0)+ bias = case.pop("bias", False)+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, softcap))+ batch = _batch(seed = 5)+ results = []+ for fused in (False, True):+ model = _model(0, softcap, bias)+ ref_model = _model(1, softcap, bias)+ run_batch = dict(batch)+ trainer = _trainer(model, ref_model, **case)+ if reference == "precomputed":+ with torch.no_grad():+ logits = ref_model(+ input_ids = batch["input_ids"], attention_mask = batch["attention_mask"]+ ).logits+ logps = torch.log_softmax(logits[:, :-1], -1).gather(+ -1, batch["input_ids"][:, 1:, None]+ )[..., 0]+ logps = (logps * batch["completion_mask"][:, 1:]).sum(1)+ run_batch["ref_chosen_logps"], run_batch["ref_rejected_logps"] = logps.chunk(2)+ trainer.precompute_ref_logps = True+ trainer.ref_model = None+ if fused and _declined_here(case):+ assert (+ dpo_module.unsloth_fused_dpo_compute_loss_v1(+ trainer, model, dict(run_batch), False, trl_semantics = TRL_SEMANTICS+ )+ is None+ )+ return+ results.append(_run(trainer, model, run_batch, fused, dpo_module))+ _same(*results)+++@needs_hook+def test_compute_loss_peft_reference(dpo_module, monkeypatch):+ peft = pytest.importorskip("peft")+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))+ batch = _batch(seed = 7)+ results = []+ for fused in (False, True):+ base = _model(0)+ model = peft.get_peft_model(+ base,+ peft.LoraConfig(+ r = 4, lora_alpha = 4, target_modules = ["q_proj", "v_proj"], init_lora_weights = False+ ),+ ).double()+ results.append(_run(_trainer(model, None), model, batch, fused, dpo_module))+ _same(*results)+++ "overrides, inputs_extra, return_outputs",+ [+ (dict(use_liger_kernel = True), {}, False),+ (dict(aux_loss_enabled = True), {}, False),+ (dict(is_vision_model = True), {}, False),+ (dict(f_divergence_type = "alpha_divergence"), {}, False),+ (dict(loss_types = ["some_future_loss"]), {}, False),+ (dict(), {"pixel_values": torch.zeros(1)}, False),+ (dict(), {}, True),+ ],+)+def test_compute_loss_declines(dpo_module, monkeypatch, overrides, inputs_extra, return_outputs):+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))+ model = _model(0)+ trainer = _trainer(model, _model(1), **overrides)+ batch = dict(_batch(seed = 1), **inputs_extra)+ semantics = dict(+ TRL_SEMANTICS or {"legacy_fdiv": False, "entropy_mean": False, "tp_size": True}+ )+ assert (+ dpo_module.unsloth_fused_dpo_compute_loss_v1(+ trainer, model, batch, return_outputs, trl_semantics = semantics+ )+ is None+ )+++def test_compute_loss_declines_without_semantics(dpo_module, monkeypatch):+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))+ model = _model(0)+ assert (+ dpo_module.unsloth_fused_dpo_compute_loss_v1(+ _trainer(model, _model(1)), model, _batch(1), False+ )+ is None+ )+++def test_legacy_fdiv_keeps_only_reverse_kl(dpo_module, monkeypatch):+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))+ legacy = {"legacy_fdiv": True, "entropy_mean": False, "tp_size": False}+ for f_div in ("forward_kl", "js_divergence"):+ model = _model(0)+ trainer = _trainer(model, _model(1), f_divergence_type = f_div)+ assert (+ dpo_module.unsloth_fused_dpo_compute_loss_v1(+ trainer, model, _batch(1), False, trl_semantics = legacy+ )+ is None+ )+++def test_compute_loss_flag_off(dpo_module, monkeypatch):+ monkeypatch.setenv("UNSLOTH_FUSED_PREFERENCE_LOSS", "0")+ model = _model(0)+ semantics = {"legacy_fdiv": False, "entropy_mean": False, "tp_size": True}+ assert (+ dpo_module.unsloth_fused_dpo_compute_loss_v1(+ _trainer(model, _model(1)), model, _batch(1), False, trl_semantics = semantics+ )+ is None+ )+++def test_rl_replacement_hooks_installed_trl_compute_loss(monkeypatch):+ hook = _RR["dpo_trainer_fused_preference_compute_loss"]+ source = _TRL_SOURCE+ monkeypatch.setenv("UNSLOTH_FUSED_PREFERENCE_LOSS", "0")+ assert hook("_compute_loss", source) == source+ monkeypatch.setenv("UNSLOTH_FUSED_PREFERENCE_LOSS", "1")+ patched = hook("_compute_loss", source)+ if TRL_SEMANTICS is None:+ # 0.29.x (truncates inside) and 1.15+ (TRL's own fused lm head) keep TRL's function.+ assert patched == source+ assert "self._truncate_inputs(" in source or "fused_lm_head=True" in source+ return+ assert (+ "unsloth_fused_dpo_compute_loss_v1(self, model, inputs, return_outputs, trl_semantics = "+ in patched+ )+ assert repr(TRL_SEMANTICS) in patched+ compile(__import__("textwrap").dedent(patched), "<patched>", "exec")+ # The flags follow the source: a release that reads _tp_size gets it, one that does not never does.+ assert TRL_SEMANTICS["tp_size"] == ("// self._tp_size" in source)+ # A TRL that logs a metric the replacement does not reproduce keeps its own function.+ extra = source.replace('self._metrics[mode]["entropy"]', 'self._metrics[mode]["entropy_new"]')+ assert hook("_compute_loss", extra) == extra+ assert hook("compute_loss", source) == source+ # Shapes the replacement does not reproduce keep TRL's function.+ for marker in ("self._truncate_inputs(", "fused_lm_head=True"):+ mutated = source.replace("mode = ", f"{marker}\n mode = ", 1)+ assert hook("_compute_loss", mutated) == mutated+ # A release with neither f-divergence spelling keeps TRL's function.+ no_fdiv = source.replace(+ "1 - torch.exp(-chosen_logratios)", "torch.expm1(chosen_logratios)"+ ).replace("-torch.exp(-chosen_logratios)", "torch.expm1(chosen_logratios)")+ assert hook("_compute_loss", no_fdiv) == no_fdiv+++def test_semantics_for_recorded_release_spellings():+ """The three behaviour switches, from the lines that differ between TRL 1.0 and 1.14."""+ sem = _RR["_dpo_v1_semantics"]+ base = _TRL_SOURCE if TRL_SEMANTICS is not None else None+ if base is None:+ pytest.skip("installed TRL source is not a hooked release")+ modern = (+ base.replace("-torch.exp(-chosen_logratios)", "1 - torch.exp(-chosen_logratios)")+ if TRL_SEMANTICS["legacy_fdiv"]+ else base+ )+ assert sem(modern)["legacy_fdiv"] is False+ legacy = modern.replace(+ "chosen_scores = 1 - torch.exp(-chosen_logratios)",+ "chosen_scores = -torch.exp(-chosen_logratios)",+ )+ assert sem(legacy)["legacy_fdiv"] is True+ assert (+ sem(+ modern + "\n entropy = per_token_entropy[shift_completion_mask.bool()].mean()\n"+ )["entropy_mean"]+ is True+ )+ assert sem(modern.replace("// self._tp_size", ""))["tp_size"] is Falsediff --git a/tests/test_fused_dpo_trl1_gpu.py b/tests/test_fused_dpo_trl1_gpu.pynew file mode 100644index 00000000000..4bbf4a02a33--- /dev/null+++ b/tests/test_fused_dpo_trl1_gpu.py@@ -0,0 +1,103 @@+# SPDX-License-Identifier: AGPL-3.0-only+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.++"""GPU: on TRL >= 1.0, UNSLOTH_FUSED_PREFERENCE_LOSS=1 must reach the fused DPO _compute_loss in a+real Unsloth DPOTrainer run, and the default (unset) must never reach it."""++from __future__ import annotations++import json+import os+import subprocess+import sys+import textwrap++import pytest++torch = pytest.importorskip("torch")+if not torch.cuda.is_available() or getattr(torch.cuda, "_unsloth_consolidated_spoof", False):+ pytest.skip("needs a real CUDA GPU", allow_module_level = True)+trl = pytest.importorskip("trl")++MODEL = os.environ.get("UNSLOTH_TEST_DPO_MODEL", "trl-internal-testing/tiny-Qwen3ForCausalLM")++SCRIPT = textwrap.dedent(+ """+ import json, sys+ import unsloth+ from unsloth import FastLanguageModel, PatchDPOTrainer+ import unsloth.models.dpo as dpo+ counts = {"fused": 0, "declined": 0}+ original = dpo.unsloth_fused_dpo_compute_loss_v1+ def counted(*args, **kwargs):+ out = original(*args, **kwargs)+ counts["fused" if out is not None else "declined"] += 1+ return out+ dpo.unsloth_fused_dpo_compute_loss_v1 = counted+ import torch+ from datasets import Dataset+ from trl import DPOConfig, DPOTrainer+ PatchDPOTrainer()+ model, tok = FastLanguageModel.from_pretrained(sys.argv[1], max_seq_length = 128, load_in_4bit = False)+ model = FastLanguageModel.get_peft_model(model, r = 8, lora_alpha = 8, target_modules = ["q_proj", "v_proj"])+ if tok.pad_token is None:+ tok.pad_token = tok.eos_token+ rows = [+ {"prompt": f"Question {i}: what is {i} plus {i}?", "chosen": f" It is {2 * i}.", "rejected": f" It is {2 * i + 1}."}+ for i in range(8)+ ]+ args = DPOConfig(+ output_dir = sys.argv[2], per_device_train_batch_size = 2, max_steps = 3, learning_rate = 1e-4,+ logging_steps = 1, report_to = "none", max_length = 64, beta = 0.1, save_strategy = "no",+ fp16 = not torch.cuda.is_bf16_supported(), bf16 = torch.cuda.is_bf16_supported(),+ )+ trainer = DPOTrainer(model = model, ref_model = None, args = args, train_dataset = Dataset.from_list(rows), processing_class = tok)+ trainer.train()+ losses = [h["loss"] for h in trainer.state.log_history if "loss" in h]+ print("RESULT " + json.dumps({"counts": counts, "losses": losses}))+ """+)+++def _run(tmp_path, flag):+ env = dict(os.environ)+ env.pop("UNSLOTH_FUSED_PREFERENCE_LOSS", None)+ if flag is not None:+ env["UNSLOTH_FUSED_PREFERENCE_LOSS"] = flag+ env.setdefault("UNSLOTH_COMPILE_LOCATION", str(tmp_path / f"cc_{flag}"))+ script = tmp_path / "run_dpo.py"+ script.write_text(SCRIPT, encoding = "utf-8")+ proc = subprocess.run(+ [sys.executable, str(script), MODEL, str(tmp_path / f"out_{flag}")],+ env = env,+ capture_output = True,+ text = True,+ timeout = 1200,+ )+ lines = [l for l in proc.stdout.splitlines() if l.startswith("RESULT ")]+ assert proc.returncode == 0 and lines, proc.stdout[-4000:] + proc.stderr[-4000:]+ return json.loads(lines[-1][len("RESULT ") :])+++def test_fused_dpo_v1_engages_only_when_requested(tmp_path):+ import inspect++ from trl import DPOTrainer++ source = (+ inspect.getsource(DPOTrainer._compute_loss) if hasattr(DPOTrainer, "_compute_loss") else ""+ )+ hooked_release = (+ "completion_mask" in source+ and "self._truncate_inputs(" not in source+ and "fused_lm_head=True" not in source+ )+ on = _run(tmp_path, "1")+ off = _run(tmp_path, None)+ assert off["counts"] == {"fused": 0, "declined": 0}+ if hooked_release:+ assert on["counts"]["fused"] >= 3 and on["counts"]["declined"] == 0, on+ else:+ assert on["counts"] == {"fused": 0, "declined": 0}, on+ for result in (on, off):+ assert result["losses"] and all(x == x and abs(x) < 1e4 for x in result["losses"]), resultdiff --git a/unsloth/models/dpo.py b/unsloth/models/dpo.pyindex 4be13424001..d5dec1f2666 100644--- a/unsloth/models/dpo.py+++ b/unsloth/models/dpo.py@@ -13,6 +13,7 @@ __all__ = ["PatchDPOTrainer","PatchKTOTrainer","fused_preference_loss_enabled",+ "unsloth_fused_dpo_compute_loss_v1","unsloth_fused_dpo_concatenated_forward","unsloth_fused_orpo_concatenated_forward",]@@ -512,3 +513,448 @@ def unsloth_fused_orpo_concatenated_forward(chosen_logits = mean_logits[:len_chosen]rejected_logits = mean_logits[len_chosen:]return (chosen_logps, rejected_logps, chosen_logits, rejected_logits, nll_loss)+++# TRL >= 1.0 DPOTrainer._compute_loss scores [chosen; rejected] from full batch x seq x vocab logits;+# under UNSLOTH_FUSED_PREFERENCE_LOSS=1 the replacement below reproduces it from hidden states.+_DPO_V1_INPUT_KEYS = frozenset(+ (+ "input_ids",+ "attention_mask",+ "completion_mask",+ "ref_chosen_logps",+ "ref_rejected_logps",+ )+)+_DPO_V1_LOSS_TYPES = frozenset(+ (+ "sigmoid",+ "hinge",+ "ipo",+ "exo_pair",+ "nca_pair",+ "robust",+ "bco_pair",+ "sppo_hard",+ "aot",+ "aot_unpaired",+ "apo_zero",+ "apo_down",+ "discopop",+ "sft",+ "sigmoid_norm",+ )+)+++def _metric_rows(n_rows, vocab_size):+ # One float32 logits chunk plus its log_softmax and exp: keep each near 256 MB.+ rows = (256 * 1024 * 1024) // max(3 * 4 * vocab_size, 1)+ return max(1, min(int(rows), max(n_rows, 1)))++[email protected]_grad()+def _entropy_and_argmax(hidden_or_logits, is_hidden, head, transforms, mask):+ """Per-position entropy and argmax of the (transformed) logits on ``mask`` rows only,+ one chunk of rows at a time. Positions off ``mask`` are 0 / -1."""+ import torch.nn.functional as F++ flat = hidden_or_logits.reshape(-1, hidden_or_logits.shape[-1])+ flat_mask = mask.reshape(-1).to(device = flat.device, dtype = torch.bool)+ index = flat_mask.nonzero(as_tuple = True)[0]+ # float32 at least, float64 kept (TRL computes its metrics in the logits dtype).+ stat_dtype = torch.promote_types(flat.dtype, torch.float32)+ entropy = torch.zeros(flat_mask.shape[0], dtype = stat_dtype, device = flat.device)+ argmax = torch.full((flat_mask.shape[0],), -1, dtype = torch.long, device = flat.device)+ if index.numel() == 0:+ return entropy.reshape(mask.shape), argmax.reshape(mask.shape)+ weight, bias = head+ vocab_size = weight.shape[0] if is_hidden else flat.shape[-1]+ rows = _metric_rows(index.numel(), vocab_size)+ scale_multiply, scale_divide, softcapping = transforms+ with torch.autocast(device_type = flat.device.type, enabled = False):+ for start in range(0, index.numel(), rows):+ idx = index[start : start + rows]+ if is_hidden:+ chunk = flat[idx].to(device = weight.device, dtype = weight.dtype)+ logits = F.linear(chunk, weight, bias).to(stat_dtype)+ if scale_multiply != 0.0:+ logits = logits * scale_multiply+ if scale_divide != 0.0:+ logits = logits / scale_divide+ if softcapping != 0.0:+ logits = softcapping * torch.tanh(logits / softcapping)+ else:+ logits = flat[idx].to(stat_dtype)+ logps = torch.log_softmax(logits, dim = -1)+ entropy[idx] = (-(torch.exp(logps) * logps).sum(-1)).to(entropy.device)+ argmax[idx] = logits.argmax(dim = -1).to(argmax.device)+ del logits, logps+ return entropy.reshape(mask.shape), argmax.reshape(mask.shape)+++def _dpo_v1_supported(trainer, model, inputs, return_outputs, trl_semantics):+ if not isinstance(trl_semantics, dict):+ return False+ if not fused_preference_loss_enabled() or _zoo_kernels() is None:+ return False+ if return_outputs:+ # prediction_step wants the real logits back in `outputs`.+ return False+ if getattr(trainer, "use_liger_kernel", False) or getattr(trainer, "aux_loss_enabled", False):+ return False+ if getattr(trainer, "is_encoder_decoder", False) or getattr(trainer, "is_vision_model", False):+ return False+ args = getattr(trainer, "args", None)+ if getattr(args, "padding_free", False):+ return False+ if not set(inputs).issubset(_DPO_V1_INPUT_KEYS):+ return False+ if "input_ids" not in inputs or "completion_mask" not in inputs:+ return False+ # alpha_divergence clamps by the logps dtype, which is float32 here and half in TRL.+ f_divergence_types = ("reverse_kl", "forward_kl", "js_divergence")+ if trl_semantics.get("legacy_fdiv", True):+ # Before TRL 1.10 these scores drop their constants and replace the log ratios in some+ # losses; only reverse_kl, where the score is the log ratio, is the same everywhere.+ f_divergence_types = ("reverse_kl",)+ if getattr(trainer, "f_divergence_type", None) not in f_divergence_types:+ return False+ loss_types = getattr(trainer, "loss_types", None)+ loss_weights = getattr(trainer, "loss_weights", None)+ if not loss_types or loss_weights is None or len(loss_types) != len(loss_weights):+ return False+ if not set(loss_types).issubset(_DPO_V1_LOSS_TYPES):+ return False+ return True+++def _dpo_v1_heads(trainer, model):+ """(unwrapped, head, transforms) for the policy, and the same for a separate reference model+ (``None`` when the reference is the policy with its adapter disabled or precomputed)."""+ unwrapped, head = _output_head(trainer, model)+ if head is None:+ return None+ transforms = _logit_transforms(unwrapped)+ if transforms is None:+ return None+ ref = None+ if not trainer.precompute_ref_logps:+ try:+ from accelerate.utils import is_peft_model+ except Exception:+ return None+ if not (is_peft_model(model) and trainer.ref_model is None):+ if trainer.ref_model is None:+ return None+ ref_unwrapped, ref_head = _output_head(trainer, trainer.ref_model)+ if ref_head is None:+ return None+ ref_transforms = _logit_transforms(ref_unwrapped)+ if ref_transforms is None:+ return None+ ref = (ref_unwrapped, ref_head, ref_transforms)+ return (unwrapped, head, transforms), ref+++def unsloth_fused_dpo_compute_loss_v1(+ trainer,+ model,+ inputs,+ return_outputs = False,+ trl_semantics = None,+):+ """TRL 1.x ``DPOTrainer._compute_loss`` for text-only decoder models, scored from hidden+ states in row chunks. Returns ``None`` (run TRL's own) whenever the inputs leave that case.++ ``trl_semantics`` is what rl_replacements read from the installed TRL's source: legacy_fdiv+ (pre-1.10 f-divergence scores), entropy_mean (pre-1.3 entropy metric) and tp_size (1.13++ divides the token count by the tensor-parallel size)."""+ if not _dpo_v1_supported(trainer, model, inputs, return_outputs, trl_semantics):+ return None+ heads = _dpo_v1_heads(trainer, model)+ if heads is None:+ return None+ (unwrapped, head, transforms), ref = heads++ import math+ import torch.nn.functional as F+ from trl.trainer.dpo_trainer import disable_gradient_checkpointing, use_adapter+ from accelerate.utils import is_peft_model++ self = trainer+ mode = "train" if self.model.training else "eval"+ device = self.accelerator.device++ _non_model_keys = {"completion_mask", "ref_chosen_logps", "ref_rejected_logps"}+ model_kwargs = {k: v for k, v in inputs.items() if k not in _non_model_keys}+ model_kwargs["use_cache"] = False++ input_ids = inputs["input_ids"]+ completion_mask = inputs["completion_mask"]+ shift_labels = input_ids[..., 1:]+ shift_completion_mask = completion_mask[..., 1:]+ score_mask = shift_completion_mask.bool()+ use_weighting = bool(getattr(self, "use_weighting", False))++ hidden, is_hidden = _forward_hidden_states(model, unwrapped, head[0], (), model_kwargs)+ shift_hidden = hidden[..., :-1, :]+ per_token_logps, mean_logits, log_sum_sq_probs = _token_logps(+ shift_hidden,+ is_hidden,+ head,+ transforms,+ shift_labels,+ score_mask,+ want_lssp = use_weighting,+ )+ per_token_entropy, predictions = _entropy_and_argmax(+ shift_hidden.detach(), is_hidden, head, transforms, score_mask+ )+ del hidden, shift_hidden++ if self.ld_alpha is None:+ logps = per_token_logps.sum(dim = 1)+ else:+ comp_pos = shift_completion_mask.cumsum(dim = 1)+ comp_lens = shift_completion_mask.sum(dim = 1).long()+ chosen_lens, rejected_lens = comp_lens.chunk(2, dim = 0)+ shared_lens = torch.minimum(chosen_lens, rejected_lens)+ shared_lens = torch.cat([shared_lens, shared_lens], dim = 0).to(device)+ shared_mask = (comp_pos > 0) & (comp_pos <= shared_lens.unsqueeze(1))+ tail_mask = comp_pos > shared_lens.unsqueeze(1)+ shared_logps = (per_token_logps * shared_mask).sum(dim = 1)+ tail_logps = (per_token_logps * tail_mask).sum(dim = 1)+ logps = shared_logps + self.ld_alpha * tail_logps+ chosen_logps, rejected_logps = logps.chunk(2, dim = 0)++ if self.precompute_ref_logps:+ ref_chosen_logps, ref_rejected_logps = (+ inputs["ref_chosen_logps"],+ inputs["ref_rejected_logps"],+ )+ else:+ ref_model_kwargs = dict(model_kwargs)+ with (+ torch.no_grad(),+ disable_gradient_checkpointing(self.model, self.args.gradient_checkpointing_kwargs),+ ):+ if ref is None:+ peft_model = self.accelerator.unwrap_model(model)+ with use_adapter(+ peft_model, adapter_name = "ref" if "ref" in peft_model.peft_config else None+ ):+ ref_hidden, ref_is_hidden = _forward_hidden_states(+ self.model, unwrapped, head[0], (), ref_model_kwargs+ )+ ref_head, ref_transforms = head, transforms+ else:+ ref_unwrapped, ref_head, ref_transforms = ref+ ref_hidden, ref_is_hidden = _forward_hidden_states(+ self.ref_model, ref_unwrapped, ref_head[0], (), ref_model_kwargs+ )+ ref_per_token_logps, _, _ = _token_logps(+ ref_hidden[..., :-1, :],+ ref_is_hidden,+ ref_head,+ ref_transforms,+ shift_labels,+ score_mask,+ )+ del ref_hidden+ if self.ld_alpha is None:+ ref_logps = ref_per_token_logps.sum(dim = 1)+ else:+ ref_shared_logps = (ref_per_token_logps * shared_mask).sum(dim = 1)+ ref_tail_logps = (ref_per_token_logps * tail_mask).sum(dim = 1)+ ref_logps = ref_shared_logps + self.ld_alpha * ref_tail_logps+ ref_chosen_logps, ref_rejected_logps = ref_logps.chunk(2, dim = 0)++ chosen_logratios = chosen_logps - ref_chosen_logps+ rejected_logratios = rejected_logps - ref_rejected_logps++ if self.f_divergence_type == "reverse_kl":+ chosen_scores = chosen_logratios+ rejected_scores = rejected_logratios+ elif self.f_divergence_type == "forward_kl":+ chosen_scores = 1 - torch.exp(-chosen_logratios)+ rejected_scores = 1 - torch.exp(-rejected_logratios)+ else: # js_divergence+ chosen_scores = math.log(2) + F.logsigmoid(chosen_logratios)+ rejected_scores = math.log(2) + F.logsigmoid(rejected_logratios)++ delta_score = chosen_scores - rejected_scores++ loss = 0.0+ for loss_type, loss_weight in zip(self.loss_types, self.loss_weights):+ if loss_type == "sigmoid":+ per_sequence_loss = -F.logsigmoid(self.beta * delta_score)+ elif loss_type == "hinge":+ per_sequence_loss = torch.relu(1 - self.beta * delta_score)+ elif loss_type == "ipo":+ chosen_mask, rejected_mask = completion_mask.chunk(2, dim = 0)+ chosen_avg_score = chosen_scores / chosen_mask.sum(dim = 1).clamp(min = 1.0)+ rejected_avg_score = rejected_scores / rejected_mask.sum(dim = 1).clamp(min = 1.0)+ ipo_delta = chosen_avg_score - rejected_avg_score+ per_sequence_loss = (ipo_delta - 1 / (2 * self.beta)) ** 2+ elif loss_type == "exo_pair":+ epsilon = torch.tensor(self.label_smoothing, device = device)+ qw = torch.sigmoid(self.beta * delta_score)+ log_qw = F.logsigmoid(self.beta * delta_score)+ log_pw = torch.log1p(-epsilon)+ ql = torch.sigmoid(-self.beta * delta_score)+ log_ql = F.logsigmoid(-self.beta * delta_score)+ log_pl = torch.log(epsilon)+ per_sequence_loss = qw * (log_qw - log_pw) + ql * (log_ql - log_pl)+ elif loss_type == "nca_pair":+ chosen_rewards = self.beta * chosen_logratios+ rejected_rewards = self.beta * rejected_logratios+ per_sequence_loss = (+ -F.logsigmoid(chosen_rewards)+ - 0.5 * F.logsigmoid(-chosen_rewards)+ - 0.5 * F.logsigmoid(-rejected_rewards)+ )+ elif loss_type == "robust":+ clean_loss_term = -(1 - self.label_smoothing) * F.logsigmoid(self.beta * delta_score)+ flipped_loss_term = -self.label_smoothing * F.logsigmoid(-self.beta * delta_score)+ per_sequence_loss = (clean_loss_term - flipped_loss_term) / (+ 1 - 2 * self.label_smoothing+ )+ elif loss_type == "bco_pair":+ chosen_rewards = self.beta * chosen_logratios+ rejected_rewards = self.beta * rejected_logratios+ per_sequence_loss = -F.logsigmoid(chosen_rewards) - F.logsigmoid(-rejected_rewards)+ elif loss_type == "sppo_hard":+ winner_margin_error = (chosen_logratios - 0.5 / self.beta) ** 2+ loser_margin_error = (rejected_logratios + 0.5 / self.beta) ** 2+ per_sequence_loss = winner_margin_error + loser_margin_error+ elif loss_type == "aot":+ logratios = chosen_logps - rejected_logps+ ref_logratios = ref_chosen_logps - ref_rejected_logps+ logratios_sorted, _ = torch.sort(logratios, dim = 0)+ ref_logratios_sorted, _ = torch.sort(ref_logratios, dim = 0)+ delta = logratios_sorted - ref_logratios_sorted+ per_sequence_loss = (+ -F.logsigmoid(self.beta * delta) * (1 - self.label_smoothing)+ - F.logsigmoid(-self.beta * delta) * self.label_smoothing+ )+ elif loss_type == "aot_unpaired":+ chosen_logratios_sorted, _ = torch.sort(chosen_logratios, dim = 0)+ rejected_logratios_sorted, _ = torch.sort(rejected_logratios, dim = 0)+ delta = chosen_logratios_sorted - rejected_logratios_sorted+ per_sequence_loss = (+ -F.logsigmoid(self.beta * delta) * (1 - self.label_smoothing)+ - F.logsigmoid(-self.beta * delta) * self.label_smoothing+ )+ elif loss_type == "apo_zero":+ losses_chosen = 1 - torch.sigmoid(self.beta * chosen_logratios)+ losses_rejected = torch.sigmoid(self.beta * rejected_logratios)+ per_sequence_loss = losses_chosen + losses_rejected+ elif loss_type == "apo_down":+ losses_chosen = torch.sigmoid(self.beta * chosen_logratios)+ losses_rejected = 1 - torch.sigmoid(self.beta * (chosen_logratios - rejected_logratios))+ per_sequence_loss = losses_chosen + losses_rejected+ elif loss_type == "discopop":+ logits = delta_score * self.beta+ log_ratio_modulation = torch.sigmoid(logits / self.args.discopop_tau)+ logistic_component = -F.logsigmoid(logits)+ exp_component = torch.exp(-logits)+ per_sequence_loss = (+ logistic_component * (1 - log_ratio_modulation)+ + exp_component * log_ratio_modulation+ )+ elif loss_type == "sft":+ # F.cross_entropy over the chosen completion tokens == -mean of their logps.+ chosen_token_logps, _ = per_token_logps.chunk(2, dim = 0)+ chosen_mask, _ = score_mask.chunk(2, dim = 0)+ batch_loss = -chosen_token_logps[chosen_mask].mean()+ per_sequence_loss = batch_loss.expand(chosen_token_logps.size(0))+ else: # sigmoid_norm+ chosen_mask, rejected_mask = completion_mask.chunk(2, dim = 0)+ chosen_avg_score = chosen_scores / chosen_mask.sum(dim = 1).clamp(min = 1.0)+ rejected_avg_score = rejected_scores / rejected_mask.sum(dim = 1).clamp(min = 1.0)+ delta = chosen_avg_score - rejected_avg_score+ per_sequence_loss = -F.logsigmoid(self.beta * delta)++ if use_weighting:+ completion_lengths = shift_completion_mask.sum(dim = 1).clamp_min(1)+ with torch.no_grad():+ # log_sum_sq_probs = logsumexp(2 l) - 2 logsumexp(l), TRL's log_denom.+ aligned_logps = (per_token_logps - log_sum_sq_probs) * shift_completion_mask+ mean_logps = aligned_logps.sum(dim = 1) / completion_lengths+ weights = torch.exp(mean_logps)+ chosen_weights, rejected_weights = weights.chunk(2, dim = 0)+ per_sequence_loss *= chosen_weights * rejected_weights++ loss += per_sequence_loss.mean() * loss_weight++ # Metrics, as TRL logs them, from the chunked statistics.+ mask = shift_completion_mask+ if trl_semantics.get("entropy_mean", False):+ entropy = per_token_entropy[mask.bool()].mean()+ entropy = self.accelerator.gather_for_metrics(entropy).mean().item()+ else:+ entropy_sum = (per_token_entropy * mask).sum()+ total_tokens = mask.sum()+ entropy_sum = self.accelerator.gather_for_metrics(entropy_sum).sum()+ total_tokens = self.accelerator.gather_for_metrics(total_tokens).sum()+ entropy = (entropy_sum / total_tokens).item() if total_tokens > 0 else 0.0+ self._metrics[mode]["entropy"].append(entropy)++ if mode == "train":+ num_tokens_in_batch = (+ self.accelerator.gather_for_metrics(inputs["attention_mask"].sum()).sum().item()+ )+ if trl_semantics.get("tp_size", False):+ num_tokens_in_batch = num_tokens_in_batch // self._tp_size+ self._total_train_tokens += num_tokens_in_batch+ self._metrics[mode]["num_tokens"] = [self._total_train_tokens]++ chosen_means, rejected_means = mean_logits.detach().chunk(2, dim = 0)+ chosen_mask, rejected_mask = shift_completion_mask.chunk(2, dim = 0)+ total_chosen_logits = chosen_means[chosen_mask.bool()].sum()+ total_chosen_tokens = chosen_mask.sum()+ total_rejected_logits = rejected_means[rejected_mask.bool()].sum()+ total_rejected_tokens = rejected_mask.sum()+ total_chosen_logits = self.accelerator.gather_for_metrics(total_chosen_logits).sum().item()+ total_chosen_tokens = self.accelerator.gather_for_metrics(total_chosen_tokens).sum().item()+ total_rejected_logits = self.accelerator.gather_for_metrics(total_rejected_logits).sum().item()+ total_rejected_tokens = self.accelerator.gather_for_metrics(total_rejected_tokens).sum().item()+ avg_chosen_logits = (+ total_chosen_logits / total_chosen_tokens if total_chosen_tokens > 0 else 0.0+ )+ avg_rejected_logits = (+ total_rejected_logits / total_rejected_tokens if total_rejected_tokens > 0 else 0.0+ )+ self._metrics[mode]["logits/chosen"].append(avg_chosen_logits)+ self._metrics[mode]["logits/rejected"].append(avg_rejected_logits)++ half = len(shift_completion_mask) // 2+ chosen_bool = shift_completion_mask[:half].bool()+ chosen_labels = shift_labels[:half]+ correct_predictions = (predictions[:half] == chosen_labels) & chosen_bool+ total_tokens = chosen_bool.sum()+ correct_tokens = correct_predictions.sum()+ correct_tokens = self.accelerator.gather_for_metrics(correct_tokens)+ total_tokens = self.accelerator.gather_for_metrics(total_tokens)+ total_sum = total_tokens.sum()+ accuracy = (correct_tokens.sum() / total_sum).item() if total_sum > 0 else 0.0+ self._metrics[mode]["mean_token_accuracy"].append(accuracy)++ chosen_rewards = self.beta * chosen_logratios.detach()+ rejected_rewards = self.beta * rejected_logratios.detach()+ agg_chosen_rewards = self.accelerator.gather(chosen_rewards)+ agg_rejected_rewards = self.accelerator.gather(rejected_rewards)+ self._metrics[mode]["rewards/chosen"].append(agg_chosen_rewards.mean().item())+ self._metrics[mode]["rewards/rejected"].append(agg_rejected_rewards.mean().item())+ reward_accuracies = (chosen_rewards > rejected_rewards).float()+ agg_reward_accuracies = self.accelerator.gather(reward_accuracies)+ self._metrics[mode]["rewards/accuracies"].append(agg_reward_accuracies.mean().item())+ margins = chosen_rewards - rejected_rewards+ agg_margins = self.accelerator.gather(margins)+ self._metrics[mode]["rewards/margins"].append(agg_margins.mean().item())+ self._metrics[mode]["logps/chosen"].append(self.accelerator.gather(chosen_logps).mean().item())+ self._metrics[mode]["logps/rejected"].append(+ self.accelerator.gather(rejected_logps).mean().item()+ )+ return lossdiff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.pyindex 8c841558222..54f2032995b 100644--- a/unsloth/models/rl_replacements.py+++ b/unsloth/models/rl_replacements.py@@ -756,7 +756,101 @@ def cpo_trainer_fused_preference_forward(function_name, function):)+# The replacement reproduces exactly these metric keys and logits reads; any other source keeps TRL's+# function (0.29.x truncates inside it, 1.15+ already uses TRL's fused lm head).+_DPO_V1_METRIC_KEYS = frozenset(+ (+ "aux_loss",+ "entropy",+ "num_tokens",+ "logits/chosen",+ "logits/rejected",+ "mean_token_accuracy",+ "rewards/chosen",+ "rewards/rejected",+ "rewards/accuracies",+ "rewards/margins",+ "logps/chosen",+ "logps/rejected",+ )+)+# Each entry: the spellings (whitespace-insensitive) one of which must appear.+_DPO_V1_SOURCE_MARKERS = (+ (+ "per_token_logps = selective_log_softmax(shift_logits, shift_labels)",+ "per_token_logps, per_token_entropies = selective_log_softmax_and_entropy( shift_logits, shift_labels,",+ ),+ (+ "ref_per_token_logps = selective_log_softmax(ref_shift_logits, shift_labels)",+ "ref_per_token_logps = selective_log_softmax( ref_shift_logits, shift_labels, row_mask=shift_completion_mask )",+ ),+ (+ "per_token_entropy = entropy_from_logits(shift_logits.detach())",+ "per_token_entropy = per_token_entropies",+ ),+ ("predictions = chosen_logits.argmax(dim=-1)",),+)+# Sources the replacement does not reproduce.+_DPO_V1_DECLINE_MARKERS = ("self._truncate_inputs(", "fused_lm_head=True")+# Releases before 1.10 drop the constants of the forward_kl / js f-divergence scores and feed+# those scores (not the log ratios) to nca_pair / bco_pair / sppo_hard / apo_down.+_DPO_V1_FDIV_SCORES_MARKER = "chosen_scores = 1 - torch.exp(-chosen_logratios)"+_DPO_V1_FDIV_LEGACY_MARKER = "chosen_scores = -torch.exp(-chosen_logratios)"+# Releases before 1.3 log the entropy as a mean of the masked per-token entropies.+_DPO_V1_ENTROPY_MEAN_MARKER = "entropy = per_token_entropy[shift_completion_mask.bool()].mean()"+++def _dpo_v1_semantics(function):+ """The TRL-release differences the fused _compute_loss must follow, read from its source, or+ None when the source is not one the replacement reproduces."""+ flat = " ".join(function.split())+ if any(marker in flat for marker in _DPO_V1_DECLINE_MARKERS):+ return None+ for spellings in _DPO_V1_SOURCE_MARKERS:+ if not any(" ".join(spelling.split()) in flat for spelling in spellings):+ return None+ if _DPO_V1_FDIV_SCORES_MARKER in flat:+ legacy_fdiv = False+ elif _DPO_V1_FDIV_LEGACY_MARKER in flat:+ legacy_fdiv = True+ else:+ return None+ return {+ "legacy_fdiv": legacy_fdiv,+ "entropy_mean": _DPO_V1_ENTROPY_MEAN_MARKER in flat,+ "tp_size": "// self._tp_size" in flat,+ }+++def dpo_trainer_fused_preference_compute_loss(function_name, function):+ if function_name != "_compute_loss":+ return function+ if os.environ.get("UNSLOTH_FUSED_PREFERENCE_LOSS", "0") != "1":+ return function+ if "_unsloth_fused_output" in function:+ return function+ if not re.search(r"def _compute_loss\(self, model, inputs, return_outputs\)", function):+ return function+ semantics = _dpo_v1_semantics(function)+ if semantics is None:+ return function+ keys = set(re.findall(r'self\._metrics\[mode\]\["([^"]+)"\]', function))+ if not keys or not keys.issubset(_DPO_V1_METRIC_KEYS):+ return function+ return _insert_after_docstring(+ function,+ [+ "from unsloth.models.dpo import unsloth_fused_dpo_compute_loss_v1",+ "_unsloth_fused_output = unsloth_fused_dpo_compute_loss_v1("+ f"self, model, inputs, return_outputs, trl_semantics = {semantics!r})",+ "if _unsloth_fused_output is not None:",+ " return _unsloth_fused_output",+ ],+ )++RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_fused_preference_forward)+RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_fused_preference_compute_loss)RL_FUNCTIONS["orpo_trainer"].append(orpo_trainer_fused_preference_forward)RL_FUNCTIONS["cpo_trainer"].append(cpo_trainer_fused_preference_forward)