Skip to content

Local UI to run and train LLMs and diffusion models. Supports GGUF, MLX, Qwen3.8, DeepSeek-V4.1, MiniMax-H3, Gemma 4, FLUX and more.

DPO: UNSLOTH_FUSED_PREFERENCE_LOSS on TRL 1.x DPOTrainer._compute_loss (#15277)

Daniel Hanb6cde16723ea
diff --git a/tests/test_fused_dpo_trl1.py b/tests/test_fused_dpo_trl1.py
new file mode 100644
index 00000000000..2c8567f1464
--- /dev/null
+++ b/tests/test_fused_dpo_trl1.py
@@ -0,0 +1,388 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""UNSLOTH_FUSED_PREFERENCE_LOSS=1 on TRL >= 1.0: ``unsloth_fused_dpo_compute_loss_v1`` must
+give TRL's ``DPOTrainer._compute_loss`` loss, gradients and logged metrics (float64, CPU)."""
+
+from __future__ import annotations
+
+import inspect
+import os
+import re
+import types
+from collections import defaultdict
+
+import pytest
+import torch
+
+pytest.importorskip("transformers")
+trl = pytest.importorskip("trl")
+
+
+def _pristine_dpo_trainer():
+ """TRL's own DPOTrainer, loaded fresh from its file: tests/conftest.py imports unsloth, which
+ replaces trl.DPOTrainer with the compiled UnslothDPOTrainer, and that is not the oracle here."""
+ import importlib.util
+ import sys
+
+ import trl.trainer.dpo_trainer as installed
+
+ name = "trl.trainer._pristine_dpo_trainer"
+ if name not in sys.modules:
+ spec = importlib.util.spec_from_file_location(name, installed.__file__)
+ module = importlib.util.module_from_spec(spec)
+ sys.modules[name] = module
+ spec.loader.exec_module(module)
+ return sys.modules[name].DPOTrainer
+
+
+DPOTrainer = _pristine_dpo_trainer()
+
+if not hasattr(DPOTrainer, "_compute_loss") or "completion_mask" not in inspect.getsource(
+ DPOTrainer._compute_loss
+):
+ pytest.skip("TRL < 1.0 DPOTrainer has no _compute_loss", allow_module_level = True)
+
+from tests.test_fused_preference_loss import ( # noqa: E402
+ PAD,
+ VOCAB,
+ HiddenStatesLlama,
+ _Accelerator,
+ _model,
+ dpo_module, # noqa: F401 (fixture)
+)
+
+
+def _rl_replacements_namespace():
+ import pathlib
+ import textwrap
+
+ text = (
+ pathlib.Path(__file__).resolve().parents[1] / "unsloth" / "models" / "rl_replacements.py"
+ ).read_text(encoding = "utf-8")
+ start = text.index("def _insert_after_docstring(")
+ end = text.index('RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_fused_preference_forward)')
+ namespace = {"os": os, "re": re, "textwrap": textwrap}
+ exec(text[start:end], namespace)
+ return namespace
+
+
+_RR = _rl_replacements_namespace()
+_TRL_SOURCE = inspect.getsource(DPOTrainer._compute_loss)
+# What the installed TRL's _compute_loss does; None = the hook is not installed on this TRL.
+TRL_SEMANTICS = _RR["_dpo_v1_semantics"](_TRL_SOURCE)
+needs_hook = pytest.mark.skipif(TRL_SEMANTICS is None, reason = "the hook does not apply to this TRL")
+
+
+def _batch(
+ seed,
+ n = 3,
+ width = 12,
+):
+ gen = torch.Generator().manual_seed(seed)
+ rows = 2 * n
+ ids = torch.full((rows, width), PAD, dtype = torch.long)
+ attention = torch.zeros((rows, width), dtype = torch.long)
+ completion = torch.zeros((rows, width), dtype = torch.long)
+ for i in range(rows):
+ prompt = int(torch.randint(2, 5, (1,), generator = gen))
+ answer = int(torch.randint(1, width - prompt + 1, (1,), generator = gen))
+ ids[i, : prompt + answer] = torch.randint(1, VOCAB, (prompt + answer,), generator = gen)
+ attention[i, : prompt + answer] = 1
+ completion[i, prompt : prompt + answer] = 1
+ return {"input_ids": ids, "attention_mask": attention, "completion_mask": completion}
+
+
+def _trainer(model, ref_model, **overrides):
+ trainer = types.SimpleNamespace()
+ trainer.__dict__.update(
+ accelerator = _Accelerator(),
+ model = model,
+ ref_model = ref_model,
+ precompute_ref_logps = False,
+ aux_loss_enabled = False,
+ use_liger_kernel = False,
+ is_encoder_decoder = False,
+ is_vision_model = False,
+ ld_alpha = None,
+ f_divergence_type = "reverse_kl",
+ f_alpha_divergence_coef = 1.0,
+ loss_types = ["sigmoid"],
+ loss_weights = [1.0],
+ beta = 0.1,
+ label_smoothing = 0.0,
+ use_weighting = False,
+ _metrics = {"train": defaultdict(list), "eval": defaultdict(list)},
+ _total_train_tokens = 0,
+ )
+ if "_tp_size" in _TRL_SOURCE:
+ # Only releases that read it set it (1.13+); older ones would raise on it.
+ trainer._tp_size = 1
+ trainer.args = types.SimpleNamespace(
+ gradient_checkpointing_kwargs = None, discopop_tau = 0.05, padding_free = False
+ )
+ for key, value in overrides.items():
+ setattr(trainer, key, value)
+ if "loss_weights" not in overrides:
+ trainer.loss_weights = [1.0] * len(trainer.loss_types)
+ return trainer
+
+
+CASES = [
+ dict(),
+ dict(loss_types = ["ipo"]),
+ dict(loss_types = ["hinge"]),
+ dict(loss_types = ["robust"], label_smoothing = 0.1),
+ dict(loss_types = ["exo_pair"], label_smoothing = 0.1),
+ dict(loss_types = ["nca_pair"]),
+ dict(loss_types = ["bco_pair"]),
+ dict(loss_types = ["sppo_hard"]),
+ dict(loss_types = ["aot"]),
+ dict(loss_types = ["aot_unpaired"]),
+ dict(loss_types = ["apo_zero"]),
+ dict(loss_types = ["apo_down"]),
+ dict(loss_types = ["discopop"]),
+ dict(loss_types = ["sigmoid_norm"]),
+ dict(loss_types = ["sigmoid", "sft"], loss_weights = [0.7, 0.3]),
+ dict(ld_alpha = 0.5),
+ dict(use_weighting = True),
+ dict(f_divergence_type = "forward_kl"),
+ dict(f_divergence_type = "js_divergence"),
+ dict(beta = 2.0),
+ dict(softcap = 3.0),
+ dict(bias = True),
+]
+
+
+def _run(trainer, model, batch, fused, module):
+ model.zero_grad(set_to_none = True)
+ model.train()
+ if fused:
+ HiddenStatesLlama.hidden_calls = 0
+ loss = module.unsloth_fused_dpo_compute_loss_v1(
+ trainer, model, dict(batch), False, trl_semantics = TRL_SEMANTICS
+ )
+ assert loss is not None, "fused path declined a supported case"
+ assert HiddenStatesLlama.hidden_calls >= 1
+ else:
+ loss = DPOTrainer._compute_loss(trainer, model, dict(batch), False)
+ loss.backward()
+ grads = {n: p.grad.clone() for n, p in model.named_parameters() if p.grad is not None}
+ metrics = {k: list(v) for k, v in trainer._metrics["train"].items()}
+ return loss.detach(), metrics, grads
+
+
+def _same(
+ base,
+ head,
+ tol = 1e-9,
+):
+ (loss_b, metrics_b, grads_b), (loss_h, metrics_h, grads_h) = base, head
+ assert torch.isfinite(loss_b)
+ torch.testing.assert_close(loss_h, loss_b, rtol = tol, atol = tol)
+ assert set(metrics_b) == set(metrics_h)
+ for key in metrics_b:
+ assert metrics_h[key] == pytest.approx(metrics_b[key], rel = 1e-7, abs = 1e-7), key
+ assert set(grads_b) == set(grads_h)
+ assert any(g.abs().sum() > 0 for g in grads_b.values())
+ for name in grads_b:
+ torch.testing.assert_close(grads_h[name], grads_b[name], rtol = tol, atol = tol, msg = name)
+
+
+def _declined_here(case):
+ # Pre-1.10 TRL: the fused path keeps forward_kl / js on TRL's own function.
+ return (
+ TRL_SEMANTICS["legacy_fdiv"] and case.get("f_divergence_type", "reverse_kl") != "reverse_kl"
+ )
+
+
+def _loss_type_known(case):
+ known = set(re.findall(r'loss_type == "([^"]+)"', _TRL_SOURCE))
+ return set(case.get("loss_types", ["sigmoid"])) <= known
+
+
+@needs_hook
+ "case", CASES, ids = lambda c: "-".join(f"{k}={v}" for k, v in c.items()) or "default"
+)
[email protected]("reference", ["ref_model", "precomputed"])
+def test_compute_loss_matches_trl(dpo_module, monkeypatch, case, reference):
+ case = dict(case)
+ if not _loss_type_known(case):
+ pytest.skip("loss type not in this TRL")
+ softcap = case.pop("softcap", 0.0)
+ bias = case.pop("bias", False)
+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, softcap))
+ batch = _batch(seed = 5)
+ results = []
+ for fused in (False, True):
+ model = _model(0, softcap, bias)
+ ref_model = _model(1, softcap, bias)
+ run_batch = dict(batch)
+ trainer = _trainer(model, ref_model, **case)
+ if reference == "precomputed":
+ with torch.no_grad():
+ logits = ref_model(
+ input_ids = batch["input_ids"], attention_mask = batch["attention_mask"]
+ ).logits
+ logps = torch.log_softmax(logits[:, :-1], -1).gather(
+ -1, batch["input_ids"][:, 1:, None]
+ )[..., 0]
+ logps = (logps * batch["completion_mask"][:, 1:]).sum(1)
+ run_batch["ref_chosen_logps"], run_batch["ref_rejected_logps"] = logps.chunk(2)
+ trainer.precompute_ref_logps = True
+ trainer.ref_model = None
+ if fused and _declined_here(case):
+ assert (
+ dpo_module.unsloth_fused_dpo_compute_loss_v1(
+ trainer, model, dict(run_batch), False, trl_semantics = TRL_SEMANTICS
+ )
+ is None
+ )
+ return
+ results.append(_run(trainer, model, run_batch, fused, dpo_module))
+ _same(*results)
+
+
+@needs_hook
+def test_compute_loss_peft_reference(dpo_module, monkeypatch):
+ peft = pytest.importorskip("peft")
+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))
+ batch = _batch(seed = 7)
+ results = []
+ for fused in (False, True):
+ base = _model(0)
+ model = peft.get_peft_model(
+ base,
+ peft.LoraConfig(
+ r = 4, lora_alpha = 4, target_modules = ["q_proj", "v_proj"], init_lora_weights = False
+ ),
+ ).double()
+ results.append(_run(_trainer(model, None), model, batch, fused, dpo_module))
+ _same(*results)
+
+
+ "overrides, inputs_extra, return_outputs",
+ [
+ (dict(use_liger_kernel = True), {}, False),
+ (dict(aux_loss_enabled = True), {}, False),
+ (dict(is_vision_model = True), {}, False),
+ (dict(f_divergence_type = "alpha_divergence"), {}, False),
+ (dict(loss_types = ["some_future_loss"]), {}, False),
+ (dict(), {"pixel_values": torch.zeros(1)}, False),
+ (dict(), {}, True),
+ ],
+)
+def test_compute_loss_declines(dpo_module, monkeypatch, overrides, inputs_extra, return_outputs):
+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))
+ model = _model(0)
+ trainer = _trainer(model, _model(1), **overrides)
+ batch = dict(_batch(seed = 1), **inputs_extra)
+ semantics = dict(
+ TRL_SEMANTICS or {"legacy_fdiv": False, "entropy_mean": False, "tp_size": True}
+ )
+ assert (
+ dpo_module.unsloth_fused_dpo_compute_loss_v1(
+ trainer, model, batch, return_outputs, trl_semantics = semantics
+ )
+ is None
+ )
+
+
+def test_compute_loss_declines_without_semantics(dpo_module, monkeypatch):
+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))
+ model = _model(0)
+ assert (
+ dpo_module.unsloth_fused_dpo_compute_loss_v1(
+ _trainer(model, _model(1)), model, _batch(1), False
+ )
+ is None
+ )
+
+
+def test_legacy_fdiv_keeps_only_reverse_kl(dpo_module, monkeypatch):
+ monkeypatch.setattr(dpo_module, "_logit_transforms", lambda unwrapped: (0.0, 0.0, 0.0))
+ legacy = {"legacy_fdiv": True, "entropy_mean": False, "tp_size": False}
+ for f_div in ("forward_kl", "js_divergence"):
+ model = _model(0)
+ trainer = _trainer(model, _model(1), f_divergence_type = f_div)
+ assert (
+ dpo_module.unsloth_fused_dpo_compute_loss_v1(
+ trainer, model, _batch(1), False, trl_semantics = legacy
+ )
+ is None
+ )
+
+
+def test_compute_loss_flag_off(dpo_module, monkeypatch):
+ monkeypatch.setenv("UNSLOTH_FUSED_PREFERENCE_LOSS", "0")
+ model = _model(0)
+ semantics = {"legacy_fdiv": False, "entropy_mean": False, "tp_size": True}
+ assert (
+ dpo_module.unsloth_fused_dpo_compute_loss_v1(
+ _trainer(model, _model(1)), model, _batch(1), False, trl_semantics = semantics
+ )
+ is None
+ )
+
+
+def test_rl_replacement_hooks_installed_trl_compute_loss(monkeypatch):
+ hook = _RR["dpo_trainer_fused_preference_compute_loss"]
+ source = _TRL_SOURCE
+ monkeypatch.setenv("UNSLOTH_FUSED_PREFERENCE_LOSS", "0")
+ assert hook("_compute_loss", source) == source
+ monkeypatch.setenv("UNSLOTH_FUSED_PREFERENCE_LOSS", "1")
+ patched = hook("_compute_loss", source)
+ if TRL_SEMANTICS is None:
+ # 0.29.x (truncates inside) and 1.15+ (TRL's own fused lm head) keep TRL's function.
+ assert patched == source
+ assert "self._truncate_inputs(" in source or "fused_lm_head=True" in source
+ return
+ assert (
+ "unsloth_fused_dpo_compute_loss_v1(self, model, inputs, return_outputs, trl_semantics = "
+ in patched
+ )
+ assert repr(TRL_SEMANTICS) in patched
+ compile(__import__("textwrap").dedent(patched), "<patched>", "exec")
+ # The flags follow the source: a release that reads _tp_size gets it, one that does not never does.
+ assert TRL_SEMANTICS["tp_size"] == ("// self._tp_size" in source)
+ # A TRL that logs a metric the replacement does not reproduce keeps its own function.
+ extra = source.replace('self._metrics[mode]["entropy"]', 'self._metrics[mode]["entropy_new"]')
+ assert hook("_compute_loss", extra) == extra
+ assert hook("compute_loss", source) == source
+ # Shapes the replacement does not reproduce keep TRL's function.
+ for marker in ("self._truncate_inputs(", "fused_lm_head=True"):
+ mutated = source.replace("mode = ", f"{marker}\n mode = ", 1)
+ assert hook("_compute_loss", mutated) == mutated
+ # A release with neither f-divergence spelling keeps TRL's function.
+ no_fdiv = source.replace(
+ "1 - torch.exp(-chosen_logratios)", "torch.expm1(chosen_logratios)"
+ ).replace("-torch.exp(-chosen_logratios)", "torch.expm1(chosen_logratios)")
+ assert hook("_compute_loss", no_fdiv) == no_fdiv
+
+
+def test_semantics_for_recorded_release_spellings():
+ """The three behaviour switches, from the lines that differ between TRL 1.0 and 1.14."""
+ sem = _RR["_dpo_v1_semantics"]
+ base = _TRL_SOURCE if TRL_SEMANTICS is not None else None
+ if base is None:
+ pytest.skip("installed TRL source is not a hooked release")
+ modern = (
+ base.replace("-torch.exp(-chosen_logratios)", "1 - torch.exp(-chosen_logratios)")
+ if TRL_SEMANTICS["legacy_fdiv"]
+ else base
+ )
+ assert sem(modern)["legacy_fdiv"] is False
+ legacy = modern.replace(
+ "chosen_scores = 1 - torch.exp(-chosen_logratios)",
+ "chosen_scores = -torch.exp(-chosen_logratios)",
+ )
+ assert sem(legacy)["legacy_fdiv"] is True
+ assert (
+ sem(
+ modern + "\n entropy = per_token_entropy[shift_completion_mask.bool()].mean()\n"
+ )["entropy_mean"]
+ is True
+ )
+ assert sem(modern.replace("// self._tp_size", ""))["tp_size"] is False
diff --git a/tests/test_fused_dpo_trl1_gpu.py b/tests/test_fused_dpo_trl1_gpu.py
new file mode 100644
index 00000000000..4bbf4a02a33
--- /dev/null
+++ b/tests/test_fused_dpo_trl1_gpu.py
@@ -0,0 +1,103 @@
+# SPDX-License-Identifier: AGPL-3.0-only
+# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
+
+"""GPU: on TRL >= 1.0, UNSLOTH_FUSED_PREFERENCE_LOSS=1 must reach the fused DPO _compute_loss in a
+real Unsloth DPOTrainer run, and the default (unset) must never reach it."""
+
+from __future__ import annotations
+
+import json
+import os
+import subprocess
+import sys
+import textwrap
+
+import pytest
+
+torch = pytest.importorskip("torch")
+if not torch.cuda.is_available() or getattr(torch.cuda, "_unsloth_consolidated_spoof", False):
+ pytest.skip("needs a real CUDA GPU", allow_module_level = True)
+trl = pytest.importorskip("trl")
+
+MODEL = os.environ.get("UNSLOTH_TEST_DPO_MODEL", "trl-internal-testing/tiny-Qwen3ForCausalLM")
+
+SCRIPT = textwrap.dedent(
+ """
+ import json, sys
+ import unsloth
+ from unsloth import FastLanguageModel, PatchDPOTrainer
+ import unsloth.models.dpo as dpo
+ counts = {"fused": 0, "declined": 0}
+ original = dpo.unsloth_fused_dpo_compute_loss_v1
+ def counted(*args, **kwargs):
+ out = original(*args, **kwargs)
+ counts["fused" if out is not None else "declined"] += 1
+ return out
+ dpo.unsloth_fused_dpo_compute_loss_v1 = counted
+ import torch
+ from datasets import Dataset
+ from trl import DPOConfig, DPOTrainer
+ PatchDPOTrainer()
+ model, tok = FastLanguageModel.from_pretrained(sys.argv[1], max_seq_length = 128, load_in_4bit = False)
+ model = FastLanguageModel.get_peft_model(model, r = 8, lora_alpha = 8, target_modules = ["q_proj", "v_proj"])
+ if tok.pad_token is None:
+ tok.pad_token = tok.eos_token
+ rows = [
+ {"prompt": f"Question {i}: what is {i} plus {i}?", "chosen": f" It is {2 * i}.", "rejected": f" It is {2 * i + 1}."}
+ for i in range(8)
+ ]
+ args = DPOConfig(
+ output_dir = sys.argv[2], per_device_train_batch_size = 2, max_steps = 3, learning_rate = 1e-4,
+ logging_steps = 1, report_to = "none", max_length = 64, beta = 0.1, save_strategy = "no",
+ fp16 = not torch.cuda.is_bf16_supported(), bf16 = torch.cuda.is_bf16_supported(),
+ )
+ trainer = DPOTrainer(model = model, ref_model = None, args = args, train_dataset = Dataset.from_list(rows), processing_class = tok)
+ trainer.train()
+ losses = [h["loss"] for h in trainer.state.log_history if "loss" in h]
+ print("RESULT " + json.dumps({"counts": counts, "losses": losses}))
+ """
+)
+
+
+def _run(tmp_path, flag):
+ env = dict(os.environ)
+ env.pop("UNSLOTH_FUSED_PREFERENCE_LOSS", None)
+ if flag is not None:
+ env["UNSLOTH_FUSED_PREFERENCE_LOSS"] = flag
+ env.setdefault("UNSLOTH_COMPILE_LOCATION", str(tmp_path / f"cc_{flag}"))
+ script = tmp_path / "run_dpo.py"
+ script.write_text(SCRIPT, encoding = "utf-8")
+ proc = subprocess.run(
+ [sys.executable, str(script), MODEL, str(tmp_path / f"out_{flag}")],
+ env = env,
+ capture_output = True,
+ text = True,
+ timeout = 1200,
+ )
+ lines = [l for l in proc.stdout.splitlines() if l.startswith("RESULT ")]
+ assert proc.returncode == 0 and lines, proc.stdout[-4000:] + proc.stderr[-4000:]
+ return json.loads(lines[-1][len("RESULT ") :])
+
+
+def test_fused_dpo_v1_engages_only_when_requested(tmp_path):
+ import inspect
+
+ from trl import DPOTrainer
+
+ source = (
+ inspect.getsource(DPOTrainer._compute_loss) if hasattr(DPOTrainer, "_compute_loss") else ""
+ )
+ hooked_release = (
+ "completion_mask" in source
+ and "self._truncate_inputs(" not in source
+ and "fused_lm_head=True" not in source
+ )
+ on = _run(tmp_path, "1")
+ off = _run(tmp_path, None)
+ assert off["counts"] == {"fused": 0, "declined": 0}
+ if hooked_release:
+ assert on["counts"]["fused"] >= 3 and on["counts"]["declined"] == 0, on
+ else:
+ assert on["counts"] == {"fused": 0, "declined": 0}, on
+ for result in (on, off):
+ assert result["losses"] and all(x == x and abs(x) < 1e4 for x in result["losses"]), result
diff --git a/unsloth/models/dpo.py b/unsloth/models/dpo.py
index 4be13424001..d5dec1f2666 100644
--- a/unsloth/models/dpo.py
+++ b/unsloth/models/dpo.py
@@ -13,6 +13,7 @@ __all__ = [
"PatchDPOTrainer",
"PatchKTOTrainer",
"fused_preference_loss_enabled",
+ "unsloth_fused_dpo_compute_loss_v1",
"unsloth_fused_dpo_concatenated_forward",
"unsloth_fused_orpo_concatenated_forward",
]
@@ -512,3 +513,448 @@ def unsloth_fused_orpo_concatenated_forward(
chosen_logits = mean_logits[:len_chosen]
rejected_logits = mean_logits[len_chosen:]
return (chosen_logps, rejected_logps, chosen_logits, rejected_logits, nll_loss)
+
+
+# TRL >= 1.0 DPOTrainer._compute_loss scores [chosen; rejected] from full batch x seq x vocab logits;
+# under UNSLOTH_FUSED_PREFERENCE_LOSS=1 the replacement below reproduces it from hidden states.
+_DPO_V1_INPUT_KEYS = frozenset(
+ (
+ "input_ids",
+ "attention_mask",
+ "completion_mask",
+ "ref_chosen_logps",
+ "ref_rejected_logps",
+ )
+)
+_DPO_V1_LOSS_TYPES = frozenset(
+ (
+ "sigmoid",
+ "hinge",
+ "ipo",
+ "exo_pair",
+ "nca_pair",
+ "robust",
+ "bco_pair",
+ "sppo_hard",
+ "aot",
+ "aot_unpaired",
+ "apo_zero",
+ "apo_down",
+ "discopop",
+ "sft",
+ "sigmoid_norm",
+ )
+)
+
+
+def _metric_rows(n_rows, vocab_size):
+ # One float32 logits chunk plus its log_softmax and exp: keep each near 256 MB.
+ rows = (256 * 1024 * 1024) // max(3 * 4 * vocab_size, 1)
+ return max(1, min(int(rows), max(n_rows, 1)))
+
+
+def _entropy_and_argmax(hidden_or_logits, is_hidden, head, transforms, mask):
+ """Per-position entropy and argmax of the (transformed) logits on ``mask`` rows only,
+ one chunk of rows at a time. Positions off ``mask`` are 0 / -1."""
+ import torch.nn.functional as F
+
+ flat = hidden_or_logits.reshape(-1, hidden_or_logits.shape[-1])
+ flat_mask = mask.reshape(-1).to(device = flat.device, dtype = torch.bool)
+ index = flat_mask.nonzero(as_tuple = True)[0]
+ # float32 at least, float64 kept (TRL computes its metrics in the logits dtype).
+ stat_dtype = torch.promote_types(flat.dtype, torch.float32)
+ entropy = torch.zeros(flat_mask.shape[0], dtype = stat_dtype, device = flat.device)
+ argmax = torch.full((flat_mask.shape[0],), -1, dtype = torch.long, device = flat.device)
+ if index.numel() == 0:
+ return entropy.reshape(mask.shape), argmax.reshape(mask.shape)
+ weight, bias = head
+ vocab_size = weight.shape[0] if is_hidden else flat.shape[-1]
+ rows = _metric_rows(index.numel(), vocab_size)
+ scale_multiply, scale_divide, softcapping = transforms
+ with torch.autocast(device_type = flat.device.type, enabled = False):
+ for start in range(0, index.numel(), rows):
+ idx = index[start : start + rows]
+ if is_hidden:
+ chunk = flat[idx].to(device = weight.device, dtype = weight.dtype)
+ logits = F.linear(chunk, weight, bias).to(stat_dtype)
+ if scale_multiply != 0.0:
+ logits = logits * scale_multiply
+ if scale_divide != 0.0:
+ logits = logits / scale_divide
+ if softcapping != 0.0:
+ logits = softcapping * torch.tanh(logits / softcapping)
+ else:
+ logits = flat[idx].to(stat_dtype)
+ logps = torch.log_softmax(logits, dim = -1)
+ entropy[idx] = (-(torch.exp(logps) * logps).sum(-1)).to(entropy.device)
+ argmax[idx] = logits.argmax(dim = -1).to(argmax.device)
+ del logits, logps
+ return entropy.reshape(mask.shape), argmax.reshape(mask.shape)
+
+
+def _dpo_v1_supported(trainer, model, inputs, return_outputs, trl_semantics):
+ if not isinstance(trl_semantics, dict):
+ return False
+ if not fused_preference_loss_enabled() or _zoo_kernels() is None:
+ return False
+ if return_outputs:
+ # prediction_step wants the real logits back in `outputs`.
+ return False
+ if getattr(trainer, "use_liger_kernel", False) or getattr(trainer, "aux_loss_enabled", False):
+ return False
+ if getattr(trainer, "is_encoder_decoder", False) or getattr(trainer, "is_vision_model", False):
+ return False
+ args = getattr(trainer, "args", None)
+ if getattr(args, "padding_free", False):
+ return False
+ if not set(inputs).issubset(_DPO_V1_INPUT_KEYS):
+ return False
+ if "input_ids" not in inputs or "completion_mask" not in inputs:
+ return False
+ # alpha_divergence clamps by the logps dtype, which is float32 here and half in TRL.
+ f_divergence_types = ("reverse_kl", "forward_kl", "js_divergence")
+ if trl_semantics.get("legacy_fdiv", True):
+ # Before TRL 1.10 these scores drop their constants and replace the log ratios in some
+ # losses; only reverse_kl, where the score is the log ratio, is the same everywhere.
+ f_divergence_types = ("reverse_kl",)
+ if getattr(trainer, "f_divergence_type", None) not in f_divergence_types:
+ return False
+ loss_types = getattr(trainer, "loss_types", None)
+ loss_weights = getattr(trainer, "loss_weights", None)
+ if not loss_types or loss_weights is None or len(loss_types) != len(loss_weights):
+ return False
+ if not set(loss_types).issubset(_DPO_V1_LOSS_TYPES):
+ return False
+ return True
+
+
+def _dpo_v1_heads(trainer, model):
+ """(unwrapped, head, transforms) for the policy, and the same for a separate reference model
+ (``None`` when the reference is the policy with its adapter disabled or precomputed)."""
+ unwrapped, head = _output_head(trainer, model)
+ if head is None:
+ return None
+ transforms = _logit_transforms(unwrapped)
+ if transforms is None:
+ return None
+ ref = None
+ if not trainer.precompute_ref_logps:
+ try:
+ from accelerate.utils import is_peft_model
+ except Exception:
+ return None
+ if not (is_peft_model(model) and trainer.ref_model is None):
+ if trainer.ref_model is None:
+ return None
+ ref_unwrapped, ref_head = _output_head(trainer, trainer.ref_model)
+ if ref_head is None:
+ return None
+ ref_transforms = _logit_transforms(ref_unwrapped)
+ if ref_transforms is None:
+ return None
+ ref = (ref_unwrapped, ref_head, ref_transforms)
+ return (unwrapped, head, transforms), ref
+
+
+def unsloth_fused_dpo_compute_loss_v1(
+ trainer,
+ model,
+ inputs,
+ return_outputs = False,
+ trl_semantics = None,
+):
+ """TRL 1.x ``DPOTrainer._compute_loss`` for text-only decoder models, scored from hidden
+ states in row chunks. Returns ``None`` (run TRL's own) whenever the inputs leave that case.
+
+ ``trl_semantics`` is what rl_replacements read from the installed TRL's source: legacy_fdiv
+ (pre-1.10 f-divergence scores), entropy_mean (pre-1.3 entropy metric) and tp_size (1.13+
+ divides the token count by the tensor-parallel size)."""
+ if not _dpo_v1_supported(trainer, model, inputs, return_outputs, trl_semantics):
+ return None
+ heads = _dpo_v1_heads(trainer, model)
+ if heads is None:
+ return None
+ (unwrapped, head, transforms), ref = heads
+
+ import math
+ import torch.nn.functional as F
+ from trl.trainer.dpo_trainer import disable_gradient_checkpointing, use_adapter
+ from accelerate.utils import is_peft_model
+
+ self = trainer
+ mode = "train" if self.model.training else "eval"
+ device = self.accelerator.device
+
+ _non_model_keys = {"completion_mask", "ref_chosen_logps", "ref_rejected_logps"}
+ model_kwargs = {k: v for k, v in inputs.items() if k not in _non_model_keys}
+ model_kwargs["use_cache"] = False
+
+ input_ids = inputs["input_ids"]
+ completion_mask = inputs["completion_mask"]
+ shift_labels = input_ids[..., 1:]
+ shift_completion_mask = completion_mask[..., 1:]
+ score_mask = shift_completion_mask.bool()
+ use_weighting = bool(getattr(self, "use_weighting", False))
+
+ hidden, is_hidden = _forward_hidden_states(model, unwrapped, head[0], (), model_kwargs)
+ shift_hidden = hidden[..., :-1, :]
+ per_token_logps, mean_logits, log_sum_sq_probs = _token_logps(
+ shift_hidden,
+ is_hidden,
+ head,
+ transforms,
+ shift_labels,
+ score_mask,
+ want_lssp = use_weighting,
+ )
+ per_token_entropy, predictions = _entropy_and_argmax(
+ shift_hidden.detach(), is_hidden, head, transforms, score_mask
+ )
+ del hidden, shift_hidden
+
+ if self.ld_alpha is None:
+ logps = per_token_logps.sum(dim = 1)
+ else:
+ comp_pos = shift_completion_mask.cumsum(dim = 1)
+ comp_lens = shift_completion_mask.sum(dim = 1).long()
+ chosen_lens, rejected_lens = comp_lens.chunk(2, dim = 0)
+ shared_lens = torch.minimum(chosen_lens, rejected_lens)
+ shared_lens = torch.cat([shared_lens, shared_lens], dim = 0).to(device)
+ shared_mask = (comp_pos > 0) & (comp_pos <= shared_lens.unsqueeze(1))
+ tail_mask = comp_pos > shared_lens.unsqueeze(1)
+ shared_logps = (per_token_logps * shared_mask).sum(dim = 1)
+ tail_logps = (per_token_logps * tail_mask).sum(dim = 1)
+ logps = shared_logps + self.ld_alpha * tail_logps
+ chosen_logps, rejected_logps = logps.chunk(2, dim = 0)
+
+ if self.precompute_ref_logps:
+ ref_chosen_logps, ref_rejected_logps = (
+ inputs["ref_chosen_logps"],
+ inputs["ref_rejected_logps"],
+ )
+ else:
+ ref_model_kwargs = dict(model_kwargs)
+ with (
+ torch.no_grad(),
+ disable_gradient_checkpointing(self.model, self.args.gradient_checkpointing_kwargs),
+ ):
+ if ref is None:
+ peft_model = self.accelerator.unwrap_model(model)
+ with use_adapter(
+ peft_model, adapter_name = "ref" if "ref" in peft_model.peft_config else None
+ ):
+ ref_hidden, ref_is_hidden = _forward_hidden_states(
+ self.model, unwrapped, head[0], (), ref_model_kwargs
+ )
+ ref_head, ref_transforms = head, transforms
+ else:
+ ref_unwrapped, ref_head, ref_transforms = ref
+ ref_hidden, ref_is_hidden = _forward_hidden_states(
+ self.ref_model, ref_unwrapped, ref_head[0], (), ref_model_kwargs
+ )
+ ref_per_token_logps, _, _ = _token_logps(
+ ref_hidden[..., :-1, :],
+ ref_is_hidden,
+ ref_head,
+ ref_transforms,
+ shift_labels,
+ score_mask,
+ )
+ del ref_hidden
+ if self.ld_alpha is None:
+ ref_logps = ref_per_token_logps.sum(dim = 1)
+ else:
+ ref_shared_logps = (ref_per_token_logps * shared_mask).sum(dim = 1)
+ ref_tail_logps = (ref_per_token_logps * tail_mask).sum(dim = 1)
+ ref_logps = ref_shared_logps + self.ld_alpha * ref_tail_logps
+ ref_chosen_logps, ref_rejected_logps = ref_logps.chunk(2, dim = 0)
+
+ chosen_logratios = chosen_logps - ref_chosen_logps
+ rejected_logratios = rejected_logps - ref_rejected_logps
+
+ if self.f_divergence_type == "reverse_kl":
+ chosen_scores = chosen_logratios
+ rejected_scores = rejected_logratios
+ elif self.f_divergence_type == "forward_kl":
+ chosen_scores = 1 - torch.exp(-chosen_logratios)
+ rejected_scores = 1 - torch.exp(-rejected_logratios)
+ else: # js_divergence
+ chosen_scores = math.log(2) + F.logsigmoid(chosen_logratios)
+ rejected_scores = math.log(2) + F.logsigmoid(rejected_logratios)
+
+ delta_score = chosen_scores - rejected_scores
+
+ loss = 0.0
+ for loss_type, loss_weight in zip(self.loss_types, self.loss_weights):
+ if loss_type == "sigmoid":
+ per_sequence_loss = -F.logsigmoid(self.beta * delta_score)
+ elif loss_type == "hinge":
+ per_sequence_loss = torch.relu(1 - self.beta * delta_score)
+ elif loss_type == "ipo":
+ chosen_mask, rejected_mask = completion_mask.chunk(2, dim = 0)
+ chosen_avg_score = chosen_scores / chosen_mask.sum(dim = 1).clamp(min = 1.0)
+ rejected_avg_score = rejected_scores / rejected_mask.sum(dim = 1).clamp(min = 1.0)
+ ipo_delta = chosen_avg_score - rejected_avg_score
+ per_sequence_loss = (ipo_delta - 1 / (2 * self.beta)) ** 2
+ elif loss_type == "exo_pair":
+ epsilon = torch.tensor(self.label_smoothing, device = device)
+ qw = torch.sigmoid(self.beta * delta_score)
+ log_qw = F.logsigmoid(self.beta * delta_score)
+ log_pw = torch.log1p(-epsilon)
+ ql = torch.sigmoid(-self.beta * delta_score)
+ log_ql = F.logsigmoid(-self.beta * delta_score)
+ log_pl = torch.log(epsilon)
+ per_sequence_loss = qw * (log_qw - log_pw) + ql * (log_ql - log_pl)
+ elif loss_type == "nca_pair":
+ chosen_rewards = self.beta * chosen_logratios
+ rejected_rewards = self.beta * rejected_logratios
+ per_sequence_loss = (
+ -F.logsigmoid(chosen_rewards)
+ - 0.5 * F.logsigmoid(-chosen_rewards)
+ - 0.5 * F.logsigmoid(-rejected_rewards)
+ )
+ elif loss_type == "robust":
+ clean_loss_term = -(1 - self.label_smoothing) * F.logsigmoid(self.beta * delta_score)
+ flipped_loss_term = -self.label_smoothing * F.logsigmoid(-self.beta * delta_score)
+ per_sequence_loss = (clean_loss_term - flipped_loss_term) / (
+ 1 - 2 * self.label_smoothing
+ )
+ elif loss_type == "bco_pair":
+ chosen_rewards = self.beta * chosen_logratios
+ rejected_rewards = self.beta * rejected_logratios
+ per_sequence_loss = -F.logsigmoid(chosen_rewards) - F.logsigmoid(-rejected_rewards)
+ elif loss_type == "sppo_hard":
+ winner_margin_error = (chosen_logratios - 0.5 / self.beta) ** 2
+ loser_margin_error = (rejected_logratios + 0.5 / self.beta) ** 2
+ per_sequence_loss = winner_margin_error + loser_margin_error
+ elif loss_type == "aot":
+ logratios = chosen_logps - rejected_logps
+ ref_logratios = ref_chosen_logps - ref_rejected_logps
+ logratios_sorted, _ = torch.sort(logratios, dim = 0)
+ ref_logratios_sorted, _ = torch.sort(ref_logratios, dim = 0)
+ delta = logratios_sorted - ref_logratios_sorted
+ per_sequence_loss = (
+ -F.logsigmoid(self.beta * delta) * (1 - self.label_smoothing)
+ - F.logsigmoid(-self.beta * delta) * self.label_smoothing
+ )
+ elif loss_type == "aot_unpaired":
+ chosen_logratios_sorted, _ = torch.sort(chosen_logratios, dim = 0)
+ rejected_logratios_sorted, _ = torch.sort(rejected_logratios, dim = 0)
+ delta = chosen_logratios_sorted - rejected_logratios_sorted
+ per_sequence_loss = (
+ -F.logsigmoid(self.beta * delta) * (1 - self.label_smoothing)
+ - F.logsigmoid(-self.beta * delta) * self.label_smoothing
+ )
+ elif loss_type == "apo_zero":
+ losses_chosen = 1 - torch.sigmoid(self.beta * chosen_logratios)
+ losses_rejected = torch.sigmoid(self.beta * rejected_logratios)
+ per_sequence_loss = losses_chosen + losses_rejected
+ elif loss_type == "apo_down":
+ losses_chosen = torch.sigmoid(self.beta * chosen_logratios)
+ losses_rejected = 1 - torch.sigmoid(self.beta * (chosen_logratios - rejected_logratios))
+ per_sequence_loss = losses_chosen + losses_rejected
+ elif loss_type == "discopop":
+ logits = delta_score * self.beta
+ log_ratio_modulation = torch.sigmoid(logits / self.args.discopop_tau)
+ logistic_component = -F.logsigmoid(logits)
+ exp_component = torch.exp(-logits)
+ per_sequence_loss = (
+ logistic_component * (1 - log_ratio_modulation)
+ + exp_component * log_ratio_modulation
+ )
+ elif loss_type == "sft":
+ # F.cross_entropy over the chosen completion tokens == -mean of their logps.
+ chosen_token_logps, _ = per_token_logps.chunk(2, dim = 0)
+ chosen_mask, _ = score_mask.chunk(2, dim = 0)
+ batch_loss = -chosen_token_logps[chosen_mask].mean()
+ per_sequence_loss = batch_loss.expand(chosen_token_logps.size(0))
+ else: # sigmoid_norm
+ chosen_mask, rejected_mask = completion_mask.chunk(2, dim = 0)
+ chosen_avg_score = chosen_scores / chosen_mask.sum(dim = 1).clamp(min = 1.0)
+ rejected_avg_score = rejected_scores / rejected_mask.sum(dim = 1).clamp(min = 1.0)
+ delta = chosen_avg_score - rejected_avg_score
+ per_sequence_loss = -F.logsigmoid(self.beta * delta)
+
+ if use_weighting:
+ completion_lengths = shift_completion_mask.sum(dim = 1).clamp_min(1)
+ with torch.no_grad():
+ # log_sum_sq_probs = logsumexp(2 l) - 2 logsumexp(l), TRL's log_denom.
+ aligned_logps = (per_token_logps - log_sum_sq_probs) * shift_completion_mask
+ mean_logps = aligned_logps.sum(dim = 1) / completion_lengths
+ weights = torch.exp(mean_logps)
+ chosen_weights, rejected_weights = weights.chunk(2, dim = 0)
+ per_sequence_loss *= chosen_weights * rejected_weights
+
+ loss += per_sequence_loss.mean() * loss_weight
+
+ # Metrics, as TRL logs them, from the chunked statistics.
+ mask = shift_completion_mask
+ if trl_semantics.get("entropy_mean", False):
+ entropy = per_token_entropy[mask.bool()].mean()
+ entropy = self.accelerator.gather_for_metrics(entropy).mean().item()
+ else:
+ entropy_sum = (per_token_entropy * mask).sum()
+ total_tokens = mask.sum()
+ entropy_sum = self.accelerator.gather_for_metrics(entropy_sum).sum()
+ total_tokens = self.accelerator.gather_for_metrics(total_tokens).sum()
+ entropy = (entropy_sum / total_tokens).item() if total_tokens > 0 else 0.0
+ self._metrics[mode]["entropy"].append(entropy)
+
+ if mode == "train":
+ num_tokens_in_batch = (
+ self.accelerator.gather_for_metrics(inputs["attention_mask"].sum()).sum().item()
+ )
+ if trl_semantics.get("tp_size", False):
+ num_tokens_in_batch = num_tokens_in_batch // self._tp_size
+ self._total_train_tokens += num_tokens_in_batch
+ self._metrics[mode]["num_tokens"] = [self._total_train_tokens]
+
+ chosen_means, rejected_means = mean_logits.detach().chunk(2, dim = 0)
+ chosen_mask, rejected_mask = shift_completion_mask.chunk(2, dim = 0)
+ total_chosen_logits = chosen_means[chosen_mask.bool()].sum()
+ total_chosen_tokens = chosen_mask.sum()
+ total_rejected_logits = rejected_means[rejected_mask.bool()].sum()
+ total_rejected_tokens = rejected_mask.sum()
+ total_chosen_logits = self.accelerator.gather_for_metrics(total_chosen_logits).sum().item()
+ total_chosen_tokens = self.accelerator.gather_for_metrics(total_chosen_tokens).sum().item()
+ total_rejected_logits = self.accelerator.gather_for_metrics(total_rejected_logits).sum().item()
+ total_rejected_tokens = self.accelerator.gather_for_metrics(total_rejected_tokens).sum().item()
+ avg_chosen_logits = (
+ total_chosen_logits / total_chosen_tokens if total_chosen_tokens > 0 else 0.0
+ )
+ avg_rejected_logits = (
+ total_rejected_logits / total_rejected_tokens if total_rejected_tokens > 0 else 0.0
+ )
+ self._metrics[mode]["logits/chosen"].append(avg_chosen_logits)
+ self._metrics[mode]["logits/rejected"].append(avg_rejected_logits)
+
+ half = len(shift_completion_mask) // 2
+ chosen_bool = shift_completion_mask[:half].bool()
+ chosen_labels = shift_labels[:half]
+ correct_predictions = (predictions[:half] == chosen_labels) & chosen_bool
+ total_tokens = chosen_bool.sum()
+ correct_tokens = correct_predictions.sum()
+ correct_tokens = self.accelerator.gather_for_metrics(correct_tokens)
+ total_tokens = self.accelerator.gather_for_metrics(total_tokens)
+ total_sum = total_tokens.sum()
+ accuracy = (correct_tokens.sum() / total_sum).item() if total_sum > 0 else 0.0
+ self._metrics[mode]["mean_token_accuracy"].append(accuracy)
+
+ chosen_rewards = self.beta * chosen_logratios.detach()
+ rejected_rewards = self.beta * rejected_logratios.detach()
+ agg_chosen_rewards = self.accelerator.gather(chosen_rewards)
+ agg_rejected_rewards = self.accelerator.gather(rejected_rewards)
+ self._metrics[mode]["rewards/chosen"].append(agg_chosen_rewards.mean().item())
+ self._metrics[mode]["rewards/rejected"].append(agg_rejected_rewards.mean().item())
+ reward_accuracies = (chosen_rewards > rejected_rewards).float()
+ agg_reward_accuracies = self.accelerator.gather(reward_accuracies)
+ self._metrics[mode]["rewards/accuracies"].append(agg_reward_accuracies.mean().item())
+ margins = chosen_rewards - rejected_rewards
+ agg_margins = self.accelerator.gather(margins)
+ self._metrics[mode]["rewards/margins"].append(agg_margins.mean().item())
+ self._metrics[mode]["logps/chosen"].append(self.accelerator.gather(chosen_logps).mean().item())
+ self._metrics[mode]["logps/rejected"].append(
+ self.accelerator.gather(rejected_logps).mean().item()
+ )
+ return loss
diff --git a/unsloth/models/rl_replacements.py b/unsloth/models/rl_replacements.py
index 8c841558222..54f2032995b 100644
--- a/unsloth/models/rl_replacements.py
+++ b/unsloth/models/rl_replacements.py
@@ -756,7 +756,101 @@ def cpo_trainer_fused_preference_forward(function_name, function):
)
+# The replacement reproduces exactly these metric keys and logits reads; any other source keeps TRL's
+# function (0.29.x truncates inside it, 1.15+ already uses TRL's fused lm head).
+_DPO_V1_METRIC_KEYS = frozenset(
+ (
+ "aux_loss",
+ "entropy",
+ "num_tokens",
+ "logits/chosen",
+ "logits/rejected",
+ "mean_token_accuracy",
+ "rewards/chosen",
+ "rewards/rejected",
+ "rewards/accuracies",
+ "rewards/margins",
+ "logps/chosen",
+ "logps/rejected",
+ )
+)
+# Each entry: the spellings (whitespace-insensitive) one of which must appear.
+_DPO_V1_SOURCE_MARKERS = (
+ (
+ "per_token_logps = selective_log_softmax(shift_logits, shift_labels)",
+ "per_token_logps, per_token_entropies = selective_log_softmax_and_entropy( shift_logits, shift_labels,",
+ ),
+ (
+ "ref_per_token_logps = selective_log_softmax(ref_shift_logits, shift_labels)",
+ "ref_per_token_logps = selective_log_softmax( ref_shift_logits, shift_labels, row_mask=shift_completion_mask )",
+ ),
+ (
+ "per_token_entropy = entropy_from_logits(shift_logits.detach())",
+ "per_token_entropy = per_token_entropies",
+ ),
+ ("predictions = chosen_logits.argmax(dim=-1)",),
+)
+# Sources the replacement does not reproduce.
+_DPO_V1_DECLINE_MARKERS = ("self._truncate_inputs(", "fused_lm_head=True")
+# Releases before 1.10 drop the constants of the forward_kl / js f-divergence scores and feed
+# those scores (not the log ratios) to nca_pair / bco_pair / sppo_hard / apo_down.
+_DPO_V1_FDIV_SCORES_MARKER = "chosen_scores = 1 - torch.exp(-chosen_logratios)"
+_DPO_V1_FDIV_LEGACY_MARKER = "chosen_scores = -torch.exp(-chosen_logratios)"
+# Releases before 1.3 log the entropy as a mean of the masked per-token entropies.
+_DPO_V1_ENTROPY_MEAN_MARKER = "entropy = per_token_entropy[shift_completion_mask.bool()].mean()"
+
+
+def _dpo_v1_semantics(function):
+ """The TRL-release differences the fused _compute_loss must follow, read from its source, or
+ None when the source is not one the replacement reproduces."""
+ flat = " ".join(function.split())
+ if any(marker in flat for marker in _DPO_V1_DECLINE_MARKERS):
+ return None
+ for spellings in _DPO_V1_SOURCE_MARKERS:
+ if not any(" ".join(spelling.split()) in flat for spelling in spellings):
+ return None
+ if _DPO_V1_FDIV_SCORES_MARKER in flat:
+ legacy_fdiv = False
+ elif _DPO_V1_FDIV_LEGACY_MARKER in flat:
+ legacy_fdiv = True
+ else:
+ return None
+ return {
+ "legacy_fdiv": legacy_fdiv,
+ "entropy_mean": _DPO_V1_ENTROPY_MEAN_MARKER in flat,
+ "tp_size": "// self._tp_size" in flat,
+ }
+
+
+def dpo_trainer_fused_preference_compute_loss(function_name, function):
+ if function_name != "_compute_loss":
+ return function
+ if os.environ.get("UNSLOTH_FUSED_PREFERENCE_LOSS", "0") != "1":
+ return function
+ if "_unsloth_fused_output" in function:
+ return function
+ if not re.search(r"def _compute_loss\(self, model, inputs, return_outputs\)", function):
+ return function
+ semantics = _dpo_v1_semantics(function)
+ if semantics is None:
+ return function
+ keys = set(re.findall(r'self\._metrics\[mode\]\["([^"]+)"\]', function))
+ if not keys or not keys.issubset(_DPO_V1_METRIC_KEYS):
+ return function
+ return _insert_after_docstring(
+ function,
+ [
+ "from unsloth.models.dpo import unsloth_fused_dpo_compute_loss_v1",
+ "_unsloth_fused_output = unsloth_fused_dpo_compute_loss_v1("
+ f"self, model, inputs, return_outputs, trl_semantics = {semantics!r})",
+ "if _unsloth_fused_output is not None:",
+ " return _unsloth_fused_output",
+ ],
+ )
+
+
RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_fused_preference_forward)
+RL_FUNCTIONS["dpo_trainer"].append(dpo_trainer_fused_preference_compute_loss)
RL_FUNCTIONS["orpo_trainer"].append(orpo_trainer_fused_preference_forward)
RL_FUNCTIONS["cpo_trainer"].append(cpo_trainer_fused_preference_forward)