Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

feat(dpo): IPO, SimPO and conservative DPO

FareedKhan-devaabecd25aec3
diff --git a/config/post_training_config.py b/config/post_training_config.py
index d6b2a66..d6cb09a 100644
--- a/config/post_training_config.py
+++ b/config/post_training_config.py
@@ -29,7 +29,7 @@ AttentionKind = Literal["gqa", "mla"]
AmpDtype = Literal["bf16", "fp16"]
OptimizerName = Literal["adamw", "muon"]
Schedule = Literal["cosine", "wsd", "linear"]
-PreferenceLoss = Literal["dpo", "orpo", "kto"]
+PreferenceLoss = Literal["dpo", "ipo", "simpo", "orpo", "kto"]
RewardSource = Literal["verifier", "rm"]
@@ -161,9 +161,11 @@ class DPOConfig(BaseModelConfig):
pref_path: str = f"{DATA_DIR}/preferences.jsonl"
test_path: str = f"{DATA_DIR}/preferences_test.jsonl"
out_ckpt: str = f"{CKPT_DIR}/dpo.pt"
- loss_type: PreferenceLoss = "dpo" # dpo | orpo | kto
+ loss_type: PreferenceLoss = "dpo" # dpo | ipo | simpo | orpo | kto
beta: float = 0.1
- orpo_lambda: float = 1.0 # ORPO odds-ratio weight (loss_type="orpo")
+ label_smoothing: float = 0.0 # > 0 = conservative DPO for noisy preference labels
+ simpo_gamma: float = 0.5 # SimPO target reward margin
+ orpo_lambda: float = 1.0 # ORPO odds-ratio weight (loss_type="orpo")
batch_size: int = 8
epochs: int = 1
eval_steps: int = 200
@@ -177,6 +179,7 @@ class DPOConfig(BaseModelConfig):
def __post_init__(self) -> None:
super().__post_init__()
_check(self.beta > 0, "beta must be positive")
+ _check(0.0 <= self.label_smoothing < 0.5, "label_smoothing must be in [0, 0.5)")
_check(16 <= self.max_len <= self.context_length, "max_len must be between 16 and context_length")
diff --git a/configs/dpo.json b/configs/dpo.json
index d9e32a8..caf028d 100644
--- a/configs/dpo.json
+++ b/configs/dpo.json
@@ -5,6 +5,8 @@
"out_ckpt": "models/dpo.pt",
"loss_type": "dpo",
"beta": 0.1,
+ "label_smoothing": 0.0,
+ "simpo_gamma": 0.5,
"orpo_lambda": 1.0,
"batch_size": 8,
"epochs": 1,
diff --git a/scripts/train_dpo.py b/scripts/train_dpo.py
index 8a80c19..b5be772 100644
--- a/scripts/train_dpo.py
+++ b/scripts/train_dpo.py
@@ -1,11 +1,14 @@
"""
-Direct Preference Optimization (and ORPO / KTO variants) on preference pairs.
+Direct Preference Optimization and its variants (IPO, SimPO, ORPO, KTO) on preference pairs.
The policy is initialized from the SFT checkpoint; a frozen deep copy of it serves as the
-DPO/KTO reference (ORPO is reference-free). Reports implicit-reward accuracy on held-out
-preferences and GSM8K dev accuracy.
+reference for DPO / IPO / KTO (SimPO and ORPO are reference-free, so no copy is made).
+Reports implicit-reward accuracy on held-out preferences.
python scripts/train_dpo.py --loss_type dpo --beta 0.1
+ python scripts/train_dpo.py --loss_type dpo --label_smoothing 0.1 # conservative DPO
+ python scripts/train_dpo.py --loss_type ipo --beta 0.1
+ python scripts/train_dpo.py --loss_type simpo --beta 2.0 --simpo_gamma 0.5
torchrun --standalone --nproc_per_node=2 scripts/train_dpo.py
"""
@@ -25,7 +28,7 @@ from config.post_training_config import DPOConfig
from data_loader.preference_dataset import get_preference_iterator
from src.post_training.cli import parse_config_with_json
from src.post_training.distributed import ddp_setup, ddp_wrap, cleanup, reduce_scalar
-from src.post_training.dpo import dpo_loss, orpo_loss, kto_loss, implicit_accuracy
+from src.post_training.dpo import dpo_loss, implicit_accuracy, ipo_loss, kto_loss, orpo_loss, simpo_loss
from src.post_training.logging_utils import MetricsLogger
from src.post_training.optim import configure_optimizer, cosine_lr, set_lr
from src.post_training.rollout import sequence_logprobs
@@ -33,6 +36,8 @@ from src.post_training.utils import (
amp_autocast, load_backbone_from_ckpt, make_frozen_copy, save_stage_ckpt, set_seed, unwrap,
)
+REFERENCE_FREE = {"orpo", "simpo"}
+
def _logps(model, ids, mask, requires_grad):
return sequence_logprobs(model, ids, mask, requires_grad=requires_grad)
@@ -49,13 +54,17 @@ def _compute_losses(policy, ref, batch, cfg, ctx):
if cfg.loss_type == "orpo":
return orpo_loss(pc, pr, ncn, nrn, orpo_lambda=cfg.orpo_lambda)
+ if cfg.loss_type == "simpo":
+ return simpo_loss(pc, pr, ncn, nrn, beta=cfg.beta, gamma=cfg.simpo_gamma, label_smoothing=cfg.label_smoothing)
with torch.no_grad(), amp_autocast(cfg.amp_dtype, ctx.device):
rsum, _ = _logps(ref, ids, mask, requires_grad=False)
rc, rr = rsum[:B], rsum[B:]
if cfg.loss_type == "kto":
return kto_loss(pc, pr, rc, rr, beta=cfg.beta)
- return dpo_loss(pc, pr, rc, rr, beta=cfg.beta)
+ if cfg.loss_type == "ipo":
+ return ipo_loss(pc, pr, rc, rr, ncn, nrn, beta=cfg.beta)
+ return dpo_loss(pc, pr, rc, rr, beta=cfg.beta, label_smoothing=cfg.label_smoothing)
@torch.no_grad()
@@ -83,7 +92,7 @@ def main():
set_seed(cfg.seed + ctx.rank)
policy = load_backbone_from_ckpt(cfg, cfg.sft_ckpt, ctx.device)
- ref = make_frozen_copy(policy, device=ctx.device) if cfg.loss_type != "orpo" else None
+ ref = make_frozen_copy(policy, device=ctx.device) if cfg.loss_type not in REFERENCE_FREE else None
policy = ddp_wrap(policy, ctx)
optimizer = configure_optimizer(unwrap(policy), cfg.lr, cfg.weight_decay)
diff --git a/src/post_training/dpo.py b/src/post_training/dpo.py
index 751f4c8..14d36bc 100644
--- a/src/post_training/dpo.py
+++ b/src/post_training/dpo.py
@@ -1,57 +1,130 @@
"""
-Direct Preference Optimization and two reference-free / unpaired variants.
+Direct Preference Optimization and its main variants.
All operate on sequence-level log-probabilities of the chosen/rejected responses (summed
-over response tokens via :func:`src.post_training.rollout.sequence_logprobs`).
-
-- ``dpo_loss`` : the standard DPO objective; aligns the policy to preferences using a
- frozen reference model and temperature ``beta``.
-- ``orpo_loss`` : ORPO -- reference-FREE; combines the SFT NLL on the chosen response with
- an odds-ratio preference term (folds SFT + alignment into one stage).
-- ``kto_loss`` : KTO -- works from a per-example desirable/undesirable signal (here read
- off the chosen/rejected pair) with a reference KL baseline.
+over response tokens via :func:`src.post_training.rollout.sequence_logprobs`), so they share
+one trainer (``scripts/train_dpo.py --loss_type ...``).
+
+- ``dpo_loss`` : the standard DPO objective (Rafailov et al. 2023); aligns the policy to
+ preferences using a frozen reference model and temperature ``beta``.
+ ``label_smoothing > 0`` gives conservative DPO for noisy labels.
+- ``ipo_loss`` : IPO (Azar et al. 2023); regresses the preference gap to a fixed target
+ instead of pushing it to infinity, so it does not overfit easy pairs.
+- ``simpo_loss`` : SimPO (Meng et al. 2024); reference-FREE, uses the *average* log-prob per
+ token as the reward (no bias toward long answers) plus a target margin.
+- ``orpo_loss`` : ORPO -- reference-FREE; combines the SFT NLL on the chosen response with
+ an odds-ratio preference term (folds SFT + alignment into one stage).
+- ``kto_loss`` : KTO -- works from a per-example desirable/undesirable signal (here read
+ off the chosen/rejected pair) with a reference KL baseline.
+
+Every function returns ``(loss, chosen_reward, rejected_reward)``; the rewards are detached
+diagnostics, and "implicit accuracy" is how often chosen_reward > rejected_reward.
"""
from __future__ import annotations
import torch
import torch.nn.functional as F
+from jaxtyping import Float
+from torch import Tensor
+
+Batch = Float[Tensor, " batch"]
+LossAndRewards = tuple[Float[Tensor, ""], Batch, Batch]
+
+
+def _preference_nll(logits: Batch, label_smoothing: float = 0.0) -> Float[Tensor, ""]:
+ """``-log sigmoid(logits)``, optionally hedged against flipped labels (conservative DPO)."""
+ if label_smoothing == 0.0:
+ return -F.logsigmoid(logits).mean()
+ return (-(1 - label_smoothing) * F.logsigmoid(logits) - label_smoothing * F.logsigmoid(-logits)).mean()
def dpo_loss(
- policy_chosen_logps: torch.Tensor,
- policy_rejected_logps: torch.Tensor,
- ref_chosen_logps: torch.Tensor,
- ref_rejected_logps: torch.Tensor,
+ policy_chosen_logps: Batch,
+ policy_rejected_logps: Batch,
+ ref_chosen_logps: Batch,
+ ref_rejected_logps: Batch,
beta: float = 0.1,
-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+ label_smoothing: float = 0.0,
+) -> LossAndRewards:
"""
Standard DPO loss. Inputs are summed response log-probs (B,).
+ ``L = -log sigmoid(beta * [(pi_c - pi_r) - (ref_c - ref_r)])``. With
+ ``label_smoothing = eps`` the loss assumes each label is flipped with probability eps
+ (conservative DPO), which keeps the model from becoming overconfident on noisy pairs.
+
Returns ``(loss, chosen_reward, rejected_reward)`` where the implicit rewards
``beta * (policy_logp - ref_logp)`` are detached diagnostics.
"""
pi_logratios = policy_chosen_logps - policy_rejected_logps
ref_logratios = ref_chosen_logps - ref_rejected_logps
logits = pi_logratios - ref_logratios
- loss = -F.logsigmoid(beta * logits).mean()
+ loss = _preference_nll(beta * logits, label_smoothing)
+ chosen_reward = beta * (policy_chosen_logps - ref_chosen_logps).detach()
+ rejected_reward = beta * (policy_rejected_logps - ref_rejected_logps).detach()
+ return loss, chosen_reward, rejected_reward
+
+
+def ipo_loss(
+ policy_chosen_logps: Batch,
+ policy_rejected_logps: Batch,
+ ref_chosen_logps: Batch,
+ ref_rejected_logps: Batch,
+ chosen_n_tokens: Batch,
+ rejected_n_tokens: Batch,
+ beta: float = 0.1,
+) -> LossAndRewards:
+ """
+ IPO: ``L = (h - 1 / (2 beta))^2`` with ``h`` the DPO log-ratio gap on per-token averages.
+
+ DPO keeps pushing ``h`` up even after a pair is clearly separated (the sigmoid never
+ saturates to zero loss), which can overfit when preferences are near-deterministic. IPO
+ asks for a fixed gap of ``1 / (2 beta)`` and stops there.
+ """
+ nc, nr = chosen_n_tokens.clamp(min=1), rejected_n_tokens.clamp(min=1)
+ h = (policy_chosen_logps / nc - policy_rejected_logps / nr) - (ref_chosen_logps / nc - ref_rejected_logps / nr)
+ loss = ((h - 1.0 / (2.0 * beta)) ** 2).mean()
chosen_reward = beta * (policy_chosen_logps - ref_chosen_logps).detach()
rejected_reward = beta * (policy_rejected_logps - ref_rejected_logps).detach()
return loss, chosen_reward, rejected_reward
-def _log1mexp(x: torch.Tensor) -> torch.Tensor:
+def simpo_loss(
+ policy_chosen_logps: Batch,
+ policy_rejected_logps: Batch,
+ chosen_n_tokens: Batch,
+ rejected_n_tokens: Batch,
+ beta: float = 2.0,
+ gamma: float = 0.5,
+ label_smoothing: float = 0.0,
+) -> LossAndRewards:
+ """
+ SimPO: ``L = -log sigmoid(beta * avg_logp(chosen) - beta * avg_logp(rejected) - gamma)``.
+
+ Two differences from DPO: no reference model (cheaper, one model in memory), and the
+ reward is the *average* log-prob per token, which removes DPO's pull toward longer
+ answers. ``gamma`` asks for a minimum reward margin. SimPO uses a larger ``beta``
+ (about 2.0 to 2.5) than DPO because the average log-prob is a much smaller number.
+ """
+ chosen_reward = beta * policy_chosen_logps / chosen_n_tokens.clamp(min=1)
+ rejected_reward = beta * policy_rejected_logps / rejected_n_tokens.clamp(min=1)
+ loss = _preference_nll(chosen_reward - rejected_reward - gamma, label_smoothing)
+ return loss, chosen_reward.detach(), rejected_reward.detach()
+
+
+def _log1mexp(x: Batch) -> Batch:
"""Numerically stable log(1 - exp(x)) for x < 0."""
return torch.where(x > -0.6931, torch.log(-torch.expm1(x)), torch.log1p(-torch.exp(x)))
def orpo_loss(
- policy_chosen_logps: torch.Tensor,
- policy_rejected_logps: torch.Tensor,
- chosen_n_tokens: torch.Tensor,
- rejected_n_tokens: torch.Tensor,
+ policy_chosen_logps: Batch,
+ policy_rejected_logps: Batch,
+ chosen_n_tokens: Batch,
+ rejected_n_tokens: Batch,
orpo_lambda: float = 1.0,
-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+) -> LossAndRewards:
"""
ORPO (reference-free). Uses per-token MEAN log-probs.
@@ -69,14 +142,14 @@ def orpo_loss(
def kto_loss(
- policy_chosen_logps: torch.Tensor,
- policy_rejected_logps: torch.Tensor,
- ref_chosen_logps: torch.Tensor,
- ref_rejected_logps: torch.Tensor,
+ policy_chosen_logps: Batch,
+ policy_rejected_logps: Batch,
+ ref_chosen_logps: Batch,
+ ref_rejected_logps: Batch,
beta: float = 0.1,
desirable_weight: float = 1.0,
undesirable_weight: float = 1.0,
-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:
+) -> LossAndRewards:
"""
KTO from paired data: chosen = desirable, rejected = undesirable, with a reference-KL
baseline estimated (detached) from the batch's mean log-ratio.
@@ -90,6 +163,6 @@ def kto_loss(
return loss, (beta * chosen_logratio).detach(), (beta * rejected_logratio).detach()
-def implicit_accuracy(chosen_reward: torch.Tensor, rejected_reward: torch.Tensor) -> torch.Tensor:
- """Fraction of pairs where the implicit (DPO/KTO/ORPO) reward prefers chosen."""
+def implicit_accuracy(chosen_reward: Batch, rejected_reward: Batch) -> Float[Tensor, ""]:
+ """Fraction of pairs where the implicit (DPO/IPO/SimPO/KTO/ORPO) reward prefers chosen."""
return (chosen_reward > rejected_reward).float().mean()
diff --git a/tests/test_post_training_losses.py b/tests/test_post_training_losses.py
new file mode 100644
index 0000000..de0cabb
--- /dev/null
+++ b/tests/test_post_training_losses.py
@@ -0,0 +1,50 @@
+"""The preference-optimization family: DPO, IPO, SimPO and conservative DPO."""
+
+from __future__ import annotations
+
+import pytest
+import torch
+import torch.nn.functional as F
+
+from src.post_training.dpo import dpo_loss, implicit_accuracy, ipo_loss, simpo_loss
+
+
+def _pairs():
+ torch.manual_seed(0)
+ pc, pr, rc, rr = (torch.randn(6) * 5 - 20 for _ in range(4))
+ n = torch.full((6,), 10.0)
+ return pc, pr, rc, rr, n
+
+
+def test_dpo_matches_the_formula_and_label_smoothing_hedges() -> None:
+ pc, pr, rc, rr, _ = _pairs()
+ loss, cr, rj = dpo_loss(pc, pr, rc, rr, beta=0.1)
+ logits = 0.1 * ((pc - pr) - (rc - rr))
+ assert torch.allclose(loss, -F.logsigmoid(logits).mean())
+ assert torch.equal(implicit_accuracy(cr, rj), ((pc - rc) > (pr - rr)).float().mean())
+ # conservative DPO: with eps = 0.5 the two directions cancel and the loss stops depending on the order
+ smoothed, _, _ = dpo_loss(pc, pr, rc, rr, beta=0.1, label_smoothing=0.3)
+ flipped, _, _ = dpo_loss(pr, pc, rr, rc, beta=0.1, label_smoothing=0.3)
+ plain_flipped, _, _ = dpo_loss(pr, pc, rr, rc, beta=0.1)
+ assert abs(smoothed - flipped) < abs(loss - plain_flipped)
+
+
+def test_ipo_is_zero_at_its_target_gap() -> None:
+ beta = 0.25
+ n = torch.ones(1)
+ target = 1.0 / (2 * beta) # IPO wants exactly this gap, not an infinite one
+ loss, _, _ = ipo_loss(torch.tensor([target]), torch.zeros(1), torch.zeros(1), torch.zeros(1), n, n, beta)
+ assert loss.item() == pytest.approx(0.0, abs=1e-7)
+ bigger, _, _ = ipo_loss(torch.tensor([target + 3]), torch.zeros(1), torch.zeros(1), torch.zeros(1), n, n, beta)
+ assert bigger.item() > 1.0 # overshooting is penalized too, unlike DPO
+
+
+def test_simpo_is_reference_free_and_length_normalized() -> None:
+ pc, pr, _, _, n = _pairs()
+ loss, cr, rj = simpo_loss(pc, pr, n, n, beta=2.0, gamma=0.5)
+ expected = -F.logsigmoid(2.0 * pc / 10 - 2.0 * pr / 10 - 0.5).mean()
+ assert torch.allclose(loss, expected)
+ # twice as long answers with twice the total log-prob get the same reward
+ loss_long, _, _ = simpo_loss(2 * pc, 2 * pr, 2 * n, 2 * n, beta=2.0, gamma=0.5)
+ assert torch.allclose(loss, loss_long)
+