A straightforward method for training your LLM, from downloading data to generating text.
feat(dpo): IPO, SimPO and conservative DPO
FareedKhan-devaabecd25aec3
diff --git a/config/post_training_config.py b/config/post_training_config.pyindex d6b2a66..d6cb09a 100644--- a/config/post_training_config.py+++ b/config/post_training_config.py@@ -29,7 +29,7 @@ AttentionKind = Literal["gqa", "mla"]AmpDtype = Literal["bf16", "fp16"]OptimizerName = Literal["adamw", "muon"]Schedule = Literal["cosine", "wsd", "linear"]-PreferenceLoss = Literal["dpo", "orpo", "kto"]+PreferenceLoss = Literal["dpo", "ipo", "simpo", "orpo", "kto"]RewardSource = Literal["verifier", "rm"]@@ -161,9 +161,11 @@ class DPOConfig(BaseModelConfig):pref_path: str = f"{DATA_DIR}/preferences.jsonl"test_path: str = f"{DATA_DIR}/preferences_test.jsonl"out_ckpt: str = f"{CKPT_DIR}/dpo.pt"- loss_type: PreferenceLoss = "dpo" # dpo | orpo | kto+ loss_type: PreferenceLoss = "dpo" # dpo | ipo | simpo | orpo | ktobeta: float = 0.1- orpo_lambda: float = 1.0 # ORPO odds-ratio weight (loss_type="orpo")+ label_smoothing: float = 0.0 # > 0 = conservative DPO for noisy preference labels+ simpo_gamma: float = 0.5 # SimPO target reward margin+ orpo_lambda: float = 1.0 # ORPO odds-ratio weight (loss_type="orpo")batch_size: int = 8epochs: int = 1eval_steps: int = 200@@ -177,6 +179,7 @@ class DPOConfig(BaseModelConfig):def __post_init__(self) -> None:super().__post_init__()_check(self.beta > 0, "beta must be positive")+ _check(0.0 <= self.label_smoothing < 0.5, "label_smoothing must be in [0, 0.5)")_check(16 <= self.max_len <= self.context_length, "max_len must be between 16 and context_length")diff --git a/configs/dpo.json b/configs/dpo.jsonindex d9e32a8..caf028d 100644--- a/configs/dpo.json+++ b/configs/dpo.json@@ -5,6 +5,8 @@"out_ckpt": "models/dpo.pt","loss_type": "dpo","beta": 0.1,+ "label_smoothing": 0.0,+ "simpo_gamma": 0.5,"orpo_lambda": 1.0,"batch_size": 8,"epochs": 1,diff --git a/scripts/train_dpo.py b/scripts/train_dpo.pyindex 8a80c19..b5be772 100644--- a/scripts/train_dpo.py+++ b/scripts/train_dpo.py@@ -1,11 +1,14 @@"""-Direct Preference Optimization (and ORPO / KTO variants) on preference pairs.+Direct Preference Optimization and its variants (IPO, SimPO, ORPO, KTO) on preference pairs.The policy is initialized from the SFT checkpoint; a frozen deep copy of it serves as the-DPO/KTO reference (ORPO is reference-free). Reports implicit-reward accuracy on held-out-preferences and GSM8K dev accuracy.+reference for DPO / IPO / KTO (SimPO and ORPO are reference-free, so no copy is made).+Reports implicit-reward accuracy on held-out preferences.python scripts/train_dpo.py --loss_type dpo --beta 0.1+ python scripts/train_dpo.py --loss_type dpo --label_smoothing 0.1 # conservative DPO+ python scripts/train_dpo.py --loss_type ipo --beta 0.1+ python scripts/train_dpo.py --loss_type simpo --beta 2.0 --simpo_gamma 0.5torchrun --standalone --nproc_per_node=2 scripts/train_dpo.py"""@@ -25,7 +28,7 @@ from config.post_training_config import DPOConfigfrom data_loader.preference_dataset import get_preference_iteratorfrom src.post_training.cli import parse_config_with_jsonfrom src.post_training.distributed import ddp_setup, ddp_wrap, cleanup, reduce_scalar-from src.post_training.dpo import dpo_loss, orpo_loss, kto_loss, implicit_accuracy+from src.post_training.dpo import dpo_loss, implicit_accuracy, ipo_loss, kto_loss, orpo_loss, simpo_lossfrom src.post_training.logging_utils import MetricsLoggerfrom src.post_training.optim import configure_optimizer, cosine_lr, set_lrfrom src.post_training.rollout import sequence_logprobs@@ -33,6 +36,8 @@ from src.post_training.utils import (amp_autocast, load_backbone_from_ckpt, make_frozen_copy, save_stage_ckpt, set_seed, unwrap,)+REFERENCE_FREE = {"orpo", "simpo"}+def _logps(model, ids, mask, requires_grad):return sequence_logprobs(model, ids, mask, requires_grad=requires_grad)@@ -49,13 +54,17 @@ def _compute_losses(policy, ref, batch, cfg, ctx):if cfg.loss_type == "orpo":return orpo_loss(pc, pr, ncn, nrn, orpo_lambda=cfg.orpo_lambda)+ if cfg.loss_type == "simpo":+ return simpo_loss(pc, pr, ncn, nrn, beta=cfg.beta, gamma=cfg.simpo_gamma, label_smoothing=cfg.label_smoothing)with torch.no_grad(), amp_autocast(cfg.amp_dtype, ctx.device):rsum, _ = _logps(ref, ids, mask, requires_grad=False)rc, rr = rsum[:B], rsum[B:]if cfg.loss_type == "kto":return kto_loss(pc, pr, rc, rr, beta=cfg.beta)- return dpo_loss(pc, pr, rc, rr, beta=cfg.beta)+ if cfg.loss_type == "ipo":+ return ipo_loss(pc, pr, rc, rr, ncn, nrn, beta=cfg.beta)+ return dpo_loss(pc, pr, rc, rr, beta=cfg.beta, label_smoothing=cfg.label_smoothing)@torch.no_grad()@@ -83,7 +92,7 @@ def main():set_seed(cfg.seed + ctx.rank)policy = load_backbone_from_ckpt(cfg, cfg.sft_ckpt, ctx.device)- ref = make_frozen_copy(policy, device=ctx.device) if cfg.loss_type != "orpo" else None+ ref = make_frozen_copy(policy, device=ctx.device) if cfg.loss_type not in REFERENCE_FREE else Nonepolicy = ddp_wrap(policy, ctx)optimizer = configure_optimizer(unwrap(policy), cfg.lr, cfg.weight_decay)diff --git a/src/post_training/dpo.py b/src/post_training/dpo.pyindex 751f4c8..14d36bc 100644--- a/src/post_training/dpo.py+++ b/src/post_training/dpo.py@@ -1,57 +1,130 @@"""-Direct Preference Optimization and two reference-free / unpaired variants.+Direct Preference Optimization and its main variants.All operate on sequence-level log-probabilities of the chosen/rejected responses (summed-over response tokens via :func:`src.post_training.rollout.sequence_logprobs`).--- ``dpo_loss`` : the standard DPO objective; aligns the policy to preferences using a- frozen reference model and temperature ``beta``.-- ``orpo_loss`` : ORPO -- reference-FREE; combines the SFT NLL on the chosen response with- an odds-ratio preference term (folds SFT + alignment into one stage).-- ``kto_loss`` : KTO -- works from a per-example desirable/undesirable signal (here read- off the chosen/rejected pair) with a reference KL baseline.+over response tokens via :func:`src.post_training.rollout.sequence_logprobs`), so they share+one trainer (``scripts/train_dpo.py --loss_type ...``).++- ``dpo_loss`` : the standard DPO objective (Rafailov et al. 2023); aligns the policy to+ preferences using a frozen reference model and temperature ``beta``.+ ``label_smoothing > 0`` gives conservative DPO for noisy labels.+- ``ipo_loss`` : IPO (Azar et al. 2023); regresses the preference gap to a fixed target+ instead of pushing it to infinity, so it does not overfit easy pairs.+- ``simpo_loss`` : SimPO (Meng et al. 2024); reference-FREE, uses the *average* log-prob per+ token as the reward (no bias toward long answers) plus a target margin.+- ``orpo_loss`` : ORPO -- reference-FREE; combines the SFT NLL on the chosen response with+ an odds-ratio preference term (folds SFT + alignment into one stage).+- ``kto_loss`` : KTO -- works from a per-example desirable/undesirable signal (here read+ off the chosen/rejected pair) with a reference KL baseline.++Every function returns ``(loss, chosen_reward, rejected_reward)``; the rewards are detached+diagnostics, and "implicit accuracy" is how often chosen_reward > rejected_reward."""from __future__ import annotationsimport torchimport torch.nn.functional as F+from jaxtyping import Float+from torch import Tensor++Batch = Float[Tensor, " batch"]+LossAndRewards = tuple[Float[Tensor, ""], Batch, Batch]+++def _preference_nll(logits: Batch, label_smoothing: float = 0.0) -> Float[Tensor, ""]:+ """``-log sigmoid(logits)``, optionally hedged against flipped labels (conservative DPO)."""+ if label_smoothing == 0.0:+ return -F.logsigmoid(logits).mean()+ return (-(1 - label_smoothing) * F.logsigmoid(logits) - label_smoothing * F.logsigmoid(-logits)).mean()def dpo_loss(- policy_chosen_logps: torch.Tensor,- policy_rejected_logps: torch.Tensor,- ref_chosen_logps: torch.Tensor,- ref_rejected_logps: torch.Tensor,+ policy_chosen_logps: Batch,+ policy_rejected_logps: Batch,+ ref_chosen_logps: Batch,+ ref_rejected_logps: Batch,beta: float = 0.1,-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+ label_smoothing: float = 0.0,+) -> LossAndRewards:"""Standard DPO loss. Inputs are summed response log-probs (B,).+ ``L = -log sigmoid(beta * [(pi_c - pi_r) - (ref_c - ref_r)])``. With+ ``label_smoothing = eps`` the loss assumes each label is flipped with probability eps+ (conservative DPO), which keeps the model from becoming overconfident on noisy pairs.+Returns ``(loss, chosen_reward, rejected_reward)`` where the implicit rewards``beta * (policy_logp - ref_logp)`` are detached diagnostics."""pi_logratios = policy_chosen_logps - policy_rejected_logpsref_logratios = ref_chosen_logps - ref_rejected_logpslogits = pi_logratios - ref_logratios- loss = -F.logsigmoid(beta * logits).mean()+ loss = _preference_nll(beta * logits, label_smoothing)+ chosen_reward = beta * (policy_chosen_logps - ref_chosen_logps).detach()+ rejected_reward = beta * (policy_rejected_logps - ref_rejected_logps).detach()+ return loss, chosen_reward, rejected_reward+++def ipo_loss(+ policy_chosen_logps: Batch,+ policy_rejected_logps: Batch,+ ref_chosen_logps: Batch,+ ref_rejected_logps: Batch,+ chosen_n_tokens: Batch,+ rejected_n_tokens: Batch,+ beta: float = 0.1,+) -> LossAndRewards:+ """+ IPO: ``L = (h - 1 / (2 beta))^2`` with ``h`` the DPO log-ratio gap on per-token averages.++ DPO keeps pushing ``h`` up even after a pair is clearly separated (the sigmoid never+ saturates to zero loss), which can overfit when preferences are near-deterministic. IPO+ asks for a fixed gap of ``1 / (2 beta)`` and stops there.+ """+ nc, nr = chosen_n_tokens.clamp(min=1), rejected_n_tokens.clamp(min=1)+ h = (policy_chosen_logps / nc - policy_rejected_logps / nr) - (ref_chosen_logps / nc - ref_rejected_logps / nr)+ loss = ((h - 1.0 / (2.0 * beta)) ** 2).mean()chosen_reward = beta * (policy_chosen_logps - ref_chosen_logps).detach()rejected_reward = beta * (policy_rejected_logps - ref_rejected_logps).detach()return loss, chosen_reward, rejected_reward-def _log1mexp(x: torch.Tensor) -> torch.Tensor:+def simpo_loss(+ policy_chosen_logps: Batch,+ policy_rejected_logps: Batch,+ chosen_n_tokens: Batch,+ rejected_n_tokens: Batch,+ beta: float = 2.0,+ gamma: float = 0.5,+ label_smoothing: float = 0.0,+) -> LossAndRewards:+ """+ SimPO: ``L = -log sigmoid(beta * avg_logp(chosen) - beta * avg_logp(rejected) - gamma)``.++ Two differences from DPO: no reference model (cheaper, one model in memory), and the+ reward is the *average* log-prob per token, which removes DPO's pull toward longer+ answers. ``gamma`` asks for a minimum reward margin. SimPO uses a larger ``beta``+ (about 2.0 to 2.5) than DPO because the average log-prob is a much smaller number.+ """+ chosen_reward = beta * policy_chosen_logps / chosen_n_tokens.clamp(min=1)+ rejected_reward = beta * policy_rejected_logps / rejected_n_tokens.clamp(min=1)+ loss = _preference_nll(chosen_reward - rejected_reward - gamma, label_smoothing)+ return loss, chosen_reward.detach(), rejected_reward.detach()+++def _log1mexp(x: Batch) -> Batch:"""Numerically stable log(1 - exp(x)) for x < 0."""return torch.where(x > -0.6931, torch.log(-torch.expm1(x)), torch.log1p(-torch.exp(x)))def orpo_loss(- policy_chosen_logps: torch.Tensor,- policy_rejected_logps: torch.Tensor,- chosen_n_tokens: torch.Tensor,- rejected_n_tokens: torch.Tensor,+ policy_chosen_logps: Batch,+ policy_rejected_logps: Batch,+ chosen_n_tokens: Batch,+ rejected_n_tokens: Batch,orpo_lambda: float = 1.0,-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+) -> LossAndRewards:"""ORPO (reference-free). Uses per-token MEAN log-probs.@@ -69,14 +142,14 @@ def orpo_loss(def kto_loss(- policy_chosen_logps: torch.Tensor,- policy_rejected_logps: torch.Tensor,- ref_chosen_logps: torch.Tensor,- ref_rejected_logps: torch.Tensor,+ policy_chosen_logps: Batch,+ policy_rejected_logps: Batch,+ ref_chosen_logps: Batch,+ ref_rejected_logps: Batch,beta: float = 0.1,desirable_weight: float = 1.0,undesirable_weight: float = 1.0,-) -> tuple[torch.Tensor, torch.Tensor, torch.Tensor]:+) -> LossAndRewards:"""KTO from paired data: chosen = desirable, rejected = undesirable, with a reference-KLbaseline estimated (detached) from the batch's mean log-ratio.@@ -90,6 +163,6 @@ def kto_loss(return loss, (beta * chosen_logratio).detach(), (beta * rejected_logratio).detach()-def implicit_accuracy(chosen_reward: torch.Tensor, rejected_reward: torch.Tensor) -> torch.Tensor:- """Fraction of pairs where the implicit (DPO/KTO/ORPO) reward prefers chosen."""+def implicit_accuracy(chosen_reward: Batch, rejected_reward: Batch) -> Float[Tensor, ""]:+ """Fraction of pairs where the implicit (DPO/IPO/SimPO/KTO/ORPO) reward prefers chosen."""return (chosen_reward > rejected_reward).float().mean()diff --git a/tests/test_post_training_losses.py b/tests/test_post_training_losses.pynew file mode 100644index 0000000..de0cabb--- /dev/null+++ b/tests/test_post_training_losses.py@@ -0,0 +1,50 @@+"""The preference-optimization family: DPO, IPO, SimPO and conservative DPO."""++from __future__ import annotations++import pytest+import torch+import torch.nn.functional as F++from src.post_training.dpo import dpo_loss, implicit_accuracy, ipo_loss, simpo_loss+++def _pairs():+ torch.manual_seed(0)+ pc, pr, rc, rr = (torch.randn(6) * 5 - 20 for _ in range(4))+ n = torch.full((6,), 10.0)+ return pc, pr, rc, rr, n+++def test_dpo_matches_the_formula_and_label_smoothing_hedges() -> None:+ pc, pr, rc, rr, _ = _pairs()+ loss, cr, rj = dpo_loss(pc, pr, rc, rr, beta=0.1)+ logits = 0.1 * ((pc - pr) - (rc - rr))+ assert torch.allclose(loss, -F.logsigmoid(logits).mean())+ assert torch.equal(implicit_accuracy(cr, rj), ((pc - rc) > (pr - rr)).float().mean())+ # conservative DPO: with eps = 0.5 the two directions cancel and the loss stops depending on the order+ smoothed, _, _ = dpo_loss(pc, pr, rc, rr, beta=0.1, label_smoothing=0.3)+ flipped, _, _ = dpo_loss(pr, pc, rr, rc, beta=0.1, label_smoothing=0.3)+ plain_flipped, _, _ = dpo_loss(pr, pc, rr, rc, beta=0.1)+ assert abs(smoothed - flipped) < abs(loss - plain_flipped)+++def test_ipo_is_zero_at_its_target_gap() -> None:+ beta = 0.25+ n = torch.ones(1)+ target = 1.0 / (2 * beta) # IPO wants exactly this gap, not an infinite one+ loss, _, _ = ipo_loss(torch.tensor([target]), torch.zeros(1), torch.zeros(1), torch.zeros(1), n, n, beta)+ assert loss.item() == pytest.approx(0.0, abs=1e-7)+ bigger, _, _ = ipo_loss(torch.tensor([target + 3]), torch.zeros(1), torch.zeros(1), torch.zeros(1), n, n, beta)+ assert bigger.item() > 1.0 # overshooting is penalized too, unlike DPO+++def test_simpo_is_reference_free_and_length_normalized() -> None:+ pc, pr, _, _, n = _pairs()+ loss, cr, rj = simpo_loss(pc, pr, n, n, beta=2.0, gamma=0.5)+ expected = -F.logsigmoid(2.0 * pc / 10 - 2.0 * pr / 10 - 0.5).mean()+ assert torch.allclose(loss, expected)+ # twice as long answers with twice the total log-prob get the same reward+ loss_long, _, _ = simpo_loss(2 * pc, 2 * pr, 2 * n, 2 * n, beta=2.0, gamma=0.5)+ assert torch.allclose(loss, loss_long)+