Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

7.0 KB
Raw Blame
"""
Direct Preference Optimization and its variants (IPO, SimPO, ORPO, KTO) on preference pairs.

The policy is initialized from the SFT checkpoint; a frozen deep copy of it serves as the
reference for DPO / IPO / KTO (SimPO and ORPO are reference-free, so no copy is made).
Reports implicit-reward accuracy on held-out preferences.

    python scripts/train_dpo.py --loss_type dpo --beta 0.1
    python scripts/train_dpo.py --loss_type dpo --label_smoothing 0.1     # conservative DPO
    python scripts/train_dpo.py --loss_type ipo --beta 0.1
    python scripts/train_dpo.py --loss_type simpo --beta 2.0 --simpo_gamma 0.5
    torchrun --standalone --nproc_per_node=2 scripts/train_dpo.py
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # run from the repo without installing

import os
import time

import torch

from config.post_training_config import DPOConfig
from data_loader.preference_dataset import get_preference_iterator
from src.post_training.cli import parse_config_with_json
from src.post_training.distributed import cleanup, ddp_setup, ddp_wrap, reduce_scalar
from src.post_training.dpo import dpo_loss, implicit_accuracy, ipo_loss, kto_loss, orpo_loss, simpo_loss
from src.post_training.logging_utils import MetricsLogger
from src.post_training.optim import configure_optimizer, cosine_lr, set_lr
from src.post_training.rollout import sequence_logprobs
from src.post_training.utils import (
    amp_autocast,
    load_backbone_from_ckpt,
    make_frozen_copy,
    save_stage_ckpt,
    set_seed,
    unwrap,
)

REFERENCE_FREE = {"orpo", "simpo"}


def _logps(model, ids, mask, requires_grad):
    return sequence_logprobs(model, ids, mask, requires_grad=requires_grad)


def _compute_losses(policy, ref, batch, cfg, ctx):
    B = batch["chosen_ids"].size(0)
    ids = torch.cat([batch["chosen_ids"], batch["rejected_ids"]], dim=0)
    mask = torch.cat([batch["chosen_mask"], batch["rejected_mask"]], dim=0)

    with amp_autocast(cfg.amp_dtype, ctx.device):
        psum, pn = _logps(policy, ids, mask, requires_grad=True)
    pc, pr, ncn, nrn = psum[:B], psum[B:], pn[:B], pn[B:]

    if cfg.loss_type == "orpo":
        return orpo_loss(pc, pr, ncn, nrn, orpo_lambda=cfg.orpo_lambda)
    if cfg.loss_type == "simpo":
        return simpo_loss(pc, pr, ncn, nrn, beta=cfg.beta, gamma=cfg.simpo_gamma, label_smoothing=cfg.label_smoothing)

    with torch.no_grad(), amp_autocast(cfg.amp_dtype, ctx.device):
        rsum, _ = _logps(ref, ids, mask, requires_grad=False)
    rc, rr = rsum[:B], rsum[B:]
    if cfg.loss_type == "kto":
        return kto_loss(pc, pr, rc, rr, beta=cfg.beta)
    if cfg.loss_type == "ipo":
        return ipo_loss(pc, pr, rc, rr, ncn, nrn, beta=cfg.beta)
    return dpo_loss(pc, pr, rc, rr, beta=cfg.beta, label_smoothing=cfg.label_smoothing)


@torch.no_grad()
def eval_implicit_acc(policy, ref, cfg, ctx, max_batches: int = 100) -> tuple[float, float]:
    if not os.path.exists(cfg.test_path):
        return float("nan"), float("nan")
    policy.eval()
    it = get_preference_iterator(cfg.test_path, cfg.batch_size, cfg.max_len, device=ctx.device,
                                 rank=ctx.rank, world_size=ctx.world_size, shuffle=False, infinite=False)
    acc, marg, n = 0.0, 0.0, 0
    for batch in it:
        loss, cr, rr = _compute_losses(policy, ref, batch, cfg, ctx)
        acc += implicit_accuracy(cr, rr).item()
        marg += (cr - rr).mean().item()
        n += 1
        if n >= max_batches:
            break
    policy.train()
    return acc / max(1, n), marg / max(1, n)


def main():
    cfg, _ = parse_config_with_json(DPOConfig, "configs/dpo.json")
    ctx = ddp_setup(cfg.device)
    set_seed(cfg.seed + ctx.rank)

    policy = load_backbone_from_ckpt(cfg, cfg.sft_ckpt, ctx.device)
    ref = make_frozen_copy(policy, device=ctx.device) if cfg.loss_type not in REFERENCE_FREE else None
    policy = ddp_wrap(policy, ctx)
    optimizer = configure_optimizer(unwrap(policy), cfg.lr, cfg.weight_decay)

    with open(cfg.pref_path, encoding="utf-8") as f:
        n_rows = sum(1 for line in f if line.strip())
    total_steps = max(1, (n_rows // (cfg.batch_size * ctx.world_size)) * cfg.epochs)

    logger = None
    if ctx.is_main:
        print(f"DPO[{cfg.loss_type}] from {cfg.sft_ckpt} | {n_rows} pairs | total_steps={total_steps} | beta={cfg.beta}")
        logger = MetricsLogger(f"dpo_{cfg.loss_type}", cfg.log_dir, use_wandb=cfg.use_wandb, wandb_project=cfg.wandb_project)

    train_it = get_preference_iterator(cfg.pref_path, cfg.batch_size, cfg.max_len, device=ctx.device,
                                       rank=ctx.rank, world_size=ctx.world_size, shuffle=True, infinite=True)

    policy.train()
    t0 = time.perf_counter()
    for step in range(total_steps):
        lr = cosine_lr(step, warmup_steps=cfg.warmup_steps, max_steps=total_steps, lr=cfg.lr, min_lr=cfg.lr * 0.1)
        set_lr(optimizer, lr)

        batch = next(train_it)
        loss, cr, rr = _compute_losses(policy, ref, batch, cfg, ctx)
        optimizer.zero_grad(set_to_none=True)
        loss.backward()
        torch.nn.utils.clip_grad_norm_(policy.parameters(), cfg.grad_clip)
        optimizer.step()

        if ctx.is_main and step % 20 == 0:
            acc = implicit_accuracy(cr, rr).item()
            dt = time.perf_counter() - t0; t0 = time.perf_counter()
            print(f"step {step}/{total_steps} | loss {loss.item():.4f} | acc {acc:.3f} | "
                  f"r_chosen {cr.mean().item():.3f} r_rejected {rr.mean().item():.3f} | {dt:.1f}s/20")
            if logger:
                logger.log(step, {"train_loss": loss.item(), "train_acc": acc,
                                  "r_chosen": cr.mean().item(), "r_rejected": rr.mean().item(), "lr": lr})

        if step > 0 and step % cfg.eval_steps == 0:
            acc, marg = eval_implicit_acc(policy, ref, cfg, ctx)
            acc, marg = reduce_scalar(acc, ctx), reduce_scalar(marg, ctx)
            if ctx.is_main:
                print(f"  [eval] step {step} | test_acc {acc:.3f} | margin {marg:.3f}")
                if logger:
                    logger.log(step, {"test_acc": acc, "test_margin": marg})

        if ctx.is_main and step > 0 and step % cfg.save_every == 0:
            save_stage_ckpt(cfg.out_ckpt, policy, optimizer, stage=f"dpo_{cfg.loss_type}", cfg=cfg, step=step,
                            metrics={"train_loss": loss.item()})

    if ctx.is_main:
        # Unwrap for the final eval: other ranks are already at cleanup(), so a collective on
        # the DDP-wrapped policy here would hang (NCCL timeout). The periodic eval runs on all ranks.
        acc, marg = eval_implicit_acc(unwrap(policy), ref, cfg, ctx)
        save_stage_ckpt(cfg.out_ckpt, policy, optimizer, stage=f"dpo_{cfg.loss_type}", cfg=cfg, step=total_steps,
                        metrics={"test_acc": acc, "test_margin": marg})
        print(f"Done DPO[{cfg.loss_type}]. test_acc {acc:.3f} margin {marg:.3f} -> {cfg.out_ckpt}")
        if logger:
            logger.close()
    cleanup(ctx)


if __name__ == "__main__":
    main()