A straightforward method for training your LLM, from downloading data to generating text.
"""
GRPO (Group Relative Policy Optimization) core -- the DeepSeekMath / DeepSeek-R1 RL algorithm,
plus the switches that turn it into the 2025 follow-ups.
GRPO drops PPO's value network. For each prompt it samples a GROUP of G completions,
scores them with a verifiable reward, and uses the group's own mean/std to compute a
relative advantage -- so the baseline is the group, not a learned critic. The update is a
clipped surrogate plus a per-token KL penalty to the reference policy.
The variants differ in a few precise places, each one a keyword here:
================= ==========================================================================
variant what changes (keyword)
================= ==========================================================================
GRPO ``adv_norm="std"``; loss averaged per answer, then over answers
(``loss_agg="seq-mean-token-mean"``, the paper's formula)
DAPO all tokens of the batch averaged together (``loss_agg="token-mean"``), a
larger upper clip ``clip_high`` ("clip-higher", keeps exploration alive),
groups with no reward spread skipped (``filter_groups`` in the trainer),
usually ``kl_coef=0``
Dr. GRPO no division by the group std (``adv_norm="none"``) and a constant length
normalizer (``loss_agg="seq-mean-token-sum-norm"``); both remove biases
toward long wrong answers and toward easy/hard prompts
GSPO one importance ratio per answer, the length-normalized sequence likelihood
ratio (``ratio_level="sequence"``), clipped with a tiny range (~3e-4)
================= ==========================================================================
The trainer default (``token-mean``, symmetric clip, std-normalized advantages) is exactly
what this repo has always run.
"""
from __future__ import annotations
from typing import Literal
import torch
from jaxtyping import Bool, Float
from torch import Tensor
from src.post_training.utils import masked_mean, masked_mean_per_row
AdvantageNorm = Literal["std", "none"]
LossAggregation = Literal["token-mean", "seq-mean-token-mean", "seq-mean-token-sum-norm"]
RatioLevel = Literal["token", "sequence"]
def group_advantages(
rewards: Float[Tensor, " n"], group_size: int, eps: float = 1e-4, scale: AdvantageNorm = "std"
) -> Float[Tensor, " n"]:
"""
Group-relative advantage: ``(r - group_mean) / (group_std + eps)``.
``rewards`` is (num_prompts * group_size,) laid out group-contiguously (all G samples
of prompt 0, then prompt 1, ...). Returns advantages of the same shape. With
``scale="none"`` (Dr. GRPO) the advantage is just ``r - group_mean``: dividing by the std
gives prompts that are almost always solved (or failed) a disproportionately large weight.
"""
r = rewards.view(-1, group_size)
adv = r - r.mean(dim=1, keepdim=True)
if scale == "std":
adv = adv / (r.std(dim=1, keepdim=True) + eps)
return adv.reshape(-1)
def k3_kl(new_logp: Float[Tensor, "*shape"], ref_logp: Float[Tensor, "*shape"]) -> Float[Tensor, "*shape"]:
"""Per-token unbiased, non-negative KL estimator (Schulman's k3) for KL(policy||ref)."""
diff = ref_logp - new_logp
return torch.exp(diff) - diff - 1.0
def _mean_over_answers(per_answer: Float[Tensor, " batch"], mask: Bool[Tensor, "batch steps"]) -> Float[Tensor, ""]:
"""Average over the answers that have at least one trained token.
``--filter_groups`` masks out every token of a skipped group. Counting those empty rows in
the average would shrink the update by the fraction of skipped groups, like a random
learning-rate cut, so they are left out.
"""
active = mask.any(dim=-1).to(per_answer.dtype)
return (per_answer * active).sum() / active.sum().clamp(min=1.0)
def aggregate_token_loss(
per_token: Float[Tensor, "batch steps"],
mask: Bool[Tensor, "batch steps"],
mode: LossAggregation = "token-mean",
max_len: int | None = None,
) -> Float[Tensor, ""]:
"""
Reduce a per-token loss to one number. The choice matters more than it looks:
- ``token-mean``: every token in the batch counts the same. Long answers get more say,
which is what DAPO wants (long wrong answers are penalized in full).
- ``seq-mean-token-mean``: average inside each answer, then across answers. Every answer
counts the same, so each token of a long answer counts less. This is the GRPO paper.
- ``seq-mean-token-sum-norm``: sum inside each answer, divide by a constant (the generation
budget ``max_len``), then average. Dr. GRPO: unbiased and still length-independent.
Answers whose mask is empty (groups skipped by ``--filter_groups``) are left out of the
per-answer averages.
"""
m = mask.to(per_token.dtype)
if mode == "token-mean":
return masked_mean(per_token, m)
if mode == "seq-mean-token-mean":
return _mean_over_answers(masked_mean_per_row(per_token, m), mask)
if mode == "seq-mean-token-sum-norm":
return _mean_over_answers((per_token * m).sum(dim=-1) / float(max_len or per_token.size(-1)), mask)
raise ValueError(f"unknown loss aggregation {mode!r}")
def grpo_loss(
new_logp: Float[Tensor, "batch steps"],
old_logp: Float[Tensor, "batch steps"],
ref_logp: Float[Tensor, "batch steps"],
advantages: Float[Tensor, " batch"],
resp_mask: Bool[Tensor, "batch steps"],
clip: float = 0.2,
kl_coef: float = 0.04,
*,
clip_high: float | None = None,
loss_agg: LossAggregation = "token-mean",
ratio_level: RatioLevel = "token",
max_len: int | None = None,
) -> tuple[Float[Tensor, ""], dict[str, float]]:
"""
Clipped surrogate + KL penalty, with the variant switches described in the module docstring.
Args:
new_logp/old_logp/ref_logp: (B, L) per-token log-probs (policy / sampling / ref).
advantages: (B,) one scalar per completion, broadcast over its tokens.
resp_mask: (B, L) bool over response tokens.
clip, clip_high: the ratio may move in ``[1 - clip, 1 + clip_high]`` (``clip_high``
defaults to ``clip``).
loss_agg: how per-token terms are averaged (token-level ratio only).
ratio_level: ``"token"`` (GRPO/DAPO) or ``"sequence"`` (GSPO).
max_len: the constant normalizer for ``seq-mean-token-sum-norm``.
Returns:
(loss, stats) with mean KL and clip fraction for logging.
"""
high = clip if clip_high is None else clip_high
adv = advantages[:, None]
kl = k3_kl(new_logp, ref_logp)
if ratio_level == "sequence":
# GSPO: s_i = exp(mean_t log(pi/pi_old)), one ratio for the whole answer.
log_ratio = masked_mean_per_row(new_logp - old_logp, resp_mask)
ratio = torch.exp(log_ratio)
surrogate = torch.min(ratio * advantages, torch.clamp(ratio, 1.0 - clip, 1.0 + high) * advantages)
loss = -_mean_over_answers(surrogate - kl_coef * masked_mean_per_row(kl, resp_mask), resp_mask)
clipped = _mean_over_answers(((ratio < 1.0 - clip) | (ratio > 1.0 + high)).float(), resp_mask)
else:
ratio = torch.exp(new_logp - old_logp)
surrogate = torch.min(ratio * adv, torch.clamp(ratio, 1.0 - clip, 1.0 + high) * adv)
loss = -aggregate_token_loss(surrogate - kl_coef * kl, resp_mask, loss_agg, max_len)
clipped = masked_mean(((ratio < 1.0 - clip) | (ratio > 1.0 + high)).float(), resp_mask)
stats = {"kl": masked_mean(kl, resp_mask).item(), "clipfrac": clipped.item()}
return loss, stats