Fine-tune LLMs from one YAML. Layer streaming trains an 8B model on a 4 GB laptop GPU.
Python cliconsumer-gpudpofine-tuningggufhuggingfacellmllmopslocal-ailocal-llmloralow-vrammachine-learningollamapeftpythonpytorchqlorasfttransformers
"""GRPO objective variants — v0.50.0 Part A.
Closed allowlist of GRPO-family RL objectives shipped for parity with
unsloth + axolotl. Schema-load-time validation + metadata only in v0.50.0;
live loss-function wiring lands in v0.50.1 (mirrors v0.27.0 MII /
v0.37.0 multipack / v0.41.0 LLaMA Pro / v0.45.0 plugins / v0.48.0 curriculum
stub-then-live pattern).
Variants:
- gspo : Group Sequence Policy Optimization (Qwen)
- dapo : Decoupled Advantage Policy Optimization (unsloth, axolotl)
- dr_grpo : Doubly Robust GRPO (unsloth, axolotl)
- bnpo : Batch Normalized Policy Optimization (unsloth)
- two_sided : Symmetric clipping w/ delta (unsloth)
- rft : Reinforced Fine-Tuning (unsloth)
- standard : Default GRPO (DeepSeek-R1 style — legacy alias)
Security:
- Closed allowlist; arbitrary string at schema level rejected.
- Empty / null-byte / non-string / oversize rejected.
- `_VARIANT_METADATA` wrapped in MappingProxyType (matches v0.36.0
_REGISTRY / v0.41.0 _OPTIMIZER_PACKAGES policy).
"""
from __future__ import annotations
import math
import types
from dataclasses import dataclass
_MAX_VARIANT_NAME_LEN = 32
SUPPORTED_GRPO_VARIANTS: frozenset[str] = frozenset({
"gspo",
"dapo",
"dr_grpo",
"bnpo",
"two_sided",
"rft",
"standard",
})
# Variants that require an explicit delta (symmetric clipping radius).
_REQUIRES_DELTA: frozenset[str] = frozenset({"two_sided"})
# Variants whose loss kernel live-wiring is deferred. v0.53.11 #123 lifts
# the 6 entries — all variants now have live math kernels via
# :func:`apply_variant_loss`. ``_DEFERRED_LIVE`` is kept (empty) for back-compat
# with callers that inspect it (e.g. ``variant_is_live_wired``).
_DEFERRED_LIVE: frozenset[str] = frozenset()
@dataclass(frozen=True)
class GRPOVariantSpec:
"""Metadata for a GRPO objective variant."""
name: str
description: str
requires_delta: bool
live_wired: bool
_VARIANT_METADATA = types.MappingProxyType({
"standard": GRPOVariantSpec(
name="standard",
description="Default GRPO (DeepSeek-R1 style)",
requires_delta=False,
live_wired=True,
),
"gspo": GRPOVariantSpec(
name="gspo",
description="Group Sequence Policy Optimization",
requires_delta=False,
live_wired=True,
),
"dapo": GRPOVariantSpec(
name="dapo",
description="Decoupled Advantage Policy Optimization",
requires_delta=False,
live_wired=True,
),
"dr_grpo": GRPOVariantSpec(
name="dr_grpo",
description="Doubly Robust GRPO",
requires_delta=False,
live_wired=True,
),
"bnpo": GRPOVariantSpec(
name="bnpo",
description="Batch Normalized Policy Optimization",
requires_delta=False,
live_wired=True,
),
"two_sided": GRPOVariantSpec(
name="two_sided",
description="Two-sided GRPO with symmetric delta clipping",
requires_delta=True,
live_wired=True,
),
"rft": GRPOVariantSpec(
name="rft",
description="Reinforced Fine-Tuning",
requires_delta=False,
live_wired=True,
),
})
def validate_grpo_variant(name: object) -> str:
"""Validate and normalise a GRPO variant name.
Returns the canonical (lower-cased) name on success. Raises
``ValueError`` with an actionable message on any failure.
"""
if isinstance(name, bool):
raise ValueError("grpo_variant must be a string, got bool")
if not isinstance(name, str):
raise ValueError(
f"grpo_variant must be a string, got {type(name).__name__}"
)
if not name:
raise ValueError("grpo_variant must be a non-empty string")
if "\x00" in name:
raise ValueError("grpo_variant must not contain null bytes")
if len(name) > _MAX_VARIANT_NAME_LEN:
raise ValueError(
f"grpo_variant exceeds {_MAX_VARIANT_NAME_LEN} chars"
)
normalised = name.lower()
if normalised not in SUPPORTED_GRPO_VARIANTS:
raise ValueError(
f"grpo_variant={name!r} is not supported. "
f"Valid: {sorted(SUPPORTED_GRPO_VARIANTS)}"
)
return normalised
def get_variant_spec(name: str) -> GRPOVariantSpec:
"""Return the :class:`GRPOVariantSpec` for ``name``.
Raises ``KeyError`` if ``name`` is not in the allowlist.
"""
normalised = validate_grpo_variant(name)
return _VARIANT_METADATA[normalised]
def variant_requires_delta(name: str) -> bool:
"""Return True if the variant requires ``grpo_delta`` to be set."""
if not isinstance(name, str):
return False
return name.lower() in _REQUIRES_DELTA
def variant_is_live_wired(name: str) -> bool:
"""Return True if the variant has a live loss kernel in this release.
All v0.50.0 additions return False (deferred to v0.50.1); only
``standard`` returns True.
"""
if not isinstance(name, str):
return False
return name.lower() not in _DEFERRED_LIVE
def validate_grpo_delta(value: object) -> float:
"""Validate the two-sided clipping delta.
Must be a finite float in ``(0, 1]``. Bool rejected (matches v0.30.0
Candidate policy).
"""
if isinstance(value, bool):
raise ValueError("grpo_delta must not be bool")
if not isinstance(value, (int, float)):
raise ValueError(
f"grpo_delta must be a number, got {type(value).__name__}"
)
fvalue = float(value)
if not math.isfinite(fvalue):
raise ValueError("grpo_delta must be finite (no NaN/Inf)")
if not (0.0 < fvalue <= 1.0):
raise ValueError(
f"grpo_delta={fvalue} must be in (0, 1]"
)
return fvalue
def apply_variant_loss(
name: str,
*,
logp_new,
logp_old,
advantages,
beta: float = 0.0,
delta: float | None = None,
completion_mask=None,
reference_logp=None,
):
"""Compute the per-batch loss tensor for a GRPO variant (v0.53.11 #123).
This is the live kernel that lifts the v0.50.0 ``NotImplementedError``
stub for the 6 deferred variants. The signature accepts the minimum
quantities each variant needs:
- ``logp_new``: policy log-probs from current model, shape ``[B, T]``.
- ``logp_old``: log-probs from rollout policy (detached), shape ``[B, T]``.
- ``advantages``: group-relative advantages, shape ``[B]`` or ``[B, T]``.
- ``beta``: KL coefficient (``GRPOConfig.beta``, i.e. ``grpo_beta``).
Every variant applies it -- see *KL penalty* below; ``0`` means no KL
term.
- ``delta``: symmetric clipping radius (used by ``two_sided`` and, as the
sequence-level clipping radius, by ``gspo``).
- ``completion_mask``: optional ``[B, T]`` 0/1 mask (1 where token is in
the completion). When supplied, length-normalising variants
(``bnpo``) divide by ``mask.sum(-1).clamp(min=1)``.
- ``reference_logp``: the frozen reference policy's per-token log-probs,
shape ``[B, T]`` (trl's ``ref_per_token_logps``). Required when
``beta > 0``; not read at all when ``beta == 0``.
Returns a scalar torch tensor (or ``None`` for the standard variant —
callers should fall through to the existing TRL ``GRPOTrainer.compute_loss``,
which applies ``beta`` itself).
KL penalty (#1232). With ``beta > 0`` every variant adds trl 0.29's
per-token KL estimate against the reference,
``kl_t = exp(ref_t - logp_t) - (ref_t - logp_t) - 1``, as
``per_token_loss + beta * kl_t`` ahead of the variant's own reduction --
where ``GRPOTrainer._compute_loss`` puts it -- so ``beta`` weighs the KL
against the policy term per token exactly as it does in trl. A policy
equal to its reference adds exactly zero. ``beta > 0`` with
``reference_logp=None`` raises ``ValueError`` instead of dropping the
term; ``beta == 0`` skips it and reproduces the pre-#1232 loss bit for
bit.
The math is intentionally minimal — these are *reference* kernels for
routing + unit tests. Production correctness on multi-billion-param
models will be validated by the v0.53.11 smoke run on SmolLM2-135M
+ gsm8k. Each variant kernel matches the canonical formula from the
unsloth / axolotl reference implementations:
- ``gspo``: Group Sequence Policy Optimization (Qwen, arXiv:2507.18071).
Sequence-level importance ratio length-normalized by completion length:
``s_i = exp((1/|y_i|) * sum (log p_new - log p_old))``, optimized with
sequence-level surrogate clipping: ``-min(s * A, clip(s, 1-eps, 1+eps) * A)``.
KL: each sequence's masked mean ``kl_t`` joins that sequence's loss
before the mean over non-empty sequences.
- ``dapo``: decoupled-clip — uses asymmetric clipping bounds
``[1-eps_lo, 1+eps_hi]`` (here ``eps_lo=0.2, eps_hi=0.28`` per the
paper). KL: per token, inside the masked token mean over the batch.
- ``dr_grpo``: GRPO without length normalisation (the doubly-robust
bias-correction term is left to v0.53.12+; the schema gate keeps
misconfigured runs out). KL: per token, inside the masked token sum
that is then averaged over the batch, like the policy term.
- ``bnpo``: length-normalised PPO loss — divides by ``mask.sum(-1)``.
KL: per token, inside the per-sequence length-normalised mean.
- ``two_sided``: symmetric clipping with operator-supplied ``delta``;
``delta=None`` raises ``ValueError`` (schema requires it). KL: per
token, inside the masked token mean over the batch.
- ``rft``: rejection-sampling fine-tuning — only positive-advantage
samples contribute (zero-advantage rows masked out before mean). KL:
on those same accepted tokens only, over the same denominator, so a
batch with no accepted completion still contributes zero.
- ``standard``: returns ``None`` so the caller delegates to the
existing v0.50.0 ``GRPOTrainerWrapper`` path.
"""
import torch # lazy import — utility module is dependency-light
normalised = validate_grpo_variant(name)
if normalised == "standard":
return None
# Defensive guards — bool rejected on every numeric kwarg per project
# policy (matches v0.30.0 Candidate / v0.41.0 lr_groups).
if isinstance(beta, bool):
raise TypeError("beta must be float, not bool")
if not isinstance(beta, (int, float)):
raise TypeError(f"beta must be a number, got {type(beta).__name__}")
beta_f = float(beta)
if not math.isfinite(beta_f) or beta_f < 0.0:
raise ValueError("beta must be a finite non-negative number")
# #1232 — at beta == 0 the reference is not read at all, so the loss is the
# pre-#1232 one bit for bit and a reference whose exp() overflows cannot
# leak NaN into it through a 0 * inf.
per_token_kl = (
_per_token_kl(normalised, beta_f, logp_new, reference_logp) if beta_f > 0.0 else None
)
if normalised == "two_sided":
if delta is None:
raise ValueError("two_sided variant requires grpo_delta (got None)")
delta_f = validate_grpo_delta(delta)
else:
delta_f = None
# Cast advantages to 2-D if needed so broadcasting against logp ratios
# is consistent across variants.
if advantages.dim() == 1:
advantages_2d = advantages.unsqueeze(-1)
else:
advantages_2d = advantages
# token-level importance ratio (PPO building block).
# Clamped from above to prevent torch.exp() overflow to +inf under extreme
# policy divergence, avoiding IEEE-754 0.0 * inf = NaN in backward autograd.
# The bound is adapted to max(|A|, 1.0) so ratio * A never overflows finfo.max
# without prematurely clamping moderate ratios in float16 (#1610).
max_log_ratio = math.log(torch.finfo(logp_new.dtype).max) - 1.0
if normalised in ("dapo", "dr_grpo", "bnpo", "two_sided"):
adv_bound = max_log_ratio - torch.log(
torch.clamp(torch.abs(advantages_2d.detach()), min=1.0)
)
log_ratio = torch.clamp_max(logp_new - logp_old, adv_bound)
ratio = torch.exp(log_ratio)
elif normalised == "gspo":
# GSPO sequence ratio is bounded at sequence level before exp().
log_ratio = logp_new - logp_old
ratio = None
else:
log_ratio = None
ratio = None
if normalised == "gspo":
# Group Sequence Policy Optimization (Qwen, arXiv:2507.18071):
# Sequence-level importance ratio length-normalized by completion length:
# s_i = exp( 1/|y_i| * sum_{t=1}^{|y_i|} (log p_new - log p_old) )
# Optimized with sequence-level surrogate clipping:
# loss = -min(s_i * A_i, clip(s_i, 1-eps, 1+eps) * A_i).mean()
if delta is not None:
eps = validate_grpo_delta(delta)
else:
eps = 0.2
if advantages.dim() == 1:
adv_seq = advantages
elif advantages.dim() == 2 and advantages.size(-1) == 1:
adv_seq = advantages.squeeze(-1)
elif completion_mask is not None:
adv_masked = torch.where(
completion_mask.bool(), advantages, torch.zeros_like(advantages)
)
adv_seq = adv_masked.sum(dim=-1) / completion_mask.to(dtype=advantages.dtype).sum(
dim=-1
).clamp(min=1.0)
else:
adv_seq = advantages.mean(dim=-1)
if completion_mask is not None:
lengths = completion_mask.to(dtype=log_ratio.dtype).sum(dim=-1)
valid_seq_mask = lengths > 0
masked_log_ratio = torch.where(
completion_mask.bool(), log_ratio, torch.zeros_like(log_ratio)
)
seq_log_ratio = masked_log_ratio.sum(dim=-1) / lengths.clamp(min=1.0)
else:
seq_log_ratio = log_ratio.mean(dim=-1)
valid_seq_mask = torch.ones(
seq_log_ratio.shape, dtype=torch.bool, device=seq_log_ratio.device
)
adv_seq_bound = max_log_ratio - torch.log(torch.clamp(torch.abs(adv_seq.detach()), min=1.0))
seq_log_ratio = torch.clamp_max(seq_log_ratio, adv_seq_bound)
seq_ratio = torch.exp(seq_log_ratio)
surr1 = seq_ratio * adv_seq
surr2 = torch.clamp(seq_ratio, min=1.0 - eps, max=1.0 + eps) * adv_seq
seq_loss = -torch.min(surr1, surr2)
if per_token_kl is not None:
# Sequence level, like the policy term: trl's sequence-level
# importance sampling adds beta * kl_t to the per-sequence loss and
# takes each sequence's masked token mean.
if completion_mask is not None:
masked_kl = torch.where(
completion_mask.bool(), per_token_kl, torch.zeros_like(per_token_kl)
)
seq_kl = masked_kl.sum(dim=-1) / lengths.clamp(min=1.0)
else:
seq_kl = per_token_kl.mean(dim=-1)
seq_loss = seq_loss + beta_f * seq_kl
masked_seq_loss = torch.where(valid_seq_mask, seq_loss, torch.zeros_like(seq_loss))
denom = valid_seq_mask.to(dtype=log_ratio.dtype).sum().clamp(min=1.0)
loss = masked_seq_loss.sum() / denom
elif normalised == "dapo":
# Decoupled clip — asymmetric bounds.
eps_lo, eps_hi = 0.2, 0.28
clipped = torch.clamp(ratio, min=1 - eps_lo, max=1 + eps_hi)
# PPO surrogate: min(ratio * A, clipped * A).
token_loss = -torch.min(ratio * advantages_2d, clipped * advantages_2d)
token_loss = _with_kl(token_loss, per_token_kl, beta_f)
loss = _masked_mean(token_loss, completion_mask)
elif normalised == "dr_grpo":
# No length normalisation — sum across tokens then mean across batch.
token_loss = -(ratio * advantages_2d)
token_loss = _with_kl(token_loss, per_token_kl, beta_f)
if completion_mask is not None:
token_loss = torch.where(
completion_mask.bool(), token_loss, torch.zeros_like(token_loss)
)
# sum-over-tokens, mean-over-batch (no division by completion length)
loss = token_loss.sum(dim=-1).mean()
elif normalised == "bnpo":
# Batch-normalised PPO with length-normalisation.
eps = 0.2
clipped = torch.clamp(ratio, min=1 - eps, max=1 + eps)
token_loss = -torch.min(ratio * advantages_2d, clipped * advantages_2d)
token_loss = _with_kl(token_loss, per_token_kl, beta_f)
loss = _masked_mean(token_loss, completion_mask, normalize_by_length=True)
elif normalised == "two_sided":
# Symmetric clipping at [1-delta, 1+delta].
clipped = torch.clamp(ratio, min=1 - delta_f, max=1 + delta_f)
token_loss = -torch.min(ratio * advantages_2d, clipped * advantages_2d)
token_loss = _with_kl(token_loss, per_token_kl, beta_f)
loss = _masked_mean(token_loss, completion_mask)
elif normalised == "rft":
# Rejection sampling fine-tuning: only positive-advantage tokens
# contribute to the gradient.
pos_bool = advantages_2d > 0
if completion_mask is not None:
pos_bool = pos_bool & completion_mask.bool()
# Standard SFT-style negative log-likelihood weighted by positive mask.
token_loss = torch.where(pos_bool, -logp_new, torch.zeros_like(logp_new))
if per_token_kl is not None:
# rft trains on the accepted completions only, so its KL covers the
# same tokens over the same denominator (#1232): a batch with no
# accepted completion still contributes zero.
token_loss = token_loss + beta_f * torch.where(
pos_bool, per_token_kl, torch.zeros_like(per_token_kl)
)
denom = pos_bool.to(dtype=logp_new.dtype).sum().clamp(min=1.0)
loss = token_loss.sum() / denom
else:
# Defensive fallback — schema rejects everything outside the allowlist.
raise ValueError(f"Unhandled grpo_variant={normalised!r}")
return loss
def _per_token_kl(variant: str, beta: float, logp_new, reference_logp):
"""trl 0.29's per-token KL estimate against the frozen reference (#1232).
``exp(ref - logp) - (ref - logp) - 1``: the expression
``GRPOTrainer._compute_loss`` weights by ``beta``. It is non-negative and
exactly zero where the policy equals the reference. The reference is
detached because it is a frozen policy, not a parameter.
"""
import torch # lazy import — utility module is dependency-light
if reference_logp is None:
# Inside the trainer this text reaches the user through the #159
# fallback warning (for a KL-free run, set grpo_beta: 0; #1247),
# so it names the batch key trl should have filled instead.
raise ValueError(
f"grpo_variant={variant!r} with beta={beta} needs reference_logp, the "
"frozen reference policy's per-token log-probs, for its KL penalty "
"(#1232); in a trl batch that is 'ref_per_token_logps', and it is missing"
)
if tuple(reference_logp.shape) != tuple(logp_new.shape):
raise ValueError(
f"reference_logp shape {tuple(reference_logp.shape)} does not match "
f"logp_new shape {tuple(logp_new.shape)}"
)
ref_minus_logp = reference_logp.detach() - logp_new
max_log_ratio = math.log(torch.finfo(logp_new.dtype).max) - 1.0
ref_minus_logp_clamped = torch.clamp_max(ref_minus_logp, max_log_ratio)
return torch.exp(ref_minus_logp_clamped) - ref_minus_logp_clamped - 1
def _with_kl(token_loss, per_token_kl, beta: float):
"""``token_loss + beta * kl_t``, trl's placement; a no-op without a KL term."""
if per_token_kl is None:
return token_loss
return token_loss + beta * per_token_kl
def variant_kl_metric(
name: str,
*,
logp_new,
advantages,
beta: float,
completion_mask=None,
reference_logp=None,
):
"""Batch-mean per-token KL for the ``kl`` metric of a variant run (#1263).
trl 0.29's stock loss logs ``(kl_t * mask).sum() / mask.sum().clamp(min=1)``.
This is the same quantity over the tokens the variant applies its KL term
to: the completion tokens, or the accepted ones for ``rft``. Returns
``None`` when ``beta == 0`` (no KL term, so trl logs nothing either) and
for ``standard``, whose loss and ``kl`` metric are trl's own.
"""
normalised = validate_grpo_variant(name)
if normalised == "standard" or float(beta) == 0.0:
return None
import torch # lazy import — utility module is dependency-light
per_token_kl = _per_token_kl(normalised, float(beta), logp_new, reference_logp).detach()
mask = completion_mask
if normalised == "rft":
advantages_2d = advantages.unsqueeze(-1) if advantages.dim() == 1 else advantages
mask = (advantages_2d > 0).to(per_token_kl.dtype).expand_as(per_token_kl)
if completion_mask is not None:
mask = mask * completion_mask
if mask is None:
return per_token_kl.mean()
masked_kl = torch.where(mask.bool(), per_token_kl, torch.zeros_like(per_token_kl))
acc_dtype = torch.promote_types(per_token_kl.dtype, torch.float32)
return masked_kl.sum(dtype=acc_dtype) / mask.to(dtype=acc_dtype).sum().clamp(min=1.0)
def _masked_mean(
token_loss,
completion_mask=None,
*,
normalize_by_length: bool = False,
):
"""Mean over a masked tensor; helper for :func:`apply_variant_loss`."""
if completion_mask is None:
return token_loss.mean()
import torch # lazy import — utility module is dependency-light
masked = torch.where(completion_mask.bool(), token_loss, torch.zeros_like(token_loss))
mask_in_dtype = completion_mask.to(dtype=token_loss.dtype)
if normalize_by_length:
lengths = mask_in_dtype.sum(dim=-1).clamp(min=1.0)
per_sample = masked.sum(dim=-1) / lengths
return per_sample.mean()
denom = mask_in_dtype.sum().clamp(min=1.0)
return masked.sum() / denom
def list_variants() -> tuple[str, ...]:
"""Return a sorted tuple of supported variant names (for CLI help)."""
return tuple(sorted(SUPPORTED_GRPO_VARIANTS))