A straightforward method for training your LLM, from downloading data to generating text.
"""
Shared helpers for the post-training stack: model construction from configs,
frozen reference/old-policy copies, checkpoint I/O (keeping the repo's existing
checkpoint shape), masked reductions, and seeding.
"""
from __future__ import annotations
import contextlib
import copy
import random
from dataclasses import asdict, is_dataclass
from typing import Any
import numpy as np
import torch
import torch.nn as nn
from src.checkpoint import (
atomic_save,
load_checkpoint,
load_model_weights,
model_state_from_checkpoint,
strip_wrapper_prefixes,
unwrap_model,
)
from src.models.factory import LanguageModel, build_model
from src.models.modern import ModernTransformer
def amp_autocast(amp_dtype: str | None, device: str):
"""Return a bf16 autocast context on CUDA when requested, else a no-op context.
bf16 needs no GradScaler (unlike fp16), which keeps the training loops clean.
"""
if amp_dtype == "bf16" and str(device).startswith("cuda") and torch.cuda.is_available():
return torch.autocast(device_type="cuda", dtype=torch.bfloat16)
return contextlib.nullcontext()
# --- Reproducibility ---------------------------------------------------------
def set_seed(seed: int) -> None:
"""Seed python / numpy / torch (incl. CUDA) for reproducible runs."""
random.seed(seed)
np.random.seed(seed)
torch.manual_seed(seed)
torch.cuda.manual_seed_all(seed)
# --- Model construction ------------------------------------------------------
def _cfg_get(cfg: Any, key: str) -> Any:
"""Read a field from either a dataclass/object (attr) or a dict."""
if isinstance(cfg, dict):
return cfg[key]
return getattr(cfg, key)
def build_model_from_config(cfg: Any) -> LanguageModel:
"""
Construct a fresh model from a config carrying the standard keys
``n_head, n_embed, context_length, vocab_size, n_blocks`` (plus ``arch`` and the modern
model's settings, when present). Works with the post-training dataclasses and with the
legacy ``default_config`` dict.
"""
return build_model(cfg)
# Kept for code that imported the old private name; it now also strips torch.compile's
# ``_orig_mod.`` prefix (issue #36).
_strip_ddp_prefix = strip_wrapper_prefixes
def load_backbone_from_ckpt(cfg: Any, ckpt_path: str, device: str) -> LanguageModel:
"""
Build a Transformer from ``cfg`` and load backbone weights from a checkpoint saved
by the pretraining script or any post-training stage (``model_state_dict`` key).
DDP (``module.``) and torch.compile (``_orig_mod.``) prefixes are stripped. Auxiliary
head weights (value/reward), if present, are ignored here because wrappers add their own
fresh heads. A checkpoint that does not cover every backbone parameter raises an error
instead of silently leaving random weights in place.
"""
model = build_model_from_config(cfg)
state = model_state_from_checkpoint(load_checkpoint(ckpt_path, map_location="cpu"))
load_model_weights(model, state, source=ckpt_path)
return model.to(device)
def unwrap(model: nn.Module) -> nn.Module:
"""Return the plain model behind DDP and torch.compile wrappers (or the model itself)."""
return unwrap_model(model)
def moe_balance_loss(model: nn.Module) -> torch.Tensor | float:
"""The Mixture-of-Experts balancing loss of the last forward pass, times its coefficient.
The modern model adds this loss itself only when it is given targets. The post-training
losses are computed outside the model, so the stages add it with this helper. It is 0 for
dense models.
"""
inner = unwrap(model)
if isinstance(inner, ModernTransformer) and inner.aux_loss is not None:
return inner.config.moe_aux_loss_coef * inner.aux_loss
return 0.0
def make_frozen_copy(model: nn.Module, device: str | None = None) -> nn.Module:
"""
Deep-copy a model, put it in eval mode, and disable all gradients. Used for the
DPO/PPO/GRPO reference model and the PPO old-policy snapshot. At the ~300M-1B
scale this is cheap on an 80GB H100 and is the clearest possible implementation.
"""
ref = copy.deepcopy(unwrap(model))
if device is not None:
ref = ref.to(device)
ref.eval()
for p in ref.parameters():
p.requires_grad_(False)
return ref
# --- Masked reductions -------------------------------------------------------
def masked_mean(values: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Mean of ``values`` over positions where ``mask`` is truthy (safe if mask empty)."""
mask = mask.to(values.dtype)
total = (values * mask).sum()
count = mask.sum().clamp(min=1.0)
return total / count
def masked_mean_per_row(values: torch.Tensor, mask: torch.Tensor) -> torch.Tensor:
"""Per-row (B,) mean of ``values`` (B,T) over masked positions."""
mask = mask.to(values.dtype)
total = (values * mask).sum(dim=-1)
count = mask.sum(dim=-1).clamp(min=1.0)
return total / count
def gather_last(values: torch.Tensor, seq_lengths: torch.Tensor) -> torch.Tensor:
"""
Given per-token ``values`` (B, T) and ``seq_lengths`` (B,), return the value at the
last real token of each row, i.e. ``values[i, seq_lengths[i]-1]``. This is the
InstructGPT convention for reading a scalar reward off a sequence model.
"""
idx = (seq_lengths - 1).clamp(min=0).long()
return values[torch.arange(values.size(0), device=values.device), idx]
# --- Checkpoint I/O ----------------------------------------------------------
def _cfg_to_dict(cfg: Any) -> Any:
return asdict(cfg) if is_dataclass(cfg) and not isinstance(cfg, type) else cfg
def save_stage_ckpt(
path: str,
model: nn.Module,
optimizer: torch.optim.Optimizer | None,
*,
stage: str,
cfg: Any,
step: int,
metrics: dict | None = None,
extra: dict | None = None,
) -> None:
"""
Save a checkpoint in the repo's existing shape (``model_state_dict`` /
``optimizer_state_dict``) plus post-training metadata (``stage``, ``cfg``, ``step``,
``metrics``). DDP and torch.compile wrappers are unwrapped first so the keys are clean
and the file loads into a bare model on any device. The write is atomic.
"""
payload = {
"model_state_dict": unwrap(model).state_dict(),
"optimizer_state_dict": optimizer.state_dict() if optimizer is not None else None,
"stage": stage,
"cfg": _cfg_to_dict(cfg),
"step": step,
"metrics": metrics or {},
"pytorch_version": torch.__version__,
"cuda_version": torch.version.cuda,
}
if extra:
payload.update(extra)
atomic_save(payload, path)