Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

6.8 KB
Raw Blame
"""
Supervised Fine-Tuning of the pretrained base on packed instruction data.

Loads the base checkpoint, trains with the prompt-masked SFT loss, periodically reports
masked dev loss, and saves an SFT checkpoint. DDP + bf16, single code path for 1 or N GPUs.

Single GPU:
    python scripts/train_sft.py
Both GPUs:
    torchrun --standalone --nproc_per_node=2 scripts/train_sft.py
"""

from __future__ import annotations

import sys
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parents[1]))  # run from the repo without installing

import contextlib
import copy
import math
import os
import time

import torch

from config.post_training_config import SFTConfig
from data_loader.sft_dataset import get_sft_batch_iterator
from src.models.lora import apply_lora, lora_parameter_count, merge_lora
from src.post_training.cli import parse_config_with_json
from src.post_training.distributed import cleanup, ddp_setup, ddp_wrap, reduce_scalar
from src.post_training.logging_utils import MetricsLogger
from src.post_training.optim import configure_optimizer, cosine_lr, set_lr
from src.post_training.sft import sft_loss
from src.post_training.utils import (
    amp_autocast,
    load_backbone_from_ckpt,
    moe_balance_loss,
    save_stage_ckpt,
    set_seed,
    unwrap,
)


@torch.no_grad()
def eval_dev(model, cfg, ctx, dev_path: str, max_batches: int = 50) -> float:
    if not os.path.exists(dev_path):
        return float("nan")
    model.eval()
    it = get_sft_batch_iterator(dev_path, cfg.batch_size, device=ctx.device,
                                rank=ctx.rank, world_size=ctx.world_size, shuffle=False, infinite=False)
    total, n = 0.0, 0
    for tokens, mask, _ in it:
        with amp_autocast(cfg.amp_dtype, ctx.device):
            logits, _ = model(tokens)
            loss = sft_loss(logits, tokens, mask)
        total += loss.item(); n += 1
        if n >= max_batches:
            break
    model.train()
    return total / max(1, n)


def model_to_save(model, use_lora: bool):
    """With LoRA, save a merged copy so the checkpoint is a plain model for the next stages."""
    plain = unwrap(model)
    return merge_lora(copy.deepcopy(plain)) if use_lora else plain


def main():
    cfg, _ = parse_config_with_json(SFTConfig, "configs/sft.json")
    ctx = ddp_setup(cfg.device)
    set_seed(cfg.seed + ctx.rank)

    model = load_backbone_from_ckpt(cfg, cfg.pretrained_ckpt, ctx.device)
    use_lora = cfg.lora_rank > 0
    if use_lora:
        wrapped = apply_lora(model, cfg.lora_rank, cfg.lora_alpha, cfg.lora_dropout)
        trainable, total = lora_parameter_count(model)
        if ctx.is_main:
            print(f"LoRA rank {cfg.lora_rank} on {len(wrapped)} layers: training {trainable:,} of "
                  f"{total:,} parameters ({100 * trainable / total:.2f}%)")
    if cfg.compile:
        model = torch.compile(model)
    model = ddp_wrap(model, ctx)
    optimizer = configure_optimizer(unwrap(model), cfg.lr, cfg.weight_decay)

    logger = None
    if ctx.is_main:
        print(f"SFT from {cfg.pretrained_ckpt} | world_size={ctx.world_size}")
        logger = MetricsLogger("sft", cfg.log_dir, use_wandb=cfg.use_wandb, wandb_project=cfg.wandb_project)

    train_it = get_sft_batch_iterator(cfg.data_path, cfg.batch_size, device=ctx.device,
                                      rank=ctx.rank, world_size=ctx.world_size, shuffle=True, infinite=True)

    # Estimate total steps for the cosine schedule from dataset size. One optimizer step uses
    # grad_accum micro-batches on every rank.
    import h5py
    with h5py.File(cfg.data_path, "r") as f:
        n_rows = f["tokens"].shape[0]
    steps_per_epoch = max(1, n_rows // (cfg.batch_size * cfg.grad_accum * ctx.world_size))
    total_steps = cfg.max_steps if cfg.max_steps > 0 else steps_per_epoch * cfg.epochs
    if ctx.is_main:
        print(f"{n_rows} packed rows | ~{steps_per_epoch} steps/epoch | total_steps={total_steps} | "
              f"effective batch = {cfg.batch_size}*{cfg.grad_accum}*{ctx.world_size} rows")

    model.train()
    t0 = time.perf_counter()
    epoch = 0
    for step in range(total_steps):
        lr = cosine_lr(step, warmup_steps=cfg.warmup_steps, max_steps=total_steps, lr=cfg.lr, min_lr=cfg.min_lr)
        set_lr(optimizer, lr)

        optimizer.zero_grad(set_to_none=True)
        loss_sum = 0.0
        for micro in range(cfg.grad_accum):
            tokens, mask, epoch = next(train_it)
            # Only sync gradients across GPUs on the last micro-batch.
            sync = micro == cfg.grad_accum - 1 or not ctx.enabled
            with (contextlib.nullcontext() if sync else model.no_sync()), amp_autocast(cfg.amp_dtype, ctx.device):
                logits, _ = model(tokens)
                ce = sft_loss(logits, tokens, mask)
                # Mixture-of-Experts models also keep their experts balanced (0 for dense models)
                loss = (ce + moe_balance_loss(model)) / cfg.grad_accum
            loss.backward()
            loss_sum += ce.item() / cfg.grad_accum  # log the plain SFT loss
        if epoch >= cfg.epochs and cfg.max_steps <= 0:
            break
        loss = torch.tensor(loss_sum)
        torch.nn.utils.clip_grad_norm_(model.parameters(), cfg.grad_clip)
        optimizer.step()

        if ctx.is_main and step % 20 == 0:
            dt = time.perf_counter() - t0; t0 = time.perf_counter()
            print(f"step {step}/{total_steps} | loss {loss.item():.4f} | ppl {math.exp(min(20, loss.item())):.2f} | lr {lr:.2e} | {dt:.1f}s/20")
            if logger:
                logger.log(step, {"train_loss": loss.item(), "lr": lr})

        if step > 0 and step % cfg.eval_steps == 0:
            dev = reduce_scalar(eval_dev(model, cfg, ctx, cfg.dev_path), ctx)
            if ctx.is_main:
                print(f"  [eval] step {step} | dev_loss {dev:.4f} | dev_ppl {math.exp(min(20, dev)):.2f}")
                if logger:
                    logger.log(step, {"dev_loss": dev})

        if ctx.is_main and step > 0 and step % cfg.save_every == 0:
            save_stage_ckpt(cfg.out_ckpt, model_to_save(model, use_lora), None if use_lora else optimizer,
                            stage="sft", cfg=cfg, step=step, metrics={"train_loss": loss.item()})

    if ctx.is_main:
        # Use the unwrapped model for the final eval: the other ranks have already reached
        # cleanup(), so calling the DDP-wrapped model here would launch a collective with no
        # peer and hang (NCCL timeout). The periodic eval above runs on all ranks, so it is fine.
        dev = eval_dev(unwrap(model), cfg, ctx, cfg.dev_path)
        save_stage_ckpt(cfg.out_ckpt, model_to_save(model, use_lora), None if use_lora else optimizer,
                        stage="sft", cfg=cfg, step=total_steps, metrics={"dev_loss": dev})
        print(f"Done SFT. dev_loss {dev:.4f} -> {cfg.out_ckpt}")
        if logger:
            logger.close()
    cleanup(ctx)


if __name__ == "__main__":
    main()