Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

11.7 KB
Raw Blame
"""
Tests for the modern decoder (src/models/modern).

Most tests check a property that must hold rather than a hard-coded number: causality,
cached decoding matching a full forward pass, rotary embeddings depending only on relative
position, grouped-query attention matching multi-head attention with copied heads, and so on.
"""

from __future__ import annotations

import math
from dataclasses import replace

import pytest
import torch
import torch.nn.functional as F

from src.models.modern import (
    GroupedQueryAttention,
    ModernConfig,
    ModernTransformer,
    MoE,
    RMSNorm,
    apply_rope,
    rope_cache,
)

BASE = ModernConfig(vocab_size=97, context_length=32, n_embed=32, n_head=4, n_blocks=2, n_kv_head=2)

VARIANTS = {
    "gqa": BASE,
    "mha": replace(BASE, n_kv_head=None),
    "mqa": replace(BASE, n_kv_head=1),
    "mla": replace(BASE, attention="mla", kv_latent_dim=12, rope_dim=4),
    "window": replace(BASE, sliding_window=5),
    "moe": replace(BASE, n_experts=4, moe_top_k=2, n_shared_experts=1),
    "gated_no_qknorm": replace(BASE, attn_gate=True, qk_norm=False, tie_embeddings=False),
}


def _model(cfg: ModernConfig, seed: int = 0) -> ModernTransformer:
    torch.manual_seed(seed)
    return ModernTransformer(cfg).eval()


@pytest.mark.parametrize("name", list(VARIANTS))
def test_forward_shapes_and_loss(name: str) -> None:
    cfg = VARIANTS[name]
    model = _model(cfg)
    idx = torch.randint(0, cfg.vocab_size, (3, 16))
    targets = torch.randint(0, cfg.vocab_size, (3, 16))
    logits, loss = model(idx, targets)
    assert logits.shape == (3, 16, cfg.vocab_size)
    assert loss is not None and loss.ndim == 0
    # a freshly initialized model should be close to a uniform guess: loss ~ ln(vocab)
    assert abs(loss.item() - math.log(cfg.vocab_size)) < 0.5


@pytest.mark.parametrize("name", list(VARIANTS))
def test_causality(name: str) -> None:
    """Changing a token must not change the predictions for earlier positions."""
    cfg = VARIANTS[name]
    model = _model(cfg)
    idx = torch.randint(0, cfg.vocab_size, (2, 20))
    changed = idx.clone()
    changed[:, 12] = (changed[:, 12] + 1) % cfg.vocab_size
    with torch.no_grad():
        a, _ = model(idx)
        b, _ = model(changed)
    assert torch.allclose(a[:, :12], b[:, :12], atol=1e-5)
    assert not torch.allclose(a[:, 12:], b[:, 12:], atol=1e-5)


@pytest.mark.parametrize("name", list(VARIANTS))
def test_kv_cache_matches_full_forward(name: str) -> None:
    """Prefill 7 tokens, then decode one token at a time: logits must equal a full pass."""
    cfg = VARIANTS[name]
    model = _model(cfg)
    idx = torch.randint(0, cfg.vocab_size, (2, 18))
    with torch.no_grad():
        full, _ = model(idx)
        cache = model.new_cache(batch_size=2)
        steps = [model.lm_head(model.forward_hidden(idx[:, :7], cache))]
        for t in range(7, 18):
            steps.append(model.lm_head(model.forward_hidden(idx[:, t : t + 1], cache)))
    assert cache.length == 18
    assert torch.allclose(torch.cat(steps, dim=1), full, atol=1e-4)


def test_sliding_window_limits_what_a_token_sees() -> None:
    """With one block and window w, position t only depends on tokens t-w+1 .. t."""
    cfg = replace(BASE, n_blocks=1, sliding_window=4)
    model = _model(cfg)
    idx = torch.randint(0, cfg.vocab_size, (1, 16))
    changed = idx.clone()
    changed[:, 5] = (changed[:, 5] + 1) % cfg.vocab_size  # outside the window of position 9..15
    with torch.no_grad():
        a, _ = model(idx)
        b, _ = model(changed)
    assert torch.allclose(a[:, 9:], b[:, 9:], atol=1e-5)
    assert not torch.allclose(a[:, 5:9], b[:, 5:9], atol=1e-5)


def test_rope_scores_depend_only_on_relative_position() -> None:
    torch.manual_seed(0)
    cos, sin = rope_cache(dim=16, max_len=64)
    q, k = torch.randn(16), torch.randn(16)

    def score(m: int, n: int) -> float:
        qm = apply_rope(q[None], cos[m : m + 1], sin[m : m + 1])[0]
        kn = apply_rope(k[None], cos[n : n + 1], sin[n : n + 1])[0]
        return float(qm @ kn)

    assert math.isclose(score(10, 3), score(40, 33), rel_tol=1e-4, abs_tol=1e-4)
    assert math.isclose(score(5, 5), float(q @ k), rel_tol=1e-4, abs_tol=1e-4)  # same position: no change


def test_rope_is_a_rotation_of_each_pair() -> None:
    """Pair (x[i], x[i + d/2]) at position m is rotated by the angle m * theta_i."""
    torch.manual_seed(0)
    dim, m = 8, 7
    cos, sin = rope_cache(dim=dim, max_len=16)
    x = torch.randn(dim)
    out = apply_rope(x[None], cos[m : m + 1], sin[m : m + 1])[0]
    half = dim // 2
    for i in range(half):
        angle = m * 10_000.0 ** (-2 * i / dim)
        rot = torch.tensor([[math.cos(angle), -math.sin(angle)], [math.sin(angle), math.cos(angle)]])
        expected = rot @ torch.stack([x[i], x[i + half]])
        assert torch.allclose(torch.stack([out[i], out[i + half]]), expected, atol=1e-5)
    assert torch.allclose(out.norm(), x.norm(), atol=1e-5)  # rotations keep the length


def test_rmsnorm_matches_torch() -> None:
    x = torch.randn(4, 10, 32)
    ours, ref = RMSNorm(32, eps=1e-6), torch.nn.RMSNorm(32, eps=1e-6)
    with torch.no_grad():
        ours.weight.uniform_(0.5, 1.5)
        ref.weight.copy_(ours.weight)
    assert torch.allclose(ours(x), ref(x), atol=1e-5)


def test_gqa_equals_mha_with_shared_heads() -> None:
    """GQA with 2 KV heads == MHA whose 4 KV heads are copies (heads 0,1 share KV 0; 2,3 share KV 1)."""
    gqa_cfg = replace(BASE, n_kv_head=2, qk_norm=False)
    mha_cfg = replace(BASE, n_kv_head=4, qk_norm=False)
    torch.manual_seed(0)
    gqa, mha = GroupedQueryAttention(gqa_cfg, 0), GroupedQueryAttention(mha_cfg, 0)
    D = BASE.head_dim
    with torch.no_grad():
        mha.q_proj.weight.copy_(gqa.q_proj.weight)
        mha.o_proj.weight.copy_(gqa.o_proj.weight)
        for proj in ("k_proj", "v_proj"):
            w = getattr(gqa, proj).weight.view(2, D, -1)  # (kv_heads, D, C)
            getattr(mha, proj).weight.copy_(w.repeat_interleave(2, dim=0).reshape(4 * D, -1))
    x = torch.randn(2, 9, BASE.n_embed)
    cos, sin = rope_cache(D, 9)
    assert torch.allclose(gqa(x, cos, sin), mha(x, cos, sin), atol=1e-5)


def test_mla_caches_much_less_than_mha() -> None:
    mha = _model(replace(BASE, n_kv_head=None)).new_cache(1)
    mla = _model(VARIANTS["mla"]).new_cache(1)
    assert mla.k.shape[-1] == 12 and mla.v.shape[-1] == 4  # latent + shared rotary key
    assert mla.bytes_per_token() * 3 < mha.bytes_per_token()


def test_moe_routes_tokens_and_balancing_loss_trains_the_router() -> None:
    torch.manual_seed(0)
    moe = MoE(n_embed=16, hidden=32, n_experts=4, top_k=2)
    x = torch.randn(2, 10, 16)
    y = moe(x)
    assert y.shape == x.shape
    assert moe.tokens_per_expert is not None and moe.tokens_per_expert.sum().item() == 2 * 10 * 2
    assert moe.aux_loss is not None and moe.aux_loss.item() >= 0.99  # == 1 only for perfect balance
    moe.aux_loss.backward()
    assert moe.router.weight.grad is not None and moe.router.weight.grad.abs().sum() > 0


def test_moe_loss_includes_aux_and_active_params_are_smaller() -> None:
    model = _model(VARIANTS["moe"])
    idx = torch.randint(0, BASE.vocab_size, (2, 8))
    logits, loss = model(idx, idx)
    plain = F.cross_entropy(logits.reshape(-1, BASE.vocab_size), idx.reshape(-1))
    assert model.aux_loss is not None
    assert torch.allclose(loss, plain + BASE.moe_aux_loss_coef * model.aux_loss)
    assert model.active_params() < model.num_params()


def test_post_training_adds_the_moe_balancing_loss() -> None:
    from src.models.transformer import Transformer
    from src.post_training.utils import moe_balance_loss

    model = _model(VARIANTS["moe"])
    logits, _ = model(torch.randint(0, BASE.vocab_size, (2, 8)))  # no targets, like the SFT trainer
    extra = moe_balance_loss(torch.nn.DataParallel(model))  # wrappers are looked through
    assert isinstance(extra, torch.Tensor) and model.aux_loss is not None
    assert torch.allclose(extra, BASE.moe_aux_loss_coef * model.aux_loss)
    extra.backward()
    assert model.blocks[0].mlp.router.weight.grad is not None
    assert moe_balance_loss(_model(BASE)) == 0.0  # dense modern model
    assert moe_balance_loss(Transformer(n_head=2, n_embed=16, context_length=8, vocab_size=32, N_BLOCKS=1)) == 0.0


def test_tied_embeddings_share_one_matrix() -> None:
    tied, untied = _model(BASE), _model(replace(BASE, tie_embeddings=False))
    assert tied.lm_head.weight is tied.token_embed.weight
    assert untied.num_params() - tied.num_params() == BASE.vocab_size * BASE.n_embed


@pytest.mark.parametrize("name", ["gqa", "moe"])
def test_gradient_checkpointing_gives_the_same_gradients(name: str) -> None:
    cfg = VARIANTS[name]
    idx = torch.randint(0, cfg.vocab_size, (2, 12))
    grads = []
    for ckpt in (False, True):
        model = _model(cfg).train()
        model.gradient_checkpointing = ckpt
        _, loss = model(idx, idx)
        loss.backward()
        grads.append(torch.cat([p.grad.flatten() for p in model.parameters() if p.grad is not None]))
    assert torch.allclose(grads[0], grads[1], atol=1e-5)


@pytest.mark.parametrize("name", ["gqa", "mla", "window"])
def test_cached_and_uncached_generation_agree(name: str) -> None:
    model = _model(VARIANTS[name])
    prompt = torch.randint(0, BASE.vocab_size, (2, 5))
    a = model.generate(prompt, 20, top_k=1, use_cache=True)
    b = model.generate(prompt, 20, top_k=1, use_cache=False)
    assert torch.equal(a, b)


def test_generation_can_run_past_the_context_window() -> None:
    model = _model(BASE)
    out = model.generate(torch.zeros(1, 4, dtype=torch.long), max_new_tokens=3 * BASE.context_length)
    assert out.shape == (1, 4 + 3 * BASE.context_length)


def test_config_validation() -> None:
    with pytest.raises(ValueError, match="divisible"):
        ModernConfig(n_embed=30, n_head=4)
    with pytest.raises(ValueError, match="n_kv_head"):
        ModernConfig(n_embed=32, n_head=4, n_kv_head=3)
    assert ModernConfig.from_mapping({"n_embed": 64, "n_head": 4, "lr": 1e-3}).n_embed == 64


def test_post_training_wrappers_accept_the_modern_model() -> None:
    from src.post_training.reward_model import RewardModel
    from src.post_training.rollout import compute_logprobs, generate_with_logprobs
    from src.post_training.value_head import TransformerWithValueHead

    model = _model(BASE)
    actor = TransformerWithValueHead(model)
    idx = torch.randint(0, BASE.vocab_size, (2, 8))
    logits, values = actor(idx)
    assert logits.shape == (2, 8, BASE.vocab_size) and values.shape == (2, 8)
    reward = RewardModel(_model(BASE))(idx, seq_lengths=torch.tensor([8, 5]))
    assert reward.shape == (2,)
    rb = generate_with_logprobs(model, idx[:, :4], max_new_tokens=6)
    lp, mask = compute_logprobs(model, rb.sequences, rb.response_mask, requires_grad=False)
    assert torch.allclose(rb.gen_logprobs[rb.response_mask[:, 4:]], lp[mask], atol=1e-4)


def test_rl_rollouts_use_the_kv_cache_and_sample_the_same_tokens() -> None:
    from src.post_training.rollout import generate_with_logprobs
    from src.post_training.value_head import TransformerWithValueHead

    model = _model(VARIANTS["gqa"])
    prompts = torch.randint(0, BASE.vocab_size, (3, 5))
    runs = []
    for use_cache in (True, False):
        torch.manual_seed(0)
        runs.append(generate_with_logprobs(model, prompts, 12, temperature=0.9, use_cache=use_cache))
    cached, plain = runs
    assert torch.equal(cached.sequences, plain.sequences)
    assert torch.allclose(cached.gen_logprobs, plain.gen_logprobs, atol=1e-5)
    # the PPO actor wraps the policy in a value head; its rollouts take the cached path too
    actor = TransformerWithValueHead(model)  # built before seeding: its init draws random numbers
    torch.manual_seed(0)
    assert torch.equal(generate_with_logprobs(actor, prompts, 12, temperature=0.9).sequences, cached.sequences)