Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

4.7 KB
Raw Blame
"""Speculative decoding and int8 weight quantization."""

from __future__ import annotations

import copy

import pytest
import torch
import torch.nn as nn
import torch.nn.functional as F

from src.inference.quantize import Int8Linear, model_size_bytes, quantize_int8
from src.inference.speculative import speculative_generate
from src.models.modern import ModernConfig, ModernTransformer
from src.models.transformer import Transformer


def _classic(seed: int, vocab: int = 13) -> Transformer:
    torch.manual_seed(seed)
    return Transformer(n_head=2, n_embed=16, context_length=32, vocab_size=vocab, N_BLOCKS=1).eval()


def _greedy(model: nn.Module, idx: torch.Tensor, n: int) -> torch.Tensor:
    for _ in range(n):
        logits, _ = model(idx[:, -32:])
        idx = torch.cat([idx, logits[:, -1].argmax(-1, keepdim=True)], dim=1)
    return idx


def test_greedy_speculative_decoding_equals_greedy_target_decoding() -> None:
    target, draft = _classic(0), _classic(1)  # an unrelated, random draft
    prompt = torch.tensor([[1, 2, 3]])
    ours, stats = speculative_generate(target, draft, prompt, 20, k=4, greedy=True)
    assert torch.equal(ours, _greedy(target, prompt, 20))
    assert stats.generated == 20 and stats.target_calls <= 20


def test_speculative_decoding_respects_the_trained_window() -> None:
    """Each model only sees its last `window` tokens; inside the window the result is exact."""
    target, draft = _classic(0), _classic(1)
    prompt = torch.tensor([[1, 2, 3]])
    ours, _ = speculative_generate(target, draft, prompt, 10, k=3, greedy=True, target_window=16, draft_window=5)
    assert torch.equal(ours, target.generate(prompt, 10, top_k=1, context_window=16))

    seen: dict[str, int] = {}

    def record(name: str):
        def hook(module: nn.Module, args: tuple[torch.Tensor, ...]) -> None:
            seen[name] = max(seen.get(name, 0), args[0].size(1))
        return hook

    target.register_forward_pre_hook(record("target"))
    draft.register_forward_pre_hook(record("draft"))
    speculative_generate(target, draft, prompt, 40, k=3, greedy=True, target_window=16, draft_window=5)
    assert seen == {"target": 16, "draft": 5}
    with pytest.raises(ValueError, match="longer than k"):
        speculative_generate(target, draft, prompt, 4, k=8, target_window=8)


def test_a_perfect_draft_is_always_accepted() -> None:
    target = _classic(0)
    g = torch.Generator().manual_seed(0)
    _, stats = speculative_generate(target, copy.deepcopy(target), torch.tensor([[1, 2]]), 24, k=4, generator=g)
    assert stats.acceptance_rate > 0.95
    assert stats.tokens_per_target_call > 4  # k accepted guesses + 1 bonus token per target pass


def test_sampling_keeps_the_target_distribution_whatever_the_draft() -> None:
    target, draft = _classic(0, vocab=6), _classic(7, vocab=6)
    prompt = torch.tensor([[1, 4, 2]])
    with torch.no_grad():
        p_target = F.softmax(target(prompt)[0][0, -1], dim=-1)
        p_draft = F.softmax(draft(prompt)[0][0, -1], dim=-1)
    assert (p_target - p_draft).abs().max() > 0.05  # the draft really is different
    g = torch.Generator().manual_seed(0)
    counts = torch.zeros(6)
    n = 3000
    for _ in range(n):
        out, _ = speculative_generate(target, draft, prompt, 1, k=3, generator=g)
        counts[out[0, -1]] += 1
    assert (counts / n - p_target).abs().max() < 0.03


def test_int8_linear_is_accurate_and_four_times_smaller() -> None:
    torch.manual_seed(0)
    linear = nn.Linear(256, 128)
    q = Int8Linear(linear)
    x = torch.randn(8, 256)
    with torch.no_grad():
        rel = (q(x) - linear(x)).norm() / linear(x).norm()
    assert rel < 0.01
    assert q.weight_int8.dtype == torch.int8
    assert model_size_bytes(q) < 0.3 * model_size_bytes(linear)


@pytest.mark.parametrize("arch", ["classic", "modern"])
def test_quantized_models_give_nearly_the_same_logits(arch: str) -> None:
    torch.manual_seed(0)
    if arch == "classic":
        model: nn.Module = Transformer(n_head=4, n_embed=64, context_length=32, vocab_size=100, N_BLOCKS=2).eval()
    else:
        model = ModernTransformer(ModernConfig(vocab_size=100, context_length=32, n_embed=64, n_head=4, n_blocks=2)).eval()
    idx = torch.randint(0, 100, (2, 16))
    with torch.no_grad():
        before = model(idx)[0]
        quantized = quantize_int8(copy.deepcopy(model))
        after = quantized(idx)[0]
    assert F.cosine_similarity(before.flatten(), after.flatten(), dim=0) > 0.999
    assert any(isinstance(m, Int8Linear) for m in quantized.modules())
    assert model_size_bytes(quantized) < model_size_bytes(model)
    if arch == "modern":  # the tied output layer is not quantized, so the tie survives
        assert quantized.lm_head.weight is quantized.token_embed.weight