A straightforward method for training your LLM, from downloading data to generating text.
feat(inference): speculative decoding and int8 weight quantization
FareedKhan-dev2ebb64cdb39c
diff --git a/src/inference/quantize.py b/src/inference/quantize.pynew file mode 100644index 0000000..461b290--- /dev/null+++ b/src/inference/quantize.py@@ -0,0 +1,74 @@+"""+Int8 weight-only quantization, written from scratch.++A float32 weight takes 4 bytes; an int8 takes 1. Weight-only quantization stores every+``nn.Linear`` weight matrix as int8 plus one float scale per output row:++ scale_r = max(|W_r|) / 127 (per output channel, "absmax")+ Q_r = round(W_r / scale_r) (integers in [-127, 127])+ W_r ~= Q_r * scale_r (dequantized on the fly in forward)++The matmul still runs in floating point, so this saves memory (about 4x for the linear+layers) rather than compute. That is the main win for inference on small devices, where+memory bandwidth, not arithmetic, limits token speed. Per-channel scales keep the error+small because each row gets its own range.++The output layer is left alone by default: when its weight is tied to the input embedding,+quantizing it would silently untie them, and the vocabulary projection is the most+sensitive layer anyway.+"""++from __future__ import annotations++from collections.abc import Iterable++import torch+import torch.nn as nn+import torch.nn.functional as F+from jaxtyping import Float+from torch import Tensor+++class Int8Linear(nn.Module):+ """A frozen ``nn.Linear`` whose weight is stored as int8 with one scale per output row."""++ weight_int8: Tensor+ scale: Tensor++ def __init__(self, linear: nn.Linear) -> None:+ super().__init__()+ self.in_features, self.out_features = linear.in_features, linear.out_features+ w = linear.weight.detach().float()+ scale = w.abs().amax(dim=1, keepdim=True).clamp(min=1e-8) / 127.0+ self.register_buffer("weight_int8", torch.round(w / scale).clamp(-127, 127).to(torch.int8))+ self.register_buffer("scale", scale)+ self.bias = None if linear.bias is None else nn.Parameter(linear.bias.detach().clone(), requires_grad=False)++ def dequantized_weight(self) -> Float[Tensor, "out in"]:+ return self.weight_int8.float() * self.scale++ def forward(self, x: Float[Tensor, "*batch in_features"]) -> Float[Tensor, "*batch out_features"]:+ weight = self.dequantized_weight().to(x.dtype)+ bias = None if self.bias is None else self.bias.to(x.dtype)+ return F.linear(x, weight, bias)+++def quantize_int8(model: nn.Module, skip: Iterable[str] = ("lm_head",)) -> nn.Module:+ """Replace every ``nn.Linear`` (except names in ``skip``) by :class:`Int8Linear`, in place."""+ skip = set(skip)+ for parent in list(model.modules()):+ for name, child in list(parent.named_children()):+ if isinstance(child, nn.Linear) and name not in skip:+ setattr(parent, name, Int8Linear(child))+ return model+++def model_size_bytes(model: nn.Module) -> int:+ """Bytes of all parameters and buffers (shared tensors counted once)."""+ seen: set[int] = set()+ total = 0+ for t in [*model.parameters(), *model.buffers()]:+ if id(t) not in seen:+ seen.add(id(t))+ total += t.numel() * t.element_size()+ return totaldiff --git a/src/inference/speculative.py b/src/inference/speculative.pynew file mode 100644index 0000000..d257827--- /dev/null+++ b/src/inference/speculative.py@@ -0,0 +1,117 @@+"""+Speculative decoding (Leviathan et al. 2023; Chen et al. 2023), written from scratch.++Generating with a big model is slow because every token needs a full forward pass, and the+passes cannot overlap: token t+1 depends on token t. Speculative decoding uses a small,+fast *draft* model to guess the next ``k`` tokens, then runs the big *target* model ONCE on+all of them. The target's probabilities decide how many guesses to keep:++ for each draft token x with draft probability q(x) and target probability p(x):+ keep x with probability min(1, p(x) / q(x))+ at the first rejection, sample a replacement from max(0, p - q) (renormalized)+ if all k are kept, sample one bonus token from the target's next distribution++This rule makes the output distribution *exactly* the target model's distribution, no+matter how bad the draft is. A good draft just means more tokens kept per target pass. With+greedy decoding the rule reduces to "keep a guess if it equals the target's argmax", so the+result is identical to plain greedy decoding with the target.++This version recomputes the prefix on every call (no KV cache) to keep the algorithm in+view; :attr:`SpeculativeStats.target_calls` counts the expensive passes.+"""++from __future__ import annotations++from dataclasses import dataclass++import torch+import torch.nn.functional as F+from jaxtyping import Float, Int+from torch import Tensor++from src.inference.sampling import filter_logits+from src.models.factory import LanguageModel+++@dataclass+class SpeculativeStats:+ proposed: int = 0 # draft tokens proposed+ accepted: int = 0 # draft tokens the target kept+ target_calls: int = 0 # forward passes of the big model+ generated: int = 0 # tokens produced++ @property+ def acceptance_rate(self) -> float:+ return self.accepted / max(1, self.proposed)++ @property+ def tokens_per_target_call(self) -> float:+ return self.generated / max(1, self.target_calls)+++def _probs(logits: Float[Tensor, "n vocab"], temperature: float, top_k: int | None, greedy: bool) -> Float[Tensor, "n vocab"]:+ if greedy:+ return F.one_hot(logits.argmax(dim=-1), logits.size(-1)).float()+ return F.softmax(filter_logits(logits.float(), temperature, top_k), dim=-1)+++def _logits(model: LanguageModel, idx: Int[Tensor, "1 seq"]) -> Float[Tensor, "1 seq vocab"]:+ out = model(idx[:, -model.context_length :])+ return out[0] if isinstance(out, tuple) else out++[email protected]_grad()+def speculative_generate(+ target: LanguageModel,+ draft: LanguageModel,+ idx: Int[Tensor, "1 seq"],+ max_new_tokens: int,+ *,+ k: int = 4,+ temperature: float = 1.0,+ top_k: int | None = None,+ greedy: bool = False,+ generator: torch.Generator | None = None,+) -> tuple[Int[Tensor, "1 total"], SpeculativeStats]:+ """Generate ``max_new_tokens`` tokens after ``idx`` (batch size 1) with draft-then-verify."""+ if idx.size(0) != 1:+ raise ValueError("speculative_generate handles one sequence at a time")+ stats = SpeculativeStats()+ start = idx.size(1)+ while idx.size(1) - start < max_new_tokens:+ n = min(k, max_new_tokens - (idx.size(1) - start))+ # 1. The draft proposes n tokens, one at a time, remembering its probabilities q.+ x, guesses, q_rows = idx, [], []+ for _ in range(n):+ q = _probs(_logits(draft, x)[:, -1, :], temperature, top_k, greedy)+ tok = torch.multinomial(q, 1, generator=generator)+ guesses.append(tok)+ q_rows.append(q[0])+ x = torch.cat([x, tok], dim=1)+ stats.proposed += n++ # 2. One target pass scores all n guesses, plus the position after them.+ p_rows = _probs(_logits(target, x)[0, -(n + 1) :, :], temperature, top_k, greedy)+ stats.target_calls += 1++ # 3. Keep each guess with probability min(1, p / q); stop at the first rejection.+ kept: list[Tensor] = []+ next_tok = None+ for i, tok in enumerate(guesses):+ t = int(tok)+ ratio = p_rows[i, t] / q_rows[i][t].clamp(min=1e-12)+ if torch.rand((), generator=generator) < ratio.clamp(max=1.0):+ kept.append(tok)+ continue+ residual = (p_rows[i] - q_rows[i]).clamp(min=0)+ residual = residual / residual.sum() if residual.sum() > 0 else p_rows[i]+ next_tok = torch.multinomial(residual[None], 1, generator=generator)+ break+ if next_tok is None: # every guess was kept: the target's last row gives a bonus token+ next_tok = torch.multinomial(p_rows[n][None], 1, generator=generator)+ stats.accepted += len(kept)+ new = torch.cat([*kept, next_tok], dim=1) if kept else next_tok+ idx = torch.cat([idx, new], dim=1)+ idx = idx[:, : start + max_new_tokens]+ stats.generated = idx.size(1) - start+ return idx, statsdiff --git a/tests/test_inference.py b/tests/test_inference.pynew file mode 100644index 0000000..4894cad--- /dev/null+++ b/tests/test_inference.py@@ -0,0 +1,90 @@+"""Speculative decoding and int8 weight quantization."""++from __future__ import annotations++import copy++import pytest+import torch+import torch.nn as nn+import torch.nn.functional as F++from src.inference.quantize import Int8Linear, model_size_bytes, quantize_int8+from src.inference.speculative import speculative_generate+from src.models.modern import ModernConfig, ModernTransformer+from src.models.transformer import Transformer+++def _classic(seed: int, vocab: int = 13) -> Transformer:+ torch.manual_seed(seed)+ return Transformer(n_head=2, n_embed=16, context_length=32, vocab_size=vocab, N_BLOCKS=1).eval()+++def _greedy(model: nn.Module, idx: torch.Tensor, n: int) -> torch.Tensor:+ for _ in range(n):+ logits, _ = model(idx[:, -32:])+ idx = torch.cat([idx, logits[:, -1].argmax(-1, keepdim=True)], dim=1)+ return idx+++def test_greedy_speculative_decoding_equals_greedy_target_decoding() -> None:+ target, draft = _classic(0), _classic(1) # an unrelated, random draft+ prompt = torch.tensor([[1, 2, 3]])+ ours, stats = speculative_generate(target, draft, prompt, 20, k=4, greedy=True)+ assert torch.equal(ours, _greedy(target, prompt, 20))+ assert stats.generated == 20 and stats.target_calls <= 20+++def test_a_perfect_draft_is_always_accepted() -> None:+ target = _classic(0)+ g = torch.Generator().manual_seed(0)+ _, stats = speculative_generate(target, copy.deepcopy(target), torch.tensor([[1, 2]]), 24, k=4, generator=g)+ assert stats.acceptance_rate > 0.95+ assert stats.tokens_per_target_call > 4 # k accepted guesses + 1 bonus token per target pass+++def test_sampling_keeps_the_target_distribution_whatever_the_draft() -> None:+ target, draft = _classic(0, vocab=6), _classic(7, vocab=6)+ prompt = torch.tensor([[1, 4, 2]])+ with torch.no_grad():+ p_target = F.softmax(target(prompt)[0][0, -1], dim=-1)+ p_draft = F.softmax(draft(prompt)[0][0, -1], dim=-1)+ assert (p_target - p_draft).abs().max() > 0.05 # the draft really is different+ g = torch.Generator().manual_seed(0)+ counts = torch.zeros(6)+ n = 3000+ for _ in range(n):+ out, _ = speculative_generate(target, draft, prompt, 1, k=3, generator=g)+ counts[out[0, -1]] += 1+ assert (counts / n - p_target).abs().max() < 0.03+++def test_int8_linear_is_accurate_and_four_times_smaller() -> None:+ torch.manual_seed(0)+ linear = nn.Linear(256, 128)+ q = Int8Linear(linear)+ x = torch.randn(8, 256)+ with torch.no_grad():+ rel = (q(x) - linear(x)).norm() / linear(x).norm()+ assert rel < 0.01+ assert q.weight_int8.dtype == torch.int8+ assert model_size_bytes(q) < 0.3 * model_size_bytes(linear)++[email protected]("arch", ["classic", "modern"])+def test_quantized_models_give_nearly_the_same_logits(arch: str) -> None:+ torch.manual_seed(0)+ if arch == "classic":+ model: nn.Module = Transformer(n_head=4, n_embed=64, context_length=32, vocab_size=100, N_BLOCKS=2).eval()+ else:+ model = ModernTransformer(ModernConfig(vocab_size=100, context_length=32, n_embed=64, n_head=4, n_blocks=2)).eval()+ idx = torch.randint(0, 100, (2, 16))+ with torch.no_grad():+ before = model(idx)[0]+ quantized = quantize_int8(copy.deepcopy(model))+ after = quantized(idx)[0]+ assert F.cosine_similarity(before.flatten(), after.flatten(), dim=0) > 0.999+ assert any(isinstance(m, Int8Linear) for m in quantized.modules())+ assert model_size_bytes(quantized) < model_size_bytes(model)+ if arch == "modern": # the tied output layer is not quantized, so the tie survives+ assert quantized.lm_head.weight is quantized.token_embed.weight