Skip to content

A straightforward method for training your LLM, from downloading data to generating text.

feat(inference): speculative decoding and int8 weight quantization

FareedKhan-dev2ebb64cdb39c
diff --git a/src/inference/quantize.py b/src/inference/quantize.py
new file mode 100644
index 0000000..461b290
--- /dev/null
+++ b/src/inference/quantize.py
@@ -0,0 +1,74 @@
+"""
+Int8 weight-only quantization, written from scratch.
+
+A float32 weight takes 4 bytes; an int8 takes 1. Weight-only quantization stores every
+``nn.Linear`` weight matrix as int8 plus one float scale per output row:
+
+ scale_r = max(|W_r|) / 127 (per output channel, "absmax")
+ Q_r = round(W_r / scale_r) (integers in [-127, 127])
+ W_r ~= Q_r * scale_r (dequantized on the fly in forward)
+
+The matmul still runs in floating point, so this saves memory (about 4x for the linear
+layers) rather than compute. That is the main win for inference on small devices, where
+memory bandwidth, not arithmetic, limits token speed. Per-channel scales keep the error
+small because each row gets its own range.
+
+The output layer is left alone by default: when its weight is tied to the input embedding,
+quantizing it would silently untie them, and the vocabulary projection is the most
+sensitive layer anyway.
+"""
+
+from __future__ import annotations
+
+from collections.abc import Iterable
+
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+from jaxtyping import Float
+from torch import Tensor
+
+
+class Int8Linear(nn.Module):
+ """A frozen ``nn.Linear`` whose weight is stored as int8 with one scale per output row."""
+
+ weight_int8: Tensor
+ scale: Tensor
+
+ def __init__(self, linear: nn.Linear) -> None:
+ super().__init__()
+ self.in_features, self.out_features = linear.in_features, linear.out_features
+ w = linear.weight.detach().float()
+ scale = w.abs().amax(dim=1, keepdim=True).clamp(min=1e-8) / 127.0
+ self.register_buffer("weight_int8", torch.round(w / scale).clamp(-127, 127).to(torch.int8))
+ self.register_buffer("scale", scale)
+ self.bias = None if linear.bias is None else nn.Parameter(linear.bias.detach().clone(), requires_grad=False)
+
+ def dequantized_weight(self) -> Float[Tensor, "out in"]:
+ return self.weight_int8.float() * self.scale
+
+ def forward(self, x: Float[Tensor, "*batch in_features"]) -> Float[Tensor, "*batch out_features"]:
+ weight = self.dequantized_weight().to(x.dtype)
+ bias = None if self.bias is None else self.bias.to(x.dtype)
+ return F.linear(x, weight, bias)
+
+
+def quantize_int8(model: nn.Module, skip: Iterable[str] = ("lm_head",)) -> nn.Module:
+ """Replace every ``nn.Linear`` (except names in ``skip``) by :class:`Int8Linear`, in place."""
+ skip = set(skip)
+ for parent in list(model.modules()):
+ for name, child in list(parent.named_children()):
+ if isinstance(child, nn.Linear) and name not in skip:
+ setattr(parent, name, Int8Linear(child))
+ return model
+
+
+def model_size_bytes(model: nn.Module) -> int:
+ """Bytes of all parameters and buffers (shared tensors counted once)."""
+ seen: set[int] = set()
+ total = 0
+ for t in [*model.parameters(), *model.buffers()]:
+ if id(t) not in seen:
+ seen.add(id(t))
+ total += t.numel() * t.element_size()
+ return total
diff --git a/src/inference/speculative.py b/src/inference/speculative.py
new file mode 100644
index 0000000..d257827
--- /dev/null
+++ b/src/inference/speculative.py
@@ -0,0 +1,117 @@
+"""
+Speculative decoding (Leviathan et al. 2023; Chen et al. 2023), written from scratch.
+
+Generating with a big model is slow because every token needs a full forward pass, and the
+passes cannot overlap: token t+1 depends on token t. Speculative decoding uses a small,
+fast *draft* model to guess the next ``k`` tokens, then runs the big *target* model ONCE on
+all of them. The target's probabilities decide how many guesses to keep:
+
+ for each draft token x with draft probability q(x) and target probability p(x):
+ keep x with probability min(1, p(x) / q(x))
+ at the first rejection, sample a replacement from max(0, p - q) (renormalized)
+ if all k are kept, sample one bonus token from the target's next distribution
+
+This rule makes the output distribution *exactly* the target model's distribution, no
+matter how bad the draft is. A good draft just means more tokens kept per target pass. With
+greedy decoding the rule reduces to "keep a guess if it equals the target's argmax", so the
+result is identical to plain greedy decoding with the target.
+
+This version recomputes the prefix on every call (no KV cache) to keep the algorithm in
+view; :attr:`SpeculativeStats.target_calls` counts the expensive passes.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+import torch
+import torch.nn.functional as F
+from jaxtyping import Float, Int
+from torch import Tensor
+
+from src.inference.sampling import filter_logits
+from src.models.factory import LanguageModel
+
+
+@dataclass
+class SpeculativeStats:
+ proposed: int = 0 # draft tokens proposed
+ accepted: int = 0 # draft tokens the target kept
+ target_calls: int = 0 # forward passes of the big model
+ generated: int = 0 # tokens produced
+
+ @property
+ def acceptance_rate(self) -> float:
+ return self.accepted / max(1, self.proposed)
+
+ @property
+ def tokens_per_target_call(self) -> float:
+ return self.generated / max(1, self.target_calls)
+
+
+def _probs(logits: Float[Tensor, "n vocab"], temperature: float, top_k: int | None, greedy: bool) -> Float[Tensor, "n vocab"]:
+ if greedy:
+ return F.one_hot(logits.argmax(dim=-1), logits.size(-1)).float()
+ return F.softmax(filter_logits(logits.float(), temperature, top_k), dim=-1)
+
+
+def _logits(model: LanguageModel, idx: Int[Tensor, "1 seq"]) -> Float[Tensor, "1 seq vocab"]:
+ out = model(idx[:, -model.context_length :])
+ return out[0] if isinstance(out, tuple) else out
+
+
+def speculative_generate(
+ target: LanguageModel,
+ draft: LanguageModel,
+ idx: Int[Tensor, "1 seq"],
+ max_new_tokens: int,
+ *,
+ k: int = 4,
+ temperature: float = 1.0,
+ top_k: int | None = None,
+ greedy: bool = False,
+ generator: torch.Generator | None = None,
+) -> tuple[Int[Tensor, "1 total"], SpeculativeStats]:
+ """Generate ``max_new_tokens`` tokens after ``idx`` (batch size 1) with draft-then-verify."""
+ if idx.size(0) != 1:
+ raise ValueError("speculative_generate handles one sequence at a time")
+ stats = SpeculativeStats()
+ start = idx.size(1)
+ while idx.size(1) - start < max_new_tokens:
+ n = min(k, max_new_tokens - (idx.size(1) - start))
+ # 1. The draft proposes n tokens, one at a time, remembering its probabilities q.
+ x, guesses, q_rows = idx, [], []
+ for _ in range(n):
+ q = _probs(_logits(draft, x)[:, -1, :], temperature, top_k, greedy)
+ tok = torch.multinomial(q, 1, generator=generator)
+ guesses.append(tok)
+ q_rows.append(q[0])
+ x = torch.cat([x, tok], dim=1)
+ stats.proposed += n
+
+ # 2. One target pass scores all n guesses, plus the position after them.
+ p_rows = _probs(_logits(target, x)[0, -(n + 1) :, :], temperature, top_k, greedy)
+ stats.target_calls += 1
+
+ # 3. Keep each guess with probability min(1, p / q); stop at the first rejection.
+ kept: list[Tensor] = []
+ next_tok = None
+ for i, tok in enumerate(guesses):
+ t = int(tok)
+ ratio = p_rows[i, t] / q_rows[i][t].clamp(min=1e-12)
+ if torch.rand((), generator=generator) < ratio.clamp(max=1.0):
+ kept.append(tok)
+ continue
+ residual = (p_rows[i] - q_rows[i]).clamp(min=0)
+ residual = residual / residual.sum() if residual.sum() > 0 else p_rows[i]
+ next_tok = torch.multinomial(residual[None], 1, generator=generator)
+ break
+ if next_tok is None: # every guess was kept: the target's last row gives a bonus token
+ next_tok = torch.multinomial(p_rows[n][None], 1, generator=generator)
+ stats.accepted += len(kept)
+ new = torch.cat([*kept, next_tok], dim=1) if kept else next_tok
+ idx = torch.cat([idx, new], dim=1)
+ idx = idx[:, : start + max_new_tokens]
+ stats.generated = idx.size(1) - start
+ return idx, stats
diff --git a/tests/test_inference.py b/tests/test_inference.py
new file mode 100644
index 0000000..4894cad
--- /dev/null
+++ b/tests/test_inference.py
@@ -0,0 +1,90 @@
+"""Speculative decoding and int8 weight quantization."""
+
+from __future__ import annotations
+
+import copy
+
+import pytest
+import torch
+import torch.nn as nn
+import torch.nn.functional as F
+
+from src.inference.quantize import Int8Linear, model_size_bytes, quantize_int8
+from src.inference.speculative import speculative_generate
+from src.models.modern import ModernConfig, ModernTransformer
+from src.models.transformer import Transformer
+
+
+def _classic(seed: int, vocab: int = 13) -> Transformer:
+ torch.manual_seed(seed)
+ return Transformer(n_head=2, n_embed=16, context_length=32, vocab_size=vocab, N_BLOCKS=1).eval()
+
+
+def _greedy(model: nn.Module, idx: torch.Tensor, n: int) -> torch.Tensor:
+ for _ in range(n):
+ logits, _ = model(idx[:, -32:])
+ idx = torch.cat([idx, logits[:, -1].argmax(-1, keepdim=True)], dim=1)
+ return idx
+
+
+def test_greedy_speculative_decoding_equals_greedy_target_decoding() -> None:
+ target, draft = _classic(0), _classic(1) # an unrelated, random draft
+ prompt = torch.tensor([[1, 2, 3]])
+ ours, stats = speculative_generate(target, draft, prompt, 20, k=4, greedy=True)
+ assert torch.equal(ours, _greedy(target, prompt, 20))
+ assert stats.generated == 20 and stats.target_calls <= 20
+
+
+def test_a_perfect_draft_is_always_accepted() -> None:
+ target = _classic(0)
+ g = torch.Generator().manual_seed(0)
+ _, stats = speculative_generate(target, copy.deepcopy(target), torch.tensor([[1, 2]]), 24, k=4, generator=g)
+ assert stats.acceptance_rate > 0.95
+ assert stats.tokens_per_target_call > 4 # k accepted guesses + 1 bonus token per target pass
+
+
+def test_sampling_keeps_the_target_distribution_whatever_the_draft() -> None:
+ target, draft = _classic(0, vocab=6), _classic(7, vocab=6)
+ prompt = torch.tensor([[1, 4, 2]])
+ with torch.no_grad():
+ p_target = F.softmax(target(prompt)[0][0, -1], dim=-1)
+ p_draft = F.softmax(draft(prompt)[0][0, -1], dim=-1)
+ assert (p_target - p_draft).abs().max() > 0.05 # the draft really is different
+ g = torch.Generator().manual_seed(0)
+ counts = torch.zeros(6)
+ n = 3000
+ for _ in range(n):
+ out, _ = speculative_generate(target, draft, prompt, 1, k=3, generator=g)
+ counts[out[0, -1]] += 1
+ assert (counts / n - p_target).abs().max() < 0.03
+
+
+def test_int8_linear_is_accurate_and_four_times_smaller() -> None:
+ torch.manual_seed(0)
+ linear = nn.Linear(256, 128)
+ q = Int8Linear(linear)
+ x = torch.randn(8, 256)
+ with torch.no_grad():
+ rel = (q(x) - linear(x)).norm() / linear(x).norm()
+ assert rel < 0.01
+ assert q.weight_int8.dtype == torch.int8
+ assert model_size_bytes(q) < 0.3 * model_size_bytes(linear)
+
+
[email protected]("arch", ["classic", "modern"])
+def test_quantized_models_give_nearly_the_same_logits(arch: str) -> None:
+ torch.manual_seed(0)
+ if arch == "classic":
+ model: nn.Module = Transformer(n_head=4, n_embed=64, context_length=32, vocab_size=100, N_BLOCKS=2).eval()
+ else:
+ model = ModernTransformer(ModernConfig(vocab_size=100, context_length=32, n_embed=64, n_head=4, n_blocks=2)).eval()
+ idx = torch.randint(0, 100, (2, 16))
+ with torch.no_grad():
+ before = model(idx)[0]
+ quantized = quantize_int8(copy.deepcopy(model))
+ after = quantized(idx)[0]
+ assert F.cosine_similarity(before.flatten(), after.flatten(), dim=0) > 0.999
+ assert any(isinstance(m, Int8Linear) for m in quantized.modules())
+ assert model_size_bytes(quantized) < model_size_bytes(model)
+ if arch == "modern": # the tied output layer is not quantized, so the tie survives
+ assert quantized.lm_head.weight is quantized.token_embed.weight