Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tools: E8 lattice quantization for the ablation harness — int{2,3,4,8}[-gN][-e8|-e8u][-rot] (#81)
ZacharyZcR0189a9f0ae97
diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.pyindex 7a870ffb..689643af 100644--- a/c/tools/quant_ablation.py+++ b/c/tools/quant_ablation.py@@ -106,33 +106,106 @@ def rotation(dim, device, seed=417):return q-def quantize_param(w, bits, group, rot=False):+def quantize_param(w, bits, group, rot=False, e8=""):if w.ndim == 3: # fused experts [E, in, out] -> move input lastx = w.transpose(1, 2).contiguous()- x = _rot_quant(x, bits, group) if rot else _quant_last_dim(x, bits, group)+ x = _rot_quant(x, bits, group, e8) if rot else _grid_or_e8(x, bits, group, e8)return x.transpose(1, 2).contiguous()if rot:- return _rot_quant(w, bits, group)- return _quant_last_dim(w, bits, group) # nn.Linear [out, in] -- input already last+ return _rot_quant(w, bits, group, e8)+ return _grid_or_e8(w, bits, group, e8) # nn.Linear [out, in] -- input already last-def _rot_quant(x, bits, group):+def _grid_or_e8(x, bits, group, e8):+ if e8:+ return _quant_e8(x.float(), group, ball=(e8 == "-e8"))+ return _quant_last_dim(x, bits, group)+++def _rot_quant(x, bits, group, e8=""):"""W -> Qn(W@Q) @ Q^T along the last (input) dim — see rotation() above."""q = rotation(x.shape[-1], x.device)- return (_quant_last_dim(x.float() @ q, bits, group) @ q.T).contiguous()+ return (_grid_or_e8(x.float() @ q, bits, group, e8) @ q.T).contiguous()-SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-rot)?(-nohead)?$")+# --------------------------------------------------------------------------------------+# E8 lattice quantization (#81 follow-up): the -rot schemes above are QuaRot (rotation ++# uniform grid). QuIP#'s 2-bit result needs the second ingredient — an E8 lattice codebook+# instead of the grid. E8 = D8 ∪ (D8 + 1/2), nearest point via Conway–Sloane: round every+# coordinate, and if the sum is odd re-round the worst coordinate the other way; repeat on+# the half-shifted copy and keep the closer of the two. `-e8` clamps points to |p|^2 <= 10,+# the E8P ball QuIP# builds its 2^16 codebook from (2 bits/weight for 8-dim blocks);+# `-e8u` leaves the lattice unbounded — an ideal-codebook upper bound, not a deployable rate.+# Scale: per group, a small MSE search over multiples of the block RMS (absmax is the wrong+# statistic for a lattice — the ball wants energy matched, not the peak).+# --------------------------------------------------------------------------------------+def _d8_nearest(y):+ f = torch.round(y)+ d = y - f+ odd = (f.sum(-1).long() & 1).bool()+ idx = d.abs().argmax(-1, keepdim=True)+ step = torch.where(d.gather(-1, idx) >= 0, 1.0, -1.0)+ flipped = f.gather(-1, idx) + step+ return f.scatter(-1, idx, torch.where(odd[..., None], flipped, f.gather(-1, idx)))+++def _e8_nearest(y):+ a = _d8_nearest(y)+ b = _d8_nearest(y - 0.5) + 0.5+ da = ((y - a) ** 2).sum(-1, keepdim=True)+ db = ((y - b) ** 2).sum(-1, keepdim=True)+ return torch.where(da <= db, a, b)+++def _e8_ball(y, r2=10.0):+ p = _e8_nearest(y)+ for _ in range(8): # shrink-and-requantize until inside+ n2 = (p ** 2).sum(-1, keepdim=True)+ over = n2 > r2 + 1e-6+ if not over.any():+ break+ y = torch.where(over, y * torch.sqrt(r2 / torch.clamp(n2, min=r2)) * 0.98, y)+ p = torch.where(over, _e8_nearest(y), p)+ return p+++def _quant_e8(x, group, ball):+ """Blocks of 8 along the input dim; per-group scale by MSE search over RMS multiples."""+ if x.shape[-1] % 8:+ raise SystemExit(f"-e8 needs input dim divisible by 8 (got {x.shape[-1]})")+ g = group or x.shape[-1]+ if g % 8:+ raise SystemExit(f"-e8 group {g} must be a multiple of 8")+ shp = x.shape+ xg = x.reshape(-1, g) # [G, g]+ rms = torch.clamp(xg.pow(2).mean(-1, keepdim=True).sqrt(), min=1e-8)+ best_out, best_err = None, None+ for k in (0.5, 0.7, 0.9, 1.1, 1.4, 1.8, 2.4):+ s = rms * k+ yb = (xg / s).reshape(-1, g // 8, 8)+ p = _e8_ball(yb) if ball else _e8_nearest(yb)+ out = (p.reshape(-1, g) * s)+ err = (out - xg).pow(2).sum(-1, keepdim=True)+ if best_err is None:+ best_out, best_err = out, err+ else:+ take = err < best_err+ best_out = torch.where(take, out, best_out)+ best_err = torch.where(take, err, best_err)+ return best_out.reshape(shp)+++SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?)?(-rot)?(-nohead)?$")def parse_scheme(name):- """'int4-g128-nohead' -> (bits=4, group=128, skip_head=True). 'fp16' -> None."""+ """'int4-g128-nohead' -> (bits, group, e8, skip_head...). 'fp16' -> None."""if name == "fp16":return Nonem = SCHEME_RE.match(name)if not m:- raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,3,4,8}}[-g<N>][-rot][-nohead])")- return int(m.group(1)), int(m.group(2) or 0), bool(m.group(3)), bool(m.group(4))+ raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,3,4,8}}[-g<N>][-e8|-e8u][-rot][-nohead])")+ return int(m.group(1)), int(m.group(2) or 0), m.group(3) or "", bool(m.group(4)), bool(m.group(5))def is_router(name):@@ -152,7 +225,7 @@ def apply_scheme(model, scheme):spec = parse_scheme(scheme)if spec is None:return 0, 0, total- bits, group, rot, skip_head = spec+ bits, group, e8, rot, skip_head = specn = qp = 0with torch.no_grad():for name, p in model.named_parameters():@@ -160,7 +233,7 @@ def apply_scheme(model, scheme):continueif skip_head and is_head_or_embed(name):continue- p.data.copy_(quantize_param(p.data.float(), bits, group, rot).to(p.dtype))+ p.data.copy_(quantize_param(p.data.float(), bits, group, rot, e8).to(p.dtype))n += 1qp += p.numel()return n, qp, total