Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
quant_ablation: add rotation preconditioning (-rot, QuaRot/QuIP#) and int3 schemes (#132, #81)
ZacharyZcRf8c0552c6da5
diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.pyindex c3c5bf9e..dd192ba9 100644--- a/c/tools/quant_ablation.py+++ b/c/tools/quant_ablation.py@@ -78,14 +78,51 @@ def _quant_last_dim(x, bits, group):return out.reshape(*out.shape[:-2], -1) if group else out-def quantize_param(w, bits, group):+# --------------------------------------------------------------------------------------+# Rotation preconditioning (QuaRot / QuIP# family, #81): multiply the input dimension by+# an orthogonal Q = diag(signs) @ H/sqrt(n) BEFORE quantizing, and by Q^T after — the+# round-trip Q4(W@Q)@Q.T measures exactly the weight error of a deployed scheme that+# stores W@Q quantized and rotates activations at runtime (W'@x' = W@x since [email protected] = I;+# the runtime cost is one O(D log D) transform per matmul INPUT, not per weight).+# Spreading outliers across the block is the point: absmax scales stop being hostage to+# one heavy coordinate, which is the failure mode #108 measured (margin erosion on MMLU).+# --------------------------------------------------------------------------------------+_ROT_CACHE = {}++def rotation(dim, device, seed=417):+ key = (dim, str(device))+ if key in _ROT_CACHE:+ return _ROT_CACHE[key]+ if dim & (dim - 1):+ raise SystemExit(f"-rot needs power-of-2 input dims (got {dim}); OLMoE dims are 2048/1024")+ h = torch.ones(1, 1, device=device, dtype=torch.float32)+ while h.shape[0] < dim: # Sylvester recursion+ h = torch.cat([torch.cat([h, h], 1), torch.cat([h, -h], 1)], 0)+ h /= h.shape[0] ** 0.5 # orthonormal+ g = torch.Generator().manual_seed(seed + dim)+ signs = (torch.randint(0, 2, (dim,), generator=g).float() * 2 - 1).to(device)+ q = signs[:, None] * h # Q = D @ H/sqrt(n), orthogonal+ _ROT_CACHE[key] = q+ return q+++def quantize_param(w, bits, group, rot=False):if w.ndim == 3: # fused experts [E, in, out] -> move input lastx = w.transpose(1, 2).contiguous()- return _quant_last_dim(x, bits, group).transpose(1, 2).contiguous()+ x = _rot_quant(x, bits, group) if rot else _quant_last_dim(x, bits, group)+ return x.transpose(1, 2).contiguous()+ if rot:+ return _rot_quant(w, bits, group)return _quant_last_dim(w, bits, group) # nn.Linear [out, in] -- input already last-SCHEME_RE = re.compile(r"^int(2|4|8)(?:-g(\d+))?(-nohead)?$")+def _rot_quant(x, bits, group):+ """W -> Qn(W@Q) @ Q^T along the last (input) dim — see rotation() above."""+ q = rotation(x.shape[-1], x.device)+ return (_quant_last_dim(x.float() @ q, bits, group) @ q.T).contiguous()+++SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-rot)?(-nohead)?$")def parse_scheme(name):@@ -94,8 +131,8 @@ def parse_scheme(name):return Nonem = SCHEME_RE.match(name)if not m:- raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,4,8}}[-g<N>][-nohead])")- return int(m.group(1)), int(m.group(2) or 0), bool(m.group(3))+ raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,3,4,8}}[-g<N>][-rot][-nohead])")+ return int(m.group(1)), int(m.group(2) or 0), bool(m.group(3)), bool(m.group(4))def is_router(name):@@ -115,7 +152,7 @@ def apply_scheme(model, scheme):spec = parse_scheme(scheme)if spec is None:return 0, 0, total- bits, group, skip_head = spec+ bits, group, rot, skip_head = specn = qp = 0with torch.no_grad():for name, p in model.named_parameters():@@ -123,7 +160,7 @@ def apply_scheme(model, scheme):continueif skip_head and is_head_or_embed(name):continue- p.data.copy_(quantize_param(p.data.float(), bits, group).to(p.dtype))+ p.data.copy_(quantize_param(p.data.float(), bits, group, rot).to(p.dtype))n += 1qp += p.numel()return n, qp, total