Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
tools: bytes-true 'iq3-pack' ablation scheme + inverse rotation (#452 step 2)
ZacharyZcRf7f68264e36a
diff --git a/c/tests/test_iq3_pack.py b/c/tests/test_iq3_pack.pyindex e2a2f318..7d70c987 100644--- a/c/tests/test_iq3_pack.py+++ b/c/tests/test_iq3_pack.py@@ -121,6 +121,12 @@ class TestIq3Pack(unittest.TestCase):self.assertTrue(np.allclose(wr @ xr, w @ x, rtol=1e-4, atol=1e-5),f"max |Ξ| = {np.abs(wr @ xr - w @ x).max()}")+ def test_unrotate_inverts_rotate(self):+ w = (np.random.randn(4, 768) * 0.05).astype(np.float32)+ back = P.unrotate_rows(P.rotate_rows(w))+ self.assertTrue(np.allclose(back, w, rtol=1e-5, atol=1e-7),+ f"max |Ξ| = {np.abs(back - w).max()}")+if __name__ == "__main__":unittest.main()diff --git a/c/tools/iq3_pack.py b/c/tools/iq3_pack.pyindex e51c9d79..c95a343a 100644--- a/c/tools/iq3_pack.py+++ b/c/tools/iq3_pack.py@@ -210,6 +210,19 @@ def rot_blocks(dim):return out+def _fwht_rows(blk, b):+ """In-place-style FWHT over rows of [..., b] (b a power of two)."""+ h = 1+ while h < b:+ blk = blk.reshape(-1, b // (2 * h), 2, h)+ u, v = blk[:, :, 0, :].copy(), blk[:, :, 1, :].copy()+ blk[:, :, 0, :] = u + v+ blk[:, :, 1, :] = u - v+ blk = blk.reshape(-1, b)+ h <<= 1+ return blk / np.sqrt(b, dtype=np.float32)++def rotate_rows(x):"""Apply the rotation to rows of float32 [..., dim] (weights W@Q oractivations Q^T x β the transform is the same). Returns a new array."""@@ -218,15 +231,19 @@ def rotate_rows(x):rows = x.reshape(-1, dim).copy()off = 0for b in rot_blocks(dim):- blk = rows[:, off:off + b] * signs(b)[None, :]- h = 1- while h < b: # in-place FWHT, vectorized over rows- blk = blk.reshape(-1, b // (2 * h), 2, h)- u, v = blk[:, :, 0, :].copy(), blk[:, :, 1, :].copy()- blk[:, :, 0, :] = u + v- blk[:, :, 1, :] = u - v- blk = blk.reshape(-1, b)- h <<= 1- rows[:, off:off + b] = blk / np.sqrt(b, dtype=np.float32)+ rows[:, off:off + b] = _fwht_rows(rows[:, off:off + b] * signs(b)[None, :], b)+ off += b+ return rows.reshape(x.shape)+++def unrotate_rows(x):+ """Inverse of rotate_rows (Q v back to v): FWHT first, then the sign flip.+ Eval-side only β the engine always works in the rotated space."""+ x = np.ascontiguousarray(x, dtype=np.float32)+ dim = x.shape[-1]+ rows = x.reshape(-1, dim).copy()+ off = 0+ for b in rot_blocks(dim):+ rows[:, off:off + b] = _fwht_rows(rows[:, off:off + b].copy(), b) * signs(b)[None, :]off += breturn rows.reshape(x.shape)diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.pyindex a602ae8b..f6995f4e 100644--- a/c/tools/quant_ablation.py+++ b/c/tools/quant_ablation.py@@ -277,10 +277,27 @@ def _quant_e8(x, group, bits, ball):SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?|-iq3)?(-rot)?(-nohead)?$")+def _quant_iq3_pack(x):+ """Bytes-true fmt=6 (#452 step 2): the tensor goes through tools/iq3_pack β+ real 98-byte blocks, the parity sign cost, fp16 super-scales, regenerated+ rotation signs β and comes back to model space via the inverse rotation.+ This is the number a SHIPPED container gets, where `int3-iq3-rot` above is+ the float simulation that chose the scheme. The rotation sign diagonal+ differs from the simulation's torch PRNG (xorshift spec, quant.h e8_signs);+ statistically equivalent, and it is what the engine regenerates."""+ import iq3_pack as IP+ w = x.detach().cpu().float().numpy()+ packed = IP.encode(IP.rotate_rows(w.reshape(-1, w.shape[-1])))+ deq = IP.decode(packed, w.shape[-1])+ return torch.from_numpy(IP.unrotate_rows(deq).reshape(w.shape)).to(x.device)++def parse_scheme(name):"""'int4-g128-nohead' -> (bits, group, e8, skip_head...). 'fp16' -> None."""if name == "fp16":return None+ if name == "iq3-pack":+ return "PACK"m = SCHEME_RE.match(name)if not m:raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,3,4,8}}[-g<N>][-e8|-e8u][-rot][-nohead])")@@ -304,8 +321,19 @@ def apply_scheme(model, scheme):spec = parse_scheme(scheme)if spec is None:return 0, 0, total- bits, group, e8, rot, skip_head = specn = qp = 0+ if spec == "PACK":+ with torch.no_grad():+ for name, p in model.named_parameters():+ if p.ndim < 2 or is_router(name):+ continue+ if p.shape[-1] % 256: # fmt=6 needs I % 256 == 0 β same rule the converter enforces+ continue+ p.data.copy_(_quant_iq3_pack(p.data).to(p.dtype))+ n += 1+ qp += p.numel()+ return n, qp, total+ bits, group, e8, rot, skip_head = specwith torch.no_grad():for name, p in model.named_parameters():if p.ndim < 2 or is_router(name):