Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tools: bytes-true 'iq3-pack' ablation scheme + inverse rotation (#452 step 2)

ZacharyZcRf7f68264e36a
diff --git a/c/tests/test_iq3_pack.py b/c/tests/test_iq3_pack.py
index e2a2f318..7d70c987 100644
--- a/c/tests/test_iq3_pack.py
+++ b/c/tests/test_iq3_pack.py
@@ -121,6 +121,12 @@ class TestIq3Pack(unittest.TestCase):
self.assertTrue(np.allclose(wr @ xr, w @ x, rtol=1e-4, atol=1e-5),
f"max |Ξ”| = {np.abs(wr @ xr - w @ x).max()}")
+ def test_unrotate_inverts_rotate(self):
+ w = (np.random.randn(4, 768) * 0.05).astype(np.float32)
+ back = P.unrotate_rows(P.rotate_rows(w))
+ self.assertTrue(np.allclose(back, w, rtol=1e-5, atol=1e-7),
+ f"max |Ξ”| = {np.abs(back - w).max()}")
+
if __name__ == "__main__":
unittest.main()
diff --git a/c/tools/iq3_pack.py b/c/tools/iq3_pack.py
index e51c9d79..c95a343a 100644
--- a/c/tools/iq3_pack.py
+++ b/c/tools/iq3_pack.py
@@ -210,6 +210,19 @@ def rot_blocks(dim):
return out
+def _fwht_rows(blk, b):
+ """In-place-style FWHT over rows of [..., b] (b a power of two)."""
+ h = 1
+ while h < b:
+ blk = blk.reshape(-1, b // (2 * h), 2, h)
+ u, v = blk[:, :, 0, :].copy(), blk[:, :, 1, :].copy()
+ blk[:, :, 0, :] = u + v
+ blk[:, :, 1, :] = u - v
+ blk = blk.reshape(-1, b)
+ h <<= 1
+ return blk / np.sqrt(b, dtype=np.float32)
+
+
def rotate_rows(x):
"""Apply the rotation to rows of float32 [..., dim] (weights W@Q or
activations Q^T x β€” the transform is the same). Returns a new array."""
@@ -218,15 +231,19 @@ def rotate_rows(x):
rows = x.reshape(-1, dim).copy()
off = 0
for b in rot_blocks(dim):
- blk = rows[:, off:off + b] * signs(b)[None, :]
- h = 1
- while h < b: # in-place FWHT, vectorized over rows
- blk = blk.reshape(-1, b // (2 * h), 2, h)
- u, v = blk[:, :, 0, :].copy(), blk[:, :, 1, :].copy()
- blk[:, :, 0, :] = u + v
- blk[:, :, 1, :] = u - v
- blk = blk.reshape(-1, b)
- h <<= 1
- rows[:, off:off + b] = blk / np.sqrt(b, dtype=np.float32)
+ rows[:, off:off + b] = _fwht_rows(rows[:, off:off + b] * signs(b)[None, :], b)
+ off += b
+ return rows.reshape(x.shape)
+
+
+def unrotate_rows(x):
+ """Inverse of rotate_rows (Q v back to v): FWHT first, then the sign flip.
+ Eval-side only β€” the engine always works in the rotated space."""
+ x = np.ascontiguousarray(x, dtype=np.float32)
+ dim = x.shape[-1]
+ rows = x.reshape(-1, dim).copy()
+ off = 0
+ for b in rot_blocks(dim):
+ rows[:, off:off + b] = _fwht_rows(rows[:, off:off + b].copy(), b) * signs(b)[None, :]
off += b
return rows.reshape(x.shape)
diff --git a/c/tools/quant_ablation.py b/c/tools/quant_ablation.py
index a602ae8b..f6995f4e 100644
--- a/c/tools/quant_ablation.py
+++ b/c/tools/quant_ablation.py
@@ -277,10 +277,27 @@ def _quant_e8(x, group, bits, ball):
SCHEME_RE = re.compile(r"^int(2|3|4|8)(?:-g(\d+))?(-e8u?|-iq3)?(-rot)?(-nohead)?$")
+def _quant_iq3_pack(x):
+ """Bytes-true fmt=6 (#452 step 2): the tensor goes through tools/iq3_pack β€”
+ real 98-byte blocks, the parity sign cost, fp16 super-scales, regenerated
+ rotation signs β€” and comes back to model space via the inverse rotation.
+ This is the number a SHIPPED container gets, where `int3-iq3-rot` above is
+ the float simulation that chose the scheme. The rotation sign diagonal
+ differs from the simulation's torch PRNG (xorshift spec, quant.h e8_signs);
+ statistically equivalent, and it is what the engine regenerates."""
+ import iq3_pack as IP
+ w = x.detach().cpu().float().numpy()
+ packed = IP.encode(IP.rotate_rows(w.reshape(-1, w.shape[-1])))
+ deq = IP.decode(packed, w.shape[-1])
+ return torch.from_numpy(IP.unrotate_rows(deq).reshape(w.shape)).to(x.device)
+
+
def parse_scheme(name):
"""'int4-g128-nohead' -> (bits, group, e8, skip_head...). 'fp16' -> None."""
if name == "fp16":
return None
+ if name == "iq3-pack":
+ return "PACK"
m = SCHEME_RE.match(name)
if not m:
raise SystemExit(f"bad scheme '{name}' (expected fp16 | int{{2,3,4,8}}[-g<N>][-e8|-e8u][-rot][-nohead])")
@@ -304,8 +321,19 @@ def apply_scheme(model, scheme):
spec = parse_scheme(scheme)
if spec is None:
return 0, 0, total
- bits, group, e8, rot, skip_head = spec
n = qp = 0
+ if spec == "PACK":
+ with torch.no_grad():
+ for name, p in model.named_parameters():
+ if p.ndim < 2 or is_router(name):
+ continue
+ if p.shape[-1] % 256: # fmt=6 needs I % 256 == 0 β€” same rule the converter enforces
+ continue
+ p.data.copy_(_quant_iq3_pack(p.data).to(p.dtype))
+ n += 1
+ qp += p.numel()
+ return n, qp, total
+ bits, group, e8, rot, skip_head = spec
with torch.no_grad():
for name, p in model.named_parameters():
if p.ndim < 2 or is_router(name):