Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
convert: --xbits e8 β produce and load the fmt=6 E8/IQ3 container (#452 step 4)
ZacharyZcR3634001d981e
diff --git a/c/colibri.c b/c/colibri.cindex ec4c2173..f7d609d0 100644--- a/c/colibri.c+++ b/c/colibri.c@@ -123,6 +123,8 @@ static int64_t qt_bytes(const QT *t){ /* byte residenti del tensore */* included below β keep the arithmetic literal here) */int64_t ng=((int64_t)t->I+63)/64;return (int64_t)t->O*ng*24 + (int64_t)t->O*ng*4; }+ if(t->fmt==6) /* E8/IQ3: 98B per 256 weights, scales in-block, .qs is a 4-byte tag */+ return (int64_t)t->O*(((int64_t)t->I+255)/256)*98 + 4;return (int64_t)t->O*((t->I+1)/2) + (int64_t)t->O*4; /* fmt=2 int4 per-row */}@@ -289,7 +291,7 @@ static void qt_cuda_reset(QT *t){t->cuda_failed=0;}static int qt_cuda_upload(QT *t){- if(t->fmt==5) return 0; /* int3-g64: no CUDA kernel yet β tensor stays CPU-side */+ if(t->fmt==5||t->fmt==6) return 0; /* int3-g64 / E8: no CUDA kernel yet β tensor stays CPU-side */const void *weights = t->fmt==0 ? (const void*)t->qf: t->fmt==1 ? (const void*)t->q8 : (const void*)t->q4;if(t->fmt==4) /* grouped int4 (#334): scales are [O, ceil(I/gs)] β the plain@@ -963,6 +965,10 @@ static int detect_group_size(int O, int I, int64_t ns){static int qt_resolve_fmt(const char *name, int O, int I, int64_t nb, int64_t ns, int *gs){int64_t exp_i8=(int64_t)O*I, exp_i4=(int64_t)O*((I+1)/2), exp_i2=(int64_t)O*((I+3)/4);int64_t exp_i3=(int64_t)O*i3_rowbytes(I); /* int3-g64 (fmt=5): 24B per 64-input group */+ /* fmt=6 (E8/IQ3, #452): scales live inside the 98B super-blocks, so the .qs+ * convention is kept with a single-float tag β ns==4 is the discriminator+ * (every other format carries at least O floats of real scales). */+ if(ns==4 && nb==(int64_t)O*e8_rowbytes(I)){ *gs=0; return 6; }/* Row formats take precedence: for tiny I the int3-g64 byte count can coincide with* a row layout (e.g. [O,48]: ceil(48/2)=24=1*24). For real tensor shapes the counts* are distinct, and the weight bytes β not the scale size β are the int3 tag, because@@ -1001,6 +1007,9 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, intelse if(fmt==5){ int64_t ng=i3_groups(I); /* int3-g64: 24B/group weights + O*ng group scales */if(t->fmt!=5||!t->q4){ t->fmt=5; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=falloc((int64_t)O*ng); }st_read_raw(&m->S,name,t->q4,drop); }+ else if(fmt==6){ /* E8/IQ3: everything in-block, .qs is the 4-byte tag */+ if(t->fmt!=6||!t->q4){ t->fmt=6; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(1); }+ st_read_raw(&m->S,name,t->q4,drop); }else { if(t->fmt!=fmt||!t->q4){ t->fmt=fmt; t->O=O; t->I=I; t->gs=0; t->q4=qalloc(nb); t->s=qsalloc(O); } st_read_raw(&m->S,name,t->q4,drop); }/* cap MUST match the scale cardinality qt_resolve_fmt already validated and* the falloc above actually reserved, per format: grouped-int4 (fmt=4) keeps@@ -1009,7 +1018,8 @@ static void qt_from_disk(Model *m, const char *name, int O, int I, int bits, int* reject a legitimate container (fmt=5 regressed exactly that way). */st_read_f32_cap(&m->S,sn,t->s,fmt==4 ? (int64_t)O*((I+gs-1)/gs) :- fmt==5 ? (int64_t)O*i3_groups(I) : (int64_t)O, drop);+ fmt==5 ? (int64_t)O*i3_groups(I) :+ fmt==6 ? (int64_t)1 : (int64_t)O, drop);} else {if(!t->qf && !t->q8 && !t->q4) qt_alloc(t,O,I,bits);if(t->fmt==0) st_read_f32_cap(&m->S,name,t->qf,(int64_t)O*I,drop);@@ -2977,6 +2987,8 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}/* ---- FASE C/D: risolvi (pin/cache/disco) e calcola, a blocchi di 64 unici ---- */float *xg=falloc((int64_t)S*D), *gg=falloc((int64_t)S*I), *uu=falloc((int64_t)S*I), *hh=falloc((int64_t)S*D);+ float *xe=NULL; /* fmt=6: x under the rotation Q^T, built once per call β all routed+ * experts of the layer share it (the placement rule in quant.h) */int *rows=malloc(S*sizeof(int)); float *rw=malloc(S*sizeof(float));#ifdef COLI_CUDA/* PIPE Inc.1b: il batch-union del prefill passa dai gruppi GPU β prima di@@ -3289,7 +3301,12 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, intngroup++; continue;}#endif- for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, x+(int64_t)rows[r]*D, D*sizeof(float));+ const float *xsrc=x;+ if(e->g.fmt==6){+ if(!xe){ xe=falloc((int64_t)S*D); memcpy(xe,x,(size_t)S*D*sizeof(float)); e8_rot_rows(xe,S,D); }+ xsrc=xe;+ }+ for(int r=0;r<nr;r++) memcpy(xg+(int64_t)r*D, xsrc+(int64_t)rows[r]*D, D*sizeof(float));double t0=now_s();#ifdef COLI_CUDAif(!group_enabled && g_cuda_enabled && e->g.cuda_eligible && e->u.cuda_eligible &&@@ -3303,6 +3320,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int#endifexpert_gate_up(gg,uu,xg,&e->g,&e->u,nr);for(int64_t z=0;z<(int64_t)nr*I;z++) gg[z]=siluf(gg[z])*uu[z];+ if(e->d.fmt==6) e8_rot_rows(gg,nr,I); /* down input is per-expert β rotate here */matmul_qt(hh, gg, &e->d, nr);for(int r=0;r<nr;r++){ float *os=out+(int64_t)rows[r]*D, wgt=rw[r], *hr=hh+(int64_t)r*D;for(int d=0;d<D;d++) os[d]+=wgt*hr[d]; }@@ -3467,7 +3485,7 @@ static void moe(Model *m, Layer *l, int layer, float *x, int S, float *out, int}shared_done:free(logits_all); free(choice); free(idxs); free(ws); free(keff); free(uniq);- free(xg); free(gg); free(uu); free(hh); free(rows); free(rw);+ free(xg); free(gg); free(uu); free(hh); free(rows); free(rw); free(xe);#ifdef COLI_CUDAfree(group_x);free(group_y);free(group_row); free(group_weight);diff --git a/c/quant.h b/c/quant.hindex 6149bb6c..505141f5 100644--- a/c/quant.h+++ b/c/quant.h@@ -1156,6 +1156,35 @@ static inline void e8_fwht(float *a, int n, const uint8_t *signbits){}static inline int e8_pow2_ceil(int n){ int p=1; while(p<n) p<<=1; return p; }+/* Rotation sign bits, regenerated β not stored. xorshift64* seeded 417+n, one+ * bit per element; tools/iq3_pack.py signs() draws the identical stream, so the+ * container carries no rotation data and the two sides cannot drift apart+ * without the oracle fixture catching it (#452). */+static inline void e8_signs(uint8_t *bits, int n){+ uint64_t s=417u+(uint64_t)n;+ for(int i=0;i<(n+7)/8;i++){+ s^=s>>12; s^=s<<25; s^=s>>27;+ bits[i]=(uint8_t)((s*2685821657736338717ULL)>>56);+ }+}+/* Apply the fmt=6 activation rotation Q^T in place, row-major [nr,dim].+ * Non-power-of-two dims tile block-diagonally; each block is the largest power+ * of two dividing the remainder (= its lowest set bit): 6144 -> 2048+4096,+ * 1536 -> 512+1024. Blocks over 32768 halve until they fit the sign buffer.+ * The converter rotates weight rows with this exact routine (W@Q and Q^T x are+ * the same transform: Q is symmetric-orthogonal up to the sign flip). */+static inline void e8_rot_rows(float *rows, int nr, int dim){+ int off=0;+ while(off<dim){+ int rem=dim-off, b=rem&(-rem);+ while(b>32768) b>>=1;+ uint8_t bits[32768/8];+ e8_signs(bits,b);+ for(int r=0;r<nr;r++) e8_fwht(rows+(int64_t)r*dim+off,b,bits);+ off+=b;+ }+}+static void matmul_e8(float *y, const float *x, const uint8_t *q, const float *unused,int S, int I, int O){(void)unused; /* scales live inside the blocks */diff --git a/c/tests/fixtures/e8_case.bin b/c/tests/fixtures/e8_case.binindex bec2cd7a..aa9573df 100644Binary files a/c/tests/fixtures/e8_case.bin and b/c/tests/fixtures/e8_case.bin differdiff --git a/c/tests/test_e8_kernel.c b/c/tests/test_e8_kernel.cindex caf8b7e4..9d7b15d0 100644--- a/c/tests/test_e8_kernel.c+++ b/c/tests/test_e8_kernel.c@@ -81,6 +81,46 @@ int main(void) {}printf("e8 kernel oracle: O=%d I=%d, dequant exact, matmul worst rel %.2e\n", O, I, worst);if (worst > 1e-5) { printf("FAIL\n"); return 1; }++ /* 3. rotation agreement (converter step, #452): the fixture carries a+ * rotated-weights section β packed W@Q from the Python side, the raw+ * activation, Python's Q^T x, and the float64 reference product. The C+ * side must reproduce the rotation (regenerated signs + block tiling ++ * FWHT) and then the matmul, or every rotated container decodes wrong. */+ const uint8_t *sec = (const uint8_t *)(yref + O);+ if ((size_t)(sec - raw) < nbytes) {+ int32_t O2, I2;+ memcpy(&O2, sec, 4); memcpy(&I2, sec + 4, 4);+ size_t rb2 = (size_t)e8_rowbytes(I2);+ const uint8_t *packed2 = sec + 8;+ const float *x2 = (const float *)(packed2 + (size_t)O2 * rb2);+ const float *xrot2 = x2 + I2;+ const float *y2ref = xrot2 + I2;+ float *xr = malloc((size_t)I2 * sizeof(float));+ memcpy(xr, x2, (size_t)I2 * sizeof(float));+ e8_rot_rows(xr, 1, I2);+ double wrot = 0;+ for (int i = 0; i < I2; i++) {+ double d = fabs((double)xr[i] - (double)xrot2[i]);+ double rel = d / (fabs((double)xrot2[i]) + 1e-6);+ if (rel > wrot) wrot = rel;+ }+ float *y2 = malloc((size_t)O2 * sizeof(float));+ matmul_e8(y2, xr, packed2, NULL, 1, I2, O2);+ double wmm = 0;+ for (int o = 0; o < O2; o++) {+ double d = fabs((double)y2[o] - (double)y2ref[o]);+ double rel = d / (fabs((double)y2ref[o]) + 1e-6);+ if (rel > wmm) wmm = rel;+ }+ printf("e8 rotation oracle: O2=%d I2=%d, rot worst rel %.2e, matmul worst rel %.2e\n",+ O2, I2, wrot, wmm);+ if (wrot > 1e-5 || wmm > 1e-5) { printf("FAIL\n"); return 1; }+ free(xr); free(y2);+ } else {+ fprintf(stderr, "fixture has no rotated section β regenerate with tools/make_e8_fixture.py\n");+ return 1;+ }printf("OK\n");free(y); free(raw);return 0;diff --git a/c/tests/test_iq3_pack.py b/c/tests/test_iq3_pack.pyindex 43aa877f..e2a2f318 100644--- a/c/tests/test_iq3_pack.py+++ b/c/tests/test_iq3_pack.py@@ -91,6 +91,36 @@ class TestIq3Pack(unittest.TestCase):with self.assertRaises(ValueError):P.encode(np.zeros((2, 300), dtype=np.float32))+ def test_signs_spec_frozen(self):+ """First bytes of the n=256 stream, computed once by hand from the+ xorshift64* spec. If this fails, the PRNG drifted and every rotated+ container in the wild decodes to garbage β the constants are the spec."""+ s = P.signs(256)+ self.assertEqual(s.shape, (256,))+ self.assertTrue(set(np.unique(s)) <= {-1.0, 1.0})+ self.assertEqual(list((s[:16] < 0).astype(int)),+ [0, 1, 1, 1, 1, 0, 1, 1, 1, 0, 1, 0, 0, 1, 1, 0])+ self.assertFalse(np.array_equal(P.signs(256), P.signs(512)[:256]),+ "streams for different sizes must differ (seed = 417+n)")++ def test_rot_blocks_tiling(self):+ self.assertEqual(P.rot_blocks(2048), [2048])+ self.assertEqual(P.rot_blocks(6144), [2048, 4096])+ self.assertEqual(P.rot_blocks(1536), [512, 1024])+ for dim in (768, 1536, 6144):+ self.assertEqual(sum(P.rot_blocks(dim)), dim)++ def test_rotation_is_orthogonal(self):+ """Q preserves norms and cancels between W@Q and Q^T x: the rotated+ matmul must reproduce the unrotated one to float precision."""+ w = (np.random.randn(8, 1536) * 0.05).astype(np.float32)+ x = np.random.randn(1536).astype(np.float32)+ wr, xr = P.rotate_rows(w), P.rotate_rows(x[None, :])[0]+ self.assertTrue(np.allclose(np.linalg.norm(wr, axis=1),+ np.linalg.norm(w, axis=1), rtol=1e-5))+ self.assertTrue(np.allclose(wr @ xr, w @ x, rtol=1e-4, atol=1e-5),+ f"max |Ξ| = {np.abs(wr @ xr - w @ x).max()}")+if __name__ == "__main__":unittest.main()diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.pyindex e9a27774..6647e071 100644--- a/c/tools/convert_fp8_to_int4.py+++ b/c/tools/convert_fp8_to_int4.py@@ -106,6 +106,22 @@ def quant_int3_g64(w, bits=3, group=64): # -> (qbytes U8 [O*ceil(I/64)*24out = np.concatenate([lo, hi], axis=2) # [O, ng, 24]return out.reshape(-1), s[:, :, 0].astype(np.float32).reshape(-1)+E8 = "e8" # CLI/bits-plumbing marker for fmt=6 (not a bit width)++def quant_e8(w): # -> (qbytes U8 [O*ceil(I/256)*98], tag f32 [1])+ """E8/IQ3 lattice (fmt=6 in colibri.c, #452): rotate the rows first (W@Q,+ block-diagonal FWHT with regenerated signs β iq3_pack.rotate_rows mirrors+ quant.h e8_rot_rows), then pack with the iq3 codec: 98B per 256 weights,+ 3.0625 bpw, every scale in-block. The .qs companion is a single float,+ the engine's fmt=6 discriminator β not a scale."""+ sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))+ import iq3_pack+ O, I = w.shape+ if I % 256:+ raise SystemExit(f"e8: input dim {I} is not a multiple of 256")+ packed = iq3_pack.encode(iq3_pack.rotate_rows(np.asarray(w, dtype=np.float32)))+ return packed.reshape(-1), np.array([6.0], dtype=np.float32)+def quant_int2(w, bits): # -> (qbytes U8 [O*ceil(I/4)], scale f32 [O]); 4/byteO, I = w.shapeqmax = (1 << (bits - 1)) - 1 # bits=2 -> qmax=1, valori [-2,1]@@ -272,7 +288,10 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,if f".{proj}.weight" in name: bits = pb; breakif w.ndim != 2: # es. bias 1D non previsto come 'q' -> tienilo f32out_dict[name] = w.astype(np.float32); continue- if bits == 3:+ if bits == E8:+ # fmt=6 E8/IQ3 β routed-expert projections only, enforced in main().+ q, s = quant_e8(w)+ elif bits == 3:# int3-g64 (fmt=5): inherently group-64, distinct from grouped-int4.q, s = quant_int3_g64(w)elif group_size > 0 and bits <= 4:@@ -311,6 +330,9 @@ def check_or_record_params(outdir, prefix, params):os.replace(tmp, path)return True+def _bits(v): # "e8" -> fmt=6 marker; anything else an int width+ return E8 if v == E8 else int(v)+def main():ap = argparse.ArgumentParser()ap.add_argument("--repo", default=None)@@ -318,7 +340,7 @@ def main():ap.add_argument("--outdir", required=False)ap.add_argument("--ebits", type=int, default=None) # bit residenti (default 4; 8 per --mtp/--indexer)ap.add_argument("--io-bits", type=int, default=8) # bit di embed/lm_head- ap.add_argument("--xbits", type=int, default=None) # bit degli expert ROUTED (streaming); default=ebits+ ap.add_argument("--xbits", type=_bits, default=None) # bit degli expert ROUTED (streaming), o "e8" (fmt=6); default=ebits# Mixed-precision: per-tensor-type bit overrides. Default = ebits (all same).# Set these higher to protect sensitive tensors from quantization error.ap.add_argument("--shared-bits", type=int, default=None,@@ -334,11 +356,11 @@ def main():ap.add_argument("--group-size", type=int, default=0, # 0 = per-row (backward compat); 128 = group-scaledhelp="group size for int4 scales: 0=per-row (default), 128=one scale per 128 elements (much better quality)")# Per-projection bit overrides for routed experts (orthogonal to the type-level flags above).- ap.add_argument("--up-bits", type=int, default=None,+ ap.add_argument("--up-bits", type=_bits, default=None,help="bits for up_proj in routed experts (e.g. 3 = int3-g64). Default=xbits")- ap.add_argument("--gate-bits", type=int, default=None,+ ap.add_argument("--gate-bits", type=_bits, default=None,help="bits for gate_proj in routed experts. Default=xbits")- ap.add_argument("--down-bits", type=int, default=None,+ ap.add_argument("--down-bits", type=_bits, default=None,help="bits for down_proj in routed experts. Default=xbits")ap.add_argument("--n-layers", type=int, default=78)ap.add_argument("--min-free-gb", type=float, default=20.0)@@ -373,6 +395,13 @@ def main():if val is not None: PROJ_BITS[proj] = valif PROJ_BITS:print(f"[per-projection expert bits] {PROJ_BITS} (others -> xbits={a.xbits})")+ # fmt=6 is all-or-nothing across the three expert projections: gate and up+ # share one rotated input row in the engine (the placement rule in quant.h),+ # so a mixed layout would need two gather buffers for zero measured benefit.+ eff = [PROJ_BITS.get(p, a.xbits) for p in ("gate_proj", "up_proj", "down_proj")]+ if any(b == E8 for b in eff) and not all(b == E8 for b in eff):+ raise SystemExit(f"e8 covers all three expert projections or none (got {eff}); "+ "use --xbits e8, or none of the e8 flags")# Build per-type bits map. If a type-specific arg is set, use it; otherwise the# converter falls back to ebits for that type.diff --git a/c/tools/iq3_pack.py b/c/tools/iq3_pack.pyindex b59451f5..e51c9d79 100644--- a/c/tools/iq3_pack.py+++ b/c/tools/iq3_pack.py@@ -171,3 +171,62 @@ def decode(packed, K):def bpw():return BLOCK_BYTES * 8 / QK+++# ---- rotation (converter side of quant.h e8_signs / e8_rot_rows) -------------+#+# Q = D @ H / sqrt(n) per power-of-two block; W@Q on weight rows here equals+# Q^T x on activations in the engine (sign-flip then FWHT β one routine both+# sides). The sign diagonal D is REGENERATED from the block size, never stored:+# both sides draw the same xorshift64* stream seeded 417+n, and the kernel+# fixture (make_e8_fixture.py) pins the agreement.++def signs(n):+ """[n] float32 in {+1,-1} β bit-exact mirror of quant.h e8_signs()."""+ s = np.uint64(417 + n)+ out = np.empty((n + 7) // 8, dtype=np.uint8)+ with np.errstate(over="ignore"):+ for i in range(len(out)):+ s ^= s >> np.uint64(12)+ s ^= (s << np.uint64(25)) & np.uint64(0xFFFFFFFFFFFFFFFF)+ s ^= s >> np.uint64(27)+ out[i] = np.uint8(((s * np.uint64(2685821657736338717)) &+ np.uint64(0xFFFFFFFFFFFFFFFF)) >> np.uint64(56))+ bits = np.unpackbits(out, bitorder="little")[:n]+ return np.where(bits == 1, -1.0, 1.0).astype(np.float32)+++def rot_blocks(dim):+ """Block tiling: each block is the largest power of two dividing the+ remainder (its lowest set bit) β 6144 -> [2048, 4096], 1536 -> [512, 1024].+ Blocks over 32768 halve, mirroring the engine's sign-buffer cap."""+ out, rem = [], dim+ while rem:+ b = rem & (-rem)+ while b > 32768:+ b >>= 1+ out.append(b)+ rem -= b+ return out+++def rotate_rows(x):+ """Apply the rotation to rows of float32 [..., dim] (weights W@Q or+ activations Q^T x β the transform is the same). Returns a new array."""+ x = np.ascontiguousarray(x, dtype=np.float32)+ dim = x.shape[-1]+ rows = x.reshape(-1, dim).copy()+ off = 0+ for b in rot_blocks(dim):+ blk = rows[:, off:off + b] * signs(b)[None, :]+ h = 1+ while h < b: # in-place FWHT, vectorized over rows+ blk = blk.reshape(-1, b // (2 * h), 2, h)+ u, v = blk[:, :, 0, :].copy(), blk[:, :, 1, :].copy()+ blk[:, :, 0, :] = u + v+ blk[:, :, 1, :] = u - v+ blk = blk.reshape(-1, b)+ h <<= 1+ rows[:, off:off + b] = blk / np.sqrt(b, dtype=np.float32)+ off += b+ return rows.reshape(x.shape)diff --git a/c/tools/make_e8_fixture.py b/c/tools/make_e8_fixture.pyindex e58d7d4d..1a146cff 100644--- a/c/tools/make_e8_fixture.py+++ b/c/tools/make_e8_fixture.py@@ -27,6 +27,17 @@ def main():# float64 reference so the C float path is compared against something betteryref = (deq.astype(np.float64) @ x.astype(np.float64)).astype(np.float32)+ # Rotated section (#452 converter step): the container stores W@Q; the+ # engine rotates activations with e8_rot_rows. I2=768 tiles as 256+512,+ # exercising the multi-block path and the regenerated signs on both sides.+ O2, I2 = 16, 768+ w2 = (rng.standard_normal((O2, I2)) * 0.05).astype(np.float32)+ packed2 = P.encode(P.rotate_rows(w2))+ x2 = (rng.standard_normal(I2) * 1.0).astype(np.float32)+ xrot2 = P.rotate_rows(x2[None, :])[0].astype(np.float32)+ y2ref = (P.decode(packed2, I2).astype(np.float64)+ @ xrot2.astype(np.float64)).astype(np.float32)+out = os.path.join(os.path.dirname(__file__), "..", "tests", "fixtures")os.makedirs(out, exist_ok=True)path = os.path.join(out, "e8_case.bin")@@ -36,8 +47,13 @@ def main():f.write(deq.tobytes())f.write(x.tobytes())f.write(yref.tobytes())+ f.write(struct.pack("<ii", O2, I2))+ f.write(packed2.tobytes())+ f.write(x2.tobytes())+ f.write(xrot2.tobytes())+ f.write(y2ref.tobytes())print(f"wrote {path}: O={O} I={I} packed={packed.nbytes}B "- f"({packed.nbytes * 8 / (O * I):.4f} bpw)")+ f"({packed.nbytes * 8 / (O * I):.4f} bpw) + rotated O2={O2} I2={I2}")if __name__ == "__main__":