Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tools: quantize glm_tiny routed experts to fmt=6 (E8/IQ3) and fmt=4 (grouped int4)
eliedcfd05480e4
diff --git a/.gitignore b/.gitignoreindex 5644439f..982972c7 100644--- a/.gitignore+++ b/.gitignore@@ -82,6 +82,8 @@ c/glm_tiny/c/glm_tiny_i2/c/glm_tiny_i4/c/glm_tiny_mix/+c/glm_tiny_fmt6/+c/glm_tiny_fmt4/c/glm_bench_medium/c/bench/diff --git a/c/tools/README.md b/c/tools/README.mdindex 01f08e59..659394ba 100644--- a/c/tools/README.md+++ b/c/tools/README.md@@ -19,3 +19,17 @@ Run them from `c/`, for example:python3 tools/convert_fp8_to_int4.py --selftestpython3 tools/make_glm_bench_model.py --output /tmp/colibri-bench```++`make_glm_oracle.py` also produces the quantized routed-expert fixtures for the+fmt=6 (E8/IQ3, rotation-bearing) and fmt=4 (grouped int4, no-rotation control)+parity gate (#3/#7). Only the routed experts are quantized; shared/dense/attn+stay f32, and the reference (`ref_glm.json` inside each fixture dir) is computed+from the dequantized weights so the engine reproduces it token-exactly:++```sh+python3 tools/make_glm_oracle.py --fmt6 # -> glm_tiny_fmt6/+python3 tools/make_glm_oracle.py --fmt4 # -> glm_tiny_fmt4/+# verify the engine loads the formats directly (32/32 expected):+SNAP=./glm_tiny_fmt6 REF=./glm_tiny_fmt6/ref_glm.json TF=1 COLI_TEMP=0 ./colibri 64 16 16+SNAP=./glm_tiny_fmt4 REF=./glm_tiny_fmt4/ref_glm.json TF=1 COLI_TEMP=0 ./colibri 64 16 16+```diff --git a/c/tools/make_glm_oracle.py b/c/tools/make_glm_oracle.pyindex 5e16f516..db6c8623 100644--- a/c/tools/make_glm_oracle.py+++ b/c/tools/make_glm_oracle.py@@ -14,7 +14,23 @@ originale invariato).EN: --fp8 writes FP8 e4m3 + 128x128 block scale_inv (real GLM-5.2-FP8 layout) instead of bf16,EN: so convert_fp8_to_int4.py can run its FP8->int4 path on a tiny model. ref_glm.json isEN: computed AFTER the FP8 round-trip, so the reference matches exactly what the converter-EN: ingests. Default: bf16 (original oracle unchanged)."""+EN: ingests. Default: bf16 (original oracle unchanged).++--fmt6 / --fmt4 (ticket #3): quantize ONLY the ROUTED experts to fmt=6 (E8/IQ3, the only+rotation-bearing format) or fmt=4 (grouped int4, the no-rotation control); shared/dense/attn+stay f32. This is the fixture that lets the token-exact parity gate (#7) exercise the E8+activation pre-rotation (E8_XE / e8_rot_rows) — the default f32 oracle never touches it.++The E8 super-block is 256 weights (98 bytes), so the routed-expert contraction dims must be+multiples of 256; these two flags therefore generate the model with hidden_size=256 and+moe_intermediate_size=256 (the f32 default keeps 128/32). Weights are packed from the+ORIGINAL weights and the ref is computed from the DEQUANTIZED weights (round-trip through the+same quantizer), so the engine's decode reproduces the reference token-exactly.++Regeneration (run from c/):+ python3 tools/make_glm_oracle.py --fmt6 # -> glm_tiny_fmt6/ (model.safetensors, config.json, ref_glm.json)+ python3 tools/make_glm_oracle.py --fmt4 # -> glm_tiny_fmt4/+ # verify: SNAP=./glm_tiny_fmt6 REF=./glm_tiny_fmt6/ref_glm.json TF=1 COLI_TEMP=0 ./colibri 64 16 16"""import json, sys, argparsefrom pathlib import Path@@ -64,19 +80,42 @@ from glm_fp8_emit import (fp8_block_quantize, fp8_block_dequantize, keep_f32,save_fp8_safetensors, unfuse_experts)ap = argparse.ArgumentParser()-ap.add_argument("--fp8", action="store_true",+_quant = ap.add_mutually_exclusive_group()+_quant.add_argument("--fp8", action="store_true",help="salva in FP8 e4m3 + 128x128 block scale_inv (layout GLM-5.2-FP8) e ""calcola ref_glm.json sul modello dopo il round-trip FP8. ""EN: write FP8 e4m3 + block scale_inv, ref computed on FP8-rounded model")+_quant.add_argument("--fmt6", action="store_true",+ help="quantizza SOLO gli expert routed in fmt=6 (E8/IQ3, #452) e lascia il "+ "resto (shared/dense/attn) f32; oracolo per esercitare la rotazione E8. "+ "EN: quantize ONLY routed experts to fmt=6 (E8/IQ3); everything else f32")+_quant.add_argument("--fmt4", action="store_true",+ help="come --fmt6 ma fmt=4 (int4 grouped gs=64, nessuna rotazione): il "+ "braccio di controllo senza rotazione. "+ "EN: like --fmt6 but fmt=4 (grouped int4, no rotation) — the control arm")args = ap.parse_args()torch.manual_seed(1234)+# E8/IQ3 (fmt=6) e il suo controllo int4-grouped (fmt=4) richiedono che le+# dimensioni di contrazione degli expert routed siano multiple di 256: il codec E8+# impacchetta 256 pesi per super-blocco (98 byte) e quant_e8 rifiuta qualsiasi altra+# I (iq3_pack.encode: `K % 256 == 0`). La config tiny di default (hidden=128,+# moe_inter=32) NON e' quantizzabile in E8, quindi le fixture quantizzate usano+# hidden=256 / moe_inter=256. Il default f32 (nessuna flag) resta INVARIATO a 128/32.+# EN: fmt=6's E8 super-block (256 weights/98B) forces routed-expert contraction dims+# to be multiples of 256; the quantized fixtures use hidden=256/moe_inter=256 while+# the f32 default is unchanged at 128/32.+if args.fmt6 or args.fmt4:+ _HIDDEN, _MOE_INTER = 256, 256+else:+ _HIDDEN, _MOE_INTER = 128, 32+cfg = GlmMoeDsaConfig(vocab_size=256,- hidden_size=128,+ hidden_size=_HIDDEN,intermediate_size=64, # MLP densa (primi 3 layer)- moe_intermediate_size=32, # expert+ moe_intermediate_size=_MOE_INTER, # expertnum_hidden_layers=5, # 3 densi + 2 sparsefirst_k_dense_replace=3,num_attention_heads=4,@@ -128,6 +167,42 @@ if args.fp8:q, s = fp8_block_quantize(p)p.copy_(fp8_block_dequantize(q, s))+# --fmt6/--fmt4: round-trip degli expert routed FUSI (gate_up_proj [E,2M,I] e+# down_proj [E,I,M]) attraverso la quantizzazione PRIMA di calcolare il riferimento,+# cosi' ref_glm.json riflette ESATTAMENTE i pesi che il motore decodifichera' dal+# container quantizzato (stesso pattern del ramo --fp8). La quantizzazione opera per+# riga sull'ultima dim (= la dim di contrazione), quindi quantizzare il fuso 3-D+# equivale a quantizzare gli expert 2-D non fusi: le righe sono identiche.+# EN: round-trip the FUSED routed experts through the quantizer before computing the+# ref (same pattern as --fp8), so the ref matches exactly what the engine decodes.+# Quantization is per-row on the last (contraction) dim, so fused and unfused encodings+# are identical.+if args.fmt6 or args.fmt4:+ # Cattura i pesi ORIGINALI (pre-round-trip) PRIMA di toccare il modello: servono+ # per il SAVE — i pesi impacchettati devono venire dai pesi originali, non da quelli+ # gia' dequantizzati (encode(decode(x)) != encode(x): il codec E8 non e' idempotente+ # byte-per-byte). state_dict() ritorna viste che aliasano i parametri, quindi clone+ # esplicito. EN: clone the original weights before the round-trip; the SAVE path must+ # pack the ORIGINAL weights (encode∘decode is not idempotent), and state_dict()+ # aliases the live parameters.+ sd_orig = {k: v.detach().clone() for k, v in model.state_dict().items()}+ import iq3_pack+ from convert_fp8_to_int4 import quant_int4_grouped+ from convert_fmt4_to_fmt2 import dequant_fmt4+ with torch.no_grad():+ for n, p in model.named_parameters():+ if ".mlp.experts." not in n or not n.endswith(("gate_up_proj", "down_proj")):+ continue+ shp = tuple(p.shape); I = shp[-1]+ w = p.detach().float().numpy().reshape(-1, I)+ if args.fmt6:+ packed = iq3_pack.encode(iq3_pack.rotate_rows(w))+ w_eff = iq3_pack.unrotate_rows(iq3_pack.decode(packed, I))+ else:+ q, s = quant_int4_grouped(w, 4, 64)+ w_eff = dequant_fmt4(q.reshape(-1, (I + 1) // 2), s, w.shape[0], I)+ p.copy_(torch.from_numpy(w_eff.reshape(shp)))+print("=== state_dict tensors (names used by the C loader) ===")for n, p in model.state_dict().items():print(f" {n:60s} {tuple(p.shape)}")@@ -154,15 +229,53 @@ print("tf_pred:", tf_pred)sd = model.state_dict()unfuse_experts(sd)-Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves-if args.fp8:- n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors")- print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) "- f"-> glm_tiny/model.safetensors")-else:+if args.fmt6 or args.fmt4:+ # Salva SOLO gli expert routed quantizzati (fmt=6 o fmt=4 + .qs); shared/dense/attn/+ # norme restano f32. I pesi impacchettati provengono dai pesi ORIGINALI (sd_orig,+ # non fusi), non da quelli gia' dequantizzati, cosi' il motore decodifica ESATTAMENTE+ # i pesi che il round-trip sopra ha usato per il riferimento. EN: quantize ONLY the+ # routed experts (fmt=6 or fmt=4 + .qs); everything else stays f32. Packed weights+ # come from the ORIGINAL (unfused) weights, not the dequantized ones, so the engine+ # decodes exactly the weights the round-trip used for the reference.+ outdir = "glm_tiny_fmt6" if args.fmt6 else "glm_tiny_fmt4"+ fmt = 6 if args.fmt6 else 4+ Path(outdir).mkdir(parents=True, exist_ok=True)+ import numpy as np+ import iq3_pack+ from convert_fp8_to_int4 import quant_int4_grouped+ sd = unfuse_experts(sd_orig)+ out = {}+ for name, t in sd.items():+ if ".mlp.experts." in name and name.endswith(+ (".gate_proj.weight", ".up_proj.weight", ".down_proj.weight")):+ w = t.detach().float().numpy()+ if fmt == 6:+ packed = iq3_pack.encode(iq3_pack.rotate_rows(w))+ out[name] = torch.from_numpy(np.ascontiguousarray(packed.reshape(-1)))+ out[name + ".qs"] = torch.tensor([6.0], dtype=torch.float32)+ else:+ q, s = quant_int4_grouped(w, 4, 64)+ out[name] = torch.from_numpy(q)+ out[name + ".qs"] = torch.from_numpy(s)+ else:+ out[name] = t.detach().contiguous()from safetensors.torch import save_file- save_file({k: v.contiguous() for k, v in sd.items()}, "glm_tiny/model.safetensors")-json.dump(cfg.to_dict(), open("glm_tiny/config.json", "w"))-json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred}, open("ref_glm.json", "w"))-print("saved: glm_tiny/ (weights + config) and ref_glm.json"- + (" [fp8]" if args.fp8 else ""))+ save_file(out, f"{outdir}/model.safetensors")+ json.dump(cfg.to_dict(), open(f"{outdir}/config.json", "w"))+ json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred},+ open(f"{outdir}/ref_glm.json", "w"))+ print(f"saved: {outdir}/ (routed experts fmt={fmt}, shared/dense/attn f32) + "+ f"{outdir}/ref_glm.json")+else:+ Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves+ if args.fp8:+ n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors")+ print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) "+ f"-> glm_tiny/model.safetensors")+ else:+ from safetensors.torch import save_file+ save_file({k: v.contiguous() for k, v in sd.items()}, "glm_tiny/model.safetensors")+ json.dump(cfg.to_dict(), open("glm_tiny/config.json", "w"))+ json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred}, open("ref_glm.json", "w"))+ print("saved: glm_tiny/ (weights + config) and ref_glm.json"+ + (" [fp8]" if args.fp8 else ""))