Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tools: quantize glm_tiny routed experts to fmt=6 (E8/IQ3) and fmt=4 (grouped int4)

eliedcfd05480e4
diff --git a/.gitignore b/.gitignore
index 5644439f..982972c7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -82,6 +82,8 @@ c/glm_tiny/
c/glm_tiny_i2/
c/glm_tiny_i4/
c/glm_tiny_mix/
+c/glm_tiny_fmt6/
+c/glm_tiny_fmt4/
c/glm_bench_medium/
c/bench/
diff --git a/c/tools/README.md b/c/tools/README.md
index 01f08e59..659394ba 100644
--- a/c/tools/README.md
+++ b/c/tools/README.md
@@ -19,3 +19,17 @@ Run them from `c/`, for example:
python3 tools/convert_fp8_to_int4.py --selftest
python3 tools/make_glm_bench_model.py --output /tmp/colibri-bench
```
+
+`make_glm_oracle.py` also produces the quantized routed-expert fixtures for the
+fmt=6 (E8/IQ3, rotation-bearing) and fmt=4 (grouped int4, no-rotation control)
+parity gate (#3/#7). Only the routed experts are quantized; shared/dense/attn
+stay f32, and the reference (`ref_glm.json` inside each fixture dir) is computed
+from the dequantized weights so the engine reproduces it token-exactly:
+
+```sh
+python3 tools/make_glm_oracle.py --fmt6 # -> glm_tiny_fmt6/
+python3 tools/make_glm_oracle.py --fmt4 # -> glm_tiny_fmt4/
+# verify the engine loads the formats directly (32/32 expected):
+SNAP=./glm_tiny_fmt6 REF=./glm_tiny_fmt6/ref_glm.json TF=1 COLI_TEMP=0 ./colibri 64 16 16
+SNAP=./glm_tiny_fmt4 REF=./glm_tiny_fmt4/ref_glm.json TF=1 COLI_TEMP=0 ./colibri 64 16 16
+```
diff --git a/c/tools/make_glm_oracle.py b/c/tools/make_glm_oracle.py
index 5e16f516..db6c8623 100644
--- a/c/tools/make_glm_oracle.py
+++ b/c/tools/make_glm_oracle.py
@@ -14,7 +14,23 @@ originale invariato).
EN: --fp8 writes FP8 e4m3 + 128x128 block scale_inv (real GLM-5.2-FP8 layout) instead of bf16,
EN: so convert_fp8_to_int4.py can run its FP8->int4 path on a tiny model. ref_glm.json is
EN: computed AFTER the FP8 round-trip, so the reference matches exactly what the converter
-EN: ingests. Default: bf16 (original oracle unchanged)."""
+EN: ingests. Default: bf16 (original oracle unchanged).
+
+--fmt6 / --fmt4 (ticket #3): quantize ONLY the ROUTED experts to fmt=6 (E8/IQ3, the only
+rotation-bearing format) or fmt=4 (grouped int4, the no-rotation control); shared/dense/attn
+stay f32. This is the fixture that lets the token-exact parity gate (#7) exercise the E8
+activation pre-rotation (E8_XE / e8_rot_rows) — the default f32 oracle never touches it.
+
+The E8 super-block is 256 weights (98 bytes), so the routed-expert contraction dims must be
+multiples of 256; these two flags therefore generate the model with hidden_size=256 and
+moe_intermediate_size=256 (the f32 default keeps 128/32). Weights are packed from the
+ORIGINAL weights and the ref is computed from the DEQUANTIZED weights (round-trip through the
+same quantizer), so the engine's decode reproduces the reference token-exactly.
+
+Regeneration (run from c/):
+ python3 tools/make_glm_oracle.py --fmt6 # -> glm_tiny_fmt6/ (model.safetensors, config.json, ref_glm.json)
+ python3 tools/make_glm_oracle.py --fmt4 # -> glm_tiny_fmt4/
+ # verify: SNAP=./glm_tiny_fmt6 REF=./glm_tiny_fmt6/ref_glm.json TF=1 COLI_TEMP=0 ./colibri 64 16 16"""
import json, sys, argparse
from pathlib import Path
@@ -64,19 +80,42 @@ from glm_fp8_emit import (fp8_block_quantize, fp8_block_dequantize, keep_f32,
save_fp8_safetensors, unfuse_experts)
ap = argparse.ArgumentParser()
-ap.add_argument("--fp8", action="store_true",
+_quant = ap.add_mutually_exclusive_group()
+_quant.add_argument("--fp8", action="store_true",
help="salva in FP8 e4m3 + 128x128 block scale_inv (layout GLM-5.2-FP8) e "
"calcola ref_glm.json sul modello dopo il round-trip FP8. "
"EN: write FP8 e4m3 + block scale_inv, ref computed on FP8-rounded model")
+_quant.add_argument("--fmt6", action="store_true",
+ help="quantizza SOLO gli expert routed in fmt=6 (E8/IQ3, #452) e lascia il "
+ "resto (shared/dense/attn) f32; oracolo per esercitare la rotazione E8. "
+ "EN: quantize ONLY routed experts to fmt=6 (E8/IQ3); everything else f32")
+_quant.add_argument("--fmt4", action="store_true",
+ help="come --fmt6 ma fmt=4 (int4 grouped gs=64, nessuna rotazione): il "
+ "braccio di controllo senza rotazione. "
+ "EN: like --fmt6 but fmt=4 (grouped int4, no rotation) — the control arm")
args = ap.parse_args()
torch.manual_seed(1234)
+# E8/IQ3 (fmt=6) e il suo controllo int4-grouped (fmt=4) richiedono che le
+# dimensioni di contrazione degli expert routed siano multiple di 256: il codec E8
+# impacchetta 256 pesi per super-blocco (98 byte) e quant_e8 rifiuta qualsiasi altra
+# I (iq3_pack.encode: `K % 256 == 0`). La config tiny di default (hidden=128,
+# moe_inter=32) NON e' quantizzabile in E8, quindi le fixture quantizzate usano
+# hidden=256 / moe_inter=256. Il default f32 (nessuna flag) resta INVARIATO a 128/32.
+# EN: fmt=6's E8 super-block (256 weights/98B) forces routed-expert contraction dims
+# to be multiples of 256; the quantized fixtures use hidden=256/moe_inter=256 while
+# the f32 default is unchanged at 128/32.
+if args.fmt6 or args.fmt4:
+ _HIDDEN, _MOE_INTER = 256, 256
+else:
+ _HIDDEN, _MOE_INTER = 128, 32
+
cfg = GlmMoeDsaConfig(
vocab_size=256,
- hidden_size=128,
+ hidden_size=_HIDDEN,
intermediate_size=64, # MLP densa (primi 3 layer)
- moe_intermediate_size=32, # expert
+ moe_intermediate_size=_MOE_INTER, # expert
num_hidden_layers=5, # 3 densi + 2 sparse
first_k_dense_replace=3,
num_attention_heads=4,
@@ -128,6 +167,42 @@ if args.fp8:
q, s = fp8_block_quantize(p)
p.copy_(fp8_block_dequantize(q, s))
+# --fmt6/--fmt4: round-trip degli expert routed FUSI (gate_up_proj [E,2M,I] e
+# down_proj [E,I,M]) attraverso la quantizzazione PRIMA di calcolare il riferimento,
+# cosi' ref_glm.json riflette ESATTAMENTE i pesi che il motore decodifichera' dal
+# container quantizzato (stesso pattern del ramo --fp8). La quantizzazione opera per
+# riga sull'ultima dim (= la dim di contrazione), quindi quantizzare il fuso 3-D
+# equivale a quantizzare gli expert 2-D non fusi: le righe sono identiche.
+# EN: round-trip the FUSED routed experts through the quantizer before computing the
+# ref (same pattern as --fp8), so the ref matches exactly what the engine decodes.
+# Quantization is per-row on the last (contraction) dim, so fused and unfused encodings
+# are identical.
+if args.fmt6 or args.fmt4:
+ # Cattura i pesi ORIGINALI (pre-round-trip) PRIMA di toccare il modello: servono
+ # per il SAVE — i pesi impacchettati devono venire dai pesi originali, non da quelli
+ # gia' dequantizzati (encode(decode(x)) != encode(x): il codec E8 non e' idempotente
+ # byte-per-byte). state_dict() ritorna viste che aliasano i parametri, quindi clone
+ # esplicito. EN: clone the original weights before the round-trip; the SAVE path must
+ # pack the ORIGINAL weights (encode∘decode is not idempotent), and state_dict()
+ # aliases the live parameters.
+ sd_orig = {k: v.detach().clone() for k, v in model.state_dict().items()}
+ import iq3_pack
+ from convert_fp8_to_int4 import quant_int4_grouped
+ from convert_fmt4_to_fmt2 import dequant_fmt4
+ with torch.no_grad():
+ for n, p in model.named_parameters():
+ if ".mlp.experts." not in n or not n.endswith(("gate_up_proj", "down_proj")):
+ continue
+ shp = tuple(p.shape); I = shp[-1]
+ w = p.detach().float().numpy().reshape(-1, I)
+ if args.fmt6:
+ packed = iq3_pack.encode(iq3_pack.rotate_rows(w))
+ w_eff = iq3_pack.unrotate_rows(iq3_pack.decode(packed, I))
+ else:
+ q, s = quant_int4_grouped(w, 4, 64)
+ w_eff = dequant_fmt4(q.reshape(-1, (I + 1) // 2), s, w.shape[0], I)
+ p.copy_(torch.from_numpy(w_eff.reshape(shp)))
+
print("=== state_dict tensors (names used by the C loader) ===")
for n, p in model.state_dict().items():
print(f" {n:60s} {tuple(p.shape)}")
@@ -154,15 +229,53 @@ print("tf_pred:", tf_pred)
sd = model.state_dict()
unfuse_experts(sd)
-Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves
-if args.fp8:
- n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors")
- print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) "
- f"-> glm_tiny/model.safetensors")
-else:
+if args.fmt6 or args.fmt4:
+ # Salva SOLO gli expert routed quantizzati (fmt=6 o fmt=4 + .qs); shared/dense/attn/
+ # norme restano f32. I pesi impacchettati provengono dai pesi ORIGINALI (sd_orig,
+ # non fusi), non da quelli gia' dequantizzati, cosi' il motore decodifica ESATTAMENTE
+ # i pesi che il round-trip sopra ha usato per il riferimento. EN: quantize ONLY the
+ # routed experts (fmt=6 or fmt=4 + .qs); everything else stays f32. Packed weights
+ # come from the ORIGINAL (unfused) weights, not the dequantized ones, so the engine
+ # decodes exactly the weights the round-trip used for the reference.
+ outdir = "glm_tiny_fmt6" if args.fmt6 else "glm_tiny_fmt4"
+ fmt = 6 if args.fmt6 else 4
+ Path(outdir).mkdir(parents=True, exist_ok=True)
+ import numpy as np
+ import iq3_pack
+ from convert_fp8_to_int4 import quant_int4_grouped
+ sd = unfuse_experts(sd_orig)
+ out = {}
+ for name, t in sd.items():
+ if ".mlp.experts." in name and name.endswith(
+ (".gate_proj.weight", ".up_proj.weight", ".down_proj.weight")):
+ w = t.detach().float().numpy()
+ if fmt == 6:
+ packed = iq3_pack.encode(iq3_pack.rotate_rows(w))
+ out[name] = torch.from_numpy(np.ascontiguousarray(packed.reshape(-1)))
+ out[name + ".qs"] = torch.tensor([6.0], dtype=torch.float32)
+ else:
+ q, s = quant_int4_grouped(w, 4, 64)
+ out[name] = torch.from_numpy(q)
+ out[name + ".qs"] = torch.from_numpy(s)
+ else:
+ out[name] = t.detach().contiguous()
from safetensors.torch import save_file
- save_file({k: v.contiguous() for k, v in sd.items()}, "glm_tiny/model.safetensors")
-json.dump(cfg.to_dict(), open("glm_tiny/config.json", "w"))
-json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred}, open("ref_glm.json", "w"))
-print("saved: glm_tiny/ (weights + config) and ref_glm.json"
- + (" [fp8]" if args.fp8 else ""))
+ save_file(out, f"{outdir}/model.safetensors")
+ json.dump(cfg.to_dict(), open(f"{outdir}/config.json", "w"))
+ json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred},
+ open(f"{outdir}/ref_glm.json", "w"))
+ print(f"saved: {outdir}/ (routed experts fmt={fmt}, shared/dense/attn f32) + "
+ f"{outdir}/ref_glm.json")
+else:
+ Path("glm_tiny").mkdir(parents=True, exist_ok=True) # safetensors/json won't create the dir themselves
+ if args.fp8:
+ n_fp8, n_tot = save_fp8_safetensors(sd, "glm_tiny/model.safetensors")
+ print(f"\nsaved FP8: {n_fp8} e4m3 tensors (+{n_tot - n_fp8} scale_inv sidecars / f32) "
+ f"-> glm_tiny/model.safetensors")
+ else:
+ from safetensors.torch import save_file
+ save_file({k: v.contiguous() for k, v in sd.items()}, "glm_tiny/model.safetensors")
+ json.dump(cfg.to_dict(), open("glm_tiny/config.json", "w"))
+ json.dump({"prompt_ids": prompt, "full_ids": full, "tf_pred": tf_pred}, open("ref_glm.json", "w"))
+ print("saved: glm_tiny/ (weights + config) and ref_glm.json"
+ + (" [fp8]" if args.fp8 else ""))