Run frontier MoE models on hardware you already own β pure C, zero deps, experts streamed from disk. Tiny engine, immense model. π¦
C
convert: block the quantizers by rows, and report progress on the local pass
Steve Markgraf8e2ee1fade2a
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.pyindex 998f8288..058615b9 100644--- a/c/tools/convert_fp8_to_int4.py+++ b/c/tools/convert_fp8_to_int4.py@@ -261,6 +261,25 @@ def dequant(f, name, keys):# record dict(PROJ_BITS) β this global is the definition those sites depend on.PROJ_BITS = {}+# Rows per quantization block. Every non-E8 format here carries per-row scales (or+# per-group scales along I), so rows never interact and blocking is BIT-IDENTICAL to+# quantizing the whole tensor β it only caps the peak of the quantizer's full-size+# temporaries. That matters on small hosts: embed/lm_head at [154880, 6144] is 3.8 GB+# as f32, and abs/divide/rint/clip each materialise another copy (~15 GB peak) on a+# box with 13 GB free. E8 is excluded β it is already blocked inside iq3_pack.encode,+# and its ".qs" companion is a single format tag, not a per-row scale.+QUANT_ROWS = int(os.environ.get("COLI_QUANT_ROWS", "8192"))++def _rowwise(fn, w, *args):+ O = w.shape[0]+ if O <= QUANT_ROWS:+ return fn(w, *args)+ qs, ss = [], []+ for r0 in range(0, O, QUANT_ROWS):+ q, s = fn(w[r0:r0 + QUANT_ROWS], *args)+ qs.append(q); ss.append(s)+ return np.concatenate(qs), np.concatenate(ss)+def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,keep_mtp=False, keep_idx=False, group_size=0, bits_map=None):from safetensors import safe_open@@ -290,15 +309,16 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,out_dict[name] = w.astype(np.float32); continueif bits == E8:# fmt=6 E8/IQ3 β routed-expert projections only, enforced in main().+ # Already row-blocked inside iq3_pack.encode.q, s = quant_e8(w)elif bits == 3:# int3-g64 (fmt=5): inherently group-64, distinct from grouped-int4.- q, s = quant_int3_g64(w)+ q, s = _rowwise(quant_int3_g64, w)elif group_size > 0 and bits <= 4:- q, s = quant_int4_grouped(w, bits, group_size)+ q, s = _rowwise(quant_int4_grouped, w, bits, group_size)else:- q, s = (quant_int2(w, bits) if bits <= 2 else- quant_int4(w, bits) if bits <= 4 else quant_int8(w, bits))+ q, s = _rowwise(quant_int2 if bits <= 2 else+ quant_int4 if bits <= 4 else quant_int8, w, bits)out_dict[name] = qout_dict[name + ".qs"] = s@@ -567,6 +587,8 @@ def main():returndone = prog.setdefault("shards", {}); prog["params"] = paramsn = 0; fresh = 0; skipped = 0+ import time as _t+ t_start = _t.time()for i, sp in enumerate(shards):key = os.path.basename(sp)prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso@@ -574,6 +596,14 @@ def main():if prev: n += 1skipped += 1continue+ # Progress + ETA: the local pass can run for days on a big model (E8 on+ # GLM-5.2 is ~50 h split across workers), and without this the loop is+ # silent until it finishes. flush because stdout is a redirected file.+ eta = ""+ if fresh:+ per = (_t.time() - t_start) / fresh+ eta = f", ETA {per * (len(shards) - i) / 3600:.1f} h"+ print(f"[{i + 1}/{len(shards)}] {key} ({free_gb(a.outdir):.0f} GB free{eta})", flush=True)out = {}convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits,keep_mtp=a.mtp, keep_idx=a.indexer,