Skip to content

Run frontier MoE models on hardware you already own β€” pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

convert: block the quantizers by rows, and report progress on the local pass

Steve Markgraf8e2ee1fade2a
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.py
index 998f8288..058615b9 100644
--- a/c/tools/convert_fp8_to_int4.py
+++ b/c/tools/convert_fp8_to_int4.py
@@ -261,6 +261,25 @@ def dequant(f, name, keys):
# record dict(PROJ_BITS) β€” this global is the definition those sites depend on.
PROJ_BITS = {}
+# Rows per quantization block. Every non-E8 format here carries per-row scales (or
+# per-group scales along I), so rows never interact and blocking is BIT-IDENTICAL to
+# quantizing the whole tensor β€” it only caps the peak of the quantizer's full-size
+# temporaries. That matters on small hosts: embed/lm_head at [154880, 6144] is 3.8 GB
+# as f32, and abs/divide/rint/clip each materialise another copy (~15 GB peak) on a
+# box with 13 GB free. E8 is excluded β€” it is already blocked inside iq3_pack.encode,
+# and its ".qs" companion is a single format tag, not a per-row scale.
+QUANT_ROWS = int(os.environ.get("COLI_QUANT_ROWS", "8192"))
+
+def _rowwise(fn, w, *args):
+ O = w.shape[0]
+ if O <= QUANT_ROWS:
+ return fn(w, *args)
+ qs, ss = [], []
+ for r0 in range(0, O, QUANT_ROWS):
+ q, s = fn(w[r0:r0 + QUANT_ROWS], *args)
+ qs.append(q); ss.append(s)
+ return np.concatenate(qs), np.concatenate(ss)
+
def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,
keep_mtp=False, keep_idx=False, group_size=0, bits_map=None):
from safetensors import safe_open
@@ -290,15 +309,16 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,
out_dict[name] = w.astype(np.float32); continue
if bits == E8:
# fmt=6 E8/IQ3 β€” routed-expert projections only, enforced in main().
+ # Already row-blocked inside iq3_pack.encode.
q, s = quant_e8(w)
elif bits == 3:
# int3-g64 (fmt=5): inherently group-64, distinct from grouped-int4.
- q, s = quant_int3_g64(w)
+ q, s = _rowwise(quant_int3_g64, w)
elif group_size > 0 and bits <= 4:
- q, s = quant_int4_grouped(w, bits, group_size)
+ q, s = _rowwise(quant_int4_grouped, w, bits, group_size)
else:
- q, s = (quant_int2(w, bits) if bits <= 2 else
- quant_int4(w, bits) if bits <= 4 else quant_int8(w, bits))
+ q, s = _rowwise(quant_int2 if bits <= 2 else
+ quant_int4 if bits <= 4 else quant_int8, w, bits)
out_dict[name] = q
out_dict[name + ".qs"] = s
@@ -567,6 +587,8 @@ def main():
return
done = prog.setdefault("shards", {}); prog["params"] = params
n = 0; fresh = 0; skipped = 0
+ import time as _t
+ t_start = _t.time()
for i, sp in enumerate(shards):
key = os.path.basename(sp)
prev = done.get(key) # None = mai visto; "" = visto, vuoto; nome = emesso
@@ -574,6 +596,14 @@ def main():
if prev: n += 1
skipped += 1
continue
+ # Progress + ETA: the local pass can run for days on a big model (E8 on
+ # GLM-5.2 is ~50 h split across workers), and without this the loop is
+ # silent until it finishes. flush because stdout is a redirected file.
+ eta = ""
+ if fresh:
+ per = (_t.time() - t_start) / fresh
+ eta = f", ETA {per * (len(shards) - i) / 3600:.1f} h"
+ print(f"[{i + 1}/{len(shards)}] {key} ({free_gb(a.outdir):.0f} GB free{eta})", flush=True)
out = {}
convert_shard(sp, out, a.n_layers, a.ebits, a.io_bits, a.xbits,
keep_mtp=a.mtp, keep_idx=a.indexer,