Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
convert: --up-bits/--gate-bits/--down-bits per-projection expert quant
FABIOTESS932f3678b014
diff --git a/c/tools/convert_fp8_to_int4.py b/c/tools/convert_fp8_to_int4.pyindex b4b96aca..447654d0 100644--- a/c/tools/convert_fp8_to_int4.py+++ b/c/tools/convert_fp8_to_int4.py@@ -237,6 +237,14 @@ def dequant(f, name, keys):return (w * sc).numpy()return f.get_tensor(name).to(torch.float32).numpy()+# Per-projection bit overrides for ROUTED experts (gate_proj/up_proj/down_proj), set from+# --up-bits/--gate-bits/--down-bits in main(). Empty = uniform xbits. Motivated by the+# measured result that up_proj tolerates int3-g64 at ~zero quality cost while int2 craters+# (OLMoE ablation, PR #168 comment): up-only int3 drops ~8% of expert bytes for free.+# NB: the resume manifests (check_or_record_params and the --indir progress file) already+# record dict(PROJ_BITS) — this global is the definition those sites depend on.+PROJ_BITS = {}+def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,keep_mtp=False, keep_idx=False, group_size=0, bits_map=None):from safetensors import safe_open@@ -258,6 +266,10 @@ def convert_shard(path, out_dict, n_layers, ebits, io_bits, xbits,# Any unknown kind that fell through classify as "q"if bits_map and kind not in bits_map and kind not in ("io", "x", "sh", "o", "kvb", "attn", "dmlp"):bits = ebits+ # Per-projection override for routed experts, applied on top of the type-level bits.+ if kind == "x" and PROJ_BITS: # e.g. up_proj -> 3 (int3-g64) while gate/down stay 4+ for proj, pb in PROJ_BITS.items():+ if f".{proj}.weight" in name: bits = pb; breakif w.ndim != 2: # es. bias 1D non previsto come 'q' -> tienilo f32out_dict[name] = w.astype(np.float32); continueif bits == 3:@@ -321,6 +333,13 @@ def main():help="bits for dense MLP (first 3 layers). Default=ebits")ap.add_argument("--group-size", type=int, default=0, # 0 = per-row (backward compat); 128 = group-scaledhelp="group size for int4 scales: 0=per-row (default), 128=one scale per 128 elements (much better quality)")+ # Per-projection bit overrides for routed experts (orthogonal to the type-level flags above).+ ap.add_argument("--up-bits", type=int, default=None,+ help="bits for up_proj in routed experts (e.g. 3 = int3-g64). Default=xbits")+ ap.add_argument("--gate-bits", type=int, default=None,+ help="bits for gate_proj in routed experts. Default=xbits")+ ap.add_argument("--down-bits", type=int, default=None,+ help="bits for down_proj in routed experts. Default=xbits")ap.add_argument("--n-layers", type=int, default=78)ap.add_argument("--min-free-gb", type=float, default=20.0)ap.add_argument("--selftest", action="store_true")@@ -350,6 +369,10 @@ def main():"embedding half -> MTP acceptance ~0% (issue #8). Use the default --ebits 8, ""or add --group-size 128 for group-scaled int4.")if a.xbits is None: a.xbits = a.ebits+ for proj, val in (("gate_proj", a.gate_bits), ("up_proj", a.up_bits), ("down_proj", a.down_bits)):+ if val is not None: PROJ_BITS[proj] = val+ if PROJ_BITS:+ print(f"[per-projection expert bits] {PROJ_BITS} (others -> xbits={a.xbits})")# Build per-type bits map. If a type-specific arg is set, use it; otherwise the# converter falls back to ebits for that type.