Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tools: Inkling bf16 -> colibri int4 converter, e2e-validated

Nicholas Beerbowerb721a1e8884c
diff --git a/c/tools/convert_inkling_int4.py b/c/tools/convert_inkling_int4.py
new file mode 100644
index 00000000..fb7985c2
--- /dev/null
+++ b/c/tools/convert_inkling_int4.py
@@ -0,0 +1,401 @@
+#!/usr/bin/env python3
+"""Convert the Thinking Machines Inkling BF16 checkpoint (TML-native tensor
+names) into a colibri snapshot readable by inkling.c.
+
+Name mapping follows transformers' checkpoint_conversion_mapping for
+inkling_mm_model verbatim (conversion_mapping.py), including the CRITICAL
+detail that every fused w13 tensor stores gate/up rows INTERLEAVED
+(g0,u0,g1,u1,...): de-interleaving = even rows first, then odd rows.
+Quantization math is bit-identical to convert_fp8_to_int4.py / the C engine
+(np.rint = lrintf, per-row symmetric scales, low nibble = even column).
+
+Output tensor policy (Stage A):
+ - routed experts (95% of the params) -> int4 (or --xbits) + `.qs` f32 row scales
+ - norms, sconv weights, rel-bias banks, router (+bias/scales) -> f32
+ - everything else (attn projections, dense/shared MLP, embed, lm_head)
+ -> bf16 passthrough (st_read_f32 in the C engine converts on load)
+ - model.audio.* / model.visual.* / model.mtp.* -> skipped (text-only)
+
+Modes:
+ --indir DIR --outdir DIR [--watch] convert local shards (resumable per
+ shard; --watch polls while `hf download` is still running and exits
+ when every shard in model.safetensors.index.json is converted)
+ --selftest numpy-only unit tests (interleave, int4)
+ --selftest-e2e HFDIR OUTBASE build a fake TML-named checkpoint from a
+ tiny HF-format snapshot (tools/make_tiny_inkling.py), then convert it
+ with passthrough and int4 experts -> OUTBASE-pass/, OUTBASE-i4/
+"""
+import argparse
+import glob
+import json
+import os
+import re
+import shutil
+import sys
+import time
+
+import numpy as np
+import torch
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+# ---------- quantization (identical to convert_fp8_to_int4.py) ----------
+
+def quant_int8_rows(w):
+ """w: f32 [O,I] -> (int8 bytes [O,I] viewed u8, f32 scales [O])"""
+ s = np.abs(w).max(axis=1, keepdims=True) / 127.0
+ s[s < 1e-8] = 1e-8
+ q = np.clip(np.rint(w / s), -128, 127).astype(np.int8)
+ return q.view(np.uint8), s[:, 0].astype(np.float32)
+
+
+def quant_int4_rows(w):
+ """w: f32 [O,I], I even -> (packed u8 [O,I/2], f32 scales [O]).
+ Low nibble = even column, high nibble = odd column, offset +8."""
+ O, I = w.shape
+ assert I % 2 == 0
+ s = np.abs(w).max(axis=1, keepdims=True) / 7.0
+ s[s < 1e-8] = 1e-8
+ q = np.clip(np.rint(w / s), -8, 7).astype(np.int32)
+ lo = (q[:, 0::2] + 8).astype(np.uint8)
+ hi = (q[:, 1::2] + 8).astype(np.uint8)
+ return (lo | (hi << 4)), s[:, 0].astype(np.float32)
+
+
+def deinterleave(t: torch.Tensor, dim: int) -> torch.Tensor:
+ """TML fused-w13 layout (g0,u0,g1,u1,...) -> grouped ([gate; up]).
+ Matches transformers' Interleave(dim) conversion op."""
+ n = t.shape[dim]
+ idx = torch.cat([torch.arange(0, n, 2), torch.arange(1, n, 2)])
+ return t.index_select(dim, idx)
+
+
+def interleave(t: torch.Tensor, dim: int) -> torch.Tensor:
+ """Inverse of deinterleave (used only to fabricate test checkpoints)."""
+ n = t.shape[dim]
+ idx = torch.empty(n, dtype=torch.long)
+ idx[0::2] = torch.arange(0, n // 2)
+ idx[1::2] = torch.arange(n // 2, n)
+ return t.index_select(dim, idx)
+
+
+# ---------- name mapping (TML native -> HF names inkling.c loads) ----------
+
+SKIP_PREFIXES = ("model.audio.", "model.visual.", "model.mtp.")
+
+RENAMES = [
+ (r"^model\.llm\.embed\.weight$", "model.embed_tokens.weight"),
+ (r"^model\.llm\.embed_norm\.weight$", "model.embed_norm.weight"),
+ (r"^model\.llm\.norm\.weight$", "model.norm.weight"),
+ (r"^model\.llm\.unembed\.weight$", "lm_head.weight"),
+ (r"^model\.llm\.layers\.", "model.layers."),
+]
+SUBS = [
+ (r"\.attn\.wq_du\.", ".self_attn.q_proj."),
+ (r"\.attn\.wk_dv\.", ".self_attn.k_proj."),
+ (r"\.attn\.wv_dv\.", ".self_attn.v_proj."),
+ (r"\.attn\.wr_du\.", ".self_attn.r_proj."),
+ (r"\.attn\.wo_ud\.", ".self_attn.o_proj."),
+ (r"\.attn\.q_norm\.", ".self_attn.q_norm."),
+ (r"\.attn\.k_norm\.", ".self_attn.k_norm."),
+ (r"\.attn\.k_sconv\.weight$", ".self_attn.k_sconv.conv1d.weight"),
+ (r"\.attn\.v_sconv\.weight$", ".self_attn.v_sconv.conv1d.weight"),
+ (r"\.attn\.rel_logits_proj\.proj$", ".self_attn.rel_logits_proj.proj"),
+ (r"\.attn_sconv\.weight$", ".attn_sconv.conv1d.weight"),
+ (r"\.mlp_sconv\.weight$", ".mlp_sconv.conv1d.weight"),
+ (r"\.attn_norm\.weight$", ".input_layernorm.weight"),
+ (r"\.mlp_norm\.weight$", ".post_attention_layernorm.weight"),
+ (r"\.mlp\.gate\.bias$", ".mlp.gate.e_score_correction_bias"),
+ (r"\.mlp\.w2_md\.weight$", ".mlp.down_proj.weight"),
+ (r"\.mlp\.experts\.w2_weight$", ".mlp.experts.down_proj"),
+ (r"\.mlp\.shared_experts\.shared_w2_weight$", ".mlp.shared_experts.down_proj"),
+]
+# names that stay f32 in the container (small / numerically sensitive;
+# the reference keeps every sconv in fp32)
+F32_RE = re.compile(
+ r"(norm\.weight$|layernorm\.weight$|rel_logits_proj\.proj$|conv1d\.weight$"
+ r"|global_scale$|e_score_correction_bias$|\.mlp\.gate\.weight$)"
+)
+
+
+def map_name(name):
+ """Simple renames only (fused w13 tensors are handled by the shard loop).
+ Returns the mapped name, or None to skip."""
+ if name.startswith(SKIP_PREFIXES):
+ return None
+ for pat, rep in RENAMES:
+ name = re.sub(pat, rep, name)
+ for pat, rep in SUBS:
+ name = re.sub(pat, rep, name)
+ return name
+
+
+# ---------- per-shard conversion ----------
+
+def quantize_expert_tensor(f, name, xbits, out, out_name):
+ """Chunked (per-expert) de-interleave/quantize of the fused 3D expert
+ tensors, so peak RAM stays at one expert slice, not one 19 GB tensor."""
+ sl = f.get_slice(name)
+ E, R, C = sl.get_shape() # w13: [E,2I,D] (interleaved), w2: [E,D,I]
+ is_w13 = name.endswith("w13_weight")
+ if xbits == 4:
+ qs = np.empty((E * R, C // 2), np.uint8)
+ else:
+ qs = np.empty((E * R, C), np.uint8)
+ sc = np.empty(E * R, np.float32)
+ for e in range(E):
+ w = sl[e] # [R, C] bf16
+ if is_w13:
+ w = deinterleave(w, 0) # -> [gate rows; up rows]
+ w = w.float().numpy()
+ q, s = quant_int4_rows(w) if xbits == 4 else quant_int8_rows(w)
+ qs[e * R:(e + 1) * R] = q
+ sc[e * R:(e + 1) * R] = s
+ out[out_name] = torch.from_numpy(qs.reshape(-1))
+ out[out_name + ".qs"] = torch.from_numpy(sc)
+
+
+def convert_shard(path, out, xbits):
+ with safe_open(path, framework="pt") as f:
+ for name in f.keys():
+ if name.startswith(SKIP_PREFIXES):
+ continue
+ base = map_name(name)
+ if name.endswith(".mlp.experts.w13_weight"):
+ tgt = base.replace(".w13_weight", ".gate_up_proj")
+ if xbits:
+ quantize_expert_tensor(f, name, xbits, out, tgt)
+ else:
+ t = f.get_tensor(name)
+ out[tgt] = deinterleave(t, 1).contiguous()
+ continue
+ if name.endswith(".mlp.experts.w2_weight"):
+ tgt = map_name(name)
+ if xbits:
+ quantize_expert_tensor(f, name, xbits, out, tgt)
+ else:
+ out[tgt] = f.get_tensor(name)
+ continue
+ if name.endswith(".mlp.shared_experts.shared_w13_weight"):
+ t = deinterleave(f.get_tensor(name), 1) # [ns, 2I, D] -> grouped
+ half = t.shape[1] // 2
+ pref = base.replace(".shared_w13_weight", "")
+ out[pref + ".gate_proj"] = t[:, :half].contiguous()
+ out[pref + ".up_proj"] = t[:, half:].contiguous()
+ continue
+ if name.endswith(".mlp.w13_dn.weight"):
+ t = deinterleave(f.get_tensor(name), 0) # [2I, D] -> grouped
+ half = t.shape[0] // 2
+ pref = base.replace(".w13_dn.weight", "")
+ out[pref + ".gate_proj.weight"] = t[:half].contiguous()
+ out[pref + ".up_proj.weight"] = t[half:].contiguous()
+ continue
+ t = f.get_tensor(name)
+ if F32_RE.search(base):
+ out[base] = t.float()
+ else:
+ out[base] = t # bf16 passthrough
+
+
+AUX_FILES = ["config.json", "tokenizer.json", "tokenizer_config.json",
+ "special_tokens_map.json", "chat_template.jinja"]
+
+
+def shard_out_name(src):
+ m = re.search(r"model-(\d+)-of-\d+\.safetensors$", os.path.basename(src))
+ return f"out-{m.group(1)}.safetensors" if m else \
+ "out-" + os.path.basename(src)
+
+
+def convert_dir(indir, outdir, xbits, watch=False, delete_src=False):
+ os.makedirs(outdir, exist_ok=True)
+ index_path = os.path.join(indir, "model.safetensors.index.json")
+ done_src = set()
+ while True:
+ shards = sorted(glob.glob(os.path.join(indir, "model-*.safetensors")))
+ if not shards:
+ single = os.path.join(indir, "model.safetensors")
+ if os.path.exists(single):
+ shards = [single]
+ for sp in shards:
+ if sp in done_src:
+ continue
+ dst = os.path.join(outdir, shard_out_name(sp))
+ if os.path.exists(dst):
+ done_src.add(sp)
+ continue
+ t0 = time.time()
+ out = {}
+ convert_shard(sp, out, xbits)
+ if out:
+ save_file(out, dst + ".tmp")
+ os.replace(dst + ".tmp", dst) # atomic: partial writes never count as done
+ else: # shard held only skipped (mm/mtp) tensors
+ open(dst, "wb").close()
+ done_src.add(sp)
+ src_gb = os.path.getsize(sp) / 1e9
+ dst_gb = os.path.getsize(dst) / 1e9
+ print(f"{os.path.basename(sp)} ({src_gb:.1f}G) -> "
+ f"{os.path.basename(dst)} ({dst_gb:.1f}G) in {time.time()-t0:.0f}s",
+ flush=True)
+ if delete_src:
+ os.remove(sp)
+ # completion check: every shard named in the index is converted
+ total = None
+ if os.path.exists(index_path):
+ wm = json.load(open(index_path))["weight_map"]
+ total = sorted(set(wm.values()))
+ if all(os.path.exists(os.path.join(outdir, shard_out_name(s))) for s in total):
+ break
+ elif shards and not watch:
+ break
+ if not watch:
+ break
+ time.sleep(60)
+ for fn in AUX_FILES:
+ src = os.path.join(indir, fn)
+ if os.path.exists(src):
+ shutil.copy(src, outdir)
+ print(f"conversion complete -> {outdir}")
+
+
+# ---------- selftests ----------
+
+def selftest():
+ torch.manual_seed(0)
+ # 1) interleave round-trip + semantics
+ g = torch.arange(0, 12).reshape(6, 2).float()
+ u = torch.arange(100, 112).reshape(6, 2).float()
+ fused = interleave(torch.cat([g, u]), 0)
+ assert torch.equal(fused[0], g[0]) and torch.equal(fused[1], u[0]), \
+ "interleave: row 0 must be gate0, row 1 up0"
+ got = deinterleave(fused, 0)
+ assert torch.equal(got[:6], g) and torch.equal(got[6:], u)
+ # 2) int4 pack/unpack round-trip against the C engine's convention
+ w = np.random.randn(16, 32).astype(np.float32)
+ q, s = quant_int4_rows(w)
+ lo = (q & 0x0F).astype(np.int32) - 8
+ hi = ((q >> 4) & 0x0F).astype(np.int32) - 8
+ deq = np.empty_like(w)
+ deq[:, 0::2] = lo
+ deq[:, 1::2] = hi
+ deq *= s[:, None]
+ ref = np.clip(np.rint(w / s[:, None]), -8, 7) * s[:, None]
+ assert np.array_equal(deq, ref), "int4 round-trip mismatch"
+ # 3) name mapping spot checks
+ cases = {
+ "model.llm.embed.weight": "model.embed_tokens.weight",
+ "model.llm.unembed.weight": "lm_head.weight",
+ "model.llm.layers.3.attn.wq_du.weight": "model.layers.3.self_attn.q_proj.weight",
+ "model.llm.layers.3.attn.k_sconv.weight": "model.layers.3.self_attn.k_sconv.conv1d.weight",
+ "model.llm.layers.3.attn_sconv.weight": "model.layers.3.attn_sconv.conv1d.weight",
+ "model.llm.layers.3.attn_norm.weight": "model.layers.3.input_layernorm.weight",
+ "model.llm.layers.3.mlp_norm.weight": "model.layers.3.post_attention_layernorm.weight",
+ "model.llm.layers.3.mlp.gate.bias": "model.layers.3.mlp.gate.e_score_correction_bias",
+ "model.llm.layers.3.mlp.gate.global_scale": "model.layers.3.mlp.gate.global_scale",
+ "model.llm.layers.3.mlp.experts.w2_weight": "model.layers.3.mlp.experts.down_proj",
+ "model.llm.layers.0.mlp.w2_md.weight": "model.layers.0.mlp.down_proj.weight",
+ "model.mtp.layers.0.input_proj.weight": None,
+ "model.audio.encoder.weight": None,
+ "model.visual.final_norm.weight": None,
+ }
+ for src, want in cases.items():
+ got = map_name(src)
+ assert got == want, f"map_name({src}) = {got}, want {want}"
+ print("SELFTEST OK")
+
+
+def selftest_e2e(hfdir, outbase):
+ """Fabricate a TML-named checkpoint from a tiny HF snapshot (inverse
+ mapping, w13 re-fused AND re-interleaved), then convert it back. Validates
+ the converter + the C engine's container reader end-to-end without the
+ real download. Tensors stay f32 so the oracle comparison is exact."""
+ src = os.path.join(hfdir, "model.safetensors")
+ fake = outbase + "-tml"
+ os.makedirs(fake, exist_ok=True)
+ inv = {}
+ with safe_open(src, framework="pt") as f:
+ names = list(f.keys())
+ tensors = {n: f.get_tensor(n) for n in names}
+ layers = {}
+ for n in names:
+ m = re.match(r"model\.layers\.(\d+)\.(.*)", n)
+ if m:
+ layers.setdefault(int(m.group(1)), {})[m.group(2)] = tensors[n]
+ for n, t in tensors.items():
+ if n == "model.embed_tokens.weight": inv["model.llm.embed.weight"] = t
+ elif n == "model.embed_norm.weight": inv["model.llm.embed_norm.weight"] = t
+ elif n == "model.norm.weight": inv["model.llm.norm.weight"] = t
+ elif n == "lm_head.weight": inv["model.llm.unembed.weight"] = t
+ for li, lt in layers.items():
+ p = f"model.llm.layers.{li}."
+ for hf, tml in [("self_attn.q_proj.weight", "attn.wq_du.weight"),
+ ("self_attn.k_proj.weight", "attn.wk_dv.weight"),
+ ("self_attn.v_proj.weight", "attn.wv_dv.weight"),
+ ("self_attn.r_proj.weight", "attn.wr_du.weight"),
+ ("self_attn.o_proj.weight", "attn.wo_ud.weight"),
+ ("self_attn.q_norm.weight", "attn.q_norm.weight"),
+ ("self_attn.k_norm.weight", "attn.k_norm.weight"),
+ ("self_attn.rel_logits_proj.proj", "attn.rel_logits_proj.proj"),
+ ("self_attn.k_sconv.conv1d.weight", "attn.k_sconv.weight"),
+ ("self_attn.v_sconv.conv1d.weight", "attn.v_sconv.weight"),
+ ("attn_sconv.conv1d.weight", "attn_sconv.weight"),
+ ("mlp_sconv.conv1d.weight", "mlp_sconv.weight"),
+ ("input_layernorm.weight", "attn_norm.weight"),
+ ("post_attention_layernorm.weight", "mlp_norm.weight"),
+ ("mlp.gate.weight", "mlp.gate.weight"),
+ ("mlp.gate.e_score_correction_bias", "mlp.gate.bias"),
+ ("mlp.gate.global_scale", "mlp.gate.global_scale"),
+ ("mlp.global_scale", "mlp.global_scale"),
+ ("mlp.down_proj.weight", "mlp.w2_md.weight"),
+ ("mlp.experts.down_proj", "mlp.experts.w2_weight"),
+ ("mlp.shared_experts.down_proj", "mlp.shared_experts.shared_w2_weight")]:
+ if hf in lt:
+ inv[p + tml] = lt[hf]
+ if "mlp.gate_proj.weight" in lt: # dense: fuse + interleave
+ inv[p + "mlp.w13_dn.weight"] = interleave(
+ torch.cat([lt["mlp.gate_proj.weight"], lt["mlp.up_proj.weight"]]), 0)
+ if "mlp.experts.gate_up_proj" in lt: # routed: re-interleave dim1
+ inv[p + "mlp.experts.w13_weight"] = interleave(lt["mlp.experts.gate_up_proj"], 1)
+ if "mlp.shared_experts.gate_proj" in lt:
+ inv[p + "mlp.shared_experts.shared_w13_weight"] = interleave(
+ torch.cat([lt["mlp.shared_experts.gate_proj"],
+ lt["mlp.shared_experts.up_proj"]], dim=1), 1)
+ save_file({k: v.contiguous() for k, v in inv.items()},
+ os.path.join(fake, "model-00001-of-00001.safetensors"))
+ shutil.copy(os.path.join(hfdir, "config.json"), fake)
+ print(f"fake TML checkpoint: {fake} ({len(inv)} tensors)")
+ for tag, xbits in [("pass", 0), ("i4", 4)]:
+ convert_dir(fake, f"{outbase}-{tag}", xbits)
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("--indir")
+ ap.add_argument("--outdir")
+ ap.add_argument("--xbits", type=int, default=4, choices=[0, 4, 8],
+ help="routed-expert bits: 4 (default), 8, or 0 = bf16 passthrough")
+ ap.add_argument("--watch", action="store_true",
+ help="poll --indir while the download is still running")
+ ap.add_argument("--delete-src", action="store_true")
+ ap.add_argument("--selftest", action="store_true")
+ ap.add_argument("--selftest-e2e", nargs=2, metavar=("HFDIR", "OUTBASE"))
+ a = ap.parse_args()
+ if a.selftest:
+ selftest(); return
+ if a.selftest_e2e:
+ selftest_e2e(*a.selftest_e2e); return
+ if not a.indir or not a.outdir:
+ ap.error("--indir and --outdir required")
+ os.makedirs(a.outdir, exist_ok=True)
+ lock = open(os.path.join(a.outdir, ".convert.lock"), "w")
+ import fcntl
+ try:
+ fcntl.flock(lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
+ except OSError:
+ sys.exit("ERROR: another converter is already using this output directory.")
+ convert_dir(a.indir, a.outdir, a.xbits, watch=a.watch, delete_src=a.delete_src)
+
+
+if __name__ == "__main__":
+ main()