Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tools: K3 tokenizer generator, numpy layer reference, container repack

Steve Markgraf8a2eccbbb107
diff --git a/c/tools/k3_ref.py b/c/tools/k3_ref.py
new file mode 100644
index 00000000..8dd4f10b
--- /dev/null
+++ b/c/tools/k3_ref.py
@@ -0,0 +1,338 @@
+#!/usr/bin/env python3
+"""Numpy reference for the first N layers of Kimi K3 — validates c/kimi_k3.c.
+
+Reads tensors straight from the HF shards (BF16 -> f32, MXFP4 dequant), runs T
+injected input rows [T, hidden] (the engine's K3_X0 hook, bypassing the
+embedding, which lives in one of the last shards) through layers 0..N-1
+exactly as modeling_kimi_linear.py does — KDA recurrence, gated NoPE MLA,
+AttnRes, LatentMoE with SiTU-GLU — and writes the same trace layout the C
+engine emits under K3_TRACE: per token, per layer, hidden[hidden] f32.
+
+ gen : k3_ref.py gen <x0.bin> --tokens 6 --dim 7168 [--seed 7]
+ run : k3_ref.py run <hf_dir> --x0 x0.bin --layers 4 --out ref.bin
+ cmp : k3_ref.py cmp <c_trace.bin> <ref.bin> --dim 7168 --layers 4
+
+Pure numpy, no torch. Everything f32 (f64 only where the C engine also
+accumulates in double: AttnRes scores, RMS means).
+"""
+import argparse
+import json
+import os
+import struct
+import sys
+
+import numpy as np
+
+E2M1 = np.array([0, .5, 1, 1.5, 2, 3, 4, 6, -0., -.5, -1, -1.5, -2, -3, -4, -6],
+ dtype=np.float32)
+
+
+class Shards:
+ def __init__(self, d):
+ self.dir = d
+ idx = json.load(open(os.path.join(d, "model.safetensors.index.json")))
+ self.wmap = idx["weight_map"]
+ self.hdrs = {}
+
+ def _hdr(self, fn):
+ if fn not in self.hdrs:
+ with open(os.path.join(self.dir, fn), "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ self.hdrs[fn] = (json.loads(f.read(n)), 8 + n)
+ return self.hdrs[fn]
+
+ def get(self, name):
+ fn = self.wmap[name]
+ h, base = self._hdr(fn)
+ v = h[name]
+ o0, o1 = v["data_offsets"]
+ with open(os.path.join(self.dir, fn), "rb") as f:
+ f.seek(base + o0)
+ raw = f.read(o1 - o0)
+ if v["dtype"] == "BF16":
+ u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16
+ a = u.view(np.float32)
+ elif v["dtype"] == "F32":
+ a = np.frombuffer(raw, dtype=np.float32)
+ elif v["dtype"] == "U8":
+ a = np.frombuffer(raw, dtype=np.uint8)
+ else:
+ raise ValueError(v["dtype"])
+ return a.reshape(v["shape"]).copy()
+
+
+def sigmoid(x):
+ return 1.0 / (1.0 + np.exp(-x))
+
+
+def silu(x):
+ return x * sigmoid(x)
+
+
+def rmsnorm(x, w, eps):
+ ms = np.mean(x.astype(np.float64) ** 2, axis=-1, keepdims=True)
+ return (x * (1.0 / np.sqrt(ms + eps))).astype(np.float32) * w
+
+
+def situ(g, u, b1, b2):
+ return (b1 * np.tanh(g / b1) * sigmoid(g)) * (b2 * np.tanh(u / b2))
+
+
+def deq_mx4(packed, scale):
+ O, half = packed.shape
+ nib = np.empty((O, half * 2), dtype=np.uint8)
+ nib[:, 0::2] = packed & 0xF
+ nib[:, 1::2] = packed >> 4
+ w = E2M1[nib]
+ sc = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127).astype(np.float32)
+ I = half * 2
+ return (w.reshape(O, I // 32, 32) * sc[:, :, None]).reshape(O, I)
+
+
+def res_mix(prefix, bres, sw, eps):
+ """entries = bres rows + prefix (last); softmax over RMS-normed dots; mix RAW."""
+ vs = list(bres) + [prefix]
+ sc = np.empty(len(vs), dtype=np.float32)
+ sw64 = sw.astype(np.float64)
+ for e, v in enumerate(vs):
+ v64 = v.astype(np.float64)
+ ms = np.mean(v64 ** 2)
+ sc[e] = np.dot(v64, sw64) / np.sqrt(ms + eps)
+ p = np.exp(sc - sc.max())
+ p /= p.sum()
+ out = np.zeros_like(prefix)
+ for e, v in enumerate(vs):
+ out += p[e] * v
+ return out
+
+
+class K3Ref:
+ def __init__(self, hf_dir, n_layers):
+ self.S = Shards(hf_dir)
+ cfg = json.load(open(os.path.join(hf_dir, "config.json")))
+ tc = cfg.get("text_config", cfg)
+ self.tc = tc
+ self.D = tc["hidden_size"]
+ self.eps = tc["rms_norm_eps"]
+ self.b1 = tc["activation_situ_beta"]
+ self.b2 = tc["activation_situ_linear_beta"]
+ self.res_bs = tc["attn_res_block_size"]
+ self.first_dense = tc["first_k_dense_replace"]
+ la = tc["linear_attn_config"]
+ self.kda_layers = set(x - 1 for x in la["kda_layers"]) # 0-indexed
+ self.kh, self.hd = la["num_heads"], la["head_dim"]
+ self.lb = la["gate_lower_bound"]
+ self.conv_k = la["short_conv_kernel_size"]
+ self.n_layers = n_layers
+ self.pfx = "language_model." if any(
+ k.startswith("language_model.") for k in list(self.S.wmap)[:5]) else ""
+
+ def t(self, name):
+ return self.S.get(self.pfx + name)
+
+ def kda(self, li, x):
+ """x [T,D] -> [T,D]; sequential recurrence, T small."""
+ p = f"model.layers.{li}.self_attn."
+ T = x.shape[0]
+ H, hd = self.kh, self.hd
+ qp = x @ self.t(p + "q_proj.weight").T
+ kp = x @ self.t(p + "k_proj.weight").T
+ vp = x @ self.t(p + "v_proj.weight").T
+ outs = []
+ for nm, arr in (("q_conv1d", qp), ("k_conv1d", kp), ("v_conv1d", vp)):
+ w = self.t(p + nm + ".weight").reshape(-1, self.conv_k) # [P,K]
+ P = arr.shape[1]
+ pad = np.zeros((self.conv_k - 1, P), dtype=np.float32)
+ xp = np.concatenate([pad, arr], axis=0) # [T+K-1, P]
+ y = np.zeros_like(arr)
+ for j in range(self.conv_k):
+ y += xp[j:j + T] * w[:, j]
+ outs.append(silu(y))
+ q, k, v = outs
+ g_low = (x @ self.t(p + "f_a_proj.weight").T) @ self.t(p + "f_b_proj.weight").T
+ beta = sigmoid(x @ self.t(p + "b_proj.weight").T) # [T,H]
+ dt = self.t(p + "dt_bias") # [H*hd]
+ A = np.exp(self.t(p + "A_log")[:H]) # padded ckpt
+ onw = self.t(p + "o_norm.weight")
+ gfull = x @ self.t(p + "g_proj.weight").T # [T,H*hd]
+ q = q.reshape(T, H, hd)
+ k = k.reshape(T, H, hd)
+ v = v.reshape(T, H, hd)
+ z = g_low.reshape(T, H, hd) + dt.reshape(H, hd)
+ alpha = np.exp(self.lb * sigmoid(A[None, :, None] * z)) # [T,H,hd]
+ qn = q / np.sqrt((q ** 2).sum(-1, keepdims=True) + 1e-6) * (hd ** -0.5)
+ kn = k / np.sqrt((k ** 2).sum(-1, keepdims=True) + 1e-6)
+ on = np.zeros((T, H, hd), dtype=np.float32)
+ for h in range(H):
+ S = np.zeros((hd, hd), dtype=np.float32) # S[k,v]
+ for t in range(T):
+ S = S * alpha[t, h][:, None]
+ kS = kn[t, h] @ S
+ vt = (v[t, h] - kS) * beta[t, h]
+ S = S + np.outer(kn[t, h], vt)
+ on[t, h] = qn[t, h] @ S
+ ms = np.mean(on.astype(np.float64) ** 2, axis=-1, keepdims=True)
+ on = (on * (1.0 / np.sqrt(ms + self.eps))).astype(np.float32) * onw
+ on = on * sigmoid(gfull.reshape(T, H, hd))
+ return on.reshape(T, H * hd) @ self.t(p + "o_proj.weight").T
+
+ def mla(self, li, x):
+ p = f"model.layers.{li}.self_attn."
+ tc = self.tc
+ T = x.shape[0]
+ H = tc["num_attention_heads"]
+ nope, rope, vh = tc["qk_nope_head_dim"], tc["qk_rope_head_dim"], tc["v_head_dim"]
+ kvl = tc["kv_lora_rank"]
+ qh = nope + rope
+ qa = rmsnorm(x @ self.t(p + "q_a_proj.weight").T,
+ self.t(p + "q_a_layernorm.weight"), self.eps)
+ q = (qa @ self.t(p + "q_b_proj.weight").T).reshape(T, H, qh)
+ ckv = x @ self.t(p + "kv_a_proj_with_mqa.weight").T
+ c = rmsnorm(ckv[:, :kvl], self.t(p + "kv_a_layernorm.weight"), self.eps)
+ k_rot = ckv[:, kvl:] # [T,rope] raw (NoPE)
+ kv = (c @ self.t(p + "kv_b_proj.weight").T).reshape(T, H, nope + vh)
+ k_pass, v = kv[..., :nope], kv[..., nope:]
+ scale = qh ** -0.5
+ sc = (np.einsum("thd,shd->hts", q[..., :nope], k_pass) +
+ np.einsum("thd,sd->hts", q[..., nope:], k_rot)) * scale
+ mask = np.tril(np.ones((T, T), dtype=bool))
+ sc = np.where(mask[None], sc, -np.inf)
+ sc = sc - sc.max(-1, keepdims=True)
+ w = np.exp(sc)
+ w /= w.sum(-1, keepdims=True)
+ ctx = np.einsum("hts,shd->thd", w.astype(np.float32), v)
+ g = sigmoid(x @ self.t(p + "g_proj.weight").T)
+ ctx = ctx.reshape(T, H * vh) * g
+ return ctx @ self.t(p + "o_proj.weight").T
+
+ def moe(self, li, x):
+ p = f"model.layers.{li}.block_sparse_moe."
+ tc = self.tc
+ T = x.shape[0]
+ K = tc["num_experts_per_token"]
+ router = self.t(p + "gate.weight")
+ bias = self.t(p + "gate.e_score_correction_bias")
+ lat_dn = self.t(p + "routed_expert_down_proj.weight")
+ lat_up = self.t(p + "routed_expert_up_proj.weight")
+ lat_nw = self.t(p + "routed_expert_norm.weight")
+ sg = self.t(p + "shared_experts.gate_proj.weight")
+ su = self.t(p + "shared_experts.up_proj.weight")
+ sd = self.t(p + "shared_experts.down_proj.weight")
+ out = np.zeros_like(x)
+ experts = {}
+ for t in range(T):
+ s = sigmoid(x[t] @ router.T)
+ sel = np.argsort(-(s + bias), kind="stable")[:K]
+ w = s[sel]
+ w = w / (w.sum() + 1e-20)
+ z = x[t] @ lat_dn.T
+ u = np.zeros(z.shape[0], dtype=np.float32)
+ for e, we in zip(sel, w):
+ if e not in experts:
+ ep = f"{p}experts.{e}."
+ W1 = deq_mx4(self.t(ep + "w1.weight_packed"), self.t(ep + "w1.weight_scale"))
+ W2 = deq_mx4(self.t(ep + "w2.weight_packed"), self.t(ep + "w2.weight_scale"))
+ W3 = deq_mx4(self.t(ep + "w3.weight_packed"), self.t(ep + "w3.weight_scale"))
+ experts[e] = (W1, W2, W3)
+ W1, W2, W3 = experts[e]
+ act = situ(z @ W1.T, z @ W3.T, self.b1, self.b2)
+ u += we * (act @ W2.T)
+ u = rmsnorm(u, lat_nw, self.eps)
+ y = u @ lat_up.T
+ act = situ(x[t] @ sg.T, x[t] @ su.T, self.b1, self.b2)
+ out[t] = y + act @ sd.T
+ return out
+
+ def dense(self, li, x):
+ p = f"model.layers.{li}.mlp."
+ act = situ(x @ self.t(p + "gate_proj.weight").T,
+ x @ self.t(p + "up_proj.weight").T, self.b1, self.b2)
+ return act @ self.t(p + "down_proj.weight").T
+
+ def run(self, x0, out_path):
+ """x0 [T,D]; per-token trace rows in the SAME order as the C engine
+ (token-major: the C engine steps one token at a time through all
+ layers). Recurrent/cache state must therefore persist across tokens —
+ we instead run all T at once per layer and emit token-major at the
+ end, which is equivalent because layer l for token t only depends on
+ layers <= l of tokens <= t."""
+ T = x0.shape[0]
+ trace = np.zeros((T, self.n_layers, self.D), dtype=np.float32)
+ hidden = x0.copy()
+ bres = []
+ prefix = None
+ for li in range(self.n_layers):
+ lp = f"model.layers.{li}."
+ in_ln = self.t(lp + "input_layernorm.weight")
+ post_ln = self.t(lp + "post_attention_layernorm.weight")
+ asw = self.t(lp + "self_attention_res_norm.weight") * \
+ self.t(lp + "self_attention_res_proj.weight").reshape(-1)
+ msw = self.t(lp + "mlp_res_norm.weight") * \
+ self.t(lp + "mlp_res_proj.weight").reshape(-1)
+ prefix = hidden.copy()
+ have_prefix = True
+ if bres:
+ hidden = np.stack([res_mix(prefix[t], [b[t] for b in bres], asw, self.eps)
+ for t in range(T)])
+ if li % self.res_bs == 0:
+ bres.append(prefix.copy())
+ have_prefix = False
+ nrm = rmsnorm(hidden, in_ln, self.eps)
+ att = self.kda(li, nrm) if li in self.kda_layers else self.mla(li, nrm)
+ prefix = (prefix + att) if have_prefix else att
+ mix = np.stack([res_mix(prefix[t], [b[t] for b in bres], msw, self.eps)
+ for t in range(T)])
+ nrm = rmsnorm(mix, post_ln, self.eps)
+ mlp = self.moe(li, nrm) if li >= self.first_dense else self.dense(li, nrm)
+ prefix = prefix + mlp
+ hidden = prefix.copy()
+ trace[:, li] = hidden
+ print(f"[ref] layer {li} done ({'KDA' if li in self.kda_layers else 'MLA'},"
+ f" {'moe' if li >= self.first_dense else 'dense'})", flush=True)
+ trace.tofile(out_path)
+ print(f"[ref] wrote {out_path}: {T} tokens x {self.n_layers} layers x {self.D}")
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ sub = ap.add_subparsers(dest="cmd", required=True)
+ g = sub.add_parser("gen")
+ g.add_argument("out")
+ g.add_argument("--tokens", type=int, default=6)
+ g.add_argument("--dim", type=int, default=7168)
+ g.add_argument("--seed", type=int, default=7)
+ g.add_argument("--scale", type=float, default=0.02)
+ r = sub.add_parser("run")
+ r.add_argument("hf_dir")
+ r.add_argument("--x0", required=True)
+ r.add_argument("--layers", type=int, default=4)
+ r.add_argument("--out", required=True)
+ c = sub.add_parser("cmp")
+ c.add_argument("a")
+ c.add_argument("b")
+ c.add_argument("--dim", type=int, default=7168)
+ c.add_argument("--layers", type=int, default=4)
+ a = ap.parse_args()
+ if a.cmd == "gen":
+ rng = np.random.default_rng(a.seed)
+ x = (rng.standard_normal((a.tokens, a.dim)) * a.scale).astype(np.float32)
+ x.tofile(a.out)
+ print(f"wrote {a.out}: {a.tokens}x{a.dim}")
+ elif a.cmd == "run":
+ x0 = np.fromfile(a.x0, dtype=np.float32).reshape(-1, 7168)
+ K3Ref(a.hf_dir, a.layers).run(x0, a.out)
+ else:
+ A = np.fromfile(a.a, dtype=np.float32).reshape(-1, a.layers, a.dim)
+ B = np.fromfile(a.b, dtype=np.float32).reshape(-1, a.layers, a.dim)
+ assert A.shape == B.shape, (A.shape, B.shape)
+ worst = 0.0
+ for li in range(a.layers):
+ x, y = A[:, li], B[:, li]
+ rel = np.linalg.norm(x - y) / (np.linalg.norm(y) + 1e-30)
+ worst = max(worst, rel)
+ print(f"layer {li}: rel_l2 {rel:.3e} (|c|={np.linalg.norm(x):.4g} |ref|={np.linalg.norm(y):.4g})")
+ print("VERDICT:", "OK" if worst < 3e-4 else "MISMATCH", f"(worst {worst:.3e})")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/c/tools/k3_repack.py b/c/tools/k3_repack.py
new file mode 100644
index 00000000..00a91803
--- /dev/null
+++ b/c/tools/k3_repack.py
@@ -0,0 +1,279 @@
+#!/usr/bin/env python3
+"""Repack a Kimi K3 HF snapshot into colibri's ideal ingestion layout.
+
+Designed so the HDD->NVMe copy IS the conversion (one read of the source):
+
+ - routed experts (MXFP4, QAT): byte-identical passthrough, written in
+ expert-id order, the six tensors of an expert back-to-back (the engine
+ reads one expert = one pread)
+ - big BF16 matrices: quantized ON THE WAY with EXACTLY the engine's
+ load-time algorithm (int8 per-row absmax/127 or int4-g64 absmax/7,
+ round-half-even, same f32 op order), stored as U8 + <name>.qs f32 scale
+ sidecar -> the engine skips its 10-15 min load-time quantization and
+ cold-starts from the container in ~1-2 min
+ - everything small or sensitive (norms, router, conv taps, dt_bias, A_log,
+ o_norm, f_a/f_b, b_proj, res_*, embed_tokens, final norm): passthrough
+ in the source dtype (BF16/F32) — quantizing these saves ~nothing and the
+ engine keeps them f32 in RAM anyway
+ - vision_tower / mm_projector (shards 95-96): dropped
+ - output: model-XXXXX-of-000094.safetensors (same shard split: one layer
+ per shard + shard 94 = embed/head), spec-valid safetensors (no holes),
+ regenerated model.safetensors.index.json, config/tokenizer files copied
+
+Resume-safe: each shard is written to .tmp and renamed; existing outputs are
+skipped. Rerun after an interruption and it continues.
+
+usage: k3_repack.py <src_dir> <dst_dir> [--bits 8] [--mla-bits N] [--head-bits N]
+ [--shards 1,2,94] [--verify-full]
+
+Classification needs the layer type (o_proj/g_proj exist on BOTH KDA and MLA
+layers) — read from config.json linear_attn_config.
+"""
+import argparse
+import json
+import os
+import shutil
+import struct
+import sys
+import time
+
+import numpy as np
+
+CHUNK_ROWS = 4096 # quantize in row blocks (bounds RAM to ~120 MB)
+COPY_BLOCK = 64 << 20 # passthrough copy block
+
+
+def read_header(path):
+ with open(path, "rb") as f:
+ n = struct.unpack("<Q", f.read(8))[0]
+ h = json.loads(f.read(n))
+ h.pop("__metadata__", None)
+ return h, 8 + n
+
+
+def bf16_to_f32(buf):
+ u = np.frombuffer(buf, dtype=np.uint16).astype(np.uint32) << 16
+ return u.view(np.float32)
+
+
+def quant_int8(rows):
+ """rows [n,I] f32 -> (int8 bytes, f32 scales [n]); engine w_load exact."""
+ am = np.abs(rows).max(axis=1)
+ s = (am / np.float32(127.0)).astype(np.float32)
+ s = np.maximum(s, np.float32(1e-20))
+ q = np.clip(np.rint(rows * (np.float32(1.0) / s)[:, None]), -127, 127).astype(np.int8)
+ return q.tobytes(), s
+
+
+def quant_int4_g64(rows):
+ """rows [n,I] f32, I%64==0 -> (packed bytes low-nibble-even biased+8,
+ f32 scales [n, I/64]); engine w_load / matmul_i4_grouped exact."""
+ n, I = rows.shape
+ g = rows.reshape(n, I // 64, 64)
+ am = np.abs(g).max(axis=2)
+ s = (am / np.float32(7.0)).astype(np.float32)
+ s = np.maximum(s, np.float32(1e-20))
+ q = np.clip(np.rint(g * (np.float32(1.0) / s)[:, :, None]), -8, 7).astype(np.int32) + 8
+ q = q.reshape(n, I)
+ packed = (q[:, 0::2] | (q[:, 1::2] << 4)).astype(np.uint8)
+ return packed.tobytes(), s
+
+
+class Classifier:
+ def __init__(self, src):
+ cfg = json.load(open(os.path.join(src, "config.json")))
+ tc = cfg.get("text_config", cfg)
+ self.kda = set(x - 1 for x in tc["linear_attn_config"]["kda_layers"])
+
+ def __call__(self, name):
+ """-> ('skip'|'expert'|'pass'|'q8class', class) where class in
+ {'bits','mla','head'} for quantized tensors."""
+ n = name
+ if n.startswith("vision_tower") or n.startswith("mm_projector"):
+ return "skip", None
+ if n.startswith("language_model."):
+ n = n[len("language_model."):]
+ if ".block_sparse_moe.experts." in n:
+ return "expert", None
+ if n == "lm_head.weight":
+ return "quant", "head"
+ parts = n.split(".")
+ li = int(parts[2]) if len(parts) > 3 and parts[1] == "layers" else -1
+ if ".self_attn." in n and n.endswith(".weight"):
+ leaf = parts[-2]
+ if leaf in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj"):
+ return "quant", "mla"
+ if leaf in ("q_proj", "k_proj", "v_proj"):
+ return "quant", "bits"
+ if leaf in ("o_proj", "g_proj"):
+ return ("quant", "bits") if li in self.kda else ("quant", "mla")
+ return "pass", None # conv taps, norms, f_a/f_b, b_proj
+ for leaf in ("routed_expert_down_proj", "routed_expert_up_proj",
+ "shared_experts.gate_proj", "shared_experts.up_proj",
+ "shared_experts.down_proj",
+ "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj"):
+ if n.endswith(leaf + ".weight"):
+ return "quant", "bits"
+ return "pass", None # router, biases, norms, embed, res_*
+
+
+def expert_sort_key(name):
+ # ...experts.<E>.<w1|w2|w3>.weight_<packed|scale>
+ p = name.split(".experts.")[1].split(".")
+ e = int(p[0])
+ mat = {"w1": 0, "w2": 1, "w3": 2}[p[1]]
+ kind = 0 if p[2].endswith("packed") else 1
+ # engine slot order: w1p w1s w2p w2s w3p w3s
+ return (e, mat, kind)
+
+
+def repack_shard(src_path, dst_path, cls, bits_map, verify_full):
+ hdr, data_start = read_header(src_path)
+ items = sorted(hdr.items(), key=lambda kv: kv[1]["data_offsets"][0])
+ non_expert, experts = [], []
+ for name, v in items:
+ kind, qc = cls(name)
+ if kind == "skip":
+ continue
+ (experts if kind == "expert" else non_expert).append((name, v, kind, qc))
+ experts.sort(key=lambda t: expert_sort_key(t[0]))
+ plan = non_expert + experts
+
+ # pass 1: output sizes -> header with final offsets
+ out_entries = [] # (name, dtype, shape, nbytes) (+ .qs after its tensor)
+ for name, v, kind, qc in plan:
+ if kind == "quant":
+ O, I = v["shape"]
+ bits = bits_map[qc]
+ if bits <= 4 and I % 64 == 0:
+ out_entries.append((name, "U8", [O, I // 2], O * (I // 2)))
+ out_entries.append((name + ".qs", "F32", [O, I // 64], O * (I // 64) * 4))
+ else:
+ out_entries.append((name, "U8", [O, I], O * I))
+ out_entries.append((name + ".qs", "F32", [O], O * 4))
+ else:
+ out_entries.append((name, v["dtype"], v["shape"],
+ v["data_offsets"][1] - v["data_offsets"][0]))
+ oh, off = {}, 0
+ for name, dt, shape, nb in out_entries:
+ oh[name] = {"dtype": dt, "shape": shape, "data_offsets": [off, off + nb]}
+ off += nb
+ hjson = json.dumps(oh, separators=(",", ":")).encode()
+ pad = (-(8 + len(hjson))) % 8
+ hjson += b" " * pad
+
+ tmp = dst_path + ".tmp"
+ written = 0
+ with open(src_path, "rb") as fin, open(tmp, "wb") as fout:
+ fout.write(struct.pack("<Q", len(hjson)) + hjson)
+ for name, v, kind, qc in plan:
+ a0, b0 = v["data_offsets"]
+ fin.seek(data_start + a0)
+ if kind == "quant":
+ O, I = v["shape"]
+ bits = bits_map[qc]
+ use4 = bits <= 4 and I % 64 == 0
+ qparts, sparts = [], []
+ for r0 in range(0, O, CHUNK_ROWS):
+ n = min(CHUNK_ROWS, O - r0)
+ raw = fin.read(n * I * 2)
+ rows = bf16_to_f32(raw).reshape(n, I)
+ qb, s = quant_int4_g64(rows) if use4 else quant_int8(rows)
+ qparts.append(qb)
+ sparts.append(s.astype(np.float32).tobytes())
+ for qb in qparts:
+ fout.write(qb)
+ for sb in sparts:
+ fout.write(sb)
+ written += sum(len(x) for x in qparts) + sum(len(x) for x in sparts)
+ else:
+ left = b0 - a0
+ while left:
+ blk = fin.read(min(COPY_BLOCK, left))
+ if not blk:
+ raise IOError(f"short read in {name}")
+ fout.write(blk)
+ left -= len(blk)
+ written += b0 - a0
+ # sanity: bytes written == declared
+ want = off
+ got = os.path.getsize(tmp) - 8 - len(hjson)
+ if got != want:
+ raise IOError(f"{dst_path}: wrote {got} data bytes, header declares {want}")
+ os.replace(tmp, dst_path)
+
+ if verify_full and experts:
+ vh, vstart = read_header(dst_path)
+ with open(src_path, "rb") as fs, open(dst_path, "rb") as fd:
+ for name, v, kind, qc in experts:
+ sa, sb = v["data_offsets"]
+ da, db = vh[name]["data_offsets"]
+ assert sb - sa == db - da, name
+ fs.seek(data_start + sa)
+ fd.seek(vstart + da)
+ left = sb - sa
+ while left:
+ n = min(COPY_BLOCK, left)
+ if fs.read(n) != fd.read(n):
+ raise IOError(f"VERIFY FAILED: {name}")
+ left -= n
+ return oh, written
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("src")
+ ap.add_argument("dst")
+ ap.add_argument("--bits", type=int, default=8)
+ ap.add_argument("--mla-bits", type=int, default=None)
+ ap.add_argument("--head-bits", type=int, default=None)
+ ap.add_argument("--shards", help="comma list of shard numbers (default 1..94)")
+ ap.add_argument("--verify-full", action="store_true",
+ help="re-read every expert byte and compare with the source")
+ a = ap.parse_args()
+ bits_map = {"bits": a.bits,
+ "mla": a.mla_bits if a.mla_bits is not None else max(a.bits, 8),
+ "head": a.head_bits if a.head_bits is not None else max(a.bits, 8)}
+ os.makedirs(a.dst, exist_ok=True)
+ cls = Classifier(a.src)
+ shard_nums = ([int(x) for x in a.shards.split(",")] if a.shards
+ else list(range(1, 95)))
+ index = {}
+ t0 = time.time()
+ total = 0
+ done_any = False
+ for i, num in enumerate(shard_nums):
+ sp = os.path.join(a.src, f"model-{num:05d}-of-000096.safetensors")
+ dname = f"model-{num:05d}-of-000094.safetensors"
+ dp = os.path.join(a.dst, dname)
+ if not os.path.exists(sp):
+ print(f"[{num:2d}] SOURCE MISSING — skipped (rerun when downloaded)", flush=True)
+ continue
+ if os.path.exists(dp):
+ oh, _ = read_header(dp)
+ for name in oh:
+ index[name] = dname
+ print(f"[{num:2d}] exists — skipped (resume)", flush=True)
+ continue
+ oh, written = repack_shard(sp, dp, cls, bits_map, a.verify_full)
+ for name in oh:
+ index[name] = dname
+ total += written
+ done_any = True
+ el = time.time() - t0
+ print(f"[{num:2d}] {os.path.getsize(dp)/1e9:6.2f} GB out "
+ f"({total/1e9:7.1f} GB total, {total/1e9/max(el,1e-9):5.2f} GB/s, {el:6.0f}s)",
+ flush=True)
+ if done_any or index:
+ with open(os.path.join(a.dst, "model.safetensors.index.json"), "w") as f:
+ json.dump({"metadata": {"total_size": total}, "weight_map": index}, f)
+ for extra in ("config.json", "generation_config.json", "tokenizer.json",
+ "tokenizer_config.json"):
+ s = os.path.join(a.src, extra)
+ if os.path.exists(s):
+ shutil.copy2(s, os.path.join(a.dst, extra))
+ print(f"done: {total/1e9:.1f} GB written in {time.time()-t0:.0f}s -> {a.dst}")
+
+
+if __name__ == "__main__":
+ main()
diff --git a/c/tools/k3_tokenizer.py b/c/tools/k3_tokenizer.py
new file mode 100644
index 00000000..8a3da0e8
--- /dev/null
+++ b/c/tools/k3_tokenizer.py
@@ -0,0 +1,225 @@
+#!/usr/bin/env python3
+"""Synthesize tokenizer.json for Kimi K3 from the repo's tiktoken.model.
+
+The K3 HF repo ships only a raw tiktoken BPE (base64 token -> rank) plus
+tokenization_kimi.py; the colibri-family engines load HuggingFace
+tokenizer.json (tok.h). This tool writes one:
+
+ - vocab: every tiktoken byte sequence mapped through the GPT-2 byte-level
+ alphabet (the same map tok.h rebuilds), id = rank
+ - merges: EMPTY on purpose. Rank-recovered merge lists cannot be made
+ exactly faithful (verified: "newlines" merges new+l+ines instead of
+ new+lines under the standard max-rank-minimizing recovery). tok.h
+ detects the empty list and switches to rank-BPE — merge the adjacent
+ pair whose concatenation has the lowest vocab id — which IS tiktoken's
+ algorithm, so encoding is exact by construction.
+ - added_tokens: ids num_base..num_base+255 from tokenizer_config.json's
+ added_tokens_decoder, gaps filled with <|reserved_token_N|>
+ - pre_tokenizer: the Kimi Split regex (contains \\p{Han} and \\p{Lu} — the
+ markers tok.h sniffs to select the kimi family), plus ByteLevel
+
+usage: k3_tokenizer.py <hf_dir> [-o out.json] [--check "text" ...]
+ --check re-encodes text with tiktoken (if installed) and with a pure-
+ python replay of the emitted tokenizer.json; both must agree.
+"""
+import argparse
+import base64
+import json
+import os
+import sys
+
+PAT = "|".join([
+ r"""[\p{Han}]+""",
+ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
+ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",
+ r"""\p{N}{1,3}""",
+ r""" ?[^\s\p{L}\p{N}]+[\r\n]*""",
+ r"""\s*[\r\n]+""",
+ r"""\s+(?!\S)""",
+ r"""\s+""",
+])
+
+NUM_RESERVED = 256
+
+
+def bytes_to_unicode():
+ bs = list(range(33, 127)) + list(range(161, 173)) + list(range(174, 256))
+ cs = bs[:]
+ n = 0
+ for b in range(256):
+ if b not in bs:
+ bs.append(b)
+ cs.append(256 + n)
+ n += 1
+ return dict(zip(bs, [chr(c) for c in cs]))
+
+
+def main():
+ ap = argparse.ArgumentParser()
+ ap.add_argument("hf_dir")
+ ap.add_argument("-o", "--out")
+ ap.add_argument("--check", nargs="*", default=None)
+ ap.add_argument("--ctest", metavar="BINARY",
+ help="cross-check the C tokenizer (tests/test_tok_kimi) against tiktoken")
+ a = ap.parse_args()
+ out = a.out or os.path.join(a.hf_dir, "tokenizer.json")
+
+ ranks = {}
+ with open(os.path.join(a.hf_dir, "tiktoken.model"), "rb") as f:
+ for line in f:
+ if not line.strip():
+ continue
+ tok, rank = line.split()
+ ranks[base64.b64decode(tok)] = int(rank)
+ num_base = len(ranks)
+ print(f"tiktoken vocab: {num_base} tokens")
+
+ b2u = bytes_to_unicode()
+
+ def bl(bs): # bytes -> byte-level string
+ return "".join(b2u[b] for b in bs)
+
+ vocab = {bl(tok): rank for tok, rank in ranks.items()}
+ merges = [] # empty on purpose -> tok.h rank-BPE (see module docstring)
+
+ tc = {}
+ tcp = os.path.join(a.hf_dir, "tokenizer_config.json")
+ if os.path.exists(tcp):
+ tc = json.load(open(tcp))
+ named = {int(k): v["content"] for k, v in tc.get("added_tokens_decoder", {}).items()}
+ special = {int(k): bool(v.get("special", False))
+ for k, v in tc.get("added_tokens_decoder", {}).items()}
+ added = []
+ for i in range(num_base, num_base + NUM_RESERVED):
+ content = named.get(i, f"<|reserved_token_{i}|>")
+ added.append({"id": i, "content": content, "single_word": False,
+ "lstrip": False, "rstrip": False, "normalized": False,
+ "special": special.get(i, True)})
+
+ tj = {
+ "version": "1.0",
+ "truncation": None,
+ "padding": None,
+ "added_tokens": added,
+ "normalizer": None,
+ "pre_tokenizer": {
+ "type": "Sequence",
+ "pretokenizers": [
+ {"type": "Split", "pattern": {"Regex": PAT},
+ "behavior": "Isolated", "invert": False},
+ {"type": "ByteLevel", "add_prefix_space": False,
+ "trim_offsets": True, "use_regex": False},
+ ],
+ },
+ "post_processor": None,
+ "decoder": {"type": "ByteLevel", "add_prefix_space": True,
+ "trim_offsets": True, "use_regex": True},
+ "model": {
+ "type": "BPE", "dropout": None, "unk_token": None,
+ "continuing_subword_prefix": None, "end_of_word_suffix": None,
+ "fuse_unk": False, "byte_fallback": False, "ignore_merges": True,
+ "vocab": vocab, "merges": merges,
+ },
+ }
+ with open(out, "w", encoding="utf-8") as f:
+ json.dump(tj, f, ensure_ascii=False)
+ print(f"wrote {out} ({os.path.getsize(out)/1e6:.1f} MB)")
+
+ corpus = [
+ "Hello, world! The quick brown fox jumps over 12345 lazy dogs.",
+ "def main():\n return {'a': 1}\n",
+ "Kimi K3 是一个开源模型。它支持中文和English mixed text!",
+ " indented\n\n\nnewlines\t\ttabs ",
+ "I'll say we've DONE it. HTTPResponse XMLHttpRequest",
+ "今天天气怎么样?我们去公园散步吧。",
+ "日本語のテキストです。カタカナとひらがな、漢字も。",
+ "한국어 텍스트도 있습니다.",
+ "naïve café résumé ÜBER Straße mañana",
+ "don't CAN'T it's I'M we'RE you'll",
+ "1 12 123 1234 12345 3.14159 -42",
+ "!!! ??? ... --- /// ***bold*** `code`",
+ " a\n\n\nb\t\t c \n d ",
+ "🚀 emoji test 🎉 ½ ² € §",
+ "iPhone macOS HTTPResponse XMLHttpRequest camelCase snake_case",
+ "々 〇 一二三 中中中English中文",
+ "\r\nwindows\r\nline\r\nendings\r\n",
+ "mixed 中文with no space英文boundaries测试",
+ ]
+ if a.ctest:
+ import struct as _st
+ import subprocess
+ try:
+ import tiktoken
+ except ImportError:
+ print("--ctest: tiktoken not installed", file=sys.stderr)
+ return
+ enc = tiktoken.Encoding(name="k3", pat_str=PAT,
+ mergeable_ranks=ranks, special_tokens={})
+ cases = "/tmp/k3_tok_cases.bin"
+ with open(cases, "wb") as f:
+ f.write(_st.pack("<I", len(corpus)))
+ for t in corpus:
+ b = t.encode("utf-8")
+ f.write(_st.pack("<I", len(b)) + b)
+ r = subprocess.run([a.ctest, out, cases], capture_output=True, text=True)
+ print(r.stderr.strip(), file=sys.stderr)
+ got = [([int(x) for x in ln.split()] if ln else [])
+ for ln in r.stdout.splitlines()]
+ ok = True
+ for t, g in zip(corpus, got):
+ want = enc.encode(t)
+ if want != g:
+ ok = False
+ print(f"C MISMATCH on {t!r}\n tiktoken: {want}\n tok.h : {g}")
+ print("ctest:", "OK — tok.h matches tiktoken on all cases" if ok else "FAILED")
+ return
+
+ if a.check is not None:
+ texts = a.check or corpus
+ try:
+ import tiktoken
+ except ImportError:
+ print("--check: tiktoken not installed, skipping", file=sys.stderr)
+ return
+ enc = tiktoken.Encoding(name="k3", pat_str=PAT,
+ mergeable_ranks=ranks, special_tokens={})
+ def bpe_py(piece_bytes):
+ """rank-BPE replay: what tok.h does with an empty merges list"""
+ s = bl(piece_bytes)
+ if s in vocab:
+ return [vocab[s]]
+ sym = [c for c in s]
+ while True:
+ best, bp = None, -1
+ for i in range(len(sym) - 1):
+ r = vocab.get(sym[i] + sym[i + 1])
+ if r is not None and (best is None or r < best):
+ best, bp = r, i
+ if bp < 0:
+ break
+ sym[bp:bp + 2] = [sym[bp] + sym[bp + 1]]
+ return [vocab[t] for t in sym if t in vocab]
+
+ import regex
+ # Python's regex module wants V1 set-difference syntax for what
+ # fancy-regex writes as [...&&[^\p{Han}]]
+ py_pat = PAT.replace(
+ r"[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]",
+ r"[[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]--[\p{Han}]]").replace(
+ r"[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]",
+ r"[[\p{Ll}\p{Lm}\p{Lo}\p{M}]--[\p{Han}]]")
+ cre = regex.compile(py_pat, flags=regex.V1)
+ ok = True
+ for t in texts:
+ want = enc.encode(t)
+ got = []
+ for mt in cre.finditer(t):
+ got += bpe_py(mt.group().encode("utf-8"))
+ if want != got:
+ ok = False
+ print(f"MISMATCH on {t!r}\n tiktoken: {want}\n replay : {got}")
+ print("check:", "OK — merges replay matches tiktoken" if ok else "FAILED")
+
+
+if __name__ == "__main__":
+ main()