Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tools: K3 tokenizer generator, numpy layer reference, container repack
Steve Markgraf8a2eccbbb107
diff --git a/c/tools/k3_ref.py b/c/tools/k3_ref.pynew file mode 100644index 00000000..8dd4f10b--- /dev/null+++ b/c/tools/k3_ref.py@@ -0,0 +1,338 @@+#!/usr/bin/env python3+"""Numpy reference for the first N layers of Kimi K3 — validates c/kimi_k3.c.++Reads tensors straight from the HF shards (BF16 -> f32, MXFP4 dequant), runs T+injected input rows [T, hidden] (the engine's K3_X0 hook, bypassing the+embedding, which lives in one of the last shards) through layers 0..N-1+exactly as modeling_kimi_linear.py does — KDA recurrence, gated NoPE MLA,+AttnRes, LatentMoE with SiTU-GLU — and writes the same trace layout the C+engine emits under K3_TRACE: per token, per layer, hidden[hidden] f32.++ gen : k3_ref.py gen <x0.bin> --tokens 6 --dim 7168 [--seed 7]+ run : k3_ref.py run <hf_dir> --x0 x0.bin --layers 4 --out ref.bin+ cmp : k3_ref.py cmp <c_trace.bin> <ref.bin> --dim 7168 --layers 4++Pure numpy, no torch. Everything f32 (f64 only where the C engine also+accumulates in double: AttnRes scores, RMS means).+"""+import argparse+import json+import os+import struct+import sys++import numpy as np++E2M1 = np.array([0, .5, 1, 1.5, 2, 3, 4, 6, -0., -.5, -1, -1.5, -2, -3, -4, -6],+ dtype=np.float32)+++class Shards:+ def __init__(self, d):+ self.dir = d+ idx = json.load(open(os.path.join(d, "model.safetensors.index.json")))+ self.wmap = idx["weight_map"]+ self.hdrs = {}++ def _hdr(self, fn):+ if fn not in self.hdrs:+ with open(os.path.join(self.dir, fn), "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ self.hdrs[fn] = (json.loads(f.read(n)), 8 + n)+ return self.hdrs[fn]++ def get(self, name):+ fn = self.wmap[name]+ h, base = self._hdr(fn)+ v = h[name]+ o0, o1 = v["data_offsets"]+ with open(os.path.join(self.dir, fn), "rb") as f:+ f.seek(base + o0)+ raw = f.read(o1 - o0)+ if v["dtype"] == "BF16":+ u = np.frombuffer(raw, dtype=np.uint16).astype(np.uint32) << 16+ a = u.view(np.float32)+ elif v["dtype"] == "F32":+ a = np.frombuffer(raw, dtype=np.float32)+ elif v["dtype"] == "U8":+ a = np.frombuffer(raw, dtype=np.uint8)+ else:+ raise ValueError(v["dtype"])+ return a.reshape(v["shape"]).copy()+++def sigmoid(x):+ return 1.0 / (1.0 + np.exp(-x))+++def silu(x):+ return x * sigmoid(x)+++def rmsnorm(x, w, eps):+ ms = np.mean(x.astype(np.float64) ** 2, axis=-1, keepdims=True)+ return (x * (1.0 / np.sqrt(ms + eps))).astype(np.float32) * w+++def situ(g, u, b1, b2):+ return (b1 * np.tanh(g / b1) * sigmoid(g)) * (b2 * np.tanh(u / b2))+++def deq_mx4(packed, scale):+ O, half = packed.shape+ nib = np.empty((O, half * 2), dtype=np.uint8)+ nib[:, 0::2] = packed & 0xF+ nib[:, 1::2] = packed >> 4+ w = E2M1[nib]+ sc = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127).astype(np.float32)+ I = half * 2+ return (w.reshape(O, I // 32, 32) * sc[:, :, None]).reshape(O, I)+++def res_mix(prefix, bres, sw, eps):+ """entries = bres rows + prefix (last); softmax over RMS-normed dots; mix RAW."""+ vs = list(bres) + [prefix]+ sc = np.empty(len(vs), dtype=np.float32)+ sw64 = sw.astype(np.float64)+ for e, v in enumerate(vs):+ v64 = v.astype(np.float64)+ ms = np.mean(v64 ** 2)+ sc[e] = np.dot(v64, sw64) / np.sqrt(ms + eps)+ p = np.exp(sc - sc.max())+ p /= p.sum()+ out = np.zeros_like(prefix)+ for e, v in enumerate(vs):+ out += p[e] * v+ return out+++class K3Ref:+ def __init__(self, hf_dir, n_layers):+ self.S = Shards(hf_dir)+ cfg = json.load(open(os.path.join(hf_dir, "config.json")))+ tc = cfg.get("text_config", cfg)+ self.tc = tc+ self.D = tc["hidden_size"]+ self.eps = tc["rms_norm_eps"]+ self.b1 = tc["activation_situ_beta"]+ self.b2 = tc["activation_situ_linear_beta"]+ self.res_bs = tc["attn_res_block_size"]+ self.first_dense = tc["first_k_dense_replace"]+ la = tc["linear_attn_config"]+ self.kda_layers = set(x - 1 for x in la["kda_layers"]) # 0-indexed+ self.kh, self.hd = la["num_heads"], la["head_dim"]+ self.lb = la["gate_lower_bound"]+ self.conv_k = la["short_conv_kernel_size"]+ self.n_layers = n_layers+ self.pfx = "language_model." if any(+ k.startswith("language_model.") for k in list(self.S.wmap)[:5]) else ""++ def t(self, name):+ return self.S.get(self.pfx + name)++ def kda(self, li, x):+ """x [T,D] -> [T,D]; sequential recurrence, T small."""+ p = f"model.layers.{li}.self_attn."+ T = x.shape[0]+ H, hd = self.kh, self.hd+ qp = x @ self.t(p + "q_proj.weight").T+ kp = x @ self.t(p + "k_proj.weight").T+ vp = x @ self.t(p + "v_proj.weight").T+ outs = []+ for nm, arr in (("q_conv1d", qp), ("k_conv1d", kp), ("v_conv1d", vp)):+ w = self.t(p + nm + ".weight").reshape(-1, self.conv_k) # [P,K]+ P = arr.shape[1]+ pad = np.zeros((self.conv_k - 1, P), dtype=np.float32)+ xp = np.concatenate([pad, arr], axis=0) # [T+K-1, P]+ y = np.zeros_like(arr)+ for j in range(self.conv_k):+ y += xp[j:j + T] * w[:, j]+ outs.append(silu(y))+ q, k, v = outs+ g_low = (x @ self.t(p + "f_a_proj.weight").T) @ self.t(p + "f_b_proj.weight").T+ beta = sigmoid(x @ self.t(p + "b_proj.weight").T) # [T,H]+ dt = self.t(p + "dt_bias") # [H*hd]+ A = np.exp(self.t(p + "A_log")[:H]) # padded ckpt+ onw = self.t(p + "o_norm.weight")+ gfull = x @ self.t(p + "g_proj.weight").T # [T,H*hd]+ q = q.reshape(T, H, hd)+ k = k.reshape(T, H, hd)+ v = v.reshape(T, H, hd)+ z = g_low.reshape(T, H, hd) + dt.reshape(H, hd)+ alpha = np.exp(self.lb * sigmoid(A[None, :, None] * z)) # [T,H,hd]+ qn = q / np.sqrt((q ** 2).sum(-1, keepdims=True) + 1e-6) * (hd ** -0.5)+ kn = k / np.sqrt((k ** 2).sum(-1, keepdims=True) + 1e-6)+ on = np.zeros((T, H, hd), dtype=np.float32)+ for h in range(H):+ S = np.zeros((hd, hd), dtype=np.float32) # S[k,v]+ for t in range(T):+ S = S * alpha[t, h][:, None]+ kS = kn[t, h] @ S+ vt = (v[t, h] - kS) * beta[t, h]+ S = S + np.outer(kn[t, h], vt)+ on[t, h] = qn[t, h] @ S+ ms = np.mean(on.astype(np.float64) ** 2, axis=-1, keepdims=True)+ on = (on * (1.0 / np.sqrt(ms + self.eps))).astype(np.float32) * onw+ on = on * sigmoid(gfull.reshape(T, H, hd))+ return on.reshape(T, H * hd) @ self.t(p + "o_proj.weight").T++ def mla(self, li, x):+ p = f"model.layers.{li}.self_attn."+ tc = self.tc+ T = x.shape[0]+ H = tc["num_attention_heads"]+ nope, rope, vh = tc["qk_nope_head_dim"], tc["qk_rope_head_dim"], tc["v_head_dim"]+ kvl = tc["kv_lora_rank"]+ qh = nope + rope+ qa = rmsnorm(x @ self.t(p + "q_a_proj.weight").T,+ self.t(p + "q_a_layernorm.weight"), self.eps)+ q = (qa @ self.t(p + "q_b_proj.weight").T).reshape(T, H, qh)+ ckv = x @ self.t(p + "kv_a_proj_with_mqa.weight").T+ c = rmsnorm(ckv[:, :kvl], self.t(p + "kv_a_layernorm.weight"), self.eps)+ k_rot = ckv[:, kvl:] # [T,rope] raw (NoPE)+ kv = (c @ self.t(p + "kv_b_proj.weight").T).reshape(T, H, nope + vh)+ k_pass, v = kv[..., :nope], kv[..., nope:]+ scale = qh ** -0.5+ sc = (np.einsum("thd,shd->hts", q[..., :nope], k_pass) ++ np.einsum("thd,sd->hts", q[..., nope:], k_rot)) * scale+ mask = np.tril(np.ones((T, T), dtype=bool))+ sc = np.where(mask[None], sc, -np.inf)+ sc = sc - sc.max(-1, keepdims=True)+ w = np.exp(sc)+ w /= w.sum(-1, keepdims=True)+ ctx = np.einsum("hts,shd->thd", w.astype(np.float32), v)+ g = sigmoid(x @ self.t(p + "g_proj.weight").T)+ ctx = ctx.reshape(T, H * vh) * g+ return ctx @ self.t(p + "o_proj.weight").T++ def moe(self, li, x):+ p = f"model.layers.{li}.block_sparse_moe."+ tc = self.tc+ T = x.shape[0]+ K = tc["num_experts_per_token"]+ router = self.t(p + "gate.weight")+ bias = self.t(p + "gate.e_score_correction_bias")+ lat_dn = self.t(p + "routed_expert_down_proj.weight")+ lat_up = self.t(p + "routed_expert_up_proj.weight")+ lat_nw = self.t(p + "routed_expert_norm.weight")+ sg = self.t(p + "shared_experts.gate_proj.weight")+ su = self.t(p + "shared_experts.up_proj.weight")+ sd = self.t(p + "shared_experts.down_proj.weight")+ out = np.zeros_like(x)+ experts = {}+ for t in range(T):+ s = sigmoid(x[t] @ router.T)+ sel = np.argsort(-(s + bias), kind="stable")[:K]+ w = s[sel]+ w = w / (w.sum() + 1e-20)+ z = x[t] @ lat_dn.T+ u = np.zeros(z.shape[0], dtype=np.float32)+ for e, we in zip(sel, w):+ if e not in experts:+ ep = f"{p}experts.{e}."+ W1 = deq_mx4(self.t(ep + "w1.weight_packed"), self.t(ep + "w1.weight_scale"))+ W2 = deq_mx4(self.t(ep + "w2.weight_packed"), self.t(ep + "w2.weight_scale"))+ W3 = deq_mx4(self.t(ep + "w3.weight_packed"), self.t(ep + "w3.weight_scale"))+ experts[e] = (W1, W2, W3)+ W1, W2, W3 = experts[e]+ act = situ(z @ W1.T, z @ W3.T, self.b1, self.b2)+ u += we * (act @ W2.T)+ u = rmsnorm(u, lat_nw, self.eps)+ y = u @ lat_up.T+ act = situ(x[t] @ sg.T, x[t] @ su.T, self.b1, self.b2)+ out[t] = y + act @ sd.T+ return out++ def dense(self, li, x):+ p = f"model.layers.{li}.mlp."+ act = situ(x @ self.t(p + "gate_proj.weight").T,+ x @ self.t(p + "up_proj.weight").T, self.b1, self.b2)+ return act @ self.t(p + "down_proj.weight").T++ def run(self, x0, out_path):+ """x0 [T,D]; per-token trace rows in the SAME order as the C engine+ (token-major: the C engine steps one token at a time through all+ layers). Recurrent/cache state must therefore persist across tokens —+ we instead run all T at once per layer and emit token-major at the+ end, which is equivalent because layer l for token t only depends on+ layers <= l of tokens <= t."""+ T = x0.shape[0]+ trace = np.zeros((T, self.n_layers, self.D), dtype=np.float32)+ hidden = x0.copy()+ bres = []+ prefix = None+ for li in range(self.n_layers):+ lp = f"model.layers.{li}."+ in_ln = self.t(lp + "input_layernorm.weight")+ post_ln = self.t(lp + "post_attention_layernorm.weight")+ asw = self.t(lp + "self_attention_res_norm.weight") * \+ self.t(lp + "self_attention_res_proj.weight").reshape(-1)+ msw = self.t(lp + "mlp_res_norm.weight") * \+ self.t(lp + "mlp_res_proj.weight").reshape(-1)+ prefix = hidden.copy()+ have_prefix = True+ if bres:+ hidden = np.stack([res_mix(prefix[t], [b[t] for b in bres], asw, self.eps)+ for t in range(T)])+ if li % self.res_bs == 0:+ bres.append(prefix.copy())+ have_prefix = False+ nrm = rmsnorm(hidden, in_ln, self.eps)+ att = self.kda(li, nrm) if li in self.kda_layers else self.mla(li, nrm)+ prefix = (prefix + att) if have_prefix else att+ mix = np.stack([res_mix(prefix[t], [b[t] for b in bres], msw, self.eps)+ for t in range(T)])+ nrm = rmsnorm(mix, post_ln, self.eps)+ mlp = self.moe(li, nrm) if li >= self.first_dense else self.dense(li, nrm)+ prefix = prefix + mlp+ hidden = prefix.copy()+ trace[:, li] = hidden+ print(f"[ref] layer {li} done ({'KDA' if li in self.kda_layers else 'MLA'},"+ f" {'moe' if li >= self.first_dense else 'dense'})", flush=True)+ trace.tofile(out_path)+ print(f"[ref] wrote {out_path}: {T} tokens x {self.n_layers} layers x {self.D}")+++def main():+ ap = argparse.ArgumentParser()+ sub = ap.add_subparsers(dest="cmd", required=True)+ g = sub.add_parser("gen")+ g.add_argument("out")+ g.add_argument("--tokens", type=int, default=6)+ g.add_argument("--dim", type=int, default=7168)+ g.add_argument("--seed", type=int, default=7)+ g.add_argument("--scale", type=float, default=0.02)+ r = sub.add_parser("run")+ r.add_argument("hf_dir")+ r.add_argument("--x0", required=True)+ r.add_argument("--layers", type=int, default=4)+ r.add_argument("--out", required=True)+ c = sub.add_parser("cmp")+ c.add_argument("a")+ c.add_argument("b")+ c.add_argument("--dim", type=int, default=7168)+ c.add_argument("--layers", type=int, default=4)+ a = ap.parse_args()+ if a.cmd == "gen":+ rng = np.random.default_rng(a.seed)+ x = (rng.standard_normal((a.tokens, a.dim)) * a.scale).astype(np.float32)+ x.tofile(a.out)+ print(f"wrote {a.out}: {a.tokens}x{a.dim}")+ elif a.cmd == "run":+ x0 = np.fromfile(a.x0, dtype=np.float32).reshape(-1, 7168)+ K3Ref(a.hf_dir, a.layers).run(x0, a.out)+ else:+ A = np.fromfile(a.a, dtype=np.float32).reshape(-1, a.layers, a.dim)+ B = np.fromfile(a.b, dtype=np.float32).reshape(-1, a.layers, a.dim)+ assert A.shape == B.shape, (A.shape, B.shape)+ worst = 0.0+ for li in range(a.layers):+ x, y = A[:, li], B[:, li]+ rel = np.linalg.norm(x - y) / (np.linalg.norm(y) + 1e-30)+ worst = max(worst, rel)+ print(f"layer {li}: rel_l2 {rel:.3e} (|c|={np.linalg.norm(x):.4g} |ref|={np.linalg.norm(y):.4g})")+ print("VERDICT:", "OK" if worst < 3e-4 else "MISMATCH", f"(worst {worst:.3e})")+++if __name__ == "__main__":+ main()diff --git a/c/tools/k3_repack.py b/c/tools/k3_repack.pynew file mode 100644index 00000000..00a91803--- /dev/null+++ b/c/tools/k3_repack.py@@ -0,0 +1,279 @@+#!/usr/bin/env python3+"""Repack a Kimi K3 HF snapshot into colibri's ideal ingestion layout.++Designed so the HDD->NVMe copy IS the conversion (one read of the source):++ - routed experts (MXFP4, QAT): byte-identical passthrough, written in+ expert-id order, the six tensors of an expert back-to-back (the engine+ reads one expert = one pread)+ - big BF16 matrices: quantized ON THE WAY with EXACTLY the engine's+ load-time algorithm (int8 per-row absmax/127 or int4-g64 absmax/7,+ round-half-even, same f32 op order), stored as U8 + <name>.qs f32 scale+ sidecar -> the engine skips its 10-15 min load-time quantization and+ cold-starts from the container in ~1-2 min+ - everything small or sensitive (norms, router, conv taps, dt_bias, A_log,+ o_norm, f_a/f_b, b_proj, res_*, embed_tokens, final norm): passthrough+ in the source dtype (BF16/F32) — quantizing these saves ~nothing and the+ engine keeps them f32 in RAM anyway+ - vision_tower / mm_projector (shards 95-96): dropped+ - output: model-XXXXX-of-000094.safetensors (same shard split: one layer+ per shard + shard 94 = embed/head), spec-valid safetensors (no holes),+ regenerated model.safetensors.index.json, config/tokenizer files copied++Resume-safe: each shard is written to .tmp and renamed; existing outputs are+skipped. Rerun after an interruption and it continues.++usage: k3_repack.py <src_dir> <dst_dir> [--bits 8] [--mla-bits N] [--head-bits N]+ [--shards 1,2,94] [--verify-full]++Classification needs the layer type (o_proj/g_proj exist on BOTH KDA and MLA+layers) — read from config.json linear_attn_config.+"""+import argparse+import json+import os+import shutil+import struct+import sys+import time++import numpy as np++CHUNK_ROWS = 4096 # quantize in row blocks (bounds RAM to ~120 MB)+COPY_BLOCK = 64 << 20 # passthrough copy block+++def read_header(path):+ with open(path, "rb") as f:+ n = struct.unpack("<Q", f.read(8))[0]+ h = json.loads(f.read(n))+ h.pop("__metadata__", None)+ return h, 8 + n+++def bf16_to_f32(buf):+ u = np.frombuffer(buf, dtype=np.uint16).astype(np.uint32) << 16+ return u.view(np.float32)+++def quant_int8(rows):+ """rows [n,I] f32 -> (int8 bytes, f32 scales [n]); engine w_load exact."""+ am = np.abs(rows).max(axis=1)+ s = (am / np.float32(127.0)).astype(np.float32)+ s = np.maximum(s, np.float32(1e-20))+ q = np.clip(np.rint(rows * (np.float32(1.0) / s)[:, None]), -127, 127).astype(np.int8)+ return q.tobytes(), s+++def quant_int4_g64(rows):+ """rows [n,I] f32, I%64==0 -> (packed bytes low-nibble-even biased+8,+ f32 scales [n, I/64]); engine w_load / matmul_i4_grouped exact."""+ n, I = rows.shape+ g = rows.reshape(n, I // 64, 64)+ am = np.abs(g).max(axis=2)+ s = (am / np.float32(7.0)).astype(np.float32)+ s = np.maximum(s, np.float32(1e-20))+ q = np.clip(np.rint(g * (np.float32(1.0) / s)[:, :, None]), -8, 7).astype(np.int32) + 8+ q = q.reshape(n, I)+ packed = (q[:, 0::2] | (q[:, 1::2] << 4)).astype(np.uint8)+ return packed.tobytes(), s+++class Classifier:+ def __init__(self, src):+ cfg = json.load(open(os.path.join(src, "config.json")))+ tc = cfg.get("text_config", cfg)+ self.kda = set(x - 1 for x in tc["linear_attn_config"]["kda_layers"])++ def __call__(self, name):+ """-> ('skip'|'expert'|'pass'|'q8class', class) where class in+ {'bits','mla','head'} for quantized tensors."""+ n = name+ if n.startswith("vision_tower") or n.startswith("mm_projector"):+ return "skip", None+ if n.startswith("language_model."):+ n = n[len("language_model."):]+ if ".block_sparse_moe.experts." in n:+ return "expert", None+ if n == "lm_head.weight":+ return "quant", "head"+ parts = n.split(".")+ li = int(parts[2]) if len(parts) > 3 and parts[1] == "layers" else -1+ if ".self_attn." in n and n.endswith(".weight"):+ leaf = parts[-2]+ if leaf in ("q_a_proj", "q_b_proj", "kv_a_proj_with_mqa", "kv_b_proj"):+ return "quant", "mla"+ if leaf in ("q_proj", "k_proj", "v_proj"):+ return "quant", "bits"+ if leaf in ("o_proj", "g_proj"):+ return ("quant", "bits") if li in self.kda else ("quant", "mla")+ return "pass", None # conv taps, norms, f_a/f_b, b_proj+ for leaf in ("routed_expert_down_proj", "routed_expert_up_proj",+ "shared_experts.gate_proj", "shared_experts.up_proj",+ "shared_experts.down_proj",+ "mlp.gate_proj", "mlp.up_proj", "mlp.down_proj"):+ if n.endswith(leaf + ".weight"):+ return "quant", "bits"+ return "pass", None # router, biases, norms, embed, res_*+++def expert_sort_key(name):+ # ...experts.<E>.<w1|w2|w3>.weight_<packed|scale>+ p = name.split(".experts.")[1].split(".")+ e = int(p[0])+ mat = {"w1": 0, "w2": 1, "w3": 2}[p[1]]+ kind = 0 if p[2].endswith("packed") else 1+ # engine slot order: w1p w1s w2p w2s w3p w3s+ return (e, mat, kind)+++def repack_shard(src_path, dst_path, cls, bits_map, verify_full):+ hdr, data_start = read_header(src_path)+ items = sorted(hdr.items(), key=lambda kv: kv[1]["data_offsets"][0])+ non_expert, experts = [], []+ for name, v in items:+ kind, qc = cls(name)+ if kind == "skip":+ continue+ (experts if kind == "expert" else non_expert).append((name, v, kind, qc))+ experts.sort(key=lambda t: expert_sort_key(t[0]))+ plan = non_expert + experts++ # pass 1: output sizes -> header with final offsets+ out_entries = [] # (name, dtype, shape, nbytes) (+ .qs after its tensor)+ for name, v, kind, qc in plan:+ if kind == "quant":+ O, I = v["shape"]+ bits = bits_map[qc]+ if bits <= 4 and I % 64 == 0:+ out_entries.append((name, "U8", [O, I // 2], O * (I // 2)))+ out_entries.append((name + ".qs", "F32", [O, I // 64], O * (I // 64) * 4))+ else:+ out_entries.append((name, "U8", [O, I], O * I))+ out_entries.append((name + ".qs", "F32", [O], O * 4))+ else:+ out_entries.append((name, v["dtype"], v["shape"],+ v["data_offsets"][1] - v["data_offsets"][0]))+ oh, off = {}, 0+ for name, dt, shape, nb in out_entries:+ oh[name] = {"dtype": dt, "shape": shape, "data_offsets": [off, off + nb]}+ off += nb+ hjson = json.dumps(oh, separators=(",", ":")).encode()+ pad = (-(8 + len(hjson))) % 8+ hjson += b" " * pad++ tmp = dst_path + ".tmp"+ written = 0+ with open(src_path, "rb") as fin, open(tmp, "wb") as fout:+ fout.write(struct.pack("<Q", len(hjson)) + hjson)+ for name, v, kind, qc in plan:+ a0, b0 = v["data_offsets"]+ fin.seek(data_start + a0)+ if kind == "quant":+ O, I = v["shape"]+ bits = bits_map[qc]+ use4 = bits <= 4 and I % 64 == 0+ qparts, sparts = [], []+ for r0 in range(0, O, CHUNK_ROWS):+ n = min(CHUNK_ROWS, O - r0)+ raw = fin.read(n * I * 2)+ rows = bf16_to_f32(raw).reshape(n, I)+ qb, s = quant_int4_g64(rows) if use4 else quant_int8(rows)+ qparts.append(qb)+ sparts.append(s.astype(np.float32).tobytes())+ for qb in qparts:+ fout.write(qb)+ for sb in sparts:+ fout.write(sb)+ written += sum(len(x) for x in qparts) + sum(len(x) for x in sparts)+ else:+ left = b0 - a0+ while left:+ blk = fin.read(min(COPY_BLOCK, left))+ if not blk:+ raise IOError(f"short read in {name}")+ fout.write(blk)+ left -= len(blk)+ written += b0 - a0+ # sanity: bytes written == declared+ want = off+ got = os.path.getsize(tmp) - 8 - len(hjson)+ if got != want:+ raise IOError(f"{dst_path}: wrote {got} data bytes, header declares {want}")+ os.replace(tmp, dst_path)++ if verify_full and experts:+ vh, vstart = read_header(dst_path)+ with open(src_path, "rb") as fs, open(dst_path, "rb") as fd:+ for name, v, kind, qc in experts:+ sa, sb = v["data_offsets"]+ da, db = vh[name]["data_offsets"]+ assert sb - sa == db - da, name+ fs.seek(data_start + sa)+ fd.seek(vstart + da)+ left = sb - sa+ while left:+ n = min(COPY_BLOCK, left)+ if fs.read(n) != fd.read(n):+ raise IOError(f"VERIFY FAILED: {name}")+ left -= n+ return oh, written+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("src")+ ap.add_argument("dst")+ ap.add_argument("--bits", type=int, default=8)+ ap.add_argument("--mla-bits", type=int, default=None)+ ap.add_argument("--head-bits", type=int, default=None)+ ap.add_argument("--shards", help="comma list of shard numbers (default 1..94)")+ ap.add_argument("--verify-full", action="store_true",+ help="re-read every expert byte and compare with the source")+ a = ap.parse_args()+ bits_map = {"bits": a.bits,+ "mla": a.mla_bits if a.mla_bits is not None else max(a.bits, 8),+ "head": a.head_bits if a.head_bits is not None else max(a.bits, 8)}+ os.makedirs(a.dst, exist_ok=True)+ cls = Classifier(a.src)+ shard_nums = ([int(x) for x in a.shards.split(",")] if a.shards+ else list(range(1, 95)))+ index = {}+ t0 = time.time()+ total = 0+ done_any = False+ for i, num in enumerate(shard_nums):+ sp = os.path.join(a.src, f"model-{num:05d}-of-000096.safetensors")+ dname = f"model-{num:05d}-of-000094.safetensors"+ dp = os.path.join(a.dst, dname)+ if not os.path.exists(sp):+ print(f"[{num:2d}] SOURCE MISSING — skipped (rerun when downloaded)", flush=True)+ continue+ if os.path.exists(dp):+ oh, _ = read_header(dp)+ for name in oh:+ index[name] = dname+ print(f"[{num:2d}] exists — skipped (resume)", flush=True)+ continue+ oh, written = repack_shard(sp, dp, cls, bits_map, a.verify_full)+ for name in oh:+ index[name] = dname+ total += written+ done_any = True+ el = time.time() - t0+ print(f"[{num:2d}] {os.path.getsize(dp)/1e9:6.2f} GB out "+ f"({total/1e9:7.1f} GB total, {total/1e9/max(el,1e-9):5.2f} GB/s, {el:6.0f}s)",+ flush=True)+ if done_any or index:+ with open(os.path.join(a.dst, "model.safetensors.index.json"), "w") as f:+ json.dump({"metadata": {"total_size": total}, "weight_map": index}, f)+ for extra in ("config.json", "generation_config.json", "tokenizer.json",+ "tokenizer_config.json"):+ s = os.path.join(a.src, extra)+ if os.path.exists(s):+ shutil.copy2(s, os.path.join(a.dst, extra))+ print(f"done: {total/1e9:.1f} GB written in {time.time()-t0:.0f}s -> {a.dst}")+++if __name__ == "__main__":+ main()diff --git a/c/tools/k3_tokenizer.py b/c/tools/k3_tokenizer.pynew file mode 100644index 00000000..8a3da0e8--- /dev/null+++ b/c/tools/k3_tokenizer.py@@ -0,0 +1,225 @@+#!/usr/bin/env python3+"""Synthesize tokenizer.json for Kimi K3 from the repo's tiktoken.model.++The K3 HF repo ships only a raw tiktoken BPE (base64 token -> rank) plus+tokenization_kimi.py; the colibri-family engines load HuggingFace+tokenizer.json (tok.h). This tool writes one:++ - vocab: every tiktoken byte sequence mapped through the GPT-2 byte-level+ alphabet (the same map tok.h rebuilds), id = rank+ - merges: EMPTY on purpose. Rank-recovered merge lists cannot be made+ exactly faithful (verified: "newlines" merges new+l+ines instead of+ new+lines under the standard max-rank-minimizing recovery). tok.h+ detects the empty list and switches to rank-BPE — merge the adjacent+ pair whose concatenation has the lowest vocab id — which IS tiktoken's+ algorithm, so encoding is exact by construction.+ - added_tokens: ids num_base..num_base+255 from tokenizer_config.json's+ added_tokens_decoder, gaps filled with <|reserved_token_N|>+ - pre_tokenizer: the Kimi Split regex (contains \\p{Han} and \\p{Lu} — the+ markers tok.h sniffs to select the kimi family), plus ByteLevel++usage: k3_tokenizer.py <hf_dir> [-o out.json] [--check "text" ...]+ --check re-encodes text with tiktoken (if installed) and with a pure-+ python replay of the emitted tokenizer.json; both must agree.+"""+import argparse+import base64+import json+import os+import sys++PAT = "|".join([+ r"""[\p{Han}]+""",+ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",+ r"""[^\r\n\p{L}\p{N}]?[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]+[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?""",+ r"""\p{N}{1,3}""",+ r""" ?[^\s\p{L}\p{N}]+[\r\n]*""",+ r"""\s*[\r\n]+""",+ r"""\s+(?!\S)""",+ r"""\s+""",+])++NUM_RESERVED = 256+++def bytes_to_unicode():+ bs = list(range(33, 127)) + list(range(161, 173)) + list(range(174, 256))+ cs = bs[:]+ n = 0+ for b in range(256):+ if b not in bs:+ bs.append(b)+ cs.append(256 + n)+ n += 1+ return dict(zip(bs, [chr(c) for c in cs]))+++def main():+ ap = argparse.ArgumentParser()+ ap.add_argument("hf_dir")+ ap.add_argument("-o", "--out")+ ap.add_argument("--check", nargs="*", default=None)+ ap.add_argument("--ctest", metavar="BINARY",+ help="cross-check the C tokenizer (tests/test_tok_kimi) against tiktoken")+ a = ap.parse_args()+ out = a.out or os.path.join(a.hf_dir, "tokenizer.json")++ ranks = {}+ with open(os.path.join(a.hf_dir, "tiktoken.model"), "rb") as f:+ for line in f:+ if not line.strip():+ continue+ tok, rank = line.split()+ ranks[base64.b64decode(tok)] = int(rank)+ num_base = len(ranks)+ print(f"tiktoken vocab: {num_base} tokens")++ b2u = bytes_to_unicode()++ def bl(bs): # bytes -> byte-level string+ return "".join(b2u[b] for b in bs)++ vocab = {bl(tok): rank for tok, rank in ranks.items()}+ merges = [] # empty on purpose -> tok.h rank-BPE (see module docstring)++ tc = {}+ tcp = os.path.join(a.hf_dir, "tokenizer_config.json")+ if os.path.exists(tcp):+ tc = json.load(open(tcp))+ named = {int(k): v["content"] for k, v in tc.get("added_tokens_decoder", {}).items()}+ special = {int(k): bool(v.get("special", False))+ for k, v in tc.get("added_tokens_decoder", {}).items()}+ added = []+ for i in range(num_base, num_base + NUM_RESERVED):+ content = named.get(i, f"<|reserved_token_{i}|>")+ added.append({"id": i, "content": content, "single_word": False,+ "lstrip": False, "rstrip": False, "normalized": False,+ "special": special.get(i, True)})++ tj = {+ "version": "1.0",+ "truncation": None,+ "padding": None,+ "added_tokens": added,+ "normalizer": None,+ "pre_tokenizer": {+ "type": "Sequence",+ "pretokenizers": [+ {"type": "Split", "pattern": {"Regex": PAT},+ "behavior": "Isolated", "invert": False},+ {"type": "ByteLevel", "add_prefix_space": False,+ "trim_offsets": True, "use_regex": False},+ ],+ },+ "post_processor": None,+ "decoder": {"type": "ByteLevel", "add_prefix_space": True,+ "trim_offsets": True, "use_regex": True},+ "model": {+ "type": "BPE", "dropout": None, "unk_token": None,+ "continuing_subword_prefix": None, "end_of_word_suffix": None,+ "fuse_unk": False, "byte_fallback": False, "ignore_merges": True,+ "vocab": vocab, "merges": merges,+ },+ }+ with open(out, "w", encoding="utf-8") as f:+ json.dump(tj, f, ensure_ascii=False)+ print(f"wrote {out} ({os.path.getsize(out)/1e6:.1f} MB)")++ corpus = [+ "Hello, world! The quick brown fox jumps over 12345 lazy dogs.",+ "def main():\n return {'a': 1}\n",+ "Kimi K3 是一个开源模型。它支持中文和English mixed text!",+ " indented\n\n\nnewlines\t\ttabs ",+ "I'll say we've DONE it. HTTPResponse XMLHttpRequest",+ "今天天气怎么样?我们去公园散步吧。",+ "日本語のテキストです。カタカナとひらがな、漢字も。",+ "한국어 텍스트도 있습니다.",+ "naïve café résumé ÜBER Straße mañana",+ "don't CAN'T it's I'M we'RE you'll",+ "1 12 123 1234 12345 3.14159 -42",+ "!!! ??? ... --- /// ***bold*** `code`",+ " a\n\n\nb\t\t c \n d ",+ "🚀 emoji test 🎉 ½ ² € §",+ "iPhone macOS HTTPResponse XMLHttpRequest camelCase snake_case",+ "々 〇 一二三 中中中English中文",+ "\r\nwindows\r\nline\r\nendings\r\n",+ "mixed 中文with no space英文boundaries测试",+ ]+ if a.ctest:+ import struct as _st+ import subprocess+ try:+ import tiktoken+ except ImportError:+ print("--ctest: tiktoken not installed", file=sys.stderr)+ return+ enc = tiktoken.Encoding(name="k3", pat_str=PAT,+ mergeable_ranks=ranks, special_tokens={})+ cases = "/tmp/k3_tok_cases.bin"+ with open(cases, "wb") as f:+ f.write(_st.pack("<I", len(corpus)))+ for t in corpus:+ b = t.encode("utf-8")+ f.write(_st.pack("<I", len(b)) + b)+ r = subprocess.run([a.ctest, out, cases], capture_output=True, text=True)+ print(r.stderr.strip(), file=sys.stderr)+ got = [([int(x) for x in ln.split()] if ln else [])+ for ln in r.stdout.splitlines()]+ ok = True+ for t, g in zip(corpus, got):+ want = enc.encode(t)+ if want != g:+ ok = False+ print(f"C MISMATCH on {t!r}\n tiktoken: {want}\n tok.h : {g}")+ print("ctest:", "OK — tok.h matches tiktoken on all cases" if ok else "FAILED")+ return++ if a.check is not None:+ texts = a.check or corpus+ try:+ import tiktoken+ except ImportError:+ print("--check: tiktoken not installed, skipping", file=sys.stderr)+ return+ enc = tiktoken.Encoding(name="k3", pat_str=PAT,+ mergeable_ranks=ranks, special_tokens={})+ def bpe_py(piece_bytes):+ """rank-BPE replay: what tok.h does with an empty merges list"""+ s = bl(piece_bytes)+ if s in vocab:+ return [vocab[s]]+ sym = [c for c in s]+ while True:+ best, bp = None, -1+ for i in range(len(sym) - 1):+ r = vocab.get(sym[i] + sym[i + 1])+ if r is not None and (best is None or r < best):+ best, bp = r, i+ if bp < 0:+ break+ sym[bp:bp + 2] = [sym[bp] + sym[bp + 1]]+ return [vocab[t] for t in sym if t in vocab]++ import regex+ # Python's regex module wants V1 set-difference syntax for what+ # fancy-regex writes as [...&&[^\p{Han}]]+ py_pat = PAT.replace(+ r"[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]",+ r"[[\p{Lu}\p{Lt}\p{Lm}\p{Lo}\p{M}]--[\p{Han}]]").replace(+ r"[\p{Ll}\p{Lm}\p{Lo}\p{M}&&[^\p{Han}]]",+ r"[[\p{Ll}\p{Lm}\p{Lo}\p{M}]--[\p{Han}]]")+ cre = regex.compile(py_pat, flags=regex.V1)+ ok = True+ for t in texts:+ want = enc.encode(t)+ got = []+ for mt in cre.finditer(t):+ got += bpe_py(mt.group().encode("utf-8"))+ if want != got:+ ok = False+ print(f"MISMATCH on {t!r}\n tiktoken: {want}\n replay : {got}")+ print("check:", "OK — merges replay matches tiktoken" if ok else "FAILED")+++if __name__ == "__main__":+ main()