Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
test(k3): tiny fixture + independent vendor oracle for Kimi K3
JustVugg03df9d86e5e2
diff --git a/.github/workflows/check.yml b/.github/workflows/check.ymlindex fd050494..7eb49655 100644--- a/.github/workflows/check.yml+++ b/.github/workflows/check.yml@@ -59,6 +59,22 @@ jobs:name: colibri-windows-exepath: c/colibri.exe+ k3-tiny:+ name: Linux (generated tiny K3 vendor oracle)+ runs-on: ubuntu-latest+ timeout-minutes: 20+ steps:+ - uses: actions/checkout@v4+ - uses: actions/setup-python@v5+ with:+ python-version: '3.13'+ cache: pip+ cache-dependency-path: c/tools/requirements-kimi-k3-tiny.txt+ - name: Install pinned tiny-fixture dependencies+ run: python -m pip install -r c/tools/requirements-kimi-k3-tiny.txt+ - name: Generate fixture and run vendor oracle+ run: make -C c kimi-k3-tiny-check+v4-tiny:name: Linux (generated tiny V4 target oracle)runs-on: ubuntu-latestdiff --git a/.gitignore b/.gitignoreindex bd86311d..b46b8a44 100644--- a/.gitignore+++ b/.gitignore@@ -137,3 +137,6 @@ c/*.obj# compiled objects*.o++# vendor modeling cache for the tiny K3 oracle (maintainer tool)+c/kimi_k3_tiny/.vendor_kimi_k3/diff --git a/c/Makefile b/c/Makefileindex 48d712d2..d221d37c 100644--- a/c/Makefile+++ b/c/Makefile@@ -606,6 +606,20 @@ endifall: colibri$(EXE)+# Tiny Kimi K3 fixture + vendor oracle. The generator is numpy-only and+# deterministic; ref.json is committed and comes from Moonshot's own modeling+# code (tools/make_kimi_k3_ref.py, maintainer-run, SHA-256-pinned), so CI+# needs neither torch nor network. kimi_k3 builds on every platform, so+# unlike the V4 targets this needs no support gate.+.PHONY: kimi-k3-tiny-generate kimi-k3-tiny-check+kimi-k3-tiny-generate:+ $(PYTHON) tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force++kimi-k3-tiny-check: kimi-k3-tiny-generate kimi_k3$(EXE)+ $(PYTHON) tests/test_kimi_k3_tiny.py \+ --binary ./kimi_k3$(EXE) \+ --fixture ./kimi_k3_tiny+.PHONY: deepseek-v4 deepseek-v4-oracle deepseek-v4-tiny-generate \deepseek-v4-tiny-check deepseek-v4-cleanifeq ($(COLI_V4_SUPPORTED),1)diff --git a/c/kimi_k3_tiny/config.json b/c/kimi_k3_tiny/config.jsonnew file mode 100644index 00000000..d42f6bce--- /dev/null+++ b/c/kimi_k3_tiny/config.json@@ -0,0 +1,61 @@+{+ "model_type": "kimi_linear",+ "architectures": [+ "KimiLinearForCausalLM"+ ],+ "hidden_size": 128,+ "num_hidden_layers": 6,+ "vocab_size": 256,+ "first_k_dense_replace": 2,+ "intermediate_size": 64,+ "num_attention_heads": 4,+ "num_key_value_heads": 4,+ "q_lora_rank": 32,+ "kv_lora_rank": 32,+ "qk_nope_head_dim": 16,+ "qk_rope_head_dim": 8,+ "v_head_dim": 16,+ "num_experts": 8,+ "num_experts_per_token": 2,+ "moe_intermediate_size": 32,+ "routed_expert_hidden_size": 32,+ "num_shared_experts": 1,+ "attn_res_block_size": 2,+ "hidden_act": "situ",+ "activation_situ_beta": 1.0,+ "activation_situ_linear_beta": 1.0,+ "moe_router_activation_func": "sigmoid",+ "topk_method": "noaux_tc",+ "use_grouped_topk": true,+ "num_expert_group": 1,+ "topk_group": 1,+ "moe_renormalize": true,+ "moe_layer_freq": 1,+ "routed_scaling_factor": 1.0,+ "latent_moe_use_norm": true,+ "mla_use_nope": true,+ "mla_use_output_gate": true,+ "rms_norm_eps": 1e-05,+ "rope_theta": 10000.0,+ "max_position_embeddings": 256,+ "tie_word_embeddings": false,+ "linear_attn_config": {+ "num_heads": 2,+ "head_dim": 16,+ "short_conv_kernel_size": 4,+ "kda_layers": [+ 1,+ 3+ ],+ "full_attn_layers": [+ 2,+ 4,+ 5,+ 6+ ],+ "use_full_rank_gate": true,+ "gate_lower_bound": -5.0+ },+ "bos_token_id": 0,+ "eos_token_id": 1+}diff --git a/c/kimi_k3_tiny/ref.json b/c/kimi_k3_tiny/ref.jsonnew file mode 100644index 00000000..d2fe6d34--- /dev/null+++ b/c/kimi_k3_tiny/ref.json@@ -0,0 +1,464 @@+{+ "schema_version": 1,+ "source": "moonshot-vendor",+ "vendor_repo": "moonshotai/Kimi-K3",+ "vendor_files": {+ "configuration_kimi_k3.py": "735eb9ebe593e17d231e08e1df7f7be9b5ee0e079f511aa201f9572077b416ae",+ "modeling_kimi_linear.py": "9e3564c70ac21854ce5a090cc946c5dc76b70d1050ef50840449181a20fff44a"+ },+ "torch_version": "2.12.1+cu130",+ "transformers_version": "4.56.2",+ "engine_env": {+ "K3_BITS": "32",+ "K3_MLA_BITS": "32",+ "K3_HEAD_BITS": "32",+ "K3_IDOT": "0",+ "COLI_TEMP": "0"+ },+ "cases": {+ "short": {+ "prompt_ids": [+ 5,+ 7,+ 9,+ 11,+ 13,+ 17,+ 19,+ 23+ ],+ "teacher_forcing_ids": [+ 4,+ 6,+ 219,+ 10,+ 69,+ 201,+ 18,+ 78,+ 157,+ 198,+ 33,+ 205,+ 110,+ 109,+ 108,+ 107+ ],+ "greedy_full_ids": [+ 5,+ 7,+ 9,+ 11,+ 13,+ 17,+ 19,+ 23,+ 78,+ 157,+ 198,+ 33,+ 205,+ 110,+ 109,+ 108+ ],+ "greedy_new_ids": [+ 78,+ 157,+ 198,+ 33,+ 205,+ 110,+ 109,+ 108+ ],+ "max_new_tokens": 8,+ "min_logit_margin": 0.2905+ },+ "chunk": {+ "prompt_ids": [+ 5,+ 12,+ 19,+ 26,+ 33,+ 40,+ 47,+ 54,+ 61,+ 68,+ 75,+ 82,+ 89,+ 96,+ 103,+ 110,+ 117,+ 124,+ 131,+ 138,+ 145,+ 152,+ 159,+ 166,+ 173,+ 180,+ 187,+ 194,+ 201,+ 208,+ 215,+ 222,+ 229,+ 236,+ 243,+ 250,+ 2,+ 8,+ 15,+ 22+ ],+ "teacher_forcing_ids": [+ 4,+ 111,+ 18,+ 246,+ 250,+ 155,+ 251,+ 53,+ 157,+ 67,+ 74,+ 81,+ 88,+ 194,+ 235,+ 109,+ 118,+ 68,+ 91,+ 70,+ 144,+ 10,+ 202,+ 165,+ 200,+ 0,+ 186,+ 157,+ 3,+ 10,+ 122,+ 56,+ 111,+ 4,+ 34,+ 203,+ 127,+ 135,+ 254,+ 146,+ 145,+ 206,+ 205,+ 81+ ],+ "greedy_full_ids": [+ 5,+ 12,+ 19,+ 26,+ 33,+ 40,+ 47,+ 54,+ 61,+ 68,+ 75,+ 82,+ 89,+ 96,+ 103,+ 110,+ 117,+ 124,+ 131,+ 138,+ 145,+ 152,+ 159,+ 166,+ 173,+ 180,+ 187,+ 194,+ 201,+ 208,+ 215,+ 222,+ 229,+ 236,+ 243,+ 250,+ 2,+ 8,+ 15,+ 22,+ 146,+ 145,+ 206,+ 205+ ],+ "greedy_new_ids": [+ 146,+ 145,+ 206,+ 205+ ],+ "max_new_tokens": 4,+ "min_logit_margin": 0.0274+ },+ "long": {+ "prompt_ids": [+ 5,+ 16,+ 27,+ 38,+ 49,+ 60,+ 71,+ 82,+ 93,+ 104,+ 115,+ 126,+ 137,+ 148,+ 159,+ 170,+ 181,+ 192,+ 203,+ 214,+ 225,+ 236,+ 247,+ 2,+ 13,+ 24,+ 35,+ 46,+ 57,+ 68,+ 79,+ 90,+ 101,+ 112,+ 123,+ 134,+ 145,+ 156,+ 167,+ 178,+ 189,+ 200,+ 211,+ 222,+ 233,+ 244,+ 255,+ 10,+ 21,+ 32,+ 43,+ 54,+ 65,+ 76,+ 87,+ 98,+ 109,+ 120,+ 131,+ 142,+ 153,+ 164,+ 175,+ 186,+ 197,+ 208,+ 219,+ 230,+ 241,+ 252,+ 7,+ 18+ ],+ "teacher_forcing_ids": [+ 4,+ 144,+ 111,+ 37,+ 255,+ 244,+ 70,+ 206,+ 202,+ 103,+ 171,+ 14,+ 165,+ 147,+ 118,+ 133,+ 180,+ 6,+ 154,+ 213,+ 97,+ 226,+ 246,+ 28,+ 184,+ 3,+ 34,+ 141,+ 56,+ 189,+ 97,+ 81,+ 66,+ 105,+ 116,+ 133,+ 48,+ 155,+ 166,+ 184,+ 163,+ 199,+ 210,+ 170,+ 61,+ 243,+ 210,+ 235,+ 20,+ 31,+ 186,+ 53,+ 254,+ 75,+ 86,+ 97,+ 65,+ 119,+ 27,+ 142,+ 254,+ 177,+ 202,+ 185,+ 4,+ 185,+ 218,+ 251,+ 54,+ 251,+ 80,+ 11,+ 10,+ 9,+ 202,+ 201+ ],+ "greedy_full_ids": [+ 5,+ 16,+ 27,+ 38,+ 49,+ 60,+ 71,+ 82,+ 93,+ 104,+ 115,+ 126,+ 137,+ 148,+ 159,+ 170,+ 181,+ 192,+ 203,+ 214,+ 225,+ 236,+ 247,+ 2,+ 13,+ 24,+ 35,+ 46,+ 57,+ 68,+ 79,+ 90,+ 101,+ 112,+ 123,+ 134,+ 145,+ 156,+ 167,+ 178,+ 189,+ 200,+ 211,+ 222,+ 233,+ 244,+ 255,+ 10,+ 21,+ 32,+ 43,+ 54,+ 65,+ 76,+ 87,+ 98,+ 109,+ 120,+ 131,+ 142,+ 153,+ 164,+ 175,+ 186,+ 197,+ 208,+ 219,+ 230,+ 241,+ 252,+ 7,+ 18,+ 11,+ 10,+ 9,+ 202+ ],+ "greedy_new_ids": [+ 11,+ 10,+ 9,+ 202+ ],+ "max_new_tokens": 4,+ "min_logit_margin": 0.0253+ }+ }+}diff --git a/c/tests/test_kimi_k3_tiny.py b/c/tests/test_kimi_k3_tiny.pynew file mode 100644index 00000000..15bfed53--- /dev/null+++ b/c/tests/test_kimi_k3_tiny.py@@ -0,0 +1,170 @@+#!/usr/bin/env python3+"""Token-exact checks for the Kimi K3 engine against an independent oracle.++Dependency-free (stdlib only). The fixture is generated by+tools/make_kimi_k3_tiny.py (numpy, deterministic); the expected tokens in+ref.json come from Moonshot's own modeling code via tools/make_kimi_k3_ref.py+(run offline by a maintainer, vendor files pinned by SHA-256), so the engine+and its oracle never share an author.++What is checked, all token-exact (argmax, no tolerances):+ - greedy generation for three cases (short; 40 tokens crossing the default+ 32-token prefill chunk; 72 tokens crossing it twice)+ - teacher forcing at every position via the K3_LOGITS dump+ - determinism (two identical runs, identical tokens)+ - the oracle BITES: one corrupted expert scale byte per layer-2 expert must+ change the output (guards against a test that silently compares nothing)++The engine runs in its exact-math configuration (ref.json's engine_env:+K3_BITS/K3_MLA_BITS/K3_HEAD_BITS=32 because the fixture stores f32 dense+weights, K3_IDOT=0 because int8-activation expert matmuls are a documented+~0.4%/group speed approximation, not an oracle target). The fixture ships+no tokenizer.json on purpose: the engine then prints raw token ids on+stdout, which is what this test parses.+"""++import argparse+import array+import json+import os+import shutil+import struct+import subprocess+import sys+import tempfile+from pathlib import Path+++def run_engine(binary, fixture, prompt_ids, max_new, env, extra_env=None):+ run_env = dict(os.environ, **env)+ if extra_env:+ run_env.update(extra_env)+ result = subprocess.run(+ [binary, str(fixture), "--ids", " ".join(map(str, prompt_ids)),+ "--ngen", str(max_new)],+ env=run_env, capture_output=True, text=True, timeout=600)+ if result.returncode != 0:+ raise AssertionError(+ f"engine exited {result.returncode}:\n{result.stderr[-2000:]}")+ # without a tokenizer.json the engine prints "id id id ... TUNE decode:"+ return [int(token) for token in result.stdout.split("TUNE")[0].split()]+++def check_greedy(binary, fixture, reference):+ for name, case in reference["cases"].items():+ got = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ want = case["greedy_new_ids"]+ if got != want:+ raise AssertionError(+ f"greedy mismatch in case '{name}': engine {got} vs "+ f"reference {want}")+ print(f"ok greedy {name}: {len(want)}/{len(want)} tokens")+++def check_determinism(binary, fixture, reference):+ case = reference["cases"]["short"]+ first = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ second = run_engine(binary, fixture, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ if first != second:+ raise AssertionError(f"non-deterministic greedy: {first} vs {second}")+ print("ok determinism: two identical runs, identical tokens")+++def check_teacher_forcing(binary, fixture, reference, vocab):+ for name, case in reference["cases"].items():+ full = case["greedy_full_ids"]+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:+ logits_path = tmp.name+ try:+ run_engine(binary, fixture, full, 0, reference["engine_env"],+ extra_env={"K3_LOGITS": logits_path})+ raw = array.array("f")+ with open(logits_path, "rb") as fh:+ raw.frombytes(fh.read())+ finally:+ os.unlink(logits_path)+ if len(raw) != len(full) * vocab:+ raise AssertionError(+ f"case '{name}': K3_LOGITS dumped {len(raw)} floats, "+ f"expected {len(full) * vocab}")+ got = []+ for position in range(len(full)):+ row = raw[position * vocab:(position + 1) * vocab]+ got.append(max(range(vocab), key=row.__getitem__))+ want = case["teacher_forcing_ids"]+ if got != want:+ first_bad = next(i for i, (a, b) in enumerate(zip(got, want))+ if a != b)+ raise AssertionError(+ f"teacher forcing mismatch in case '{name}' at position "+ f"{first_bad}: engine {got[first_bad]} vs reference "+ f"{want[first_bad]}")+ print(f"ok teacher forcing {name}: {len(want)}/{len(want)} positions")+++def check_oracle_bites(binary, fixture, reference, experts):+ """One corrupted scale byte per layer-2 expert must change the output.+ Without this, a comparator bug that accepts anything would pass silently."""+ corrupt = Path(tempfile.mkdtemp(prefix="kimi-k3-tiny-bite-")) / "fixture"+ shutil.copytree(fixture, corrupt)+ try:+ model = corrupt / "model.safetensors"+ raw = bytearray(model.read_bytes())+ header_len = struct.unpack("<Q", bytes(raw[:8]))[0]+ header = json.loads(bytes(raw[8:8 + header_len]))+ for expert in range(experts):+ name = (f"model.layers.2.block_sparse_moe.experts.{expert}"+ f".w1.weight_scale")+ lo, _ = header[name]["data_offsets"]+ offset = 8 + header_len + lo+ raw[offset] = (raw[offset] + 3) % 256 # 2^3 on one group+ model.write_bytes(bytes(raw))+ diverged = 0+ for case in reference["cases"].values():+ got = run_engine(binary, corrupt, case["prompt_ids"],+ case["max_new_tokens"], reference["engine_env"])+ if got != case["greedy_new_ids"]:+ diverged += 1+ if not diverged:+ raise AssertionError(+ "corrupted expert scales produced identical tokens in every "+ "case: the oracle does not bite")+ print(f"ok oracle bites: corrupted expert scales diverge in "+ f"{diverged}/{len(reference['cases'])} cases")+ finally:+ shutil.rmtree(corrupt.parent)+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--binary", required=True)+ parser.add_argument("--fixture", required=True)+ args = parser.parse_args()++ fixture = Path(args.fixture)+ reference = json.loads((fixture / "ref.json").read_text())+ if reference.get("source") != "moonshot-vendor":+ raise AssertionError(+ "ref.json must come from the independent vendor implementation "+ f"(source={reference.get('source')!r})")+ config = json.loads((fixture / "config.json").read_text())+ if (fixture / "tokenizer.json").exists():+ raise AssertionError(+ "fixture must not ship tokenizer.json: raw-id stdout is the "+ "comparison channel")++ check_greedy(args.binary, fixture, reference)+ check_determinism(args.binary, fixture, reference)+ check_teacher_forcing(args.binary, fixture, reference,+ config["vocab_size"])+ check_oracle_bites(args.binary, fixture, reference,+ config["num_experts"])+ print("kimi-k3-tiny-check: all checks passed")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/make_kimi_k3_ref.py b/c/tools/make_kimi_k3_ref.pynew file mode 100644index 00000000..211f1a8c--- /dev/null+++ b/c/tools/make_kimi_k3_ref.py@@ -0,0 +1,372 @@+#!/usr/bin/env python3+"""Produce ref.json for the tiny Kimi K3 fixture from Moonshot's own code.++Maintainer tool, run offline once per fixture change; CI never executes it+(tests/test_kimi_k3_tiny.py only reads the committed ref.json). The point is+independence: the reference tokens come from the vendor's published+implementation (modeling_kimi_linear.py on the Kimi-K3 Hugging Face repo,+pinned below by SHA-256), so the engine and its oracle never share an author.+A shared misunderstanding cannot pass both.++Parity is by construction, the same discipline as make_deepseek_v4_tiny.py:+- dense weights are stored f32 in the fixture and the engine is tested with+ K3_BITS=32 / K3_MLA_BITS=32 / K3_HEAD_BITS=32, so both sides compute from+ identical bytes;+- routed experts are stored packed MXFP4 g32; the engine decodes them+ natively while this script dequantizes THE SAME BYTES (low nibble = even+ column, w = e2m1 * 2^(scale-127)) into the vendor model's f32 Linear+ weights. Either way the mathematical weights are equal, so token-level+ argmax must agree wherever the logit margin exceeds float noise -- and the+ generator engineers wide margins, which this script VERIFIES per emitted+ position instead of assuming (--min-margin, refuse on violation).++Requires: torch (CPU is fine), einops, numpy. Network only if the vendor+files are not already cached next to --vendor-dir.+"""++import argparse+import hashlib+import importlib+import json+import struct+import sys+import urllib.request+from pathlib import Path++import numpy as np++VENDOR_REPO = "moonshotai/Kimi-K3"+# Pinned provenance: refuse to run against silently changed vendor code.+VENDOR_FILES = {+ "configuration_kimi_k3.py":+ "735eb9ebe593e17d231e08e1df7f7be9b5ee0e079f511aa201f9572077b416ae",+ "modeling_kimi_linear.py":+ "9e3564c70ac21854ce5a090cc946c5dc76b70d1050ef50840449181a20fff44a",+}++MX4_LUT = np.array([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,+ -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],+ dtype=np.float32)+++def read_safetensors(path):+ with open(path, "rb") as fh:+ header_len = struct.unpack("<Q", fh.read(8))[0]+ header = json.loads(fh.read(header_len))+ base = 8 + header_len+ raw = fh.read()+ dtypes = {"F32": np.float32, "U8": np.uint8}+ out = {}+ for name, meta in header.items():+ if name == "__metadata__":+ continue+ lo, hi = meta["data_offsets"]+ dtype = dtypes[meta["dtype"]]+ arr = np.frombuffer(raw, dtype=dtype,+ count=(hi - lo) // np.dtype(dtype).itemsize,+ offset=lo)+ out[name] = arr.reshape(meta["shape"]).copy()+ return out+++def mxfp4_dequant(packed, scale):+ """Exact peer of quant.h:matmul_mxfp4's decode: packed [O, I/2] with the+ LOW nibble on the even column, scale [O, I/32] as 2^(s-127) per group."""+ rows, half = packed.shape+ cols = half * 2+ codes = np.empty((rows, cols), dtype=np.uint8)+ codes[:, 0::2] = packed & 0x0F+ codes[:, 1::2] = packed >> 4+ values = MX4_LUT[codes]+ exponents = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127)+ return (values.reshape(rows, cols // 32, 32) *+ exponents[:, :, None]).reshape(rows, cols).astype(np.float32)+++def fetch_vendor(vendor_dir):+ vendor_dir.mkdir(parents=True, exist_ok=True)+ for name, expected in VENDOR_FILES.items():+ path = vendor_dir / name+ if not path.exists():+ url = f"https://huggingface.co/{VENDOR_REPO}/raw/main/{name}"+ print(f"[ref] downloading {url}", file=sys.stderr)+ with urllib.request.urlopen(url, timeout=60) as resp:+ path.write_bytes(resp.read())+ digest = hashlib.sha256(path.read_bytes()).hexdigest()+ if digest != expected:+ raise SystemExit(+ f"{name}: sha256 {digest} does not match the pinned "+ f"{expected}; the vendor file changed upstream. Review it, "+ f"then update VENDOR_FILES deliberately.")+ init = vendor_dir / "__init__.py"+ if not init.exists():+ init.write_text("")+++def load_vendor(vendor_dir):+ """Load the vendor files as a synthetic package, so their relative+ imports work regardless of the directory's (dotted) name."""+ import importlib.machinery+ import importlib.util+ pkg_name = "kimi_k3_vendor"+ spec = importlib.machinery.ModuleSpec(pkg_name, None, is_package=True)+ package = importlib.util.module_from_spec(spec)+ package.__path__ = [str(vendor_dir)]+ sys.modules[pkg_name] = package++ def load(stem):+ file_spec = importlib.util.spec_from_file_location(+ f"{pkg_name}.{stem}", vendor_dir / f"{stem}.py")+ module = importlib.util.module_from_spec(file_spec)+ sys.modules[f"{pkg_name}.{stem}"] = module+ file_spec.loader.exec_module(module)+ return module++ load("configuration_kimi_k3")+ module = load("modeling_kimi_linear")+ patch_kda_cpu(module)+ return module+++def patch_kda_cpu(module):+ """The vendor calls fla's chunk_kda/fused_recurrent_kda, which are Triton+ kernels and need a GPU. Reroute both to fla's OWN naive torch+ implementation (fla/ops/kda/naive.py) plus fla's OWN naive gate+ (fla/ops/kda/gate.py) -- the reference implementations fla tests its+ kernels against. No colibri-authored math enters the oracle: the+ preprocessing below mirrors exactly the use_*_in_kernel flags the vendor+ passes, using fla's published formulas."""+ import torch+ import torch.nn.functional as F+ from fla.ops.kda.naive import naive_recurrent_kda+ from fla.ops.kda.gate import naive_kda_gate, naive_kda_lowerbound_gate++ def kda_cpu(q, k, v, g, beta, A_log=None, dt_bias=None, scale=None,+ initial_state=None, output_final_state=False,+ use_qk_l2norm_in_kernel=False, use_gate_in_kernel=False,+ use_beta_sigmoid_in_kernel=False, safe_gate=False,+ lower_bound=None, transpose_state_layout=False,+ cu_seqlens=None, **_ignored):+ # our reference forward is single-sequence, cache-free+ assert cu_seqlens is None, "packed sequences not expected here"+ assert initial_state is None, "cache-free reference only"+ if use_qk_l2norm_in_kernel:+ q = F.normalize(q.float(), p=2, dim=-1)+ k = F.normalize(k.float(), p=2, dim=-1)+ if use_gate_in_kernel:+ if safe_gate or lower_bound is not None:+ g = naive_kda_lowerbound_gate(g, A_log, dt_bias,+ lower_bound+ if lower_bound is not None+ else -5.0)+ else:+ g = naive_kda_gate(g, A_log, dt_bias)+ if use_beta_sigmoid_in_kernel:+ beta = torch.sigmoid(beta.float())+ o, state = naive_recurrent_kda(+ q=q, k=k, v=v, g=g, beta=beta, scale=scale,+ initial_state=None, output_final_state=output_final_state)+ return o, state++ module.chunk_kda = kda_cpu+ module.fused_recurrent_kda = kda_cpu++ # fla's ShortConvolution supports only the triton/cuda backends. Its+ # documented semantics are a causal depthwise conv (nn.Conv1d with+ # groups=D, padding=K-1, output truncated to T) plus SiLU. Re-express+ # exactly that in stock torch.+ def shortconv_cpu(self, x, residual=None, mask=None, cache=None,+ output_final_state=False, cu_seqlens=None,+ chunk_indices=None, **_ignored):+ assert mask is None and cache is None and cu_seqlens is None+ assert not output_final_state, "cache-free reference only"+ length = x.shape[1]+ y = F.conv1d(x.transpose(1, 2).float(), self.weight.float(),+ None if self.bias is None else self.bias.float(),+ groups=self.groups,+ padding=self.kernel_size[0] - 1)[..., :length]+ if self.activation in ("silu", "swish"):+ y = F.silu(y)+ y = y.transpose(1, 2).to(x.dtype)+ if residual is not None:+ y = y + residual+ return y, None++ module.ShortConvolution.forward = shortconv_cpu++ # FusedRMSNormGated is triton-only too. Its kernel computes+ # y = rmsnorm(x) * weight * act(g) with act(g) = sigmoid(g) (or+ # g*sigmoid(g) for swish), per fused_norm_gate.py's kernel body.+ def rmsnorm_gated_cpu(self, x, g, residual=None, prenorm=False,+ residual_in_fp32=False):+ assert residual is None and not prenorm+ xf = x.float()+ y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)+ if self.weight is not None:+ y = y * self.weight.float()+ gf = g.float()+ if self.activation in ("swish", "silu"):+ y = y * gf * torch.sigmoid(gf)+ else:+ y = y * torch.sigmoid(gf)+ return y.to(x.dtype)++ module.FusedRMSNormGated.forward = rmsnorm_gated_cpu+++def build_model(module, config_dict):+ import torch+ config_cls = module.KimiLinearConfig+ config = config_cls(**dict(config_dict, attn_implementation="eager",+ use_cache=False))+ # transformers 4.56 resolves the attention backend from the private+ # attribute; the constructor kwarg alone is not authoritative there.+ config._attn_implementation = "eager"+ torch.manual_seed(0) # init values are all overwritten by the load below+ model = module.KimiLinearForCausalLM(config)+ # KimiLinearModel.__init__ force-overwrites any requested attention+ # implementation with flash_attention_2 (modeling_kimi_linear.py:1110).+ # The MLA forward re-reads self.config._attn_implementation at call time+ # and has a full eager path, so restoring it after construction is+ # effective and touches no vendor math.+ model.config._attn_implementation = "eager"+ model.eval()+ return model+++def to_state_dict(tensors, config_dict):+ """Fixture names -> vendor parameter names, with MXFP4 dequantized."""+ import torch+ kda_heads = config_dict["linear_attn_config"]["num_heads"]+ state, packed = {}, {}+ for name, arr in tensors.items():+ if name.endswith(".weight_packed"):+ packed[name[:-len(".weight_packed")]] = arr+ continue+ if name.endswith(".weight_scale"):+ continue+ state[name] = torch.from_numpy(np.ascontiguousarray(arr))+ for base, pk in packed.items():+ scale = tensors[base + ".weight_scale"]+ state[base + ".weight"] = torch.from_numpy(mxfp4_dequant(pk, scale))+ # The checkpoint stores A_log zero-padded to head_dim; the model wants+ # one entry per KDA head (the engine reads the same first slice).+ for name in list(state):+ if name.endswith("self_attn.A_log"):+ state[name] = state[name][:kda_heads].clone()+ return state+++def forward_argmax(model, ids):+ import torch+ with torch.no_grad():+ logits = model(input_ids=torch.tensor([ids], dtype=torch.long),+ use_cache=False).logits[0]+ return logits+++def case_reference(model, prompt, max_new, min_margin):+ import torch+ sequence = list(prompt)+ margins = []+ for _ in range(max_new):+ logits = forward_argmax(model, sequence)[-1]+ top2 = torch.topk(logits, 2).values+ margins.append(float(top2[0] - top2[1]))+ sequence.append(int(logits.argmax()))+ full_logits = forward_argmax(model, sequence)+ teacher = full_logits.argmax(-1).tolist()+ tf_top2 = torch.topk(full_logits, 2, dim=-1).values+ tf_margins = (tf_top2[:, 0] - tf_top2[:, 1]).tolist()+ worst = min(margins + tf_margins)+ if worst < min_margin:+ raise SystemExit(+ f"logit margin {worst:.4f} below --min-margin {min_margin}: the "+ f"fixture rides a near-tie and token-exactness would be luck, "+ f"not correctness. Re-tune the generator instead of lowering "+ f"the margin.")+ return {+ "prompt_ids": list(prompt),+ "teacher_forcing_ids": teacher,+ "greedy_full_ids": sequence,+ "greedy_new_ids": sequence[len(prompt):],+ "max_new_tokens": max_new,+ "min_logit_margin": round(worst, 4),+ }+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--fixture", default="./kimi_k3_tiny")+ parser.add_argument("--vendor-dir", default=None,+ help="dir with the pinned vendor .py files "+ "(default: <fixture>/.vendor_kimi_k3)")+ parser.add_argument("--output", default=None,+ help="default: <fixture>/ref.json")+ parser.add_argument("--min-margin", type=float, default=0.05)+ args = parser.parse_args()++ fixture = Path(args.fixture)+ vendor_dir = (Path(args.vendor_dir) if args.vendor_dir+ else fixture / ".vendor_kimi_k3")+ output = Path(args.output) if args.output else fixture / "ref.json"++ config_dict = json.loads((fixture / "config.json").read_text())+ tensors = read_safetensors(fixture / "model.safetensors")++ fetch_vendor(vendor_dir)+ module = load_vendor(vendor_dir)+ model = build_model(module, config_dict)+ state = to_state_dict(tensors, config_dict)+ missing, unexpected = model.load_state_dict(state, strict=False)+ # Refuse silence: every fixture tensor must land, and every model weight+ # must be fed. Anything else is a drifted name map, not a detail.+ if missing or unexpected:+ raise SystemExit(f"state dict mismatch:\n missing: {missing}\n"+ f" unexpected: {unexpected}")++ eos = config_dict.get("eos_token_id", 1)+ vocab = config_dict["vocab_size"]+ cases = {+ # mirrors make_deepseek_v4_tiny.py: short, chunk-crossing, long+ "short": ([5, 7, 9, 11, 13, 17, 19, 23], 8),+ "chunk": ([(5 + i * 7) % vocab for i in range(40)], 4),+ "long": ([(5 + i * 11) % vocab for i in range(72)], 4),+ }+ reference = {}+ for name, (prompt, max_new) in cases.items():+ prompt = [t if t != eos else t + 1 for t in prompt]+ reference[name] = case_reference(model, prompt, max_new,+ args.min_margin)+ if eos in reference[name]["greedy_new_ids"]:+ raise SystemExit(f"case {name}: greedy emitted EOS; the fixture "+ f"must keep EOS unattractive")+ print(f"[ref] {name}: prompt {len(prompt)} -> +{max_new}, "+ f"min margin {reference[name]['min_logit_margin']}",+ file=sys.stderr)++ import torch+ import transformers+ output.write_text(json.dumps({+ "schema_version": 1,+ "source": "moonshot-vendor",+ "vendor_repo": VENDOR_REPO,+ "vendor_files": VENDOR_FILES,+ "torch_version": torch.__version__,+ "transformers_version": transformers.__version__,+ "engine_env": {+ # exact-math configuration: dense weights at f32 (the fixture+ # stores f32; default K3_BITS=4 would requantize at load) and+ # float expert matmuls (default K3_IDOT=1 quantizes activations+ # to int8 per group of 32, a deliberate ~0.4%/group speed+ # approximation that is NOT an oracle target)+ "K3_BITS": "32", "K3_MLA_BITS": "32", "K3_HEAD_BITS": "32",+ "K3_IDOT": "0", "COLI_TEMP": "0"},+ "cases": reference,+ }, indent=1) + "\n")+ print(f"[ref] wrote {output}", file=sys.stderr)+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/make_kimi_k3_tiny.py b/c/tools/make_kimi_k3_tiny.pynew file mode 100644index 00000000..68022cd3--- /dev/null+++ b/c/tools/make_kimi_k3_tiny.py@@ -0,0 +1,302 @@+#!/usr/bin/env python3+"""Generate the tiny Kimi K3 checkpoint used by tests/test_kimi_k3_tiny.py.++numpy-only and deterministic: CI regenerates model.safetensors from SEED and+byte-identical weights come out on every platform (NumPy's PCG64 stream and+elementwise float32 arithmetic are platform-stable). The reference tokens in+ref.json are NOT produced here: they come from Moonshot's own modeling code+(tools/make_kimi_k3_ref.py, run offline once, provenance pinned by SHA-256),+so the engine and its oracle never share an author.++The fixture is deliberately the smallest checkpoint that still crosses every+distinct execution path of kimi_k3.c:++ layer 0: KDA (Gated DeltaNet) + dense MLP+ layer 1: gated MLA + dense MLP+ layer 2: KDA + sparse MoE (routed MXFP4 experts)+ layer 3-5: gated MLA + sparse MoE++Two engine contracts the emission must honour byte-for-byte:+ - routed experts are native MXFP4 g32: e2m1 nibble pairs packed two per+ byte plus one e8m0 exponent byte per group of 32. kimi_k3.c checks the+ six per-expert tensor sizes exactly and refuses a container that does+ not match (w1/w3 packed [moe_inter, latent/2], scale [moe_inter,+ latent/32]; w2 packed [latent, moe_inter/2], scale [latent, moe_inter/32]).+ - the loader preads expert bytes raw from recorded fd/offset and leans on+ the back-to-back layout of the six tensors, so their emission order is+ load-bearing (w1 packed, w1 scale, w2 packed, w2 scale, w3 packed,+ w3 scale) and the safetensors writer must preserve insertion order.++No tokenizer.json is written on purpose: without one the engine prints raw+generated token ids on stdout, which is exactly what the token-exact test+wants to parse.++Greedy determinism is engineered, not hoped for: lm_head is a scaled cyclic+shift of embed_tokens, giving every position a wide top-1 logit margin+instead of relying on accidental near-ties between random rows, and the EOS+row is zeroed so fixed-length regressions can detect early truncation. The+margin is then verified, not assumed: make_kimi_k3_ref.py asserts a minimum+top1-top2 logit gap at every emitted position.+"""++import argparse+import json+import struct+import sys+from pathlib import Path++import numpy as np++SEED = 20260823++VOCAB = 256+HIDDEN = 128+LAYERS = 6+FIRST_DENSE = 2 # layers 0..1 use the dense MLP path+DENSE_INTER = 64+HEADS = 4+Q_RANK = 32+KV_RANK = 32+QK_NOPE = 16+QK_ROPE = 8+V_HEAD = 16+EXPERTS = 8+TOPK = 2+MOE_INTER = 32 # engine requires a multiple of 32+LATENT = 32 # engine requires a multiple of 32+SHARED = 1+RES_BLOCK = 2+KDA_HEADS = 2+KDA_HD = 16+CONV_K = 4 # engine requires <= 8+KDA_LAYERS = [1, 3] # 1-indexed, per the engine's config schema+EOS_ID = 1++# One config for two readers. kimi_k3.c consumes the numeric core and+# linear_attn_config.kda_layers; the extra semantic keys (router activation,+# topk method, situ, gated NoPE MLA, full_attn_layers, use_full_rank_gate)+# are ignored by the engine but consumed by Moonshot's KimiLinearConfig, so+# the vendor reference and the engine read the SAME file and cannot drift.+# Values mirror the real Kimi-K3 checkpoint's text_config (the behavior the+# engine hardcodes), with only the dimensions shrunk.+CONFIG = {+ "model_type": "kimi_linear",+ "architectures": ["KimiLinearForCausalLM"],+ "hidden_size": HIDDEN,+ "num_hidden_layers": LAYERS,+ "vocab_size": VOCAB,+ "first_k_dense_replace": FIRST_DENSE,+ "intermediate_size": DENSE_INTER,+ "num_attention_heads": HEADS,+ "num_key_value_heads": HEADS,+ "q_lora_rank": Q_RANK,+ "kv_lora_rank": KV_RANK,+ "qk_nope_head_dim": QK_NOPE,+ "qk_rope_head_dim": QK_ROPE,+ "v_head_dim": V_HEAD,+ "num_experts": EXPERTS,+ "num_experts_per_token": TOPK,+ "moe_intermediate_size": MOE_INTER,+ "routed_expert_hidden_size": LATENT,+ "num_shared_experts": SHARED,+ "attn_res_block_size": RES_BLOCK,+ "hidden_act": "situ",+ "activation_situ_beta": 1.0,+ "activation_situ_linear_beta": 1.0,+ "moe_router_activation_func": "sigmoid",+ "topk_method": "noaux_tc",+ "use_grouped_topk": True,+ "num_expert_group": 1,+ "topk_group": 1,+ "moe_renormalize": True,+ "moe_layer_freq": 1,+ "routed_scaling_factor": 1.0,+ "latent_moe_use_norm": True,+ "mla_use_nope": True,+ "mla_use_output_gate": True,+ "rms_norm_eps": 1e-5,+ "rope_theta": 10000.0,+ "max_position_embeddings": 256,+ "tie_word_embeddings": False,+ "linear_attn_config": {+ "num_heads": KDA_HEADS,+ "head_dim": KDA_HD,+ "short_conv_kernel_size": CONV_K,+ "kda_layers": KDA_LAYERS,+ "full_attn_layers": [x for x in range(1, LAYERS + 1)+ if x not in KDA_LAYERS],+ "use_full_rank_gate": True,+ "gate_lower_bound": -5.0,+ },+ "bos_token_id": 0,+ "eos_token_id": EOS_ID,+}++ST_DTYPE = {np.dtype("float32"): "F32", np.dtype("uint8"): "U8"}+++def build_tensors(rng):+ """Every weight of the tiny model, in emission order."""+ proj = KDA_HEADS * KDA_HD+ qk_head = QK_NOPE + QK_ROPE+ shared_inter = MOE_INTER * SHARED+ tensors = {}++ def put(name, arr):+ assert name not in tensors, name+ tensors[name] = np.ascontiguousarray(arr)++ def normal(*shape, s=0.02):+ return (rng.standard_normal(shape) * s).astype(np.float32)++ def ones(n):+ return np.ones(n, dtype=np.float32)++ def mxfp4(rows, cols):+ """One packed MXFP4 matrix. All 16 e2m1 codes are finite, so any+ nibble pattern is legal; exponents stay near 127 (2^0) to keep the+ dequantized magnitudes tame and never hit the 0xFF NaN code."""+ packed = rng.integers(0, 256, size=(rows, cols // 2), dtype=np.uint8)+ scale = rng.integers(124, 131, size=(rows, cols // 32), dtype=np.uint8)+ return packed, scale++ embed = normal(VOCAB, HIDDEN, s=0.13)+ put("model.embed_tokens.weight", embed)+ put("model.norm.weight", ones(HIDDEN))+ put("model.output_attn_res_norm.weight", ones(HIDDEN))+ put("model.output_attn_res_proj.weight", normal(1, HIDDEN, s=0.1))+ # A scaled cyclic shift of the embedding: token t points firmly at+ # (t+1) % VOCAB, so greedy argmax never rides a near-tie between random+ # rows. EOS is deliberately unattractive (zero row) so every+ # fixed-length regression can detect early truncation.+ lm_head = np.roll(embed, -1, axis=0) * 4.0+ lm_head[EOS_ID] = 0.0+ put("lm_head.weight", lm_head)++ for i in range(LAYERS):+ p = f"model.layers.{i}."+ put(p + "input_layernorm.weight", ones(HIDDEN))+ put(p + "post_attention_layernorm.weight", ones(HIDDEN))+ put(p + "self_attention_res_norm.weight", ones(HIDDEN))+ put(p + "self_attention_res_proj.weight", normal(1, HIDDEN, s=0.1))+ put(p + "mlp_res_norm.weight", ones(HIDDEN))+ put(p + "mlp_res_proj.weight", normal(1, HIDDEN, s=0.1))++ a = p + "self_attn."+ if (i + 1) in KDA_LAYERS: # kda_layers is 1-indexed+ for n in ("q_proj", "k_proj", "v_proj", "g_proj"):+ put(a + n + ".weight", normal(proj, HIDDEN, s=0.05))+ put(a + "o_proj.weight", normal(HIDDEN, proj, s=0.05))+ for n in ("q_conv1d", "k_conv1d", "v_conv1d"):+ put(a + n + ".weight", normal(proj, 1, CONV_K, s=0.3))+ put(a + "f_a_proj.weight", normal(KDA_HD, HIDDEN, s=0.05))+ put(a + "f_b_proj.weight", normal(proj, KDA_HD, s=0.05))+ put(a + "b_proj.weight", normal(KDA_HEADS, HIDDEN, s=0.05))+ put(a + "dt_bias", normal(proj, s=0.1))+ put(a + "o_norm.weight", ones(KDA_HD))+ # stored zero-padded to kda_hd; the engine reads the first+ # kda_heads entries and exponentiates them+ put(a + "A_log", normal(KDA_HD, s=0.1))+ else:+ put(a + "q_a_proj.weight", normal(Q_RANK, HIDDEN, s=0.05))+ put(a + "q_b_proj.weight", normal(HEADS * qk_head, Q_RANK, s=0.05))+ put(a + "kv_a_proj_with_mqa.weight",+ normal(KV_RANK + QK_ROPE, HIDDEN, s=0.05))+ put(a + "kv_b_proj.weight",+ normal(HEADS * (QK_NOPE + V_HEAD), KV_RANK, s=0.05))+ put(a + "o_proj.weight", normal(HIDDEN, HEADS * V_HEAD, s=0.05))+ put(a + "g_proj.weight", normal(HEADS * V_HEAD, HIDDEN, s=0.05))+ put(a + "q_a_layernorm.weight", ones(Q_RANK))+ put(a + "kv_a_layernorm.weight", ones(KV_RANK))++ if i < FIRST_DENSE:+ put(p + "mlp.gate_proj.weight", normal(DENSE_INTER, HIDDEN, s=0.05))+ put(p + "mlp.up_proj.weight", normal(DENSE_INTER, HIDDEN, s=0.05))+ put(p + "mlp.down_proj.weight", normal(HIDDEN, DENSE_INTER, s=0.05))+ else:+ b = p + "block_sparse_moe."+ # closed-form router weights: wide, stable expert preferences per+ # hidden direction, so top-k selection cannot ride a near-tie+ gate = 0.05 * np.sin(+ np.arange(EXPERTS * HIDDEN, dtype=np.float32) * 0.017 + i+ ).reshape(EXPERTS, HIDDEN).astype(np.float32)+ put(b + "gate.weight", gate)+ put(b + "gate.e_score_correction_bias",+ np.linspace(-0.03, 0.03, EXPERTS, dtype=np.float32))+ put(b + "routed_expert_norm.weight", ones(LATENT))+ put(b + "routed_expert_down_proj.weight",+ normal(LATENT, HIDDEN, s=0.05))+ put(b + "routed_expert_up_proj.weight",+ normal(HIDDEN, LATENT, s=0.05))+ put(b + "shared_experts.gate_proj.weight",+ normal(shared_inter, HIDDEN, s=0.05))+ put(b + "shared_experts.up_proj.weight",+ normal(shared_inter, HIDDEN, s=0.05))+ put(b + "shared_experts.down_proj.weight",+ normal(HIDDEN, shared_inter, s=0.05))+ for e in range(EXPERTS):+ q = b + f"experts.{e}."+ w1p, w1s = mxfp4(MOE_INTER, LATENT)+ w2p, w2s = mxfp4(LATENT, MOE_INTER)+ w3p, w3s = mxfp4(MOE_INTER, LATENT)+ # order is load-bearing: the engine preads these six raw and+ # back-to-back from recorded offsets+ put(q + "w1.weight_packed", w1p)+ put(q + "w1.weight_scale", w1s)+ put(q + "w2.weight_packed", w2p)+ put(q + "w2.weight_scale", w2s)+ put(q + "w3.weight_packed", w3p)+ put(q + "w3.weight_scale", w3s)+ return tensors+++def write_safetensors(path, tensors):+ """Hand-rolled writer: preserves dict insertion order, which the expert+ contiguity contract depends on (the safetensors library reorders)."""+ header, offset = {}, 0+ for name, arr in tensors.items():+ header[name] = {+ "dtype": ST_DTYPE[arr.dtype],+ "shape": list(arr.shape),+ "data_offsets": [offset, offset + arr.nbytes],+ }+ offset += arr.nbytes+ blob = json.dumps(header, separators=(",", ":"), sort_keys=False).encode()+ blob += b" " * ((-len(blob)) % 8)+ with open(path, "wb") as fh:+ fh.write(struct.pack("<Q", len(blob)))+ fh.write(blob)+ for arr in tensors.values():+ fh.write(arr.tobytes())+++def main():+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])+ parser.add_argument("--output", default="./kimi_k3_tiny")+ parser.add_argument("--force", action="store_true",+ help="overwrite an existing model.safetensors")+ args = parser.parse_args()++ out = Path(args.output)+ out.mkdir(parents=True, exist_ok=True)+ model_path = out / "model.safetensors"+ if model_path.exists() and not args.force:+ print(f"{model_path} exists; use --force to regenerate",+ file=sys.stderr)+ return 1++ rng = np.random.default_rng(SEED)+ tensors = build_tensors(rng)+ write_safetensors(model_path, tensors)+ (out / "config.json").write_text(json.dumps(CONFIG, indent=2) + "\n")++ total = sum(a.nbytes for a in tensors.values())+ print(f"{out}: {len(tensors)} tensors, {total / 1e6:.1f} MB, "+ f"{LAYERS} layers ({LAYERS - FIRST_DENSE} sparse, "+ f"KDA at {KDA_LAYERS}) x {EXPERTS} experts, latent {LATENT}, "+ f"seed {SEED}")+ return 0+++if __name__ == "__main__":+ sys.exit(main())diff --git a/c/tools/requirements-kimi-k3-ref.txt b/c/tools/requirements-kimi-k3-ref.txtnew file mode 100644index 00000000..7f473708--- /dev/null+++ b/c/tools/requirements-kimi-k3-ref.txt@@ -0,0 +1,10 @@+# maintainer-only: regenerate kimi_k3_tiny/ref.json from Moonshot's own+# modeling code (tools/make_kimi_k3_ref.py). CI never needs these.+# transformers is pinned to the version the vendor code targets+# (OutputRecorder and friends moved in 5.x).+torch+transformers==4.56.2+einops+fla-core+numpy+packagingdiff --git a/c/tools/requirements-kimi-k3-tiny.txt b/c/tools/requirements-kimi-k3-tiny.txtnew file mode 100644index 00000000..4e66e16e--- /dev/null+++ b/c/tools/requirements-kimi-k3-tiny.txt@@ -0,0 +1,2 @@+# tiny K3 fixture generation (CI): deterministic numpy-only weights+numpy>=1.24