Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

test(k3): tiny fixture + independent vendor oracle for Kimi K3

JustVugg03df9d86e5e2
diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml
index fd050494..7eb49655 100644
--- a/.github/workflows/check.yml
+++ b/.github/workflows/check.yml
@@ -59,6 +59,22 @@ jobs:
name: colibri-windows-exe
path: c/colibri.exe
+ k3-tiny:
+ name: Linux (generated tiny K3 vendor oracle)
+ runs-on: ubuntu-latest
+ timeout-minutes: 20
+ steps:
+ - uses: actions/checkout@v4
+ - uses: actions/setup-python@v5
+ with:
+ python-version: '3.13'
+ cache: pip
+ cache-dependency-path: c/tools/requirements-kimi-k3-tiny.txt
+ - name: Install pinned tiny-fixture dependencies
+ run: python -m pip install -r c/tools/requirements-kimi-k3-tiny.txt
+ - name: Generate fixture and run vendor oracle
+ run: make -C c kimi-k3-tiny-check
+
v4-tiny:
name: Linux (generated tiny V4 target oracle)
runs-on: ubuntu-latest
diff --git a/.gitignore b/.gitignore
index bd86311d..b46b8a44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -137,3 +137,6 @@ c/*.obj
# compiled objects
*.o
+
+# vendor modeling cache for the tiny K3 oracle (maintainer tool)
+c/kimi_k3_tiny/.vendor_kimi_k3/
diff --git a/c/Makefile b/c/Makefile
index 48d712d2..d221d37c 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -606,6 +606,20 @@ endif
all: colibri$(EXE)
+# Tiny Kimi K3 fixture + vendor oracle. The generator is numpy-only and
+# deterministic; ref.json is committed and comes from Moonshot's own modeling
+# code (tools/make_kimi_k3_ref.py, maintainer-run, SHA-256-pinned), so CI
+# needs neither torch nor network. kimi_k3 builds on every platform, so
+# unlike the V4 targets this needs no support gate.
+.PHONY: kimi-k3-tiny-generate kimi-k3-tiny-check
+kimi-k3-tiny-generate:
+ $(PYTHON) tools/make_kimi_k3_tiny.py --output ./kimi_k3_tiny --force
+
+kimi-k3-tiny-check: kimi-k3-tiny-generate kimi_k3$(EXE)
+ $(PYTHON) tests/test_kimi_k3_tiny.py \
+ --binary ./kimi_k3$(EXE) \
+ --fixture ./kimi_k3_tiny
+
.PHONY: deepseek-v4 deepseek-v4-oracle deepseek-v4-tiny-generate \
deepseek-v4-tiny-check deepseek-v4-clean
ifeq ($(COLI_V4_SUPPORTED),1)
diff --git a/c/kimi_k3_tiny/config.json b/c/kimi_k3_tiny/config.json
new file mode 100644
index 00000000..d42f6bce
--- /dev/null
+++ b/c/kimi_k3_tiny/config.json
@@ -0,0 +1,61 @@
+{
+ "model_type": "kimi_linear",
+ "architectures": [
+ "KimiLinearForCausalLM"
+ ],
+ "hidden_size": 128,
+ "num_hidden_layers": 6,
+ "vocab_size": 256,
+ "first_k_dense_replace": 2,
+ "intermediate_size": 64,
+ "num_attention_heads": 4,
+ "num_key_value_heads": 4,
+ "q_lora_rank": 32,
+ "kv_lora_rank": 32,
+ "qk_nope_head_dim": 16,
+ "qk_rope_head_dim": 8,
+ "v_head_dim": 16,
+ "num_experts": 8,
+ "num_experts_per_token": 2,
+ "moe_intermediate_size": 32,
+ "routed_expert_hidden_size": 32,
+ "num_shared_experts": 1,
+ "attn_res_block_size": 2,
+ "hidden_act": "situ",
+ "activation_situ_beta": 1.0,
+ "activation_situ_linear_beta": 1.0,
+ "moe_router_activation_func": "sigmoid",
+ "topk_method": "noaux_tc",
+ "use_grouped_topk": true,
+ "num_expert_group": 1,
+ "topk_group": 1,
+ "moe_renormalize": true,
+ "moe_layer_freq": 1,
+ "routed_scaling_factor": 1.0,
+ "latent_moe_use_norm": true,
+ "mla_use_nope": true,
+ "mla_use_output_gate": true,
+ "rms_norm_eps": 1e-05,
+ "rope_theta": 10000.0,
+ "max_position_embeddings": 256,
+ "tie_word_embeddings": false,
+ "linear_attn_config": {
+ "num_heads": 2,
+ "head_dim": 16,
+ "short_conv_kernel_size": 4,
+ "kda_layers": [
+ 1,
+ 3
+ ],
+ "full_attn_layers": [
+ 2,
+ 4,
+ 5,
+ 6
+ ],
+ "use_full_rank_gate": true,
+ "gate_lower_bound": -5.0
+ },
+ "bos_token_id": 0,
+ "eos_token_id": 1
+}
diff --git a/c/kimi_k3_tiny/ref.json b/c/kimi_k3_tiny/ref.json
new file mode 100644
index 00000000..d2fe6d34
--- /dev/null
+++ b/c/kimi_k3_tiny/ref.json
@@ -0,0 +1,464 @@
+{
+ "schema_version": 1,
+ "source": "moonshot-vendor",
+ "vendor_repo": "moonshotai/Kimi-K3",
+ "vendor_files": {
+ "configuration_kimi_k3.py": "735eb9ebe593e17d231e08e1df7f7be9b5ee0e079f511aa201f9572077b416ae",
+ "modeling_kimi_linear.py": "9e3564c70ac21854ce5a090cc946c5dc76b70d1050ef50840449181a20fff44a"
+ },
+ "torch_version": "2.12.1+cu130",
+ "transformers_version": "4.56.2",
+ "engine_env": {
+ "K3_BITS": "32",
+ "K3_MLA_BITS": "32",
+ "K3_HEAD_BITS": "32",
+ "K3_IDOT": "0",
+ "COLI_TEMP": "0"
+ },
+ "cases": {
+ "short": {
+ "prompt_ids": [
+ 5,
+ 7,
+ 9,
+ 11,
+ 13,
+ 17,
+ 19,
+ 23
+ ],
+ "teacher_forcing_ids": [
+ 4,
+ 6,
+ 219,
+ 10,
+ 69,
+ 201,
+ 18,
+ 78,
+ 157,
+ 198,
+ 33,
+ 205,
+ 110,
+ 109,
+ 108,
+ 107
+ ],
+ "greedy_full_ids": [
+ 5,
+ 7,
+ 9,
+ 11,
+ 13,
+ 17,
+ 19,
+ 23,
+ 78,
+ 157,
+ 198,
+ 33,
+ 205,
+ 110,
+ 109,
+ 108
+ ],
+ "greedy_new_ids": [
+ 78,
+ 157,
+ 198,
+ 33,
+ 205,
+ 110,
+ 109,
+ 108
+ ],
+ "max_new_tokens": 8,
+ "min_logit_margin": 0.2905
+ },
+ "chunk": {
+ "prompt_ids": [
+ 5,
+ 12,
+ 19,
+ 26,
+ 33,
+ 40,
+ 47,
+ 54,
+ 61,
+ 68,
+ 75,
+ 82,
+ 89,
+ 96,
+ 103,
+ 110,
+ 117,
+ 124,
+ 131,
+ 138,
+ 145,
+ 152,
+ 159,
+ 166,
+ 173,
+ 180,
+ 187,
+ 194,
+ 201,
+ 208,
+ 215,
+ 222,
+ 229,
+ 236,
+ 243,
+ 250,
+ 2,
+ 8,
+ 15,
+ 22
+ ],
+ "teacher_forcing_ids": [
+ 4,
+ 111,
+ 18,
+ 246,
+ 250,
+ 155,
+ 251,
+ 53,
+ 157,
+ 67,
+ 74,
+ 81,
+ 88,
+ 194,
+ 235,
+ 109,
+ 118,
+ 68,
+ 91,
+ 70,
+ 144,
+ 10,
+ 202,
+ 165,
+ 200,
+ 0,
+ 186,
+ 157,
+ 3,
+ 10,
+ 122,
+ 56,
+ 111,
+ 4,
+ 34,
+ 203,
+ 127,
+ 135,
+ 254,
+ 146,
+ 145,
+ 206,
+ 205,
+ 81
+ ],
+ "greedy_full_ids": [
+ 5,
+ 12,
+ 19,
+ 26,
+ 33,
+ 40,
+ 47,
+ 54,
+ 61,
+ 68,
+ 75,
+ 82,
+ 89,
+ 96,
+ 103,
+ 110,
+ 117,
+ 124,
+ 131,
+ 138,
+ 145,
+ 152,
+ 159,
+ 166,
+ 173,
+ 180,
+ 187,
+ 194,
+ 201,
+ 208,
+ 215,
+ 222,
+ 229,
+ 236,
+ 243,
+ 250,
+ 2,
+ 8,
+ 15,
+ 22,
+ 146,
+ 145,
+ 206,
+ 205
+ ],
+ "greedy_new_ids": [
+ 146,
+ 145,
+ 206,
+ 205
+ ],
+ "max_new_tokens": 4,
+ "min_logit_margin": 0.0274
+ },
+ "long": {
+ "prompt_ids": [
+ 5,
+ 16,
+ 27,
+ 38,
+ 49,
+ 60,
+ 71,
+ 82,
+ 93,
+ 104,
+ 115,
+ 126,
+ 137,
+ 148,
+ 159,
+ 170,
+ 181,
+ 192,
+ 203,
+ 214,
+ 225,
+ 236,
+ 247,
+ 2,
+ 13,
+ 24,
+ 35,
+ 46,
+ 57,
+ 68,
+ 79,
+ 90,
+ 101,
+ 112,
+ 123,
+ 134,
+ 145,
+ 156,
+ 167,
+ 178,
+ 189,
+ 200,
+ 211,
+ 222,
+ 233,
+ 244,
+ 255,
+ 10,
+ 21,
+ 32,
+ 43,
+ 54,
+ 65,
+ 76,
+ 87,
+ 98,
+ 109,
+ 120,
+ 131,
+ 142,
+ 153,
+ 164,
+ 175,
+ 186,
+ 197,
+ 208,
+ 219,
+ 230,
+ 241,
+ 252,
+ 7,
+ 18
+ ],
+ "teacher_forcing_ids": [
+ 4,
+ 144,
+ 111,
+ 37,
+ 255,
+ 244,
+ 70,
+ 206,
+ 202,
+ 103,
+ 171,
+ 14,
+ 165,
+ 147,
+ 118,
+ 133,
+ 180,
+ 6,
+ 154,
+ 213,
+ 97,
+ 226,
+ 246,
+ 28,
+ 184,
+ 3,
+ 34,
+ 141,
+ 56,
+ 189,
+ 97,
+ 81,
+ 66,
+ 105,
+ 116,
+ 133,
+ 48,
+ 155,
+ 166,
+ 184,
+ 163,
+ 199,
+ 210,
+ 170,
+ 61,
+ 243,
+ 210,
+ 235,
+ 20,
+ 31,
+ 186,
+ 53,
+ 254,
+ 75,
+ 86,
+ 97,
+ 65,
+ 119,
+ 27,
+ 142,
+ 254,
+ 177,
+ 202,
+ 185,
+ 4,
+ 185,
+ 218,
+ 251,
+ 54,
+ 251,
+ 80,
+ 11,
+ 10,
+ 9,
+ 202,
+ 201
+ ],
+ "greedy_full_ids": [
+ 5,
+ 16,
+ 27,
+ 38,
+ 49,
+ 60,
+ 71,
+ 82,
+ 93,
+ 104,
+ 115,
+ 126,
+ 137,
+ 148,
+ 159,
+ 170,
+ 181,
+ 192,
+ 203,
+ 214,
+ 225,
+ 236,
+ 247,
+ 2,
+ 13,
+ 24,
+ 35,
+ 46,
+ 57,
+ 68,
+ 79,
+ 90,
+ 101,
+ 112,
+ 123,
+ 134,
+ 145,
+ 156,
+ 167,
+ 178,
+ 189,
+ 200,
+ 211,
+ 222,
+ 233,
+ 244,
+ 255,
+ 10,
+ 21,
+ 32,
+ 43,
+ 54,
+ 65,
+ 76,
+ 87,
+ 98,
+ 109,
+ 120,
+ 131,
+ 142,
+ 153,
+ 164,
+ 175,
+ 186,
+ 197,
+ 208,
+ 219,
+ 230,
+ 241,
+ 252,
+ 7,
+ 18,
+ 11,
+ 10,
+ 9,
+ 202
+ ],
+ "greedy_new_ids": [
+ 11,
+ 10,
+ 9,
+ 202
+ ],
+ "max_new_tokens": 4,
+ "min_logit_margin": 0.0253
+ }
+ }
+}
diff --git a/c/tests/test_kimi_k3_tiny.py b/c/tests/test_kimi_k3_tiny.py
new file mode 100644
index 00000000..15bfed53
--- /dev/null
+++ b/c/tests/test_kimi_k3_tiny.py
@@ -0,0 +1,170 @@
+#!/usr/bin/env python3
+"""Token-exact checks for the Kimi K3 engine against an independent oracle.
+
+Dependency-free (stdlib only). The fixture is generated by
+tools/make_kimi_k3_tiny.py (numpy, deterministic); the expected tokens in
+ref.json come from Moonshot's own modeling code via tools/make_kimi_k3_ref.py
+(run offline by a maintainer, vendor files pinned by SHA-256), so the engine
+and its oracle never share an author.
+
+What is checked, all token-exact (argmax, no tolerances):
+ - greedy generation for three cases (short; 40 tokens crossing the default
+ 32-token prefill chunk; 72 tokens crossing it twice)
+ - teacher forcing at every position via the K3_LOGITS dump
+ - determinism (two identical runs, identical tokens)
+ - the oracle BITES: one corrupted expert scale byte per layer-2 expert must
+ change the output (guards against a test that silently compares nothing)
+
+The engine runs in its exact-math configuration (ref.json's engine_env:
+K3_BITS/K3_MLA_BITS/K3_HEAD_BITS=32 because the fixture stores f32 dense
+weights, K3_IDOT=0 because int8-activation expert matmuls are a documented
+~0.4%/group speed approximation, not an oracle target). The fixture ships
+no tokenizer.json on purpose: the engine then prints raw token ids on
+stdout, which is what this test parses.
+"""
+
+import argparse
+import array
+import json
+import os
+import shutil
+import struct
+import subprocess
+import sys
+import tempfile
+from pathlib import Path
+
+
+def run_engine(binary, fixture, prompt_ids, max_new, env, extra_env=None):
+ run_env = dict(os.environ, **env)
+ if extra_env:
+ run_env.update(extra_env)
+ result = subprocess.run(
+ [binary, str(fixture), "--ids", " ".join(map(str, prompt_ids)),
+ "--ngen", str(max_new)],
+ env=run_env, capture_output=True, text=True, timeout=600)
+ if result.returncode != 0:
+ raise AssertionError(
+ f"engine exited {result.returncode}:\n{result.stderr[-2000:]}")
+ # without a tokenizer.json the engine prints "id id id ... TUNE decode:"
+ return [int(token) for token in result.stdout.split("TUNE")[0].split()]
+
+
+def check_greedy(binary, fixture, reference):
+ for name, case in reference["cases"].items():
+ got = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ want = case["greedy_new_ids"]
+ if got != want:
+ raise AssertionError(
+ f"greedy mismatch in case '{name}': engine {got} vs "
+ f"reference {want}")
+ print(f"ok greedy {name}: {len(want)}/{len(want)} tokens")
+
+
+def check_determinism(binary, fixture, reference):
+ case = reference["cases"]["short"]
+ first = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ second = run_engine(binary, fixture, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ if first != second:
+ raise AssertionError(f"non-deterministic greedy: {first} vs {second}")
+ print("ok determinism: two identical runs, identical tokens")
+
+
+def check_teacher_forcing(binary, fixture, reference, vocab):
+ for name, case in reference["cases"].items():
+ full = case["greedy_full_ids"]
+ with tempfile.NamedTemporaryFile(suffix=".f32", delete=False) as tmp:
+ logits_path = tmp.name
+ try:
+ run_engine(binary, fixture, full, 0, reference["engine_env"],
+ extra_env={"K3_LOGITS": logits_path})
+ raw = array.array("f")
+ with open(logits_path, "rb") as fh:
+ raw.frombytes(fh.read())
+ finally:
+ os.unlink(logits_path)
+ if len(raw) != len(full) * vocab:
+ raise AssertionError(
+ f"case '{name}': K3_LOGITS dumped {len(raw)} floats, "
+ f"expected {len(full) * vocab}")
+ got = []
+ for position in range(len(full)):
+ row = raw[position * vocab:(position + 1) * vocab]
+ got.append(max(range(vocab), key=row.__getitem__))
+ want = case["teacher_forcing_ids"]
+ if got != want:
+ first_bad = next(i for i, (a, b) in enumerate(zip(got, want))
+ if a != b)
+ raise AssertionError(
+ f"teacher forcing mismatch in case '{name}' at position "
+ f"{first_bad}: engine {got[first_bad]} vs reference "
+ f"{want[first_bad]}")
+ print(f"ok teacher forcing {name}: {len(want)}/{len(want)} positions")
+
+
+def check_oracle_bites(binary, fixture, reference, experts):
+ """One corrupted scale byte per layer-2 expert must change the output.
+ Without this, a comparator bug that accepts anything would pass silently."""
+ corrupt = Path(tempfile.mkdtemp(prefix="kimi-k3-tiny-bite-")) / "fixture"
+ shutil.copytree(fixture, corrupt)
+ try:
+ model = corrupt / "model.safetensors"
+ raw = bytearray(model.read_bytes())
+ header_len = struct.unpack("<Q", bytes(raw[:8]))[0]
+ header = json.loads(bytes(raw[8:8 + header_len]))
+ for expert in range(experts):
+ name = (f"model.layers.2.block_sparse_moe.experts.{expert}"
+ f".w1.weight_scale")
+ lo, _ = header[name]["data_offsets"]
+ offset = 8 + header_len + lo
+ raw[offset] = (raw[offset] + 3) % 256 # 2^3 on one group
+ model.write_bytes(bytes(raw))
+ diverged = 0
+ for case in reference["cases"].values():
+ got = run_engine(binary, corrupt, case["prompt_ids"],
+ case["max_new_tokens"], reference["engine_env"])
+ if got != case["greedy_new_ids"]:
+ diverged += 1
+ if not diverged:
+ raise AssertionError(
+ "corrupted expert scales produced identical tokens in every "
+ "case: the oracle does not bite")
+ print(f"ok oracle bites: corrupted expert scales diverge in "
+ f"{diverged}/{len(reference['cases'])} cases")
+ finally:
+ shutil.rmtree(corrupt.parent)
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--binary", required=True)
+ parser.add_argument("--fixture", required=True)
+ args = parser.parse_args()
+
+ fixture = Path(args.fixture)
+ reference = json.loads((fixture / "ref.json").read_text())
+ if reference.get("source") != "moonshot-vendor":
+ raise AssertionError(
+ "ref.json must come from the independent vendor implementation "
+ f"(source={reference.get('source')!r})")
+ config = json.loads((fixture / "config.json").read_text())
+ if (fixture / "tokenizer.json").exists():
+ raise AssertionError(
+ "fixture must not ship tokenizer.json: raw-id stdout is the "
+ "comparison channel")
+
+ check_greedy(args.binary, fixture, reference)
+ check_determinism(args.binary, fixture, reference)
+ check_teacher_forcing(args.binary, fixture, reference,
+ config["vocab_size"])
+ check_oracle_bites(args.binary, fixture, reference,
+ config["num_experts"])
+ print("kimi-k3-tiny-check: all checks passed")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/make_kimi_k3_ref.py b/c/tools/make_kimi_k3_ref.py
new file mode 100644
index 00000000..211f1a8c
--- /dev/null
+++ b/c/tools/make_kimi_k3_ref.py
@@ -0,0 +1,372 @@
+#!/usr/bin/env python3
+"""Produce ref.json for the tiny Kimi K3 fixture from Moonshot's own code.
+
+Maintainer tool, run offline once per fixture change; CI never executes it
+(tests/test_kimi_k3_tiny.py only reads the committed ref.json). The point is
+independence: the reference tokens come from the vendor's published
+implementation (modeling_kimi_linear.py on the Kimi-K3 Hugging Face repo,
+pinned below by SHA-256), so the engine and its oracle never share an author.
+A shared misunderstanding cannot pass both.
+
+Parity is by construction, the same discipline as make_deepseek_v4_tiny.py:
+- dense weights are stored f32 in the fixture and the engine is tested with
+ K3_BITS=32 / K3_MLA_BITS=32 / K3_HEAD_BITS=32, so both sides compute from
+ identical bytes;
+- routed experts are stored packed MXFP4 g32; the engine decodes them
+ natively while this script dequantizes THE SAME BYTES (low nibble = even
+ column, w = e2m1 * 2^(scale-127)) into the vendor model's f32 Linear
+ weights. Either way the mathematical weights are equal, so token-level
+ argmax must agree wherever the logit margin exceeds float noise -- and the
+ generator engineers wide margins, which this script VERIFIES per emitted
+ position instead of assuming (--min-margin, refuse on violation).
+
+Requires: torch (CPU is fine), einops, numpy. Network only if the vendor
+files are not already cached next to --vendor-dir.
+"""
+
+import argparse
+import hashlib
+import importlib
+import json
+import struct
+import sys
+import urllib.request
+from pathlib import Path
+
+import numpy as np
+
+VENDOR_REPO = "moonshotai/Kimi-K3"
+# Pinned provenance: refuse to run against silently changed vendor code.
+VENDOR_FILES = {
+ "configuration_kimi_k3.py":
+ "735eb9ebe593e17d231e08e1df7f7be9b5ee0e079f511aa201f9572077b416ae",
+ "modeling_kimi_linear.py":
+ "9e3564c70ac21854ce5a090cc946c5dc76b70d1050ef50840449181a20fff44a",
+}
+
+MX4_LUT = np.array([0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0,
+ -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0],
+ dtype=np.float32)
+
+
+def read_safetensors(path):
+ with open(path, "rb") as fh:
+ header_len = struct.unpack("<Q", fh.read(8))[0]
+ header = json.loads(fh.read(header_len))
+ base = 8 + header_len
+ raw = fh.read()
+ dtypes = {"F32": np.float32, "U8": np.uint8}
+ out = {}
+ for name, meta in header.items():
+ if name == "__metadata__":
+ continue
+ lo, hi = meta["data_offsets"]
+ dtype = dtypes[meta["dtype"]]
+ arr = np.frombuffer(raw, dtype=dtype,
+ count=(hi - lo) // np.dtype(dtype).itemsize,
+ offset=lo)
+ out[name] = arr.reshape(meta["shape"]).copy()
+ return out
+
+
+def mxfp4_dequant(packed, scale):
+ """Exact peer of quant.h:matmul_mxfp4's decode: packed [O, I/2] with the
+ LOW nibble on the even column, scale [O, I/32] as 2^(s-127) per group."""
+ rows, half = packed.shape
+ cols = half * 2
+ codes = np.empty((rows, cols), dtype=np.uint8)
+ codes[:, 0::2] = packed & 0x0F
+ codes[:, 1::2] = packed >> 4
+ values = MX4_LUT[codes]
+ exponents = np.ldexp(np.float32(1.0), scale.astype(np.int32) - 127)
+ return (values.reshape(rows, cols // 32, 32) *
+ exponents[:, :, None]).reshape(rows, cols).astype(np.float32)
+
+
+def fetch_vendor(vendor_dir):
+ vendor_dir.mkdir(parents=True, exist_ok=True)
+ for name, expected in VENDOR_FILES.items():
+ path = vendor_dir / name
+ if not path.exists():
+ url = f"https://huggingface.co/{VENDOR_REPO}/raw/main/{name}"
+ print(f"[ref] downloading {url}", file=sys.stderr)
+ with urllib.request.urlopen(url, timeout=60) as resp:
+ path.write_bytes(resp.read())
+ digest = hashlib.sha256(path.read_bytes()).hexdigest()
+ if digest != expected:
+ raise SystemExit(
+ f"{name}: sha256 {digest} does not match the pinned "
+ f"{expected}; the vendor file changed upstream. Review it, "
+ f"then update VENDOR_FILES deliberately.")
+ init = vendor_dir / "__init__.py"
+ if not init.exists():
+ init.write_text("")
+
+
+def load_vendor(vendor_dir):
+ """Load the vendor files as a synthetic package, so their relative
+ imports work regardless of the directory's (dotted) name."""
+ import importlib.machinery
+ import importlib.util
+ pkg_name = "kimi_k3_vendor"
+ spec = importlib.machinery.ModuleSpec(pkg_name, None, is_package=True)
+ package = importlib.util.module_from_spec(spec)
+ package.__path__ = [str(vendor_dir)]
+ sys.modules[pkg_name] = package
+
+ def load(stem):
+ file_spec = importlib.util.spec_from_file_location(
+ f"{pkg_name}.{stem}", vendor_dir / f"{stem}.py")
+ module = importlib.util.module_from_spec(file_spec)
+ sys.modules[f"{pkg_name}.{stem}"] = module
+ file_spec.loader.exec_module(module)
+ return module
+
+ load("configuration_kimi_k3")
+ module = load("modeling_kimi_linear")
+ patch_kda_cpu(module)
+ return module
+
+
+def patch_kda_cpu(module):
+ """The vendor calls fla's chunk_kda/fused_recurrent_kda, which are Triton
+ kernels and need a GPU. Reroute both to fla's OWN naive torch
+ implementation (fla/ops/kda/naive.py) plus fla's OWN naive gate
+ (fla/ops/kda/gate.py) -- the reference implementations fla tests its
+ kernels against. No colibri-authored math enters the oracle: the
+ preprocessing below mirrors exactly the use_*_in_kernel flags the vendor
+ passes, using fla's published formulas."""
+ import torch
+ import torch.nn.functional as F
+ from fla.ops.kda.naive import naive_recurrent_kda
+ from fla.ops.kda.gate import naive_kda_gate, naive_kda_lowerbound_gate
+
+ def kda_cpu(q, k, v, g, beta, A_log=None, dt_bias=None, scale=None,
+ initial_state=None, output_final_state=False,
+ use_qk_l2norm_in_kernel=False, use_gate_in_kernel=False,
+ use_beta_sigmoid_in_kernel=False, safe_gate=False,
+ lower_bound=None, transpose_state_layout=False,
+ cu_seqlens=None, **_ignored):
+ # our reference forward is single-sequence, cache-free
+ assert cu_seqlens is None, "packed sequences not expected here"
+ assert initial_state is None, "cache-free reference only"
+ if use_qk_l2norm_in_kernel:
+ q = F.normalize(q.float(), p=2, dim=-1)
+ k = F.normalize(k.float(), p=2, dim=-1)
+ if use_gate_in_kernel:
+ if safe_gate or lower_bound is not None:
+ g = naive_kda_lowerbound_gate(g, A_log, dt_bias,
+ lower_bound
+ if lower_bound is not None
+ else -5.0)
+ else:
+ g = naive_kda_gate(g, A_log, dt_bias)
+ if use_beta_sigmoid_in_kernel:
+ beta = torch.sigmoid(beta.float())
+ o, state = naive_recurrent_kda(
+ q=q, k=k, v=v, g=g, beta=beta, scale=scale,
+ initial_state=None, output_final_state=output_final_state)
+ return o, state
+
+ module.chunk_kda = kda_cpu
+ module.fused_recurrent_kda = kda_cpu
+
+ # fla's ShortConvolution supports only the triton/cuda backends. Its
+ # documented semantics are a causal depthwise conv (nn.Conv1d with
+ # groups=D, padding=K-1, output truncated to T) plus SiLU. Re-express
+ # exactly that in stock torch.
+ def shortconv_cpu(self, x, residual=None, mask=None, cache=None,
+ output_final_state=False, cu_seqlens=None,
+ chunk_indices=None, **_ignored):
+ assert mask is None and cache is None and cu_seqlens is None
+ assert not output_final_state, "cache-free reference only"
+ length = x.shape[1]
+ y = F.conv1d(x.transpose(1, 2).float(), self.weight.float(),
+ None if self.bias is None else self.bias.float(),
+ groups=self.groups,
+ padding=self.kernel_size[0] - 1)[..., :length]
+ if self.activation in ("silu", "swish"):
+ y = F.silu(y)
+ y = y.transpose(1, 2).to(x.dtype)
+ if residual is not None:
+ y = y + residual
+ return y, None
+
+ module.ShortConvolution.forward = shortconv_cpu
+
+ # FusedRMSNormGated is triton-only too. Its kernel computes
+ # y = rmsnorm(x) * weight * act(g) with act(g) = sigmoid(g) (or
+ # g*sigmoid(g) for swish), per fused_norm_gate.py's kernel body.
+ def rmsnorm_gated_cpu(self, x, g, residual=None, prenorm=False,
+ residual_in_fp32=False):
+ assert residual is None and not prenorm
+ xf = x.float()
+ y = xf * torch.rsqrt(xf.pow(2).mean(-1, keepdim=True) + self.eps)
+ if self.weight is not None:
+ y = y * self.weight.float()
+ gf = g.float()
+ if self.activation in ("swish", "silu"):
+ y = y * gf * torch.sigmoid(gf)
+ else:
+ y = y * torch.sigmoid(gf)
+ return y.to(x.dtype)
+
+ module.FusedRMSNormGated.forward = rmsnorm_gated_cpu
+
+
+def build_model(module, config_dict):
+ import torch
+ config_cls = module.KimiLinearConfig
+ config = config_cls(**dict(config_dict, attn_implementation="eager",
+ use_cache=False))
+ # transformers 4.56 resolves the attention backend from the private
+ # attribute; the constructor kwarg alone is not authoritative there.
+ config._attn_implementation = "eager"
+ torch.manual_seed(0) # init values are all overwritten by the load below
+ model = module.KimiLinearForCausalLM(config)
+ # KimiLinearModel.__init__ force-overwrites any requested attention
+ # implementation with flash_attention_2 (modeling_kimi_linear.py:1110).
+ # The MLA forward re-reads self.config._attn_implementation at call time
+ # and has a full eager path, so restoring it after construction is
+ # effective and touches no vendor math.
+ model.config._attn_implementation = "eager"
+ model.eval()
+ return model
+
+
+def to_state_dict(tensors, config_dict):
+ """Fixture names -> vendor parameter names, with MXFP4 dequantized."""
+ import torch
+ kda_heads = config_dict["linear_attn_config"]["num_heads"]
+ state, packed = {}, {}
+ for name, arr in tensors.items():
+ if name.endswith(".weight_packed"):
+ packed[name[:-len(".weight_packed")]] = arr
+ continue
+ if name.endswith(".weight_scale"):
+ continue
+ state[name] = torch.from_numpy(np.ascontiguousarray(arr))
+ for base, pk in packed.items():
+ scale = tensors[base + ".weight_scale"]
+ state[base + ".weight"] = torch.from_numpy(mxfp4_dequant(pk, scale))
+ # The checkpoint stores A_log zero-padded to head_dim; the model wants
+ # one entry per KDA head (the engine reads the same first slice).
+ for name in list(state):
+ if name.endswith("self_attn.A_log"):
+ state[name] = state[name][:kda_heads].clone()
+ return state
+
+
+def forward_argmax(model, ids):
+ import torch
+ with torch.no_grad():
+ logits = model(input_ids=torch.tensor([ids], dtype=torch.long),
+ use_cache=False).logits[0]
+ return logits
+
+
+def case_reference(model, prompt, max_new, min_margin):
+ import torch
+ sequence = list(prompt)
+ margins = []
+ for _ in range(max_new):
+ logits = forward_argmax(model, sequence)[-1]
+ top2 = torch.topk(logits, 2).values
+ margins.append(float(top2[0] - top2[1]))
+ sequence.append(int(logits.argmax()))
+ full_logits = forward_argmax(model, sequence)
+ teacher = full_logits.argmax(-1).tolist()
+ tf_top2 = torch.topk(full_logits, 2, dim=-1).values
+ tf_margins = (tf_top2[:, 0] - tf_top2[:, 1]).tolist()
+ worst = min(margins + tf_margins)
+ if worst < min_margin:
+ raise SystemExit(
+ f"logit margin {worst:.4f} below --min-margin {min_margin}: the "
+ f"fixture rides a near-tie and token-exactness would be luck, "
+ f"not correctness. Re-tune the generator instead of lowering "
+ f"the margin.")
+ return {
+ "prompt_ids": list(prompt),
+ "teacher_forcing_ids": teacher,
+ "greedy_full_ids": sequence,
+ "greedy_new_ids": sequence[len(prompt):],
+ "max_new_tokens": max_new,
+ "min_logit_margin": round(worst, 4),
+ }
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--fixture", default="./kimi_k3_tiny")
+ parser.add_argument("--vendor-dir", default=None,
+ help="dir with the pinned vendor .py files "
+ "(default: <fixture>/.vendor_kimi_k3)")
+ parser.add_argument("--output", default=None,
+ help="default: <fixture>/ref.json")
+ parser.add_argument("--min-margin", type=float, default=0.05)
+ args = parser.parse_args()
+
+ fixture = Path(args.fixture)
+ vendor_dir = (Path(args.vendor_dir) if args.vendor_dir
+ else fixture / ".vendor_kimi_k3")
+ output = Path(args.output) if args.output else fixture / "ref.json"
+
+ config_dict = json.loads((fixture / "config.json").read_text())
+ tensors = read_safetensors(fixture / "model.safetensors")
+
+ fetch_vendor(vendor_dir)
+ module = load_vendor(vendor_dir)
+ model = build_model(module, config_dict)
+ state = to_state_dict(tensors, config_dict)
+ missing, unexpected = model.load_state_dict(state, strict=False)
+ # Refuse silence: every fixture tensor must land, and every model weight
+ # must be fed. Anything else is a drifted name map, not a detail.
+ if missing or unexpected:
+ raise SystemExit(f"state dict mismatch:\n missing: {missing}\n"
+ f" unexpected: {unexpected}")
+
+ eos = config_dict.get("eos_token_id", 1)
+ vocab = config_dict["vocab_size"]
+ cases = {
+ # mirrors make_deepseek_v4_tiny.py: short, chunk-crossing, long
+ "short": ([5, 7, 9, 11, 13, 17, 19, 23], 8),
+ "chunk": ([(5 + i * 7) % vocab for i in range(40)], 4),
+ "long": ([(5 + i * 11) % vocab for i in range(72)], 4),
+ }
+ reference = {}
+ for name, (prompt, max_new) in cases.items():
+ prompt = [t if t != eos else t + 1 for t in prompt]
+ reference[name] = case_reference(model, prompt, max_new,
+ args.min_margin)
+ if eos in reference[name]["greedy_new_ids"]:
+ raise SystemExit(f"case {name}: greedy emitted EOS; the fixture "
+ f"must keep EOS unattractive")
+ print(f"[ref] {name}: prompt {len(prompt)} -> +{max_new}, "
+ f"min margin {reference[name]['min_logit_margin']}",
+ file=sys.stderr)
+
+ import torch
+ import transformers
+ output.write_text(json.dumps({
+ "schema_version": 1,
+ "source": "moonshot-vendor",
+ "vendor_repo": VENDOR_REPO,
+ "vendor_files": VENDOR_FILES,
+ "torch_version": torch.__version__,
+ "transformers_version": transformers.__version__,
+ "engine_env": {
+ # exact-math configuration: dense weights at f32 (the fixture
+ # stores f32; default K3_BITS=4 would requantize at load) and
+ # float expert matmuls (default K3_IDOT=1 quantizes activations
+ # to int8 per group of 32, a deliberate ~0.4%/group speed
+ # approximation that is NOT an oracle target)
+ "K3_BITS": "32", "K3_MLA_BITS": "32", "K3_HEAD_BITS": "32",
+ "K3_IDOT": "0", "COLI_TEMP": "0"},
+ "cases": reference,
+ }, indent=1) + "\n")
+ print(f"[ref] wrote {output}", file=sys.stderr)
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/make_kimi_k3_tiny.py b/c/tools/make_kimi_k3_tiny.py
new file mode 100644
index 00000000..68022cd3
--- /dev/null
+++ b/c/tools/make_kimi_k3_tiny.py
@@ -0,0 +1,302 @@
+#!/usr/bin/env python3
+"""Generate the tiny Kimi K3 checkpoint used by tests/test_kimi_k3_tiny.py.
+
+numpy-only and deterministic: CI regenerates model.safetensors from SEED and
+byte-identical weights come out on every platform (NumPy's PCG64 stream and
+elementwise float32 arithmetic are platform-stable). The reference tokens in
+ref.json are NOT produced here: they come from Moonshot's own modeling code
+(tools/make_kimi_k3_ref.py, run offline once, provenance pinned by SHA-256),
+so the engine and its oracle never share an author.
+
+The fixture is deliberately the smallest checkpoint that still crosses every
+distinct execution path of kimi_k3.c:
+
+ layer 0: KDA (Gated DeltaNet) + dense MLP
+ layer 1: gated MLA + dense MLP
+ layer 2: KDA + sparse MoE (routed MXFP4 experts)
+ layer 3-5: gated MLA + sparse MoE
+
+Two engine contracts the emission must honour byte-for-byte:
+ - routed experts are native MXFP4 g32: e2m1 nibble pairs packed two per
+ byte plus one e8m0 exponent byte per group of 32. kimi_k3.c checks the
+ six per-expert tensor sizes exactly and refuses a container that does
+ not match (w1/w3 packed [moe_inter, latent/2], scale [moe_inter,
+ latent/32]; w2 packed [latent, moe_inter/2], scale [latent, moe_inter/32]).
+ - the loader preads expert bytes raw from recorded fd/offset and leans on
+ the back-to-back layout of the six tensors, so their emission order is
+ load-bearing (w1 packed, w1 scale, w2 packed, w2 scale, w3 packed,
+ w3 scale) and the safetensors writer must preserve insertion order.
+
+No tokenizer.json is written on purpose: without one the engine prints raw
+generated token ids on stdout, which is exactly what the token-exact test
+wants to parse.
+
+Greedy determinism is engineered, not hoped for: lm_head is a scaled cyclic
+shift of embed_tokens, giving every position a wide top-1 logit margin
+instead of relying on accidental near-ties between random rows, and the EOS
+row is zeroed so fixed-length regressions can detect early truncation. The
+margin is then verified, not assumed: make_kimi_k3_ref.py asserts a minimum
+top1-top2 logit gap at every emitted position.
+"""
+
+import argparse
+import json
+import struct
+import sys
+from pathlib import Path
+
+import numpy as np
+
+SEED = 20260823
+
+VOCAB = 256
+HIDDEN = 128
+LAYERS = 6
+FIRST_DENSE = 2 # layers 0..1 use the dense MLP path
+DENSE_INTER = 64
+HEADS = 4
+Q_RANK = 32
+KV_RANK = 32
+QK_NOPE = 16
+QK_ROPE = 8
+V_HEAD = 16
+EXPERTS = 8
+TOPK = 2
+MOE_INTER = 32 # engine requires a multiple of 32
+LATENT = 32 # engine requires a multiple of 32
+SHARED = 1
+RES_BLOCK = 2
+KDA_HEADS = 2
+KDA_HD = 16
+CONV_K = 4 # engine requires <= 8
+KDA_LAYERS = [1, 3] # 1-indexed, per the engine's config schema
+EOS_ID = 1
+
+# One config for two readers. kimi_k3.c consumes the numeric core and
+# linear_attn_config.kda_layers; the extra semantic keys (router activation,
+# topk method, situ, gated NoPE MLA, full_attn_layers, use_full_rank_gate)
+# are ignored by the engine but consumed by Moonshot's KimiLinearConfig, so
+# the vendor reference and the engine read the SAME file and cannot drift.
+# Values mirror the real Kimi-K3 checkpoint's text_config (the behavior the
+# engine hardcodes), with only the dimensions shrunk.
+CONFIG = {
+ "model_type": "kimi_linear",
+ "architectures": ["KimiLinearForCausalLM"],
+ "hidden_size": HIDDEN,
+ "num_hidden_layers": LAYERS,
+ "vocab_size": VOCAB,
+ "first_k_dense_replace": FIRST_DENSE,
+ "intermediate_size": DENSE_INTER,
+ "num_attention_heads": HEADS,
+ "num_key_value_heads": HEADS,
+ "q_lora_rank": Q_RANK,
+ "kv_lora_rank": KV_RANK,
+ "qk_nope_head_dim": QK_NOPE,
+ "qk_rope_head_dim": QK_ROPE,
+ "v_head_dim": V_HEAD,
+ "num_experts": EXPERTS,
+ "num_experts_per_token": TOPK,
+ "moe_intermediate_size": MOE_INTER,
+ "routed_expert_hidden_size": LATENT,
+ "num_shared_experts": SHARED,
+ "attn_res_block_size": RES_BLOCK,
+ "hidden_act": "situ",
+ "activation_situ_beta": 1.0,
+ "activation_situ_linear_beta": 1.0,
+ "moe_router_activation_func": "sigmoid",
+ "topk_method": "noaux_tc",
+ "use_grouped_topk": True,
+ "num_expert_group": 1,
+ "topk_group": 1,
+ "moe_renormalize": True,
+ "moe_layer_freq": 1,
+ "routed_scaling_factor": 1.0,
+ "latent_moe_use_norm": True,
+ "mla_use_nope": True,
+ "mla_use_output_gate": True,
+ "rms_norm_eps": 1e-5,
+ "rope_theta": 10000.0,
+ "max_position_embeddings": 256,
+ "tie_word_embeddings": False,
+ "linear_attn_config": {
+ "num_heads": KDA_HEADS,
+ "head_dim": KDA_HD,
+ "short_conv_kernel_size": CONV_K,
+ "kda_layers": KDA_LAYERS,
+ "full_attn_layers": [x for x in range(1, LAYERS + 1)
+ if x not in KDA_LAYERS],
+ "use_full_rank_gate": True,
+ "gate_lower_bound": -5.0,
+ },
+ "bos_token_id": 0,
+ "eos_token_id": EOS_ID,
+}
+
+ST_DTYPE = {np.dtype("float32"): "F32", np.dtype("uint8"): "U8"}
+
+
+def build_tensors(rng):
+ """Every weight of the tiny model, in emission order."""
+ proj = KDA_HEADS * KDA_HD
+ qk_head = QK_NOPE + QK_ROPE
+ shared_inter = MOE_INTER * SHARED
+ tensors = {}
+
+ def put(name, arr):
+ assert name not in tensors, name
+ tensors[name] = np.ascontiguousarray(arr)
+
+ def normal(*shape, s=0.02):
+ return (rng.standard_normal(shape) * s).astype(np.float32)
+
+ def ones(n):
+ return np.ones(n, dtype=np.float32)
+
+ def mxfp4(rows, cols):
+ """One packed MXFP4 matrix. All 16 e2m1 codes are finite, so any
+ nibble pattern is legal; exponents stay near 127 (2^0) to keep the
+ dequantized magnitudes tame and never hit the 0xFF NaN code."""
+ packed = rng.integers(0, 256, size=(rows, cols // 2), dtype=np.uint8)
+ scale = rng.integers(124, 131, size=(rows, cols // 32), dtype=np.uint8)
+ return packed, scale
+
+ embed = normal(VOCAB, HIDDEN, s=0.13)
+ put("model.embed_tokens.weight", embed)
+ put("model.norm.weight", ones(HIDDEN))
+ put("model.output_attn_res_norm.weight", ones(HIDDEN))
+ put("model.output_attn_res_proj.weight", normal(1, HIDDEN, s=0.1))
+ # A scaled cyclic shift of the embedding: token t points firmly at
+ # (t+1) % VOCAB, so greedy argmax never rides a near-tie between random
+ # rows. EOS is deliberately unattractive (zero row) so every
+ # fixed-length regression can detect early truncation.
+ lm_head = np.roll(embed, -1, axis=0) * 4.0
+ lm_head[EOS_ID] = 0.0
+ put("lm_head.weight", lm_head)
+
+ for i in range(LAYERS):
+ p = f"model.layers.{i}."
+ put(p + "input_layernorm.weight", ones(HIDDEN))
+ put(p + "post_attention_layernorm.weight", ones(HIDDEN))
+ put(p + "self_attention_res_norm.weight", ones(HIDDEN))
+ put(p + "self_attention_res_proj.weight", normal(1, HIDDEN, s=0.1))
+ put(p + "mlp_res_norm.weight", ones(HIDDEN))
+ put(p + "mlp_res_proj.weight", normal(1, HIDDEN, s=0.1))
+
+ a = p + "self_attn."
+ if (i + 1) in KDA_LAYERS: # kda_layers is 1-indexed
+ for n in ("q_proj", "k_proj", "v_proj", "g_proj"):
+ put(a + n + ".weight", normal(proj, HIDDEN, s=0.05))
+ put(a + "o_proj.weight", normal(HIDDEN, proj, s=0.05))
+ for n in ("q_conv1d", "k_conv1d", "v_conv1d"):
+ put(a + n + ".weight", normal(proj, 1, CONV_K, s=0.3))
+ put(a + "f_a_proj.weight", normal(KDA_HD, HIDDEN, s=0.05))
+ put(a + "f_b_proj.weight", normal(proj, KDA_HD, s=0.05))
+ put(a + "b_proj.weight", normal(KDA_HEADS, HIDDEN, s=0.05))
+ put(a + "dt_bias", normal(proj, s=0.1))
+ put(a + "o_norm.weight", ones(KDA_HD))
+ # stored zero-padded to kda_hd; the engine reads the first
+ # kda_heads entries and exponentiates them
+ put(a + "A_log", normal(KDA_HD, s=0.1))
+ else:
+ put(a + "q_a_proj.weight", normal(Q_RANK, HIDDEN, s=0.05))
+ put(a + "q_b_proj.weight", normal(HEADS * qk_head, Q_RANK, s=0.05))
+ put(a + "kv_a_proj_with_mqa.weight",
+ normal(KV_RANK + QK_ROPE, HIDDEN, s=0.05))
+ put(a + "kv_b_proj.weight",
+ normal(HEADS * (QK_NOPE + V_HEAD), KV_RANK, s=0.05))
+ put(a + "o_proj.weight", normal(HIDDEN, HEADS * V_HEAD, s=0.05))
+ put(a + "g_proj.weight", normal(HEADS * V_HEAD, HIDDEN, s=0.05))
+ put(a + "q_a_layernorm.weight", ones(Q_RANK))
+ put(a + "kv_a_layernorm.weight", ones(KV_RANK))
+
+ if i < FIRST_DENSE:
+ put(p + "mlp.gate_proj.weight", normal(DENSE_INTER, HIDDEN, s=0.05))
+ put(p + "mlp.up_proj.weight", normal(DENSE_INTER, HIDDEN, s=0.05))
+ put(p + "mlp.down_proj.weight", normal(HIDDEN, DENSE_INTER, s=0.05))
+ else:
+ b = p + "block_sparse_moe."
+ # closed-form router weights: wide, stable expert preferences per
+ # hidden direction, so top-k selection cannot ride a near-tie
+ gate = 0.05 * np.sin(
+ np.arange(EXPERTS * HIDDEN, dtype=np.float32) * 0.017 + i
+ ).reshape(EXPERTS, HIDDEN).astype(np.float32)
+ put(b + "gate.weight", gate)
+ put(b + "gate.e_score_correction_bias",
+ np.linspace(-0.03, 0.03, EXPERTS, dtype=np.float32))
+ put(b + "routed_expert_norm.weight", ones(LATENT))
+ put(b + "routed_expert_down_proj.weight",
+ normal(LATENT, HIDDEN, s=0.05))
+ put(b + "routed_expert_up_proj.weight",
+ normal(HIDDEN, LATENT, s=0.05))
+ put(b + "shared_experts.gate_proj.weight",
+ normal(shared_inter, HIDDEN, s=0.05))
+ put(b + "shared_experts.up_proj.weight",
+ normal(shared_inter, HIDDEN, s=0.05))
+ put(b + "shared_experts.down_proj.weight",
+ normal(HIDDEN, shared_inter, s=0.05))
+ for e in range(EXPERTS):
+ q = b + f"experts.{e}."
+ w1p, w1s = mxfp4(MOE_INTER, LATENT)
+ w2p, w2s = mxfp4(LATENT, MOE_INTER)
+ w3p, w3s = mxfp4(MOE_INTER, LATENT)
+ # order is load-bearing: the engine preads these six raw and
+ # back-to-back from recorded offsets
+ put(q + "w1.weight_packed", w1p)
+ put(q + "w1.weight_scale", w1s)
+ put(q + "w2.weight_packed", w2p)
+ put(q + "w2.weight_scale", w2s)
+ put(q + "w3.weight_packed", w3p)
+ put(q + "w3.weight_scale", w3s)
+ return tensors
+
+
+def write_safetensors(path, tensors):
+ """Hand-rolled writer: preserves dict insertion order, which the expert
+ contiguity contract depends on (the safetensors library reorders)."""
+ header, offset = {}, 0
+ for name, arr in tensors.items():
+ header[name] = {
+ "dtype": ST_DTYPE[arr.dtype],
+ "shape": list(arr.shape),
+ "data_offsets": [offset, offset + arr.nbytes],
+ }
+ offset += arr.nbytes
+ blob = json.dumps(header, separators=(",", ":"), sort_keys=False).encode()
+ blob += b" " * ((-len(blob)) % 8)
+ with open(path, "wb") as fh:
+ fh.write(struct.pack("<Q", len(blob)))
+ fh.write(blob)
+ for arr in tensors.values():
+ fh.write(arr.tobytes())
+
+
+def main():
+ parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
+ parser.add_argument("--output", default="./kimi_k3_tiny")
+ parser.add_argument("--force", action="store_true",
+ help="overwrite an existing model.safetensors")
+ args = parser.parse_args()
+
+ out = Path(args.output)
+ out.mkdir(parents=True, exist_ok=True)
+ model_path = out / "model.safetensors"
+ if model_path.exists() and not args.force:
+ print(f"{model_path} exists; use --force to regenerate",
+ file=sys.stderr)
+ return 1
+
+ rng = np.random.default_rng(SEED)
+ tensors = build_tensors(rng)
+ write_safetensors(model_path, tensors)
+ (out / "config.json").write_text(json.dumps(CONFIG, indent=2) + "\n")
+
+ total = sum(a.nbytes for a in tensors.values())
+ print(f"{out}: {len(tensors)} tensors, {total / 1e6:.1f} MB, "
+ f"{LAYERS} layers ({LAYERS - FIRST_DENSE} sparse, "
+ f"KDA at {KDA_LAYERS}) x {EXPERTS} experts, latent {LATENT}, "
+ f"seed {SEED}")
+ return 0
+
+
+if __name__ == "__main__":
+ sys.exit(main())
diff --git a/c/tools/requirements-kimi-k3-ref.txt b/c/tools/requirements-kimi-k3-ref.txt
new file mode 100644
index 00000000..7f473708
--- /dev/null
+++ b/c/tools/requirements-kimi-k3-ref.txt
@@ -0,0 +1,10 @@
+# maintainer-only: regenerate kimi_k3_tiny/ref.json from Moonshot's own
+# modeling code (tools/make_kimi_k3_ref.py). CI never needs these.
+# transformers is pinned to the version the vendor code targets
+# (OutputRecorder and friends moved in 5.x).
+torch
+transformers==4.56.2
+einops
+fla-core
+numpy
+packaging
diff --git a/c/tools/requirements-kimi-k3-tiny.txt b/c/tools/requirements-kimi-k3-tiny.txt
new file mode 100644
index 00000000..4e66e16e
--- /dev/null
+++ b/c/tools/requirements-kimi-k3-tiny.txt
@@ -0,0 +1,2 @@
+# tiny K3 fixture generation (CI): deterministic numpy-only weights
+numpy>=1.24