Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

Gate the Kimi pre-tokenizer, which nothing was watching

ZacharyZcR3a2978407820
diff --git a/c/Makefile b/c/Makefile
index 56390905..ab808231 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -550,6 +550,9 @@ tests/test_json$(EXE): tests/test_json.c json.h
tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
+tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h
$(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS)
diff --git a/c/tests/test_tok_kimi_tiny.c b/c/tests/test_tok_kimi_tiny.c
new file mode 100644
index 00000000..04b2e9a1
--- /dev/null
+++ b/c/tests/test_tok_kimi_tiny.c
@@ -0,0 +1,93 @@
+/* Kimi (K3) pre-tokenizer against a tiny in-repo vocabulary.
+ *
+ * WHY THIS EXISTS. o200k has had a gate since it was written
+ * (tests/test_tok_o200k.c, 40/40 encode + decode off a 4 KB fixture). The Kimi
+ * family has had none: tests/test_tok_kimi.c needs a real tokenizer.json and a
+ * cases.bin produced by tiktoken, so it has no build rule and has never run in
+ * CI -- c/Makefile lists it among the files that "deliberately have no rule".
+ *
+ * That matters because the two families share one implementation. tok.h's
+ * o2_letters_masked() and pretok_chunk_o2fam() serve o200k and Kimi from the
+ * same lines, differing by a flag; a change made for one silently reshapes the
+ * other, and only the o200k half had anything watching.
+ *
+ * WHAT IS ASSERTED, AND WHAT IS NOT. Not token ids from the real Kimi
+ * tokenizer -- reproducing those needs the actual vocabulary, which is what
+ * made the existing test unrunnable. What is asserted is the pre-tokenizer
+ * BOUNDARY, which is a property of the rules and is what the shared code
+ * decides:
+ *
+ * - a Han run is its own chunk, and does not merge with adjacent Latin
+ * - Han never joins a letter run (it is \p{Lo}; the Kimi classes mask it out)
+ * - rule D takes no '/' tail, unlike o200k
+ *
+ * The fixture makes those boundaries observable: "中文" is one vocabulary entry,
+ * so a correct Han-run split encodes it as ONE id, and any other split falls
+ * back to per-byte ids. The same input through the o200k fixture takes the
+ * other path, so the two are contrasted rather than asserted in isolation.
+ */
+#define _GNU_SOURCE
+#include <stdio.h>
+#include <string.h>
+#include "../tok.h"
+
+static int fails;
+
+static void expect_n(Tok *T, const char *what, const char *text, int want) {
+ int ids[128];
+ int n = tok_encode(T, text, (int)strlen(text), ids, 128);
+ if (n != want) {
+ printf(" FAIL %-28s %-12s got %d ids, want %d:", what, text, n, want);
+ for (int i = 0; i < n; i++) printf(" %d", ids[i]);
+ printf("\n");
+ fails++;
+ } else {
+ printf(" ok %-28s %-12s %d ids\n", what, text, n);
+ }
+}
+
+/* The two families must not agree here -- if they do, the Kimi flag stopped
+ * reaching the splitter and this whole file would pass vacuously. */
+static void expect_differs(Tok *A, Tok *B, const char *text) {
+ int a[128], b[128];
+ int na = tok_encode(A, text, (int)strlen(text), a, 128);
+ int nb = tok_encode(B, text, (int)strlen(text), b, 128);
+ int same = (na == nb) && memcmp(a, b, (size_t)na * sizeof(int)) == 0;
+ if (same) {
+ printf(" FAIL kimi and o200k agree on %-12s (%d ids) -- flag not reaching the splitter\n", text, na);
+ fails++;
+ } else {
+ printf(" ok kimi != o200k on %-12s (%d vs %d ids)\n", text, na, nb);
+ }
+}
+
+int main(void) {
+ Tok K, O;
+ tok_load(&K, "tests/tok_kimi_tiny.json");
+ tok_load(&O, "tests/tok_o200k_tiny.json");
+
+ if (!K.kimi) {
+ printf(" FAIL fixture did not select the Kimi family (kimi=%d)\n", K.kimi);
+ return 1;
+ }
+ if (O.kimi) {
+ printf(" FAIL o200k fixture selected the Kimi family\n");
+ return 1;
+ }
+ printf(" ok family detection: kimi=%d / o200k kimi=%d\n", K.kimi, O.kimi);
+
+ /* Han run is one chunk and hits the vocabulary entry. */
+ expect_n(&K, "Han run is one chunk", "\xe4\xb8\xad\xe6\x96\x87", 1);
+ /* Han does not absorb the Latin that follows: 中文 + abc, not one blob. */
+ expect_n(&K, "Han does not eat Latin", "\xe4\xb8\xad\xe6\x96\x87" "abc", 3);
+ /* A Han codepoint at the far end of the range (U+9FA5) is still a Han run,
+ * so it does not merge with the Latin that follows: one id each. */
+ expect_n(&K, "U+9FA5 splits from Latin", "\xe9\xbe\xa5" "a", 2);
+
+ /* Same inputs must NOT take the o200k path. */
+ expect_differs(&K, &O, "\xe4\xb8\xad\xe6\x96\x87");
+ expect_differs(&K, &O, "\xe4\xb8\xad\xe6\x96\x87" "abc");
+
+ printf(fails ? "test_tok_kimi_tiny: %d failure(s)\n" : "test_tok_kimi_tiny: ok\n", fails);
+ return fails != 0;
+}
diff --git a/c/tests/tok_kimi_tiny.json b/c/tests/tok_kimi_tiny.json
new file mode 100644
index 00000000..5522f8dc
--- /dev/null
+++ b/c/tests/tok_kimi_tiny.json
@@ -0,0 +1,418 @@
+{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 274,
+ "content": "<|endoftext|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 275,
+ "content": "<|message_user|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ }
+ ],
+ "normalizer": null,
+ "pre_tokenizer": {
+ "type": "Sequence",
+ "pretokenizers": [
+ {
+ "type": "Split",
+ "pattern": {
+ "Regex": "[\\p{Han}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"
+ },
+ "behavior": "Isolated",
+ "invert": false
+ },
+ {
+ "type": "ByteLevel",
+ "add_prefix_space": false,
+ "trim_offsets": true,
+ "use_regex": false
+ }
+ ]
+ },
+ "post_processor": null,
+ "decoder": {
+ "type": "ByteLevel",
+ "add_prefix_space": true,
+ "trim_offsets": true,
+ "use_regex": true
+ },
+ "model": {
+ "type": "BPE",
+ "dropout": null,
+ "unk_token": null,
+ "continuing_subword_prefix": null,
+ "end_of_word_suffix": null,
+ "fuse_unk": false,
+ "byte_fallback": false,
+ "ignore_merges": true,
+ "vocab": {
+ "Ā": 0,
+ "ā": 1,
+ "Ă": 2,
+ "ă": 3,
+ "Ą": 4,
+ "ą": 5,
+ "Ć": 6,
+ "ć": 7,
+ "Ĉ": 8,
+ "ĉ": 9,
+ "Ċ": 10,
+ "ċ": 11,
+ "Č": 12,
+ "č": 13,
+ "Ď": 14,
+ "ď": 15,
+ "Đ": 16,
+ "đ": 17,
+ "Ē": 18,
+ "ē": 19,
+ "Ĕ": 20,
+ "ĕ": 21,
+ "Ė": 22,
+ "ė": 23,
+ "Ę": 24,
+ "ę": 25,
+ "Ě": 26,
+ "ě": 27,
+ "Ĝ": 28,
+ "ĝ": 29,
+ "Ğ": 30,
+ "ğ": 31,
+ "Ġ": 32,
+ "!": 33,
+ "\"": 34,
+ "#": 35,
+ "$": 36,
+ "%": 37,
+ "&": 38,
+ "'": 39,
+ "(": 40,
+ ")": 41,
+ "*": 42,
+ "+": 43,
+ ",": 44,
+ "-": 45,
+ ".": 46,
+ "/": 47,
+ "0": 48,
+ "1": 49,
+ "2": 50,
+ "3": 51,
+ "4": 52,
+ "5": 53,
+ "6": 54,
+ "7": 55,
+ "8": 56,
+ "9": 57,
+ ":": 58,
+ ";": 59,
+ "<": 60,
+ "=": 61,
+ ">": 62,
+ "?": 63,
+ "@": 64,
+ "A": 65,
+ "B": 66,
+ "C": 67,
+ "D": 68,
+ "E": 69,
+ "F": 70,
+ "G": 71,
+ "H": 72,
+ "I": 73,
+ "J": 74,
+ "K": 75,
+ "L": 76,
+ "M": 77,
+ "N": 78,
+ "O": 79,
+ "P": 80,
+ "Q": 81,
+ "R": 82,
+ "S": 83,
+ "T": 84,
+ "U": 85,
+ "V": 86,
+ "W": 87,
+ "X": 88,
+ "Y": 89,
+ "Z": 90,
+ "[": 91,
+ "\\": 92,
+ "]": 93,
+ "^": 94,
+ "_": 95,
+ "`": 96,
+ "a": 97,
+ "b": 98,
+ "c": 99,
+ "d": 100,
+ "e": 101,
+ "f": 102,
+ "g": 103,
+ "h": 104,
+ "i": 105,
+ "j": 106,
+ "k": 107,
+ "l": 108,
+ "m": 109,
+ "n": 110,
+ "o": 111,
+ "p": 112,
+ "q": 113,
+ "r": 114,
+ "s": 115,
+ "t": 116,
+ "u": 117,
+ "v": 118,
+ "w": 119,
+ "x": 120,
+ "y": 121,
+ "z": 122,
+ "{": 123,
+ "|": 124,
+ "}": 125,
+ "~": 126,
+ "ġ": 127,
+ "Ģ": 128,
+ "ģ": 129,
+ "Ĥ": 130,
+ "ĥ": 131,
+ "Ħ": 132,
+ "ħ": 133,
+ "Ĩ": 134,
+ "ĩ": 135,
+ "Ī": 136,
+ "ī": 137,
+ "Ĭ": 138,
+ "ĭ": 139,
+ "Į": 140,
+ "į": 141,
+ "İ": 142,
+ "ı": 143,
+ "IJ": 144,
+ "ij": 145,
+ "Ĵ": 146,
+ "ĵ": 147,
+ "Ķ": 148,
+ "ķ": 149,
+ "ĸ": 150,
+ "Ĺ": 151,
+ "ĺ": 152,
+ "Ļ": 153,
+ "ļ": 154,
+ "Ľ": 155,
+ "ľ": 156,
+ "Ŀ": 157,
+ "ŀ": 158,
+ "Ł": 159,
+ "ł": 160,
+ "¡": 161,
+ "¢": 162,
+ "£": 163,
+ "¤": 164,
+ "¥": 165,
+ "¦": 166,
+ "§": 167,
+ "¨": 168,
+ "©": 169,
+ "ª": 170,
+ "«": 171,
+ "¬": 172,
+ "Ń": 173,
+ "®": 174,
+ "¯": 175,
+ "°": 176,
+ "±": 177,
+ "²": 178,
+ "³": 179,
+ "´": 180,
+ "µ": 181,
+ "¶": 182,
+ "·": 183,
+ "¸": 184,
+ "¹": 185,
+ "º": 186,
+ "»": 187,
+ "¼": 188,
+ "½": 189,
+ "¾": 190,
+ "¿": 191,
+ "À": 192,
+ "Á": 193,
+ "Â": 194,
+ "Ã": 195,
+ "Ä": 196,
+ "Å": 197,
+ "Æ": 198,
+ "Ç": 199,
+ "È": 200,
+ "É": 201,
+ "Ê": 202,
+ "Ë": 203,
+ "Ì": 204,
+ "Í": 205,
+ "Î": 206,
+ "Ï": 207,
+ "Ð": 208,
+ "Ñ": 209,
+ "Ò": 210,
+ "Ó": 211,
+ "Ô": 212,
+ "Õ": 213,
+ "Ö": 214,
+ "×": 215,
+ "Ø": 216,
+ "Ù": 217,
+ "Ú": 218,
+ "Û": 219,
+ "Ü": 220,
+ "Ý": 221,
+ "Þ": 222,
+ "ß": 223,
+ "à": 224,
+ "á": 225,
+ "â": 226,
+ "ã": 227,
+ "ä": 228,
+ "å": 229,
+ "æ": 230,
+ "ç": 231,
+ "è": 232,
+ "é": 233,
+ "ê": 234,
+ "ë": 235,
+ "ì": 236,
+ "í": 237,
+ "î": 238,
+ "ï": 239,
+ "ð": 240,
+ "ñ": 241,
+ "ò": 242,
+ "ó": 243,
+ "ô": 244,
+ "õ": 245,
+ "ö": 246,
+ "÷": 247,
+ "ø": 248,
+ "ù": 249,
+ "ú": 250,
+ "û": 251,
+ "ü": 252,
+ "ý": 253,
+ "þ": 254,
+ "ÿ": 255,
+ "he": 256,
+ "ll": 257,
+ "hell": 258,
+ "hello": 259,
+ "Wo": 260,
+ "Wor": 261,
+ "World": 262,
+ "ld": 263,
+ "th": 264,
+ "the": 265,
+ "Ġthe": 266,
+ "12": 267,
+ "123": 268,
+ "ab": 269,
+ "'s": 270,
+ "Ġa": 271,
+ "./": 272,
+ "ĊĊ": 273,
+ "ä¸Ń": 274,
+ "æĸĩ": 275,
+ "ä¸Ńæĸĩ": 276,
+ "æĹ¥æľ¬": 277,
+ "ãģĤ": 278,
+ "é¾¥": 279
+ },
+ "merges": [
+ [
+ "h",
+ "e"
+ ],
+ [
+ "l",
+ "l"
+ ],
+ [
+ "he",
+ "ll"
+ ],
+ [
+ "hell",
+ "o"
+ ],
+ [
+ "W",
+ "o"
+ ],
+ [
+ "Wo",
+ "r"
+ ],
+ [
+ "Wor",
+ "ld"
+ ],
+ [
+ "l",
+ "d"
+ ],
+ [
+ "t",
+ "h"
+ ],
+ [
+ "th",
+ "e"
+ ],
+ [
+ "Ġ",
+ "the"
+ ],
+ [
+ "1",
+ "2"
+ ],
+ [
+ "12",
+ "3"
+ ],
+ [
+ "a",
+ "b"
+ ],
+ [
+ "'",
+ "s"
+ ],
+ [
+ "Ġ",
+ "a"
+ ],
+ [
+ ".",
+ "/"
+ ],
+ [
+ "Ċ",
+ "Ċ"
+ ]
+ ]
+ }
+}
\ No newline at end of file