Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
Gate the Kimi pre-tokenizer, which nothing was watching
ZacharyZcR3a2978407820
diff --git a/c/Makefile b/c/Makefileindex 56390905..ab808231 100644--- a/c/Makefile+++ b/c/Makefile@@ -550,6 +550,9 @@ tests/test_json$(EXE): tests/test_json.c json.htests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)++tests/test_tok_kimi_tiny$(EXE): tests/test_tok_kimi_tiny.c tok.h tok_unicode.h tok_unicode_o200k.h json.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)tests/test_st_pread$(EXE): tests/test_st_pread.c st.h json.h compat.h$(CC) $(CFLAGS) -DST_PREAD_CHUNK=7 $< -o $@ $(LDFLAGS)diff --git a/c/tests/test_tok_kimi_tiny.c b/c/tests/test_tok_kimi_tiny.cnew file mode 100644index 00000000..04b2e9a1--- /dev/null+++ b/c/tests/test_tok_kimi_tiny.c@@ -0,0 +1,93 @@+/* Kimi (K3) pre-tokenizer against a tiny in-repo vocabulary.+ *+ * WHY THIS EXISTS. o200k has had a gate since it was written+ * (tests/test_tok_o200k.c, 40/40 encode + decode off a 4 KB fixture). The Kimi+ * family has had none: tests/test_tok_kimi.c needs a real tokenizer.json and a+ * cases.bin produced by tiktoken, so it has no build rule and has never run in+ * CI -- c/Makefile lists it among the files that "deliberately have no rule".+ *+ * That matters because the two families share one implementation. tok.h's+ * o2_letters_masked() and pretok_chunk_o2fam() serve o200k and Kimi from the+ * same lines, differing by a flag; a change made for one silently reshapes the+ * other, and only the o200k half had anything watching.+ *+ * WHAT IS ASSERTED, AND WHAT IS NOT. Not token ids from the real Kimi+ * tokenizer -- reproducing those needs the actual vocabulary, which is what+ * made the existing test unrunnable. What is asserted is the pre-tokenizer+ * BOUNDARY, which is a property of the rules and is what the shared code+ * decides:+ *+ * - a Han run is its own chunk, and does not merge with adjacent Latin+ * - Han never joins a letter run (it is \p{Lo}; the Kimi classes mask it out)+ * - rule D takes no '/' tail, unlike o200k+ *+ * The fixture makes those boundaries observable: "中文" is one vocabulary entry,+ * so a correct Han-run split encodes it as ONE id, and any other split falls+ * back to per-byte ids. The same input through the o200k fixture takes the+ * other path, so the two are contrasted rather than asserted in isolation.+ */+#define _GNU_SOURCE+#include <stdio.h>+#include <string.h>+#include "../tok.h"++static int fails;++static void expect_n(Tok *T, const char *what, const char *text, int want) {+ int ids[128];+ int n = tok_encode(T, text, (int)strlen(text), ids, 128);+ if (n != want) {+ printf(" FAIL %-28s %-12s got %d ids, want %d:", what, text, n, want);+ for (int i = 0; i < n; i++) printf(" %d", ids[i]);+ printf("\n");+ fails++;+ } else {+ printf(" ok %-28s %-12s %d ids\n", what, text, n);+ }+}++/* The two families must not agree here -- if they do, the Kimi flag stopped+ * reaching the splitter and this whole file would pass vacuously. */+static void expect_differs(Tok *A, Tok *B, const char *text) {+ int a[128], b[128];+ int na = tok_encode(A, text, (int)strlen(text), a, 128);+ int nb = tok_encode(B, text, (int)strlen(text), b, 128);+ int same = (na == nb) && memcmp(a, b, (size_t)na * sizeof(int)) == 0;+ if (same) {+ printf(" FAIL kimi and o200k agree on %-12s (%d ids) -- flag not reaching the splitter\n", text, na);+ fails++;+ } else {+ printf(" ok kimi != o200k on %-12s (%d vs %d ids)\n", text, na, nb);+ }+}++int main(void) {+ Tok K, O;+ tok_load(&K, "tests/tok_kimi_tiny.json");+ tok_load(&O, "tests/tok_o200k_tiny.json");++ if (!K.kimi) {+ printf(" FAIL fixture did not select the Kimi family (kimi=%d)\n", K.kimi);+ return 1;+ }+ if (O.kimi) {+ printf(" FAIL o200k fixture selected the Kimi family\n");+ return 1;+ }+ printf(" ok family detection: kimi=%d / o200k kimi=%d\n", K.kimi, O.kimi);++ /* Han run is one chunk and hits the vocabulary entry. */+ expect_n(&K, "Han run is one chunk", "\xe4\xb8\xad\xe6\x96\x87", 1);+ /* Han does not absorb the Latin that follows: 中文 + abc, not one blob. */+ expect_n(&K, "Han does not eat Latin", "\xe4\xb8\xad\xe6\x96\x87" "abc", 3);+ /* A Han codepoint at the far end of the range (U+9FA5) is still a Han run,+ * so it does not merge with the Latin that follows: one id each. */+ expect_n(&K, "U+9FA5 splits from Latin", "\xe9\xbe\xa5" "a", 2);++ /* Same inputs must NOT take the o200k path. */+ expect_differs(&K, &O, "\xe4\xb8\xad\xe6\x96\x87");+ expect_differs(&K, &O, "\xe4\xb8\xad\xe6\x96\x87" "abc");++ printf(fails ? "test_tok_kimi_tiny: %d failure(s)\n" : "test_tok_kimi_tiny: ok\n", fails);+ return fails != 0;+}diff --git a/c/tests/tok_kimi_tiny.json b/c/tests/tok_kimi_tiny.jsonnew file mode 100644index 00000000..5522f8dc--- /dev/null+++ b/c/tests/tok_kimi_tiny.json@@ -0,0 +1,418 @@+{+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [+ {+ "id": 274,+ "content": "<|endoftext|>",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 275,+ "content": "<|message_user|>",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ }+ ],+ "normalizer": null,+ "pre_tokenizer": {+ "type": "Sequence",+ "pretokenizers": [+ {+ "type": "Split",+ "pattern": {+ "Regex": "[\\p{Han}]+|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}&&[^\\p{Han}]]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"+ },+ "behavior": "Isolated",+ "invert": false+ },+ {+ "type": "ByteLevel",+ "add_prefix_space": false,+ "trim_offsets": true,+ "use_regex": false+ }+ ]+ },+ "post_processor": null,+ "decoder": {+ "type": "ByteLevel",+ "add_prefix_space": true,+ "trim_offsets": true,+ "use_regex": true+ },+ "model": {+ "type": "BPE",+ "dropout": null,+ "unk_token": null,+ "continuing_subword_prefix": null,+ "end_of_word_suffix": null,+ "fuse_unk": false,+ "byte_fallback": false,+ "ignore_merges": true,+ "vocab": {+ "Ā": 0,+ "ā": 1,+ "Ă": 2,+ "ă": 3,+ "Ą": 4,+ "ą": 5,+ "Ć": 6,+ "ć": 7,+ "Ĉ": 8,+ "ĉ": 9,+ "Ċ": 10,+ "ċ": 11,+ "Č": 12,+ "č": 13,+ "Ď": 14,+ "ď": 15,+ "Đ": 16,+ "đ": 17,+ "Ē": 18,+ "ē": 19,+ "Ĕ": 20,+ "ĕ": 21,+ "Ė": 22,+ "ė": 23,+ "Ę": 24,+ "ę": 25,+ "Ě": 26,+ "ě": 27,+ "Ĝ": 28,+ "ĝ": 29,+ "Ğ": 30,+ "ğ": 31,+ "Ġ": 32,+ "!": 33,+ "\"": 34,+ "#": 35,+ "$": 36,+ "%": 37,+ "&": 38,+ "'": 39,+ "(": 40,+ ")": 41,+ "*": 42,+ "+": 43,+ ",": 44,+ "-": 45,+ ".": 46,+ "/": 47,+ "0": 48,+ "1": 49,+ "2": 50,+ "3": 51,+ "4": 52,+ "5": 53,+ "6": 54,+ "7": 55,+ "8": 56,+ "9": 57,+ ":": 58,+ ";": 59,+ "<": 60,+ "=": 61,+ ">": 62,+ "?": 63,+ "@": 64,+ "A": 65,+ "B": 66,+ "C": 67,+ "D": 68,+ "E": 69,+ "F": 70,+ "G": 71,+ "H": 72,+ "I": 73,+ "J": 74,+ "K": 75,+ "L": 76,+ "M": 77,+ "N": 78,+ "O": 79,+ "P": 80,+ "Q": 81,+ "R": 82,+ "S": 83,+ "T": 84,+ "U": 85,+ "V": 86,+ "W": 87,+ "X": 88,+ "Y": 89,+ "Z": 90,+ "[": 91,+ "\\": 92,+ "]": 93,+ "^": 94,+ "_": 95,+ "`": 96,+ "a": 97,+ "b": 98,+ "c": 99,+ "d": 100,+ "e": 101,+ "f": 102,+ "g": 103,+ "h": 104,+ "i": 105,+ "j": 106,+ "k": 107,+ "l": 108,+ "m": 109,+ "n": 110,+ "o": 111,+ "p": 112,+ "q": 113,+ "r": 114,+ "s": 115,+ "t": 116,+ "u": 117,+ "v": 118,+ "w": 119,+ "x": 120,+ "y": 121,+ "z": 122,+ "{": 123,+ "|": 124,+ "}": 125,+ "~": 126,+ "ġ": 127,+ "Ģ": 128,+ "ģ": 129,+ "Ĥ": 130,+ "ĥ": 131,+ "Ħ": 132,+ "ħ": 133,+ "Ĩ": 134,+ "ĩ": 135,+ "Ī": 136,+ "ī": 137,+ "Ĭ": 138,+ "ĭ": 139,+ "Į": 140,+ "į": 141,+ "İ": 142,+ "ı": 143,+ "IJ": 144,+ "ij": 145,+ "Ĵ": 146,+ "ĵ": 147,+ "Ķ": 148,+ "ķ": 149,+ "ĸ": 150,+ "Ĺ": 151,+ "ĺ": 152,+ "Ļ": 153,+ "ļ": 154,+ "Ľ": 155,+ "ľ": 156,+ "Ŀ": 157,+ "ŀ": 158,+ "Ł": 159,+ "ł": 160,+ "¡": 161,+ "¢": 162,+ "£": 163,+ "¤": 164,+ "¥": 165,+ "¦": 166,+ "§": 167,+ "¨": 168,+ "©": 169,+ "ª": 170,+ "«": 171,+ "¬": 172,+ "Ń": 173,+ "®": 174,+ "¯": 175,+ "°": 176,+ "±": 177,+ "²": 178,+ "³": 179,+ "´": 180,+ "µ": 181,+ "¶": 182,+ "·": 183,+ "¸": 184,+ "¹": 185,+ "º": 186,+ "»": 187,+ "¼": 188,+ "½": 189,+ "¾": 190,+ "¿": 191,+ "À": 192,+ "Á": 193,+ "Â": 194,+ "Ã": 195,+ "Ä": 196,+ "Å": 197,+ "Æ": 198,+ "Ç": 199,+ "È": 200,+ "É": 201,+ "Ê": 202,+ "Ë": 203,+ "Ì": 204,+ "Í": 205,+ "Î": 206,+ "Ï": 207,+ "Ð": 208,+ "Ñ": 209,+ "Ò": 210,+ "Ó": 211,+ "Ô": 212,+ "Õ": 213,+ "Ö": 214,+ "×": 215,+ "Ø": 216,+ "Ù": 217,+ "Ú": 218,+ "Û": 219,+ "Ü": 220,+ "Ý": 221,+ "Þ": 222,+ "ß": 223,+ "à": 224,+ "á": 225,+ "â": 226,+ "ã": 227,+ "ä": 228,+ "å": 229,+ "æ": 230,+ "ç": 231,+ "è": 232,+ "é": 233,+ "ê": 234,+ "ë": 235,+ "ì": 236,+ "í": 237,+ "î": 238,+ "ï": 239,+ "ð": 240,+ "ñ": 241,+ "ò": 242,+ "ó": 243,+ "ô": 244,+ "õ": 245,+ "ö": 246,+ "÷": 247,+ "ø": 248,+ "ù": 249,+ "ú": 250,+ "û": 251,+ "ü": 252,+ "ý": 253,+ "þ": 254,+ "ÿ": 255,+ "he": 256,+ "ll": 257,+ "hell": 258,+ "hello": 259,+ "Wo": 260,+ "Wor": 261,+ "World": 262,+ "ld": 263,+ "th": 264,+ "the": 265,+ "Ġthe": 266,+ "12": 267,+ "123": 268,+ "ab": 269,+ "'s": 270,+ "Ġa": 271,+ "./": 272,+ "ĊĊ": 273,+ "ä¸Ń": 274,+ "æĸĩ": 275,+ "ä¸Ńæĸĩ": 276,+ "æĹ¥æľ¬": 277,+ "ãģĤ": 278,+ "é¾¥": 279+ },+ "merges": [+ [+ "h",+ "e"+ ],+ [+ "l",+ "l"+ ],+ [+ "he",+ "ll"+ ],+ [+ "hell",+ "o"+ ],+ [+ "W",+ "o"+ ],+ [+ "Wo",+ "r"+ ],+ [+ "Wor",+ "ld"+ ],+ [+ "l",+ "d"+ ],+ [+ "t",+ "h"+ ],+ [+ "th",+ "e"+ ],+ [+ "Ġ",+ "the"+ ],+ [+ "1",+ "2"+ ],+ [+ "12",+ "3"+ ],+ [+ "a",+ "b"+ ],+ [+ "'",+ "s"+ ],+ [+ "Ġ",+ "a"+ ],+ [+ ".",+ "/"+ ],+ [+ "Ċ",+ "Ċ"+ ]+ ]+ }+}\ No newline at end of file