Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tests: o200k tokenizer coverage, no model download
Nicholas Beerbower9e2d41567c5e
diff --git a/c/Makefile b/c/Makefileindex 27902cb2..9887c74f 100644--- a/c/Makefile+++ b/c/Makefile@@ -166,7 +166,7 @@ elsePYTHON ?= python3endifCUDA_OBJ =-TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)+TEST_BINS = tests/test_json$(EXE) tests/test_st$(EXE) tests/test_tier$(EXE) tests/test_grammar$(EXE) tests/test_schema_gbnf$(EXE) tests/test_decode_batch$(EXE) tests/test_idot$(EXE) tests/test_i4_grouped$(EXE) tests/test_stops$(EXE) tests/test_tok_o200k$(EXE) tests/test_kv_alloc$(EXE) tests/test_i4_acc512$(EXE) tests/test_compat_direct$(EXE)ifneq (,$(LINUX))TEST_BINS += tests/test_uring$(EXE)endif@@ -293,6 +293,9 @@ iobench$(EXE): iobench.c compat.htests/test_json$(EXE): tests/test_json.c json.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_st$(EXE): tests/test_st.c st.h json.h compat.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/tests/test_tok_o200k.c b/c/tests/test_tok_o200k.cnew file mode 100644index 00000000..b86bda90--- /dev/null+++ b/c/tests/test_tok_o200k.c@@ -0,0 +1,61 @@+/* o200k pre-tokenizer validation against HF-tokenizers-generated expectations.+ * Self-contained for the test-c harness: loads tests/tok_o200k_tiny.json (a+ * synthetic byte-level BPE whose Split regex is the o200k pattern — a few KB,+ * no model download) and scores tests/tok_o200k_cases.txt, whose expected ids+ * were produced by HF `tokenizers` on the same file. Guards the case-aware+ * letter matcher, contractions, digit groups, the [\r\n/]* punctuation tail,+ * whitespace branches, and added-token atomicity; round-trips every case.+ * The cl100k path is untouched by construction (dispatch requires \p{Lu} in+ * the tokenizer's own Split pattern) and stays covered by the GLM oracle. */+#define _GNU_SOURCE+#include "../tok.h"++int main(void) {+ Tok T;+ tok_load(&T, "tests/tok_o200k_tiny.json");+ if (!T.o200k) { fprintf(stderr, "test_tok_o200k: o200k pattern not detected\n"); return 1; }+ FILE *f = fopen("tests/tok_o200k_cases.txt", "rb");+ if (!f) { perror("tests/tok_o200k_cases.txt"); return 1; }+ char *line = NULL; size_t cap = 0; ssize_t nr;+ int pass = 0, tot = 0, dpass = 0;+ while ((nr = getline(&line, &cap, f)) >= 0) {+ if (nr > 0 && line[nr-1] == '\n') line[--nr] = 0;+ if (nr == 0) continue;+ char *tab = strchr(line, '\t'); if (!tab) continue;+ *tab = 0;+ const char *text = line, *idstr = tab + 1;+ char tbuf[4096]; int tn = 0;+ for (const char *q = text; *q && tn < 4095; q++) {+ if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; }+ else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; }+ else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; }+ else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; }+ else tbuf[tn++] = *q;+ }+ tbuf[tn] = 0;+ int exp[512], ne = 0;+ for (const char *q = idstr; *q; ) {+ while (*q == ',' || *q == ' ') q++;+ if (!*q) break;+ exp[ne++] = atoi(q);+ while (*q && *q != ',') q++;+ }+ int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512);+ int ok = (ng == ne);+ for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]);+ tot++; if (ok) pass++;+ char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191);+ int drt = (dn == tn) && !memcmp(dec, tbuf, tn);+ if (drt) dpass++;+ if (!ok || !drt) {+ fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne);+ for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]);+ fprintf(stderr, "\n got(%d):", ng);+ for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]);+ fprintf(stderr, "\n decode_ok=%d\n", drt);+ }+ }+ fclose(f);+ printf("test_tok_o200k: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot);+ return (pass == tot && dpass == tot) ? 0 : 2;+}diff --git a/c/tests/tok_o200k_cases.txt b/c/tests/tok_o200k_cases.txtnew file mode 100644index 00000000..38e37fa5--- /dev/null+++ b/c/tests/tok_o200k_cases.txt@@ -0,0 +1,40 @@+hello world 259,32,119,111,114,263+HelloWorld 72,101,257,111,262+XMLHttpRequest 88,77,76,72,116,116,112,82,101,113,117,101,115,116+helloWORLDhello 259,87,79,82,76,68,259+dog's 100,111,103,270+DON'T 68,79,78,39,84+don't 100,111,110,39,116+I'll've 73,39,257,39,118,101+O'Brien 79,39,66,114,105,101,110+the theatre 265,32,116,256,97,116,114,101+12345 268,52,53+a1b22c333d4444 97,49,98,50,50,99,51,51,51,100,52,52,52,52+3.14 51,46,49,52+http://x.com/a/b 104,116,116,112,58,47,47,120,46,99,111,109,47,97,47,98+path/to/file 112,97,264,47,116,111,47,102,105,108,101+a//b///c 97,47,47,98,47,47,47,99+!!\r\n//x 33,33,13,10,47,47,120+one\ntwo\r\nthree 111,110,101,10,116,119,111,13,10,264,114,101,101+ \n x 32,32,10,32,32,120+ 32,32,32+a b c 97,32,32,98,32,32,32,99+tab\there 116,269,9,256,114,101+Café 67,97,102,101,204,129+naiveBayes 110,97,105,118,101,66,97,121,101,115+Éclair 195,137,99,108,97,105,114+北京大学 229,140,151,228,186,172,229,164,167,229,173,166+ΑΒαβ 206,145,206,146,206,177,206,178+Иван 208,152,208,178,208,176,208,189+ẞßscharf 225,186,158,195,159,115,99,104,97,114,102+i̇stanbul 105,204,135,115,116,97,110,98,117,108+hello<|endoftext|>world 259,274,119,111,114,263+<|message_user|>hi 275,104,105+mixedCASEand123 109,105,120,101,100,67,65,83,69,97,110,100,268+'s 270+ 's 32,39,115+A 65+aB 97,66+Ab 65,98+AB 65,66+ab 269diff --git a/c/tests/tok_o200k_tiny.json b/c/tests/tok_o200k_tiny.jsonnew file mode 100644index 00000000..829301ec--- /dev/null+++ b/c/tests/tok_o200k_tiny.json@@ -0,0 +1 @@+{"version": "1.0", "truncation": null, "padding": null, "added_tokens": [{"id": 274, "content": "<|endoftext|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}, {"id": 275, "content": "<|message_user|>", "single_word": false, "lstrip": false, "rstrip": false, "normalized": false, "special": true}], "normalizer": null, "pre_tokenizer": {"type": "Sequence", "pretokenizers": [{"type": "Split", "pattern": {"Regex": "[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]*[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]+(?i:'s|'t|'re|'ve|'m|'ll|'d)?|[^\\r\\n\\p{L}\\p{N}]?[\\p{Lu}\\p{Lt}\\p{Lm}\\p{Lo}\\p{M}]+[\\p{Ll}\\p{Lm}\\p{Lo}\\p{M}]*(?i:'s|'t|'re|'ve|'m|'ll|'d)?|\\p{N}{1,3}| ?[^\\s\\p{L}\\p{N}]+[\\r\\n/]*|\\s*[\\r\\n]+|\\s+(?!\\S)|\\s+"}, "behavior": "Isolated", "invert": false}, {"type": "ByteLevel", "add_prefix_space": false, "trim_offsets": true, "use_regex": false}]}, "post_processor": null, "decoder": {"type": "ByteLevel", "add_prefix_space": true, "trim_offsets": true, "use_regex": true}, "model": {"type": "BPE", "dropout": null, "unk_token": null, "continuing_subword_prefix": null, "end_of_word_suffix": null, "fuse_unk": false, "byte_fallback": false, "ignore_merges": true, "vocab": {"Ā": 0, "ā": 1, "Ă": 2, "ă": 3, "Ą": 4, "ą": 5, "Ć": 6, "ć": 7, "Ĉ": 8, "ĉ": 9, "Ċ": 10, "ċ": 11, "Č": 12, "č": 13, "Ď": 14, "ď": 15, "Đ": 16, "đ": 17, "Ē": 18, "ē": 19, "Ĕ": 20, "ĕ": 21, "Ė": 22, "ė": 23, "Ę": 24, "ę": 25, "Ě": 26, "ě": 27, "Ĝ": 28, "ĝ": 29, "Ğ": 30, "ğ": 31, "Ġ": 32, "!": 33, "\"": 34, "#": 35, "$": 36, "%": 37, "&": 38, "'": 39, "(": 40, ")": 41, "*": 42, "+": 43, ",": 44, "-": 45, ".": 46, "/": 47, "0": 48, "1": 49, "2": 50, "3": 51, "4": 52, "5": 53, "6": 54, "7": 55, "8": 56, "9": 57, ":": 58, ";": 59, "<": 60, "=": 61, ">": 62, "?": 63, "@": 64, "A": 65, "B": 66, "C": 67, "D": 68, "E": 69, "F": 70, "G": 71, "H": 72, "I": 73, "J": 74, "K": 75, "L": 76, "M": 77, "N": 78, "O": 79, "P": 80, "Q": 81, "R": 82, "S": 83, "T": 84, "U": 85, "V": 86, "W": 87, "X": 88, "Y": 89, "Z": 90, "[": 91, "\\": 92, "]": 93, "^": 94, "_": 95, "`": 96, "a": 97, "b": 98, "c": 99, "d": 100, "e": 101, "f": 102, "g": 103, "h": 104, "i": 105, "j": 106, "k": 107, "l": 108, "m": 109, "n": 110, "o": 111, "p": 112, "q": 113, "r": 114, "s": 115, "t": 116, "u": 117, "v": 118, "w": 119, "x": 120, "y": 121, "z": 122, "{": 123, "|": 124, "}": 125, "~": 126, "ġ": 127, "Ģ": 128, "ģ": 129, "Ĥ": 130, "ĥ": 131, "Ħ": 132, "ħ": 133, "Ĩ": 134, "ĩ": 135, "Ī": 136, "ī": 137, "Ĭ": 138, "ĭ": 139, "Į": 140, "į": 141, "İ": 142, "ı": 143, "IJ": 144, "ij": 145, "Ĵ": 146, "ĵ": 147, "Ķ": 148, "ķ": 149, "ĸ": 150, "Ĺ": 151, "ĺ": 152, "Ļ": 153, "ļ": 154, "Ľ": 155, "ľ": 156, "Ŀ": 157, "ŀ": 158, "Ł": 159, "ł": 160, "¡": 161, "¢": 162, "£": 163, "¤": 164, "¥": 165, "¦": 166, "§": 167, "¨": 168, "©": 169, "ª": 170, "«": 171, "¬": 172, "Ń": 173, "®": 174, "¯": 175, "°": 176, "±": 177, "²": 178, "³": 179, "´": 180, "µ": 181, "¶": 182, "·": 183, "¸": 184, "¹": 185, "º": 186, "»": 187, "¼": 188, "½": 189, "¾": 190, "¿": 191, "À": 192, "Á": 193, "Â": 194, "Ã": 195, "Ä": 196, "Å": 197, "Æ": 198, "Ç": 199, "È": 200, "É": 201, "Ê": 202, "Ë": 203, "Ì": 204, "Í": 205, "Î": 206, "Ï": 207, "Ð": 208, "Ñ": 209, "Ò": 210, "Ó": 211, "Ô": 212, "Õ": 213, "Ö": 214, "×": 215, "Ø": 216, "Ù": 217, "Ú": 218, "Û": 219, "Ü": 220, "Ý": 221, "Þ": 222, "ß": 223, "à": 224, "á": 225, "â": 226, "ã": 227, "ä": 228, "å": 229, "æ": 230, "ç": 231, "è": 232, "é": 233, "ê": 234, "ë": 235, "ì": 236, "í": 237, "î": 238, "ï": 239, "ð": 240, "ñ": 241, "ò": 242, "ó": 243, "ô": 244, "õ": 245, "ö": 246, "÷": 247, "ø": 248, "ù": 249, "ú": 250, "û": 251, "ü": 252, "ý": 253, "þ": 254, "ÿ": 255, "he": 256, "ll": 257, "hell": 258, "hello": 259, "Wo": 260, "Wor": 261, "World": 262, "ld": 263, "th": 264, "the": 265, "Ġthe": 266, "12": 267, "123": 268, "ab": 269, "'s": 270, "Ġa": 271, "./": 272, "ĊĊ": 273}, "merges": [["h", "e"], ["l", "l"], ["he", "ll"], ["hell", "o"], ["W", "o"], ["Wo", "r"], ["Wor", "ld"], ["l", "d"], ["t", "h"], ["th", "e"], ["Ġ", "the"], ["1", "2"], ["12", "3"], ["a", "b"], ["'", "s"], ["Ġ", "a"], [".", "/"], ["Ċ", "Ċ"]]}}\ No newline at end of file