Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
tok.h: the GPT-2 pre-tokenizer family, so OLMoE tokenizes like HF
JustVugg26e000bffb14
diff --git a/c/Makefile b/c/Makefileindex edc703f6..06d00d6d 100644--- a/c/Makefile+++ b/c/Makefile@@ -1261,6 +1261,9 @@ tests/test_json$(EXE): tests/test_json.c json.htests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_tok_gpt2$(EXE): tests/test_tok_gpt2.c tok.h tok_unicode.h tok_unicode_o200k.h json.h+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)+tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)diff --git a/c/tests/test_tok_gpt2.c b/c/tests/test_tok_gpt2.cnew file mode 100644index 00000000..7a27ffe2--- /dev/null+++ b/c/tests/test_tok_gpt2.c@@ -0,0 +1,65 @@+/* GPT-2 pre-tokenizer family (bare ByteLevel with use_regex, no Split: OLMoE,+ * GPT-NeoX) against HF-tokenizers-generated expectations. tests/tok_gpt2_tiny.json+ * is a byte-level BPE trained with `tokenizers` on a few sentences (14 KB, no+ * model download) with exactly OLMoE's pre_tokenizer block, and+ * tests/tok_gpt2_cases.txt holds the ids HF produced for each case. Guards the+ * rules where GPT-2 and cl100k differ, which tok.h used to get wrong for OLMoE:+ * " 4" and " 20" as one piece, unbounded digit runs, a non-space prefix NOT glued+ * to the word ("(abc"), case-sensitive contractions, whitespace runs before text,+ * and added-token boundaries. Round-trips every case. Measured on the real OLMoE+ * tokenizer: 1560/1708 cases identical before this family, 1708/1708 after. */+#define _GNU_SOURCE+#include "../tok.h"++int main(void) {+ Tok T;+ tok_load(&T, "tests/tok_gpt2_tiny.json");+ if (!T.gpt2) { fprintf(stderr, "test_tok_gpt2: GPT-2 family (bare ByteLevel) not detected\n"); return 1; }+ FILE *f = fopen("tests/tok_gpt2_cases.txt", "rb");+ if (!f) { perror("tests/tok_gpt2_cases.txt"); return 1; }+ /* fgets, not getline: MinGW's UCRT lacks getline and this must run on+ * the windows job. Case lines are short; 8 KB is generous. */+ char line[8192];+ int pass = 0, tot = 0, dpass = 0;+ while (fgets(line, sizeof(line), f)) {+ size_t nr = strlen(line);+ while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0;+ if (nr == 0) continue;+ char *tab = strchr(line, '\t'); if (!tab) continue;+ *tab = 0;+ const char *text = line, *idstr = tab + 1;+ char tbuf[4096]; int tn = 0;+ for (const char *q = text; *q && tn < 4095; q++) {+ if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; }+ else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; }+ else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; }+ else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; }+ else tbuf[tn++] = *q;+ }+ tbuf[tn] = 0;+ int exp[512], ne = 0;+ for (const char *q = idstr; *q; ) {+ while (*q == ',' || *q == ' ') q++;+ if (!*q) break;+ exp[ne++] = atoi(q);+ while (*q && *q != ',') q++;+ }+ int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512);+ int ok = (ng == ne);+ for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]);+ tot++; if (ok) pass++;+ char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191);+ int drt = (dn == tn) && !memcmp(dec, tbuf, tn);+ if (drt) dpass++;+ if (!ok || !drt) {+ fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne);+ for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]);+ fprintf(stderr, "\n got(%d):", ng);+ for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]);+ fprintf(stderr, "\n decode_ok=%d\n", drt);+ }+ }+ fclose(f);+ printf("test_tok_gpt2: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot);+ return (pass == tot && dpass == tot) ? 0 : 2;+}diff --git a/c/tests/tok_gpt2_cases.txt b/c/tests/tok_gpt2_cases.txtnew file mode 100644index 00000000..c632bf97--- /dev/null+++ b/c/tests/tok_gpt2_cases.txt@@ -0,0 +1,27 @@+ 4 262+x = 4 << 20 87,347,262,282,281+MAX_BODY = 4 << 20 301,62,384,347,262,282,281+1234567 apples 16,294,383,401+(abc 7,386+[def] {ghi} 58,285,60,349,389,92+'S 6,50+'s 288+It's 40,83,288+don't. 372,289,13+they're 83,256,88,290+ x 220,278+x 87,350+ 350+a\n\nb 64,198,198,65+a.\nb 64,13,198,65+Hello, world! 385,11,381,0+\t\tif x:\n\t\t\treturn 197,197,314,278,25,343,197,197,400+café 388+<|endoftext|>x.<|endoftext|> 405,87,13,405+x.<|endoftext|> 87,13,405+ <|endoftext|> 350,405+ été 42 220,342,83,342,366+a b c 64,220,276,350,220,66+end.\n 68,324,13,198+\n\n 198,198+x\r\ny 87,201,198,88diff --git a/c/tests/tok_gpt2_tiny.json b/c/tests/tok_gpt2_tiny.jsonnew file mode 100644index 00000000..c1c2accc--- /dev/null+++ b/c/tests/tok_gpt2_tiny.json@@ -0,0 +1,1056 @@+{+ "version": "1.0",+ "truncation": null,+ "padding": null,+ "added_tokens": [+ {+ "id": 405,+ "content": "<|endoftext|>",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ },+ {+ "id": 406,+ "content": "<|padding|>",+ "single_word": false,+ "lstrip": false,+ "rstrip": false,+ "normalized": false,+ "special": true+ }+ ],+ "normalizer": {+ "type": "NFC"+ },+ "pre_tokenizer": {+ "type": "ByteLevel",+ "add_prefix_space": false,+ "trim_offsets": true,+ "use_regex": true+ },+ "post_processor": null,+ "decoder": {+ "type": "ByteLevel",+ "add_prefix_space": true,+ "trim_offsets": true,+ "use_regex": true+ },+ "model": {+ "type": "BPE",+ "dropout": null,+ "unk_token": null,+ "continuing_subword_prefix": null,+ "end_of_word_suffix": null,+ "fuse_unk": false,+ "byte_fallback": false,+ "ignore_merges": false,+ "vocab": {+ "!": 0,+ "\"": 1,+ "#": 2,+ "$": 3,+ "%": 4,+ "&": 5,+ "'": 6,+ "(": 7,+ ")": 8,+ "*": 9,+ "+": 10,+ ",": 11,+ "-": 12,+ ".": 13,+ "/": 14,+ "0": 15,+ "1": 16,+ "2": 17,+ "3": 18,+ "4": 19,+ "5": 20,+ "6": 21,+ "7": 22,+ "8": 23,+ "9": 24,+ ":": 25,+ ";": 26,+ "<": 27,+ "=": 28,+ ">": 29,+ "?": 30,+ "@": 31,+ "A": 32,+ "B": 33,+ "C": 34,+ "D": 35,+ "E": 36,+ "F": 37,+ "G": 38,+ "H": 39,+ "I": 40,+ "J": 41,+ "K": 42,+ "L": 43,+ "M": 44,+ "N": 45,+ "O": 46,+ "P": 47,+ "Q": 48,+ "R": 49,+ "S": 50,+ "T": 51,+ "U": 52,+ "V": 53,+ "W": 54,+ "X": 55,+ "Y": 56,+ "Z": 57,+ "[": 58,+ "\\": 59,+ "]": 60,+ "^": 61,+ "_": 62,+ "`": 63,+ "a": 64,+ "b": 65,+ "c": 66,+ "d": 67,+ "e": 68,+ "f": 69,+ "g": 70,+ "h": 71,+ "i": 72,+ "j": 73,+ "k": 74,+ "l": 75,+ "m": 76,+ "n": 77,+ "o": 78,+ "p": 79,+ "q": 80,+ "r": 81,+ "s": 82,+ "t": 83,+ "u": 84,+ "v": 85,+ "w": 86,+ "x": 87,+ "y": 88,+ "z": 89,+ "{": 90,+ "|": 91,+ "}": 92,+ "~": 93,+ "¡": 94,+ "¢": 95,+ "£": 96,+ "¤": 97,+ "¥": 98,+ "¦": 99,+ "§": 100,+ "¨": 101,+ "©": 102,+ "ª": 103,+ "«": 104,+ "¬": 105,+ "®": 106,+ "¯": 107,+ "°": 108,+ "±": 109,+ "²": 110,+ "³": 111,+ "´": 112,+ "µ": 113,+ "¶": 114,+ "·": 115,+ "¸": 116,+ "¹": 117,+ "º": 118,+ "»": 119,+ "¼": 120,+ "½": 121,+ "¾": 122,+ "¿": 123,+ "À": 124,+ "Á": 125,+ "Â": 126,+ "Ã": 127,+ "Ä": 128,+ "Å": 129,+ "Æ": 130,+ "Ç": 131,+ "È": 132,+ "É": 133,+ "Ê": 134,+ "Ë": 135,+ "Ì": 136,+ "Í": 137,+ "Î": 138,+ "Ï": 139,+ "Ð": 140,+ "Ñ": 141,+ "Ò": 142,+ "Ó": 143,+ "Ô": 144,+ "Õ": 145,+ "Ö": 146,+ "×": 147,+ "Ø": 148,+ "Ù": 149,+ "Ú": 150,+ "Û": 151,+ "Ü": 152,+ "Ý": 153,+ "Þ": 154,+ "ß": 155,+ "à": 156,+ "á": 157,+ "â": 158,+ "ã": 159,+ "ä": 160,+ "å": 161,+ "æ": 162,+ "ç": 163,+ "è": 164,+ "é": 165,+ "ê": 166,+ "ë": 167,+ "ì": 168,+ "í": 169,+ "î": 170,+ "ï": 171,+ "ð": 172,+ "ñ": 173,+ "ò": 174,+ "ó": 175,+ "ô": 176,+ "õ": 177,+ "ö": 178,+ "÷": 179,+ "ø": 180,+ "ù": 181,+ "ú": 182,+ "û": 183,+ "ü": 184,+ "ý": 185,+ "þ": 186,+ "ÿ": 187,+ "Ā": 188,+ "ā": 189,+ "Ă": 190,+ "ă": 191,+ "Ą": 192,+ "ą": 193,+ "Ć": 194,+ "ć": 195,+ "Ĉ": 196,+ "ĉ": 197,+ "Ċ": 198,+ "ċ": 199,+ "Č": 200,+ "č": 201,+ "Ď": 202,+ "ď": 203,+ "Đ": 204,+ "đ": 205,+ "Ē": 206,+ "ē": 207,+ "Ĕ": 208,+ "ĕ": 209,+ "Ė": 210,+ "ė": 211,+ "Ę": 212,+ "ę": 213,+ "Ě": 214,+ "ě": 215,+ "Ĝ": 216,+ "ĝ": 217,+ "Ğ": 218,+ "ğ": 219,+ "Ġ": 220,+ "ġ": 221,+ "Ģ": 222,+ "ģ": 223,+ "Ĥ": 224,+ "ĥ": 225,+ "Ħ": 226,+ "ħ": 227,+ "Ĩ": 228,+ "ĩ": 229,+ "Ī": 230,+ "ī": 231,+ "Ĭ": 232,+ "ĭ": 233,+ "Į": 234,+ "į": 235,+ "İ": 236,+ "ı": 237,+ "IJ": 238,+ "ij": 239,+ "Ĵ": 240,+ "ĵ": 241,+ "Ķ": 242,+ "ķ": 243,+ "ĸ": 244,+ "Ĺ": 245,+ "ĺ": 246,+ "Ļ": 247,+ "ļ": 248,+ "Ľ": 249,+ "ľ": 250,+ "Ŀ": 251,+ "ŀ": 252,+ "Ł": 253,+ "ł": 254,+ "Ń": 255,+ "he": 256,+ "Ġt": 257,+ "es": 258,+ "re": 259,+ "ve": 260,+ "wo": 261,+ "Ġ4": 262,+ "ĠI": 263,+ "Ġa": 264,+ "Ġf": 265,+ "20": 266,+ "<<": 267,+ "de": 268,+ "do": 269,+ "in": 270,+ "la": 271,+ "ll": 272,+ "ow": 273,+ "ove": 274,+ "Ġ1": 275,+ "Ġb": 276,+ "Ġs": 277,+ "Ġx": 278,+ "Ġhe": 279,+ "Ġwo": 280,+ "Ġ20": 281,+ "Ġ<<": 282,+ "Ġthe": 283,+ "ĠIt": 284,+ "def": 285,+ "ine": 286,+ "'d": 287,+ "'s": 288,+ "'t": 289,+ "'re": 290,+ "'ve": 291,+ "'ll": 292,+ "):": 293,+ "23": 294,+ "45": 295,+ "67": 296,+ "AX": 297,+ "BO": 298,+ "DY": 299,+ "He": 300,+ "MAX": 301,+ "The": 302,+ "ab": 303,+ "ac": 304,+ "af": 305,+ "aÃ": 306,+ "bla": 307,+ "bove": 308,+ "ck": 309,+ "caf": 310,+ "gh": 311,+ "go": 312,+ "hre": 313,+ "if": 314,+ "im": 315,+ "is": 316,+ "ick": 317,+ "ju": 318,+ "kn": 319,+ "ld": 320,+ "les": 321,+ "line": 322,+ "mp": 323,+ "nd": 324,+ "nk": 325,+ "naÃ": 326,+ "op": 327,+ "ox": 328,+ "pp": 329,+ "pac": 330,+ "qu": 331,+ "rn": 332,+ "row": 333,+ "rld": 334,+ "tu": 335,+ "tes": 336,+ "top": 337,+ "we": 338,+ "ytes": 339,+ "zy": 340,+ "¯ve": 341,+ "é": 342,+ "Ċĉ": 343,+ "Ġ#": 344,+ "Ġ(": 345,+ "Ġ+": 346,+ "Ġ=": 347,+ "Ġ[": 348,+ "Ġ{": 349,+ "ĠĠ": 350,+ "Ġdo": 351,+ "Ġla": 352,+ "Ġove": 353,+ "Ġgo": 354,+ "Ġis": 355,+ "Ġju": 356,+ "Ġkn": 357,+ "Ġline": 358,+ "ĠnaÃ": 359,+ "Ġqu": 360,+ "Ġwe": 361,+ "Ġtwo": 362,+ "Ġthre": 363,+ "Ġtim": 364,+ "retu": 365,+ "Ġ42": 366,+ "Ġabove": 367,+ "Ġand": 368,+ "Ġapp": 369,+ "Ġfine": 370,+ "Ġfox": 371,+ "don": 372,+ "llo": 373,+ "Ġ123": 374,+ "Ġbrow": 375,+ "Ġbytes": 376,+ "Ġspac": 377,+ "Ġstop": 378,+ "Ġhere": 379,+ "Ġwon": 380,+ "Ġworld": 381,+ "Ġthey": 382,+ "4567": 383,+ "BODY": 384,+ "Hello": 385,+ "abc": 386,+ "blank": 387,+ "café": 388,+ "ghi": 389,+ "mps": 390,+ "Ġdog": 391,+ "Ġlazy": 392,+ "Ġover": 393,+ "Ġjumps": 394,+ "Ġknow": 395,+ "Ġnaïve": 396,+ "Ġquick": 397,+ "Ġthree": 398,+ "Ġtimes": 399,+ "return": 400,+ "Ġapples": 401,+ "Ġ1234567": 402,+ "Ġbrown": 403,+ "Ġspaces": 404+ },+ "merges": [+ [+ "h",+ "e"+ ],+ [+ "Ġ",+ "t"+ ],+ [+ "e",+ "s"+ ],+ [+ "r",+ "e"+ ],+ [+ "v",+ "e"+ ],+ [+ "w",+ "o"+ ],+ [+ "Ġ",+ "4"+ ],+ [+ "Ġ",+ "I"+ ],+ [+ "Ġ",+ "a"+ ],+ [+ "Ġ",+ "f"+ ],+ [+ "2",+ "0"+ ],+ [+ "<",+ "<"+ ],+ [+ "d",+ "e"+ ],+ [+ "d",+ "o"+ ],+ [+ "i",+ "n"+ ],+ [+ "l",+ "a"+ ],+ [+ "l",+ "l"+ ],+ [+ "o",+ "w"+ ],+ [+ "o",+ "ve"+ ],+ [+ "Ġ",+ "1"+ ],+ [+ "Ġ",+ "b"+ ],+ [+ "Ġ",+ "s"+ ],+ [+ "Ġ",+ "x"+ ],+ [+ "Ġ",+ "he"+ ],+ [+ "Ġ",+ "wo"+ ],+ [+ "Ġ",+ "20"+ ],+ [+ "Ġ",+ "<<"+ ],+ [+ "Ġt",+ "he"+ ],+ [+ "ĠI",+ "t"+ ],+ [+ "de",+ "f"+ ],+ [+ "in",+ "e"+ ],+ [+ "'",+ "d"+ ],+ [+ "'",+ "s"+ ],+ [+ "'",+ "t"+ ],+ [+ "'",+ "re"+ ],+ [+ "'",+ "ve"+ ],+ [+ "'",+ "ll"+ ],+ [+ ")",+ ":"+ ],+ [+ "2",+ "3"+ ],+ [+ "4",+ "5"+ ],+ [+ "6",+ "7"+ ],+ [+ "A",+ "X"+ ],+ [+ "B",+ "O"+ ],+ [+ "D",+ "Y"+ ],+ [+ "H",+ "e"+ ],+ [+ "M",+ "AX"+ ],+ [+ "T",+ "he"+ ],+ [+ "a",+ "b"+ ],+ [+ "a",+ "c"+ ],+ [+ "a",+ "f"+ ],+ [+ "a",+ "Ã"+ ],+ [+ "b",+ "la"+ ],+ [+ "b",+ "ove"+ ],+ [+ "c",+ "k"+ ],+ [+ "c",+ "af"+ ],+ [+ "g",+ "h"+ ],+ [+ "g",+ "o"+ ],+ [+ "h",+ "re"+ ],+ [+ "i",+ "f"+ ],+ [+ "i",+ "m"+ ],+ [+ "i",+ "s"+ ],+ [+ "i",+ "ck"+ ],+ [+ "j",+ "u"+ ],+ [+ "k",+ "n"+ ],+ [+ "l",+ "d"+ ],+ [+ "l",+ "es"+ ],+ [+ "l",+ "ine"+ ],+ [+ "m",+ "p"+ ],+ [+ "n",+ "d"+ ],+ [+ "n",+ "k"+ ],+ [+ "n",+ "aÃ"+ ],+ [+ "o",+ "p"+ ],+ [+ "o",+ "x"+ ],+ [+ "p",+ "p"+ ],+ [+ "p",+ "ac"+ ],+ [+ "q",+ "u"+ ],+ [+ "r",+ "n"+ ],+ [+ "r",+ "ow"+ ],+ [+ "r",+ "ld"+ ],+ [+ "t",+ "u"+ ],+ [+ "t",+ "es"+ ],+ [+ "t",+ "op"+ ],+ [+ "w",+ "e"+ ],+ [+ "y",+ "tes"+ ],+ [+ "z",+ "y"+ ],+ [+ "¯",+ "ve"+ ],+ [+ "Ã",+ "©"+ ],+ [+ "Ċ",+ "ĉ"+ ],+ [+ "Ġ",+ "#"+ ],+ [+ "Ġ",+ "("+ ],+ [+ "Ġ",+ "+"+ ],+ [+ "Ġ",+ "="+ ],+ [+ "Ġ",+ "["+ ],+ [+ "Ġ",+ "{"+ ],+ [+ "Ġ",+ "Ġ"+ ],+ [+ "Ġ",+ "do"+ ],+ [+ "Ġ",+ "la"+ ],+ [+ "Ġ",+ "ove"+ ],+ [+ "Ġ",+ "go"+ ],+ [+ "Ġ",+ "is"+ ],+ [+ "Ġ",+ "ju"+ ],+ [+ "Ġ",+ "kn"+ ],+ [+ "Ġ",+ "line"+ ],+ [+ "Ġ",+ "naÃ"+ ],+ [+ "Ġ",+ "qu"+ ],+ [+ "Ġ",+ "we"+ ],+ [+ "Ġt",+ "wo"+ ],+ [+ "Ġt",+ "hre"+ ],+ [+ "Ġt",+ "im"+ ],+ [+ "re",+ "tu"+ ],+ [+ "Ġ4",+ "2"+ ],+ [+ "Ġa",+ "bove"+ ],+ [+ "Ġa",+ "nd"+ ],+ [+ "Ġa",+ "pp"+ ],+ [+ "Ġf",+ "ine"+ ],+ [+ "Ġf",+ "ox"+ ],+ [+ "do",+ "n"+ ],+ [+ "ll",+ "o"+ ],+ [+ "Ġ1",+ "23"+ ],+ [+ "Ġb",+ "row"+ ],+ [+ "Ġb",+ "ytes"+ ],+ [+ "Ġs",+ "pac"+ ],+ [+ "Ġs",+ "top"+ ],+ [+ "Ġhe",+ "re"+ ],+ [+ "Ġwo",+ "n"+ ],+ [+ "Ġwo",+ "rld"+ ],+ [+ "Ġthe",+ "y"+ ],+ [+ "45",+ "67"+ ],+ [+ "BO",+ "DY"+ ],+ [+ "He",+ "llo"+ ],+ [+ "ab",+ "c"+ ],+ [+ "bla",+ "nk"+ ],+ [+ "caf",+ "é"+ ],+ [+ "gh",+ "i"+ ],+ [+ "mp",+ "s"+ ],+ [+ "Ġdo",+ "g"+ ],+ [+ "Ġla",+ "zy"+ ],+ [+ "Ġove",+ "r"+ ],+ [+ "Ġju",+ "mps"+ ],+ [+ "Ġkn",+ "ow"+ ],+ [+ "ĠnaÃ",+ "¯ve"+ ],+ [+ "Ġqu",+ "ick"+ ],+ [+ "Ġthre",+ "e"+ ],+ [+ "Ġtim",+ "es"+ ],+ [+ "retu",+ "rn"+ ],+ [+ "Ġapp",+ "les"+ ],+ [+ "Ġ123",+ "4567"+ ],+ [+ "Ġbrow",+ "n"+ ],+ [+ "Ġspac",+ "es"+ ]+ ]+ }+}\ No newline at end of filediff --git a/c/tok.h b/c/tok.hindex 54c5c699..83b649df 100644--- a/c/tok.h+++ b/c/tok.h@@ -58,6 +58,8 @@ typedef struct {int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */int kimi; /* 1 = Kimi (K3) family: o200k rules + a leading \p{Han}-run rule,* Han excluded from the letter classes, no '/' tail in the punct rule */+ int gpt2; /* 1 = GPT-2 family (OLMoE / GPT-NeoX): a bare ByteLevel pre_tokenizer+ * with use_regex and no Split, so HF applies the original GPT-2 pattern */int rankbpe; /* 1 = no merges list (tiktoken-derived vocab): merge the adjacent* pair whose CONCATENATION has the lowest vocab id — exactly* tiktoken's byte_pair_encode, no recovered merges to diverge */@@ -218,6 +220,13 @@ static void tok_load(Tok *T, const char *path){* case-category classes (\p{Lu}...) which cl100k does not use */jval *pt=json_get(root,"pre_tokenizer");if(pt){+ /* No Split at all, just ByteLevel with use_regex (the default): HF runs+ * the original GPT-2 regex, not cl100k. OLMoE's tokenizer.json is this. */+ jval *pty=json_get(pt,"type");+ if(pty && pty->t==J_STR && pty->str && !strcmp(pty->str,"ByteLevel")){+ jval *ur=json_get(pt,"use_regex");+ if(!ur || (ur->t==J_BOOL && ur->boolean)) T->gpt2=1;+ }jval *ps=json_get(pt,"pretokenizers");if(ps&&ps->t==J_ARR) for(int i=0;i<ps->len;i++){jval *pat=json_get(ps->kids[i],"pattern");@@ -532,6 +541,52 @@ static void pretok_chunk_kimi(Tok *T, const unsigned char *p, int a, int b, intfree(cp); free(off);}+/* ---------- pre-tokenizer GPT-2 (bare ByteLevel with use_regex: OLMoE, GPT-NeoX) ----------+ * The original GPT-2 pattern, which HF applies when the pre_tokenizer is a+ * ByteLevel with use_regex=true and no Split in front of it:+ * 's|'t|'re|'ve|'m|'ll|'d | ?\p{L}+ | ?\p{N}+ | ?[^\s\p{L}\p{N}]+ | \s+(?!\S) | \s++ * It differs from cl100k in every rule: the contractions are case-sensitive,+ * letters and digits take ONE optional leading space (not any non-letter),+ * digit runs are unbounded, the punctuation run has no newline tail, and+ * there is no \s*[\r\n]+ rule. Measured on OLMoE against `tokenizers`:+ * 1560/1708 cases identical under the cl100k rules, every miss a " 4" /+ * " 20"-style number or a non-space prefix glued to a word. */+static void pretok_chunk_gpt2(Tok *T, const unsigned char *p, int a, int b, int *out, int *no, int max){+ int nb=b-a; if(nb<=0) return;+ uint32_t *cp=malloc((nb+1)*sizeof(uint32_t)); int *off=malloc((nb+2)*sizeof(int)); int n=0;+ for(int i=a;i<b;){ uint32_t c; int k=u8_next(p,b,i,&c); off[n]=i; cp[n]=c; n++; i+=k; }+ off[n]=b;+ int i=0;+ while(i<n){+ int start=i; uint32_t c=cp[i];+ /* 1) 's|'t|'re|'ve|'m|'ll|'d -- case-sensitive, unlike cl100k's (?i:) */+ if(c=='\'' && i+1<n){+ uint32_t d=cp[i+1];+ if(i+2<n){ uint32_t d2=cp[i+2];+ if((d=='r'&&d2=='e')||(d=='v'&&d2=='e')||(d=='l'&&d2=='l')){ i+=3; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } }+ if(d=='s'||d=='t'||d=='m'||d=='d'){ i+=2; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }+ }+ /* 2) ' ?\p{L}+' 3) ' ?\p{N}+' 4) ' ?[^\s\p{L}\p{N}]+' : one optional space, then a run of one class */+ {+ int j=i; if(c==' ' && j+1<n) j++;+ uint32_t d=cp[j];+ if(is_L(d)){ while(j<n && is_L(cp[j])) j++; i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }+ if(is_N(d)){ while(j<n && is_N(cp[j])) j++; i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }+ if(!is_S(d)){ while(j<n && !is_S(cp[j]) && !is_L(cp[j]) && !is_N(cp[j])) j++; i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }+ }+ /* 5) \s+(?!\S): a run followed by a non-space keeps its last char for the+ * next piece (which takes it as its optional space); at the end of the+ * input the whole run. 6) \s+: a single whitespace before a non-space. */+ {+ int r=i; while(r<n && is_S(cp[r])) r++;+ if(r>i){ int end=(r<n)? r-1 : r; if(end<=i) end=i+1; i=end; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }+ }+ i++; /* unreachable: every codepoint is a letter, a digit, whitespace or none of them */+ bpe_piece(T,p,off[start],off[i],out,no,max);+ }+ free(cp); free(off);+}+/* ---------- encode: testo -> id (split sugli added token, poi pretok+BPE) ---------- */static int tok_encode(Tok *T, const char *text, int len, int *out, int max){const unsigned char *p=(const unsigned char*)text; int no=0; int i=0;@@ -546,7 +601,8 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){}int chunk_end = (hitpos<0) ? len : hitpos;if(chunk_end>i){- if(T->kimi) pretok_chunk_kimi(T,p,i,chunk_end,out,&no,max);+ if(T->gpt2) pretok_chunk_gpt2(T,p,i,chunk_end,out,&no,max);+ else if(T->kimi) pretok_chunk_kimi(T,p,i,chunk_end,out,&no,max);else if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max);else pretok_chunk(T,p,i,chunk_end,out,&no,max);}