Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

tok.h: the GPT-2 pre-tokenizer family, so OLMoE tokenizes like HF

JustVugg26e000bffb14
diff --git a/c/Makefile b/c/Makefile
index edc703f6..06d00d6d 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1261,6 +1261,9 @@ tests/test_json$(EXE): tests/test_json.c json.h
tests/test_tok_o200k$(EXE): tests/test_tok_o200k.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+tests/test_tok_gpt2$(EXE): tests/test_tok_gpt2.c tok.h tok_unicode.h tok_unicode_o200k.h json.h
+ $(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
+
tests/test_k3_ram_budget$(EXE): tests/test_k3_ram_budget.c kimi_k3.c st.h tok.h quant.h compat.h omp_tune.h route_trace.h kv_prefix.h serve_codec.h
$(CC) $(CFLAGS) $< -o $@ $(LDFLAGS)
diff --git a/c/tests/test_tok_gpt2.c b/c/tests/test_tok_gpt2.c
new file mode 100644
index 00000000..7a27ffe2
--- /dev/null
+++ b/c/tests/test_tok_gpt2.c
@@ -0,0 +1,65 @@
+/* GPT-2 pre-tokenizer family (bare ByteLevel with use_regex, no Split: OLMoE,
+ * GPT-NeoX) against HF-tokenizers-generated expectations. tests/tok_gpt2_tiny.json
+ * is a byte-level BPE trained with `tokenizers` on a few sentences (14 KB, no
+ * model download) with exactly OLMoE's pre_tokenizer block, and
+ * tests/tok_gpt2_cases.txt holds the ids HF produced for each case. Guards the
+ * rules where GPT-2 and cl100k differ, which tok.h used to get wrong for OLMoE:
+ * " 4" and " 20" as one piece, unbounded digit runs, a non-space prefix NOT glued
+ * to the word ("(abc"), case-sensitive contractions, whitespace runs before text,
+ * and added-token boundaries. Round-trips every case. Measured on the real OLMoE
+ * tokenizer: 1560/1708 cases identical before this family, 1708/1708 after. */
+#define _GNU_SOURCE
+#include "../tok.h"
+
+int main(void) {
+ Tok T;
+ tok_load(&T, "tests/tok_gpt2_tiny.json");
+ if (!T.gpt2) { fprintf(stderr, "test_tok_gpt2: GPT-2 family (bare ByteLevel) not detected\n"); return 1; }
+ FILE *f = fopen("tests/tok_gpt2_cases.txt", "rb");
+ if (!f) { perror("tests/tok_gpt2_cases.txt"); return 1; }
+ /* fgets, not getline: MinGW's UCRT lacks getline and this must run on
+ * the windows job. Case lines are short; 8 KB is generous. */
+ char line[8192];
+ int pass = 0, tot = 0, dpass = 0;
+ while (fgets(line, sizeof(line), f)) {
+ size_t nr = strlen(line);
+ while (nr > 0 && (line[nr-1] == '\n' || line[nr-1] == '\r')) line[--nr] = 0;
+ if (nr == 0) continue;
+ char *tab = strchr(line, '\t'); if (!tab) continue;
+ *tab = 0;
+ const char *text = line, *idstr = tab + 1;
+ char tbuf[4096]; int tn = 0;
+ for (const char *q = text; *q && tn < 4095; q++) {
+ if (q[0]=='\\' && q[1]=='n') { tbuf[tn++]='\n'; q++; }
+ else if (q[0]=='\\' && q[1]=='t') { tbuf[tn++]='\t'; q++; }
+ else if (q[0]=='\\' && q[1]=='r') { tbuf[tn++]='\r'; q++; }
+ else if (q[0]=='\\' && q[1]=='\\') { tbuf[tn++]='\\'; q++; }
+ else tbuf[tn++] = *q;
+ }
+ tbuf[tn] = 0;
+ int exp[512], ne = 0;
+ for (const char *q = idstr; *q; ) {
+ while (*q == ',' || *q == ' ') q++;
+ if (!*q) break;
+ exp[ne++] = atoi(q);
+ while (*q && *q != ',') q++;
+ }
+ int got[512]; int ng = tok_encode(&T, tbuf, tn, got, 512);
+ int ok = (ng == ne);
+ for (int i = 0; i < ng && ok; i++) ok = (got[i] == exp[i]);
+ tot++; if (ok) pass++;
+ char dec[8192]; int dn = tok_decode(&T, got, ng, dec, 8191);
+ int drt = (dn == tn) && !memcmp(dec, tbuf, tn);
+ if (drt) dpass++;
+ if (!ok || !drt) {
+ fprintf(stderr, "MISMATCH text=%s\n exp(%d):", text, ne);
+ for (int i = 0; i < ne; i++) fprintf(stderr, " %d", exp[i]);
+ fprintf(stderr, "\n got(%d):", ng);
+ for (int i = 0; i < ng; i++) fprintf(stderr, " %d", got[i]);
+ fprintf(stderr, "\n decode_ok=%d\n", drt);
+ }
+ }
+ fclose(f);
+ printf("test_tok_gpt2: ENCODE %d/%d DECODE %d/%d\n", pass, tot, dpass, tot);
+ return (pass == tot && dpass == tot) ? 0 : 2;
+}
diff --git a/c/tests/tok_gpt2_cases.txt b/c/tests/tok_gpt2_cases.txt
new file mode 100644
index 00000000..c632bf97
--- /dev/null
+++ b/c/tests/tok_gpt2_cases.txt
@@ -0,0 +1,27 @@
+ 4 262
+x = 4 << 20 87,347,262,282,281
+MAX_BODY = 4 << 20 301,62,384,347,262,282,281
+1234567 apples 16,294,383,401
+(abc 7,386
+[def] {ghi} 58,285,60,349,389,92
+'S 6,50
+'s 288
+It's 40,83,288
+don't. 372,289,13
+they're 83,256,88,290
+ x 220,278
+x 87,350
+ 350
+a\n\nb 64,198,198,65
+a.\nb 64,13,198,65
+Hello, world! 385,11,381,0
+\t\tif x:\n\t\t\treturn 197,197,314,278,25,343,197,197,400
+café 388
+<|endoftext|>x.<|endoftext|> 405,87,13,405
+x.<|endoftext|> 87,13,405
+ <|endoftext|> 350,405
+ été 42 220,342,83,342,366
+a b c 64,220,276,350,220,66
+end.\n 68,324,13,198
+\n\n 198,198
+x\r\ny 87,201,198,88
diff --git a/c/tests/tok_gpt2_tiny.json b/c/tests/tok_gpt2_tiny.json
new file mode 100644
index 00000000..c1c2accc
--- /dev/null
+++ b/c/tests/tok_gpt2_tiny.json
@@ -0,0 +1,1056 @@
+{
+ "version": "1.0",
+ "truncation": null,
+ "padding": null,
+ "added_tokens": [
+ {
+ "id": 405,
+ "content": "<|endoftext|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ },
+ {
+ "id": 406,
+ "content": "<|padding|>",
+ "single_word": false,
+ "lstrip": false,
+ "rstrip": false,
+ "normalized": false,
+ "special": true
+ }
+ ],
+ "normalizer": {
+ "type": "NFC"
+ },
+ "pre_tokenizer": {
+ "type": "ByteLevel",
+ "add_prefix_space": false,
+ "trim_offsets": true,
+ "use_regex": true
+ },
+ "post_processor": null,
+ "decoder": {
+ "type": "ByteLevel",
+ "add_prefix_space": true,
+ "trim_offsets": true,
+ "use_regex": true
+ },
+ "model": {
+ "type": "BPE",
+ "dropout": null,
+ "unk_token": null,
+ "continuing_subword_prefix": null,
+ "end_of_word_suffix": null,
+ "fuse_unk": false,
+ "byte_fallback": false,
+ "ignore_merges": false,
+ "vocab": {
+ "!": 0,
+ "\"": 1,
+ "#": 2,
+ "$": 3,
+ "%": 4,
+ "&": 5,
+ "'": 6,
+ "(": 7,
+ ")": 8,
+ "*": 9,
+ "+": 10,
+ ",": 11,
+ "-": 12,
+ ".": 13,
+ "/": 14,
+ "0": 15,
+ "1": 16,
+ "2": 17,
+ "3": 18,
+ "4": 19,
+ "5": 20,
+ "6": 21,
+ "7": 22,
+ "8": 23,
+ "9": 24,
+ ":": 25,
+ ";": 26,
+ "<": 27,
+ "=": 28,
+ ">": 29,
+ "?": 30,
+ "@": 31,
+ "A": 32,
+ "B": 33,
+ "C": 34,
+ "D": 35,
+ "E": 36,
+ "F": 37,
+ "G": 38,
+ "H": 39,
+ "I": 40,
+ "J": 41,
+ "K": 42,
+ "L": 43,
+ "M": 44,
+ "N": 45,
+ "O": 46,
+ "P": 47,
+ "Q": 48,
+ "R": 49,
+ "S": 50,
+ "T": 51,
+ "U": 52,
+ "V": 53,
+ "W": 54,
+ "X": 55,
+ "Y": 56,
+ "Z": 57,
+ "[": 58,
+ "\\": 59,
+ "]": 60,
+ "^": 61,
+ "_": 62,
+ "`": 63,
+ "a": 64,
+ "b": 65,
+ "c": 66,
+ "d": 67,
+ "e": 68,
+ "f": 69,
+ "g": 70,
+ "h": 71,
+ "i": 72,
+ "j": 73,
+ "k": 74,
+ "l": 75,
+ "m": 76,
+ "n": 77,
+ "o": 78,
+ "p": 79,
+ "q": 80,
+ "r": 81,
+ "s": 82,
+ "t": 83,
+ "u": 84,
+ "v": 85,
+ "w": 86,
+ "x": 87,
+ "y": 88,
+ "z": 89,
+ "{": 90,
+ "|": 91,
+ "}": 92,
+ "~": 93,
+ "¡": 94,
+ "¢": 95,
+ "£": 96,
+ "¤": 97,
+ "¥": 98,
+ "¦": 99,
+ "§": 100,
+ "¨": 101,
+ "©": 102,
+ "ª": 103,
+ "«": 104,
+ "¬": 105,
+ "®": 106,
+ "¯": 107,
+ "°": 108,
+ "±": 109,
+ "²": 110,
+ "³": 111,
+ "´": 112,
+ "µ": 113,
+ "¶": 114,
+ "·": 115,
+ "¸": 116,
+ "¹": 117,
+ "º": 118,
+ "»": 119,
+ "¼": 120,
+ "½": 121,
+ "¾": 122,
+ "¿": 123,
+ "À": 124,
+ "Á": 125,
+ "Â": 126,
+ "Ã": 127,
+ "Ä": 128,
+ "Å": 129,
+ "Æ": 130,
+ "Ç": 131,
+ "È": 132,
+ "É": 133,
+ "Ê": 134,
+ "Ë": 135,
+ "Ì": 136,
+ "Í": 137,
+ "Î": 138,
+ "Ï": 139,
+ "Ð": 140,
+ "Ñ": 141,
+ "Ò": 142,
+ "Ó": 143,
+ "Ô": 144,
+ "Õ": 145,
+ "Ö": 146,
+ "×": 147,
+ "Ø": 148,
+ "Ù": 149,
+ "Ú": 150,
+ "Û": 151,
+ "Ü": 152,
+ "Ý": 153,
+ "Þ": 154,
+ "ß": 155,
+ "à": 156,
+ "á": 157,
+ "â": 158,
+ "ã": 159,
+ "ä": 160,
+ "å": 161,
+ "æ": 162,
+ "ç": 163,
+ "è": 164,
+ "é": 165,
+ "ê": 166,
+ "ë": 167,
+ "ì": 168,
+ "í": 169,
+ "î": 170,
+ "ï": 171,
+ "ð": 172,
+ "ñ": 173,
+ "ò": 174,
+ "ó": 175,
+ "ô": 176,
+ "õ": 177,
+ "ö": 178,
+ "÷": 179,
+ "ø": 180,
+ "ù": 181,
+ "ú": 182,
+ "û": 183,
+ "ü": 184,
+ "ý": 185,
+ "þ": 186,
+ "ÿ": 187,
+ "Ā": 188,
+ "ā": 189,
+ "Ă": 190,
+ "ă": 191,
+ "Ą": 192,
+ "ą": 193,
+ "Ć": 194,
+ "ć": 195,
+ "Ĉ": 196,
+ "ĉ": 197,
+ "Ċ": 198,
+ "ċ": 199,
+ "Č": 200,
+ "č": 201,
+ "Ď": 202,
+ "ď": 203,
+ "Đ": 204,
+ "đ": 205,
+ "Ē": 206,
+ "ē": 207,
+ "Ĕ": 208,
+ "ĕ": 209,
+ "Ė": 210,
+ "ė": 211,
+ "Ę": 212,
+ "ę": 213,
+ "Ě": 214,
+ "ě": 215,
+ "Ĝ": 216,
+ "ĝ": 217,
+ "Ğ": 218,
+ "ğ": 219,
+ "Ġ": 220,
+ "ġ": 221,
+ "Ģ": 222,
+ "ģ": 223,
+ "Ĥ": 224,
+ "ĥ": 225,
+ "Ħ": 226,
+ "ħ": 227,
+ "Ĩ": 228,
+ "ĩ": 229,
+ "Ī": 230,
+ "ī": 231,
+ "Ĭ": 232,
+ "ĭ": 233,
+ "Į": 234,
+ "į": 235,
+ "İ": 236,
+ "ı": 237,
+ "IJ": 238,
+ "ij": 239,
+ "Ĵ": 240,
+ "ĵ": 241,
+ "Ķ": 242,
+ "ķ": 243,
+ "ĸ": 244,
+ "Ĺ": 245,
+ "ĺ": 246,
+ "Ļ": 247,
+ "ļ": 248,
+ "Ľ": 249,
+ "ľ": 250,
+ "Ŀ": 251,
+ "ŀ": 252,
+ "Ł": 253,
+ "ł": 254,
+ "Ń": 255,
+ "he": 256,
+ "Ġt": 257,
+ "es": 258,
+ "re": 259,
+ "ve": 260,
+ "wo": 261,
+ "Ġ4": 262,
+ "ĠI": 263,
+ "Ġa": 264,
+ "Ġf": 265,
+ "20": 266,
+ "<<": 267,
+ "de": 268,
+ "do": 269,
+ "in": 270,
+ "la": 271,
+ "ll": 272,
+ "ow": 273,
+ "ove": 274,
+ "Ġ1": 275,
+ "Ġb": 276,
+ "Ġs": 277,
+ "Ġx": 278,
+ "Ġhe": 279,
+ "Ġwo": 280,
+ "Ġ20": 281,
+ "Ġ<<": 282,
+ "Ġthe": 283,
+ "ĠIt": 284,
+ "def": 285,
+ "ine": 286,
+ "'d": 287,
+ "'s": 288,
+ "'t": 289,
+ "'re": 290,
+ "'ve": 291,
+ "'ll": 292,
+ "):": 293,
+ "23": 294,
+ "45": 295,
+ "67": 296,
+ "AX": 297,
+ "BO": 298,
+ "DY": 299,
+ "He": 300,
+ "MAX": 301,
+ "The": 302,
+ "ab": 303,
+ "ac": 304,
+ "af": 305,
+ "aÃ": 306,
+ "bla": 307,
+ "bove": 308,
+ "ck": 309,
+ "caf": 310,
+ "gh": 311,
+ "go": 312,
+ "hre": 313,
+ "if": 314,
+ "im": 315,
+ "is": 316,
+ "ick": 317,
+ "ju": 318,
+ "kn": 319,
+ "ld": 320,
+ "les": 321,
+ "line": 322,
+ "mp": 323,
+ "nd": 324,
+ "nk": 325,
+ "naÃ": 326,
+ "op": 327,
+ "ox": 328,
+ "pp": 329,
+ "pac": 330,
+ "qu": 331,
+ "rn": 332,
+ "row": 333,
+ "rld": 334,
+ "tu": 335,
+ "tes": 336,
+ "top": 337,
+ "we": 338,
+ "ytes": 339,
+ "zy": 340,
+ "¯ve": 341,
+ "é": 342,
+ "Ċĉ": 343,
+ "Ġ#": 344,
+ "Ġ(": 345,
+ "Ġ+": 346,
+ "Ġ=": 347,
+ "Ġ[": 348,
+ "Ġ{": 349,
+ "ĠĠ": 350,
+ "Ġdo": 351,
+ "Ġla": 352,
+ "Ġove": 353,
+ "Ġgo": 354,
+ "Ġis": 355,
+ "Ġju": 356,
+ "Ġkn": 357,
+ "Ġline": 358,
+ "ĠnaÃ": 359,
+ "Ġqu": 360,
+ "Ġwe": 361,
+ "Ġtwo": 362,
+ "Ġthre": 363,
+ "Ġtim": 364,
+ "retu": 365,
+ "Ġ42": 366,
+ "Ġabove": 367,
+ "Ġand": 368,
+ "Ġapp": 369,
+ "Ġfine": 370,
+ "Ġfox": 371,
+ "don": 372,
+ "llo": 373,
+ "Ġ123": 374,
+ "Ġbrow": 375,
+ "Ġbytes": 376,
+ "Ġspac": 377,
+ "Ġstop": 378,
+ "Ġhere": 379,
+ "Ġwon": 380,
+ "Ġworld": 381,
+ "Ġthey": 382,
+ "4567": 383,
+ "BODY": 384,
+ "Hello": 385,
+ "abc": 386,
+ "blank": 387,
+ "café": 388,
+ "ghi": 389,
+ "mps": 390,
+ "Ġdog": 391,
+ "Ġlazy": 392,
+ "Ġover": 393,
+ "Ġjumps": 394,
+ "Ġknow": 395,
+ "Ġnaïve": 396,
+ "Ġquick": 397,
+ "Ġthree": 398,
+ "Ġtimes": 399,
+ "return": 400,
+ "Ġapples": 401,
+ "Ġ1234567": 402,
+ "Ġbrown": 403,
+ "Ġspaces": 404
+ },
+ "merges": [
+ [
+ "h",
+ "e"
+ ],
+ [
+ "Ġ",
+ "t"
+ ],
+ [
+ "e",
+ "s"
+ ],
+ [
+ "r",
+ "e"
+ ],
+ [
+ "v",
+ "e"
+ ],
+ [
+ "w",
+ "o"
+ ],
+ [
+ "Ġ",
+ "4"
+ ],
+ [
+ "Ġ",
+ "I"
+ ],
+ [
+ "Ġ",
+ "a"
+ ],
+ [
+ "Ġ",
+ "f"
+ ],
+ [
+ "2",
+ "0"
+ ],
+ [
+ "<",
+ "<"
+ ],
+ [
+ "d",
+ "e"
+ ],
+ [
+ "d",
+ "o"
+ ],
+ [
+ "i",
+ "n"
+ ],
+ [
+ "l",
+ "a"
+ ],
+ [
+ "l",
+ "l"
+ ],
+ [
+ "o",
+ "w"
+ ],
+ [
+ "o",
+ "ve"
+ ],
+ [
+ "Ġ",
+ "1"
+ ],
+ [
+ "Ġ",
+ "b"
+ ],
+ [
+ "Ġ",
+ "s"
+ ],
+ [
+ "Ġ",
+ "x"
+ ],
+ [
+ "Ġ",
+ "he"
+ ],
+ [
+ "Ġ",
+ "wo"
+ ],
+ [
+ "Ġ",
+ "20"
+ ],
+ [
+ "Ġ",
+ "<<"
+ ],
+ [
+ "Ġt",
+ "he"
+ ],
+ [
+ "ĠI",
+ "t"
+ ],
+ [
+ "de",
+ "f"
+ ],
+ [
+ "in",
+ "e"
+ ],
+ [
+ "'",
+ "d"
+ ],
+ [
+ "'",
+ "s"
+ ],
+ [
+ "'",
+ "t"
+ ],
+ [
+ "'",
+ "re"
+ ],
+ [
+ "'",
+ "ve"
+ ],
+ [
+ "'",
+ "ll"
+ ],
+ [
+ ")",
+ ":"
+ ],
+ [
+ "2",
+ "3"
+ ],
+ [
+ "4",
+ "5"
+ ],
+ [
+ "6",
+ "7"
+ ],
+ [
+ "A",
+ "X"
+ ],
+ [
+ "B",
+ "O"
+ ],
+ [
+ "D",
+ "Y"
+ ],
+ [
+ "H",
+ "e"
+ ],
+ [
+ "M",
+ "AX"
+ ],
+ [
+ "T",
+ "he"
+ ],
+ [
+ "a",
+ "b"
+ ],
+ [
+ "a",
+ "c"
+ ],
+ [
+ "a",
+ "f"
+ ],
+ [
+ "a",
+ "Ã"
+ ],
+ [
+ "b",
+ "la"
+ ],
+ [
+ "b",
+ "ove"
+ ],
+ [
+ "c",
+ "k"
+ ],
+ [
+ "c",
+ "af"
+ ],
+ [
+ "g",
+ "h"
+ ],
+ [
+ "g",
+ "o"
+ ],
+ [
+ "h",
+ "re"
+ ],
+ [
+ "i",
+ "f"
+ ],
+ [
+ "i",
+ "m"
+ ],
+ [
+ "i",
+ "s"
+ ],
+ [
+ "i",
+ "ck"
+ ],
+ [
+ "j",
+ "u"
+ ],
+ [
+ "k",
+ "n"
+ ],
+ [
+ "l",
+ "d"
+ ],
+ [
+ "l",
+ "es"
+ ],
+ [
+ "l",
+ "ine"
+ ],
+ [
+ "m",
+ "p"
+ ],
+ [
+ "n",
+ "d"
+ ],
+ [
+ "n",
+ "k"
+ ],
+ [
+ "n",
+ "aÃ"
+ ],
+ [
+ "o",
+ "p"
+ ],
+ [
+ "o",
+ "x"
+ ],
+ [
+ "p",
+ "p"
+ ],
+ [
+ "p",
+ "ac"
+ ],
+ [
+ "q",
+ "u"
+ ],
+ [
+ "r",
+ "n"
+ ],
+ [
+ "r",
+ "ow"
+ ],
+ [
+ "r",
+ "ld"
+ ],
+ [
+ "t",
+ "u"
+ ],
+ [
+ "t",
+ "es"
+ ],
+ [
+ "t",
+ "op"
+ ],
+ [
+ "w",
+ "e"
+ ],
+ [
+ "y",
+ "tes"
+ ],
+ [
+ "z",
+ "y"
+ ],
+ [
+ "¯",
+ "ve"
+ ],
+ [
+ "Ã",
+ "©"
+ ],
+ [
+ "Ċ",
+ "ĉ"
+ ],
+ [
+ "Ġ",
+ "#"
+ ],
+ [
+ "Ġ",
+ "("
+ ],
+ [
+ "Ġ",
+ "+"
+ ],
+ [
+ "Ġ",
+ "="
+ ],
+ [
+ "Ġ",
+ "["
+ ],
+ [
+ "Ġ",
+ "{"
+ ],
+ [
+ "Ġ",
+ "Ġ"
+ ],
+ [
+ "Ġ",
+ "do"
+ ],
+ [
+ "Ġ",
+ "la"
+ ],
+ [
+ "Ġ",
+ "ove"
+ ],
+ [
+ "Ġ",
+ "go"
+ ],
+ [
+ "Ġ",
+ "is"
+ ],
+ [
+ "Ġ",
+ "ju"
+ ],
+ [
+ "Ġ",
+ "kn"
+ ],
+ [
+ "Ġ",
+ "line"
+ ],
+ [
+ "Ġ",
+ "naÃ"
+ ],
+ [
+ "Ġ",
+ "qu"
+ ],
+ [
+ "Ġ",
+ "we"
+ ],
+ [
+ "Ġt",
+ "wo"
+ ],
+ [
+ "Ġt",
+ "hre"
+ ],
+ [
+ "Ġt",
+ "im"
+ ],
+ [
+ "re",
+ "tu"
+ ],
+ [
+ "Ġ4",
+ "2"
+ ],
+ [
+ "Ġa",
+ "bove"
+ ],
+ [
+ "Ġa",
+ "nd"
+ ],
+ [
+ "Ġa",
+ "pp"
+ ],
+ [
+ "Ġf",
+ "ine"
+ ],
+ [
+ "Ġf",
+ "ox"
+ ],
+ [
+ "do",
+ "n"
+ ],
+ [
+ "ll",
+ "o"
+ ],
+ [
+ "Ġ1",
+ "23"
+ ],
+ [
+ "Ġb",
+ "row"
+ ],
+ [
+ "Ġb",
+ "ytes"
+ ],
+ [
+ "Ġs",
+ "pac"
+ ],
+ [
+ "Ġs",
+ "top"
+ ],
+ [
+ "Ġhe",
+ "re"
+ ],
+ [
+ "Ġwo",
+ "n"
+ ],
+ [
+ "Ġwo",
+ "rld"
+ ],
+ [
+ "Ġthe",
+ "y"
+ ],
+ [
+ "45",
+ "67"
+ ],
+ [
+ "BO",
+ "DY"
+ ],
+ [
+ "He",
+ "llo"
+ ],
+ [
+ "ab",
+ "c"
+ ],
+ [
+ "bla",
+ "nk"
+ ],
+ [
+ "caf",
+ "é"
+ ],
+ [
+ "gh",
+ "i"
+ ],
+ [
+ "mp",
+ "s"
+ ],
+ [
+ "Ġdo",
+ "g"
+ ],
+ [
+ "Ġla",
+ "zy"
+ ],
+ [
+ "Ġove",
+ "r"
+ ],
+ [
+ "Ġju",
+ "mps"
+ ],
+ [
+ "Ġkn",
+ "ow"
+ ],
+ [
+ "ĠnaÃ",
+ "¯ve"
+ ],
+ [
+ "Ġqu",
+ "ick"
+ ],
+ [
+ "Ġthre",
+ "e"
+ ],
+ [
+ "Ġtim",
+ "es"
+ ],
+ [
+ "retu",
+ "rn"
+ ],
+ [
+ "Ġapp",
+ "les"
+ ],
+ [
+ "Ġ123",
+ "4567"
+ ],
+ [
+ "Ġbrow",
+ "n"
+ ],
+ [
+ "Ġspac",
+ "es"
+ ]
+ ]
+ }
+}
\ No newline at end of file
diff --git a/c/tok.h b/c/tok.h
index 54c5c699..83b649df 100644
--- a/c/tok.h
+++ b/c/tok.h
@@ -58,6 +58,8 @@ typedef struct {
int o200k; /* pre_tokenizer regex family: 0 = cl100k (GLM), 1 = o200k (Inkling) */
int kimi; /* 1 = Kimi (K3) family: o200k rules + a leading \p{Han}-run rule,
* Han excluded from the letter classes, no '/' tail in the punct rule */
+ int gpt2; /* 1 = GPT-2 family (OLMoE / GPT-NeoX): a bare ByteLevel pre_tokenizer
+ * with use_regex and no Split, so HF applies the original GPT-2 pattern */
int rankbpe; /* 1 = no merges list (tiktoken-derived vocab): merge the adjacent
* pair whose CONCATENATION has the lowest vocab id — exactly
* tiktoken's byte_pair_encode, no recovered merges to diverge */
@@ -218,6 +220,13 @@ static void tok_load(Tok *T, const char *path){
* case-category classes (\p{Lu}...) which cl100k does not use */
jval *pt=json_get(root,"pre_tokenizer");
if(pt){
+ /* No Split at all, just ByteLevel with use_regex (the default): HF runs
+ * the original GPT-2 regex, not cl100k. OLMoE's tokenizer.json is this. */
+ jval *pty=json_get(pt,"type");
+ if(pty && pty->t==J_STR && pty->str && !strcmp(pty->str,"ByteLevel")){
+ jval *ur=json_get(pt,"use_regex");
+ if(!ur || (ur->t==J_BOOL && ur->boolean)) T->gpt2=1;
+ }
jval *ps=json_get(pt,"pretokenizers");
if(ps&&ps->t==J_ARR) for(int i=0;i<ps->len;i++){
jval *pat=json_get(ps->kids[i],"pattern");
@@ -532,6 +541,52 @@ static void pretok_chunk_kimi(Tok *T, const unsigned char *p, int a, int b, int
free(cp); free(off);
}
+/* ---------- pre-tokenizer GPT-2 (bare ByteLevel with use_regex: OLMoE, GPT-NeoX) ----------
+ * The original GPT-2 pattern, which HF applies when the pre_tokenizer is a
+ * ByteLevel with use_regex=true and no Split in front of it:
+ * 's|'t|'re|'ve|'m|'ll|'d | ?\p{L}+ | ?\p{N}+ | ?[^\s\p{L}\p{N}]+ | \s+(?!\S) | \s+
+ * It differs from cl100k in every rule: the contractions are case-sensitive,
+ * letters and digits take ONE optional leading space (not any non-letter),
+ * digit runs are unbounded, the punctuation run has no newline tail, and
+ * there is no \s*[\r\n]+ rule. Measured on OLMoE against `tokenizers`:
+ * 1560/1708 cases identical under the cl100k rules, every miss a " 4" /
+ * " 20"-style number or a non-space prefix glued to a word. */
+static void pretok_chunk_gpt2(Tok *T, const unsigned char *p, int a, int b, int *out, int *no, int max){
+ int nb=b-a; if(nb<=0) return;
+ uint32_t *cp=malloc((nb+1)*sizeof(uint32_t)); int *off=malloc((nb+2)*sizeof(int)); int n=0;
+ for(int i=a;i<b;){ uint32_t c; int k=u8_next(p,b,i,&c); off[n]=i; cp[n]=c; n++; i+=k; }
+ off[n]=b;
+ int i=0;
+ while(i<n){
+ int start=i; uint32_t c=cp[i];
+ /* 1) 's|'t|'re|'ve|'m|'ll|'d -- case-sensitive, unlike cl100k's (?i:) */
+ if(c=='\'' && i+1<n){
+ uint32_t d=cp[i+1];
+ if(i+2<n){ uint32_t d2=cp[i+2];
+ if((d=='r'&&d2=='e')||(d=='v'&&d2=='e')||(d=='l'&&d2=='l')){ i+=3; bpe_piece(T,p,off[start],off[i],out,no,max); continue; } }
+ if(d=='s'||d=='t'||d=='m'||d=='d'){ i+=2; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
+ }
+ /* 2) ' ?\p{L}+' 3) ' ?\p{N}+' 4) ' ?[^\s\p{L}\p{N}]+' : one optional space, then a run of one class */
+ {
+ int j=i; if(c==' ' && j+1<n) j++;
+ uint32_t d=cp[j];
+ if(is_L(d)){ while(j<n && is_L(cp[j])) j++; i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
+ if(is_N(d)){ while(j<n && is_N(cp[j])) j++; i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
+ if(!is_S(d)){ while(j<n && !is_S(cp[j]) && !is_L(cp[j]) && !is_N(cp[j])) j++; i=j; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
+ }
+ /* 5) \s+(?!\S): a run followed by a non-space keeps its last char for the
+ * next piece (which takes it as its optional space); at the end of the
+ * input the whole run. 6) \s+: a single whitespace before a non-space. */
+ {
+ int r=i; while(r<n && is_S(cp[r])) r++;
+ if(r>i){ int end=(r<n)? r-1 : r; if(end<=i) end=i+1; i=end; bpe_piece(T,p,off[start],off[i],out,no,max); continue; }
+ }
+ i++; /* unreachable: every codepoint is a letter, a digit, whitespace or none of them */
+ bpe_piece(T,p,off[start],off[i],out,no,max);
+ }
+ free(cp); free(off);
+}
+
/* ---------- encode: testo -> id (split sugli added token, poi pretok+BPE) ---------- */
static int tok_encode(Tok *T, const char *text, int len, int *out, int max){
const unsigned char *p=(const unsigned char*)text; int no=0; int i=0;
@@ -546,7 +601,8 @@ static int tok_encode(Tok *T, const char *text, int len, int *out, int max){
}
int chunk_end = (hitpos<0) ? len : hitpos;
if(chunk_end>i){
- if(T->kimi) pretok_chunk_kimi(T,p,i,chunk_end,out,&no,max);
+ if(T->gpt2) pretok_chunk_gpt2(T,p,i,chunk_end,out,&no,max);
+ else if(T->kimi) pretok_chunk_kimi(T,p,i,chunk_end,out,&no,max);
else if(T->o200k) pretok_chunk_o200k(T,p,i,chunk_end,out,&no,max);
else pretok_chunk(T,p,i,chunk_end,out,&no,max);
}