Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦
C
fix(qwen36): read tokenizer.json merges in string and pair spellings
Léon Avic Simmonscfd51066dfdc
diff --git a/.gitignore b/.gitignoreindex 0fe3c15b..8837be44 100644--- a/.gitignore+++ b/.gitignore@@ -63,6 +63,8 @@ c/tests/test_*!c/tests/test_*.cu!c/tests/test_*.mm!c/tests/test_*.py+# scratch fixture directories the test binaries write at run time+c/tests/tmp_*# bench_* and fuzz_* build to extensionless binaries in the same directory and# were not covered here, which is how c/tests/bench_omp_grain reached the tree.c/tests/bench_*diff --git a/c/Makefile b/c/Makefileindex 141b4260..6aef99b1 100644--- a/c/Makefile+++ b/c/Makefile@@ -1067,6 +1067,11 @@ tests/test_qwen36_dense_batch$(EXE): tests/test_qwen36_dense_batch.c qwen36.c qwtests/test_qwen36_json_escape$(EXE): tests/test_qwen36_json_escape.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)+# Both tokenizer.json merge spellings ("a b" strings and ["a","b"] pairs)+# must index the same merge table; the pair form is what Qwen3.6 ships.+tests/test_qwen36_tok_merges$(EXE): tests/test_qwen36_tok_merges.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)+# Reproducible local timing evidence; intentionally not a noisy CI perf gate.tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)diff --git a/c/qwen36.c b/c/qwen36.cindex 6ee29bf2..baec143e 100644--- a/c/qwen36.c+++ b/c/qwen36.c@@ -293,11 +293,29 @@ static void load_tokenizer(const char *path){jval *merges = json_get(model, "merges");if (merges && merges->t==J_ARR){for (int r=0;r<merges->len;r++){- const char *e = merges->kids[r]->str; if(!e) continue;- const char *sp = strchr(e, ' '); if(!sp) continue;- int la=(int)(sp-e), lb=(int)strlen(sp+1);+ /* Two on-disk spellings for one merge table: legacy tokenizer.json+ * writes "a b" strings, tokenizers >= 0.20 (transformers 4.45+,+ * the Qwen3.6 checkpoints included) writes ["a","b"] pairs. The+ * string-only reader SILENTLY indexed zero merges from the pair+ * form, and encode_text degraded to one token per byte-symbol --+ * 24 tokens for a 24-char prompt, real-model run -- because+ * bpe_piece treats an empty merge table as "nothing to merge",+ * not as an error. */+ jval *mk = merges->kids[r];+ const char *a, *b;+ int la, lb;+ if (mk && mk->t==J_STR && mk->str){+ const char *sp = strchr(mk->str, ' '); if(!sp) continue;+ a = mk->str; la = (int)(sp - mk->str);+ b = sp + 1; lb = (int)strlen(b);+ } else if (mk && mk->t==J_ARR && mk->len==2 &&+ mk->kids[0] && mk->kids[0]->t==J_STR && mk->kids[0]->str &&+ mk->kids[1] && mk->kids[1]->t==J_STR && mk->kids[1]->str){+ a = mk->kids[0]->str; la = (int)strlen(a);+ b = mk->kids[1]->str; lb = (int)strlen(b);+ } else continue;char *key=malloc(la+1+lb+1);- memcpy(key,e,la); key[la]=0x1F; memcpy(key+la+1,sp+1,lb); key[la+1+lb]=0;+ memcpy(key,a,la); key[la]=0x1F; memcpy(key+la+1,b,lb); key[la+1+lb]=0;smap_put(&g_merge, key, r);}}diff --git a/c/tests/test_qwen36_tok_merges.c b/c/tests/test_qwen36_tok_merges.cnew file mode 100644index 00000000..91ec10bb--- /dev/null+++ b/c/tests/test_qwen36_tok_merges.c@@ -0,0 +1,77 @@+/* Tokenizer merge-table format gate. tokenizer.json spells its merges two+ * ways: legacy files write "a b" strings, tokenizers >= 0.20 (transformers+ * 4.45+, the Qwen3.6 checkpoints included) writes ["a","b"] pairs. The+ * string-only reader silently indexed ZERO merges from the pair form and+ * encode_text degraded to one token per byte-symbol -- observed as a 24-token+ * encoding of a 24-char prompt on the first real-model run, with the model+ * fed a token stream it never saw in training. bpe_piece treats an empty+ * merge table as "nothing to merge", so nothing ever refused.+ *+ * Both spellings of the SAME tiny model must produce the SAME merged ids:+ * "in in" -> [in, Ġin], exercising both a plain merge (i+n) and a chained+ * one (Ġ + in, rank-ordered after i+n produces the intermediate). */+#ifndef _GNU_SOURCE+#define _GNU_SOURCE+#endif++#define main qwen36_main_unused+#include "../qwen36.c"+#undef main++#include <sys/stat.h>++#define CHECK(condition) do { \+ if (!(condition)) { \+ fprintf(stderr, "%s:%d: check failed: %s\n", \+ __FILE__, __LINE__, #condition); \+ exit(1); \+ } \+} while (0)++static const char *tok_pairs =+ "{\"model\":{\"vocab\":{\"i\":0,\"n\":1,\"\\u0120\":2,\"in\":3,"+ "\"\\u0120in\":4},\"merges\":[[\"i\",\"n\"],[\"\\u0120\",\"in\"]]}}";+static const char *tok_strings =+ "{\"model\":{\"vocab\":{\"i\":0,\"n\":1,\"\\u0120\":2,\"in\":3,"+ "\"\\u0120in\":4},\"merges\":[\"i n\",\"\\u0120 in\"]}}";++static void write_file(const char *path, const char *text) {+ FILE *f = fopen(path, "wb");+ CHECK(f != NULL);+ CHECK(fwrite(text, 1, strlen(text), f) == strlen(text));+ CHECK(fclose(f) == 0);+}++static void check_encoding(const char *tok_path, const char *label) {+ load_tokenizer(tok_path);+ CHECK(g_tok != NULL && g_tok_n == 5);+ int *ids = NULL, n = 0;+ encode_text("in in", &ids, &n);+ if (n != 2 || ids[0] != 3 || ids[1] != 4) {+ fprintf(stderr, "%s: expected [3, 4], got %d ids:", label, n);+ for (int i = 0; i < n; i++) fprintf(stderr, " %d", ids[i]);+ fprintf(stderr, "\n");+ exit(1);+ }+ free(ids);+}++int main(void) {+ const char *dir = "tests/tmp_tok_merges";+#ifdef _WIN32+ _mkdir(dir);+#else+ mkdir(dir, 0700);+#endif+ char pairs_path[256], strings_path[256];+ snprintf(pairs_path, sizeof(pairs_path), "%s/tokenizer_pairs.json", dir);+ snprintf(strings_path, sizeof(strings_path), "%s/tokenizer_strings.json", dir);+ write_file(pairs_path, tok_pairs);+ write_file(strings_path, tok_strings);+ /* load_tokenizer is re-entrant enough for a test: smap_init re-allocates+ * fresh tables and neither fixture carries added_tokens. */+ check_encoding(strings_path, "string merges");+ check_encoding(pairs_path, "pair merges");+ printf("test_qwen36_tok_merges: OK\n");+ return 0;+}