Skip to content

Run frontier MoE models on hardware you already own — pure C, zero deps, experts streamed from disk. Tiny engine, immense model. 🐦

C

fix(qwen36): read tokenizer.json merges in string and pair spellings

Léon Avic Simmonscfd51066dfdc
diff --git a/.gitignore b/.gitignore
index 0fe3c15b..8837be44 100644
--- a/.gitignore
+++ b/.gitignore
@@ -63,6 +63,8 @@ c/tests/test_*
!c/tests/test_*.cu
!c/tests/test_*.mm
!c/tests/test_*.py
+# scratch fixture directories the test binaries write at run time
+c/tests/tmp_*
# bench_* and fuzz_* build to extensionless binaries in the same directory and
# were not covered here, which is how c/tests/bench_omp_grain reached the tree.
c/tests/bench_*
diff --git a/c/Makefile b/c/Makefile
index 141b4260..6aef99b1 100644
--- a/c/Makefile
+++ b/c/Makefile
@@ -1067,6 +1067,11 @@ tests/test_qwen36_dense_batch$(EXE): tests/test_qwen36_dense_batch.c qwen36.c qw
tests/test_qwen36_json_escape$(EXE): tests/test_qwen36_json_escape.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
+# Both tokenizer.json merge spellings ("a b" strings and ["a","b"] pairs)
+# must index the same merge table; the pair form is what Qwen3.6 ships.
+tests/test_qwen36_tok_merges$(EXE): tests/test_qwen36_tok_merges.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
+ $(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
+
# Reproducible local timing evidence; intentionally not a noisy CI perf gate.
tests/bench_qwen36_dense_batch$(EXE): tests/bench_qwen36_dense_batch.c qwen36.c qwen36_tier.h st.h json.h compat.h $(QWEN36_TIER_SRC) $(CUDA_OBJ)
$(CC) $(QWEN36_CFLAGS) $< $(QWEN36_TIER_SRC) $(CUDA_OBJ) -o $@ $(QWEN36_LDFLAGS)
diff --git a/c/qwen36.c b/c/qwen36.c
index 6ee29bf2..baec143e 100644
--- a/c/qwen36.c
+++ b/c/qwen36.c
@@ -293,11 +293,29 @@ static void load_tokenizer(const char *path){
jval *merges = json_get(model, "merges");
if (merges && merges->t==J_ARR){
for (int r=0;r<merges->len;r++){
- const char *e = merges->kids[r]->str; if(!e) continue;
- const char *sp = strchr(e, ' '); if(!sp) continue;
- int la=(int)(sp-e), lb=(int)strlen(sp+1);
+ /* Two on-disk spellings for one merge table: legacy tokenizer.json
+ * writes "a b" strings, tokenizers >= 0.20 (transformers 4.45+,
+ * the Qwen3.6 checkpoints included) writes ["a","b"] pairs. The
+ * string-only reader SILENTLY indexed zero merges from the pair
+ * form, and encode_text degraded to one token per byte-symbol --
+ * 24 tokens for a 24-char prompt, real-model run -- because
+ * bpe_piece treats an empty merge table as "nothing to merge",
+ * not as an error. */
+ jval *mk = merges->kids[r];
+ const char *a, *b;
+ int la, lb;
+ if (mk && mk->t==J_STR && mk->str){
+ const char *sp = strchr(mk->str, ' '); if(!sp) continue;
+ a = mk->str; la = (int)(sp - mk->str);
+ b = sp + 1; lb = (int)strlen(b);
+ } else if (mk && mk->t==J_ARR && mk->len==2 &&
+ mk->kids[0] && mk->kids[0]->t==J_STR && mk->kids[0]->str &&
+ mk->kids[1] && mk->kids[1]->t==J_STR && mk->kids[1]->str){
+ a = mk->kids[0]->str; la = (int)strlen(a);
+ b = mk->kids[1]->str; lb = (int)strlen(b);
+ } else continue;
char *key=malloc(la+1+lb+1);
- memcpy(key,e,la); key[la]=0x1F; memcpy(key+la+1,sp+1,lb); key[la+1+lb]=0;
+ memcpy(key,a,la); key[la]=0x1F; memcpy(key+la+1,b,lb); key[la+1+lb]=0;
smap_put(&g_merge, key, r);
}
}
diff --git a/c/tests/test_qwen36_tok_merges.c b/c/tests/test_qwen36_tok_merges.c
new file mode 100644
index 00000000..91ec10bb
--- /dev/null
+++ b/c/tests/test_qwen36_tok_merges.c
@@ -0,0 +1,77 @@
+/* Tokenizer merge-table format gate. tokenizer.json spells its merges two
+ * ways: legacy files write "a b" strings, tokenizers >= 0.20 (transformers
+ * 4.45+, the Qwen3.6 checkpoints included) writes ["a","b"] pairs. The
+ * string-only reader silently indexed ZERO merges from the pair form and
+ * encode_text degraded to one token per byte-symbol -- observed as a 24-token
+ * encoding of a 24-char prompt on the first real-model run, with the model
+ * fed a token stream it never saw in training. bpe_piece treats an empty
+ * merge table as "nothing to merge", so nothing ever refused.
+ *
+ * Both spellings of the SAME tiny model must produce the SAME merged ids:
+ * "in in" -> [in, Ġin], exercising both a plain merge (i+n) and a chained
+ * one (Ġ + in, rank-ordered after i+n produces the intermediate). */
+#ifndef _GNU_SOURCE
+#define _GNU_SOURCE
+#endif
+
+#define main qwen36_main_unused
+#include "../qwen36.c"
+#undef main
+
+#include <sys/stat.h>
+
+#define CHECK(condition) do { \
+ if (!(condition)) { \
+ fprintf(stderr, "%s:%d: check failed: %s\n", \
+ __FILE__, __LINE__, #condition); \
+ exit(1); \
+ } \
+} while (0)
+
+static const char *tok_pairs =
+ "{\"model\":{\"vocab\":{\"i\":0,\"n\":1,\"\\u0120\":2,\"in\":3,"
+ "\"\\u0120in\":4},\"merges\":[[\"i\",\"n\"],[\"\\u0120\",\"in\"]]}}";
+static const char *tok_strings =
+ "{\"model\":{\"vocab\":{\"i\":0,\"n\":1,\"\\u0120\":2,\"in\":3,"
+ "\"\\u0120in\":4},\"merges\":[\"i n\",\"\\u0120 in\"]}}";
+
+static void write_file(const char *path, const char *text) {
+ FILE *f = fopen(path, "wb");
+ CHECK(f != NULL);
+ CHECK(fwrite(text, 1, strlen(text), f) == strlen(text));
+ CHECK(fclose(f) == 0);
+}
+
+static void check_encoding(const char *tok_path, const char *label) {
+ load_tokenizer(tok_path);
+ CHECK(g_tok != NULL && g_tok_n == 5);
+ int *ids = NULL, n = 0;
+ encode_text("in in", &ids, &n);
+ if (n != 2 || ids[0] != 3 || ids[1] != 4) {
+ fprintf(stderr, "%s: expected [3, 4], got %d ids:", label, n);
+ for (int i = 0; i < n; i++) fprintf(stderr, " %d", ids[i]);
+ fprintf(stderr, "\n");
+ exit(1);
+ }
+ free(ids);
+}
+
+int main(void) {
+ const char *dir = "tests/tmp_tok_merges";
+#ifdef _WIN32
+ _mkdir(dir);
+#else
+ mkdir(dir, 0700);
+#endif
+ char pairs_path[256], strings_path[256];
+ snprintf(pairs_path, sizeof(pairs_path), "%s/tokenizer_pairs.json", dir);
+ snprintf(strings_path, sizeof(strings_path), "%s/tokenizer_strings.json", dir);
+ write_file(pairs_path, tok_pairs);
+ write_file(strings_path, tok_strings);
+ /* load_tokenizer is re-entrant enough for a test: smap_init re-allocates
+ * fresh tables and neither fixture carries added_tokens. */
+ check_encoding(strings_path, "string merges");
+ check_encoding(pairs_path, "pair merges");
+ printf("test_qwen36_tok_merges: OK\n");
+ return 0;
+}