37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* tok_check.c - checks the tokenizer: encodes a text (special tokens
* parsed), compares the ids with a reference file of int32 ids when given,
* and decodes them back, which must give the text again.
*
* Usage: tok_check <model.jns> <text> [reference.tokens]
*/
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include <time.h>
#include "../src/llm/jns.h"
#include "../src/llm/tokenizer.h"
static char *read_file(const char *path, size_t *len)
{
FILE *f = fopen(path, "rb");
if (!f)
return NULL;
fseek(f, 0, SEEK_END);
long n = ftell(f);
fseek(f, 0, SEEK_SET);
char *b = malloc((size_t)n + 1);
if (b && fread(b, 1, (size_t)n, f) != (size_t)n) {
free(b);
b = NULL;
}
fclose(f);
*len = (size_t)n;
return b;
}
static double now(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return (double)ts.tv_sec + 1e-9 * (double)ts.tv_nsec;
}
int main(int argc, char **argv)
{
if (argc < 3) {
fprintf(stderr, "usage: tok_check <model.jns> <text> [reference]\n");
return 2;
}
char err[256] = "";
struct janas_jns j;
if (janas_jns_open(argv[1], &j, err, sizeof(err)) != 0) {
fprintf(stderr, "%s\n", err);
return 1;
}
double t0 = now();
struct janas_tokenizer *t = janas_tokenizer_create(&j, err, sizeof(err));
if (!t) {
fprintf(stderr, "%s\n", err);
return 1;
}
double t1 = now();
size_t len;
char *text = read_file(argv[2], &len);
if (!text)
return 1;
int32_t *ids = malloc((len + 16) * sizeof(int32_t));
long n = janas_tokenizer_encode(t, text, len, 1, ids, len + 16);
double t2 = now();
printf("vocabulary %u tokens, loaded in %.1f ms\n",
janas_tokenizer_n_vocab(t), 1e3 * (t1 - t0));
printf("%zu bytes -> %ld tokens in %.2f ms (%.1f MB/s)\n", len, n,
1e3 * (t2 - t1), (double)len / (t2 - t1) / 1e6);
int bad = 0;
if (argc > 3) {
size_t rl;
int32_t *ref = (int32_t *)read_file(argv[3], &rl);
long rn = ref ? (long)(rl / 4) : 0;
long m = rn < n ? rn : n, i = 0;
while (i < m && ids[i] == ref[i])
i++;
if (i == m && rn == n) {
printf("reference: %ld tokens, identical\n", rn);
} else {
bad = 1;
printf("reference: %ld tokens, first difference at %ld\n", rn, i);
for (long k = i > 3 ? i - 3 : 0; k < i + 6; k++) {
char a[64] = "", b[64] = "";
size_t la =
k < n ? janas_tokenizer_decode(t, ids[k], a, 63) : 0;
size_t lb =
k < rn ? janas_tokenizer_decode(t, ref[k], b, 63) : 0;
a[la < 63 ? la : 63] = 0;
b[lb < 63 ? lb : 63] = 0;
printf(" %6ld own %6d [%s] ref %6d [%s]\n", k,
k < n ? ids[k] : -1, a, k < rn ? ref[k] : -1, b);
}
}
free(ref);
}
size_t w = 0;
for (long i = 0; i < n; i++) {
char buf[256];
size_t l = janas_tokenizer_decode(t, ids[i], buf, sizeof(buf));
if (w + l > len || memcmp(text + w, buf, l) != 0) {
printf("round trip: differs at token %ld (byte %zu)\n", i, w);
bad = 1;
break;
}
w += l;
}
if (!bad || w == len)
printf("round trip: %s\n", w == len ? "identical" : "short");
free(ids);
free(text);
janas_tokenizer_destroy(t);
janas_jns_close(&j);
return bad;
}