37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* gen_unicode.c - generates src/llm/unicode_data.h: code point ranges of the
* Unicode letter (L*), number (N*) and mark (M*) categories and of the
* White_Space property, for the pre-tokenizer of src/llm/tokenizer.c.
*
* It reads the Unicode Character Database: the general categories from
* UnicodeData.txt (a code point it does not list is unassigned, Cn), the
* version from the first line of DerivedAge.txt. Debian and its kin ship
* the database in /usr/share/unicode (package unicode-data); it is also at
* https://www.unicode.org/Public/<version>/ucd/.
*
* Usage: gen_unicode <ucd-directory> > src/llm/unicode_data.h
*/
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#define N_CP 0x110000
static const uint32_t WHITE_SPACE[][2] = {
{0x09, 0x0D}, {0x20, 0x20}, {0x85, 0x85}, {0xA0, 0xA0},
{0x1680, 0x1680}, {0x2000, 0x200A}, {0x2028, 0x2029}, {0x202F, 0x202F},
{0x205F, 0x205F}, {0x3000, 0x3000},
};
/* the first letter of each code point's general category; 'C' when the
database does not list it */
static char major[N_CP];
static FILE *open_in(const char *dir, const char *name)
{
char path[4096];
snprintf(path, sizeof path, "%s/%s", dir, name);
FILE *f = fopen(path, "r");
if (!f)
perror(path);
return f;
}
/* Fields: code;name;category;... A range is two lines, its name ending in
", First>" and ", Last>". */
static int read_categories(const char *dir)
{
FILE *f = open_in(dir, "UnicodeData.txt");
if (!f)
return -1;
memset(major, 'C', sizeof major);
char line[1024];
long first = -1, lines = 0;
while (fgets(line, sizeof line, f)) {
char *name = strchr(line, ';');
char *cat = name ? strchr(name + 1, ';') : NULL;
char *end;
long cp = strtol(line, &end, 16);
if (!cat || end != name || cp < 0 || cp >= N_CP) {
fprintf(stderr, "UnicodeData.txt: bad line: %s", line);
fclose(f);
return -1;
}
size_t name_n = (size_t)(cat - name - 1);
if (name_n >= 7 && !memcmp(cat - 7, ", Last>", 7)) {
if (first < 0 || first > cp) {
fprintf(stderr, "UnicodeData.txt: range without a start\n");
fclose(f);
return -1;
}
memset(major + first, cat[1], (size_t)(cp - first + 1));
first = -1;
} else {
major[cp] = cat[1];
if (name_n >= 8 && !memcmp(cat - 8, ", First>", 8))
first = cp;
}
lines++;
}
fclose(f);
if (lines == 0) {
fprintf(stderr, "UnicodeData.txt: empty\n");
return -1;
}
return 0;
}
/* The version from "# DerivedAge-15.1.0.txt". */
static int read_version(const char *dir, char *out, size_t out_n)
{
FILE *f = open_in(dir, "DerivedAge.txt");
if (!f)
return -1;
char line[256];
int ok = fgets(line, sizeof line, f) != NULL;
fclose(f);
const char *p = ok ? strstr(line, "DerivedAge-") : NULL;
const char *e = p ? strstr(p, ".txt") : NULL;
if (!e || (size_t)(e - p - 11) >= out_n || e - p - 11 == 0) {
fprintf(stderr, "DerivedAge.txt: no version on its first line\n");
return -1;
}
memcpy(out, p + 11, (size_t)(e - p - 11));
out[e - p - 11] = 0;
return 0;
}
static void emit_head(const char *name)
{
printf("static const uint32_t %s[][2] = {\n", name);
}
static void emit_category(const char *name, char c)
{
emit_head(name);
long start = -1;
for (long cp = 0; cp <= N_CP; cp++) {
int ok = cp < N_CP && major[cp] == c;
if (ok && start < 0) {
start = cp;
} else if (!ok && start >= 0) {
printf(" {0x%lX, 0x%lX},\n", start, cp - 1);
start = -1;
}
}
printf("};\n");
}
int main(int argc, char **argv)
{
if (argc != 2) {
fprintf(stderr, "usage: gen_unicode <ucd-directory> "
"> src/llm/unicode_data.h\n");
return 2;
}
char version[32];
if (read_version(argv[1], version, sizeof version) != 0 ||
read_categories(argv[1]) != 0)
return 1;
printf("/* SPDX-License-Identifier: GPL-3.0-or-later\n"
" Copyright (C) 2026 Maurizio Cammalleri */\n");
printf("/* Generated by tools/gen_unicode.c from Unicode %s. "
"Do not edit. */\n",
version);
printf("#ifndef JANAS_LLM_UNICODE_DATA_H\n#define JANAS_LLM_UNICODE_DATA_H"
"\n\n");
printf("#include <stdint.h>\n\n");
emit_category("janas_uni_letter", 'L');
emit_category("janas_uni_number", 'N');
emit_category("janas_uni_mark", 'M');
emit_head("janas_uni_space");
for (size_t i = 0; i < sizeof WHITE_SPACE / sizeof *WHITE_SPACE; i++)
printf(" {0x%X, 0x%X},\n", WHITE_SPACE[i][0], WHITE_SPACE[i][1]);
printf("};\n\n#endif\n");
return ferror(stdout) ? 1 : 0;
}