37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* bench_kernels.c - every weight type's product, on one core and on many,
* over 1 to 8 vectors, in GB/s of weights read: to set beside the memory's
* bandwidth (about 81 GB/s on the development machine). A product of one
* vector reads each weight once and does little with it, so a kernel well
* below the others on one core is spending its time computing, not waiting
* for memory - which is how the Q3_K, IQ4_XS and IQ3_S kernels were found
* to convert their scale through a function call (6 Oct 2026).
*
* Usage: bench_kernels [MiB [threads [first_cpu]]]
* BENCH_MEM=driver puts the weights in memory allocated by the GPU's driver
* (the memory a GPU sharing the host's reads fastest), BENCH_MEM=huge2m in
* ordinary memory aligned to 2 MB and collapsed into huge pages: the CPU's
* speed on each, against malloc's (13 Oct 2026).
* BENCH_TYPES=Q4_K,Q2_0 measures the types named alone; BENCH_NV=8,16,64
* the vectors a product takes (1,2,4,8 by default, 512 at most: a prompt's
* block): where the prompt's tiles (JANAS_CPU_TILES) pay against the rows'
* products. Defaults: 256 MiB of weights per type, 12 threads on CPUs 0-11; the
* single core is first_cpu (default 2).
* The weights are random bytes below 0x7c, so that every fp16 scale is a
* finite number; the speed of a kernel does not depend on the values.
*/
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#if defined(__linux__)
#include <sys/mman.h>
#endif
#include <time.h>
#include "common/aligned.h"
#include "common/pool.h"
#include "llm/matvec.h"
#include "llm/gpu.h"
#include "llm/quant.h"
static double now(void)
{
struct timespec ts;
clock_gettime(CLOCK_MONOTONIC, &ts);
return ts.tv_sec + ts.tv_nsec * 1e-9;
}
#ifndef MADV_COLLAPSE
#define MADV_COLLAPSE 25
#endif
/* where the weights go (BENCH_MEM): malloc, the GPU driver, 2 MB pages */
static struct janas_gpu *gpu;
static int mem_kind; /* 0 malloc, 1 driver, 2 huge2m */
static unsigned char *weights(size_t bytes)
{
if (mem_kind == 1)
return janas_gpu_alloc(gpu, bytes);
if (mem_kind == 2) {
size_t al = (size_t)2 << 20;
return janas_aligned_alloc(al, (bytes + al - 1) / al * al);
}
return malloc(bytes);
}
static const struct {
const char *name;
int type;
} TYPES[] = {{"Q4_K", JANAS_Q4_K}, {"Q5_K", JANAS_Q5_K},
{"Q6_K", JANAS_Q6_K}, {"Q3_K", JANAS_Q3_K},
{"Q8_0", JANAS_Q8_0}, {"Q4_0", JANAS_Q4_0},
{"Q4_1", JANAS_Q4_1}, {"Q5_1", JANAS_Q5_1},
{"IQ4_NL", JANAS_IQ4_NL}, {"IQ4_XS", JANAS_IQ4_XS},
{"IQ3_S", JANAS_IQ3_S}, {"MXFP4", JANAS_MXFP4},
{"Q5_0", JANAS_Q5_0}, {"Q2_0", JANAS_Q2_0},
{"Q2_K", JANAS_Q2_K}, {"IQ3_XXS", JANAS_IQ3_XXS},
{"IQ2_XXS", JANAS_IQ2_XXS}, {"IQ2_XS", JANAS_IQ2_XS},
{"IQ2_S", JANAS_IQ2_S}, {"IQ1_S", JANAS_IQ1_S},
{"IQ1_M", JANAS_IQ1_M}};
/* BENCH_TYPES=Q4_K,Q2_0 measures those types alone (all when unset). */
static int wanted(const char *name)
{
const char *list = getenv("BENCH_TYPES");
if (!list || !*list)
return 1;
size_t n = strlen(name);
for (const char *p = list; *p;) {
const char *e = strchr(p, ',');
size_t len = e ? (size_t)(e - p) : strlen(p);
if (len == n && !strncmp(p, name, n))
return 1;
p += len + (e != NULL);
}
return 0;
}
/* Seconds per product, the best of passes run for at least 0.3 s. */
static double timed(struct janas_pool *pool, struct janas_matvec_task *t)
{
double best = 1e9, start = now();
int runs = 0;
while (runs < 3 || now() - start < 0.3) {
double t0 = now();
janas_matvec_q4k_group(pool, t, 1);
double dt = now() - t0;
if (dt < best)
best = dt;
runs++;
}
return best;
}
int main(int argc, char **argv)
{
size_t mib = argc > 1 ? (size_t)atol(argv[1]) : 256;
int nt = argc > 2 ? atoi(argv[2]) : 12;
int one = argc > 3 ? atoi(argv[3]) : 2;
if (mib < 1 || nt < 1 || nt > 256 || one < 0) {
fprintf(stderr, "usage: bench_kernels [MiB [threads [first_cpu]]]\n");
return 2;
}
const char *bm = getenv("BENCH_MEM");
mem_kind = !bm ? 0
: !strcmp(bm, "driver") ? 1
: !strcmp(bm, "huge2m") ? 2
: 0;
if (mem_kind == 1) {
char err[200];
gpu = janas_gpu_create(err, sizeof err);
if (!gpu) {
fprintf(stderr, "bench_kernels: no GPU: %s\n", err);
return 1;
}
}
int cpus[256];
for (int i = 0; i < nt; i++)
cpus[i] = i;
struct janas_pool *many = janas_pool_create(nt, cpus);
struct janas_pool *single = janas_pool_create(1, &one);
if (!many || !single) {
fprintf(stderr, "bench_kernels: no thread pool\n");
return 1;
}
enum { MAX_NV = 512 };
size_t nvs[8] = {1, 2, 4, 8}, n_nv = 4;
const char *ev = getenv("BENCH_NV");
if (ev) {
n_nv = 0;
for (const char *c = ev; *c && n_nv < 8;) {
long n = strtol(c, (char **)&c, 10);
if (n >= 1 && n <= MAX_NV)
nvs[n_nv++] = (size_t)n;
while (*c == ',' || *c == ' ')
c++;
if (*c && (*c < '0' || *c > '9'))
break;
}
if (!n_nv) {
fprintf(stderr, "bench_kernels: BENCH_NV lists 1 to 512\n");
return 1;
}
}
const size_t cols = 2048, nb = cols / JANAS_QK;
struct janas_block_q8k *x = calloc(MAX_NV * nb, sizeof(*x));
float *v = malloc(MAX_NV * cols * sizeof(float));
srand(1);
for (size_t i = 0; i < MAX_NV * cols; i++)
v[i] = (float)rand() / RAND_MAX - 0.5f;
for (int k = 0; k < MAX_NV; k++)
janas_q8k_quantize(v + k * cols, x + k * nb, cols);
printf("GB/s of weights, %zu MiB a type, 2048 columns; 1 core (CPU %d) "
"and %d threads; weights in %s\n",
mib, one, nt,
mem_kind == 1 ? "the GPU driver's memory"
: mem_kind == 2 ? "2 MB pages"
: "malloc's memory");
printf("%-7s", "type");
for (int p = 0; p < 2; p++) {
for (size_t i = 0; i < n_nv; i++) {
char h[16];
snprintf(h, sizeof(h), "%sx%zu", p ? "N" : "1", nvs[i]);
printf(" %8s", h);
}
printf(p ? "\n" : " |");
}
for (size_t k = 0; k < sizeof TYPES / sizeof *TYPES; k++) {
if (!wanted(TYPES[k].name))
continue;
size_t bs = janas_qtype_block_size(TYPES[k].type);
size_t rows = (mib << 20) / (nb * bs), bytes = rows * nb * bs;
unsigned char *w = weights(bytes);
float *y = malloc(MAX_NV * rows * sizeof(float));
if (!w || !y) {
fprintf(stderr, "bench_kernels: out of memory\n");
return 1;
}
for (size_t i = 0; i < bytes; i++)
w[i] = (unsigned char)(rand() % 0x7c);
/* MXFP4's scale byte is a power of two: the ones of real weights,
not subnormal floats, whose products take the slow path */
if (TYPES[k].type == JANAS_MXFP4)
for (size_t i = 0; i < bytes; i += sizeof(struct janas_block_mxfp4))
w[i] = (unsigned char)(118 + rand() % 12);
/* the IQ types' scale of a block, at its start (IQ1_M: the top
nibbles of its four scale words), a normal fp16 as in real
weights: a subnormal one sends every float product of the block
down the slow path (IQ1_M read 0.3 GB/s on random bytes) */
int t = TYPES[k].type;
if (t == JANAS_IQ2_XXS || t == JANAS_IQ2_XS || t == JANAS_IQ2_S ||
t == JANAS_IQ3_XXS || t == JANAS_IQ1_S)
for (size_t i = 0; i < bytes; i += bs) {
uint16_t d = (uint16_t)(0x2000 + rand() % 0x1000);
memcpy(w + i, &d, 2);
}
if (t == JANAS_IQ1_M)
for (size_t i = 0; i < bytes; i += bs)
w[i + bs - 1] = (unsigned char)(0x20 | (w[i + bs - 1] & 0x0f));
#if defined(__linux__)
if (mem_kind == 2)
madvise(w, bytes, MADV_COLLAPSE);
#endif
printf("%-7s", TYPES[k].name);
for (int p = 0; p < 2; p++) {
for (size_t i = 0; i < n_nv; i++) {
struct janas_matvec_task t = {.type = TYPES[k].type,
.w = w,
.x = x,
.y = y,
.rows = rows,
.cols = cols,
.n_vec = nvs[i],
.x_stride = nb,
.y_stride = rows};
double s = timed(p ? many : single, &t);
printf(" %8.1f", bytes / s / 1e9);
}
printf(p ? "\n" : " |");
}
fflush(stdout);
if (mem_kind == 2) /* the driver's goes with the GPU */
janas_aligned_free(w);
else if (mem_kind == 0)
free(w);
free(y);
}
free(x);
free(v);
janas_pool_destroy(single);
janas_pool_destroy(many);
janas_gpu_destroy(gpu);
return 0;
}