37 tok/s from Qwen3-Next-80B (48 GB) on a 32 GB laptop, 24 without drafts vs 2.8 for llama.cpp; reading 62 vs 16 tok/s. Faster than llama.cpp on all 13 models compared in Q4_K_M. LLMs bigger than your RAM, pure C, CPU or GPU.
C avx2ccpu-inferenceggufinference-enginelinuxllmlocal-llmmixture-of-expertsmoenvmequantizationqwen3simdspeculative-decodingvulkan
/* SPDX-License-Identifier: GPL-3.0-or-later
Copyright (C) 2026 Maurizio Cammalleri */
/*
* test_npu.c - the NPU layer's parts that need no NPU: the rows' INT8
* quantization (scales in FP16, q s within half a step of the weight) and
* the IR writer (the compiler's input laid out as the driver reads it).
* The NPU itself: npu_check.
*/
#include <math.h>
#include <stdint.h>
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "common/npu.h"
#include "common/npu_ir.h"
static float h2f(uint16_t h)
{
uint32_t sign = (uint32_t)(h & 0x8000u) << 16, e = (h >> 10) & 31,
mant = h & 0x3ffu, b;
if (!e && !mant)
b = sign;
else if (!e) {
float f = ldexpf((float)mant, -24);
return sign ? -f : f;
} else
b = sign | ((e - 15 + 127) << 23) | (mant << 13);
float f;
memcpy(&f, &b, 4);
return f;
}
int main(void)
{
int failures = 0;
/* quantization */
enum { N = 7, K = 300 };
float *w = malloc(sizeof(float) * N * K);
int8_t *q = malloc(N * K);
uint16_t s[N];
unsigned r = 12345;
for (int i = 0; i < N * K; i++) {
r = r * 1103515245u + 12345u;
w[i] = ((float)(r >> 8) / (float)(1u << 24) - 0.5f) * (1 + i / K);
}
for (int k = 0; k < K; k++)
w[3 * K + k] = 0; /* a row of zeros */
w[5 * K + 17] = 1e-6f, w[5 * K + 18] = -3e-5f; /* tiny scale */
for (int k = 19; k < K; k++)
w[5 * K + k] = 0;
for (int k = 0; k < 17; k++)
w[5 * K + k] = 0;
janas_npu_quant_rows(w, N, K, q, s);
for (int i = 0; i < N; i++) {
float sc = h2f(s[i]), worst = 0;
for (int k = 0; k < K; k++) {
float e = fabsf(q[i * K + k] * sc - w[i * K + k]);
worst = fmaxf(worst, e);
if (q[i * K + k] == -128)
failures++, printf("FAIL row %d: -128\n", i);
}
float amax = 0;
for (int k = 0; k < K; k++)
amax = fmaxf(amax, fabsf(w[i * K + k]));
/* half a step: the scale rounds up, nothing is clipped */
float bound = sc * 0.5f * 1.0001f;
if (sc * 127 < amax)
failures++, printf("FAIL row %d: scale below the row's max\n", i);
if (i == 3 && (sc != 0 || worst != 0))
failures++, printf("FAIL zero row: scale %g\n", sc);
if (worst > bound)
failures++, printf("FAIL row %d: error %g > %g\n", i, worst, bound);
}
/* the IR writer */
struct npu_ir g;
npu_ir_begin(&g, "t");
int xd[2] = {4, 8}, wd[2] = {3, 8};
int8_t wq[24] = {0};
int x = npu_ir_param(&g, "x", "f32", 2, xd);
int c = npu_ir_const(&g, "w", "i8", 2, wd, wq, sizeof wq);
int m = npu_ir_matmul_tb(&g, "mm", npu_ir_convert(&g, "x16", x, "f16"),
npu_ir_convert(&g, "w16", c, "f16"), NULL);
npu_ir_result(&g, m);
if (npu_ir_end(&g) || g.nl != 6 || g.ne != 5 || g.nbin != 24)
failures++, printf("FAIL IR: %d layers, %d edges\n", g.nl, g.ne);
if (!strstr(g.xml, "<edge from-layer=\"4\" from-port=\"2\" to-layer=\"5\" "
"to-port=\"0\"/>"))
failures++, printf("FAIL IR: matmul's edge\n");
int bad[2] = {3, 9};
struct npu_ir h;
npu_ir_begin(&h, "u");
int a = npu_ir_param(&h, "a", "f16", 2, xd),
b = npu_ir_param(&h, "b", "f16", 2, bad);
if (npu_ir_matmul_tb(&h, "mm", a, b, NULL) >= 0 || !npu_ir_end(&h))
failures++, printf("FAIL IR: mismatched k accepted\n");
npu_ir_free(&h);
size_t bytes = 0;
uint8_t *mb = npu_ir_model(&g, 8, 3, &bytes);
uint64_t sx, sb;
uint32_t parts;
uint16_t maj, min;
memcpy(&maj, mb, 2), memcpy(&min, mb + 2, 2), memcpy(&parts, mb + 4, 4);
memcpy(&sx, mb + 8, 8);
memcpy(&sb, mb + 16 + sx, 8);
if (!mb || maj != 8 || min != 3 || parts != 2 || sx != g.len || sb != 24 ||
bytes != 24 + sx + 24)
failures++, printf("FAIL model layout\n");
free(mb);
npu_ir_free(&g);
free(w);
free(q);
printf("%s\n", failures ? "test_npu: FAILED" : "test_npu: ok");
return failures != 0;
}